chore: import upstream snapshot with attribution

tests/parser/engine/conftest.py

@@ -0,0 +1,53 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from vllm.entrypoints.openai.chat_completion.protocol import (
+    ChatCompletionRequest,
+)
+
+
+@pytest.fixture()
+def should_do_global_cleanup_after_test() -> bool:
+    return False
+
+
+def make_mock_tokenizer(
+    vocab: dict[str, int],
+    special_tokens: list[str] | None = None,
+) -> MagicMock:
+    """Create a mock tokenizer with the given vocabulary.
+
+    Args:
+        vocab: Mapping of token text to token ID.
+        special_tokens: Which tokens to mark as special.  When ``None``
+            (the default), every key in *vocab* is treated as special —
+            convenient when the vocab only contains delimiter tokens.
+
+    The returned mock supports get_vocab(), encode(), and decode().
+    decode() maps known token IDs back to their text and falls back to
+    chr(id) for ASCII IDs or ``<id>`` for others.
+    """
+    id_to_text = {v: k for k, v in vocab.items()}
+    tokenizer = MagicMock()
+    tokenizer.encode.return_value = [1, 2, 3]
+    tokenizer.get_vocab.return_value = dict(vocab)
+    tokenizer.decode.side_effect = lambda ids: "".join(
+        id_to_text.get(i, chr(i) if i < 128 else f"<{i}>") for i in ids
+    )
+    st = special_tokens if special_tokens is not None else list(vocab.keys())
+    tokenizer.all_special_tokens = st
+    tokenizer.all_special_ids = [vocab[t] for t in st if t in vocab]
+    return tokenizer
+
+
+@pytest.fixture
+def mock_request():
+    req = MagicMock(spec=ChatCompletionRequest)
+    req.tools = []
+    req.tool_choice = "auto"
+    req.include_reasoning = True
+    return req

tests/parser/engine/replay_harness.py

@@ -0,0 +1,426 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Data-driven replay harness for parser engine testing.
+
+Replays token sequences through parsers at different chunk sizes to
+verify chunk-size invariance: the same token sequence must produce
+identical output regardless of how tokens are batched.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+
+from vllm.entrypoints.openai.chat_completion.protocol import (
+    ChatCompletionRequest,
+)
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage
+
+
+@dataclass
+class Sample:
+    """One test sample loaded from a JSONL file."""
+
+    id: str
+    description: str
+    source: str
+    vocab: dict[str, int]
+    tokens: list[tuple[int, str]]
+    expected_reasoning: str | None
+    expected_content: str | None
+    expected_tool_calls: list[dict] | None
+    tools: list[dict] | None = None
+    chat_template_kwargs: dict | None = None
+    prompt_token_ids: list[int] | None = None
+
+
+@dataclass
+class ParseOutput:
+    """Accumulated parse output from replaying a token stream."""
+
+    reasoning: str = ""
+    content: str = ""
+    tool_calls: list[dict] = field(default_factory=list)
+
+
+class MockTokenizer:
+    """Lightweight tokenizer mock that avoids unittest.mock overhead.
+
+    Used by ``benchmarks/benchmark_parsers.py`` in tight timing loops,
+    so hot-path methods (``decode``, ``get_vocab``) must be cheap.
+    MagicMock's call-recording machinery added ~40% overhead to small-
+    sample benchmarks, inflating the per-token cost of the parser engine.
+    """
+
+    __slots__ = (
+        "_vocab",
+        "_token_ids",
+        "_token_decode_map",
+        "_special_ids",
+        "eos_token_id",
+        "bos_token_id",
+        "pad_token_id",
+        "_all_special_tokens",
+    )
+
+    def __init__(
+        self,
+        vocab: dict[str, int],
+        tokens: list[tuple[int, str]],
+    ) -> None:
+        self._vocab = vocab
+        self._token_ids = [tid for tid, _ in tokens]
+        self._token_decode_map = {tid: text for tid, text in tokens}
+        self._special_ids = set(vocab.values())
+        self._all_special_tokens = list(vocab.keys())
+        self.eos_token_id = None
+        self.bos_token_id = None
+        self.pad_token_id = None
+
+    def set_vocab(self, vocab: dict[str, int]) -> None:
+        self._vocab = vocab
+
+    def get_vocab(self) -> dict[str, int]:
+        return self._vocab
+
+    @property
+    def all_special_tokens(self) -> list[str]:
+        return self._all_special_tokens
+
+    @property
+    def all_special_ids(self) -> list[int]:
+        return [self._vocab[t] for t in self._all_special_tokens if t in self._vocab]
+
+    def encode(self, text: str, **kwargs) -> list[int]:
+        return self._token_ids
+
+    def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str:
+        parts: list[str] = []
+        for tid in ids:
+            if skip_special_tokens and tid in self._special_ids:
+                continue
+            text = self._token_decode_map.get(tid, f"?{tid}?")
+            parts.append(text)
+        return "".join(parts)
+
+
+CHUNK_SIZES = [1, 2, 3, 5, 11, 23, None]
+
+
+def make_mock_tokenizer(sample: Sample) -> MockTokenizer:
+    """Build a mock tokenizer from a sample's vocab and token data."""
+    return MockTokenizer(
+        vocab=dict(sample.vocab),
+        tokens=sample.tokens,
+    )
+
+
+def _test_request(
+    tools: list[dict] | None = None,
+) -> ChatCompletionRequest:
+    return ChatCompletionRequest(
+        model="test-model",
+        messages=[{"role": "user", "content": "test"}],
+        tools=tools,
+    )
+
+
+DUMMY_TOOLS = [
+    {
+        "type": "function",
+        "function": {"name": "stub", "parameters": {"type": "object"}},
+    },
+]
+
+
+def parse_non_streaming(
+    parser,
+    sample: Sample,
+    request: ChatCompletionRequest,
+) -> ParseOutput:
+    """Run ``parser.parse()`` and return a :class:`ParseOutput`."""
+    full_text = "".join(text for _, text in sample.tokens)
+    reasoning, content, tool_calls = parser.parse(
+        full_text,
+        request,
+        enable_auto_tools=True,
+    )
+    tc_list: list[dict] = []
+    if tool_calls:
+        for tc in tool_calls:
+            tc_list.append({"name": tc.name, "arguments": tc.arguments})
+    return ParseOutput(
+        reasoning=reasoning or "",
+        content=content or "",
+        tool_calls=tc_list,
+    )
+
+
+def replay_streaming(
+    parser,
+    tokens: list[tuple[int, str]],
+    chunk_size: int | None = None,
+    holdback_chars: int = 0,
+    finished_on_last: bool = False,
+    tools: list[dict] | None = None,
+    prompt_token_ids: list[int] | None = None,
+) -> list[DeltaMessage | None]:
+    """Feed tokens through ``parser.parse_delta()`` at a given chunk size.
+
+    Args:
+        parser: A :class:`Parser` instance with ``parse_delta()`` method.
+        tokens: List of ``(token_id, decoded_text)`` pairs.
+        chunk_size: Number of tokens per batch. ``None`` means all at once.
+        holdback_chars: Simulate detokenizer holdback by holding back
+            this many characters of decoded text between batches.
+        finished_on_last: When True, pass ``finished=True`` on the last
+            ``parse_delta()`` call, matching real server behavior.
+        tools: Optional tool definitions to include on the request,
+            matching the serving layer where tools set
+            ``tool_choice`` to ``"auto"``.
+
+    Returns:
+        List of ``DeltaMessage`` results from each ``parse_delta()`` call.
+    """
+    if chunk_size is None:
+        chunk_size = len(tokens)
+
+    results: list[DeltaMessage | None] = []
+    all_ids = [tid for tid, _ in tokens]
+    all_texts = [text for _, text in tokens]
+
+    request = _test_request(tools=tools)
+    first_prompt_ids = prompt_token_ids if prompt_token_ids is not None else []
+
+    if holdback_chars <= 0:
+        chunks = list(range(0, len(tokens), chunk_size))
+        for i, start in enumerate(chunks):
+            batch_end = min(start + chunk_size, len(tokens))
+            batch_ids = all_ids[start:batch_end]
+            delta_text = "".join(all_texts[start:batch_end])
+            is_last = i == len(chunks) - 1
+
+            result = parser.parse_delta(
+                delta_text,
+                batch_ids,
+                request,
+                prompt_token_ids=first_prompt_ids if start == 0 else None,
+                finished=finished_on_last and is_last,
+            )
+            results.append(result)
+        return results
+
+    emitted_up_to = 0
+    is_first = True
+
+    for start in range(0, len(tokens), chunk_size):
+        batch_end = min(start + chunk_size, len(tokens))
+
+        if batch_end < len(tokens):
+            held_chars = 0
+            safe_end = batch_end
+            while safe_end > emitted_up_to and held_chars < holdback_chars:
+                safe_end -= 1
+                held_chars += len(all_texts[safe_end])
+        else:
+            safe_end = batch_end
+
+        if safe_end <= emitted_up_to:
+            continue
+
+        batch_ids = all_ids[emitted_up_to:safe_end]
+        delta_text = "".join(all_texts[emitted_up_to:safe_end])
+        emitted_up_to = safe_end
+
+        is_last_chunk = batch_end >= len(tokens)
+        result = parser.parse_delta(
+            delta_text,
+            batch_ids,
+            request,
+            prompt_token_ids=first_prompt_ids if is_first else None,
+            finished=finished_on_last and is_last_chunk,
+        )
+        results.append(result)
+        is_first = False
+
+    if emitted_up_to < len(tokens):
+        batch_ids = all_ids[emitted_up_to:]
+        delta_text = "".join(all_texts[emitted_up_to:])
+        result = parser.parse_delta(
+            delta_text,
+            batch_ids,
+            request,
+            prompt_token_ids=first_prompt_ids if is_first else None,
+            finished=finished_on_last,
+        )
+        results.append(result)
+
+    return results
+
+
+def replay_with_text_holdback(
+    parser,
+    tokens: list[tuple[int, str]],
+    text_delay: int = 1,
+    tools: list[dict] | None = None,
+    prompt_token_ids: list[int] | None = None,
+) -> list[DeltaMessage | None]:
+    """Replay token-by-token with text arriving *text_delay* steps late.
+
+    Simulates the production detokenizer holdback where token IDs arrive
+    immediately but decoded text is delayed.  On the last token all
+    remaining held-back text is flushed, matching real server behavior::
+
+        step 0:   ids=[tok0], text=""               (held back)
+        step 1:   ids=[tok1], text=tok0_text         (tok0 released)
+        ...
+        step N-1: ids=[tokN-1], text=remaining_texts (flush all)
+
+    This exercises the TokenIDScanner deferred-terminal path that
+    ``replay_streaming`` (which keeps text and IDs aligned) does not.
+    """
+    results: list[DeltaMessage | None] = []
+    request = _test_request(tools=tools)
+    first_prompt_ids = prompt_token_ids if prompt_token_ids is not None else []
+
+    n = len(tokens)
+    held_texts: list[str] = []
+
+    for i in range(n):
+        token_id = tokens[i][0]
+        held_texts.append(tokens[i][1])
+
+        is_last = i == n - 1
+        if is_last:
+            delta_text = "".join(held_texts)
+            held_texts.clear()
+        elif len(held_texts) > text_delay:
+            delta_text = held_texts.pop(0)
+        else:
+            delta_text = ""
+
+        result = parser.parse_delta(
+            delta_text,
+            [token_id],
+            request,
+            prompt_token_ids=first_prompt_ids if i == 0 else None,
+            finished=is_last,
+        )
+        results.append(result)
+
+    return results
+
+
+def accumulate_deltas(
+    deltas: Sequence[DeltaMessage | None],
+) -> dict:
+    reasoning_parts: list[str] = []
+    content_parts: list[str] = []
+    tool_calls_by_idx: dict[int, dict] = {}
+
+    for delta in deltas:
+        if delta is None:
+            continue
+        if delta.reasoning:
+            reasoning_parts.append(delta.reasoning)
+        if delta.content:
+            content_parts.append(delta.content)
+        if delta.tool_calls:
+            for tc in delta.tool_calls:
+                if tc.function and tc.function.name:
+                    existing = tool_calls_by_idx.get(tc.index)
+                    if existing is None:
+                        tool_calls_by_idx[tc.index] = {
+                            "name": tc.function.name,
+                            "_args_parts": [tc.function.arguments or ""],
+                        }
+                    else:
+                        existing["_args_parts"].append(tc.function.arguments or "")
+                elif tc.function and tc.function.arguments:
+                    existing = tool_calls_by_idx.get(tc.index)
+                    if existing is not None:
+                        existing["_args_parts"].append(tc.function.arguments)
+
+    return {
+        "reasoning": "".join(reasoning_parts),
+        "content": "".join(content_parts),
+        "tool_calls": [
+            {"name": tc["name"], "arguments": "".join(tc["_args_parts"])}
+            for tc in tool_calls_by_idx.values()
+        ],
+    }
+
+
+def collect_output(results: list[DeltaMessage | None]) -> ParseOutput:
+    """Accumulate ``DeltaMessage`` results into a :class:`ParseOutput`."""
+    result = accumulate_deltas(results)
+    return ParseOutput(
+        reasoning=result["reasoning"],
+        content=result["content"],
+        tool_calls=result["tool_calls"],
+    )
+
+
+def assert_parse_output(actual: ParseOutput, sample: Sample) -> None:
+    """Compare actual parse output against expected values from a sample."""
+    if sample.expected_reasoning is not None:
+        assert actual.reasoning == sample.expected_reasoning, (
+            f"Reasoning mismatch:\n"
+            f"  expected: {sample.expected_reasoning!r}\n"
+            f"  actual:   {actual.reasoning!r}"
+        )
+
+    if sample.expected_content is not None:
+        assert actual.content == sample.expected_content, (
+            f"Content mismatch:\n"
+            f"  expected: {sample.expected_content!r}\n"
+            f"  actual:   {actual.content!r}"
+        )
+    if sample.expected_tool_calls is not None:
+        assert len(actual.tool_calls) == len(sample.expected_tool_calls), (
+            f"Tool call count mismatch: "
+            f"expected {len(sample.expected_tool_calls)}, "
+            f"got {len(actual.tool_calls)}"
+        )
+        for i, (expected_tc, actual_tc) in enumerate(
+            zip(sample.expected_tool_calls, actual.tool_calls)
+        ):
+            assert actual_tc["name"] == expected_tc["name"], (
+                f"Tool call {i} name mismatch: "
+                f"expected {expected_tc['name']!r}, "
+                f"got {actual_tc['name']!r}"
+            )
+            if "arguments" in expected_tc:
+                expected_args = expected_tc["arguments"]
+                actual_args_str = actual_tc.get("arguments", "{}")
+                if isinstance(expected_args, dict):
+                    try:
+                        actual_args = json.loads(actual_args_str)
+                    except json.JSONDecodeError as e:
+                        raise AssertionError(
+                            f"Tool call {i} arguments not valid JSON: "
+                            f"{actual_args_str!r}"
+                        ) from e
+                    assert actual_args == expected_args, (
+                        f"Tool call {i} arguments mismatch:\n"
+                        f"  expected: {expected_args}\n"
+                        f"  actual:   {actual_args}"
+                    )
+
+
+def assert_no_terminal_leakage(
+    actual: ParseOutput,
+    terminals: list[str],
+    context: str = "",
+) -> None:
+    """Assert that none of *terminals* appear in reasoning or content."""
+    suffix = f" ({context})" if context else ""
+    for terminal in terminals:
+        assert terminal not in actual.reasoning, (
+            f"{terminal!r} leaked into reasoning{suffix}"
+        )
+        assert terminal not in actual.content, (
+            f"{terminal!r} leaked into content{suffix}"
+        )

tests/parser/engine/streaming_helpers.py

@@ -0,0 +1,137 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Shared streaming simulation helpers for parser engine tests."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage
+
+
+def _build_token_id_map(parser) -> dict[str, int]:
+    """Map special token text to token IDs from the parser's config."""
+    token_id_map: dict[str, int] = {}
+    cfg = getattr(parser, "parser_engine_config", None)
+    vocab = getattr(parser, "vocab", None)
+    if cfg is not None and vocab is not None:
+        for text in (cfg.token_id_terminals or {}).values():
+            tid = vocab.get(text)
+            if tid is not None:
+                token_id_map[text] = tid
+    return token_id_map
+
+
+def simulate_tool_streaming(
+    parser,
+    request,
+    chunks: list[str],
+) -> list[tuple[DeltaMessage | None, str]]:
+    """Feed text chunks through ``extract_tool_calls_streaming()``."""
+    token_id_map = _build_token_id_map(parser)
+
+    results: list[tuple[Any, str]] = []
+    previous_text = ""
+    previous_token_ids: list[int] = []
+
+    for chunk in chunks:
+        current_text = previous_text + chunk
+
+        delta_token_ids: list[int] = [
+            tid for text, tid in token_id_map.items() if text in chunk
+        ]
+
+        current_token_ids = previous_token_ids + delta_token_ids
+
+        delta = parser.extract_tool_calls_streaming(
+            previous_text=previous_text,
+            current_text=current_text,
+            delta_text=chunk,
+            previous_token_ids=tuple(previous_token_ids),
+            current_token_ids=tuple(current_token_ids),
+            delta_token_ids=tuple(delta_token_ids),
+            request=request,
+        )
+        results.append((delta, current_text))
+        previous_text = current_text
+        previous_token_ids = list(current_token_ids)
+
+    return results
+
+
+def collect_tool_arguments(
+    results: list[tuple[DeltaMessage | None, str]],
+) -> str:
+    """Concatenate all streamed argument fragments."""
+    args_text = ""
+    for delta, _ in results:
+        if delta and delta.tool_calls:
+            for tc in delta.tool_calls:
+                if tc.function and tc.function.arguments:
+                    args_text += tc.function.arguments
+    return args_text
+
+
+def collect_content(
+    results: list[tuple[DeltaMessage | None, str]],
+) -> str:
+    """Concatenate all streamed content parts."""
+    parts: list[str] = []
+    for delta, _ in results:
+        if delta and delta.content:
+            parts.append(delta.content)
+    return "".join(parts)
+
+
+def collect_function_name(
+    results: list[tuple[DeltaMessage | None, str]],
+) -> str | None:
+    """Return first function name from deltas."""
+    for delta, _ in results:
+        if delta and delta.tool_calls:
+            for tc in delta.tool_calls:
+                if tc.function and tc.function.name:
+                    return tc.function.name
+    return None
+
+
+def simulate_reasoning_streaming(
+    parser,
+    chunks: list[str],
+    delta_token_ids_per_chunk: list[tuple[int, ...]] | None = None,
+) -> tuple[str, str]:
+    """Feed chunks through ``extract_reasoning_streaming()``.
+
+    Returns ``(reasoning_text, content_text)`` tuple.
+    """
+    token_id_map = (
+        _build_token_id_map(parser) if delta_token_ids_per_chunk is None else {}
+    )
+
+    reasoning_parts: list[str] = []
+    content_parts: list[str] = []
+    prev_text = ""
+    prev_ids: list[int] = []
+    for i, chunk in enumerate(chunks):
+        cur_text = prev_text + chunk
+        if delta_token_ids_per_chunk is not None:
+            d_ids = delta_token_ids_per_chunk[i]
+        else:
+            d_ids = tuple(tid for text, tid in token_id_map.items() if text in chunk)
+        cur_ids = prev_ids + list(d_ids)
+        delta = parser.extract_reasoning_streaming(
+            previous_text=prev_text,
+            current_text=cur_text,
+            delta_text=chunk,
+            previous_token_ids=tuple(prev_ids),
+            current_token_ids=tuple(cur_ids),
+            delta_token_ids=d_ids,
+        )
+        if delta:
+            if delta.reasoning:
+                reasoning_parts.append(delta.reasoning)
+            if delta.content:
+                content_parts.append(delta.content)
+        prev_text = cur_text
+        prev_ids = list(cur_ids)
+    return "".join(reasoning_parts), "".join(content_parts)

tests/parser/engine/test_deepseek_v32.py

@@ -0,0 +1,268 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests for DeepSeek V3.2 parser engine semantics.
+
+V3.2 uses the same DSML parameter format as V4 but wraps tool calls in
+``<｜DSML｜function_calls>`` instead of ``<｜DSML｜tool_calls>`` and has
+no reasoning (``<think>``/``</think>``) support.
+"""
+
+import json
+
+import pytest
+
+from tests.parser.engine.conftest import make_mock_tokenizer
+from tests.parser.engine.streaming_helpers import (
+    collect_content,
+    collect_function_name,
+    collect_tool_arguments,
+    simulate_tool_streaming,
+)
+from vllm.parser.deepseek_v4 import (
+    DSML_INVOKE_END,
+    DSML_INVOKE_NAME_END,
+    DSML_INVOKE_PREFIX,
+)
+from vllm.parser.deepseek_v32 import (
+    DSML_FUNC_END,
+    DSML_FUNC_START,
+    DeepSeekV32Parser,
+)
+from vllm.parser.engine.parser_engine_config import ParserState
+
+_PARAM_OPEN = '｜DSML｜parameter name="{name}" string="{is_str}">'
+_PARAM_CLOSE = "</｜DSML｜parameter>"
+
+
+def _param(name: str, is_str: str, value: str) -> str:
+    return f"<{_PARAM_OPEN.format(name=name, is_str=is_str)}{value}{_PARAM_CLOSE}"
+
+
+def _invoke(name: str, *params: str) -> str:
+    body = "\n".join(params)
+    return (
+        f"{DSML_INVOKE_PREFIX}{name}{DSML_INVOKE_NAME_END}\n{body}\n{DSML_INVOKE_END}"
+    )
+
+
+def _func_calls(*invocations: str) -> str:
+    body = "\n".join(invocations)
+    return f"{DSML_FUNC_START}\n{body}\n{DSML_FUNC_END}"
+
+
+def _make_tool(name, properties):
+    from vllm.entrypoints.openai.chat_completion.protocol import (
+        ChatCompletionToolsParam,
+    )
+
+    return ChatCompletionToolsParam(
+        type="function",
+        function={
+            "name": name,
+            "parameters": {
+                "type": "object",
+                "properties": properties,
+            },
+        },
+    )
+
+
+@pytest.fixture
+def mock_tokenizer():
+    return make_mock_tokenizer({})
+
+
+@pytest.fixture
+def mock_request():
+    from unittest.mock import MagicMock
+
+    from vllm.entrypoints.openai.chat_completion.protocol import (
+        ChatCompletionRequest,
+    )
+
+    req = MagicMock(spec=ChatCompletionRequest)
+    req.tools = []
+    req.tool_choice = "auto"
+    return req
+
+
+# ── Non-streaming extraction ────────────────────────────────────────
+
+
+class TestNonStreaming:
+    def test_no_tool_call(self, mock_tokenizer, mock_request):
+        parser = DeepSeekV32Parser(mock_tokenizer)
+        result = parser.extract_tool_calls("Hello world", mock_request)
+        assert not result.tools_called
+        assert result.content == "Hello world"
+
+    def test_single_tool(self, mock_tokenizer, mock_request):
+        text = _func_calls(
+            _invoke("get_weather", _param("city", "true", "SF")),
+        )
+        parser = DeepSeekV32Parser(mock_tokenizer)
+        result = parser.extract_tool_calls(text, mock_request)
+        assert result.tools_called
+        assert len(result.tool_calls) == 1
+        assert result.tool_calls[0].function.name == "get_weather"
+        args = json.loads(result.tool_calls[0].function.arguments)
+        assert args == {"city": "SF"}
+
+    def test_parallel_tools(self, mock_tokenizer, mock_request):
+        text = _func_calls(
+            _invoke("get_weather", _param("city", "true", "SF")),
+            _invoke("get_weather", _param("city", "true", "NYC")),
+        )
+        parser = DeepSeekV32Parser(mock_tokenizer)
+        result = parser.extract_tool_calls(text, mock_request)
+        assert result.tools_called
+        assert len(result.tool_calls) == 2
+        assert json.loads(result.tool_calls[0].function.arguments) == {"city": "SF"}
+        assert json.loads(result.tool_calls[1].function.arguments) == {"city": "NYC"}
+
+    def test_content_before_tool_call(self, mock_tokenizer, mock_request):
+        text = "Let me check. " + _func_calls(
+            _invoke("search", _param("q", "true", "vllm")),
+        )
+        parser = DeepSeekV32Parser(mock_tokenizer)
+        result = parser.extract_tool_calls(text, mock_request)
+        assert result.tools_called
+        assert result.content is not None
+        assert "Let me check" in result.content
+
+    def test_non_string_params_json_parsed(self, mock_tokenizer, mock_request):
+        text = _func_calls(
+            _invoke(
+                "toggle",
+                _param("enabled", "false", "true"),
+                _param("count", "false", "42"),
+            ),
+        )
+        parser = DeepSeekV32Parser(mock_tokenizer)
+        result = parser.extract_tool_calls(text, mock_request)
+        args = json.loads(result.tool_calls[0].function.arguments)
+        assert args["enabled"] is True
+        assert args["count"] == 42
+
+    def test_wrapper_unwrapping(self, mock_tokenizer, mock_request):
+        tool = _make_tool("get_weather", {"location": {"type": "string"}})
+        mock_request.tools = [tool]
+        text = _func_calls(
+            _invoke(
+                "get_weather",
+                _param("arguments", "false", '{"location":"Beijing"}'),
+            ),
+        )
+        parser = DeepSeekV32Parser(mock_tokenizer, tools=[tool])
+        result = parser.extract_tool_calls(text, mock_request)
+        args = json.loads(result.tool_calls[0].function.arguments)
+        assert args == {"location": "Beijing"}
+
+
+# ── Initial state ────────────────────────────────────────────────────
+
+
+class TestInitialState:
+    def test_always_content(self, mock_tokenizer):
+        parser = DeepSeekV32Parser(mock_tokenizer)
+        cfg = parser.parser_engine_config
+        assert cfg.initial_state == ParserState.CONTENT
+
+    def test_ignores_thinking_kwargs(self, mock_tokenizer):
+        parser = DeepSeekV32Parser(
+            mock_tokenizer,
+            chat_template_kwargs={"thinking": True, "enable_thinking": True},
+        )
+        cfg = parser.parser_engine_config
+        assert cfg.initial_state == ParserState.CONTENT
+
+
+# ── Streaming ────────────────────────────────────────────────────────
+
+
+class TestStreaming:
+    def test_single_tool_streaming(self, mock_tokenizer, mock_request):
+        text = _func_calls(
+            _invoke("get_weather", _param("city", "true", "SF")),
+        )
+        parser = DeepSeekV32Parser(mock_tokenizer)
+        results = simulate_tool_streaming(parser, mock_request, list(text))
+        assert collect_function_name(results) == "get_weather"
+        args_json = collect_tool_arguments(results)
+        assert json.loads(args_json) == {"city": "SF"}
+
+    def test_content_before_tool_streaming(self, mock_tokenizer, mock_request):
+        text = "Checking... " + _func_calls(
+            _invoke("fn", _param("k", "true", "v")),
+        )
+        parser = DeepSeekV32Parser(mock_tokenizer)
+        results = simulate_tool_streaming(parser, mock_request, list(text))
+        content = collect_content(results)
+        assert "Checking" in content
+
+    def test_parallel_tools_streaming(self, mock_tokenizer, mock_request):
+        text = _func_calls(
+            _invoke("fn_a", _param("x", "true", "1")),
+            _invoke("fn_b", _param("y", "true", "2")),
+        )
+        parser = DeepSeekV32Parser(mock_tokenizer)
+        results = simulate_tool_streaming(parser, mock_request, list(text))
+
+        names = []
+        for delta, _ in results:
+            if delta and delta.tool_calls:
+                for tc in delta.tool_calls:
+                    if tc.function and tc.function.name:
+                        names.append(tc.function.name)
+        assert "fn_a" in names
+        assert "fn_b" in names
+
+    def test_no_tool_content_only(self, mock_tokenizer, mock_request):
+        text = "Just some text, no tools."
+        parser = DeepSeekV32Parser(mock_tokenizer)
+        results = simulate_tool_streaming(parser, mock_request, list(text))
+        content = collect_content(results)
+        assert "Just some text" in content
+        args = collect_tool_arguments(results)
+        assert args == ""
+
+    def test_streaming_wrapper_unwrap_consistency(self, mock_tokenizer, mock_request):
+        tool = _make_tool("get_weather", {"location": {"type": "string"}})
+        mock_request.tools = [tool]
+        parser = DeepSeekV32Parser(mock_tokenizer, tools=[tool])
+
+        chunks = [
+            DSML_FUNC_START,
+            _invoke(
+                "get_weather",
+                _param("arguments", "false", '{"location": "NYC"}'),
+            ),
+            DSML_FUNC_END,
+        ]
+
+        results = simulate_tool_streaming(parser, mock_request, chunks)
+        streamed_args = collect_tool_arguments(results)
+
+        final_delta, _ = results[-1]
+        finish_delta = parser.finish_streaming()
+        extracted = parser._build_extracted_result(final_delta, finish_delta)
+
+        assert extracted.tools_called is True
+        assert len(extracted.tool_calls) == 1
+        final_args = extracted.tool_calls[0].function.arguments
+        assert json.loads(final_args) == {"location": "NYC"}
+        assert '"arguments"' not in streamed_args
+        assert final_args.startswith(streamed_args)
+
+    def test_missing_invoke_end(self, mock_tokenizer, mock_request):
+        text = (
+            f"{DSML_FUNC_START}\n"
+            f"{DSML_INVOKE_PREFIX}fn{DSML_INVOKE_NAME_END}\n"
+            f"{_param('k', 'true', 'v')}\n"
+            f"{DSML_FUNC_END}"
+        )
+        parser = DeepSeekV32Parser(mock_tokenizer)
+        results = simulate_tool_streaming(parser, mock_request, list(text))
+        assert collect_function_name(results) == "fn"
+        args = json.loads(collect_tool_arguments(results))
+        assert args == {"k": "v"}

tests/parser/engine/test_deepseek_v4.py

@@ -0,0 +1,922 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests for DeepSeek V4-specific parser engine semantics."""
+
+import json
+
+import pytest
+
+from tests.parser.engine.conftest import make_mock_tokenizer
+from tests.parser.engine.replay_harness import (
+    DUMMY_TOOLS,
+    MockTokenizer,
+    _test_request,
+    collect_output,
+    replay_streaming,
+)
+from tests.parser.engine.streaming_helpers import (
+    collect_content,
+    collect_function_name,
+    collect_tool_arguments,
+    simulate_reasoning_streaming,
+    simulate_tool_streaming,
+)
+from vllm.parser.abstract_parser import DelegatingParser
+from vllm.parser.deepseek_v4 import (
+    DSML_INVOKE_END,
+    DSML_INVOKE_NAME_END,
+    DSML_INVOKE_PREFIX,
+    DSML_THINK_END,
+    DSML_THINK_START,
+    DSML_TOOL_END,
+    DSML_TOOL_START,
+    DeepSeekV4Parser,
+    _dsml_arg_converter,
+    _unwrap_wrapper_args,
+    deepseek_v4_config,
+)
+from vllm.parser.engine.registered_adapters import (
+    DeepSeekV4ParserReasoningAdapter,
+    DeepSeekV4ParserToolAdapter,
+)
+
+_THINK_START_ID = 50
+_THINK_END_ID = 51
+
+_PARAM_OPEN = '｜DSML｜parameter name="{name}" string="{is_str}">'
+_PARAM_CLOSE = "</｜DSML｜parameter>"
+
+
+def _param(name: str, is_str: str, value: str) -> str:
+    return f"<{_PARAM_OPEN.format(name=name, is_str=is_str)}{value}{_PARAM_CLOSE}"
+
+
+@pytest.fixture
+def mock_tokenizer():
+    return make_mock_tokenizer(
+        {
+            DSML_THINK_START: _THINK_START_ID,
+            DSML_THINK_END: _THINK_END_ID,
+        }
+    )
+
+
+# ── Arg converter unit tests ─────────────────────────────────────────
+
+
+class TestArgConverter:
+    def _raw(self, *params: tuple[str, str, str]) -> str:
+        lines = [_param(n, s, v) for n, s, v in params]
+        return "\n" + "\n".join(lines) + "\n"
+
+    def test_string_param(self):
+        raw = self._raw(("city", "true", "杭州"))
+        result = json.loads(_dsml_arg_converter(raw, partial=False))
+        assert result == {"city": "杭州"}
+
+    def test_string_with_spaces_and_quotes(self):
+        raw = self._raw(("msg", "true", 'He said "hello world"'))
+        result = json.loads(_dsml_arg_converter(raw, partial=False))
+        assert result["msg"] == 'He said "hello world"'
+
+    def test_integer_param(self):
+        raw = self._raw(("count", "false", "42"))
+        result = json.loads(_dsml_arg_converter(raw, partial=False))
+        assert result["count"] == 42
+        assert isinstance(result["count"], int)
+
+    def test_float_param(self):
+        raw = self._raw(("ratio", "false", "3.14"))
+        result = json.loads(_dsml_arg_converter(raw, partial=False))
+        assert abs(result["ratio"] - 3.14) < 1e-9
+
+    def test_bool_param(self):
+        raw = self._raw(("flag", "false", "true"))
+        result = json.loads(_dsml_arg_converter(raw, partial=False))
+        assert result["flag"] is True
+
+    def test_array_param(self):
+        raw = self._raw(("items", "false", '["a", "b", "c"]'))
+        result = json.loads(_dsml_arg_converter(raw, partial=False))
+        assert result["items"] == ["a", "b", "c"]
+
+    def test_object_param(self):
+        raw = self._raw(("opts", "false", '{"key": "val"}'))
+        result = json.loads(_dsml_arg_converter(raw, partial=False))
+        assert result["opts"] == {"key": "val"}
+
+    def test_mixed_types(self):
+        raw = self._raw(
+            ("location", "true", "Tokyo"),
+            ("limit", "false", "10"),
+            ("active", "false", "false"),
+        )
+        result = json.loads(_dsml_arg_converter(raw, partial=False))
+        assert result == {"location": "Tokyo", "limit": 10, "active": False}
+
+    def test_empty_args(self):
+        result = json.loads(_dsml_arg_converter("", partial=False))
+        assert result == {}
+
+    def test_invalid_json_fallback(self):
+        raw = self._raw(("data", "false", "[broken"))
+        result = json.loads(_dsml_arg_converter(raw, partial=False))
+        assert result["data"] == "[broken"
+
+    def test_chinese_chars_preserved_in_json(self):
+        raw = self._raw(("query", "true", "你好世界"))
+        raw_json = _dsml_arg_converter(raw, partial=False)
+        assert "你好世界" in raw_json
+        result = json.loads(raw_json)
+        assert result["query"] == "你好世界"
+
+    def test_partial_complete_plus_in_progress(self):
+        raw = self._raw(("city", "true", "Tokyo"))
+        raw += f"<{_PARAM_OPEN.format(name='unit', is_str='true')}celsi"
+        result = json.loads(_dsml_arg_converter(raw, partial=True))
+        assert result["city"] == "Tokyo"
+        assert result["unit"] == "celsi"
+
+    def test_partial_no_in_progress(self):
+        raw = self._raw(("city", "true", "Tokyo"))
+        result = json.loads(_dsml_arg_converter(raw, partial=True))
+        assert result == {"city": "Tokyo"}
+
+    def test_partial_value_with_angle_bracket(self):
+        raw = f"<{_PARAM_OPEN.format(name='code', is_str='true')}a<b"
+        result = json.loads(_dsml_arg_converter(raw, partial=True))
+        assert result == {"code": "a<b"}
+
+    def test_partial_value_with_angle_bracket_and_complete_param(self):
+        raw = self._raw(("city", "true", "Tokyo"))
+        raw += f"<{_PARAM_OPEN.format(name='expr', is_str='true')}x<5"
+        result = json.loads(_dsml_arg_converter(raw, partial=True))
+        assert result["city"] == "Tokyo"
+        assert result["expr"] == "x<5"
+
+    def test_null_string_false(self):
+        raw = self._raw(("val", "false", "null"))
+        result = json.loads(_dsml_arg_converter(raw, partial=False))
+        assert result["val"] is None
+
+    def test_string_true_not_json_parsed(self):
+        raw = self._raw(("n", "true", "42"))
+        result = json.loads(_dsml_arg_converter(raw, partial=False))
+        assert result["n"] == "42"
+        assert isinstance(result["n"], str)
+
+
+# ── Bare </think> absorption and duplicate <think> absorption ─────────
+
+
+class TestThinkTagAbsorption:
+    def test_bare_think_end_not_leaked(self, mock_tokenizer):
+        parser = DeepSeekV4Parser(mock_tokenizer)
+        chunks = ["</think>", "Here is the direct answer."]
+        reasoning, content = simulate_reasoning_streaming(parser, chunks)
+        assert reasoning == ""
+        assert "</think>" not in content
+        assert "Here is the direct answer" in content
+
+    def test_duplicate_think_start_absorbed(self, mock_tokenizer):
+        parser = DeepSeekV4Parser(
+            mock_tokenizer, chat_template_kwargs={"thinking": True}
+        )
+        chunks = [
+            "<think>\n",
+            "Some reasoning.\n",
+            "</think>\n",
+            "Answer.",
+        ]
+        reasoning, content = simulate_reasoning_streaming(parser, chunks)
+        assert "Some reasoning" in reasoning
+        assert "Answer" in content
+
+
+# ── Missing </｜DSML｜invoke> before </｜DSML｜tool_calls> ────────────
+
+
+class TestMissingInvokeEnd:
+    def test_non_streaming(self, mock_tokenizer, mock_request):
+        parser = DeepSeekV4Parser(mock_tokenizer)
+        text = (
+            f"{DSML_TOOL_START}"
+            f"{DSML_INVOKE_PREFIX}get_weather{DSML_INVOKE_NAME_END}\n"
+            f"{_param('location', 'true', 'NYC')}\n"
+            f"{DSML_TOOL_END}"
+        )
+        result = parser.extract_tool_calls(text, mock_request)
+
+        assert result.tools_called is True
+        assert len(result.tool_calls) == 1
+        assert result.tool_calls[0].function.name == "get_weather"
+        args = json.loads(result.tool_calls[0].function.arguments)
+        assert args == {"location": "NYC"}
+
+    def test_streaming_with_trailing_content(self, mock_tokenizer, mock_request):
+        parser = DeepSeekV4Parser(mock_tokenizer)
+        chunks = [
+            DSML_TOOL_START,
+            f"{DSML_INVOKE_PREFIX}get_weather{DSML_INVOKE_NAME_END}\n"
+            f"{_param('location', 'true', 'NYC')}\n",
+            DSML_TOOL_END,
+            "Done.",
+        ]
+
+        results = simulate_tool_streaming(parser, mock_request, chunks)
+
+        assert collect_function_name(results) == "get_weather"
+        args = json.loads(collect_tool_arguments(results))
+        assert args == {"location": "NYC"}
+        assert "Done." in collect_content(results)
+
+
+# ── Thinking mode initial state ──────────────────────────────────────
+
+
+class TestThinkingModeConfig:
+    def test_thinking_true_starts_in_reasoning(self):
+        cfg = deepseek_v4_config(thinking=True)
+        assert cfg.initial_state.name == "REASONING"
+
+    def test_thinking_false_starts_in_content(self):
+        cfg = deepseek_v4_config(thinking=False)
+        assert cfg.initial_state.name == "CONTENT"
+
+    def test_enable_thinking_kwarg(self, mock_tokenizer):
+        p = DeepSeekV4Parser(
+            mock_tokenizer, chat_template_kwargs={"enable_thinking": True}
+        )
+        assert p.parser_engine_config.initial_state.name == "REASONING"
+
+    def test_no_thinking_kwarg_defaults_to_content(self, mock_tokenizer):
+        p = DeepSeekV4Parser(mock_tokenizer)
+        assert p.parser_engine_config.initial_state.name == "CONTENT"
+
+    def test_thinking_mode_reasoning_without_tags(self, mock_tokenizer):
+        parser = DeepSeekV4Parser(
+            mock_tokenizer, chat_template_kwargs={"thinking": True}
+        )
+        chunks = [
+            "\n\nLet me consider ",
+            "this carefully.\n",
+            "</think>\n",
+            "Here is the result.",
+        ]
+        reasoning, content = simulate_reasoning_streaming(parser, chunks)
+        assert "Let me consider" in reasoning
+        assert "Here is the result" in content
+
+    def test_thinking_mode_all_reasoning_no_end_tag(self, mock_tokenizer):
+        parser = DeepSeekV4Parser(
+            mock_tokenizer, chat_template_kwargs={"thinking": True}
+        )
+        chunks = ["I'll review ", "the PR."]
+        reasoning, content = simulate_reasoning_streaming(parser, chunks)
+        assert "review" in reasoning
+        assert "the PR" in reasoning
+        assert content == ""
+
+    def test_reasoning_effort_none_overrides_enable_thinking(self, mock_tokenizer):
+        p = DeepSeekV4Parser(
+            mock_tokenizer,
+            chat_template_kwargs={
+                "enable_thinking": True,
+                "reasoning_effort": "none",
+            },
+        )
+        assert p.parser_engine_config.initial_state.name == "CONTENT"
+
+
+# ── Implicit reasoning end (missing </think> before tool calls) ─────
+
+
+class TestImplicitReasoningEnd:
+    """Tool call markers end reasoning implicitly when </think> is missing.
+
+    DeepSeek V4 models occasionally omit </think> before emitting tool calls.
+    The (REASONING, TOOL_START) transition handles this gracefully.
+    """
+
+    @pytest.fixture
+    def thinking_parser(self, mock_tokenizer):
+        return DeepSeekV4Parser(mock_tokenizer, chat_template_kwargs={"thinking": True})
+
+    def _reasoning_then_tool(self, reasoning_text: str) -> str:
+        return reasoning_text + _tool_calls(
+            _invoke("get_weather", ("location", "true", "NYC")),
+        )
+
+    def test_non_streaming_extract_reasoning_implicit_end(self, thinking_parser):
+        text = self._reasoning_then_tool("Let me look up the weather.\n\n")
+        reasoning, content = thinking_parser.extract_reasoning(text, None)
+        assert reasoning == "Let me look up the weather."
+        assert DSML_TOOL_START not in reasoning
+        assert DSML_INVOKE_PREFIX not in reasoning
+        assert content is None
+
+    def test_non_streaming_extract_tool_calls_implicit_end(
+        self, thinking_parser, mock_request
+    ):
+        text = self._reasoning_then_tool("Let me look up the weather.\n\n")
+        result = thinking_parser.extract_tool_calls(text, mock_request)
+        assert result.tools_called is True
+        assert len(result.tool_calls) == 1
+        assert result.tool_calls[0].function.name == "get_weather"
+        args = json.loads(result.tool_calls[0].function.arguments)
+        assert args == {"location": "NYC"}
+
+    def test_non_streaming_parse_implicit_end(self, thinking_parser, mock_request):
+        text = self._reasoning_then_tool("Let me look up the weather.\n\n")
+        reasoning, content, tool_calls = thinking_parser.parse(text, mock_request)
+        assert reasoning == "Let me look up the weather."
+        assert content is None
+        assert tool_calls is not None
+        assert len(tool_calls) == 1
+        assert tool_calls[0].name == "get_weather"
+        args = json.loads(tool_calls[0].arguments)
+        assert args == {"location": "NYC"}
+
+    def test_streaming_reasoning_implicit_end(self, thinking_parser):
+        chunks = [
+            "Let me look up the weather.\n\n",
+            DSML_TOOL_START,
+            DSML_INVOKE_PREFIX + "get_weather" + DSML_INVOKE_NAME_END,
+        ]
+        reasoning, content = simulate_reasoning_streaming(thinking_parser, chunks)
+        assert reasoning == "Let me look up the weather."
+        assert DSML_TOOL_START not in reasoning
+        assert DSML_INVOKE_PREFIX not in reasoning
+
+    def test_streaming_tool_extraction_implicit_end(
+        self, thinking_parser, mock_request
+    ):
+        chunks = [
+            "Let me check.\n\n",
+            DSML_TOOL_START,
+            DSML_INVOKE_PREFIX
+            + "get_weather"
+            + DSML_INVOKE_NAME_END
+            + "\n"
+            + _param("location", "true", "NYC")
+            + "\n"
+            + DSML_INVOKE_END,
+            DSML_TOOL_END,
+        ]
+        results = simulate_tool_streaming(thinking_parser, mock_request, chunks)
+        assert collect_function_name(results) == "get_weather"
+        args = json.loads(collect_tool_arguments(results))
+        assert args == {"location": "NYC"}
+
+    def test_thinking_false_explicit_think_then_tool_call(self, mock_tokenizer):
+        parser = DeepSeekV4Parser(mock_tokenizer)
+        chunks = [
+            DSML_THINK_START,
+            "Let me check the weather.",
+            DSML_TOOL_START,
+            DSML_INVOKE_PREFIX + "get_weather" + DSML_INVOKE_NAME_END,
+        ]
+        reasoning, content = simulate_reasoning_streaming(parser, chunks)
+        assert "Let me check the weather" in reasoning
+        assert DSML_TOOL_START not in reasoning
+        assert DSML_THINK_START not in reasoning
+
+    def test_non_streaming_parallel_tools_after_implicit_end(
+        self, thinking_parser, mock_request
+    ):
+        text = "I need both.\n\n" + _tool_calls(
+            _invoke("get_weather", ("location", "true", "NYC")),
+            _invoke("get_time", ("timezone", "true", "EST")),
+        )
+        result = thinking_parser.extract_tool_calls(text, mock_request)
+        assert result.tools_called is True
+        assert len(result.tool_calls) == 2
+        assert result.tool_calls[0].function.name == "get_weather"
+        assert result.tool_calls[1].function.name == "get_time"
+
+    def test_streaming_implicit_end_trailing_whitespace_stripped(self, thinking_parser):
+        chunks = [
+            "Reasoning.\n\n\n",
+            DSML_TOOL_START,
+            DSML_INVOKE_PREFIX + "func" + DSML_INVOKE_NAME_END,
+        ]
+        reasoning, content = simulate_reasoning_streaming(thinking_parser, chunks)
+        assert reasoning == "Reasoning."
+
+
+# ── Wrapper argument unwrapping ──────────────────────────────────────
+
+
+class TestWrapperUnwrapping:
+    def test_unwrap_arguments_wrapper(self):
+        from vllm.entrypoints.openai.chat_completion.protocol import (
+            ChatCompletionToolsParam,
+        )
+
+        tool = ChatCompletionToolsParam(
+            type="function",
+            function={
+                "name": "get_weather",
+                "parameters": {
+                    "type": "object",
+                    "properties": {"location": {"type": "string"}},
+                },
+            },
+        )
+
+        result = _unwrap_wrapper_args(
+            '{"arguments": {"location": "Beijing"}}',
+            [tool],
+            "get_weather",
+        )
+        assert json.loads(result) == {"location": "Beijing"}
+
+    def test_unwrap_input_wrapper(self):
+        from vllm.entrypoints.openai.chat_completion.protocol import (
+            ChatCompletionToolsParam,
+        )
+
+        tool = ChatCompletionToolsParam(
+            type="function",
+            function={
+                "name": "get_weather",
+                "parameters": {
+                    "type": "object",
+                    "properties": {"location": {"type": "string"}},
+                },
+            },
+        )
+
+        result = _unwrap_wrapper_args(
+            '{"input": {"location": "Beijing"}}',
+            [tool],
+            "get_weather",
+        )
+        assert json.loads(result) == {"location": "Beijing"}
+
+    def test_no_unwrap_when_key_in_schema(self):
+        from vllm.entrypoints.openai.chat_completion.protocol import (
+            ChatCompletionToolsParam,
+        )
+
+        tool = ChatCompletionToolsParam(
+            type="function",
+            function={
+                "name": "func",
+                "parameters": {
+                    "type": "object",
+                    "properties": {"arguments": {"type": "string"}},
+                },
+            },
+        )
+
+        result = _unwrap_wrapper_args(
+            '{"arguments": "some value"}',
+            [tool],
+            "func",
+        )
+        assert json.loads(result) == {"arguments": "some value"}
+
+    def test_no_unwrap_when_no_tools(self):
+        result = _unwrap_wrapper_args(
+            '{"arguments": {"location": "Beijing"}}',
+            None,
+            "get_weather",
+        )
+        assert json.loads(result) == {"arguments": {"location": "Beijing"}}
+
+    def test_unwrap_json_string_inner(self):
+        from vllm.entrypoints.openai.chat_completion.protocol import (
+            ChatCompletionToolsParam,
+        )
+
+        tool = ChatCompletionToolsParam(
+            type="function",
+            function={
+                "name": "get_weather",
+                "parameters": {
+                    "type": "object",
+                    "properties": {"location": {"type": "string"}},
+                },
+            },
+        )
+
+        result = _unwrap_wrapper_args(
+            '{"arguments": "{\\"location\\": \\"Beijing\\"}"}',
+            [tool],
+            "get_weather",
+        )
+        assert json.loads(result) == {"location": "Beijing"}
+
+
+# ── Parallel tool call wrapper unwrapping ───────────────────────────
+
+
+def _make_tool(name, properties):
+    from vllm.entrypoints.openai.chat_completion.protocol import (  # noqa: E501
+        ChatCompletionToolsParam,
+    )
+
+    return ChatCompletionToolsParam(
+        type="function",
+        function={
+            "name": name,
+            "parameters": {
+                "type": "object",
+                "properties": properties,
+            },
+        },
+    )
+
+
+def _invoke(name, *params):
+    body = "\n".join(_param(n, s, v) for n, s, v in params)
+    return (
+        f"{DSML_INVOKE_PREFIX}{name}{DSML_INVOKE_NAME_END}\n{body}\n{DSML_INVOKE_END}"
+    )
+
+
+def _tool_calls(*invokes):
+    return DSML_TOOL_START + "\n".join(invokes) + DSML_TOOL_END
+
+
+class TestParallelUnwrapping:
+    @pytest.fixture
+    def weather_tool(self):
+        return _make_tool(
+            "get_weather",
+            {
+                "location": {"type": "string"},
+                "unit": {"type": "string"},
+            },
+        )
+
+    @pytest.fixture
+    def time_tool(self):
+        return _make_tool(
+            "get_time",
+            {"timezone": {"type": "string"}},
+        )
+
+    @pytest.mark.parametrize(
+        "weather_args, expected",
+        [
+            (
+                '{"location": "NYC", "unit": "celsius"}',
+                {"location": "NYC", "unit": "celsius"},
+            ),
+            ('{"location": "NYC"}', {"location": "NYC"}),
+        ],
+        ids=["all_props", "subset_props"],
+    )
+    def test_unwrap_parallel_uses_correct_schema(
+        self,
+        mock_tokenizer,
+        mock_request,
+        weather_tool,
+        time_tool,
+        weather_args,
+        expected,
+    ):
+        tools = [weather_tool, time_tool]
+        parser = DeepSeekV4Parser(mock_tokenizer, tools=tools)
+        mock_request.tools = tools
+
+        text = _tool_calls(
+            _invoke("get_weather", ("arguments", "false", weather_args)),
+            _invoke("get_time", ("timezone", "true", "EST")),
+        )
+
+        result = parser.extract_tool_calls(text, mock_request)
+
+        assert result.tools_called is True
+        assert len(result.tool_calls) == 2
+        assert result.tool_calls[0].function.name == "get_weather"
+        args0 = json.loads(result.tool_calls[0].function.arguments)
+        assert args0 == expected
+        assert result.tool_calls[1].function.name == "get_time"
+        args1 = json.loads(result.tool_calls[1].function.arguments)
+        assert args1 == {"timezone": "EST"}
+
+    def test_unwrap_parallel_streaming(
+        self, mock_tokenizer, mock_request, weather_tool, time_tool
+    ):
+        tools = [weather_tool, time_tool]
+        parser = DeepSeekV4Parser(mock_tokenizer, tools=tools)
+        mock_request.tools = tools
+
+        chunks = [
+            DSML_TOOL_START,
+            _invoke(
+                "get_weather",
+                ("arguments", "false", '{"location": "NYC"}'),
+            ),
+            _invoke("get_time", ("timezone", "true", "EST")),
+            DSML_TOOL_END,
+        ]
+
+        results = simulate_tool_streaming(parser, mock_request, chunks)
+        final_delta, _ = results[-1]
+        finish_delta = parser.finish_streaming()
+        extracted = parser._build_extracted_result(final_delta, finish_delta)
+
+        assert extracted.tools_called is True
+        assert len(extracted.tool_calls) == 2
+        args0 = json.loads(extracted.tool_calls[0].function.arguments)
+        assert args0 == {"location": "NYC"}
+        args1 = json.loads(extracted.tool_calls[1].function.arguments)
+        assert args1 == {"timezone": "EST"}
+
+    def test_no_unwrap_parallel_when_no_match(
+        self, mock_tokenizer, mock_request, weather_tool, time_tool
+    ):
+        tools = [weather_tool, time_tool]
+        parser = DeepSeekV4Parser(mock_tokenizer, tools=tools)
+        mock_request.tools = tools
+
+        text = _tool_calls(
+            _invoke(
+                "get_weather",
+                ("arguments", "false", '{"unknown_key": "val"}'),
+            ),
+            _invoke("get_time", ("timezone", "true", "EST")),
+        )
+
+        result = parser.extract_tool_calls(text, mock_request)
+
+        assert len(result.tool_calls) == 2
+        args0 = json.loads(result.tool_calls[0].function.arguments)
+        assert args0 == {"arguments": {"unknown_key": "val"}}
+        args1 = json.loads(result.tool_calls[1].function.arguments)
+        assert args1 == {"timezone": "EST"}
+
+    def test_unwrap_single_tool_still_works(self, mock_tokenizer, mock_request):
+        tool = _make_tool("get_weather", {"location": {"type": "string"}})
+        tools = [tool]
+        parser = DeepSeekV4Parser(mock_tokenizer, tools=tools)
+        mock_request.tools = tools
+
+        text = _tool_calls(
+            _invoke(
+                "get_weather",
+                ("arguments", "false", '{"location": "Beijing"}'),
+            ),
+        )
+
+        result = parser.extract_tool_calls(text, mock_request)
+
+        assert result.tools_called is True
+        assert len(result.tool_calls) == 1
+        args = json.loads(result.tool_calls[0].function.arguments)
+        assert args == {"location": "Beijing"}
+
+
+# ── Streaming wrapper consistency ─────────────────────────────────────
+
+
+class TestStreamingWrapperConsistency:
+    """Streamed arg deltas must stay consistent with final extraction
+    when wrapper params like 'arguments' are unwrapped."""
+
+    def test_streaming_wrapper_unwrap_consistency(self, mock_tokenizer, mock_request):
+        tool = _make_tool("get_weather", {"location": {"type": "string"}})
+        tools = [tool]
+        parser = DeepSeekV4Parser(mock_tokenizer, tools=tools)
+        mock_request.tools = tools
+
+        chunks = [
+            DSML_TOOL_START,
+            _invoke(
+                "get_weather",
+                ("arguments", "false", '{"location": "NYC"}'),
+            ),
+            DSML_TOOL_END,
+        ]
+
+        results = simulate_tool_streaming(parser, mock_request, chunks)
+        streamed_args = collect_tool_arguments(results)
+
+        final_delta, _ = results[-1]
+        finish_delta = parser.finish_streaming()
+        extracted = parser._build_extracted_result(final_delta, finish_delta)
+
+        assert extracted.tools_called is True
+        assert len(extracted.tool_calls) == 1
+
+        final_args = extracted.tool_calls[0].function.arguments
+        assert json.loads(final_args) == {"location": "NYC"}
+
+        assert '"arguments"' not in streamed_args, (
+            f"Streamed args should not contain wrapper key, got: {streamed_args!r}"
+        )
+
+        assert final_args.startswith(streamed_args), (
+            f"Extracted args {final_args!r} "
+            f"should start with streamed args {streamed_args!r}"
+        )
+
+
+# ── DelegatingParser: large delta with </think> + tool calls ─────────
+
+_DSV4_FULL_VOCAB = {
+    DSML_THINK_START: 128821,
+    DSML_THINK_END: 128822,
+    DSML_TOOL_START: 128823,
+    DSML_TOOL_END: 128824,
+}
+
+
+class _DeepSeekV4Delegating(DelegatingParser):
+    reasoning_parser_cls = DeepSeekV4ParserReasoningAdapter
+    tool_parser_cls = DeepSeekV4ParserToolAdapter
+
+
+def _dsv4_tokens(
+    reasoning: str,
+    tool_name: str,
+    params: list[tuple[str, str, str]],
+) -> list[tuple[int, str]]:
+    """Build a token sequence: reasoning + </think> + DSML tool block."""
+    tokens: list[tuple[int, str]] = []
+    tid = 100
+
+    for word in reasoning.split(" "):
+        prefix = " " if tokens else ""
+        tokens.append((tid, prefix + word))
+        tid += 1
+
+    tokens.append((_DSV4_FULL_VOCAB[DSML_THINK_END], DSML_THINK_END))
+
+    tokens.append((tid, "\n\n"))
+    tid += 1
+
+    tokens.append((_DSV4_FULL_VOCAB[DSML_TOOL_START], DSML_TOOL_START))
+
+    tokens.append((tid, "\n"))
+    tid += 1
+
+    invoke_prefix_text = f"{DSML_INVOKE_PREFIX}{tool_name}{DSML_INVOKE_NAME_END}"
+    tokens.append((tid, invoke_prefix_text))
+    tid += 1
+
+    tokens.append((tid, "\n"))
+    tid += 1
+
+    for name, is_str, value in params:
+        param_text = _param(name, is_str, value)
+        tokens.append((tid, param_text))
+        tid += 1
+        tokens.append((tid, "\n"))
+        tid += 1
+
+    tokens.append((tid, DSML_INVOKE_END))
+    tid += 1
+
+    tokens.append((tid, "\n"))
+    tid += 1
+
+    tokens.append((_DSV4_FULL_VOCAB[DSML_TOOL_END], DSML_TOOL_END))
+
+    return tokens
+
+
+class TestDelegatingParserLargeDelta:
+    """Regression: tool calls lost when </think> + DSML arrive in same delta.
+
+    The DelegatingParser used by the serving layer splits reasoning and
+    tool parsing across two separate engine instances.  When </think> and
+    the entire DSML tool block arrive in a single large streaming delta,
+    the content transfer from reasoning adapter to tool adapter must
+    preserve the tool call text.
+    """
+
+    @pytest.fixture
+    def dsv4_tokens(self):
+        return _dsv4_tokens(
+            reasoning="The user wants the current weather in Berlin.",
+            tool_name="get_weather",
+            params=[
+                ("location", "true", "Berlin"),
+                ("units", "true", "celsius"),
+            ],
+        )
+
+    @pytest.fixture
+    def dsv4_tokenizer(self, dsv4_tokens):
+        return MockTokenizer(
+            vocab=dict(_DSV4_FULL_VOCAB),
+            tokens=dsv4_tokens,
+        )
+
+    @pytest.mark.parametrize(
+        "chunk_size",
+        [1, 2, 3, 5, None],
+        ids=lambda c: f"chunk={c}",
+    )
+    def test_tool_calls_extracted_at_all_chunk_sizes(
+        self, dsv4_tokenizer, dsv4_tokens, chunk_size
+    ):
+        parser = _DeepSeekV4Delegating(
+            dsv4_tokenizer,
+            chat_template_kwargs={"thinking": True},
+        )
+        deltas = replay_streaming(
+            parser,
+            dsv4_tokens,
+            chunk_size=chunk_size,
+            finished_on_last=True,
+            tools=DUMMY_TOOLS,
+        )
+        output = collect_output(deltas)
+
+        assert "The user wants" in output.reasoning
+        assert len(output.tool_calls) == 1, (
+            f"Expected 1 tool call but got {len(output.tool_calls)}; "
+            f"reasoning={output.reasoning!r}, content={output.content!r}"
+        )
+        assert output.tool_calls[0]["name"] == "get_weather"
+        args = json.loads(output.tool_calls[0]["arguments"])
+        assert args == {"location": "Berlin", "units": "celsius"}
+
+    def test_eos_drop_token_does_not_swallow_tool_calls(self):
+        """Tool calls must survive when an EOS DROP token's ID is in
+        delta_token_ids but its text is absent from delta_text.
+
+        At large stream_interval the EOS token ID arrives in the same
+        delta as </think> + tool calls but the detokenizer strips the
+        EOS text.  The scanner's _rebuild_from_anchors defers all text
+        after </think> when it can't find the EOS anchor text.  The
+        reasoning adapter's finish_streaming must flush deferred text
+        as content (with skip_tool_parsing), not as tool calls.
+        """
+        eos_text = "<｜end▁of▁sentence｜>"
+        eos_id = 128801
+        vocab = {
+            DSML_THINK_START: 128821,
+            DSML_THINK_END: 128822,
+            eos_text: eos_id,
+        }
+
+        reasoning = "The user wants weather."
+        tool_block = (
+            "\n\n"
+            + DSML_TOOL_START
+            + "\n"
+            + DSML_INVOKE_PREFIX
+            + "get_weather"
+            + DSML_INVOKE_NAME_END
+            + "\n"
+            + _param("location", "true", "Berlin")
+            + "\n"
+            + DSML_INVOKE_END
+            + "\n"
+            + DSML_TOOL_END
+        )
+        # delta_text does NOT include EOS text (detokenizer strips it)
+        full_text = reasoning + DSML_THINK_END + tool_block
+        # Build token list: word-split reasoning, then special tokens,
+        # then word-split tool block content, then EOS.
+        # EOS ID is present but its text is NOT in delta_text.
+        tokens: list[tuple[int, str]] = []
+        tid = 100
+        for word in reasoning.split(" "):
+            pfx = " " if tokens else ""
+            tokens.append((tid, pfx + word))
+            tid += 1
+        tokens.append((128822, DSML_THINK_END))
+        for ch in tool_block:
+            tokens.append((tid, ch))
+            tid += 1
+        tokens.append((eos_id, eos_text))
+
+        all_ids = [t[0] for t in tokens]
+        tokenizer = MockTokenizer(vocab=vocab, tokens=tokens)
+        request = _test_request(tools=DUMMY_TOOLS)
+
+        # All-in-one delta: EOS ID in token_ids but text NOT in
+        # delta_text (detokenizer strips EOS).  This is the scenario
+        # at large stream_interval.
+        parser = _DeepSeekV4Delegating(
+            tokenizer,
+            chat_template_kwargs={"thinking": True},
+        )
+        deltas = [
+            parser.parse_delta(
+                full_text,
+                all_ids,
+                request,
+                prompt_token_ids=[],
+                finished=True,
+            )
+        ]
+
+        output = collect_output(deltas)
+
+        assert "The user wants" in output.reasoning
+        assert len(output.tool_calls) == 1, (
+            f"Expected 1 tool call but got {len(output.tool_calls)}; "
+            f"reasoning={output.reasoning!r}, content={output.content!r}"
+        )
+        assert output.tool_calls[0]["name"] == "get_weather"
+        args = json.loads(output.tool_calls[0]["arguments"])
+        assert args == {"location": "Berlin"}

tests/parser/engine/test_delegating_replay.py

@@ -0,0 +1,196 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Replay tests for DelegatingParser with engine adapters.
+
+Exercises DelegatingParser in engine-adapter mode to verify that delegated
+routing produces correct output across chunk sizes.
+See test_replay.py for tests that target engine parsers directly.
+
+Parser discovery is automatic: any engine parser in ``registered_adapters``
+that has both tool and reasoning adapters and a builder in
+``trace_builder._BUILDERS`` is picked up with zero manual wiring.
+"""
+
+from __future__ import annotations
+
+from typing import NamedTuple
+
+import pytest
+from pydantic import TypeAdapter
+
+from tests.parser.engine.replay_harness import (
+    CHUNK_SIZES,
+    DUMMY_TOOLS,
+    MockTokenizer,
+    _test_request,
+    assert_no_terminal_leakage,
+    assert_parse_output,
+    collect_output,
+    make_mock_tokenizer,
+    parse_non_streaming,
+    replay_streaming,
+)
+from tests.parser.engine.trace_builder import _BUILDERS, build_samples
+from vllm.entrypoints.openai.chat_completion.protocol import (
+    ChatCompletionToolsParam,
+)
+from vllm.parser.abstract_parser import DelegatingParser, Parser
+from vllm.parser.engine import registered_adapters as _adapters_mod
+from vllm.parser.engine.adapters import (
+    ParserEngineReasoningAdapter,
+    ParserEngineToolAdapter,
+)
+
+_TOOLS_VALIDATOR = TypeAdapter(list[ChatCompletionToolsParam])
+
+# ── Pairing discovery ────────────────────────────────────────────────
+
+
+class _PairingInfo(NamedTuple):
+    parser_cls: type[Parser]
+    name: str
+    samples: tuple
+
+
+def _discover_pairings() -> list[_PairingInfo]:
+    """Discover valid delegating pairings from registered engine adapters.
+
+    Groups tool and reasoning adapters by their engine class, then builds
+    a DelegatingParser subclass for each engine that has both adapters
+    and a test builder.
+    """
+    bare_tok = MockTokenizer(vocab={}, tokens=[])
+    engines: dict[type, dict[str, type]] = {}
+    for obj in vars(_adapters_mod).values():
+        if not isinstance(obj, type):
+            continue
+        if (
+            issubclass(obj, ParserEngineToolAdapter)
+            and obj is not ParserEngineToolAdapter
+        ):
+            tool_adapter: type[ParserEngineToolAdapter] = obj
+            engines.setdefault(tool_adapter._parser_engine_cls, {})["tool"] = obj
+        elif (
+            issubclass(obj, ParserEngineReasoningAdapter)
+            and obj is not ParserEngineReasoningAdapter
+        ):
+            reasoning_adapter: type[ParserEngineReasoningAdapter] = obj
+            engines.setdefault(reasoning_adapter._parser_engine_cls, {})[
+                "reasoning"
+            ] = obj
+
+    found: list[_PairingInfo] = []
+    missing_builders: list[str] = []
+    for engine_cls, adapters in engines.items():
+        if "tool" not in adapters or "reasoning" not in adapters:
+            continue
+        cfg = engine_cls(bare_tok, None).parser_engine_config
+        if cfg.name not in _BUILDERS:
+            missing_builders.append(f"{engine_cls.__name__} (config.name={cfg.name!r})")
+            continue
+
+        parser_cls = type(
+            f"_Delegating{engine_cls.__name__}",
+            (DelegatingParser,),
+            {
+                "reasoning_parser_cls": adapters["reasoning"],
+                "tool_parser_cls": adapters["tool"],
+            },
+        )
+        found.append(
+            _PairingInfo(
+                parser_cls=parser_cls,
+                name=cfg.name,
+                samples=build_samples(cfg.name),
+            )
+        )
+    if missing_builders:
+        raise RuntimeError(
+            f"Engine adapters in registered_adapters have no test builder "
+            f"in trace_builder._BUILDERS: {', '.join(missing_builders)}. "
+            f"Add a builder to _BUILDERS for each new parser."
+        )
+    found.sort(key=lambda p: p.name)
+    return found
+
+
+_PAIRINGS = _discover_pairings()
+
+_ALL_SAMPLES = [(p.parser_cls, s) for p in _PAIRINGS for s in p.samples]
+
+
+@pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}")
+@pytest.mark.parametrize(
+    "parser_cls,sample",
+    _ALL_SAMPLES,
+    ids=lambda v: v.id if hasattr(v, "id") else "",
+)
+def test_delegating_replay(parser_cls, sample, chunk_size):
+    tokenizer = make_mock_tokenizer(sample)
+    validated_tools = (
+        _TOOLS_VALIDATOR.validate_python(sample.tools) if sample.tools else None
+    )
+    parser = parser_cls(
+        tokenizer,
+        validated_tools,
+        chat_template_kwargs=sample.chat_template_kwargs,
+    )
+
+    deltas = replay_streaming(
+        parser,
+        sample.tokens,
+        chunk_size=chunk_size,
+        finished_on_last=True,
+        tools=sample.tools,
+        prompt_token_ids=sample.prompt_token_ids,
+    )
+    output = collect_output(deltas)
+    assert_parse_output(output, sample)
+
+
+_TOOL_CALL_SAMPLES = [
+    (p.parser_cls, p.name, s)
+    for p in _PAIRINGS
+    for s in p.samples
+    if s.expected_tool_calls
+]
+
+
+@pytest.mark.parametrize(
+    "parser_cls,parser_name,sample",
+    _TOOL_CALL_SAMPLES,
+    ids=lambda v: v.id if hasattr(v, "id") else "",
+)
+def test_delegating_parse_tool_choice_none(parser_cls, parser_name, sample):
+    """Non-streaming parse() with tool_choice='none' via DelegatingParser
+    must not leak special tokens into content."""
+    tokenizer = make_mock_tokenizer(sample)
+    validated_tools = (
+        _TOOLS_VALIDATOR.validate_python(sample.tools) if sample.tools else None
+    )
+    parser = parser_cls(
+        tokenizer,
+        validated_tools,
+        chat_template_kwargs=sample.chat_template_kwargs,
+    )
+
+    request = _test_request(tools=DUMMY_TOOLS)
+    request.tool_choice = "none"
+
+    output = parse_non_streaming(parser, sample, request)
+
+    assert output.tool_calls == [], (
+        f"Expected no tool calls but got {output.tool_calls}"
+    )
+
+    cfg = parser._tool_parser._parser_engine.parser_engine_config
+    terminals = sorted(
+        v
+        for v in set(cfg.terminals.values()) | set(cfg.token_id_terminals.values())
+        if len(v) > 1
+    )
+    assert_no_terminal_leakage(
+        output,
+        terminals,
+        context=f"parser={parser_name}",
+    )

tests/parser/engine/test_engine.py

@@ -0,0 +1,846 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests for the streaming parser engine core pipeline."""
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from tests.parser.engine.conftest import make_mock_tokenizer
+from vllm.parser.engine.events import EventType, SemanticEvent
+from vllm.parser.engine.incremental_lexer import (
+    LexerShape,
+    TerminalDef,
+    terminals_from_literals,
+)
+from vllm.parser.engine.parser_engine_config import (
+    ParserEngineConfig,
+    ParserState,
+    Transition,
+)
+from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
+
+
+def _hermes_config() -> ParserEngineConfig:
+    """Simple Hermes-style config: <tool_call>JSON</tool_call>."""
+    return ParserEngineConfig(
+        name="hermes_test",
+        terminals={
+            "TOOL_START": "<tool_call>",
+            "TOOL_END": "</tool_call>",
+        },
+        token_id_terminals={
+            "TOOL_START": "<tool_call>",
+            "TOOL_END": "</tool_call>",
+        },
+        transitions={
+            (ParserState.CONTENT, "TOOL_START"): Transition(
+                ParserState.TOOL_ARGS,
+                (EventType.TOOL_CALL_START,),
+            ),
+            (ParserState.TOOL_ARGS, "TOOL_END"): Transition(
+                ParserState.CONTENT,
+                (EventType.TOOL_CALL_END,),
+            ),
+        },
+        content_events={
+            ParserState.CONTENT: EventType.TEXT_CHUNK,
+            ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
+        },
+    )
+
+
+def _think_config() -> ParserEngineConfig:
+    """Simple think-tag reasoning config: <think>...</think>."""
+    return ParserEngineConfig(
+        name="think_test",
+        terminals={
+            "THINK_START": "<think>",
+            "THINK_END": "</think>",
+        },
+        transitions={
+            (ParserState.CONTENT, "THINK_START"): Transition(
+                ParserState.REASONING,
+                (EventType.REASONING_START,),
+            ),
+            (ParserState.REASONING, "THINK_END"): Transition(
+                ParserState.CONTENT,
+                (EventType.REASONING_END,),
+            ),
+        },
+    )
+
+
+class TestNonStreaming:
+    def test_plain_text(self):
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+        events = engine.parse_complete("Hello, world!")
+        assert len(events) == 1
+        assert events[0].type == EventType.TEXT_CHUNK
+        assert events[0].value == "Hello, world!"
+
+    def test_single_tool_call(self):
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+        text = (
+            '<tool_call>{"name": "get_weather",'
+            ' "arguments": {"city": "SF"}}'
+            "</tool_call>"
+        )
+        events = engine.parse_complete(text)
+
+        types = [e.type for e in events]
+        assert EventType.TOOL_CALL_START in types
+        assert EventType.TOOL_CALL_END in types
+        assert EventType.ARG_VALUE_CHUNK in types
+
+        arg_text = "".join(
+            e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
+        )
+        assert '"name": "get_weather"' in arg_text
+        assert '"city": "SF"' in arg_text
+
+    def test_text_then_tool_call(self):
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+        text = 'Sure!<tool_call>{"name": "add"}</tool_call>'
+        events = engine.parse_complete(text)
+
+        types = [e.type for e in events]
+        assert types[0] == EventType.TEXT_CHUNK
+        assert events[0].value == "Sure!"
+        assert EventType.TOOL_CALL_START in types
+        assert EventType.TOOL_CALL_END in types
+
+    def test_multiple_tool_calls(self):
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+        text = (
+            '<tool_call>{"name": "a"}</tool_call><tool_call>{"name": "b"}</tool_call>'
+        )
+        events = engine.parse_complete(text)
+
+        starts = [e for e in events if e.type == EventType.TOOL_CALL_START]
+        ends = [e for e in events if e.type == EventType.TOOL_CALL_END]
+        assert len(starts) == 2
+        assert len(ends) == 2
+        assert starts[0].tool_index == 0
+        assert starts[1].tool_index == 1
+
+    def test_reasoning(self):
+        engine = StreamingParserEngine(_think_config(), tokenizer=None)
+        text = "<think>Let me think...</think>The answer is 42."
+        events = engine.parse_complete(text)
+
+        types = [e.type for e in events]
+        assert types[0] == EventType.REASONING_START
+        assert EventType.REASONING_CHUNK in types
+        assert EventType.REASONING_END in types
+        assert EventType.TEXT_CHUNK in types
+
+        reasoning = "".join(
+            e.value for e in events if e.type == EventType.REASONING_CHUNK
+        )
+        assert "Let me think..." in reasoning
+
+        content = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
+        assert "The answer is 42." in content
+
+
+class TestStreaming:
+    @staticmethod
+    def _feed_chars(
+        engine: StreamingParserEngine,
+        text: str,
+    ) -> list[SemanticEvent]:
+        """Feed text one character at a time."""
+        all_events = []
+        for ch in text:
+            all_events.extend(engine.feed(ch, []))
+        all_events.extend(engine.finish())
+        return all_events
+
+    @staticmethod
+    def _feed_chunks(
+        engine: StreamingParserEngine,
+        text: str,
+        chunk_size: int,
+    ) -> list[SemanticEvent]:
+        """Feed text in fixed-size chunks."""
+        all_events = []
+        for i in range(0, len(text), chunk_size):
+            chunk = text[i : i + chunk_size]
+            all_events.extend(engine.feed(chunk, []))
+        all_events.extend(engine.finish())
+        return all_events
+
+    def test_char_by_char_tool_call(self):
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+        text = '<tool_call>{"name": "add", "arguments": {"a": 1}}</tool_call>'
+        events = self._feed_chars(engine, text)
+
+        types = [e.type for e in events]
+        assert EventType.TOOL_CALL_START in types
+        assert EventType.TOOL_CALL_END in types
+        assert EventType.ARG_VALUE_CHUNK in types
+
+        arg_text = "".join(
+            e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
+        )
+        assert '"name": "add"' in arg_text
+
+    @pytest.mark.parametrize(
+        "text",
+        [
+            '<tool_call>{"name": "get", "arguments": {"x": "hello"}}</tool_call>',
+            '<tool_call>{"name": "f", "arguments": '
+            '{"items": [1, [2, 3]], "obj": {"k": "v"}}}'
+            "</tool_call>",
+        ],
+        ids=["flat_args", "nested_arrays"],
+    )
+    def test_chunk_sizes_produce_same_content(self, text):
+        """Different chunk sizes must produce identical concatenated content."""
+        results = {}
+        for chunk_size in [1, 2, 3, 5, 7, len(text)]:
+            engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+            events = self._feed_chunks(engine, text, chunk_size)
+            arg_text = "".join(
+                e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
+            )
+            results[chunk_size] = arg_text
+
+        values = list(results.values())
+        for v in values[1:]:
+            assert v == values[0], f"Mismatch: {results}"
+
+    def test_prefix_buffering_prevents_premature_emit(self):
+        """Text like '<tool_' should be buffered, not emitted as content."""
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+
+        events1 = engine.feed("<tool_", [])
+        content_events = [e for e in events1 if e.type == EventType.TEXT_CHUNK]
+        assert len(content_events) == 0, "Should buffer partial tag"
+
+        events2 = engine.feed("call>", [])
+        starts = [e for e in events2 if e.type == EventType.TOOL_CALL_START]
+        assert len(starts) == 1
+
+    def test_prefix_buffering_flush_on_mismatch(self):
+        """Text like '<tool_box' should eventually flush as content."""
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+
+        events1 = engine.feed("<tool_", [])
+        assert len([e for e in events1 if e.type == EventType.TEXT_CHUNK]) == 0
+
+        events2 = engine.feed("box>rest", [])
+        events2.extend(engine.finish())
+        content = "".join(e.value for e in events2 if e.type == EventType.TEXT_CHUNK)
+        assert content == "<tool_box>rest"
+
+    def test_reasoning_streaming(self):
+        engine = StreamingParserEngine(_think_config(), tokenizer=None)
+        events = self._feed_chars(engine, "<think>hmm</think>answer")
+
+        reasoning = "".join(
+            e.value for e in events if e.type == EventType.REASONING_CHUNK
+        )
+        content = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
+        assert "hmm" in reasoning
+        assert "answer" in content
+
+    def test_text_between_tool_calls(self):
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+        text = (
+            'Hi<tool_call>{"name":"a"}</tool_call>'
+            'mid<tool_call>{"name":"b"}</tool_call>end'
+        )
+        events = self._feed_chunks(engine, text, 3)
+
+        texts = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
+        assert "Hi" in texts
+        assert "mid" in texts
+        assert "end" in texts
+
+        starts = [e for e in events if e.type == EventType.TOOL_CALL_START]
+        assert len(starts) == 2
+
+    def test_unmatched_close_brace_does_not_poison_depth(self):
+        """A stray } in malformed JSON must not kill streaming for
+        all subsequent content."""
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+        engine.feed("<tool_call>", [])
+
+        malformed = '}{{"a": 1}}'
+        events = self._feed_chars(engine, malformed + "</tool_call>")
+
+        arg_chunks = [e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK]
+        assert len(arg_chunks) > 1, (
+            "Content after stray } should still stream incrementally"
+        )
+        arg_text = "".join(arg_chunks)
+        assert '"a": 1' in arg_text
+
+    def test_json_args_no_premature_close_brace(self):
+        """Closing braces of the top-level JSON shouldn't be streamed
+        until confirmed by the end tag."""
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+
+        engine.feed("<tool_call>", [])
+        events = engine.feed('{"name": "f"}', [])
+
+        arg_text = "".join(
+            e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
+        )
+        assert "}" not in arg_text, "Top-level } should be held back"
+
+        events2 = engine.feed("</tool_call>", [])
+        arg_text2 = "".join(
+            e.value for e in events2 if e.type == EventType.ARG_VALUE_CHUNK
+        )
+        assert "}" in arg_text2, "} should flush on end tag"
+
+
+_START_ID = 50
+_END_ID = 51
+_TOOL_START_ID = 60
+_TOOL_END_ID = 61
+
+
+def _make_think_tokenizer():
+    tok = MagicMock()
+    tok.encode.return_value = [1, 2, 3]
+    tok.get_vocab.return_value = {"<think>": _START_ID, "</think>": _END_ID}
+    tok.decode.side_effect = lambda ids: {
+        _START_ID: "<think>",
+        _END_ID: "</think>",
+    }.get(ids[0], f"tok{ids[0]}")
+    return tok
+
+
+def _make_hermes_tokenizer():
+    """Tokenizer that resolves tool_call tags to special IDs."""
+    _special = {_TOOL_START_ID: "<tool_call>", _TOOL_END_ID: "</tool_call>"}
+    tok = MagicMock()
+    tok.encode.return_value = [1, 2, 3]
+    tok.get_vocab.return_value = {
+        "<tool_call>": _TOOL_START_ID,
+        "</tool_call>": _TOOL_END_ID,
+    }
+    tok.decode.side_effect = lambda ids: "".join(
+        _special.get(i, chr(i) if i < 128 else f"<{i}>") for i in ids
+    )
+    return tok
+
+
+class TestLexerBufferFlush:
+    """Lexer buffer must be flushed before PreLexedTerminal transitions."""
+
+    def test_buffered_prefix_emitted_in_current_state(self):
+        """Text buffered by the lexer (e.g. '<') must be emitted as
+        REASONING_CHUNK before THINK_END transitions to CONTENT."""
+        engine = StreamingParserEngine(_think_config(), _make_think_tokenizer())
+
+        events = engine.feed("<think>", [_START_ID])
+        assert any(e.type == EventType.REASONING_START for e in events)
+
+        events = engine.feed("reasoning text<", [])
+        reasoning_text = "".join(
+            e.value for e in events if e.type == EventType.REASONING_CHUNK
+        )
+        assert "reasoning text" in reasoning_text
+
+        events = engine.feed("</think>", [_END_ID])
+        event_types = [e.type for e in events]
+        if EventType.REASONING_CHUNK in event_types:
+            rc_idx = event_types.index(EventType.REASONING_CHUNK)
+            re_idx = event_types.index(EventType.REASONING_END)
+            assert rc_idx < re_idx, (
+                "'<' must be emitted as REASONING_CHUNK before REASONING_END"
+            )
+            flushed = events[rc_idx].value
+            assert "<" in flushed
+
+    def test_empty_buffer_no_extra_events(self):
+        """When the lexer buffer is empty, flushing is a no-op."""
+        engine = StreamingParserEngine(_think_config(), _make_think_tokenizer())
+
+        engine.feed("<think>", [_START_ID])
+        engine.feed("clean text", [])
+
+        events = engine.feed("</think>", [_END_ID])
+        assert any(e.type == EventType.REASONING_END for e in events)
+        chunk_events = [e for e in events if e.type == EventType.REASONING_CHUNK]
+        assert all(e.value for e in chunk_events)
+
+
+class TestTokenIdFiltering:
+    """When token IDs are available, lex-matched terminals that also
+    have token_id_terminal entries should be demoted to content."""
+
+    def test_lex_matched_terminal_demoted_after_token_ids_seen(self):
+        """After receiving token IDs, text that matches a token-ID
+        terminal should be treated as content, not trigger a transition."""
+        engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
+
+        # First feed with a non-special token ID to set _ever_had_token_ids
+        engine.feed("prefix ", [1])
+
+        # Now feed text containing <tool_call> as literal text
+        events = engine.feed(
+            "Use <tool_call> to invoke tools.</tool_call>", [2, 3, 4, 5]
+        )
+        events.extend(engine.finish())
+
+        types = [e.type for e in events]
+        assert EventType.TOOL_CALL_START not in types
+        assert EventType.TEXT_CHUNK in types
+
+        text = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
+        assert "<tool_call>" in text
+
+    def test_scanner_matched_terminal_bypasses_filter(self):
+        """PreLexedTerminals from the scanner bypass the filter and
+        still trigger state transitions."""
+        engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
+
+        events = engine.feed("<tool_call>", [_TOOL_START_ID])
+        assert any(e.type == EventType.TOOL_CALL_START for e in events)
+
+        events = engine.feed('{"name": "f"}', [2, 3])
+        events.extend(engine.feed("</tool_call>", [_TOOL_END_ID]))
+        events.extend(engine.finish())
+        assert any(e.type == EventType.TOOL_CALL_END for e in events)
+
+    def test_no_filtering_without_token_ids(self):
+        """When no token IDs are ever provided (non-streaming),
+        text matching still triggers transitions."""
+        engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
+
+        events = engine.feed('<tool_call>{"name": "f"}</tool_call>', [])
+        events.extend(engine.finish())
+
+        types = [e.type for e in events]
+        assert EventType.TOOL_CALL_START in types
+        assert EventType.TOOL_CALL_END in types
+
+    def test_mixed_text_then_real_tool_call(self):
+        """Text mentioning tool syntax followed by a real special-token
+        tool call."""
+        engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
+
+        events1 = engine.feed("Mention <tool_call> in text. ", [1, 2, 3, 4])
+        events2 = engine.feed("<tool_call>", [_TOOL_START_ID])
+        events3 = engine.feed('{"name": "a"}', [5, 6])
+        events4 = engine.feed("</tool_call>", [_TOOL_END_ID])
+        events4.extend(engine.finish())
+
+        all_events = events1 + events2 + events3 + events4
+
+        content = "".join(e.value for e in all_events if e.type == EventType.TEXT_CHUNK)
+        assert "<tool_call>" in content
+
+        assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_START) == 1
+        assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_END) == 1
+
+
+def _func_prefix_config() -> ParserEngineConfig:
+    """Config mixing token-ID terminals (TOOL_START/END) with
+    text-only terminals (FUNC_PREFIX) and fallback transitions."""
+    return ParserEngineConfig(
+        name="func_prefix_test",
+        terminals={
+            "TOOL_START": "<tool_call>",
+            "TOOL_END": "</tool_call>",
+            "FUNC_PREFIX": "<function=",
+            "FUNC_END": "</function>",
+            "CLOSE_ANGLE": ">",
+        },
+        token_id_terminals={
+            "TOOL_START": "<tool_call>",
+            "TOOL_END": "</tool_call>",
+        },
+        transitions={
+            (ParserState.CONTENT, "TOOL_START"): Transition(
+                ParserState.TOOL_PREAMBLE,
+                (EventType.TOOL_CALL_START,),
+            ),
+            (ParserState.CONTENT, "FUNC_PREFIX"): Transition(
+                ParserState.TOOL_NAME,
+                (EventType.TOOL_CALL_START,),
+                skip_in_token_id_mode=True,
+            ),
+            (ParserState.TOOL_PREAMBLE, "FUNC_PREFIX"): Transition(
+                ParserState.TOOL_NAME,
+                (),
+            ),
+            (ParserState.TOOL_NAME, "CLOSE_ANGLE"): Transition(
+                ParserState.TOOL_ARGS,
+                (),
+            ),
+            (ParserState.TOOL_ARGS, "FUNC_END"): Transition(
+                ParserState.TOOL_BETWEEN,
+                (EventType.TOOL_CALL_END,),
+            ),
+            (ParserState.TOOL_BETWEEN, "TOOL_END"): Transition(
+                ParserState.CONTENT,
+                (),
+            ),
+            (ParserState.TOOL_BETWEEN, "TOOL_START"): Transition(
+                ParserState.TOOL_PREAMBLE,
+                (EventType.TOOL_CALL_START,),
+            ),
+            (ParserState.TOOL_BETWEEN, "FUNC_PREFIX"): Transition(
+                ParserState.TOOL_NAME,
+                (EventType.TOOL_CALL_START,),
+                skip_in_token_id_mode=True,
+            ),
+        },
+        content_events={
+            ParserState.CONTENT: EventType.TEXT_CHUNK,
+            ParserState.TOOL_NAME: EventType.TOOL_NAME,
+            ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
+        },
+    )
+
+
+def _make_func_prefix_tokenizer():
+    return make_mock_tokenizer(
+        {
+            "<tool_call>": _TOOL_START_ID,
+            "</tool_call>": _TOOL_END_ID,
+        }
+    )
+
+
+class TestTextOnlyFallbackFiltering:
+    """When token IDs are available, transitions marked
+    skip_in_token_id_mode should be skipped."""
+
+    def test_func_prefix_in_prose_demoted_in_strict_mode(self):
+        """<function=get_time> in prose should NOT trigger a tool call
+        when strict mode is active."""
+        engine = StreamingParserEngine(
+            _func_prefix_config(), _make_func_prefix_tokenizer()
+        )
+        engine.feed("prefix ", [1])
+
+        events = engine.feed("Use <function=get_time> to check.", [2, 3, 4, 5])
+        events.extend(engine.finish())
+
+        types = [e.type for e in events]
+        assert EventType.TOOL_CALL_START not in types
+        assert EventType.TEXT_CHUNK in types
+        text = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
+        assert "<function=" in text
+
+    def test_normal_flow_after_tool_start_still_works(self):
+        """TOOL_START (special token) -> FUNC_PREFIX (text) should
+        still parse a tool call normally in strict mode."""
+        engine = StreamingParserEngine(
+            _func_prefix_config(), _make_func_prefix_tokenizer()
+        )
+
+        events1 = engine.feed("<tool_call>", [_TOOL_START_ID])
+        assert any(e.type == EventType.TOOL_CALL_START for e in events1)
+
+        events2 = engine.feed("<function=get_weather>", [2, 3])
+        events3 = engine.feed("args", [4])
+        events4 = engine.feed("</function>", [5, 6])
+        events4.extend(engine.feed("</tool_call>", [_TOOL_END_ID]))
+        events4.extend(engine.finish())
+
+        all_events = events1 + events2 + events3 + events4
+        assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_START) == 1
+        assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_END) == 1
+
+    def test_fallback_fires_without_token_ids(self):
+        """When no token IDs are provided, fallback transitions should
+        still fire normally."""
+        engine = StreamingParserEngine(
+            _func_prefix_config(), _make_func_prefix_tokenizer()
+        )
+
+        events = engine.feed("<function=get_time>args</function>", [])
+        events.extend(engine.finish())
+
+        types = [e.type for e in events]
+        assert EventType.TOOL_CALL_START in types
+        assert EventType.TOOL_CALL_END in types
+
+    def test_tool_between_fallback_blocked_in_strict_mode(self):
+        """The (TOOL_BETWEEN, FUNC_PREFIX) fallback should also be
+        blocked in strict mode."""
+        engine = StreamingParserEngine(
+            _func_prefix_config(), _make_func_prefix_tokenizer()
+        )
+
+        engine.feed("<tool_call>", [_TOOL_START_ID])
+        engine.feed("<function=a>", [2, 3])
+        engine.feed("args", [4])
+        engine.feed("</function>", [5, 6])
+        engine.feed("</tool_call>", [_TOOL_END_ID])
+
+        events = engine.feed("<function=b>more</function>", [7, 8, 9])
+        events.extend(engine.finish())
+
+        types = [e.type for e in events]
+        assert EventType.TOOL_CALL_START not in types
+
+
+class TestNoUnusedTokenizerAttr:
+    """StreamingParserEngine no longer stores a redundant _tokenizer."""
+
+    def test_no_tokenizer_attribute(self):
+        config = ParserEngineConfig(name="test")
+        engine = StreamingParserEngine(config, tokenizer=None)
+        assert not hasattr(engine, "_tokenizer")
+
+
+class TestArgsResetOnReentry:
+    """When leaving TOOL_ARGS and later re-entering (e.g. two tool
+    calls), the entering-TOOL_ARGS block resets args tracking.  The
+    redundant reset on exit was removed."""
+
+    @staticmethod
+    def _multi_tool_config() -> ParserEngineConfig:
+        return ParserEngineConfig(
+            name="multi_tool",
+            terminals={
+                "TOOL_START": "<tool_call>",
+                "TOOL_END": "</tool_call>",
+                "TOOL_SEP": "<tool_sep>",
+            },
+            transitions={
+                (ParserState.CONTENT, "TOOL_START"): Transition(
+                    ParserState.TOOL_ARGS,
+                    (EventType.TOOL_CALL_START,),
+                ),
+                (ParserState.TOOL_ARGS, "TOOL_END"): Transition(
+                    ParserState.TOOL_BETWEEN,
+                    (EventType.TOOL_CALL_END,),
+                ),
+                (ParserState.TOOL_BETWEEN, "TOOL_SEP"): Transition(
+                    ParserState.TOOL_ARGS,
+                    (EventType.TOOL_CALL_START,),
+                ),
+            },
+            content_events={
+                ParserState.CONTENT: EventType.TEXT_CHUNK,
+                ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
+            },
+        )
+
+    def test_args_tracking_across_reentry(self):
+        engine = StreamingParserEngine(self._multi_tool_config(), tokenizer=None)
+
+        events = engine.feed(
+            '<tool_call>{"city": "SF"}</tool_call>'
+            "<tool_sep>"
+            '{"name": "bar"}</tool_call>',
+            [],
+        )
+
+        tool_starts = [e for e in events if e.type == EventType.TOOL_CALL_START]
+        tool_ends = [e for e in events if e.type == EventType.TOOL_CALL_END]
+        arg_chunks = [e for e in events if e.type == EventType.ARG_VALUE_CHUNK]
+
+        assert len(tool_starts) == 2
+        assert len(tool_ends) == 2
+        assert tool_starts[0].tool_index == 0
+        assert tool_starts[1].tool_index == 1
+
+        first_args = "".join(e.value for e in arg_chunks if e.tool_index == 0)
+        second_args = "".join(e.value for e in arg_chunks if e.tool_index == 1)
+        assert '"city"' in first_args
+        assert '"name"' in second_args
+
+    def test_brace_depth_resets_on_reentry(self):
+        """Verify _args_brace_depth resets when re-entering TOOL_ARGS."""
+        engine = StreamingParserEngine(self._multi_tool_config(), tokenizer=None)
+        engine.feed("<tool_call>", [])
+        assert engine.state == ParserState.TOOL_ARGS
+        assert engine._args_brace_depth == 0
+
+        engine.feed('{"a": 1}', [])
+        engine.feed("</tool_call>", [])
+        assert engine.state == ParserState.TOOL_BETWEEN
+
+        engine.feed("<tool_sep>", [])
+        assert engine.state == ParserState.TOOL_ARGS
+        assert engine._args_brace_depth == 0
+        assert engine._args_in_string is False
+        assert engine._args_escape_next is False
+
+
+class TestToolPreambleFinish:
+    """finish() in TOOL_PREAMBLE state emits TOOL_CALL_END when a tool
+    call was started (tool_index >= 0), but not when tool_index is -1."""
+
+    @staticmethod
+    def _preamble_with_tool_call_start_config() -> ParserEngineConfig:
+        return ParserEngineConfig(
+            name="preamble_tcs",
+            terminals={"TOOL_START": "<tool_call>"},
+            transitions={
+                (ParserState.CONTENT, "TOOL_START"): Transition(
+                    ParserState.TOOL_PREAMBLE,
+                    (EventType.TOOL_CALL_START,),
+                ),
+            },
+            content_events={ParserState.CONTENT: EventType.TEXT_CHUNK},
+        )
+
+    @staticmethod
+    def _preamble_without_tool_call_start_config() -> ParserEngineConfig:
+        return ParserEngineConfig(
+            name="preamble_no_tcs",
+            terminals={"TOOL_CALLS_START": "<tool_calls>"},
+            transitions={
+                (ParserState.CONTENT, "TOOL_CALLS_START"): Transition(
+                    ParserState.TOOL_PREAMBLE,
+                    (),
+                ),
+            },
+            content_events={ParserState.CONTENT: EventType.TEXT_CHUNK},
+        )
+
+    def test_finish_emits_tool_call_end_with_tool_index(self):
+        config = self._preamble_with_tool_call_start_config()
+        engine = StreamingParserEngine(config, tokenizer=None)
+
+        engine.feed("<tool_call>", [])
+        assert engine.state == ParserState.TOOL_PREAMBLE
+        assert engine.tool_index == 0
+
+        finish_events = engine.finish()
+        end_events = [e for e in finish_events if e.type == EventType.TOOL_CALL_END]
+        assert len(end_events) == 1
+        assert end_events[0].tool_index == 0
+
+    def test_finish_no_tool_call_end_without_tool_index(self):
+        config = self._preamble_without_tool_call_start_config()
+        engine = StreamingParserEngine(config, tokenizer=None)
+
+        engine.feed("<tool_calls>", [])
+        assert engine.state == ParserState.TOOL_PREAMBLE
+        assert engine.tool_index == -1
+
+        finish_events = engine.finish()
+        end_events = [e for e in finish_events if e.type == EventType.TOOL_CALL_END]
+        assert len(end_events) == 0
+        assert engine.state == ParserState.CONTENT
+
+
+class TestRegexTerminalInfraRemoved:
+    """TerminalDef.priority, LexerShape.regex_terminals, and the regex
+    matching loop were removed."""
+
+    def test_terminal_def_no_priority(self):
+        import regex as re
+
+        td = TerminalDef(name="X", pattern=re.compile("x"))
+        assert not hasattr(td, "priority")
+
+    def test_lexer_shape_no_regex_terminals(self):
+        shape = LexerShape([])
+        assert not hasattr(shape, "regex_terminals")
+
+    def test_terminals_from_literals_still_works(self):
+        literals = {"TOOL_START": "<tool_call>", "TOOL_END": "</tool_call>"}
+        defs = terminals_from_literals(literals)
+        assert len(defs) == 2
+        names = {d.name for d in defs}
+        assert names == {"TOOL_START", "TOOL_END"}
+        for d in defs:
+            assert d.is_literal
+            assert d.literal in ("<tool_call>", "</tool_call>")
+
+
+class TestMultiCharTerminalInArgs:
+    """Regression: multi-char terminals falling through in TOOL_ARGS
+    must be fed char-by-char via _feed_args_text, not _feed_args_char."""
+
+    @staticmethod
+    def _newline_config() -> ParserEngineConfig:
+        return ParserEngineConfig(
+            name="newline_test",
+            terminals={
+                "TOOL_START": "<tool_call>",
+                "TOOL_END": "</tool_call>",
+                "NEWLINE": "\n",
+            },
+            transitions={
+                (ParserState.CONTENT, "TOOL_START"): Transition(
+                    ParserState.TOOL_ARGS,
+                    (EventType.TOOL_CALL_START,),
+                ),
+                (ParserState.TOOL_ARGS, "TOOL_END"): Transition(
+                    ParserState.CONTENT,
+                    (EventType.TOOL_CALL_END,),
+                ),
+            },
+            content_events={
+                ParserState.CONTENT: EventType.TEXT_CHUNK,
+                ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
+            },
+        )
+
+    def test_newline_in_args_parsed_correctly(self):
+        engine = StreamingParserEngine(self._newline_config(), tokenizer=None)
+        text = '<tool_call>{"name": "f",\n"arguments": {"a": 1}}</tool_call>'
+        events = engine.parse_complete(text)
+
+        arg_text = "".join(
+            e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
+        )
+        assert '"name": "f"' in arg_text
+        assert '"arguments"' in arg_text
+
+    def test_newline_in_args_streaming(self):
+        engine = StreamingParserEngine(self._newline_config(), tokenizer=None)
+        all_events = TestStreaming._feed_chars(
+            engine, '<tool_call>{"name": "f",\n"a": 1}</tool_call>'
+        )
+
+        arg_text = "".join(
+            e.value for e in all_events if e.type == EventType.ARG_VALUE_CHUNK
+        )
+        assert '"name": "f"' in arg_text
+        assert '"a": 1' in arg_text
+
+
+class TestSkipToolParsing:
+    """When skip_tool_parsing is set, tool tags become content."""
+
+    def test_tool_tags_emitted_as_content(self):
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+        engine.skip_tool_parsing = True
+
+        text = '<tool_call>{"name": "f"}</tool_call>'
+        events = engine.parse_complete(text)
+
+        types = [e.type for e in events]
+        assert EventType.TOOL_CALL_START not in types
+        assert EventType.TOOL_CALL_END not in types
+
+        content = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
+        assert "<tool_call>" in content
+        assert "</tool_call>" in content
+
+    def test_skip_tool_streaming(self):
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+        engine.skip_tool_parsing = True
+
+        all_events = TestStreaming._feed_chars(
+            engine, '<tool_call>{"name": "f"}</tool_call>'
+        )
+
+        types = [e.type for e in all_events]
+        assert EventType.TOOL_CALL_START not in types
+
+        content = "".join(e.value for e in all_events if e.type == EventType.TEXT_CHUNK)
+        assert "<tool_call>" in content
+
+    def test_reset_preserves_skip_tool_parsing(self):
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+        engine.skip_tool_parsing = True
+        engine.reset()
+        assert engine.skip_tool_parsing is True

tests/parser/engine/test_minimax_m2.py

@@ -0,0 +1,264 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import json
+
+import pytest
+
+from tests.parser.engine.conftest import make_mock_tokenizer
+from tests.parser.engine.streaming_helpers import (
+    collect_function_name,
+    collect_tool_arguments,
+    simulate_tool_streaming,
+)
+from vllm.entrypoints.openai.chat_completion.protocol import (
+    ChatCompletionToolsParam,
+    FunctionDefinition,
+)
+from vllm.parser.minimax_m2 import (
+    THINK_END,
+    TOOL_CALL_END,
+    TOOL_CALL_START,
+    MinimaxM2Parser,
+)
+
+
+@pytest.fixture
+def mock_tokenizer():
+    return make_mock_tokenizer(
+        {
+            THINK_END: 99,
+            TOOL_CALL_START: 100,
+            TOOL_CALL_END: 101,
+        }
+    )
+
+
+@pytest.fixture
+def parser(mock_tokenizer):
+    return MinimaxM2Parser(mock_tokenizer)
+
+
+def make_tools(*names: str):
+    return [
+        ChatCompletionToolsParam(
+            function=FunctionDefinition(
+                name=name,
+                parameters={
+                    "type": "object",
+                    "properties": {},
+                },
+            ),
+        )
+        for name in names
+    ]
+
+
+class TestNonStreaming:
+    def test_no_tool_calls(self, parser, mock_request):
+        result = parser.extract_tool_calls(
+            "</think>This is a regular response without tool calls.",
+            mock_request,
+        )
+        assert result.tools_called is False
+        assert result.tool_calls == []
+        assert result.content == "This is a regular response without tool calls."
+
+    def test_single_tool_call(self, parser, mock_request):
+        result = parser.extract_tool_calls(
+            '<minimax:tool_call><invoke name="get_weather">'
+            '<parameter name="city">Seattle</parameter>'
+            "</invoke></minimax:tool_call>",
+            mock_request,
+        )
+
+        assert result.tools_called is True
+        assert len(result.tool_calls) == 1
+        assert result.tool_calls[0].function.name == "get_weather"
+        assert json.loads(result.tool_calls[0].function.arguments) == {
+            "city": "Seattle",
+        }
+
+    def test_multiple_invokes(self, parser, mock_request):
+        result = parser.extract_tool_calls(
+            "<minimax:tool_call>"
+            '<invoke name="search"><parameter name="q">OpenAI</parameter></invoke>'
+            '<invoke name="search"><parameter name="q">vLLM</parameter></invoke>'
+            "</minimax:tool_call>",
+            mock_request,
+        )
+
+        assert result.tools_called is True
+        assert [tc.function.name for tc in result.tool_calls] == ["search", "search"]
+        assert json.loads(result.tool_calls[0].function.arguments) == {"q": "OpenAI"}
+        assert json.loads(result.tool_calls[1].function.arguments) == {"q": "vLLM"}
+
+    def test_schema_type_coercion(self, mock_tokenizer, mock_request):
+        tools = [
+            ChatCompletionToolsParam(
+                function=FunctionDefinition(
+                    name="forecast",
+                    parameters={
+                        "type": "object",
+                        "properties": {
+                            "days": {"type": "integer"},
+                            "include_hourly": {"type": "boolean"},
+                        },
+                    },
+                ),
+            )
+        ]
+        parser = MinimaxM2Parser(mock_tokenizer, tools=tools)
+        mock_request.tools = tools
+
+        result = parser.extract_tool_calls(
+            '<minimax:tool_call><invoke name="forecast">'
+            '<parameter name="days">5</parameter>'
+            '<parameter name="include_hourly">true</parameter>'
+            "</invoke></minimax:tool_call>",
+            mock_request,
+        )
+
+        assert json.loads(result.tool_calls[0].function.arguments) == {
+            "days": 5,
+            "include_hourly": True,
+        }
+
+    def test_invalid_tool_name_is_rejected(self, mock_tokenizer, mock_request):
+        tools = make_tools("search")
+        parser = MinimaxM2Parser(mock_tokenizer)
+        mock_request.tools = tools
+
+        result = parser.extract_tool_calls(
+            '<minimax:tool_call><invoke name="img_gen">'
+            '<parameter name="prompt">a cat</parameter>'
+            "</invoke></minimax:tool_call>",
+            mock_request,
+        )
+
+        assert result.tools_called is False
+        assert result.tool_calls == []
+
+    def test_mixed_tool_names_only_return_valid(self, mock_tokenizer, mock_request):
+        tools = make_tools("search")
+        parser = MinimaxM2Parser(mock_tokenizer)
+        mock_request.tools = tools
+
+        result = parser.extract_tool_calls(
+            "<minimax:tool_call>"
+            '<invoke name="img_gen"><parameter name="prompt">cat</parameter></invoke>'
+            '<invoke name="search"><parameter name="query">news</parameter></invoke>'
+            "</minimax:tool_call>",
+            mock_request,
+        )
+
+        assert result.tools_called is True
+        assert [tc.function.name for tc in result.tool_calls] == ["search"]
+        assert json.loads(result.tool_calls[0].function.arguments) == {
+            "query": "news",
+        }
+
+
+class TestStreaming:
+    def test_streaming_single_tool_call(self, parser, mock_request):
+        results = simulate_tool_streaming(
+            parser,
+            mock_request,
+            [
+                "<minimax:tool_call>",
+                '<invoke name="get_weather">',
+                '<parameter name="city">Seattle</parameter>',
+                "</invoke></minimax:tool_call>",
+            ],
+        )
+
+        assert collect_function_name(results) == "get_weather"
+        assert json.loads(collect_tool_arguments(results)) == {
+            "city": "Seattle",
+        }
+
+    def test_streaming_multiple_invokes(self, parser, mock_request):
+        results = simulate_tool_streaming(
+            parser,
+            mock_request,
+            [
+                "<minimax:tool_call>",
+                '<invoke name="a"><parameter name="x">1</parameter></invoke>',
+                '<invoke name="b"><parameter name="y">2</parameter></invoke>',
+                "</minimax:tool_call>",
+            ],
+        )
+
+        tool_names = [
+            tc.function.name
+            for delta, _ in results
+            if delta and delta.tool_calls
+            for tc in delta.tool_calls
+            if tc.function and tc.function.name
+        ]
+        assert tool_names == ["a", "b"]
+
+    def test_streaming_invoke_prefix_split_before_quote(self, parser, mock_request):
+        results = simulate_tool_streaming(
+            parser,
+            mock_request,
+            [
+                "<minimax:tool_call>",
+                "<invoke name=",
+                '"get_weather">',
+                '<parameter name="city">Seattle</parameter>',
+                "</invoke></minimax:tool_call>",
+            ],
+        )
+
+        assert collect_function_name(results) == "get_weather"
+        assert json.loads(collect_tool_arguments(results)) == {
+            "city": "Seattle",
+        }
+
+    def test_streaming_invalid_tool_name_is_rejected(
+        self, mock_tokenizer, mock_request
+    ):
+        tools = make_tools("search")
+        parser = MinimaxM2Parser(mock_tokenizer)
+        mock_request.tools = tools
+
+        results = simulate_tool_streaming(
+            parser,
+            mock_request,
+            [
+                "<minimax:tool_call>",
+                '<invoke name="img_gen">',
+                '<parameter name="prompt">cat</parameter>',
+                "</invoke></minimax:tool_call>",
+            ],
+        )
+
+        assert collect_function_name(results) is None
+        assert collect_tool_arguments(results) == ""
+
+
+class TestReasoning:
+    def test_extract_reasoning_without_start_token(self, parser, mock_request):
+        reasoning, content = parser.extract_reasoning(
+            "This is reasoning</think>This is content",
+            mock_request,
+        )
+
+        assert reasoning == "This is reasoning"
+        assert content == "This is content"
+
+    def test_extract_reasoning_without_end_token(self, parser, mock_request):
+        reasoning, content = parser.extract_reasoning(
+            "This is still reasoning",
+            mock_request,
+        )
+
+        assert reasoning == "This is still reasoning"
+        assert content is None
+
+    def test_extract_content_ids_without_end_token(self, parser):
+        assert parser.extract_content_ids([1, 2, 3]) == []
+
+    def test_extract_content_ids_after_end_token(self, parser):
+        assert parser.extract_content_ids([1, 99, 2, 3]) == [2, 3]

tests/parser/engine/test_nemotron_v3.py

@@ -0,0 +1,303 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests for the engine-based Nemotron V3 parser.
+
+Validates that ``NemotronV3Parser`` correctly handles:
+- ``<think>``/``</think>`` reasoning with ``<tool_call>`` XML tool calls
+  (same format as Qwen3)
+- Nemotron-specific reasoning/content swap when ``enable_thinking=False``
+  or ``force_nonempty_content=True``
+"""
+
+import json
+from unittest.mock import MagicMock
+
+import pytest
+
+from tests.parser.engine.conftest import make_mock_tokenizer
+from tests.parser.engine.streaming_helpers import (
+    collect_function_name,
+    collect_tool_arguments,
+    simulate_tool_streaming,
+)
+from vllm.entrypoints.openai.chat_completion.protocol import (
+    ChatCompletionRequest,
+)
+from vllm.parser.nemotron_v3 import NemotronV3Parser
+
+_THINK_START_ID = 50
+_THINK_END_ID = 51
+_TOOL_CALL_ID = 60
+_TOOL_CALL_END_ID = 61
+_TEXT_ID = 100
+
+_VOCAB = {
+    "<think>": _THINK_START_ID,
+    "</think>": _THINK_END_ID,
+    "<tool_call>": _TOOL_CALL_ID,
+    "</tool_call>": _TOOL_CALL_END_ID,
+}
+
+
+def _make_request(**chat_template_kwargs):
+    request = MagicMock(spec=ChatCompletionRequest)
+    request.tools = []
+    request.tool_choice = "auto"
+    request.include_reasoning = True
+    request.chat_template_kwargs = chat_template_kwargs or None
+    return request
+
+
+@pytest.fixture
+def parser():
+    return NemotronV3Parser(make_mock_tokenizer(_VOCAB))
+
+
+class TestNemotronSwap:
+    def test_enable_thinking_false_swaps(self, parser):
+        """When enable_thinking=False, model output without think tags
+        should have reasoning swapped to content."""
+        text = "The answer is 42."
+        request = _make_request(enable_thinking=False)
+        reasoning, content = parser.extract_reasoning(text, request)
+        assert content == "The answer is 42."
+        assert reasoning is None
+
+    def test_force_nonempty_content_swaps(self, parser):
+        """force_nonempty_content=True triggers swap when content empty."""
+        text = "The answer is 42."
+        request = _make_request(force_nonempty_content=True)
+        reasoning, content = parser.extract_reasoning(text, request)
+        assert content == "The answer is 42."
+        assert reasoning is None
+
+    def test_no_swap_when_content_exists(self, parser):
+        """With enable_thinking=False but real </think> giving content,
+        no swap occurs."""
+        text = "Some reasoning.</think>Actual content here."
+        request = _make_request(enable_thinking=False)
+        reasoning, content = parser.extract_reasoning(text, request)
+        assert reasoning == "Some reasoning."
+        assert content == "Actual content here."
+
+    def test_no_swap_when_enable_thinking_true(self, parser):
+        """Normal thinking mode: no swap, even when content is empty."""
+        text = "Still thinking..."
+        request = _make_request(enable_thinking=True)
+        reasoning, content = parser.extract_reasoning(text, request)
+        assert reasoning == "Still thinking..."
+        assert content is None
+
+    def test_no_swap_with_none_request(self, parser):
+        """Graceful handling when request is None."""
+        text = "Some text."
+        reasoning, content = parser.extract_reasoning(text, None)
+        assert reasoning == "Some text."
+        assert content is None
+
+    def test_no_swap_with_no_kwargs(self, parser):
+        """No swap when chat_template_kwargs is absent."""
+        text = "Some text."
+        request = _make_request()
+        reasoning, content = parser.extract_reasoning(text, request)
+        assert reasoning == "Some text."
+        assert content is None
+
+    def test_swap_with_whitespace_only_content(self, parser):
+        """Swap occurs when content is whitespace-only."""
+        text = "The answer.</think>   "
+        request = _make_request(enable_thinking=False)
+        reasoning, content = parser.extract_reasoning(text, request)
+        assert content == "The answer."
+        assert reasoning == "   "
+
+
+class TestNonStreamingToolCalls:
+    def test_single_tool_call(self, parser):
+        text = (
+            "<tool_call>\n"
+            "<function=get_weather>\n"
+            "<parameter=city>Tokyo</parameter>\n"
+            "</function>\n"
+            "</tool_call>"
+        )
+        request = _make_request()
+        result = parser.extract_tool_calls(text, request)
+        assert result.tools_called is True
+        assert len(result.tool_calls) == 1
+        assert result.tool_calls[0].function.name == "get_weather"
+        args = json.loads(result.tool_calls[0].function.arguments)
+        assert args == {"city": "Tokyo"}
+
+    def test_parallel_tool_calls(self, parser):
+        text = (
+            "<tool_call>\n"
+            "<function=get_weather>\n"
+            "<parameter=city>Tokyo</parameter>\n"
+            "</function>\n"
+            "</tool_call>"
+            "<tool_call>\n"
+            "<function=get_time>\n"
+            "<parameter=timezone>Asia/Tokyo</parameter>\n"
+            "</function>\n"
+            "</tool_call>"
+        )
+        request = _make_request()
+        result = parser.extract_tool_calls(text, request)
+        assert result.tools_called is True
+        assert len(result.tool_calls) == 2
+        assert result.tool_calls[0].function.name == "get_weather"
+        assert result.tool_calls[1].function.name == "get_time"
+
+    def test_no_tool_calls(self, parser):
+        request = _make_request()
+        result = parser.extract_tool_calls("Hello, how can I help?", request)
+        assert result.tools_called is False
+        # Parser starts in REASONING state, so plain text is classified
+        # as reasoning (not content) when there are no tool calls.
+        assert result.content is None
+
+
+class TestStreaming:
+    def test_streaming_tool_calls(self, parser):
+        request = _make_request()
+        chunks = [
+            "<tool_call>\n",
+            "<function=get_weather>\n",
+            "<parameter=city>Tokyo",
+            "</parameter>\n",
+            "</function>\n",
+            "</tool_call>",
+        ]
+        results = simulate_tool_streaming(parser, request, chunks)
+        name = collect_function_name(results)
+        assert name == "get_weather"
+        args_text = collect_tool_arguments(results)
+        assert args_text
+        parsed = json.loads(args_text)
+        assert parsed == {"city": "Tokyo"}
+
+
+class TestParseDeltaTokenIdFiltering:
+    """parse_delta must not trigger tool call parsing when <tool_call>
+    appears as regular text rather than as a special token ID."""
+
+    def test_tool_call_text_in_reasoning_is_not_parsed(self, parser):
+        """Literal <tool_call> in model reasoning should be content,
+        not a tool call."""
+        request = _make_request()
+
+        text = (
+            "The test uses <tool_call> syntax:\n"
+            "<tool_call>\n"
+            "<function=Bash>\n"
+            "<parameter=command>ls</parameter>\n"
+            "</function>\n"
+            "</tool_call>"
+        )
+        result = parser.parse_delta(
+            delta_text=text,
+            delta_token_ids=[_TEXT_ID] * 6,
+            request=request,
+            prompt_token_ids=[],
+            finished=True,
+        )
+
+        assert result is not None
+        assert result.reasoning is not None
+        assert "<tool_call>" in result.reasoning
+        assert not result.tool_calls
+
+    def test_special_token_id_still_triggers_tool_call(self, parser):
+        """When the scanner matches a special token ID, the tool call
+        must still be parsed correctly."""
+        request = _make_request()
+
+        parser.parse_delta(
+            delta_text="Let me check.",
+            delta_token_ids=[_TEXT_ID, _TEXT_ID, _TEXT_ID],
+            request=request,
+            prompt_token_ids=[],
+            finished=False,
+        )
+
+        parser.parse_delta(
+            delta_text="<tool_call>",
+            delta_token_ids=[_TOOL_CALL_ID],
+            request=request,
+            finished=False,
+        )
+
+        parser.parse_delta(
+            delta_text=(
+                "\n<function=get_weather>\n"
+                "<parameter=city>Tokyo</parameter>\n"
+                "</function>\n"
+            ),
+            delta_token_ids=[_TEXT_ID] * 5,
+            request=request,
+            finished=False,
+        )
+
+        parser.parse_delta(
+            delta_text="</tool_call>",
+            delta_token_ids=[_TOOL_CALL_END_ID],
+            request=request,
+            finished=True,
+        )
+
+        assert any(s.name == "get_weather" for s in parser._tool_slots)
+
+    def test_text_discussion_then_real_tool_call(self, parser):
+        """Model discusses tool syntax in reasoning, then makes a real
+        tool call via special tokens."""
+        request = _make_request()
+
+        r1 = parser.parse_delta(
+            delta_text="Use <tool_call> to invoke tools.",
+            delta_token_ids=[_TEXT_ID] * 6,
+            request=request,
+            prompt_token_ids=[],
+            finished=False,
+        )
+
+        r2 = parser.parse_delta(
+            delta_text="</think>",
+            delta_token_ids=[_THINK_END_ID],
+            request=request,
+            finished=False,
+        )
+
+        r3 = parser.parse_delta(
+            delta_text="<tool_call>",
+            delta_token_ids=[_TOOL_CALL_ID],
+            request=request,
+            finished=False,
+        )
+
+        r4 = parser.parse_delta(
+            delta_text=("\n<function=test>\n<parameter=x>1</parameter>\n</function>\n"),
+            delta_token_ids=[_TEXT_ID] * 4,
+            request=request,
+            finished=False,
+        )
+
+        r5 = parser.parse_delta(
+            delta_text="</tool_call>",
+            delta_token_ids=[_TOOL_CALL_END_ID],
+            request=request,
+            finished=True,
+        )
+
+        results = [r1, r2, r3, r4, r5]
+        reasoning = "".join(r.reasoning for r in results if r and r.reasoning)
+        assert "<tool_call>" in reasoning
+
+        names = [
+            tc.function.name
+            for r in results
+            if r and r.tool_calls
+            for tc in r.tool_calls
+            if tc.function and tc.function.name
+        ]
+        assert "test" in names

tests/parser/engine/test_qwen3_reasoning.py

@@ -0,0 +1,583 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests for the engine-based Qwen3 reasoning parser.
+
+Validates that ``Qwen3Parser`` correctly handles
+``<think>``/``</think>`` reasoning with Qwen3-specific extensions:
+- ``<tool_call>`` as implicit reasoning end (terminal + token ID)
+- Stripping ``<think>`` from generated output (old template compat)
+- No terminal text (``</think>``, ``<tool_call>``) leaks into output
+"""
+
+import dataclasses
+
+import pytest
+
+from tests.parser.engine.conftest import make_mock_tokenizer
+from tests.parser.engine.streaming_helpers import simulate_reasoning_streaming
+from vllm.parser.abstract_parser import DelegatingParser
+from vllm.parser.engine.parser_engine_config import ParserState
+from vllm.parser.engine.registered_adapters import (
+    Qwen3ParserReasoningAdapter,
+    Qwen3ParserToolAdapter,
+)
+from vllm.parser.qwen3 import Qwen3Parser, qwen3_config
+
+_THINK_START_ID = 50
+_THINK_END_ID = 51
+_TOOL_CALL_ID = 60
+_TOOL_CALL_END_ID = 61
+_TEXT_ID = 100
+
+_QWEN3_VOCAB = {
+    "<think>": _THINK_START_ID,
+    "</think>": _THINK_END_ID,
+    "<tool_call>": _TOOL_CALL_ID,
+    "</tool_call>": _TOOL_CALL_END_ID,
+}
+
+
+class _Qwen3DelegatingParser(DelegatingParser):
+    reasoning_parser_cls = Qwen3ParserReasoningAdapter
+    tool_parser_cls = Qwen3ParserToolAdapter
+
+
+@pytest.fixture
+def mock_tokenizer():
+    return make_mock_tokenizer(_QWEN3_VOCAB)
+
+
+@pytest.fixture
+def parser(mock_tokenizer):
+    return Qwen3Parser(mock_tokenizer)
+
+
+class TestNonStreaming:
+    def test_reasoning_then_content(self, parser):
+        text = "<think>Let me analyze.</think>The answer is 42."
+        reasoning, content = parser.extract_reasoning(text, None)
+        assert reasoning == "Let me analyze."
+        assert content == "The answer is 42."
+
+    def test_no_start_token_in_output(self, parser):
+        """Qwen3.5+ style: <think> in prompt, only </think> in output."""
+        text = "Let me think about this.</think>The answer is 42."
+        reasoning, content = parser.extract_reasoning(text, None)
+        assert reasoning == "Let me think about this."
+        assert content == "The answer is 42."
+
+    def test_reasoning_only(self, parser):
+        text = "<think>Still thinking...</think>"
+        reasoning, content = parser.extract_reasoning(text, None)
+        assert reasoning == "Still thinking..."
+        assert content is None
+
+    def test_no_end_tag_all_reasoning(self, parser):
+        """No </think> means truncated output — everything is reasoning."""
+        text = "Hello, no reasoning here."
+        reasoning, content = parser.extract_reasoning(text, None)
+        assert reasoning == "Hello, no reasoning here."
+        assert content is None
+
+    def test_multiline_reasoning(self, parser):
+        text = (
+            "<think>Step 1: parse.\nStep 2: compute.\nStep 3: output.</think>Result: 7."
+        )
+        reasoning, content = parser.extract_reasoning(text, None)
+        assert "Step 1" in reasoning
+        assert "Step 3" in reasoning
+        assert content == "Result: 7."
+
+    def test_tool_call_implicit_end(self, parser):
+        """<tool_call> without </think> acts as implicit reasoning end."""
+        text = (
+            "<think>I need to read the file.\n\n"
+            "<tool_call>\n<function=bash>\n"
+            "<parameter=cmd>ls</parameter>\n"
+            "</function>\n</tool_call>"
+        )
+        reasoning, content = parser.extract_reasoning(text, None)
+        assert reasoning == "I need to read the file.\n\n"
+        assert "</think>" not in reasoning
+        assert "<tool_call>" not in reasoning
+
+    def test_tool_call_implicit_end_no_think(self, parser):
+        """<tool_call> as implicit end, no <think> in output."""
+        text = (
+            "I need to read the file.\n\n"
+            "<tool_call>\n<function=bash>\n"
+            "<parameter=cmd>ls</parameter>\n"
+            "</function>\n</tool_call>"
+        )
+        reasoning, content = parser.extract_reasoning(text, None)
+        assert reasoning == "I need to read the file.\n\n"
+        assert "<tool_call>" not in reasoning
+
+    def test_live_scenario_think_end_before_tool_call(self, parser):
+        """Real model output: </think> immediately before <tool_call>.
+
+        Regression test for the bug where </think> and <parameter=...>
+        leaked into reasoning content.
+        """
+        text = (
+            "The user wants to see what files are in the current directory"
+            " and their contents. Let me start by listing the directory."
+            "</think><tool_call><function=read>"
+            "<parameter=filePath>/Users/test/demo</parameter>"
+            "</function></tool_call>"
+        )
+        reasoning, content = parser.extract_reasoning(text, None)
+        expected_reasoning = (
+            "The user wants to see what files are in the current directory"
+            " and their contents. Let me start by listing the directory."
+        )
+        assert reasoning == expected_reasoning
+        assert "</think>" not in reasoning
+        assert "<tool_call>" not in reasoning
+        assert "<parameter=" not in reasoning
+
+    def test_no_terminal_text_in_reasoning(self, parser):
+        """Terminal text must never appear in reasoning output."""
+        text = "Reasoning here.</think>Content here."
+        reasoning, content = parser.extract_reasoning(text, None)
+        assert "</think>" not in (reasoning or "")
+        assert "<think>" not in (reasoning or "")
+
+    def test_no_terminal_text_in_content(self, parser):
+        """Terminal text must never appear in content output."""
+        text = "Reasoning here.</think>Content here."
+        reasoning, content = parser.extract_reasoning(text, None)
+        assert "</think>" not in (content or "")
+        assert "<think>" not in (content or "")
+
+    def test_duplicate_think_end_absorbed(self, parser):
+        """Duplicate </think> in CONTENT state must not leak."""
+        text = "Reasoning here.</think>Content here.</think>More content."
+        reasoning, content = parser.extract_reasoning(text, None)
+        assert reasoning == "Reasoning here."
+        assert content == "Content here.More content."
+
+
+class TestIsReasoningEnd:
+    def test_think_end_token(self, parser):
+        assert parser.is_reasoning_end([_THINK_START_ID, 1, _THINK_END_ID])
+
+    def test_no_end_token(self, parser):
+        assert not parser.is_reasoning_end([_THINK_START_ID, 1, 2])
+
+    def test_start_after_end_means_not_ended(self, parser):
+        assert not parser.is_reasoning_end([_THINK_END_ID, _THINK_START_ID, 1])
+
+    def test_tool_call_as_implicit_end(self, parser):
+        """Unpaired <tool_call> is implicit reasoning end."""
+        assert parser.is_reasoning_end([_THINK_START_ID, 1, _TOOL_CALL_ID])
+
+    def test_prompt_tool_example_before_generation_think_not_end(self, parser):
+        """Tool examples before the generation <think> must not end reasoning."""
+        assert not parser.is_reasoning_end([_TOOL_CALL_ID, _TEXT_ID, _THINK_START_ID])
+
+    def test_paired_tool_call_not_end(self, parser):
+        """Paired <tool_call>...</tool_call> (from template) is NOT end."""
+        assert not parser.is_reasoning_end(
+            [_THINK_START_ID, 1, _TOOL_CALL_ID, 2, _TOOL_CALL_END_ID]
+        )
+
+    def test_tool_call_after_think_end(self, parser):
+        """<tool_call> after </think> — already ended."""
+        assert parser.is_reasoning_end(
+            [_THINK_START_ID, 1, _THINK_END_ID, _TOOL_CALL_ID]
+        )
+
+    def test_empty_ids(self, parser):
+        assert not parser.is_reasoning_end([])
+
+
+class TestDelegatingPromptDetection:
+    def test_prompt_tool_example_does_not_skip_streaming_reasoning(
+        self, mock_tokenizer, mock_request
+    ):
+        parser = _Qwen3DelegatingParser(mock_tokenizer)
+        prompt_ids = [_TOOL_CALL_ID, _TEXT_ID, _THINK_START_ID]
+
+        delta = parser.parse_delta(
+            "thinking",
+            [_TEXT_ID],
+            mock_request,
+            prompt_token_ids=prompt_ids,
+            finished=False,
+        )
+
+        assert delta is not None
+        assert delta.reasoning == "thinking"
+        assert delta.content is None
+
+
+class TestStreaming:
+    def test_basic_streaming(self, parser):
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            ["<think>", "thinking", " hard", "</think>", "done"],
+            [
+                (_THINK_START_ID,),
+                (1,),
+                (2,),
+                (_THINK_END_ID,),
+                (3,),
+            ],
+        )
+        assert reasoning == "thinking hard"
+        assert content == "done"
+
+    def test_streaming_no_start_token(self, parser):
+        """Qwen3.5 style: no <think> in output, just reasoning then </think>."""
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            ["reasoning ", "text", "</think>", "content"],
+            [
+                (1,),
+                (2,),
+                (_THINK_END_ID,),
+                (3,),
+            ],
+        )
+        assert reasoning == "reasoning text"
+        assert content == "content"
+
+    def test_streaming_start_token_stripped(self, parser):
+        """<think> in output (old template) should be stripped."""
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            ["<think>reasoning", "</think>", "content"],
+            [
+                (_THINK_START_ID, 1),
+                (_THINK_END_ID,),
+                (2,),
+            ],
+        )
+        assert reasoning == "reasoning"
+        assert content == "content"
+
+    def test_streaming_tool_call_implicit_end(self, parser):
+        """<tool_call> ends reasoning implicitly during streaming."""
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            ["I need to check.", "<tool_call>", "\n<function=test>"],
+            [
+                (1,),
+                (_TOOL_CALL_ID,),
+                (2,),
+            ],
+        )
+        assert reasoning == "I need to check."
+        assert "<tool_call>" not in reasoning
+        assert "</think>" not in reasoning
+        assert content is not None
+
+    def test_streaming_content_after_think_end(self, parser):
+        """Content deltas after </think> are routed as content."""
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            ["reasoning", "</think>", "content1", " content2"],
+            [
+                (1,),
+                (_THINK_END_ID,),
+                (2,),
+                (3,),
+            ],
+        )
+        assert reasoning == "reasoning"
+        assert content == "content1 content2"
+
+    def test_streaming_content_after_tool_call(self, parser):
+        """Content deltas after <tool_call> are routed as content."""
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            ["thinking", "<tool_call>", "<function=f>"],
+            [
+                (1,),
+                (_TOOL_CALL_ID,),
+                (2,),
+            ],
+        )
+        assert reasoning == "thinking"
+        assert "<tool_call>" not in reasoning
+        assert content is not None
+
+    def test_streaming_end_grouped_with_content(self, parser):
+        """</think> grouped with following content in one delta."""
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            ["reasoning", "</think>the answer"],
+            [
+                (1,),
+                (_THINK_END_ID, 2),
+            ],
+        )
+        assert reasoning == "reasoning"
+        assert content == "the answer"
+
+    def test_streaming_think_and_end_in_one_delta(self, parser):
+        """<think> and </think> in the same delta."""
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            ["<think>reasoning</think>"],
+            [
+                (_THINK_START_ID, 1, _THINK_END_ID),
+            ],
+        )
+        assert reasoning == "reasoning"
+        assert content == ""
+
+    def test_streaming_pure_content_no_think(self, parser):
+        """No think tokens at all — everything is reasoning (truncated)."""
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            ["hello ", "world"],
+            [
+                (1,),
+                (2,),
+            ],
+        )
+        assert reasoning == "hello world"
+        assert content == ""
+
+    def test_streaming_think_end_and_tool_call_same_delta(self, parser):
+        """</think> and <tool_call> in the same delta — no leakage.
+
+        Regression test: the old override split at <tool_call> without
+        stripping </think>, causing </think> to leak into reasoning.
+        """
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            [
+                "Let me list the directory.",
+                "</think><tool_call>",
+                "<function=read>",
+                "<parameter=filePath>/tmp</parameter>",
+            ],
+            [
+                (1,),
+                (_THINK_END_ID, _TOOL_CALL_ID),
+                (2,),
+                (3,),
+            ],
+        )
+        assert reasoning == "Let me list the directory."
+        assert "</think>" not in reasoning
+        assert "<tool_call>" not in reasoning
+        assert "<parameter=" not in reasoning
+        assert content is not None
+
+    def test_streaming_no_terminal_text_leaks(self, parser):
+        """Terminal text must never appear in reasoning or content."""
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            ["reasoning", "</think>", "content"],
+            [
+                (1,),
+                (_THINK_END_ID,),
+                (2,),
+            ],
+        )
+        assert "</think>" not in reasoning
+        assert "</think>" not in content
+        assert "<think>" not in reasoning
+
+    def test_streaming_duplicate_think_end_absorbed(self, parser):
+        """Duplicate </think> token in CONTENT state must not leak."""
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            ["reasoning", "</think>", "content", "</think>", "more"],
+            [
+                (1,),
+                (_THINK_END_ID,),
+                (2,),
+                (_THINK_END_ID,),
+                (3,),
+            ],
+        )
+        assert reasoning == "reasoning"
+        assert content == "contentmore"
+
+
+class TestTrailingWhitespaceStripping:
+    """When strip_trailing_reasoning_whitespace is True,
+    trailing whitespace before </think> must be stripped.
+
+    Models often generate trailing newlines before </think>, and these
+    accumulate across multi-turn conversations via a feedback loop.
+    """
+
+    @pytest.fixture
+    def parser_with_strip(self):
+        cfg = dataclasses.replace(
+            qwen3_config(),
+            strip_trailing_reasoning_whitespace=True,
+        )
+        return Qwen3Parser(make_mock_tokenizer(_QWEN3_VOCAB), parser_engine_config=cfg)
+
+    def test_non_streaming_trailing_newline(self, parser_with_strip):
+        text = "Reasoning here.\n</think>Content."
+        reasoning, content = parser_with_strip.extract_reasoning(text, None)
+        assert reasoning == "Reasoning here."
+        assert content == "Content."
+
+    def test_non_streaming_multiple_trailing_newlines(self, parser_with_strip):
+        text = "Reasoning here.\n\n\n</think>Content."
+        reasoning, content = parser_with_strip.extract_reasoning(text, None)
+        assert reasoning == "Reasoning here."
+        assert content == "Content."
+
+    def test_non_streaming_internal_newlines_preserved(self, parser_with_strip):
+        text = "Step 1.\n\nStep 2.\n\nStep 3.</think>Answer."
+        reasoning, content = parser_with_strip.extract_reasoning(text, None)
+        assert reasoning == "Step 1.\n\nStep 2.\n\nStep 3."
+        assert content == "Answer."
+
+    def test_non_streaming_only_newlines_becomes_none(self, parser_with_strip):
+        text = "\n\n\n</think>Content."
+        reasoning, content = parser_with_strip.extract_reasoning(text, None)
+        assert reasoning is None
+        assert content == "Content."
+
+    def test_streaming_trailing_newline_stripped(self, parser_with_strip):
+        reasoning, content = simulate_reasoning_streaming(
+            parser_with_strip,
+            ["thinking.\n", "</think>", "done"],
+            [
+                (1,),
+                (_THINK_END_ID,),
+                (2,),
+            ],
+        )
+        assert reasoning == "thinking."
+        assert content == "done"
+
+    def test_streaming_multiple_trailing_newlines_stripped(self, parser_with_strip):
+        reasoning, content = simulate_reasoning_streaming(
+            parser_with_strip,
+            ["thinking.\n", "\n", "\n", "</think>", "done"],
+            [
+                (1,),
+                (2,),
+                (3,),
+                (_THINK_END_ID,),
+                (4,),
+            ],
+        )
+        assert reasoning == "thinking."
+        assert content == "done"
+
+    def test_streaming_internal_newlines_preserved(self, parser_with_strip):
+        reasoning, content = simulate_reasoning_streaming(
+            parser_with_strip,
+            ["Step 1.\n", "\nStep 2.\n", "</think>", "Answer"],
+            [
+                (1,),
+                (2,),
+                (_THINK_END_ID,),
+                (3,),
+            ],
+        )
+        assert reasoning == "Step 1.\n\nStep 2."
+        assert content == "Answer"
+
+    def test_streaming_trailing_newlines_before_tool_call(self, parser_with_strip):
+        """Trailing newlines before implicit <tool_call> end are stripped."""
+        reasoning, content = simulate_reasoning_streaming(
+            parser_with_strip,
+            ["I'll check.\n\n", "<tool_call>", "<function=test>"],
+            [
+                (1,),
+                (_TOOL_CALL_ID,),
+                (2,),
+            ],
+        )
+        assert reasoning == "I'll check."
+        assert "<tool_call>" not in reasoning
+
+
+class TestWhitespaceStrippingDisabled:
+    """When strip_trailing_reasoning_whitespace is False,
+    trailing whitespace in reasoning must be preserved."""
+
+    @pytest.fixture
+    def parser_no_strip(self):
+        cfg = dataclasses.replace(
+            qwen3_config(),
+            strip_trailing_reasoning_whitespace=False,
+        )
+        return Qwen3Parser(make_mock_tokenizer(_QWEN3_VOCAB), parser_engine_config=cfg)
+
+    def test_non_streaming_preserves_trailing_newline(self, parser_no_strip):
+        text = "Reasoning here.\n</think>Content."
+        reasoning, content = parser_no_strip.extract_reasoning(text, None)
+        assert reasoning == "Reasoning here.\n"
+        assert content == "Content."
+
+    def test_streaming_preserves_trailing_newlines(self, parser_no_strip):
+        reasoning, content = simulate_reasoning_streaming(
+            parser_no_strip,
+            ["thinking.\n", "\n", "</think>", "done"],
+            [
+                (1,),
+                (2,),
+                (_THINK_END_ID,),
+                (3,),
+            ],
+        )
+        assert reasoning == "thinking.\n\n"
+        assert content == "done"
+
+
+class TestThinkingDisabled:
+    """When ``enable_thinking=False``, the chat template pre-fills a closed
+    ``<think>\\n\\n</think>\\n\\n`` block.  The model output starts in content
+    state, so the parser's initial state must be CONTENT — not REASONING.
+    """
+
+    def test_thinking_disabled_initial_state_is_content(self, mock_tokenizer):
+        p = Qwen3Parser(
+            mock_tokenizer,
+            chat_template_kwargs={"enable_thinking": False},
+        )
+        assert p.parser_engine_config.initial_state == ParserState.CONTENT
+
+    def test_thinking_enabled_initial_state_is_reasoning(self, mock_tokenizer):
+        p = Qwen3Parser(
+            mock_tokenizer,
+            chat_template_kwargs={"enable_thinking": True},
+        )
+        assert p.parser_engine_config.initial_state == ParserState.REASONING
+
+    def test_default_initial_state_is_reasoning(self, mock_tokenizer):
+        p = Qwen3Parser(mock_tokenizer)
+        assert p.parser_engine_config.initial_state == ParserState.REASONING
+
+    def test_thinking_disabled_streaming_content_only(self, mock_tokenizer):
+        """Plain text with thinking disabled must stream as content, not
+        reasoning.  Before the fix, the REASONING initial state caused all
+        output to be emitted as reasoning chunks."""
+        p = Qwen3Parser(
+            mock_tokenizer,
+            chat_template_kwargs={"enable_thinking": False},
+        )
+        reasoning, content = simulate_reasoning_streaming(
+            p,
+            ["The answer", " is 42."],
+            [
+                (_TEXT_ID,),
+                (_TEXT_ID,),
+            ],
+        )
+        assert content == "The answer is 42."
+        assert reasoning == ""
+
+    def test_thinking_disabled_non_streaming(self, mock_tokenizer):
+        p = Qwen3Parser(
+            mock_tokenizer,
+            chat_template_kwargs={"enable_thinking": False},
+        )
+        reasoning, content = p.extract_reasoning("The answer is 42.", None)
+        assert reasoning is None
+        assert content == "The answer is 42."

tests/parser/engine/test_replay.py

@@ -0,0 +1,598 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Replay tests for engine parsers (holdback, skip-tool-parsing, adapters).
+
+Replays dynamically built token sequences at different chunk sizes and
+holdback depths to verify chunk-size invariance and terminal-token hygiene.
+
+Parser discovery is automatic: any ``ParserEngine`` subclass registered in
+``registered_adapters`` that also has a builder in ``trace_builder._BUILDERS``
+is picked up with zero manual wiring.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+from typing import NamedTuple
+
+import pytest
+
+from tests.parser.engine.replay_harness import (
+    DUMMY_TOOLS,
+    MockTokenizer,
+    Sample,
+    _test_request,
+    assert_no_terminal_leakage,
+    assert_parse_output,
+    collect_output,
+    make_mock_tokenizer,
+    parse_non_streaming,
+    replay_streaming,
+    replay_with_text_holdback,
+)
+from tests.parser.engine.trace_builder import _BUILDERS, build_samples
+from vllm.parser.engine import registered_adapters as _adapters_mod
+from vllm.parser.engine.parser_engine import ParserEngine
+from vllm.parser.engine.parser_engine_config import ParserState
+
+# ── Parser discovery ─────────────────────────────────────────────────
+
+
+class _ParserInfo(NamedTuple):
+    parser_cls: type[ParserEngine]
+    name: str
+    samples: tuple
+    terminals: list[str]
+    tool_end: str
+    think_end: str
+    tool_start: str
+
+
+def _discover_parsers() -> list[_ParserInfo]:
+    """Discover engine parsers from registered_adapters that have test builders.
+
+    Returns one ``_ParserInfo`` per parser, sorted by config name.
+    Raises ``RuntimeError`` if any registered parser lacks a builder.
+    """
+    bare_tok = MockTokenizer(vocab={}, tokens=[])
+    found: list[_ParserInfo] = []
+    missing_builders: list[str] = []
+    for obj in vars(_adapters_mod).values():
+        if not (
+            isinstance(obj, type)
+            and issubclass(obj, ParserEngine)
+            and obj is not ParserEngine
+        ):
+            continue
+        cfg = obj(bare_tok, None).parser_engine_config
+        if cfg.name not in _BUILDERS:
+            missing_builders.append(f"{obj.__name__} (config.name={cfg.name!r})")
+            continue
+        tool_end = cfg.token_id_terminals.get("TOOL_END")
+        if not tool_end:
+            raise RuntimeError(
+                f"{obj.__name__} config missing 'TOOL_END' in token_id_terminals"
+            )
+        all_vals = set(cfg.terminals.values()) | set(cfg.token_id_terminals.values())
+        found.append(
+            _ParserInfo(
+                parser_cls=obj,
+                name=cfg.name,
+                samples=build_samples(cfg.name),
+                terminals=sorted(v for v in all_vals if len(v) > 1),
+                tool_end=tool_end,
+                think_end=cfg.terminals.get("THINK_END", ""),
+                tool_start=(
+                    cfg.terminals["TOOL_SECTION_START"]
+                    if (ParserState.CONTENT, "TOOL_SECTION_START") in cfg.transitions
+                    else cfg.terminals.get("TOOL_START", "")
+                ),
+            )
+        )
+    if missing_builders:
+        raise RuntimeError(
+            f"Engine parsers in registered_adapters have no test builder "
+            f"in trace_builder._BUILDERS: {', '.join(missing_builders)}. "
+            f"Add a builder to _BUILDERS for each new parser."
+        )
+    found.sort(key=lambda p: p.name)
+    return found
+
+
+_PARSERS = _discover_parsers()
+
+
+def _make_parser(parser_cls: type[ParserEngine], tokenizer, sample: Sample, **extra):
+    kwargs = dict(extra)
+    if sample.chat_template_kwargs:
+        kwargs["chat_template_kwargs"] = sample.chat_template_kwargs
+    return parser_cls(tokenizer, sample.tools, **kwargs)
+
+
+_ENGINE_PARSERS: dict[str, type[ParserEngine]] = {
+    f"{p.name}_engine": p.parser_cls for p in _PARSERS
+}
+
+# ── Parametrize sample lists ─────────────────────────────────────────
+
+HOLDBACK_CONFIGS = [6, 12, 24]
+
+_REPLAY_SAMPLES = [(p.parser_cls, s, p.terminals) for p in _PARSERS for s in p.samples]
+
+
+@pytest.mark.parametrize("holdback", HOLDBACK_CONFIGS, ids=lambda h: f"holdback{h}")
+@pytest.mark.parametrize("chunk_size", [3, 5, 10], ids=lambda c: f"chunk{c}")
+@pytest.mark.parametrize(
+    "parser_cls,sample,terminals",
+    _REPLAY_SAMPLES,
+    ids=lambda v: v.id if hasattr(v, "id") else "",
+)
+class TestReplayWithHoldback:
+    """Replay all parsers with simulated detokenizer holdback."""
+
+    def test_replay(self, parser_cls, sample, terminals, chunk_size, holdback):
+        tokenizer = make_mock_tokenizer(sample)
+        parser = _make_parser(parser_cls, tokenizer, sample)
+        deltas = replay_streaming(
+            parser,
+            sample.tokens,
+            chunk_size=chunk_size,
+            holdback_chars=holdback,
+            prompt_token_ids=sample.prompt_token_ids,
+        )
+        output = collect_output(deltas)
+
+        assert_parse_output(output, sample)
+        assert_no_terminal_leakage(
+            output,
+            terminals,
+            context=f"chunk_size={chunk_size}, holdback={holdback}",
+        )
+
+
+TEXT_HOLDBACK_DELAYS = [1, 2, 3]
+
+
+@pytest.mark.parametrize("delay", TEXT_HOLDBACK_DELAYS, ids=lambda d: f"delay{d}")
+@pytest.mark.parametrize(
+    "parser_cls,sample,terminals",
+    _REPLAY_SAMPLES,
+    ids=lambda v: v.id if hasattr(v, "id") else "",
+)
+class TestTextHoldback:
+    """Replay with production-like text/token-ID misalignment.
+
+    In production the detokenizer sends token IDs immediately but holds
+    back text by N tokens.  This exercises the TokenIDScanner deferred
+    terminal path that aligned-holdback tests do not cover.
+    """
+
+    def test_replay(self, parser_cls, sample, terminals, delay):
+        tokenizer = make_mock_tokenizer(sample)
+        parser = _make_parser(parser_cls, tokenizer, sample)
+        deltas = replay_with_text_holdback(
+            parser,
+            sample.tokens,
+            text_delay=delay,
+            prompt_token_ids=sample.prompt_token_ids,
+        )
+        output = collect_output(deltas)
+
+        assert_parse_output(output, sample)
+        assert_no_terminal_leakage(
+            output,
+            terminals,
+            context=f"text_delay={delay}",
+        )
+
+
+@pytest.mark.parametrize(
+    "chunk_size", [1, 2, 3, 5, 10, 19, 20, None], ids=lambda c: f"chunk{c}"
+)
+@pytest.mark.parametrize(
+    "parser_cls,sample,terminals",
+    _REPLAY_SAMPLES,
+    ids=lambda v: v.id if hasattr(v, "id") else "",
+)
+class TestReplay:
+    """Replay all parsers at varied chunk sizes without holdback."""
+
+    def test_replay(self, parser_cls, sample, terminals, chunk_size):
+        tokenizer = make_mock_tokenizer(sample)
+        parser = _make_parser(parser_cls, tokenizer, sample)
+        deltas = replay_streaming(
+            parser,
+            sample.tokens,
+            chunk_size=chunk_size,
+            prompt_token_ids=sample.prompt_token_ids,
+        )
+        output = collect_output(deltas)
+
+        assert_parse_output(output, sample)
+        assert_no_terminal_leakage(output, terminals)
+
+
+_DEFERRAL_SAMPLES = [
+    (p.parser_cls, s, p.tool_end)
+    for p in _PARSERS
+    for s in p.samples
+    if s.expected_tool_calls
+]
+
+
+@pytest.mark.parametrize(
+    "parser_cls,sample,tool_end_text",
+    _DEFERRAL_SAMPLES,
+    ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""),
+)
+class TestDeferralFinish:
+    """Test that parse_delta(finished=True) resolves deferred scanner state.
+
+    Simulates a production failure where delta_text is missing the
+    tool-call-end text but delta_token_ids has the token, causing the
+    scanner to defer it.  Without finish(), the deferred state is lost
+    and tool call arguments are empty.
+    """
+
+    def test_misaligned_last_delta_with_finish(self, parser_cls, sample, tool_end_text):
+        tokenizer = make_mock_tokenizer(sample)
+        parser = _make_parser(parser_cls, tokenizer, sample)
+
+        request = _test_request()
+
+        all_ids = [tid for tid, _ in sample.tokens]
+        all_texts = [text for _, text in sample.tokens]
+
+        tool_end_id = sample.vocab.get(tool_end_text)
+        split_idx = None
+        for i in range(len(all_ids) - 1, -1, -1):
+            if all_ids[i] == tool_end_id:
+                split_idx = i
+                break
+
+        if split_idx is None:
+            pytest.skip(f"no {tool_end_text} token found")
+
+        first_ids = all_ids[:split_idx]
+        first_text = "".join(all_texts[:split_idx])
+
+        last_ids = all_ids[split_idx:]
+        last_text_missing = "".join(all_texts[split_idx:]).replace(tool_end_text, "")
+
+        result1 = parser.parse_delta(
+            first_text,
+            first_ids,
+            request,
+            prompt_token_ids=[],
+            finished=False,
+        )
+        result2 = parser.parse_delta(
+            last_text_missing, last_ids, request, finished=True
+        )
+
+        output = collect_output([result1, result2])
+
+        tool_calls_only = dataclasses.replace(
+            sample, expected_reasoning=None, expected_content=None
+        )
+        assert_parse_output(output, tool_calls_only)
+
+
+@pytest.mark.parametrize(
+    "parser_cls,sample",
+    [(p.parser_cls, p.samples[0]) for p in _PARSERS],
+    ids=[p.name for p in _PARSERS],
+)
+class TestParserEngineAdjustRequest:
+    """Verify ParserEngine and its adapters set skip_special_tokens=False."""
+
+    def test_adjust_request_disables_skip_special_tokens(self, parser_cls, sample):
+        tokenizer = make_mock_tokenizer(sample)
+        parser = parser_cls(tokenizer, sample.tools)
+        request = _test_request()
+        assert request.skip_special_tokens is True
+        adjusted = parser.adjust_request(request)
+        assert adjusted.skip_special_tokens is False
+
+
+_TOOL_CALL_SAMPLES = [
+    (p.parser_cls, s, p.think_end, p.tool_start)
+    for p in _PARSERS
+    for s in p.samples
+    if s.expected_tool_calls and s.expected_reasoning
+]
+
+
+def _tool_suppression_expectations(
+    sample, think_end: str, tool_start: str, *, include_tool_block: bool
+) -> tuple[str, str]:
+    """Expected (reasoning, content) when tool calls are not extracted.
+
+    With ``include_tool_block=True`` (skip_tool_parsing / reasoning
+    adapter first pass), tool terminal text is preserved as content so
+    a second-pass parser can see it.
+
+    With ``include_tool_block=False`` (_suppress_tool_calls /
+    tool_choice='none'), the state machine consumes tool blocks and
+    only non-tool content survives.
+    """
+    full_text = "".join(text for _, text in sample.tokens)
+    reasoning = sample.expected_reasoning
+    idx = full_text.find(reasoning)
+    if idx < 0:
+        return (full_text, "")
+    after_reasoning = full_text[idx + len(reasoning) :]
+    if think_end:
+        pos = after_reasoning.find(think_end)
+        if pos >= 0:
+            if include_tool_block:
+                return (reasoning, after_reasoning[pos + len(think_end) :])
+            after_reasoning = after_reasoning[pos + len(think_end) :]
+    if tool_start:
+        pos = after_reasoning.find(tool_start)
+        if pos >= 0:
+            if include_tool_block:
+                return (reasoning, after_reasoning[pos:])
+            return (reasoning, after_reasoning[:pos])
+    if include_tool_block:
+        return (full_text, "")
+    return (reasoning, after_reasoning)
+
+
+@pytest.mark.parametrize("chunk_size", [1, 5, None], ids=lambda c: f"chunk{c}")
+@pytest.mark.parametrize(
+    "mode",
+    ["skip_tool_parsing", "suppress_tool_calls"],
+    ids=["skip_tool_parsing", "suppress_tool_calls"],
+)
+@pytest.mark.parametrize(
+    "parser_cls,sample,think_end,tool_start",
+    _TOOL_CALL_SAMPLES,
+    ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""),
+)
+class TestToolCallFilteringReplay:
+    """Replay with tool calls not extracted, in both filtering modes.
+
+    ``skip_tool_parsing`` (reasoning adapter first pass): tool terminal
+    text is preserved as content for a second-pass tool parser.
+
+    ``suppress_tool_calls`` (tool_choice='none'): tool call blocks are
+    consumed by the state machine and do not leak into content.
+    """
+
+    def test_replay(self, parser_cls, sample, think_end, tool_start, mode, chunk_size):
+        tokenizer = make_mock_tokenizer(sample)
+        kwargs = {}
+        if sample.chat_template_kwargs:
+            kwargs["chat_template_kwargs"] = sample.chat_template_kwargs
+        parser = parser_cls(tokenizer, **kwargs)
+
+        request = _test_request()
+        request.tools = DUMMY_TOOLS
+        if mode == "skip_tool_parsing":
+            parser.skip_tool_parsing = True
+        else:
+            request.tool_choice = "none"
+
+        all_ids = [tid for tid, _ in sample.tokens]
+        all_texts = [text for _, text in sample.tokens]
+        if chunk_size is None:
+            chunk_size = len(all_ids)
+
+        results = []
+        chunks = list(range(0, len(all_ids), chunk_size))
+        for i, start in enumerate(chunks):
+            end = min(start + chunk_size, len(all_ids))
+            is_last = i == len(chunks) - 1
+            result = parser.parse_delta(
+                "".join(all_texts[start:end]),
+                all_ids[start:end],
+                request,
+                prompt_token_ids=(sample.prompt_token_ids or [])
+                if start == 0
+                else None,
+                finished=is_last,
+            )
+            results.append(result)
+
+        output = collect_output(results)
+
+        include_block = mode == "skip_tool_parsing"
+        expected_reasoning, expected_content = _tool_suppression_expectations(
+            sample, think_end, tool_start, include_tool_block=include_block
+        )
+
+        assert output.reasoning == expected_reasoning, (
+            f"Reasoning mismatch (mode={mode}):\n"
+            f"  expected: {expected_reasoning!r}\n"
+            f"  actual:   {output.reasoning!r}"
+        )
+        assert output.tool_calls == [], (
+            f"Expected no tool calls (mode={mode}) but got {output.tool_calls}"
+        )
+        assert output.content == expected_content, (
+            f"Content mismatch (mode={mode}):\n"
+            f"  expected: {expected_content!r}\n"
+            f"  actual:   {output.content!r}"
+        )
+
+
+@pytest.mark.parametrize(
+    "parser_cls,sample,think_end,tool_start",
+    _TOOL_CALL_SAMPLES,
+    ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""),
+)
+class TestToolCallFilteringNonStreaming:
+    """Non-streaming parse() with tool_choice='none' must suppress tool
+    calls and not leak special tokens into content."""
+
+    def test_parse(self, parser_cls, sample, think_end, tool_start):
+        tokenizer = make_mock_tokenizer(sample)
+        kwargs = {}
+        if sample.chat_template_kwargs:
+            kwargs["chat_template_kwargs"] = sample.chat_template_kwargs
+        parser = parser_cls(tokenizer, **kwargs)
+
+        request = _test_request()
+        request.tools = DUMMY_TOOLS
+        request.tool_choice = "none"
+
+        output = parse_non_streaming(parser, sample, request)
+
+        expected_reasoning, expected_content = _tool_suppression_expectations(
+            sample, think_end, tool_start, include_tool_block=False
+        )
+        assert output.reasoning == expected_reasoning, (
+            f"Reasoning mismatch:\n"
+            f"  expected: {expected_reasoning!r}\n"
+            f"  actual:   {output.reasoning!r}"
+        )
+        assert output.tool_calls == [], (
+            f"Expected no tool calls but got {output.tool_calls}"
+        )
+        assert output.content == expected_content, (
+            f"Content mismatch:\n"
+            f"  expected: {expected_content!r}\n"
+            f"  actual:   {output.content!r}"
+        )
+
+
+_WS_TOOL_SAMPLES = [(t[0], t[1]) for t in _TOOL_CALL_SAMPLES if "whitespace" in t[1].id]
+
+
+@pytest.mark.parametrize(
+    "parser_cls,sample",
+    _WS_TOOL_SAMPLES,
+    ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""),
+)
+class TestToolChoiceNoneStreamingParity:
+    """Streaming and non-streaming must return the same content
+    when tool_choice='none' suppresses tool calls."""
+
+    def test_content_matches(self, parser_cls, sample):
+        tokenizer = make_mock_tokenizer(sample)
+        kwargs = {}
+        if sample.chat_template_kwargs:
+            kwargs["chat_template_kwargs"] = sample.chat_template_kwargs
+        request = _test_request()
+        request.tools = DUMMY_TOOLS
+        request.tool_choice = "none"
+
+        ns_output = parse_non_streaming(
+            parser_cls(tokenizer, **kwargs),
+            sample,
+            request,
+        )
+
+        s_parser = parser_cls(tokenizer, **kwargs)
+        results = []
+        for i, (tid, text) in enumerate(sample.tokens):
+            is_last = i == len(sample.tokens) - 1
+            results.append(
+                s_parser.parse_delta(
+                    text,
+                    [tid],
+                    request,
+                    prompt_token_ids=(sample.prompt_token_ids or [])
+                    if i == 0
+                    else None,
+                    finished=is_last,
+                )
+            )
+        s_output = collect_output(results)
+
+        assert ns_output.content == s_output.content, (
+            f"Streaming/non-streaming content mismatch:\n"
+            f"  streaming:     {s_output.content!r}\n"
+            f"  non-streaming: {ns_output.content!r}"
+        )
+
+
+_DROP_TOKENS = {"<bos>": 99990, "<eos>": 99991}
+
+
+def _inject_drop_tokens(sample):
+    """Insert <bos> at stream start and <eos> between the first two tokens."""
+    new_vocab = {**sample.vocab, **_DROP_TOKENS}
+    tokens = list(sample.tokens)
+    tokens.insert(0, (99990, "<bos>"))
+    if len(tokens) >= 3:
+        tokens.insert(2, (99991, "<eos>"))
+    else:
+        tokens.append((99991, "<eos>"))
+    return dataclasses.replace(sample, vocab=new_vocab, tokens=tokens)
+
+
+class TestDropTokenReplay:
+    """Verify unconfigured special tokens are silently dropped across
+    all parsers and chunk sizes."""
+
+    @pytest.mark.parametrize(
+        "parser_info",
+        _PARSERS,
+        ids=[p.name for p in _PARSERS],
+    )
+    @pytest.mark.parametrize("chunk_size", [1, 3, None])
+    def test_drop_tokens_removed_from_output(self, parser_info, chunk_size):
+        for sample in parser_info.samples:
+            injected = _inject_drop_tokens(sample)
+            tokenizer = make_mock_tokenizer(injected)
+            parser = _make_parser(parser_info.parser_cls, tokenizer, sample)
+
+            results = replay_streaming(
+                parser,
+                injected.tokens,
+                chunk_size=chunk_size,
+                tools=sample.tools,
+                prompt_token_ids=sample.prompt_token_ids,
+            )
+            output = collect_output(results)
+
+            assert_no_terminal_leakage(
+                output,
+                list(_DROP_TOKENS.keys()),
+                context=f"parser={parser_info.name}, chunk={chunk_size}",
+            )
+            assert_parse_output(output, sample)
+
+
+class TestDropTokenNonStreaming:
+    """Non-streaming parse() must also strip unconfigured special tokens."""
+
+    @pytest.mark.parametrize(
+        "parser_info",
+        _PARSERS,
+        ids=[p.name for p in _PARSERS],
+    )
+    def test_drop_tokens_removed_from_output(self, parser_info):
+        for sample in parser_info.samples:
+            injected = _inject_drop_tokens(sample)
+            tokenizer = make_mock_tokenizer(injected)
+            parser = _make_parser(parser_info.parser_cls, tokenizer, sample)
+
+            request = _test_request(tools=sample.tools)
+            output = parse_non_streaming(parser, injected, request)
+
+            assert_no_terminal_leakage(
+                output,
+                list(_DROP_TOKENS.keys()),
+                context=f"parser={parser_info.name}",
+            )
+
+
+class TestAdapterReferences:
+    """Verify make_adapters sets reasoning/tool parser class refs on parser engine
+    parser classes so the serving layer finds them and calls adjust_request."""
+
+    @pytest.mark.parametrize(
+        "parser_name",
+        list(_ENGINE_PARSERS.keys()),
+    )
+    def test_adapter_cls_refs_set(self, parser_name):
+        parser_cls = _ENGINE_PARSERS[parser_name]
+        assert parser_cls.reasoning_parser_cls is not None, (
+            f"{parser_name}: reasoning_parser_cls is None"
+        )
+        assert parser_cls.tool_parser_cls is not None, (
+            f"{parser_name}: tool_parser_cls is None"
+        )

tests/parser/engine/test_seed_oss.py

@@ -0,0 +1,189 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests for the engine-based seed_oss parser.
+
+seed_oss is Qwen3 with four overridden wrapper tokens, so the shared grammar
+(arg types, multiline values, parallel calls, streaming mechanics, …) is
+already covered by ``test_qwen3.py``/``test_qwen3_reasoning.py``. These tests
+cover only what is seed_oss-specific: that the ``seed:`` token overrides are
+wired through, the reasoning→tool boundary holds with them, the malformed
+header from #46314 no longer drops sibling calls, and the registered adapters
+resolve. Seed-specific budget-reflect tags inside reasoning are also covered
+here because the old dedicated parser tests exercised them.
+"""
+
+import json
+
+import pytest
+
+from tests.parser.engine.conftest import make_mock_tokenizer
+from tests.parser.engine.streaming_helpers import (
+    collect_function_name,
+    collect_tool_arguments,
+    simulate_reasoning_streaming,
+    simulate_tool_streaming,
+)
+from vllm.parser.engine.registered_adapters import (
+    SeedOssParserReasoningAdapter,
+    SeedOssParserToolAdapter,
+)
+from vllm.parser.seed_oss import SeedOssParser
+
+TOOL_CALL_START = "<seed:tool_call>"
+TOOL_CALL_END = "</seed:tool_call>"
+THINK_START = "<seed:think>"
+THINK_END = "</seed:think>"
+
+_THINK_END_ID = 51
+_TOOL_CALL_ID = 60
+
+_SEED_OSS_VOCAB = {
+    THINK_START: 50,
+    THINK_END: _THINK_END_ID,
+    TOOL_CALL_START: _TOOL_CALL_ID,
+    TOOL_CALL_END: 61,
+}
+
+
+@pytest.fixture
+def mock_tokenizer():
+    return make_mock_tokenizer(_SEED_OSS_VOCAB)
+
+
+@pytest.fixture
+def tool_parser(mock_tokenizer):
+    return SeedOssParser(
+        mock_tokenizer, chat_template_kwargs={"enable_thinking": False}
+    )
+
+
+@pytest.fixture
+def parser(mock_tokenizer):
+    return SeedOssParser(mock_tokenizer)
+
+
+def test_token_overrides_wired(parser):
+    assert parser.parser_engine_config.name == "seed_oss"
+    assert parser.reasoning_start_str == THINK_START
+    assert parser.reasoning_end_str == THINK_END
+
+
+def test_single_tool_call(tool_parser, mock_request):
+    text = (
+        f"{TOOL_CALL_START}\n<function=get_weather>\n"
+        "<parameter=city>Tokyo</parameter>\n"
+        f"</function>\n{TOOL_CALL_END}"
+    )
+    result = tool_parser.extract_tool_calls(text, mock_request)
+
+    assert result.tools_called is True
+    assert result.tool_calls[0].function.name == "get_weather"
+    assert json.loads(result.tool_calls[0].function.arguments) == {"city": "Tokyo"}
+
+
+def test_malformed_function_end_does_not_drop_siblings(tool_parser, mock_request):
+    """Regression for #46314: a malformed ``</function>`` with no closing ``>``
+    on the header must not discard the other, well-formed calls."""
+    text = (
+        f"{TOOL_CALL_START}\n<function=broken</function>\n{TOOL_CALL_END}"
+        f"{TOOL_CALL_START}\n<function=get_weather>\n"
+        "<parameter=city>Tokyo</parameter>\n"
+        f"</function>\n{TOOL_CALL_END}"
+    )
+    result = tool_parser.extract_tool_calls(text, mock_request)
+
+    weather = next(tc for tc in result.tool_calls if tc.function.name == "get_weather")
+    assert json.loads(weather.function.arguments) == {"city": "Tokyo"}
+
+
+def test_basic_streaming(tool_parser, mock_request):
+    chunks = [
+        f"{TOOL_CALL_START}\n",
+        "<function=get_weather>\n",
+        "<parameter=city>Tokyo",
+        "</parameter>\n",
+        "</function>\n",
+        f"{TOOL_CALL_END}",
+    ]
+    results = simulate_tool_streaming(tool_parser, mock_request, chunks)
+
+    assert collect_function_name(results) == "get_weather"
+    assert json.loads(collect_tool_arguments(results)) == {"city": "Tokyo"}
+
+
+def test_reasoning_then_tool_call(parser):
+    text = (
+        f"{THINK_START}I need to read the file.{THINK_END}"
+        f"{TOOL_CALL_START}\n<function=read>\n"
+        "<parameter=path>/tmp/x</parameter>\n"
+        f"</function>\n{TOOL_CALL_END}"
+    )
+    reasoning, _ = parser.extract_reasoning(text, None)
+    assert reasoning == "I need to read the file."
+    assert TOOL_CALL_START not in reasoning
+
+
+def test_streaming_think_end_and_tool_call_same_delta(parser):
+    """``</seed:think>`` and ``<seed:tool_call>`` arriving in one delta must
+    not leak the terminal tokens into the reasoning text."""
+    reasoning, content = simulate_reasoning_streaming(
+        parser,
+        [
+            "Let me list the directory.",
+            f"{THINK_END}{TOOL_CALL_START}",
+            "<function=read>",
+        ],
+        [(1,), (_THINK_END_ID, _TOOL_CALL_ID), (2,)],
+    )
+    assert reasoning == "Let me list the directory."
+    assert THINK_END not in reasoning
+    assert TOOL_CALL_START not in reasoning
+    assert content is not None
+
+
+def test_end_to_end_through_registered_adapters(mock_tokenizer, mock_request):
+    reasoning_parser = SeedOssParserReasoningAdapter(mock_tokenizer)
+    tool_parser = SeedOssParserToolAdapter(mock_tokenizer)
+    text = (
+        f"{THINK_START}Plan the call.{THINK_END}"
+        f"{TOOL_CALL_START}\n<function=get_weather>\n"
+        "<parameter=city>Tokyo</parameter>\n"
+        f"</function>\n{TOOL_CALL_END}"
+    )
+    reasoning, remaining = reasoning_parser.extract_reasoning(text, mock_request)
+    assert reasoning == "Plan the call."
+
+    tool_result = tool_parser.extract_tool_calls(remaining, mock_request)
+    assert tool_result.tool_calls[0].function.name == "get_weather"
+    assert json.loads(tool_result.tool_calls[0].function.arguments) == {"city": "Tokyo"}
+
+
+def test_budget_reflect_tags_do_not_break_adapter_pipeline(
+    mock_tokenizer,
+    mock_request,
+):
+    reasoning_parser = SeedOssParserReasoningAdapter(mock_tokenizer)
+    tool_parser = SeedOssParserToolAdapter(mock_tokenizer)
+    text = (
+        f"{THINK_START}"
+        "The user's current thinking budget is 512.</seed:cot_budget_reflect>\n"
+        "I need the weather.\n"
+        "<seed:cot_budget_reflect>I have used 131 tokens."
+        "</seed:cot_budget_reflect>\n"
+        f"{THINK_END}"
+        f"{TOOL_CALL_START}\n<function=get_weather>\n"
+        "<parameter=city>Barcelona</parameter>\n"
+        f"</function>\n{TOOL_CALL_END}"
+    )
+
+    reasoning, remaining = reasoning_parser.extract_reasoning(text, mock_request)
+    assert reasoning is not None
+    assert "current thinking budget is 512" in reasoning
+    assert "<seed:cot_budget_reflect>" in reasoning
+    assert "</seed:cot_budget_reflect>" in reasoning
+
+    tool_result = tool_parser.extract_tool_calls(remaining, mock_request)
+    assert tool_result.tool_calls[0].function.name == "get_weather"
+    assert json.loads(tool_result.tool_calls[0].function.arguments) == {
+        "city": "Barcelona"
+    }

tests/parser/engine/test_token_id_scanner.py

@@ -0,0 +1,998 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests for TokenIDScanner."""
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from vllm.parser.engine.events import EventType
+from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
+from vllm.parser.engine.token_id_scanner import (
+    PreLexedTerminal,
+    TextChunk,
+    TokenIDScanner,
+)
+from vllm.parser.gemma4 import gemma4_config
+
+CHANNEL_START = "<|channel>"
+CHANNEL_END = "<channel|>"
+CHANNEL_START_ID = 100
+CHANNEL_END_ID = 101
+REGULAR_TOKEN_ID = 200
+TOOL_START = "<tool_call>"
+TOOL_END = "</tool_call>"
+TOOL_START_ID = 110
+TOOL_END_ID = 111
+
+
+@pytest.fixture
+def tokenizer():
+    tok = MagicMock()
+    tok.get_vocab.return_value = {
+        CHANNEL_START: CHANNEL_START_ID,
+        CHANNEL_END: CHANNEL_END_ID,
+    }
+    tok.decode.side_effect = lambda ids: {
+        CHANNEL_START_ID: CHANNEL_START,
+        CHANNEL_END_ID: CHANNEL_END,
+        REGULAR_TOKEN_ID: "regular",
+    }.get(ids[0], f"<unk:{ids[0]}>")
+    return tok
+
+
+@pytest.fixture
+def scanner(tokenizer):
+    return TokenIDScanner(
+        token_id_to_terminal={
+            CHANNEL_START_ID: "THINK_START",
+            CHANNEL_END_ID: "THINK_END",
+        },
+        tokenizer=tokenizer,
+    )
+
+
+class TestJoinDecodedTextReturnsStr:
+    """_join_decoded_text always returns str."""
+
+    @pytest.fixture
+    def bare_scanner(self):
+        return TokenIDScanner({}, tokenizer=None)
+
+    def test_mixed_items(self, bare_scanner):
+        items = [
+            TextChunk("hello "),
+            PreLexedTerminal("TOOL_START", 42, "<tool_call>"),
+            TextChunk(" world"),
+        ]
+        result = bare_scanner._join_decoded_text(items)
+        assert isinstance(result, str)
+        assert result == "hello <tool_call> world"
+
+    def test_empty_list(self, bare_scanner):
+        result = bare_scanner._join_decoded_text([])
+        assert isinstance(result, str)
+        assert result == ""
+
+    def test_only_text_chunks(self, bare_scanner):
+        result = bare_scanner._join_decoded_text([TextChunk("abc"), TextChunk("def")])
+        assert result == "abcdef"
+
+
+class TestHoldbackTextRecovery:
+    def test_holdback_text_with_special_token_text_absent(self, scanner):
+        """Terminal deferred when its text is absent from delta_text."""
+        result = scanner.scan(
+            delta_text="processed is appropriate.",
+            delta_token_ids=[CHANNEL_END_ID],
+        )
+
+        assert len(result) == 0
+
+        result2 = scanner.scan(
+            delta_text="<channel|>Understood.",
+            delta_token_ids=[20, 21],
+        )
+        pre_lexed = [r for r in result2 if isinstance(r, PreLexedTerminal)]
+        assert len(pre_lexed) == 1
+        assert pre_lexed[0].terminal == "THINK_END"
+        texts = [r.text for r in result2 if isinstance(r, TextChunk)]
+        combined = "".join(texts)
+        assert "processed is appropriate." in combined
+        assert "Understood." in combined
+
+    def test_holdback_text_with_special_token_text_present(self, scanner):
+        """Hold-back text + special token text both in delta_text."""
+        result = scanner.scan(
+            delta_text="holdback text<channel|>",
+            delta_token_ids=[CHANNEL_END_ID],
+        )
+
+        assert len(result) == 2
+        assert isinstance(result[0], TextChunk)
+        assert result[0].text == "holdback text"
+        assert isinstance(result[1], PreLexedTerminal)
+        assert result[1].terminal == "THINK_END"
+
+    def test_no_holdback_text(self, scanner):
+        """delta_text is exactly the special token text."""
+        result = scanner.scan(
+            delta_text="<channel|>",
+            delta_token_ids=[CHANNEL_END_ID],
+        )
+
+        assert len(result) == 1
+        assert isinstance(result[0], PreLexedTerminal)
+        assert result[0].terminal == "THINK_END"
+
+    def test_empty_delta_text(self, scanner):
+        """Empty delta_text defers the terminal until text arrives."""
+        result = scanner.scan(
+            delta_text="",
+            delta_token_ids=[CHANNEL_END_ID],
+        )
+
+        assert len(result) == 0
+
+        flushed = scanner.flush_pending()
+        assert len(flushed) == 1
+        assert isinstance(flushed[0], PreLexedTerminal)
+        assert flushed[0].terminal == "THINK_END"
+
+    def test_empty_delta_text_drops_individual_decode_text(self, tokenizer):
+        """Empty delta_text with multiple tokens: all results deferred."""
+        tool_start_id = 400
+        tok_a = 201
+        tok_b = 202
+        tokenizer.decode.side_effect = lambda ids: {
+            tool_start_id: "<|tool_call>",
+            tok_a: "call:",
+            tok_b: "get_weather",
+        }.get(ids[0], "?")
+
+        scanner = TokenIDScanner(
+            token_id_to_terminal={tool_start_id: "TOOL_START"},
+            tokenizer=tokenizer,
+        )
+
+        result = scanner.scan(
+            delta_text="",
+            delta_token_ids=[tool_start_id, tok_a, tok_b],
+        )
+
+        assert len(result) == 0
+
+        flushed = scanner.flush_pending()
+        assert len(flushed) == 1
+        assert isinstance(flushed[0], PreLexedTerminal)
+        assert flushed[0].terminal == "TOOL_START"
+
+    def test_holdback_before_start_tag(self, scanner):
+        result = scanner.scan(
+            delta_text="prefix text<|channel>",
+            delta_token_ids=[CHANNEL_START_ID],
+        )
+
+        assert len(result) == 2
+        assert isinstance(result[0], TextChunk)
+        assert result[0].text == "prefix text"
+        assert isinstance(result[1], PreLexedTerminal)
+        assert result[1].terminal == "THINK_START"
+
+    def test_multi_token_batch_special_in_middle(self, scanner, tokenizer):
+        """Multi-token batch with special token in the middle."""
+        tok_a = 201
+        tok_b = 202
+        tokenizer.decode.side_effect = lambda ids: {
+            tok_a: "wordA",
+            tok_b: "wordB",
+            CHANNEL_END_ID: CHANNEL_END,
+        }.get(ids[0], "?")
+
+        scanner_multi = TokenIDScanner(
+            token_id_to_terminal={CHANNEL_END_ID: "THINK_END"},
+            tokenizer=tokenizer,
+        )
+
+        result = scanner_multi.scan(
+            delta_text="holdback wordA<channel|> wordB",
+            delta_token_ids=[tok_a, CHANNEL_END_ID, tok_b],
+        )
+
+        texts = [r.text for r in result if isinstance(r, TextChunk)]
+        terminals = [r.terminal for r in result if isinstance(r, PreLexedTerminal)]
+        assert "THINK_END" in terminals
+        assert "holdback wordA" in "".join(texts)
+
+    def test_multi_token_batch_special_token_text_absent(self, scanner, tokenizer):
+        """Multi-token batch where special token text is absent."""
+        tok_a = 201
+        tok_b = 202
+        tokenizer.decode.side_effect = lambda ids: {
+            tok_a: "alpha",
+            tok_b: "beta",
+            CHANNEL_END_ID: CHANNEL_END,
+        }.get(ids[0], "?")
+
+        scanner_multi = TokenIDScanner(
+            token_id_to_terminal={CHANNEL_END_ID: "THINK_END"},
+            tokenizer=tokenizer,
+        )
+
+        result = scanner_multi.scan(
+            delta_text="holdback alpha",
+            delta_token_ids=[tok_a, CHANNEL_END_ID, tok_b],
+        )
+
+        assert len(result) == 0
+
+        result2 = scanner_multi.scan(
+            delta_text="<channel|> more text",
+            delta_token_ids=[300],
+        )
+        pre_lexed = [r for r in result2 if isinstance(r, PreLexedTerminal)]
+        assert len(pre_lexed) == 1
+        assert pre_lexed[0].terminal == "THINK_END"
+        text_chunks = [r for r in result2 if isinstance(r, TextChunk)]
+        combined = "".join(t.text for t in text_chunks)
+        assert "holdback alpha" in combined
+        assert "more text" in combined
+
+    def test_holdback_with_content_after_special_token(self, tokenizer):
+        """Hold-back + special token + content after in one delta."""
+        tok_content = 210
+        tokenizer.decode.side_effect = lambda ids: {
+            CHANNEL_END_ID: CHANNEL_END,
+            tok_content: "content start",
+        }.get(ids[0], "?")
+
+        scanner = TokenIDScanner(
+            token_id_to_terminal={CHANNEL_END_ID: "THINK_END"},
+            tokenizer=tokenizer,
+        )
+
+        result = scanner.scan(
+            delta_text="reasoning end.<channel|>content start",
+            delta_token_ids=[CHANNEL_END_ID, tok_content],
+        )
+
+        pre_lexed = [r for r in result if isinstance(r, PreLexedTerminal)]
+        assert len(pre_lexed) == 1
+        assert pre_lexed[0].terminal == "THINK_END"
+
+        text_chunks = [r for r in result if isinstance(r, TextChunk)]
+        combined = "".join(t.text for t in text_chunks)
+        assert "reasoning end." in combined
+
+
+class TestEndToEndReasoningHoldback:
+    """End-to-end engine tests with detokenizer hold-back."""
+
+    def test_reasoning_content_not_truncated(self):
+        config = gemma4_config()
+        tok = MagicMock()
+        vocab = {
+            CHANNEL_START: CHANNEL_START_ID,
+            CHANNEL_END: CHANNEL_END_ID,
+        }
+        tok.get_vocab.return_value = vocab
+        tok.decode.side_effect = lambda ids: {
+            CHANNEL_START_ID: CHANNEL_START,
+            CHANNEL_END_ID: CHANNEL_END,
+        }.get(ids[0], f"tok{ids[0]}")
+
+        engine = StreamingParserEngine(config, tok)
+        all_events = []
+
+        all_events.extend(engine.feed(CHANNEL_START, [CHANNEL_START_ID]))
+        all_events.extend(
+            engine.feed(
+                "thought\nThe request was received and ",
+                [10, 11, 12, 13, 14],
+            )
+        )
+        # CHANNEL_END token arrives but its text is held back.
+        all_events.extend(
+            engine.feed(
+                "processed is appropriate.",
+                [CHANNEL_END_ID],
+            )
+        )
+        # Detokenizer flushes the held-back text.
+        all_events.extend(
+            engine.feed(
+                "<channel|>Understood.",
+                [20, 21],
+            )
+        )
+
+        all_events.extend(engine.finish())
+
+        reasoning_text = "".join(
+            e.value for e in all_events if e.type == EventType.REASONING_CHUNK
+        )
+        content_text = "".join(
+            e.value for e in all_events if e.type == EventType.TEXT_CHUNK
+        )
+
+        assert "processed is appropriate." in reasoning_text
+        assert "Understood." in content_text
+
+    def test_backtick_content_not_truncated(self):
+        config = gemma4_config()
+        tok = MagicMock()
+        vocab = {
+            CHANNEL_START: CHANNEL_START_ID,
+            CHANNEL_END: CHANNEL_END_ID,
+        }
+        tok.get_vocab.return_value = vocab
+        tok.decode.side_effect = lambda ids: {
+            CHANNEL_START_ID: CHANNEL_START,
+            CHANNEL_END_ID: CHANNEL_END,
+        }.get(ids[0], f"tok{ids[0]}")
+
+        engine = StreamingParserEngine(config, tok)
+        all_events = []
+
+        all_events.extend(engine.feed(CHANNEL_START, [CHANNEL_START_ID]))
+        all_events.extend(
+            engine.feed(
+                "thought\n1/10 completed. Next: ",
+                [10, 11, 12, 13],
+            )
+        )
+        all_events.extend(
+            engine.feed(
+                "`hostname`.\n",
+                [CHANNEL_END_ID],
+            )
+        )
+        all_events.extend(
+            engine.feed(
+                "<channel|>tool output",
+                [20, 21],
+            )
+        )
+
+        all_events.extend(engine.finish())
+
+        reasoning_text = "".join(
+            e.value for e in all_events if e.type == EventType.REASONING_CHUNK
+        )
+
+        assert "`hostname`." in reasoning_text
+
+
+_CHANNEL_START_TAG = "<|channel>"
+_CHANNEL_END_TAG = "<channel|>"
+_TOOL_START_TAG = "<|tool_call>"
+_TOOL_END_TAG = "<tool_call|>"
+_QUOTE_TAG = '<|"|>'
+
+_CHANNEL_START_TID = 100
+_CHANNEL_END_TID = 101
+_TOOL_START_TID = 102
+_TOOL_END_TID = 103
+_QUOTE_TID = 104
+_TOK = list(range(200, 215))
+
+
+def _gemma4_vocab() -> dict[str, int]:
+    return {
+        _CHANNEL_START_TAG: _CHANNEL_START_TID,
+        _CHANNEL_END_TAG: _CHANNEL_END_TID,
+        _TOOL_START_TAG: _TOOL_START_TID,
+        _TOOL_END_TAG: _TOOL_END_TID,
+        _QUOTE_TAG: _QUOTE_TID,
+    }
+
+
+def _make_gemma4_tokenizer(
+    extra_decode: dict[int, str] | None = None,
+) -> MagicMock:
+    special = {
+        _CHANNEL_START_TID: _CHANNEL_START_TAG,
+        _CHANNEL_END_TID: _CHANNEL_END_TAG,
+        _TOOL_START_TID: _TOOL_START_TAG,
+        _TOOL_END_TID: _TOOL_END_TAG,
+        _QUOTE_TID: _QUOTE_TAG,
+    }
+    decode_map = {**special, **(extra_decode or {})}
+
+    tok = MagicMock()
+    tok.get_vocab.return_value = _gemma4_vocab()
+    tok.decode.side_effect = lambda ids: decode_map.get(ids[0], f"tok{ids[0]}")
+    return tok
+
+
+def _collect_events(engine, deltas):
+    from vllm.parser.engine.events import SemanticEvent
+
+    all_events: list[SemanticEvent] = []
+    for delta_text, delta_token_ids in deltas:
+        all_events.extend(engine.feed(delta_text, delta_token_ids))
+    all_events.extend(engine.finish())
+    return all_events
+
+
+def _reasoning_text(events) -> str:
+    return "".join(e.value for e in events if e.type == EventType.REASONING_CHUNK)
+
+
+def _content_text(events) -> str:
+    return "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
+
+
+def _arg_text(events) -> str:
+    return "".join(e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK)
+
+
+def _has_event(events, event_type) -> bool:
+    return any(e.type == event_type for e in events)
+
+
+class TestMultiTokenBoundaryPreservation:
+    """No text lost at state boundaries with multi-token deltas."""
+
+    def test_empty_delta_text_at_channel_end_unified(self):
+        """Empty delta_text when CHANNEL_END arrives; text comes later."""
+        tok = _make_gemma4_tokenizer()
+        engine = StreamingParserEngine(gemma4_config(), tok)
+
+        events = _collect_events(
+            engine,
+            [
+                ("", [_CHANNEL_START_TID]),
+                ("<|channel>thought\nSome reasoning.", [_TOK[0], _TOK[1]]),
+                ("", [_CHANNEL_END_TID]),
+                ("<channel|>Final answer.", [_TOK[2], _TOK[3]]),
+            ],
+        )
+
+        reasoning = _reasoning_text(events)
+        content = _content_text(events)
+        assert "Some reasoning." in reasoning
+        assert "Final answer." in content
+        assert _has_event(events, EventType.REASONING_START)
+        assert _has_event(events, EventType.REASONING_END)
+
+    def test_deferred_channel_end_flushed_at_finish_unified(self):
+        """Deferred CHANNEL_END flushed at end-of-stream."""
+        tok = _make_gemma4_tokenizer()
+        engine = StreamingParserEngine(gemma4_config(), tok)
+
+        events = _collect_events(
+            engine,
+            [
+                (_CHANNEL_START_TAG, [_CHANNEL_START_TID]),
+                ("thought\nReasoning text.", [_TOK[0]]),
+                (" Final thought.", [_CHANNEL_END_TID]),
+            ],
+        )
+
+        reasoning = _reasoning_text(events)
+        assert "Reasoning text. Final thought." in reasoning
+        assert _has_event(events, EventType.REASONING_END)
+
+    def test_reasoning_to_tool_call_handoff_unified(self):
+        """Full reasoning -> content -> tool call flow."""
+        tok = _make_gemma4_tokenizer()
+        engine = StreamingParserEngine(gemma4_config(), tok)
+
+        events = _collect_events(
+            engine,
+            [
+                (_CHANNEL_START_TAG, [_CHANNEL_START_TID]),
+                ("thought\nI need to check the weather.", [_TOK[0], _TOK[1], _TOK[2]]),
+                (_CHANNEL_END_TAG, [_CHANNEL_END_TID]),
+                ("Let me call a tool.", [_TOK[3], _TOK[4]]),
+                (_TOOL_START_TAG, [_TOOL_START_TID]),
+                ("call:get_weather{city:", [_TOK[5], _TOK[6]]),
+                ('<|"|>SF<|"|>}', [_QUOTE_TID, _TOK[7], _QUOTE_TID, _TOK[8]]),
+                (_TOOL_END_TAG, [_TOOL_END_TID]),
+            ],
+        )
+
+        reasoning = _reasoning_text(events)
+        content = _content_text(events)
+
+        assert "I need to check the weather." in reasoning
+        assert "Let me call a tool." in content
+        assert _has_event(events, EventType.REASONING_START)
+        assert _has_event(events, EventType.REASONING_END)
+        assert _has_event(events, EventType.TOOL_CALL_START)
+        assert _has_event(events, EventType.TOOL_CALL_END)
+        assert "SF" in _arg_text(events)
+
+    def test_multiple_tool_calls_rapid_transitions_unified(self):
+        """Two back-to-back tool calls with correct tool_index tracking."""
+        tok = _make_gemma4_tokenizer()
+        engine = StreamingParserEngine(gemma4_config(), tok)
+
+        events = _collect_events(
+            engine,
+            [
+                (_TOOL_START_TAG, [_TOOL_START_TID]),
+                ("call:get_weather{city:", [_TOK[0], _TOK[1]]),
+                ('<|"|>NYC<|"|>}', [_QUOTE_TID, _TOK[2], _QUOTE_TID, _TOK[3]]),
+                (_TOOL_END_TAG, [_TOOL_END_TID]),
+                (_TOOL_START_TAG, [_TOOL_START_TID]),
+                ("call:get_time{tz:", [_TOK[4], _TOK[5]]),
+                ('<|"|>EST<|"|>}', [_QUOTE_TID, _TOK[6], _QUOTE_TID, _TOK[7]]),
+                (_TOOL_END_TAG, [_TOOL_END_TID]),
+            ],
+        )
+
+        starts = [e for e in events if e.type == EventType.TOOL_CALL_START]
+        ends = [e for e in events if e.type == EventType.TOOL_CALL_END]
+        assert len(starts) == 2
+        assert len(ends) == 2
+        assert starts[0].tool_index == 0
+        assert starts[1].tool_index == 1
+
+        names = "".join(e.value for e in events if e.type == EventType.TOOL_NAME)
+        assert "get_weather" in names
+        assert "get_time" in names
+
+    def test_deferred_channel_end_before_tool_call_unified(self):
+        """Deferred CHANNEL_END followed by a tool call."""
+        tok = _make_gemma4_tokenizer()
+        engine = StreamingParserEngine(gemma4_config(), tok)
+
+        events = _collect_events(
+            engine,
+            [
+                (_CHANNEL_START_TAG, [_CHANNEL_START_TID]),
+                ("thought\nNeed to call a tool.", [_TOK[0], _TOK[1]]),
+                (" Let me proceed.", [_CHANNEL_END_TID]),
+                (_CHANNEL_END_TAG, [_TOK[2]]),
+                (_TOOL_START_TAG, [_TOOL_START_TID]),
+                ("call:get_weather{city:", [_TOK[3], _TOK[4]]),
+                ('<|"|>Tokyo<|"|>}', [_QUOTE_TID, _TOK[5], _QUOTE_TID, _TOK[6]]),
+                (_TOOL_END_TAG, [_TOOL_END_TID]),
+            ],
+        )
+
+        reasoning = _reasoning_text(events)
+        assert "Need to call a tool. Let me proceed." in reasoning
+        assert _has_event(events, EventType.REASONING_END)
+        assert _has_event(events, EventType.TOOL_CALL_START)
+        assert _has_event(events, EventType.TOOL_CALL_END)
+        assert "Tokyo" in _arg_text(events)
+
+
+class TestStreamInterval10:
+    """Tests with stream_interval=10 (large multi-token batches)."""
+
+    def test_channel_end_mid_batch_text_present(self):
+        """<channel|> mid-batch with its text present in delta_text."""
+        tok = _make_gemma4_tokenizer({_TOK[i]: f"word{i} " for i in range(15)})
+        engine = StreamingParserEngine(gemma4_config(), tok)
+
+        events: list = []
+        events.extend(
+            engine.feed(
+                "<|channel>thought\nword0 word1 word2 word3 word4 "
+                "word5 word6 word7 word8 ",
+                [
+                    _CHANNEL_START_TID,
+                    _TOK[0],
+                    _TOK[1],
+                    _TOK[2],
+                    _TOK[3],
+                    _TOK[4],
+                    _TOK[5],
+                    _TOK[6],
+                    _TOK[7],
+                    _TOK[8],
+                ],
+            )
+        )
+
+        events.extend(
+            engine.feed(
+                "word9 word10 word11 <channel|>word12 word13 word14 word0 word1 word2 ",
+                [
+                    _TOK[9],
+                    _TOK[10],
+                    _TOK[11],
+                    _CHANNEL_END_TID,
+                    _TOK[12],
+                    _TOK[13],
+                    _TOK[14],
+                    _TOK[0],
+                    _TOK[1],
+                    _TOK[2],
+                ],
+            )
+        )
+
+        events.extend(engine.finish())
+
+        reasoning = _reasoning_text(events)
+        content = _content_text(events)
+
+        for w in ("word9", "word10", "word11"):
+            assert w in reasoning, f"{w!r} missing from reasoning"
+
+        for w in ("word12", "word13", "word14"):
+            assert w in content, f"{w!r} missing from content"
+
+        assert _has_event(events, EventType.REASONING_END)
+
+    def test_channel_end_and_tool_start_same_batch_unified(self):
+        """Both <channel|> and <|tool_call> in a single batch."""
+        tok = _make_gemma4_tokenizer({_TOK[i]: f"w{i} " for i in range(15)})
+        engine = StreamingParserEngine(gemma4_config(), tok)
+
+        events: list = []
+
+        events.extend(
+            engine.feed(
+                "<|channel>thought\nw0 w1 w2 w3 w4 w5 w6 w7 w8 ",
+                [
+                    _CHANNEL_START_TID,
+                    _TOK[0],
+                    _TOK[1],
+                    _TOK[2],
+                    _TOK[3],
+                    _TOK[4],
+                    _TOK[5],
+                    _TOK[6],
+                    _TOK[7],
+                    _TOK[8],
+                ],
+            )
+        )
+
+        events.extend(
+            engine.feed(
+                "w9 w10 <channel|>w11 <|tool_call>",
+                [
+                    _TOK[9],
+                    _TOK[10],
+                    _CHANNEL_END_TID,
+                    _TOK[11],
+                    _TOOL_START_TID,
+                    _TOK[12],
+                    _TOK[13],
+                    _TOK[14],
+                    _TOK[0],
+                    _TOK[1],
+                ],
+            )
+        )
+        events.extend(engine.finish())
+
+        reasoning = _reasoning_text(events)
+
+        assert "w9" in reasoning
+        assert "w10" in reasoning
+        assert _has_event(events, EventType.REASONING_END)
+        assert _has_event(events, EventType.TOOL_CALL_START)
+
+    def test_channel_end_mid_batch_text_absent(self):
+        """<channel|> mid-batch with its text absent from delta_text."""
+        tok = _make_gemma4_tokenizer({_TOK[i]: f"word{i} " for i in range(15)})
+        engine = StreamingParserEngine(gemma4_config(), tok)
+
+        events: list = []
+        events.extend(
+            engine.feed(
+                "<|channel>thought\nword0 word1 word2 word3 word4 "
+                "word5 word6 word7 word8 ",
+                [
+                    _CHANNEL_START_TID,
+                    _TOK[0],
+                    _TOK[1],
+                    _TOK[2],
+                    _TOK[3],
+                    _TOK[4],
+                    _TOK[5],
+                    _TOK[6],
+                    _TOK[7],
+                    _TOK[8],
+                ],
+            )
+        )
+
+        events.extend(
+            engine.feed(
+                "word9 word10 word11 ",
+                [
+                    _TOK[9],
+                    _TOK[10],
+                    _TOK[11],
+                    _CHANNEL_END_TID,
+                    _TOK[12],
+                    _TOK[13],
+                    _TOK[14],
+                    _TOK[0],
+                    _TOK[1],
+                    _TOK[2],
+                ],
+            )
+        )
+
+        events.extend(
+            engine.feed(
+                "<channel|>word12 word13 word14 word0 word1 word2 ",
+                [_TOK[3], _TOK[4], _TOK[5]],
+            )
+        )
+
+        events.extend(engine.finish())
+
+        reasoning = _reasoning_text(events)
+        content = _content_text(events)
+
+        for w in ("word9", "word10", "word11"):
+            assert w in reasoning, f"{w!r} missing from reasoning"
+
+        for w in ("word12", "word13", "word14"):
+            assert w in content, f"{w!r} missing from content"
+
+        assert _has_event(events, EventType.REASONING_END)
+
+    def test_tool_end_mid_batch_text_absent_unified(self):
+        """<tool_call|> mid-batch with text absent."""
+        tok = _make_gemma4_tokenizer({_TOK[i]: f"w{i}" for i in range(15)})
+        engine = StreamingParserEngine(gemma4_config(), tok)
+
+        events: list = []
+        events.extend(
+            engine.feed(
+                _CHANNEL_START_TAG,
+                [_CHANNEL_START_TID],
+            )
+        )
+        events.extend(
+            engine.feed(
+                "thought\nNeed a tool.",
+                [_TOK[0], _TOK[1]],
+            )
+        )
+        events.extend(
+            engine.feed(
+                _TOOL_START_TAG,
+                [_TOOL_START_TID],
+            )
+        )
+        events.extend(
+            engine.feed(
+                "call:get_weather{city:",
+                [_TOK[2], _TOK[3], _TOK[4]],
+            )
+        )
+
+        events.extend(
+            engine.feed(
+                '<|"|>San Francisco<|"|>}',
+                [
+                    _QUOTE_TID,
+                    _TOK[5],
+                    _TOK[6],
+                    _QUOTE_TID,
+                    _TOK[7],
+                    _TOOL_END_TID,
+                    _TOK[8],
+                    _TOK[9],
+                    _TOK[10],
+                    _TOK[11],
+                ],
+            )
+        )
+
+        events.extend(
+            engine.feed(
+                "<tool_call|>w8w9w10w11w12",
+                [_TOK[12], _TOK[13]],
+            )
+        )
+
+        events.extend(engine.finish())
+
+        assert _has_event(events, EventType.TOOL_CALL_END)
+        assert "San Francisco" in _arg_text(events)
+
+    def test_large_batch_holdback_spans_two_batches(self):
+        """Holdback text spanning two batches with <channel|> in the second."""
+        tok = _make_gemma4_tokenizer({_TOK[i]: f"w{i} " for i in range(15)})
+        engine = StreamingParserEngine(gemma4_config(), tok)
+
+        events: list = []
+
+        events.extend(
+            engine.feed(
+                "<|channel>thought\nThe user asked about machine learning "
+                "and I need to think about the best approach to",
+                [
+                    _CHANNEL_START_TID,
+                    _TOK[0],
+                    _TOK[1],
+                    _TOK[2],
+                    _TOK[3],
+                    _TOK[4],
+                    _TOK[5],
+                    _TOK[6],
+                    _TOK[7],
+                    _TOK[8],
+                ],
+            )
+        )
+
+        events.extend(
+            engine.feed(
+                " explain this complex topic. Let me organize my thoughts.",
+                [
+                    _TOK[9],
+                    _TOK[10],
+                    _TOK[11],
+                    _TOK[12],
+                    _TOK[13],
+                    _TOK[14],
+                    _CHANNEL_END_TID,
+                    _TOK[0],
+                    _TOK[1],
+                    _TOK[2],
+                ],
+            )
+        )
+
+        events.extend(
+            engine.feed(
+                "<channel|>w0 w1 w2 Here is what I recommend: start with "
+                "the fundamentals and build up from there.",
+                [
+                    _TOK[3],
+                    _TOK[4],
+                    _TOK[5],
+                    _TOK[6],
+                    _TOK[7],
+                    _TOK[8],
+                    _TOK[9],
+                    _TOK[10],
+                    _TOK[11],
+                    _TOK[12],
+                ],
+            )
+        )
+
+        events.extend(engine.finish())
+
+        reasoning = _reasoning_text(events)
+        content = _content_text(events)
+
+        assert "organize my thoughts." in reasoning
+        assert "explain" in reasoning
+        assert "recommend" in content
+
+        assert _has_event(events, EventType.REASONING_START)
+        assert _has_event(events, EventType.REASONING_END)
+
+
+class TestRebuildFromAnchorsLiteralLookalike:
+    """Literal token text in prose must not be consumed as an anchor."""
+
+    @pytest.fixture
+    def tool_scanner(self):
+        tok = MagicMock()
+        tok.get_vocab.return_value = {
+            TOOL_START: TOOL_START_ID,
+            TOOL_END: TOOL_END_ID,
+        }
+        tok.decode.side_effect = lambda ids: {
+            TOOL_START_ID: TOOL_START,
+            TOOL_END_ID: TOOL_END,
+        }.get(ids[0], f"t{ids[0]}")
+        return TokenIDScanner(
+            {TOOL_START_ID: "TOOL_START", TOOL_END_ID: "TOOL_END"},
+            tok,
+        )
+
+    def test_literal_before_real_anchor(self, tool_scanner):
+        delta_text = 'Use <tool_call> like this: <tool_call>{"name":"f"}</tool_call>'
+        delta_token_ids = [1, 2, 3, 4, 5, TOOL_START_ID, 6, 7, TOOL_END_ID]
+        items = tool_scanner.scan(delta_text, delta_token_ids)
+
+        text_parts = [it.text for it in items if isinstance(it, TextChunk)]
+        terminals = [it for it in items if isinstance(it, PreLexedTerminal)]
+
+        assert len(terminals) == 2
+        assert terminals[0].terminal == "TOOL_START"
+        assert terminals[1].terminal == "TOOL_END"
+
+        joined_text = "".join(text_parts)
+        assert "<tool_call>" in joined_text
+        assert '{"name":"f"}' in joined_text
+
+    def test_multiple_tool_calls_with_literal_between(self, tool_scanner):
+        delta_text = (
+            '<tool_call>{"name":"a"}</tool_call>'
+            " see <tool_call> syntax "
+            '<tool_call>{"name":"b"}</tool_call>'
+        )
+        delta_token_ids = [
+            TOOL_START_ID,
+            1,
+            TOOL_END_ID,
+            2,
+            3,
+            4,
+            TOOL_START_ID,
+            5,
+            TOOL_END_ID,
+        ]
+        items = tool_scanner.scan(delta_text, delta_token_ids)
+
+        terminals = [it for it in items if isinstance(it, PreLexedTerminal)]
+        assert len(terminals) == 4
+
+        text_parts = [it.text for it in items if isinstance(it, TextChunk)]
+        joined_text = "".join(text_parts)
+        assert "<tool_call> syntax" in joined_text
+
+
+class TestRebuildFromAnchorsCascadingDeferral:
+    """Missing middle anchor defers only itself, not subsequent ones."""
+
+    @pytest.fixture
+    def bare_scanner(self):
+        tok = MagicMock()
+        tok.decode.side_effect = lambda ids: f"t{ids[0]}"
+        return TokenIDScanner({}, tok)
+
+    def test_middle_anchor_missing_does_not_cascade(self, bare_scanner):
+        a = PreLexedTerminal("TOOL_START", TOOL_START_ID, TOOL_START)
+        b = PreLexedTerminal("THINK_END", CHANNEL_END_ID, CHANNEL_END)
+        c = PreLexedTerminal("TOOL_END", TOOL_END_ID, TOOL_END)
+        delta_text = f"prefix{TOOL_START}middle{TOOL_END}suffix"
+        results = [a, b, c]
+
+        rebuilt = bare_scanner._rebuild_from_anchors(delta_text, results)
+
+        terminals = [r for r in rebuilt if isinstance(r, PreLexedTerminal)]
+        texts = [r for r in rebuilt if isinstance(r, TextChunk)]
+        joined = "".join(t.text for t in texts)
+
+        assert len(terminals) == 2
+        assert terminals[0].terminal == "TOOL_START"
+        assert terminals[1].terminal == "TOOL_END"
+        assert "prefix" in joined
+        assert "middle" in joined
+        assert "suffix" in joined
+        assert len(bare_scanner._deferred_terminals) == 1
+        assert bare_scanner._deferred_terminals[0].terminal == "THINK_END"
+        assert bare_scanner._deferred_post_text == ""
+
+    def test_first_anchor_missing_rest_still_emitted(self, bare_scanner):
+        a = PreLexedTerminal("THINK_END", CHANNEL_END_ID, CHANNEL_END)
+        b = PreLexedTerminal("TOOL_START", TOOL_START_ID, TOOL_START)
+        delta_text = f"text{TOOL_START}more"
+        results = [a, b]
+
+        rebuilt = bare_scanner._rebuild_from_anchors(delta_text, results)
+
+        terminals = [r for r in rebuilt if isinstance(r, PreLexedTerminal)]
+        assert len(terminals) == 1
+        assert terminals[0].terminal == "TOOL_START"
+        assert len(bare_scanner._deferred_terminals) == 1
+        assert bare_scanner._deferred_terminals[0].terminal == "THINK_END"
+
+    def test_last_anchor_missing_preceding_still_emitted(self, bare_scanner):
+        a = PreLexedTerminal("TOOL_START", TOOL_START_ID, TOOL_START)
+        b = PreLexedTerminal("THINK_END", CHANNEL_END_ID, CHANNEL_END)
+        delta_text = f"text{TOOL_START}more"
+        results = [a, b]
+
+        rebuilt = bare_scanner._rebuild_from_anchors(delta_text, results)
+
+        terminals = [r for r in rebuilt if isinstance(r, PreLexedTerminal)]
+        assert len(terminals) == 1
+        assert terminals[0].terminal == "TOOL_START"
+        texts = [r for r in rebuilt if isinstance(r, TextChunk)]
+        joined = "".join(t.text for t in texts)
+        assert "text" in joined
+        assert bare_scanner._deferred_post_text == "more"
+        assert len(bare_scanner._deferred_terminals) == 1
+        assert bare_scanner._deferred_terminals[0].terminal == "THINK_END"

tests/parser/engine/test_ufffd_reasoning_transition.py

@@ -0,0 +1,181 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Regression test for U+FFFD leak at reasoning→content transition.
+
+When byte-fallback tokens span the reasoning/content boundary,
+decoding isolated content-side token IDs via tokenizer.decode()
+produces U+FFFD (Unicode replacement character).  The fix flushes
+the reasoning parser's engine lexer instead.
+
+Reproduces the bug at various chunk sizes and validates that the
+fix prevents U+FFFD from leaking into streamed content.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from tests.parser.engine.replay_harness import (
+    CHUNK_SIZES,
+    MockTokenizer,
+    collect_output,
+    replay_streaming,
+)
+from vllm.parser.abstract_parser import DelegatingParser
+from vllm.parser.engine.registered_adapters import (
+    Glm47MoeParserReasoningAdapter,
+    Glm47MoeParserToolAdapter,
+    Qwen3ParserReasoningAdapter,
+    Qwen3ParserToolAdapter,
+)
+
+
+class ByteFallbackMockTokenizer(MockTokenizer):
+    """MockTokenizer that returns U+FFFD for specified token IDs.
+
+    Simulates byte-fallback tokenizer behavior where isolated
+    partial-byte tokens decode to the Unicode replacement character.
+    """
+
+    def __init__(
+        self,
+        vocab: dict[str, int],
+        tokens: list[tuple[int, str]],
+        ufffd_token_ids: set[int],
+    ) -> None:
+        super().__init__(vocab, tokens)
+        self._ufffd_token_ids = frozenset(ufffd_token_ids)
+
+    def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str:
+        parts: list[str] = []
+        for tid in ids:
+            if skip_special_tokens and tid in self._special_ids:
+                continue
+            if tid in self._ufffd_token_ids:
+                parts.append("�")
+            else:
+                text = self._token_decode_map.get(tid, f"?{tid}?")
+                parts.append(text)
+        return "".join(parts)
+
+
+# ── Model-specific DelegatingParser subclasses ───────────────────────
+
+
+class _Glm47Delegating(DelegatingParser):
+    reasoning_parser_cls = Glm47MoeParserReasoningAdapter
+    tool_parser_cls = Glm47MoeParserToolAdapter
+
+
+class _Qwen3Delegating(DelegatingParser):
+    reasoning_parser_cls = Qwen3ParserReasoningAdapter
+    tool_parser_cls = Qwen3ParserToolAdapter
+
+
+# ── Shared test data ─────────────────────────────────────────────────
+
+_SHARED_TOKENS: list[tuple[int, str]] = [
+    (100, "Let me"),
+    (101, " think"),
+    (102, " about"),
+    (103, " Samsung."),
+    (51, "</think>"),
+    (200, "삼성"),
+    (201, "전자의"),
+    (202, " 주가를"),
+    (203, " 분석합니다."),
+]
+
+_SHARED_UFFFD_IDS: set[int] = {200}
+
+EXPECTED_REASONING = "Let me think about Samsung."
+EXPECTED_CONTENT = "삼성전자의 주가를 분석합니다."
+
+_MODEL_CONFIGS = [
+    pytest.param(
+        {
+            "<think>": 50,
+            "</think>": 51,
+            "<tool_call>": 60,
+            "</tool_call>": 61,
+            "<arg_key>": 62,
+            "</arg_key>": 63,
+            "<arg_value>": 64,
+            "</arg_value>": 65,
+        },
+        _Glm47Delegating,
+        id="glm47",
+    ),
+    pytest.param(
+        {
+            "<think>": 50,
+            "</think>": 51,
+            "<tool_call>": 60,
+            "</tool_call>": 61,
+        },
+        _Qwen3Delegating,
+        id="qwen3",
+    ),
+]
+
+
+# ── Tests ────────────────────────────────────────────────────────────
+
+
+class TestUfffdReasoningTransition:
+    """U+FFFD must not appear at the reasoning→content transition."""
+
+    @pytest.mark.parametrize("vocab,delegating_cls", _MODEL_CONFIGS)
+    @pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}")
+    def test_no_ufffd(self, chunk_size, vocab, delegating_cls):
+        tokenizer = ByteFallbackMockTokenizer(vocab, _SHARED_TOKENS, _SHARED_UFFFD_IDS)
+        parser = delegating_cls(tokenizer)
+        deltas = replay_streaming(
+            parser,
+            _SHARED_TOKENS,
+            chunk_size=chunk_size,
+            finished_on_last=True,
+        )
+        output = collect_output(deltas)
+
+        assert "�" not in output.content, (
+            f"U+FFFD leaked into content: {output.content!r}"
+        )
+        assert output.content == EXPECTED_CONTENT
+        assert output.reasoning == EXPECTED_REASONING
+
+    def test_byte_fallback_tokenizer_produces_ufffd(self):
+        """Validate the fixture: decode() returns U+FFFD for isolated
+        byte-fallback token IDs, proving the old code path would leak."""
+        vocab = dict(_MODEL_CONFIGS[0].values[0])
+        tokenizer = ByteFallbackMockTokenizer(vocab, _SHARED_TOKENS, _SHARED_UFFFD_IDS)
+        assert tokenizer.decode([200]) == "�"
+
+    @pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}")
+    def test_multiple_ufffd_tokens_at_boundary(self, chunk_size):
+        """Multiple consecutive byte-fallback tokens at the boundary."""
+        tokens: list[tuple[int, str]] = [
+            (100, "Reasoning."),
+            (51, "</think>"),
+            (200, "삼"),
+            (201, "성"),
+            (202, "전자"),
+        ]
+        ufffd_ids: set[int] = {200, 201}
+        vocab = dict(_MODEL_CONFIGS[0].values[0])
+
+        tokenizer = ByteFallbackMockTokenizer(vocab, tokens, ufffd_ids)
+        parser = _Glm47Delegating(tokenizer)
+        deltas = replay_streaming(
+            parser,
+            tokens,
+            chunk_size=chunk_size,
+            finished_on_last=True,
+        )
+        output = collect_output(deltas)
+
+        assert "�" not in output.content, (
+            f"U+FFFD leaked into content: {output.content!r}"
+        )
+        assert output.content == "삼성전자"
+        assert output.reasoning == "Reasoning."

tests/parser/engine/trace_builder.py

@@ -0,0 +1,991 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""On-demand trace builder for parser engine testing and benchmarks.
+
+Generates token sequences programmatically from model-agnostic scenario
+definitions.  Each model format handler knows how to render scenarios
+into the model's output format, tokenize them with correct special token
+IDs, and compute expected parse outputs.
+
+Every generated sample is self-validated by replaying it through the
+real parser before being returned.
+"""
+
+from __future__ import annotations
+
+import functools
+import json
+from dataclasses import dataclass
+from typing import Any
+
+from tests.parser.engine.replay_harness import (
+    MockTokenizer,
+    Sample,
+    assert_parse_output,
+    collect_output,
+    replay_streaming,
+)
+from vllm.entrypoints.openai.chat_completion.protocol import (
+    ChatCompletionToolsParam,
+)
+from vllm.parser.engine.registered_adapters import (
+    DeepSeekV4Parser,
+    DeepSeekV32Parser,
+    Gemma4Parser,
+    Glm47MoeParser,
+    KimiK2Parser,
+    MinimaxM2Parser,
+    NemotronV3Parser,
+    Qwen3Parser,
+    SeedOssParser,
+)
+
+# ── Data structures ──────────────────────────────────────────────────
+
+
+@dataclass
+class ToolCallSpec:
+    name: str
+    arguments: dict[str, Any]
+
+
+@dataclass
+class Scenario:
+    id: str
+    description: str
+    reasoning: str | None = None
+    content: str | None = None
+    tool_calls: list[ToolCallSpec] | None = None
+    after_tool_response: bool = False
+
+
+# ── Scenarios ────────────────────────────────────────────────────────
+
+_READ_TOOL = ToolCallSpec("read_file", {"path": "/tmp/test.txt"})
+_BASH_TOOL = ToolCallSpec(
+    "bash", {"command": "hostname", "description": "Get hostname"}
+)
+_WEATHER_TOOL = ToolCallSpec(
+    "get_weather",
+    {"city": "Dallas", "state": "TX", "unit": "fahrenheit"},
+)
+_COMPLEX_TOOL = ToolCallSpec(
+    "search",
+    {
+        "query": "vllm parser",
+        "filters": {"language": "python", "min_stars": 100},
+        "tags": ["ml", "inference"],
+        "limit": 10,
+        "verbose": True,
+    },
+)
+
+SCENARIOS: list[Scenario] = [
+    Scenario(
+        id="think-then-tool",
+        description="Reasoning then single tool call",
+        reasoning="Let me check the file.",
+        tool_calls=[_READ_TOOL],
+    ),
+    Scenario(
+        id="think-then-parallel-tools",
+        description="Reasoning then two parallel tool calls",
+        reasoning="I need to run both commands.",
+        tool_calls=[_BASH_TOOL, _WEATHER_TOOL],
+    ),
+    Scenario(
+        id="think-then-content",
+        description="Reasoning then content response",
+        reasoning="Let me think about this carefully.",
+        content="The answer is 42.",
+    ),
+    Scenario(
+        id="content-only",
+        description="Plain content response without reasoning",
+        content="Hello! How can I help you today?",
+    ),
+    Scenario(
+        id="tool-only",
+        description="Tool call without reasoning",
+        tool_calls=[_READ_TOOL],
+    ),
+    Scenario(
+        id="complex-json-args",
+        description="Tool call with nested objects, arrays, numbers, booleans",
+        reasoning="This needs a complex query.",
+        tool_calls=[_COMPLEX_TOOL],
+    ),
+    Scenario(
+        id="whitespace-before-tool",
+        description="Whitespace-only content before tool call",
+        content="\n\n",
+        tool_calls=[_WEATHER_TOOL],
+    ),
+    Scenario(
+        id="think-content-tool",
+        description="Reasoning, content, then tool call",
+        reasoning="Let me analyze and then fetch data.",
+        content="Checking the weather now.",
+        tool_calls=[_WEATHER_TOOL],
+    ),
+    Scenario(
+        id="think-whitespace-tool",
+        description="Reasoning, whitespace-only gap, then tool call",
+        reasoning="Let me check the file contents.",
+        content="\n\n",
+        tool_calls=[_READ_TOOL],
+    ),
+    Scenario(
+        id="empty-reasoning-content",
+        description="Empty reasoning section followed by content",
+        reasoning="",
+        content="The epoch timestamp is 1779111346.",
+    ),
+    Scenario(
+        id="tool-after-tool-response",
+        description="Tool call immediately after tool response (agentic flow)",
+        tool_calls=[_READ_TOOL],
+        after_tool_response=True,
+    ),
+    Scenario(
+        id="empty-tool-block",
+        description="Empty tool block followed by content (edge case recovery)",
+        content="Content after empty tools.",
+        tool_calls=[],
+    ),
+]
+
+
+# ── Tokenization ─────────────────────────────────────────────────────
+
+
+def _word_split(text: str) -> list[str]:
+    """Split text into word-like tokens, preserving all characters."""
+    if not text:
+        return []
+    parts: list[str] = []
+    current = ""
+    for ch in text:
+        if ch in " \t\n\r" and current and current[-1] not in " \t\n\r":
+            parts.append(current)
+            current = ch
+        else:
+            current += ch
+    if current:
+        parts.append(current)
+    return parts
+
+
+def _tokenize(
+    segments: list[tuple[str, bool]],
+    vocab: dict[str, int],
+    start_id: int = 100,
+) -> list[tuple[int, str]]:
+    """Build token list from segments.
+
+    Each segment is ``(text, is_special)``.  Special segments use vocab
+    IDs; content segments are word-split with sequential IDs.
+    """
+    tokens: list[tuple[int, str]] = []
+    next_id = start_id
+
+    for text, is_special in segments:
+        if not text:
+            continue
+        if is_special:
+            tid = vocab.get(text)
+            if tid is None:
+                raise ValueError(f"Special token {text!r} not in vocab")
+            tokens.append((tid, text))
+        else:
+            for word in _word_split(text):
+                tokens.append((next_id, word))
+                next_id += 1
+
+    return tokens
+
+
+# ── Tool definitions ─────────────────────────────────────────────────
+
+
+def _infer_schema(value: object) -> dict:
+    """Infer a JSON Schema from a Python value, recursing into dicts/lists."""
+    if isinstance(value, bool):
+        return {"type": "boolean"}
+    if isinstance(value, int):
+        return {"type": "integer"}
+    if isinstance(value, float):
+        return {"type": "number"}
+    if isinstance(value, str):
+        return {"type": "string"}
+    if isinstance(value, dict):
+        return {
+            "type": "object",
+            "properties": {k: _infer_schema(v) for k, v in value.items()},
+        }
+    if isinstance(value, list) and value:
+        return {"type": "array", "items": _infer_schema(value[0])}
+    if isinstance(value, list):
+        return {"type": "array"}
+    return {}
+
+
+def _tool_defs(tool_calls: list[ToolCallSpec]) -> list[dict]:
+    """Generate OpenAI-style tool definitions from tool call specs."""
+    seen: set[str] = set()
+    tools: list[dict] = []
+    for tc in tool_calls:
+        if tc.name in seen:
+            continue
+        seen.add(tc.name)
+        properties = {k: _infer_schema(v) for k, v in tc.arguments.items()}
+        tools.append(
+            {
+                "type": "function",
+                "function": {
+                    "name": tc.name,
+                    "parameters": {
+                        "type": "object",
+                        "properties": properties,
+                    },
+                },
+            }
+        )
+    return tools
+
+
+# ── Format handlers ──────────────────────────────────────────────────
+
+
+def _expected_tc(scenario: Scenario) -> list[dict] | None:
+    if not scenario.tool_calls:
+        return None
+    return [{"name": tc.name, "arguments": tc.arguments} for tc in scenario.tool_calls]
+
+
+def _expected_tools(scenario: Scenario) -> list[dict] | None:
+    return _tool_defs(scenario.tool_calls) if scenario.tool_calls else None
+
+
+def _validate_sample(sample: Sample, parser_cls: type, **kwargs) -> None:
+    """Replay sample through the real parser and assert correctness."""
+    tokenizer = MockTokenizer(vocab=dict(sample.vocab), tokens=sample.tokens)
+    parser = parser_cls(tokenizer, sample.tools, **kwargs)
+    deltas = replay_streaming(
+        parser,
+        sample.tokens,
+        chunk_size=1,
+        tools=sample.tools,
+        prompt_token_ids=sample.prompt_token_ids,
+    )
+    output = collect_output(deltas)
+    assert_parse_output(output, sample)
+
+
+def _validate_tools(
+    tools: list[dict] | None,
+) -> list[ChatCompletionToolsParam] | None:
+    if not tools:
+        return None
+    return [ChatCompletionToolsParam.model_validate(t) for t in tools]
+
+
+def _make_sample(
+    sample_id: str,
+    description: str,
+    vocab: dict[str, int],
+    segments: list[tuple[str, bool]],
+    expected_reasoning: str | None,
+    expected_content: str | None,
+    expected_tool_calls: list[dict] | None,
+    tools: list[dict] | None,
+    chat_template_kwargs: dict | None = None,
+    prompt_token_ids: list[int] | None = None,
+) -> Sample:
+    tokens = _tokenize(segments, vocab)
+    return Sample(
+        id=sample_id,
+        description=description,
+        source="trace-builder",
+        vocab=dict(vocab),
+        tokens=tokens,
+        expected_reasoning=expected_reasoning,
+        expected_content=expected_content,
+        expected_tool_calls=expected_tool_calls,
+        tools=_validate_tools(tools),
+        chat_template_kwargs=chat_template_kwargs,
+        prompt_token_ids=prompt_token_ids,
+    )
+
+
+# ── Qwen3 (XML tool format, starts in REASONING) ────────────────────
+
+_QWEN3_VOCAB: dict[str, int] = {
+    "<think>": 50,
+    "</think>": 51,
+    "<tool_call>": 60,
+    "</tool_call>": 61,
+}
+
+
+def _qwen3_arg_value(value: Any) -> str:
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, (int, float)):
+        return str(value)
+    if isinstance(value, str):
+        return value
+    return json.dumps(value, ensure_ascii=False)
+
+
+def _qwen3_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]:
+    parts = [f"\n<function={tc.name}>"]
+    for key, value in tc.arguments.items():
+        parts.append(f"\n<parameter={key}>{_qwen3_arg_value(value)}</parameter>")
+    parts.append("\n</function>\n")
+    return [
+        ("<tool_call>", True),
+        ("".join(parts), False),
+        ("</tool_call>", True),
+    ]
+
+
+def _qwen3_segments(scenario: Scenario) -> list[tuple[str, bool]]:
+    segs: list[tuple[str, bool]] = []
+    if scenario.reasoning is not None:
+        segs.append((scenario.reasoning, False))
+    if scenario.content is not None or scenario.tool_calls is not None:
+        segs.append(("</think>", True))
+    if scenario.tool_calls is not None and not scenario.tool_calls:
+        segs.append(("<tool_call>", True))
+        segs.append(("</tool_call>", True))
+    if scenario.content is not None:
+        segs.append((scenario.content, False))
+    if scenario.tool_calls:
+        for tc in scenario.tool_calls:
+            segs.extend(_qwen3_tool_segments(tc))
+    return segs
+
+
+def _qwen3_expected_content(scenario: Scenario) -> str | None:
+    if (
+        scenario.content is not None
+        and scenario.tool_calls
+        and not scenario.content.strip()
+    ):
+        return ""
+    return scenario.content
+
+
+def _build_qwen3(
+    scenario: Scenario,
+    name: str = "qwen3",
+    parser_cls: type = Qwen3Parser,
+    strip_trailing_ws: bool = False,
+    validate: bool = True,
+) -> Sample:
+    expected_reasoning: str | None
+    if scenario.reasoning is not None:
+        r = scenario.reasoning
+        if strip_trailing_ws:
+            r = r.rstrip()
+        expected_reasoning = r
+    else:
+        expected_reasoning = ""
+
+    sample = _make_sample(
+        sample_id=f"{name}-{scenario.id}",
+        description=scenario.description,
+        vocab=_QWEN3_VOCAB,
+        segments=_qwen3_segments(scenario),
+        expected_reasoning=expected_reasoning,
+        expected_content=_qwen3_expected_content(scenario),
+        expected_tool_calls=_expected_tc(scenario),
+        tools=_expected_tools(scenario),
+    )
+    if validate:
+        _validate_sample(sample, parser_cls)
+    return sample
+
+
+# ── MiniMax M2 (XML invoke format, starts in REASONING) ──────────────
+
+_MINIMAX_M2_VOCAB: dict[str, int] = {
+    "<think>": 50,
+    "</think>": 51,
+    "<minimax:tool_call>": 60,
+    "</minimax:tool_call>": 61,
+}
+
+
+def _minimax_m2_arg_value(value: Any) -> str:
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, (int, float)):
+        return str(value)
+    if isinstance(value, str):
+        return value
+    return json.dumps(value, ensure_ascii=False)
+
+
+def _minimax_m2_tool_segments(tool_calls: list[ToolCallSpec]) -> list[tuple[str, bool]]:
+    segs: list[tuple[str, bool]] = [("<minimax:tool_call>", True)]
+    for tc in tool_calls:
+        segs.append((f'<invoke name="{tc.name}">', False))
+        for key, value in tc.arguments.items():
+            segs.append(
+                (
+                    f'<parameter name="{key}">'
+                    f"{_minimax_m2_arg_value(value)}"
+                    "</parameter>",
+                    False,
+                )
+            )
+        segs.append(("</invoke>", False))
+    segs.append(("</minimax:tool_call>", True))
+    return segs
+
+
+def _minimax_m2_segments(scenario: Scenario) -> list[tuple[str, bool]]:
+    segs: list[tuple[str, bool]] = []
+    if scenario.reasoning is not None:
+        segs.append((scenario.reasoning, False))
+    if scenario.content is not None or scenario.tool_calls is not None:
+        segs.append(("</think>", True))
+    if scenario.tool_calls is not None and not scenario.tool_calls:
+        segs.append(("<minimax:tool_call>", True))
+        segs.append(("</minimax:tool_call>", True))
+    if scenario.content is not None:
+        segs.append((scenario.content, False))
+    if scenario.tool_calls:
+        segs.extend(_minimax_m2_tool_segments(scenario.tool_calls))
+    return segs
+
+
+def _build_minimax_m2(scenario: Scenario, validate: bool = True) -> Sample:
+    expected_reasoning: str | None
+    if scenario.reasoning is not None:
+        expected_reasoning = scenario.reasoning.rstrip()
+    else:
+        expected_reasoning = ""
+
+    sample = _make_sample(
+        sample_id=f"minimax_m2-{scenario.id}",
+        description=scenario.description,
+        vocab=_MINIMAX_M2_VOCAB,
+        segments=_minimax_m2_segments(scenario),
+        expected_reasoning=expected_reasoning,
+        expected_content=_qwen3_expected_content(scenario),
+        expected_tool_calls=_expected_tc(scenario),
+        tools=_expected_tools(scenario),
+    )
+    if validate:
+        _validate_sample(sample, MinimaxM2Parser)
+    return sample
+
+
+# ── Gemma4 (channel reasoning, custom arg format) ────────────────────
+
+_GEMMA4_VOCAB: dict[str, int] = {
+    "<|channel>": 50,
+    "<channel|>": 51,
+    "<|tool_call>": 48,
+    "<tool_call|>": 49,
+    '<|"|>': 52,
+    "<|turn>": 53,
+    "<|tool_response>": 54,
+}
+_GEMMA4_THOUGHT_PREFIX = "thought\n"
+_GEMMA4_QUOTE = '<|"|>'
+
+
+def _gemma4_value_segments(value: Any) -> list[tuple[str, bool]]:
+    """Render a value in Gemma4 arg format as segments."""
+    if isinstance(value, str):
+        return [(_GEMMA4_QUOTE, True), (value, False), (_GEMMA4_QUOTE, True)]
+    if isinstance(value, bool):
+        return [("true" if value else "false", False)]
+    if isinstance(value, (int, float)):
+        return [(str(value), False)]
+    if isinstance(value, dict):
+        segs: list[tuple[str, bool]] = [("{", False)]
+        for i, (k, v) in enumerate(value.items()):
+            if i > 0:
+                segs.append((",", False))
+            segs.append((f"{k}:", False))
+            segs.extend(_gemma4_value_segments(v))
+        segs.append(("}", False))
+        return segs
+    if isinstance(value, list):
+        segs = [("[", False)]
+        for i, item in enumerate(value):
+            if i > 0:
+                segs.append((",", False))
+            segs.extend(_gemma4_value_segments(item))
+        segs.append(("]", False))
+        return segs
+    return [(json.dumps(value, ensure_ascii=False), False)]
+
+
+def _gemma4_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]:
+    segs: list[tuple[str, bool]] = [
+        ("<|tool_call>", True),
+        (f"call:{tc.name}", False),
+        ("{", False),
+    ]
+    for i, (key, value) in enumerate(tc.arguments.items()):
+        if i > 0:
+            segs.append((",", False))
+        segs.append((f"{key}:", False))
+        segs.extend(_gemma4_value_segments(value))
+    segs.append(("}", False))
+    segs.append(("<tool_call|>", True))
+    return segs
+
+
+def _gemma4_segments(scenario: Scenario) -> list[tuple[str, bool]]:
+    segs: list[tuple[str, bool]] = []
+    if scenario.reasoning is not None:
+        segs.append(("<|channel>", True))
+        segs.append((_GEMMA4_THOUGHT_PREFIX, False))
+        segs.append((scenario.reasoning, False))
+        segs.append(("<channel|>", True))
+    if scenario.tool_calls is not None and not scenario.tool_calls:
+        segs.append(("<|tool_call>", True))
+        segs.append(("<tool_call|>", True))
+    if scenario.content is not None:
+        segs.append((scenario.content, False))
+    if scenario.tool_calls:
+        for tc in scenario.tool_calls:
+            segs.extend(_gemma4_tool_segments(tc))
+    return segs
+
+
+def _build_gemma4(scenario: Scenario, validate: bool = True) -> Sample:
+    prompt_token_ids = None
+    if scenario.after_tool_response:
+        prompt_token_ids = [_GEMMA4_VOCAB["<|tool_response>"]]
+    sample = _make_sample(
+        sample_id=f"gemma4-{scenario.id}",
+        description=scenario.description,
+        vocab=_GEMMA4_VOCAB,
+        segments=_gemma4_segments(scenario),
+        expected_reasoning=scenario.reasoning,
+        expected_content=_qwen3_expected_content(scenario),
+        expected_tool_calls=_expected_tc(scenario),
+        tools=_expected_tools(scenario),
+        prompt_token_ids=prompt_token_ids,
+    )
+    if validate:
+        _validate_sample(sample, Gemma4Parser)
+    return sample
+
+
+def _build_nemotron_v3(scenario: Scenario, validate: bool = True) -> Sample:
+    return _build_qwen3(
+        scenario,
+        name="nemotron_v3",
+        parser_cls=NemotronV3Parser,
+        strip_trailing_ws=True,
+        validate=validate,
+    )
+
+
+# ── Seed-OSS (Qwen3 XML grammar with Seed wrapper tokens) ────────────
+
+_SEED_OSS_VOCAB: dict[str, int] = {
+    "<seed:think>": 50,
+    "</seed:think>": 51,
+    "<seed:tool_call>": 60,
+    "</seed:tool_call>": 61,
+}
+
+
+def _seed_oss_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]:
+    parts = [f"\n<function={tc.name}>"]
+    for key, value in tc.arguments.items():
+        parts.append(f"\n<parameter={key}>{_qwen3_arg_value(value)}</parameter>")
+    parts.append("\n</function>\n")
+    return [
+        ("<seed:tool_call>", True),
+        ("".join(parts), False),
+        ("</seed:tool_call>", True),
+    ]
+
+
+def _seed_oss_segments(scenario: Scenario) -> list[tuple[str, bool]]:
+    segs: list[tuple[str, bool]] = []
+    if scenario.reasoning is not None:
+        segs.append((scenario.reasoning, False))
+    if scenario.content is not None or scenario.tool_calls is not None:
+        segs.append(("</seed:think>", True))
+    if scenario.tool_calls is not None and not scenario.tool_calls:
+        segs.append(("<seed:tool_call>", True))
+        segs.append(("</seed:tool_call>", True))
+    if scenario.content is not None:
+        segs.append((scenario.content, False))
+    if scenario.tool_calls:
+        for tc in scenario.tool_calls:
+            segs.extend(_seed_oss_tool_segments(tc))
+    return segs
+
+
+def _build_seed_oss(scenario: Scenario, validate: bool = True) -> Sample:
+    sample = _make_sample(
+        sample_id=f"seed_oss-{scenario.id}",
+        description=scenario.description,
+        vocab=_SEED_OSS_VOCAB,
+        segments=_seed_oss_segments(scenario),
+        expected_reasoning=scenario.reasoning if scenario.reasoning is not None else "",
+        expected_content=_qwen3_expected_content(scenario),
+        expected_tool_calls=_expected_tc(scenario),
+        tools=_expected_tools(scenario),
+    )
+    if validate:
+        _validate_sample(sample, SeedOssParser)
+    return sample
+
+
+# ── DeepSeek V4 (DSML tool format) ──────────────────────────────────
+
+_DSML = "｜DSML｜"
+_DSV4_VOCAB: dict[str, int] = {
+    "<think>": 128821,
+    "</think>": 128822,
+    f"<{_DSML}tool_calls>": 128823,
+    f"</{_DSML}tool_calls>": 128824,
+}
+
+
+def _dsv4_param_text(key: str, value: Any) -> str:
+    is_string = isinstance(value, str)
+    if is_string:
+        val_str = value
+    elif isinstance(value, bool):
+        val_str = "true" if value else "false"
+    elif isinstance(value, (int, float)):
+        val_str = str(value)
+    else:
+        val_str = json.dumps(value, ensure_ascii=False)
+    string_attr = "true" if is_string else "false"
+    return (
+        f'<{_DSML}parameter name="{key}" string="{string_attr}">'
+        f"{val_str}</{_DSML}parameter>\n"
+    )
+
+
+def _dsv4_tool_text(tc: ToolCallSpec) -> str:
+    parts = [f'<{_DSML}invoke name="{tc.name}">\n']
+    for key, value in tc.arguments.items():
+        parts.append(_dsv4_param_text(key, value))
+    parts.append(f"</{_DSML}invoke>\n")
+    return "".join(parts)
+
+
+def _dsml_tool_segs(
+    scenario: Scenario,
+    tag: str,
+) -> list[tuple[str, bool]]:
+    if not scenario.tool_calls:
+        return []
+    parts = ["\n"]
+    for tc in scenario.tool_calls:
+        parts.append(_dsv4_tool_text(tc))
+    return [
+        (f"<{_DSML}{tag}>", True),
+        ("".join(parts), False),
+        (f"</{_DSML}{tag}>", True),
+    ]
+
+
+def _dsv4_segments(scenario: Scenario, thinking: bool) -> list[tuple[str, bool]]:
+    segs: list[tuple[str, bool]] = []
+
+    if thinking:
+        if scenario.reasoning is not None:
+            segs.append((scenario.reasoning, False))
+        if scenario.content is not None or scenario.tool_calls:
+            segs.append(("</think>", True))
+    else:
+        if scenario.reasoning is not None:
+            segs.append(("<think>", True))
+            segs.append((scenario.reasoning, False))
+            segs.append(("</think>", True))
+
+    if scenario.content is not None:
+        segs.append((scenario.content, False))
+
+    segs.extend(_dsml_tool_segs(scenario, "tool_calls"))
+    return segs
+
+
+def _build_deepseek_v4(scenario: Scenario, validate: bool = True) -> Sample:
+    thinking = scenario.reasoning is not None
+    chat_kwargs = {"thinking": True} if thinking else None
+
+    if thinking:
+        expected_reasoning: str | None = scenario.reasoning or ""
+    else:
+        expected_reasoning = None
+
+    sample = _make_sample(
+        sample_id=f"deepseek_v4-{scenario.id}",
+        description=scenario.description,
+        vocab=_DSV4_VOCAB,
+        segments=_dsv4_segments(scenario, thinking),
+        expected_reasoning=expected_reasoning,
+        expected_content=_qwen3_expected_content(scenario),
+        expected_tool_calls=_expected_tc(scenario),
+        tools=_expected_tools(scenario),
+        chat_template_kwargs=chat_kwargs,
+    )
+    if validate:
+        kwargs = {}
+        if chat_kwargs:
+            kwargs["chat_template_kwargs"] = chat_kwargs
+        _validate_sample(sample, DeepSeekV4Parser, **kwargs)
+    return sample
+
+
+# ── DeepSeek V3.2 (DSML tool format, no reasoning) ──────────────────
+
+_DSV32_VOCAB: dict[str, int] = {
+    f"<{_DSML}function_calls>": 128830,
+    f"</{_DSML}function_calls>": 128831,
+}
+
+
+def _dsv32_segments(scenario: Scenario) -> list[tuple[str, bool]]:
+    segs: list[tuple[str, bool]] = []
+
+    if scenario.content is not None:
+        segs.append((scenario.content, False))
+
+    segs.extend(_dsml_tool_segs(scenario, "function_calls"))
+    return segs
+
+
+def _build_deepseek_v32(scenario: Scenario, validate: bool = True) -> Sample | None:
+    if scenario.reasoning is not None:
+        return None
+
+    sample = _make_sample(
+        sample_id=f"deepseek_v32-{scenario.id}",
+        description=scenario.description,
+        vocab=_DSV32_VOCAB,
+        segments=_dsv32_segments(scenario),
+        expected_reasoning=None,
+        expected_content=_qwen3_expected_content(scenario),
+        expected_tool_calls=_expected_tc(scenario),
+        tools=_expected_tools(scenario),
+    )
+    if validate:
+        _validate_sample(sample, DeepSeekV32Parser)
+    return sample
+
+
+# ── GLM-4.7 MoE (XML tool format, starts in REASONING) ──────────────
+
+_GLM47_MOE_VOCAB: dict[str, int] = {
+    "<think>": 50,
+    "</think>": 51,
+    "<tool_call>": 60,
+    "</tool_call>": 61,
+    "<arg_key>": 62,
+    "</arg_key>": 63,
+    "<arg_value>": 64,
+    "</arg_value>": 65,
+}
+
+
+def _glm47_moe_arg_value(value: Any) -> str:
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, (int, float)):
+        return str(value)
+    if isinstance(value, str):
+        return value
+    return json.dumps(value, ensure_ascii=False)
+
+
+def _glm47_moe_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]:
+    segs: list[tuple[str, bool]] = [
+        ("<tool_call>", True),
+        (tc.name, False),
+    ]
+    for key, value in tc.arguments.items():
+        segs.extend(
+            [
+                ("<arg_key>", True),
+                (key, False),
+                ("</arg_key>", True),
+                ("<arg_value>", True),
+                (_glm47_moe_arg_value(value), False),
+                ("</arg_value>", True),
+            ]
+        )
+    segs.append(("</tool_call>", True))
+    return segs
+
+
+def _glm47_moe_segments(scenario: Scenario) -> list[tuple[str, bool]]:
+    segs: list[tuple[str, bool]] = []
+    if scenario.reasoning is not None:
+        segs.append((scenario.reasoning, False))
+    if scenario.content is not None or scenario.tool_calls:
+        segs.append(("</think>", True))
+    if scenario.content is not None:
+        segs.append((scenario.content, False))
+    if scenario.tool_calls:
+        for tc in scenario.tool_calls:
+            segs.extend(_glm47_moe_tool_segments(tc))
+    return segs
+
+
+def _build_glm47_moe(scenario: Scenario, validate: bool = True) -> Sample:
+    sample = _make_sample(
+        sample_id=f"glm47_moe-{scenario.id}",
+        description=scenario.description,
+        vocab=_GLM47_MOE_VOCAB,
+        segments=_glm47_moe_segments(scenario),
+        expected_reasoning=scenario.reasoning if scenario.reasoning is not None else "",
+        expected_content=_qwen3_expected_content(scenario),
+        expected_tool_calls=_expected_tc(scenario),
+        tools=_expected_tools(scenario),
+    )
+    if validate:
+        _validate_sample(sample, Glm47MoeParser)
+    return sample
+
+
+# ── Kimi K2 (native tool-call section, starts in REASONING) ──────────
+
+_KIMI_K2_VOCAB: dict[str, int] = {
+    "<think>": 50,
+    "</think>": 51,
+    "<|tool_calls_section_begin|>": 60,
+    "<|tool_calls_section_end|>": 61,
+    "<|tool_call_begin|>": 62,
+    "<|tool_call_end|>": 63,
+    "<|tool_call_argument_begin|>": 64,
+}
+
+
+def _kimi_k2_tool_segments(
+    tool_calls: list[ToolCallSpec],
+) -> list[tuple[str, bool]]:
+    segs: list[tuple[str, bool]] = [("<|tool_calls_section_begin|>", True)]
+    for index, tc in enumerate(tool_calls):
+        args = json.dumps(tc.arguments, ensure_ascii=False, separators=(",", ":"))
+        segs.extend(
+            [
+                ("<|tool_call_begin|>", True),
+                (f"functions.{tc.name}:{index}\n", False),
+                ("<|tool_call_argument_begin|>", True),
+                (args, False),
+                ("<|tool_call_end|>", True),
+            ]
+        )
+    segs.append(("<|tool_calls_section_end|>", True))
+    return segs
+
+
+def _kimi_k2_segments(scenario: Scenario) -> list[tuple[str, bool]]:
+    segs: list[tuple[str, bool]] = []
+    if scenario.reasoning is not None:
+        segs.append(("<think>", True))
+        segs.append((scenario.reasoning, False))
+    if scenario.content is not None or scenario.tool_calls is not None:
+        segs.append(("</think>", True))
+    if scenario.content is not None:
+        segs.append((scenario.content, False))
+    if scenario.tool_calls is not None:
+        segs.extend(_kimi_k2_tool_segments(scenario.tool_calls))
+    return segs
+
+
+def _build_kimi_k2(
+    scenario: Scenario,
+    validate: bool = True,
+    thinking: bool = True,
+) -> Sample:
+    expected_reasoning = (
+        scenario.reasoning.rstrip()
+        if (thinking and scenario.reasoning is not None)
+        else None
+    )
+    if thinking and scenario.reasoning is None:
+        expected_reasoning = ""
+
+    sample = _make_sample(
+        sample_id=f"kimi_k2-{scenario.id}",
+        description=scenario.description,
+        vocab=_KIMI_K2_VOCAB,
+        segments=_kimi_k2_segments(scenario),
+        expected_reasoning=expected_reasoning,
+        expected_content=_qwen3_expected_content(scenario),
+        expected_tool_calls=_expected_tc(scenario),
+        tools=_expected_tools(scenario),
+        chat_template_kwargs=None if thinking else {"thinking": False},
+    )
+    if validate:
+        _validate_sample(
+            sample,
+            KimiK2Parser,
+            chat_template_kwargs=sample.chat_template_kwargs,
+        )
+    return sample
+
+
+_KIMI_K2_SCENARIOS = [
+    *SCENARIOS,
+    Scenario(
+        id="trailing-reasoning-whitespace",
+        description="Reasoning trailing whitespace is stripped",
+        reasoning="Reasoning with trailing whitespace. \n\t",
+        content="Done.",
+    ),
+]
+
+
+# ── Registry and public API ──────────────────────────────────────────
+
+_BUILDERS: dict[str, Any] = {
+    "deepseek_v32": _build_deepseek_v32,
+    "deepseek_v4": _build_deepseek_v4,
+    "gemma4": _build_gemma4,
+    "minimax_m2": _build_minimax_m2,
+    "nemotron_v3": _build_nemotron_v3,
+    "seed_oss": _build_seed_oss,
+    "glm47_moe": _build_glm47_moe,
+    "kimi_k2": _build_kimi_k2,
+    "qwen3": _build_qwen3,
+}
+
+
+@functools.cache
+def build_samples(model: str) -> tuple[Sample, ...]:
+    """Build all scenario samples for a model, self-validated."""
+    builder = _BUILDERS[model]
+    scenarios = _KIMI_K2_SCENARIOS if model == "kimi_k2" else SCENARIOS
+    return tuple(s for s in (builder(sc) for sc in scenarios) if s is not None)
+
+
+def build_sample(model: str, scenario: Scenario) -> Sample | None:
+    """Build a single sample for one model + scenario."""
+    return _BUILDERS[model](scenario)
+
+
+def build_scaling_sample(
+    model: str, token_count: int, validate: bool = False
+) -> Sample:
+    """Build a sample with approximately *token_count* tokens."""
+    sentence = "The quick brown fox jumps over the lazy dog. "
+    text = sentence * (token_count // 10 + 1)
+    scenario = Scenario(
+        id=f"scaling-{token_count}",
+        description=f"Scaling test with ~{token_count} tokens",
+        reasoning=text,
+        tool_calls=[_READ_TOOL],
+    )
+    return _BUILDERS[model](scenario, validate=validate)