wehub-resource-sync
192 lines
7.5 KiB
Python
192 lines
7.5 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
import json
|
|
from collections.abc import Iterable, Sequence
|
|
from typing import TYPE_CHECKING
|
|
|
|
from transformers import PreTrainedTokenizerBase
|
|
|
|
from vllm.entrypoints.mcp.tool_server import ToolServer
|
|
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
|
from vllm.logger import init_logger
|
|
from vllm.reasoning import ReasoningParser
|
|
|
|
if TYPE_CHECKING:
|
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
|
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
no_func_reasoning_tag = {
|
|
"type": "structural_tag",
|
|
"format": {
|
|
"type": "triggered_tags",
|
|
"tags": [
|
|
{
|
|
"begin": "<|channel|>analysis<|message|>",
|
|
"content": {"type": "any_text"},
|
|
"end": "<|end|>",
|
|
}
|
|
],
|
|
"triggers": ["<|channel|>analysis"],
|
|
"stop_after_first": False,
|
|
},
|
|
}
|
|
|
|
|
|
def from_builtin_tool_to_tag(tool: str) -> list[dict]:
|
|
tag = [
|
|
{
|
|
"begin": f"<|channel|>commentary to={tool}",
|
|
"content": {"type": "any_text"},
|
|
"end": "<|end|>",
|
|
},
|
|
{
|
|
"begin": f"<|channel|>analysis to={tool}",
|
|
"content": {"type": "any_text"},
|
|
"end": "<|end|>",
|
|
},
|
|
]
|
|
return tag
|
|
|
|
|
|
def tag_with_builtin_funcs(no_func_reasoning_tag, builtin_tool_list: list[str]) -> dict:
|
|
import copy
|
|
|
|
new_tag = copy.deepcopy(no_func_reasoning_tag)
|
|
new_tag["format"]["triggers"].append("<|channel|>commentary to=")
|
|
|
|
for tool in builtin_tool_list:
|
|
new_tag["format"]["tags"].extend(from_builtin_tool_to_tag(tool))
|
|
return new_tag
|
|
|
|
|
|
class GptOssReasoningParser(ReasoningParser):
|
|
"""
|
|
Reasoning parser for GptOss model.
|
|
|
|
The GptOss model uses harmony to extract reasoning content and this parser
|
|
is only used for detecting the end of the reasoning content.
|
|
"""
|
|
|
|
def __init__(self, tokenizer: PreTrainedTokenizerBase, *args, **kwargs):
|
|
super().__init__(tokenizer, *args, **kwargs)
|
|
# The model can output some special tokens between "final" and "<|message|>"
|
|
# So we need to look for both sequences to determine the end of reasoning.
|
|
self.reasoning_end_token_ids_prefix = self.model_tokenizer.encode(
|
|
"<|channel|>final"
|
|
)
|
|
self.reasoning_end_token_ids_suffix = self.model_tokenizer.encode("<|message|>")
|
|
# We also need to check for the <|end|> token to avoid false positives from
|
|
# previous messages in multi-turn conversations.
|
|
self.eom_token_id = self.vocab["<|end|>"]
|
|
self.reasoning_max_num_between_tokens = 20
|
|
|
|
def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
|
|
end_token_ids_prefix = self.reasoning_end_token_ids_prefix
|
|
end_token_ids_suffix = self.reasoning_end_token_ids_suffix
|
|
assert len(end_token_ids_prefix) > 0, "reasoning_end_token_ids_prefix is empty"
|
|
assert len(end_token_ids_suffix) > 0, "reasoning_end_token_ids_suffix is empty"
|
|
# Check if the end sequence is present in the input_ids.
|
|
# We search from the end of input_ids to find the last match.
|
|
for i in range(len(input_ids) - len(end_token_ids_prefix), -1, -1):
|
|
if input_ids[i] == self.eom_token_id:
|
|
# We looped backwards far enough to find the end of a previous message,
|
|
# which means we have searched the entirety of the current message
|
|
# and can exit early without searching further back into prior
|
|
# messages of the conversation.
|
|
return False
|
|
if input_ids[i : i + len(end_token_ids_prefix)] == end_token_ids_prefix:
|
|
# We have found the prefix, now we look for the suffix after the prefix.
|
|
suffix_start = i + len(end_token_ids_prefix)
|
|
for j in range(
|
|
suffix_start, len(input_ids) - len(end_token_ids_suffix) + 1
|
|
):
|
|
if j - suffix_start >= self.reasoning_max_num_between_tokens:
|
|
break
|
|
if (
|
|
input_ids[j : j + len(end_token_ids_suffix)]
|
|
== end_token_ids_suffix
|
|
):
|
|
return True
|
|
return False
|
|
|
|
def is_reasoning_end_streaming(
|
|
self, input_ids: Sequence[int], delta_ids: Iterable[int]
|
|
) -> bool:
|
|
# The pattern window covers the end-of-reasoning marker itself.
|
|
# We add len(delta_ids) so that under speculative decoding (where
|
|
# a single step can accept many tokens) the entire accepted chunk
|
|
# is always inside the scan region.
|
|
delta_ids = tuple(delta_ids)
|
|
pattern_len = (
|
|
len(self.reasoning_end_token_ids_prefix)
|
|
+ self.reasoning_max_num_between_tokens
|
|
+ len(self.reasoning_end_token_ids_suffix)
|
|
)
|
|
window = pattern_len + len(delta_ids)
|
|
n = len(input_ids)
|
|
if n <= window:
|
|
return self.is_reasoning_end(input_ids)
|
|
return self.is_reasoning_end(input_ids[n - window :])
|
|
|
|
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
|
raise NotImplementedError(
|
|
"GptOssReasoningParser only provides boundary detection. "
|
|
"Use HarmonyParser for output parsing."
|
|
)
|
|
|
|
def extract_reasoning_streaming(
|
|
self,
|
|
previous_text: str,
|
|
current_text: str,
|
|
delta_text: str,
|
|
previous_token_ids: Sequence[int],
|
|
current_token_ids: Sequence[int],
|
|
delta_token_ids: Sequence[int],
|
|
) -> DeltaMessage | None:
|
|
raise NotImplementedError(
|
|
"GptOssReasoningParser only provides boundary detection. "
|
|
"Use HarmonyParser for output parsing."
|
|
)
|
|
|
|
def extract_reasoning(
|
|
self,
|
|
model_output: str,
|
|
request: "ChatCompletionRequest | ResponsesRequest",
|
|
) -> tuple[str | None, str | None]:
|
|
raise NotImplementedError(
|
|
"GptOssReasoningParser only provides boundary detection. "
|
|
"Use HarmonyParser for output parsing."
|
|
)
|
|
|
|
# This function prepares the structural tag to format reasoning output
|
|
def prepare_structured_tag(
|
|
self, original_tag: str | None, tool_server: ToolServer | None
|
|
) -> str | None:
|
|
if original_tag is None:
|
|
if tool_server is None:
|
|
return json.dumps(no_func_reasoning_tag)
|
|
else:
|
|
builtin_tool_list: list[str] = []
|
|
if tool_server.has_tool("browser"):
|
|
builtin_tool_list.append("browser")
|
|
if tool_server.has_tool("python"):
|
|
builtin_tool_list.append("python")
|
|
if tool_server.has_tool("container"):
|
|
builtin_tool_list.append("container")
|
|
|
|
if len(builtin_tool_list) > 0:
|
|
logger.info("Builtin_tool_list: %s", builtin_tool_list)
|
|
func_tag = json.dumps(
|
|
tag_with_builtin_funcs(no_func_reasoning_tag, builtin_tool_list)
|
|
)
|
|
else:
|
|
logger.info("Builtin_tool_list is empty")
|
|
func_tag = json.dumps(no_func_reasoning_tag)
|
|
|
|
return func_tag
|
|
else:
|
|
# There is potential risk for appending the tag to the original tag
|
|
return original_tag
|