chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,9 @@
|
||||
"""Definitions of pydantic models for API entry points and configurations
|
||||
|
||||
Note
|
||||
----
|
||||
We use the following convention
|
||||
|
||||
- filename_protocol If the classes can appear in an API endpoint
|
||||
- filename_config For other config classes
|
||||
"""
|
||||
@@ -0,0 +1,278 @@
|
||||
"""The standard conversation protocol in MLC LLM"""
|
||||
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, List, Optional, Tuple, Type, TypeVar, Union # noqa: UP035
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
||||
# The message placeholders in the message prompts according to roles.
|
||||
class MessagePlaceholders(Enum):
|
||||
"""The message placeholders in the message prompts according to roles."""
|
||||
|
||||
SYSTEM = "{system_message}"
|
||||
USER = "{user_message}"
|
||||
ASSISTANT = "{assistant_message}"
|
||||
TOOL = "{tool_message}"
|
||||
FUNCTION = "{function_string}"
|
||||
|
||||
|
||||
T = TypeVar("T", bound="BaseModel")
|
||||
|
||||
|
||||
class Conversation(BaseModel):
|
||||
"""Class that specifies the convention template of conversation
|
||||
and contains the conversation history.
|
||||
|
||||
Given a conversation template, the corresponding prompt generated out
|
||||
from it is usually in the following format:
|
||||
|
||||
<<system>><<messages[0][0]>><<role_content_sep>><<messages[0][1]>><<seps[0]>>
|
||||
<<messages[1][0]>><<role_content_sep>><<messages[1][1]>><<seps[1]>>
|
||||
...
|
||||
<<messages[2][0]>><<role_content_sep>><<messages[2][1]>><<seps[0]>>
|
||||
<<roles[1]>><<role_empty_sep>>
|
||||
"""
|
||||
|
||||
# Optional name of the template.
|
||||
name: Optional[str] = None
|
||||
# The system prompt template, it optionally contains the system
|
||||
# message placeholder, and the placeholder will be replaced with
|
||||
# the system message below.
|
||||
system_template: str = MessagePlaceholders.SYSTEM.value
|
||||
# The content of the system prompt (without the template format).
|
||||
system_message: str = ""
|
||||
# The system token ids to be prepended at the beginning of tokenized
|
||||
# generated prompt.
|
||||
system_prefix_token_ids: Optional[List[int]] = None # noqa: UP006
|
||||
# Whether or not to append user role and separator after the system message.
|
||||
# This is mainly for [INST] [/INST] style prompt format
|
||||
add_role_after_system_message: bool = True
|
||||
|
||||
# The conversation roles
|
||||
roles: Dict[str, str] # noqa: UP006
|
||||
|
||||
# The roles prompt template, it optionally contains the defaults
|
||||
# message placeholders and will be replaced by actual content
|
||||
role_templates: Dict[str, str] # noqa: UP006
|
||||
|
||||
# The conversation history messages.
|
||||
# Each message is a pair of strings, denoting "(role, content)".
|
||||
# The content can be None.
|
||||
messages: List[Tuple[str, Optional[Union[str, List[Dict]]]]] = Field(default_factory=lambda: []) # noqa: UP006
|
||||
|
||||
# The separators between messages when concatenating into a single prompt.
|
||||
# List size should be either 1 or 2.
|
||||
# - When size is 1, the separator will be used between adjacent messages.
|
||||
# - When size is 2, seps[0] is used after user message, and
|
||||
# seps[1] is used after assistant message.
|
||||
seps: List[str] # noqa: UP006
|
||||
|
||||
# The separator between the role and the content in a message.
|
||||
role_content_sep: str = ""
|
||||
# The separator between the role and empty contents.
|
||||
role_empty_sep: str = ""
|
||||
|
||||
# The stop criteria
|
||||
stop_str: List[str] = Field(default_factory=lambda: []) # noqa: UP006
|
||||
stop_token_ids: List[int] = Field(default_factory=lambda: []) # noqa: UP006
|
||||
|
||||
# When True, strip `<think>...</think>` blocks (and any trailing whitespace)
|
||||
# from historical assistant messages before rendering the prompt, mirroring
|
||||
# Qwen3's official HF chat template. Only historical turns before the last
|
||||
# user message are affected; reasoning on the most recent assistant turn is
|
||||
# preserved for tool-call prefill scenarios.
|
||||
strip_reasoning_in_history: bool = False
|
||||
|
||||
# Function call fields
|
||||
function_string: str = ""
|
||||
# whether using function calling or not, helps check for output message format in API call
|
||||
use_function_calling: bool = False
|
||||
|
||||
def __init__(self, role_templates: Optional[Dict[str, str]] = None, **kwargs): # noqa: UP006
|
||||
# Defaults templates which would be overridden by model specific templates
|
||||
_role_templates: Dict[str, str] = { # noqa: UP006
|
||||
"user": MessagePlaceholders.USER.value,
|
||||
"assistant": MessagePlaceholders.ASSISTANT.value,
|
||||
"tool": MessagePlaceholders.TOOL.value,
|
||||
}
|
||||
if role_templates is not None:
|
||||
_role_templates.update(role_templates)
|
||||
super().__init__(role_templates=_role_templates, **kwargs)
|
||||
|
||||
@field_validator("seps")
|
||||
@classmethod
|
||||
def check_message_seps(cls, seps: List[str]) -> List[str]: # noqa: UP006
|
||||
"""Check if the input message separators has size 1 or 2."""
|
||||
if len(seps) == 0 or len(seps) > 2:
|
||||
raise ValueError("seps should have size 1 or 2.")
|
||||
return seps
|
||||
|
||||
def to_json_dict(self) -> Dict[str, Any]: # noqa: UP006
|
||||
"""Convert to a json dictionary"""
|
||||
return self.model_dump(by_alias=True, exclude_none=True)
|
||||
|
||||
@classmethod
|
||||
def from_json_dict(cls: Type[T], json_dict: Dict[str, Any]) -> T: # noqa: UP006
|
||||
"""Convert from a json dictionary"""
|
||||
return Conversation.model_validate(json_dict)
|
||||
|
||||
def as_prompt(self, config=None) -> List[Any]: # noqa: UP006
|
||||
"""Convert the conversation template and history messages to
|
||||
a single prompt.
|
||||
|
||||
Returns
|
||||
-------
|
||||
prompts : List[Union[str, "mlc_llm.serve.data.Data"]]
|
||||
The prompts converted from the conversation messages.
|
||||
We use Any in the signature to avoid cyclic import.
|
||||
"""
|
||||
from ..serve import data
|
||||
|
||||
# - Get the system message.
|
||||
system_msg = self.system_template.replace(
|
||||
MessagePlaceholders.SYSTEM.value, self.system_message
|
||||
)
|
||||
|
||||
# - Get the message strings.
|
||||
message_list: List[Union[str, data.Data]] = [] # noqa: UP006
|
||||
separators = list(self.seps)
|
||||
if len(separators) == 1:
|
||||
separators.append(separators[0])
|
||||
|
||||
if system_msg != "":
|
||||
message_list.append(system_msg)
|
||||
|
||||
messages = (
|
||||
_strip_reasoning_in_history(self.messages)
|
||||
if self.strip_reasoning_in_history
|
||||
else self.messages
|
||||
)
|
||||
|
||||
for i, (role, content) in enumerate(messages):
|
||||
if role not in self.roles.keys():
|
||||
raise ValueError(f'Role "{role}" is not a supported role in {self.roles.keys()}')
|
||||
separator = separators[role == "assistant"] # check assistant role
|
||||
|
||||
if content is None:
|
||||
message_list.append(self.roles[role] + self.role_empty_sep)
|
||||
continue
|
||||
|
||||
role_prefix = (
|
||||
""
|
||||
# Do not append role prefix if this is the first message and there
|
||||
# is already a system message
|
||||
if (not self.add_role_after_system_message and system_msg != "" and i == 0)
|
||||
else self.roles[role] + self.role_content_sep
|
||||
)
|
||||
if isinstance(content, str):
|
||||
message_list.append(
|
||||
role_prefix
|
||||
+ self.role_templates[role].replace(
|
||||
MessagePlaceholders[role.upper()].value, content
|
||||
)
|
||||
+ separator
|
||||
)
|
||||
continue
|
||||
|
||||
message_list.append(role_prefix)
|
||||
|
||||
for item in content:
|
||||
assert isinstance(item, dict), "Content should be a string or a list of dicts"
|
||||
assert "type" in item, "Content item should have a type field"
|
||||
if item["type"] == "text":
|
||||
message = self.role_templates[role].replace(
|
||||
MessagePlaceholders[role.upper()].value, item["text"]
|
||||
)
|
||||
message_list.append(message)
|
||||
elif item["type"] == "image_url":
|
||||
assert config is not None, "Model config is required"
|
||||
image_url = _get_url_from_item(item)
|
||||
message_list.append(data.ImageData.from_url(image_url, config))
|
||||
message_list.append("\n")
|
||||
else:
|
||||
raise ValueError(f"Unsupported content type: {item['type']}")
|
||||
|
||||
message_list.append(separator)
|
||||
|
||||
prompt = _combine_consecutive_messages(message_list)
|
||||
|
||||
if not any(isinstance(item, data.ImageData) for item in message_list):
|
||||
# Replace the last function string placeholder with actual function string
|
||||
prompt[0] = self.function_string.join(
|
||||
prompt[0].rsplit(MessagePlaceholders.FUNCTION.value, 1)
|
||||
)
|
||||
# Replace with remaining function string placeholders with empty string
|
||||
prompt[0] = prompt[0].replace(MessagePlaceholders.FUNCTION.value, "")
|
||||
|
||||
return prompt
|
||||
|
||||
|
||||
def _get_url_from_item(item: Dict) -> str: # noqa: UP006
|
||||
image_url: str
|
||||
assert "image_url" in item, "Content item should have an image_url field"
|
||||
if isinstance(item["image_url"], str):
|
||||
image_url = item["image_url"]
|
||||
elif isinstance(item["image_url"], dict):
|
||||
assert "url" in item["image_url"], (
|
||||
"Content image_url item should be a string or a dict with a url field"
|
||||
)
|
||||
image_url = item["image_url"]["url"]
|
||||
else:
|
||||
raise ValueError(
|
||||
"Content image_url item type not supported. "
|
||||
"Should be a string or a dict with a url field."
|
||||
)
|
||||
return image_url
|
||||
|
||||
|
||||
def _strip_reasoning_in_history(
|
||||
messages: List[Tuple[str, Optional[Union[str, List[Dict]]]]], # noqa: UP006
|
||||
) -> List[Tuple[str, Optional[Union[str, List[Dict]]]]]: # noqa: UP006
|
||||
"""Strip `<think>...</think>` blocks from assistant messages that precede
|
||||
the last user message, matching Qwen3's HF chat-template behavior. The last
|
||||
assistant message (if any) is preserved so tool-call prefill continuations
|
||||
keep their reasoning context.
|
||||
"""
|
||||
last_user_idx = -1
|
||||
for i, (role, _) in enumerate(messages):
|
||||
if role == "user":
|
||||
last_user_idx = i
|
||||
|
||||
result: List[Tuple[str, Optional[Union[str, List[Dict]]]]] = [] # noqa: UP006
|
||||
for i, (role, content) in enumerate(messages):
|
||||
if (
|
||||
role == "assistant"
|
||||
and i < last_user_idx
|
||||
and isinstance(content, str)
|
||||
and "</think>" in content
|
||||
):
|
||||
content = content.split("</think>")[-1].lstrip("\n")
|
||||
result.append((role, content))
|
||||
return result
|
||||
|
||||
|
||||
def _combine_consecutive_messages(messages: List[Any]) -> List[Any]: # noqa: UP006
|
||||
"""Combining consecutive strings into one.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
messages : List[Union[str, "mlc_llm.serve.data.Data"]]
|
||||
The input messages to be combined.
|
||||
We use Any in the signature to avoid cyclic import.
|
||||
|
||||
Returns
|
||||
-------
|
||||
updated_messages : List[Union[str, "mlc_llm.serve.data.Data"]]
|
||||
The combined messages
|
||||
"""
|
||||
if len(messages) == 0:
|
||||
return []
|
||||
|
||||
combined_messages = [messages[0]]
|
||||
for message in messages[1:]:
|
||||
if isinstance(message, str) and isinstance(combined_messages[-1], str):
|
||||
combined_messages[-1] += message
|
||||
else:
|
||||
combined_messages.append(message)
|
||||
return combined_messages
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Debug protocols in MLC LLM"""
|
||||
|
||||
from typing import Literal, Optional
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class DisaggConfig(BaseModel):
|
||||
"""The class of metadata used in microserving APIs."""
|
||||
|
||||
kind: Optional[Literal["prepare_receive", "remote_send", "start_generation"]] = None
|
||||
# "kv_append_metadata" is base64-encoded and is thus a string.
|
||||
kv_append_metadata: Optional[str] = None
|
||||
# "kv_window_begin" and "kv_window_end" denote the KV interval of interests.
|
||||
# "kv_window_end" supports Python style negative indexing.
|
||||
# The concrete meaning varies for different special request kind:
|
||||
# - For "prepare_receive", the begin is always 0, and "[0:end]" denotes
|
||||
# the KV range to prefill on a prefill instance.
|
||||
# - For "remote_send", "[begin:end]" means the KV range to compute prefill
|
||||
# and send to the decode instance.
|
||||
# - For "start_generation", the end is always None, and "[begin:]" denotes
|
||||
# the KV range to prefill locally on the decode instance.
|
||||
kv_window_begin: Optional[int] = None
|
||||
kv_window_end: Optional[int] = None
|
||||
# KV data destination group offset
|
||||
dst_group_offset: Optional[int] = None
|
||||
|
||||
|
||||
class DebugConfig(BaseModel):
|
||||
"""The class of debug options.
|
||||
|
||||
These optionals are available to engine
|
||||
but won't be available to serving endpoint
|
||||
unless an explicit --enable-debug passed
|
||||
"""
|
||||
|
||||
ignore_eos: bool = False
|
||||
pinned_system_prompt: bool = False
|
||||
special_request: Optional[Literal["query_engine_metrics"]] = None
|
||||
grammar_execution_mode: Literal["constraint", "jump_forward"] = "jump_forward"
|
||||
disagg_config: Optional[DisaggConfig] = None
|
||||
|
||||
"""Special request indicators
|
||||
|
||||
Special requests are handled by engine differently and do not go
|
||||
through the normal engine step flow.
|
||||
|
||||
The results to these requests are returned as field of "usage"
|
||||
"""
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Error protocols in MLC LLM"""
|
||||
|
||||
from http import HTTPStatus
|
||||
from typing import Optional
|
||||
|
||||
import fastapi
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class BadRequestError(ValueError):
|
||||
"""The exception for bad requests in engines."""
|
||||
|
||||
def __init__(self, *args: object) -> None:
|
||||
super().__init__(*args)
|
||||
|
||||
|
||||
class ErrorResponse(BaseModel):
|
||||
"""The class of error response."""
|
||||
|
||||
object: str = "error"
|
||||
message: str
|
||||
code: Optional[int] = None
|
||||
|
||||
|
||||
def create_error_response(status_code: HTTPStatus, message: str) -> fastapi.responses.JSONResponse:
|
||||
"""Create a JSON response that reports error with regarding the input message."""
|
||||
return fastapi.responses.JSONResponse(
|
||||
ErrorResponse(message=message, code=status_code.value).model_dump_json(by_alias=True),
|
||||
status_code=status_code.value,
|
||||
)
|
||||
|
||||
|
||||
async def bad_request_error_handler(_request: fastapi.Request, e: BadRequestError):
|
||||
"""The handler of BadRequestError that converts an exception into error response."""
|
||||
return create_error_response(status_code=HTTPStatus.BAD_REQUEST, message=e.args[0])
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Low-level generation config class"""
|
||||
|
||||
from typing import Dict, List, Optional # noqa: UP035
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .debug_protocol import DebugConfig
|
||||
from .openai_api_protocol import RequestResponseFormat
|
||||
|
||||
|
||||
class GenerationConfig(BaseModel):
|
||||
"""The generation configuration dataclass.
|
||||
|
||||
This is a config class used by Engine internally.
|
||||
"""
|
||||
|
||||
n: int = 1
|
||||
temperature: Optional[float] = None
|
||||
top_p: Optional[float] = None
|
||||
frequency_penalty: Optional[float] = None
|
||||
presence_penalty: Optional[float] = None
|
||||
repetition_penalty: Optional[float] = None
|
||||
logprobs: bool = False
|
||||
top_logprobs: int = 0
|
||||
logit_bias: Optional[Dict[int, float]] = None # noqa: UP006
|
||||
# internally we use -1 to represent infinite
|
||||
max_tokens: int = -1
|
||||
seed: Optional[int] = None
|
||||
stop_strs: Optional[List[str]] = None # noqa: UP006
|
||||
stop_token_ids: Optional[List[int]] = None # noqa: UP006
|
||||
response_format: Optional[RequestResponseFormat] = None
|
||||
debug_config: Optional[Optional[DebugConfig]] = None
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Protocols in MLC LLM for MicroServing."""
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from mlc_llm.protocol.openai_api_protocol import CompletionRequest
|
||||
|
||||
|
||||
class PrepRecvRequest(CompletionRequest):
|
||||
"""The extra request body for prep_recv request in MicroServing.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
kv_window_end : int
|
||||
[0, kv_window_end] denotes the KV range of the prompt to prefill on
|
||||
a prefill instance.
|
||||
The entries of this KV range will be allocated on the decode instance.
|
||||
"""
|
||||
|
||||
end: int
|
||||
|
||||
|
||||
class PrepRecvResponse(BaseModel):
|
||||
"""The response body for prep_recv request in MicroServing.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
prefix_matched_length : int
|
||||
The matched common prefix length on the decode instance when
|
||||
prefix cache is enabled, or 0 if there is no prefix cache.
|
||||
|
||||
kv_append_metadata : str
|
||||
The metadata of the KV range on the destination decode instance.
|
||||
"""
|
||||
|
||||
kv_append_metadata: str
|
||||
prefix_matched_length: int
|
||||
|
||||
|
||||
class RemoteSendRequest(CompletionRequest):
|
||||
"""The extra request body for remote_send request in MicroServing.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
kv_window_begin : int
|
||||
Denote the start of the KV range to prefill.
|
||||
|
||||
kv_window_end : int
|
||||
Denote the end of the KV range to prefill.
|
||||
|
||||
kv_append_metadata : str
|
||||
The metadata of the KV range on the destination decode instance.
|
||||
|
||||
dst_group_offset : int
|
||||
The node group offset of the destination decode instance.
|
||||
"""
|
||||
|
||||
begin: int
|
||||
end: int
|
||||
kv_addr_info: str
|
||||
recv_rank: int
|
||||
|
||||
|
||||
class StartGenerateRequest(CompletionRequest):
|
||||
"""The extra request body for start_generate request in MicroServing.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
kv_window_begin : int
|
||||
Denote the start of the KV range to prefill on the decode instance.
|
||||
"""
|
||||
|
||||
begin: int
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Schema for mlc-chat-config"""
|
||||
|
||||
from typing import Any, Dict, List, Literal, Optional, Union # noqa: UP035
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from mlc_llm.support.constants import MLC_CHAT_CONFIG_VERSION
|
||||
|
||||
from .conversation_protocol import Conversation
|
||||
|
||||
MLC_CHAT_SYSTEM_DEFAULT = {
|
||||
"pad_token_id": 0,
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"temperature": 1.0,
|
||||
"presence_penalty": 0.0,
|
||||
"frequency_penalty": 0.0,
|
||||
"repetition_penalty": 1.0,
|
||||
"top_p": 1.0,
|
||||
}
|
||||
"""system default values."""
|
||||
|
||||
|
||||
class MLCChatConfig(BaseModel):
|
||||
"""Fields in the dumped `mlc-chat-config.json` file."""
|
||||
|
||||
# Version control
|
||||
version: str = MLC_CHAT_CONFIG_VERSION
|
||||
|
||||
# use alias to avoid protected namespace conflict with pydantic
|
||||
field_model_type: str = Field(alias="model_type")
|
||||
quantization: str
|
||||
# use alias to avoid protected namespace conflict with pydantic
|
||||
field_model_config: Dict[str, Any] = Field(alias="model_config") # noqa: UP006
|
||||
vocab_size: int
|
||||
context_window_size: int
|
||||
sliding_window_size: int
|
||||
prefill_chunk_size: int
|
||||
attention_sink_size: int
|
||||
tensor_parallel_shards: int
|
||||
pipeline_parallel_stages: int = 1
|
||||
# Configuration of text generation
|
||||
active_vocab_size: int = None
|
||||
temperature: Optional[float] = None
|
||||
presence_penalty: Optional[float] = None
|
||||
frequency_penalty: Optional[float] = None
|
||||
repetition_penalty: Optional[float] = None
|
||||
top_p: Optional[float] = None
|
||||
# Tokenizer configuration
|
||||
tokenizer_files: List[str] = Field(default_factory=list) # noqa: UP006
|
||||
# The content of tokenizer.TokenizerInfo
|
||||
tokenizer_info: Dict[str, Any] = Field(default_factory=dict) # noqa: UP006
|
||||
# conversation template
|
||||
conv_template: Conversation
|
||||
# extra fields from generation_config.json
|
||||
# NOTE: they are not being used for now in MLCEngine
|
||||
# but we keep them for book-keep purposes
|
||||
pad_token_id: Optional[int] = None
|
||||
bos_token_id: Optional[int] = None
|
||||
eos_token_id: Optional[Union[int, List[int]]] = None # noqa: UP006
|
||||
|
||||
field_model_task: Literal["chat", "embedding"] = Field(default="chat", alias="model_task")
|
||||
embedding_metadata: Optional[Dict[str, Any]] = None # noqa: UP006
|
||||
|
||||
def get_system_defaults_for_missing_fields(self) -> Dict[str, Any]: # noqa: UP006
|
||||
"""Apply system default value for fields that are None
|
||||
|
||||
Note
|
||||
----
|
||||
We implement default setting in this way so we can lazily create
|
||||
MLCChatConfig, override its optional values then
|
||||
apply_system_defaults in the end.
|
||||
"""
|
||||
res = {}
|
||||
for key, value in MLC_CHAT_SYSTEM_DEFAULT.items():
|
||||
if getattr(self, key) is None:
|
||||
res[key] = value
|
||||
return res
|
||||
@@ -0,0 +1,462 @@
|
||||
"""Protocols in MLC LLM for OpenAI API.
|
||||
Adapted from FastChat's OpenAI protocol:
|
||||
https://github.com/lm-sys/FastChat/blob/main/fastchat/protocol/openai_api_protocol.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
from typing import Any, Dict, List, Literal, Optional, Tuple, Union # noqa: UP035
|
||||
|
||||
import shortuuid
|
||||
from pydantic import BaseModel, Field, field_validator, model_validator
|
||||
|
||||
from .conversation_protocol import Conversation
|
||||
from .debug_protocol import DebugConfig
|
||||
from .error_protocol import BadRequestError
|
||||
|
||||
################ Commons ################
|
||||
|
||||
|
||||
# OPenAI API compatible limits
|
||||
CHAT_COMPLETION_MAX_TOP_LOGPROBS = 20
|
||||
COMPLETION_MAX_TOP_LOGPROBS = 5
|
||||
|
||||
|
||||
class ListResponse(BaseModel):
|
||||
object: str = "list"
|
||||
data: List[Any] # noqa: UP006
|
||||
|
||||
|
||||
class TopLogProbs(BaseModel):
|
||||
token: str
|
||||
logprob: float
|
||||
bytes: Optional[List[int]] # noqa: UP006
|
||||
|
||||
|
||||
class LogProbsContent(BaseModel):
|
||||
token: str
|
||||
logprob: float
|
||||
bytes: Optional[List[int]] # noqa: UP006
|
||||
top_logprobs: List[TopLogProbs] = [] # noqa: UP006
|
||||
|
||||
|
||||
class LogProbs(BaseModel):
|
||||
content: List[LogProbsContent] # noqa: UP006
|
||||
|
||||
|
||||
class CompletionLogProbs(BaseModel):
|
||||
# The position of the token in the concatenated str: prompt + completion_text
|
||||
# TODO(vvchernov): skip optional after support
|
||||
text_offset: Optional[List[int]] # noqa: UP006
|
||||
token_logprobs: List[float] # noqa: UP006
|
||||
tokens: List[str] # noqa: UP006
|
||||
top_logprobs: List[Dict[str, float]] # noqa: UP006
|
||||
|
||||
|
||||
class CompletionUsage(BaseModel):
|
||||
prompt_tokens: int
|
||||
completion_tokens: int
|
||||
total_tokens: int
|
||||
extra: Optional[Dict[str, Any]] = None # noqa: UP006
|
||||
"""Extra metrics and info that may be returned by debug_config
|
||||
"""
|
||||
|
||||
|
||||
class StreamOptions(BaseModel):
|
||||
include_usage: Optional[bool]
|
||||
|
||||
|
||||
################ v1/embeddings ################
|
||||
|
||||
|
||||
class EmbeddingRequest(BaseModel):
|
||||
"""OpenAI "v1/embeddings" request protocol.
|
||||
API reference: https://platform.openai.com/docs/api-reference/embeddings/create
|
||||
"""
|
||||
|
||||
input: Union[str, List[str], List[int], List[List[int]]] # noqa: UP006
|
||||
model: Optional[str] = None
|
||||
encoding_format: Literal["float", "base64"] = "float"
|
||||
dimensions: Optional[int] = None
|
||||
user: Optional[str] = None
|
||||
|
||||
@field_validator("input")
|
||||
@classmethod
|
||||
def validate_input(cls, v):
|
||||
"""Check that the input is not an empty list.
|
||||
|
||||
Note: empty strings are allowed — encoder models produce valid
|
||||
embeddings from [CLS]+[SEP] tokens alone.
|
||||
"""
|
||||
if isinstance(v, list) and len(v) == 0:
|
||||
raise ValueError("Input list must not be empty.")
|
||||
return v
|
||||
|
||||
|
||||
class EmbeddingObject(BaseModel):
|
||||
object: str = "embedding"
|
||||
embedding: Union[List[float], str] # noqa: UP006
|
||||
index: int
|
||||
|
||||
|
||||
class EmbeddingUsage(BaseModel):
|
||||
prompt_tokens: int
|
||||
total_tokens: int
|
||||
|
||||
|
||||
class EmbeddingResponse(BaseModel):
|
||||
"""OpenAI "v1/embeddings" response protocol.
|
||||
API reference: https://platform.openai.com/docs/api-reference/embeddings/object
|
||||
"""
|
||||
|
||||
object: str = "list"
|
||||
data: List[EmbeddingObject] # noqa: UP006
|
||||
model: Optional[str] = None
|
||||
usage: EmbeddingUsage
|
||||
|
||||
|
||||
################ v1/models ################
|
||||
|
||||
|
||||
class ModelResponse(BaseModel):
|
||||
"""OpenAI "v1/models" response protocol.
|
||||
API reference: https://platform.openai.com/docs/api-reference/models/object
|
||||
"""
|
||||
|
||||
id: str
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
object: str = "model"
|
||||
owned_by: str = "MLC-LLM"
|
||||
|
||||
|
||||
################ v1/completions ################
|
||||
|
||||
|
||||
class RequestResponseFormat(BaseModel):
|
||||
type: Literal["text", "json_object"] = "text"
|
||||
json_schema: Optional[str] = Field(default=None, alias="schema")
|
||||
"""This field is named json_schema instead of schema because BaseModel defines a method called
|
||||
schema. During construction of RequestResponseFormat, key "schema" still should be used:
|
||||
`RequestResponseFormat(type="json_object", schema="{}")`
|
||||
"""
|
||||
|
||||
|
||||
class CompletionRequest(BaseModel):
|
||||
"""OpenAI completion request protocol.
|
||||
API reference: https://platform.openai.com/docs/api-reference/completions/create
|
||||
"""
|
||||
|
||||
model: Optional[str] = None
|
||||
prompt: Union[str, List[int]] # noqa: UP006
|
||||
best_of: int = 1
|
||||
echo: bool = False
|
||||
frequency_penalty: Optional[float] = None
|
||||
presence_penalty: Optional[float] = None
|
||||
logprobs: Optional[int] = None
|
||||
logit_bias: Optional[Dict[int, float]] = None # noqa: UP006
|
||||
max_tokens: Optional[int] = None
|
||||
n: int = 1
|
||||
seed: Optional[int] = None
|
||||
stop: Optional[Union[str, List[str]]] = None # noqa: UP006
|
||||
stream: bool = False
|
||||
stream_options: Optional[StreamOptions] = None
|
||||
suffix: Optional[str] = None
|
||||
temperature: Optional[float] = None
|
||||
top_p: Optional[float] = None
|
||||
user: Optional[str] = None
|
||||
response_format: Optional[RequestResponseFormat] = None
|
||||
debug_config: Optional[DebugConfig] = None
|
||||
|
||||
@field_validator("frequency_penalty", "presence_penalty")
|
||||
@classmethod
|
||||
def check_penalty_range(cls, penalty_value: Optional[float]) -> Optional[float]:
|
||||
"""Check if the penalty value is in range [-2, 2]."""
|
||||
if penalty_value and (penalty_value < -2 or penalty_value > 2):
|
||||
raise ValueError("Penalty value should be in range [-2, 2].")
|
||||
return penalty_value
|
||||
|
||||
@field_validator("logit_bias")
|
||||
@classmethod
|
||||
def check_logit_bias(
|
||||
cls,
|
||||
logit_bias_value: Optional[Dict[int, float]], # noqa: UP006
|
||||
) -> Optional[Dict[int, float]]: # noqa: UP006
|
||||
"""Check if the logit bias key is given as an integer."""
|
||||
if logit_bias_value is None:
|
||||
return None
|
||||
for token_id, bias in logit_bias_value.items():
|
||||
if abs(bias) > 100:
|
||||
raise ValueError(
|
||||
"Logit bias value should be in range [-100, 100], while value "
|
||||
f"{bias} is given for token id {token_id}"
|
||||
)
|
||||
return logit_bias_value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_logprobs(self) -> "CompletionRequest":
|
||||
"""Check if the logprobs requirements are valid."""
|
||||
if self.logprobs is not None and (
|
||||
self.logprobs < 0 or self.logprobs > COMPLETION_MAX_TOP_LOGPROBS
|
||||
):
|
||||
raise ValueError(f'"logprobs" must be in range [0, {COMPLETION_MAX_TOP_LOGPROBS}]')
|
||||
return self
|
||||
|
||||
|
||||
class CompletionResponseChoice(BaseModel):
|
||||
finish_reason: Optional[Literal["stop", "length", "preempt"]] = None
|
||||
index: int = 0
|
||||
logprobs: Optional[CompletionLogProbs] = None
|
||||
text: str
|
||||
|
||||
|
||||
class CompletionResponse(BaseModel):
|
||||
"""OpenAI completion response protocol.
|
||||
API reference: https://platform.openai.com/docs/api-reference/completions/object
|
||||
"""
|
||||
|
||||
id: str
|
||||
choices: List[CompletionResponseChoice] # noqa: UP006
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
model: Optional[str] = None
|
||||
object: str = "text_completion"
|
||||
usage: Optional[CompletionUsage] = None
|
||||
|
||||
|
||||
################ v1/chat/completions ################
|
||||
|
||||
|
||||
class ChatFunction(BaseModel):
|
||||
description: Optional[str] = None
|
||||
name: str
|
||||
parameters: Dict # noqa: UP006
|
||||
|
||||
|
||||
class ChatTool(BaseModel):
|
||||
type: Literal["function"]
|
||||
function: ChatFunction
|
||||
|
||||
|
||||
class ChatFunctionCall(BaseModel):
|
||||
name: str
|
||||
arguments: Union[None, Dict[str, Any]] = None # noqa: UP006
|
||||
|
||||
|
||||
class ChatToolCall(BaseModel):
|
||||
id: str = Field(default_factory=lambda: f"call_{shortuuid.random()}")
|
||||
type: Literal["function"]
|
||||
function: ChatFunctionCall
|
||||
|
||||
|
||||
class ChatCompletionMessage(BaseModel):
|
||||
content: Optional[Union[str, List[Dict]]] = None # noqa: UP006
|
||||
role: Literal["system", "user", "assistant", "tool"]
|
||||
name: Optional[str] = None
|
||||
tool_calls: Optional[List[ChatToolCall]] = None # noqa: UP006
|
||||
tool_call_id: Optional[str] = None
|
||||
|
||||
|
||||
class ChatCompletionRequest(BaseModel):
|
||||
"""OpenAI chat completion request protocol.
|
||||
API reference: https://platform.openai.com/docs/api-reference/chat/create
|
||||
"""
|
||||
|
||||
messages: List[ChatCompletionMessage] # noqa: UP006
|
||||
model: Optional[str] = None
|
||||
frequency_penalty: Optional[float] = None
|
||||
presence_penalty: Optional[float] = None
|
||||
logprobs: bool = False
|
||||
top_logprobs: int = 0
|
||||
logit_bias: Optional[Dict[int, float]] = None # noqa: UP006
|
||||
max_tokens: Optional[int] = None
|
||||
n: int = 1
|
||||
seed: Optional[int] = None
|
||||
stop: Optional[Union[str, List[str]]] = None # noqa: UP006
|
||||
stream: bool = False
|
||||
stream_options: Optional[StreamOptions] = None
|
||||
temperature: Optional[float] = None
|
||||
top_p: Optional[float] = None
|
||||
tools: Optional[List[ChatTool]] = None # noqa: UP006
|
||||
tool_choice: Optional[Union[Literal["none", "auto"], Dict]] = None # noqa: UP006
|
||||
user: Optional[str] = None
|
||||
response_format: Optional[RequestResponseFormat] = None
|
||||
# NOTE: debug_config is not part of OpenAI protocol
|
||||
# we add it to enable extra debug options
|
||||
debug_config: Optional[DebugConfig] = None
|
||||
|
||||
@field_validator("frequency_penalty", "presence_penalty")
|
||||
@classmethod
|
||||
def check_penalty_range(cls, penalty_value: Optional[float]) -> Optional[float]:
|
||||
"""Check if the penalty value is in range [-2, 2]."""
|
||||
if penalty_value and (penalty_value < -2 or penalty_value > 2):
|
||||
raise ValueError("Penalty value should be in range [-2, 2].")
|
||||
return penalty_value
|
||||
|
||||
@field_validator("logit_bias")
|
||||
@classmethod
|
||||
def check_logit_bias(
|
||||
cls,
|
||||
logit_bias_value: Optional[Dict[int, float]], # noqa: UP006
|
||||
) -> Optional[Dict[int, float]]: # noqa: UP006
|
||||
"""Check if the logit bias key is given as an integer."""
|
||||
if logit_bias_value is None:
|
||||
return None
|
||||
for token_id, bias in logit_bias_value.items():
|
||||
if abs(bias) > 100:
|
||||
raise ValueError(
|
||||
"Logit bias value should be in range [-100, 100], while value "
|
||||
f"{bias} is given for token id {token_id}"
|
||||
)
|
||||
return logit_bias_value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_logprobs(self) -> "ChatCompletionRequest":
|
||||
"""Check if the logprobs requirements are valid."""
|
||||
if self.top_logprobs < 0 or self.top_logprobs > CHAT_COMPLETION_MAX_TOP_LOGPROBS:
|
||||
raise ValueError(
|
||||
f'"top_logprobs" must be in range [0, {CHAT_COMPLETION_MAX_TOP_LOGPROBS}]'
|
||||
)
|
||||
if not self.logprobs and self.top_logprobs > 0:
|
||||
raise ValueError('"logprobs" must be True to support "top_logprobs"')
|
||||
return self
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_stream_options(self) -> "ChatCompletionRequest":
|
||||
"""Check stream options"""
|
||||
if self.stream_options is None:
|
||||
return self
|
||||
if not self.stream:
|
||||
raise ValueError("stream must be set to True when stream_options is present")
|
||||
return self
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_debug_config(self) -> "ChatCompletionRequest":
|
||||
"""Check debug config"""
|
||||
if self.debug_config is None:
|
||||
return self
|
||||
|
||||
if self.debug_config.special_request is None:
|
||||
return self
|
||||
|
||||
if not self.stream:
|
||||
raise ValueError("DebugConfig.special_request requires stream=True")
|
||||
|
||||
if self.stream_options is None or not self.stream_options.include_usage:
|
||||
raise ValueError("DebugConfig.special_request requires include_usage in stream_options")
|
||||
|
||||
return self
|
||||
|
||||
def check_message_validity(self) -> None:
|
||||
"""Check if the given chat messages are valid. Return error message if invalid."""
|
||||
for i, message in enumerate(self.messages):
|
||||
if message.role == "system" and i != 0:
|
||||
raise BadRequestError(
|
||||
f"System prompt at position {i} in the message list is invalid."
|
||||
)
|
||||
if message.tool_call_id is not None:
|
||||
if message.role != "tool":
|
||||
raise BadRequestError("Non-tool message having `tool_call_id` is invalid.")
|
||||
if isinstance(message.content, list):
|
||||
if message.role != "user":
|
||||
raise BadRequestError("Non-user message having a list of content is invalid.")
|
||||
if message.tool_calls is not None:
|
||||
if message.role != "assistant":
|
||||
raise BadRequestError("Non-assistant message having `tool_calls` is invalid.")
|
||||
raise BadRequestError("Assistant message having `tool_calls` is not supported yet.")
|
||||
|
||||
def check_function_call_usage(self, conv_template: Conversation) -> None:
|
||||
"""Check if function calling is used and update the conversation template.
|
||||
Return error message if invalid request format for function calling.
|
||||
"""
|
||||
|
||||
# return if no tools are provided or tool_choice is set to none
|
||||
if self.tools is None or (isinstance(self.tool_choice, str) and self.tool_choice == "none"):
|
||||
conv_template.use_function_calling = False
|
||||
return
|
||||
|
||||
# select the tool based on the tool_choice if specified
|
||||
if isinstance(self.tool_choice, dict):
|
||||
if self.tool_choice["type"] != "function":
|
||||
raise BadRequestError("Only 'function' tool choice is supported")
|
||||
|
||||
if len(self.tool_choice["function"]) > 1:
|
||||
raise BadRequestError("Only one tool is supported when tool_choice is specified")
|
||||
|
||||
for tool in self.tools:
|
||||
if tool.function.name == self.tool_choice["function"]["name"]:
|
||||
conv_template.use_function_calling = True
|
||||
conv_template.function_string = tool.function.model_dump_json(by_alias=True)
|
||||
return
|
||||
|
||||
raise BadRequestError(
|
||||
f"The tool_choice function {self.tool_choice['function']['name']}"
|
||||
" is not found in the tools list"
|
||||
)
|
||||
|
||||
if isinstance(self.tool_choice, str) and self.tool_choice != "auto":
|
||||
raise BadRequestError(f"Invalid tool_choice value: {self.tool_choice}")
|
||||
|
||||
function_list = []
|
||||
for tool in self.tools:
|
||||
if tool.type != "function":
|
||||
raise BadRequestError("Only 'function' tool type is supported")
|
||||
function_list.append(tool.function.model_dump(by_alias=True))
|
||||
|
||||
conv_template.use_function_calling = True
|
||||
conv_template.function_string = json.dumps(function_list)
|
||||
|
||||
|
||||
class ChatCompletionResponseChoice(BaseModel):
|
||||
finish_reason: Optional[Literal["stop", "length", "tool_calls", "error"]] = None
|
||||
index: int = 0
|
||||
message: ChatCompletionMessage
|
||||
logprobs: Optional[LogProbs] = None
|
||||
|
||||
|
||||
class ChatCompletionStreamResponseChoice(BaseModel):
|
||||
finish_reason: Optional[Literal["stop", "length", "tool_calls", "error"]] = None
|
||||
index: int = 0
|
||||
delta: ChatCompletionMessage
|
||||
logprobs: Optional[LogProbs] = None
|
||||
|
||||
|
||||
class ChatCompletionResponse(BaseModel):
|
||||
"""OpenAI completion response protocol.
|
||||
API reference: https://platform.openai.com/docs/api-reference/chat/object
|
||||
"""
|
||||
|
||||
id: str
|
||||
choices: List[ChatCompletionResponseChoice] # noqa: UP006
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
model: Optional[str] = None
|
||||
system_fingerprint: str
|
||||
object: Literal["chat.completion"] = "chat.completion"
|
||||
usage: Optional[CompletionUsage] = None
|
||||
|
||||
|
||||
class ChatCompletionStreamResponse(BaseModel):
|
||||
"""OpenAI completion stream response protocol.
|
||||
API reference: https://platform.openai.com/docs/api-reference/chat/streaming
|
||||
"""
|
||||
|
||||
id: str
|
||||
choices: List[ChatCompletionStreamResponseChoice] # noqa: UP006
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
model: Optional[str] = None
|
||||
system_fingerprint: str
|
||||
object: Literal["chat.completion.chunk"] = "chat.completion.chunk"
|
||||
usage: Optional[CompletionUsage] = None
|
||||
|
||||
|
||||
def openai_api_get_unsupported_fields(
|
||||
request: Union[CompletionRequest, ChatCompletionRequest],
|
||||
) -> List[str]: # noqa: UP006
|
||||
"""Get the unsupported fields in the request."""
|
||||
unsupported_field_default_values: List[Tuple[str, Any]] = [ # noqa: UP006
|
||||
("best_of", 1),
|
||||
]
|
||||
|
||||
unsupported_fields: List[str] = [] # noqa: UP006
|
||||
for field, value in unsupported_field_default_values:
|
||||
if hasattr(request, field) and getattr(request, field) != value:
|
||||
unsupported_fields.append(field)
|
||||
return unsupported_fields
|
||||
Reference in New Issue
Block a user