wehub-resource-sync
378 lines
12 KiB
Python
378 lines
12 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
# Adapted from
|
|
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
|
|
import time
|
|
from http import HTTPStatus
|
|
from typing import Any, ClassVar, Literal, TypeAlias
|
|
|
|
import regex as re
|
|
from pydantic import (
|
|
BaseModel,
|
|
ConfigDict,
|
|
Field,
|
|
model_serializer,
|
|
model_validator,
|
|
)
|
|
|
|
from vllm.entrypoints.chat_utils import make_tool_call_id
|
|
from vllm.exceptions import VLLMValidationError
|
|
from vllm.logger import init_logger
|
|
from vllm.utils import random_uuid
|
|
from vllm.utils.import_utils import resolve_obj_by_qualname
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
class OpenAIBaseModel(BaseModel):
|
|
# OpenAI API does allow extra fields
|
|
model_config = ConfigDict(extra="allow")
|
|
|
|
# Cache class field names
|
|
field_names: ClassVar[set[str] | None] = None
|
|
|
|
@model_validator(mode="wrap")
|
|
@classmethod
|
|
def __log_extra_fields__(cls, data, handler):
|
|
result = handler(data)
|
|
if not isinstance(data, dict):
|
|
return result
|
|
field_names = cls.field_names
|
|
if field_names is None:
|
|
# Get all class field names and their potential aliases
|
|
field_names = set()
|
|
for field_name, field in cls.model_fields.items():
|
|
field_names.add(field_name)
|
|
if alias := getattr(field, "alias", None):
|
|
field_names.add(alias)
|
|
cls.field_names = field_names
|
|
|
|
# Compare against both field names and aliases
|
|
if any(k not in field_names for k in data):
|
|
logger.debug(
|
|
"The following fields were present in the request but ignored: %s",
|
|
data.keys() - field_names,
|
|
)
|
|
return result
|
|
|
|
|
|
class ErrorInfo(OpenAIBaseModel):
|
|
message: str
|
|
type: str
|
|
param: str | None = None
|
|
code: int
|
|
|
|
|
|
class ErrorResponse(OpenAIBaseModel):
|
|
error: ErrorInfo
|
|
|
|
|
|
class ModelPermission(OpenAIBaseModel):
|
|
id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}")
|
|
object: str = "model_permission"
|
|
created: int = Field(default_factory=lambda: int(time.time()))
|
|
allow_create_engine: bool = False
|
|
allow_sampling: bool = True
|
|
allow_logprobs: bool = True
|
|
allow_search_indices: bool = False
|
|
allow_view: bool = True
|
|
allow_fine_tuning: bool = False
|
|
organization: str = "*"
|
|
group: str | None = None
|
|
is_blocking: bool = False
|
|
|
|
|
|
class ModelCard(OpenAIBaseModel):
|
|
id: str
|
|
object: str = "model"
|
|
created: int = Field(default_factory=lambda: int(time.time()))
|
|
owned_by: str = "vllm"
|
|
root: str | None = None
|
|
parent: str | None = None
|
|
max_model_len: int | None = None
|
|
permission: list[ModelPermission] = Field(default_factory=list)
|
|
|
|
|
|
class ModelList(OpenAIBaseModel):
|
|
object: str = "list"
|
|
data: list[ModelCard] = Field(default_factory=list)
|
|
|
|
|
|
class PromptTokenUsageInfo(OpenAIBaseModel):
|
|
cached_tokens: int | None = None
|
|
multimodal_tokens: dict[str, int] | None = None
|
|
"""Prompt tokens contributed by each input modality, keyed by modality name
|
|
(e.g. `image`, `audio`, `video`). A breakdown of the multimodal
|
|
placeholder tokens already counted in `prompt_tokens`; `None` when the
|
|
request has no multimodal input."""
|
|
|
|
|
|
class UsageInfo(OpenAIBaseModel):
|
|
prompt_tokens: int = 0
|
|
total_tokens: int = 0
|
|
completion_tokens: int | None = 0
|
|
prompt_tokens_details: PromptTokenUsageInfo | None = None
|
|
|
|
|
|
class PerRequestTimingMetrics(OpenAIBaseModel):
|
|
time_to_first_token_ms: float | None = None
|
|
generation_time_ms: float | None = None
|
|
queue_time_ms: float | None = None
|
|
mean_itl_ms: float | None = None
|
|
tokens_per_second: float | None = None
|
|
|
|
|
|
class RequestResponseMetadata(BaseModel):
|
|
request_id: str
|
|
final_usage_info: UsageInfo | None = None
|
|
|
|
|
|
class JsonSchemaResponseFormat(OpenAIBaseModel):
|
|
name: str
|
|
description: str | None = None
|
|
# schema is the field in openai but that causes conflicts with pydantic so
|
|
# instead use json_schema with an alias
|
|
json_schema: dict[str, Any] | None = Field(default=None, alias="schema")
|
|
strict: bool | None = None
|
|
|
|
|
|
class LegacyStructuralTag(OpenAIBaseModel):
|
|
begin: str
|
|
# schema is the field, but that causes conflicts with pydantic so
|
|
# instead use structural_tag_schema with an alias
|
|
structural_tag_schema: dict[str, Any] | None = Field(default=None, alias="schema")
|
|
end: str
|
|
|
|
|
|
class LegacyStructuralTagResponseFormat(OpenAIBaseModel):
|
|
type: Literal["structural_tag"]
|
|
structures: list[LegacyStructuralTag]
|
|
triggers: list[str]
|
|
|
|
|
|
class StructuralTagResponseFormat(OpenAIBaseModel):
|
|
type: Literal["structural_tag"]
|
|
format: Any
|
|
|
|
|
|
AnyStructuralTagResponseFormat: TypeAlias = (
|
|
LegacyStructuralTagResponseFormat | StructuralTagResponseFormat
|
|
)
|
|
|
|
|
|
class ResponseFormat(OpenAIBaseModel):
|
|
# type must be "json_schema", "json_object", or "text"
|
|
type: Literal["text", "json_object", "json_schema"]
|
|
json_schema: JsonSchemaResponseFormat | None = None
|
|
|
|
|
|
AnyResponseFormat: TypeAlias = (
|
|
ResponseFormat | StructuralTagResponseFormat | LegacyStructuralTagResponseFormat
|
|
)
|
|
|
|
|
|
def validate_structural_tag_response_format(
|
|
response_format: AnyStructuralTagResponseFormat | dict[str, Any],
|
|
) -> None:
|
|
"""Validate structural tags before they are sent to the engine.
|
|
|
|
Engine-side validation reports malformed structural tags as generation
|
|
failures. OpenAI request parsing should classify them as bad requests.
|
|
"""
|
|
import json
|
|
|
|
from pydantic import TypeAdapter, ValidationError
|
|
|
|
if isinstance(response_format, dict):
|
|
try:
|
|
response_format = TypeAdapter(
|
|
AnyStructuralTagResponseFormat
|
|
).validate_python(response_format)
|
|
except ValidationError as exc:
|
|
raise VLLMValidationError(
|
|
"Invalid response_format structural_tag specification.",
|
|
parameter="response_format",
|
|
) from exc
|
|
|
|
try:
|
|
payload = json.dumps(response_format.model_dump(by_alias=True))
|
|
validate_structural_tag_payload(payload, parameter="response_format")
|
|
except (TypeError, ValueError) as exc:
|
|
raise VLLMValidationError(
|
|
"Invalid response_format structural_tag specification.",
|
|
parameter="response_format",
|
|
) from exc
|
|
|
|
|
|
def validate_structural_tag_payload(payload: Any, *, parameter: str) -> None:
|
|
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
|
from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar
|
|
|
|
if isinstance(payload, str) and not payload:
|
|
raise VLLMValidationError(
|
|
f"Invalid {parameter} structural_tag specification.",
|
|
parameter=parameter,
|
|
)
|
|
|
|
try:
|
|
validate_xgrammar_grammar(
|
|
SamplingParams(
|
|
structured_outputs=StructuredOutputsParams(structural_tag=payload)
|
|
)
|
|
)
|
|
except (TypeError, ValueError) as exc:
|
|
raise VLLMValidationError(
|
|
f"Invalid {parameter} structural_tag specification.",
|
|
parameter=parameter,
|
|
) from exc
|
|
|
|
|
|
def validate_structured_outputs_structural_tag(
|
|
structured_outputs: Any,
|
|
) -> None:
|
|
from vllm.sampling_params import StructuredOutputsParams
|
|
|
|
if isinstance(structured_outputs, StructuredOutputsParams):
|
|
structural_tag = structured_outputs.structural_tag
|
|
elif isinstance(structured_outputs, dict):
|
|
structural_tag = structured_outputs.get("structural_tag")
|
|
else:
|
|
return
|
|
if structural_tag is not None:
|
|
validate_structural_tag_payload(
|
|
structural_tag,
|
|
parameter="structured_outputs",
|
|
)
|
|
|
|
|
|
class StreamOptions(OpenAIBaseModel):
|
|
include_usage: bool | None = False
|
|
continuous_usage_stats: bool | None = False
|
|
|
|
|
|
class FunctionDefinition(OpenAIBaseModel):
|
|
name: str
|
|
description: str | None = None
|
|
parameters: dict[str, Any] | None = None
|
|
strict: bool | None = None
|
|
defer_loading: bool | None = None
|
|
|
|
@model_serializer(mode="wrap")
|
|
def _serialize(self, handler):
|
|
data = handler(self)
|
|
if self.strict is None:
|
|
data.pop("strict", None)
|
|
if self.defer_loading is None:
|
|
data.pop("defer_loading", None)
|
|
return data
|
|
|
|
|
|
# extra="forbid" is a workaround to have kwargs as a field,
|
|
# see https://github.com/pydantic/pydantic/issues/3125
|
|
class LogitsProcessorConstructor(BaseModel):
|
|
qualname: str
|
|
args: list[Any] | None = None
|
|
kwargs: dict[str, Any] | None = None
|
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
LogitsProcessors = list[str | LogitsProcessorConstructor]
|
|
|
|
|
|
def get_logits_processors(
|
|
processors: LogitsProcessors | None, pattern: str | None
|
|
) -> list[Any] | None:
|
|
if processors and pattern:
|
|
logits_processors = []
|
|
for processor in processors:
|
|
qualname = processor if isinstance(processor, str) else processor.qualname
|
|
if not re.match(pattern, qualname):
|
|
raise ValueError(
|
|
f"Logits processor '{qualname}' is not allowed by this "
|
|
"server. See --logits-processor-pattern engine argument "
|
|
"for more information."
|
|
)
|
|
try:
|
|
logits_processor = resolve_obj_by_qualname(qualname)
|
|
except Exception as e:
|
|
raise ValueError(
|
|
f"Logits processor '{qualname}' could not be resolved: {e}"
|
|
) from e
|
|
if isinstance(processor, LogitsProcessorConstructor):
|
|
logits_processor = logits_processor(
|
|
*processor.args or [], **processor.kwargs or {}
|
|
)
|
|
logits_processors.append(logits_processor)
|
|
return logits_processors
|
|
elif processors:
|
|
raise ValueError(
|
|
"The `logits_processors` argument is not supported by this "
|
|
"server. See --logits-processor-pattern engine argument "
|
|
"for more information."
|
|
)
|
|
return None
|
|
|
|
|
|
class FunctionCall(OpenAIBaseModel):
|
|
# Internal field to preserve native tool call ID from tool parser.
|
|
# Excluded from serialization to maintain OpenAI API compatibility
|
|
# (function object should only contain 'name' and 'arguments').
|
|
id: str | None = Field(default=None, exclude=True)
|
|
name: str
|
|
arguments: str
|
|
|
|
|
|
class ToolCall(OpenAIBaseModel):
|
|
id: str = Field(default_factory=make_tool_call_id)
|
|
type: Literal["function"] = "function"
|
|
function: FunctionCall
|
|
|
|
|
|
class DeltaFunctionCall(BaseModel):
|
|
name: str | None = None
|
|
arguments: str | None = None
|
|
|
|
|
|
# a tool call delta where everything is optional
|
|
class DeltaToolCall(OpenAIBaseModel):
|
|
id: str | None = None
|
|
type: Literal["function"] | None = None
|
|
index: int
|
|
function: DeltaFunctionCall | None = None
|
|
|
|
|
|
class ExtractedToolCallInformation(BaseModel):
|
|
# indicate if tools were called
|
|
tools_called: bool
|
|
|
|
# extracted tool calls
|
|
tool_calls: list[ToolCall]
|
|
|
|
# content - per OpenAI spec, content AND tool calls can be returned rarely
|
|
# But some models will do this intentionally
|
|
content: str | None = None
|
|
|
|
|
|
class DeltaMessage(OpenAIBaseModel):
|
|
role: str | None = None
|
|
content: str | None = None
|
|
reasoning: str | None = None
|
|
tool_calls: list[DeltaToolCall] = Field(default_factory=list)
|
|
|
|
@model_serializer(mode="wrap")
|
|
def _serialize(self, handler):
|
|
data = handler(self)
|
|
if len(data.get("tool_calls", [])) == 0:
|
|
data.pop("tool_calls", None)
|
|
return data
|
|
|
|
|
|
class GenerationError(Exception):
|
|
"""raised when finish_reason indicates internal server error (500)"""
|
|
|
|
def __init__(self, message: str = "Internal server error"):
|
|
super().__init__(message)
|
|
self.status_code = HTTPStatus.INTERNAL_SERVER_ERROR
|