# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio import io import time from collections.abc import AsyncGenerator, AsyncIterator from collections.abc import Sequence as GenericSequence from http import HTTPStatus from typing import Any, Final, cast import numpy as np import pybase64 as base64 from fastapi import Request from vllm.engine.protocol import EngineClient from vllm.entrypoints.chat_utils import ( ChatTemplateContentFormatOption, ConversationMessage, make_tool_call_id, ) from vllm.entrypoints.generate.base.serving import ( GenerateBaseServing, GenerationError, build_per_request_timing_metrics, clamp_prompt_logprobs, format_token_id_placeholder, ) from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionLogProb, ChatCompletionLogProbs, ChatCompletionLogProbsContent, ChatCompletionNamedToolChoiceParam, ChatCompletionRequest, ChatCompletionResponse, ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice, ChatCompletionStreamResponse, ChatMessage, ) from vllm.entrypoints.openai.engine.protocol import ( DeltaMessage, ErrorResponse, FunctionCall, PerRequestTimingMetrics, PromptTokenUsageInfo, RequestResponseMetadata, ToolCall, UsageInfo, ) from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.entrypoints.serve.utils.tool_calls_utils import ( maybe_filter_parallel_tool_calls, ) from vllm.inputs import EngineInput, MultiModalPlaceholders from vllm.logger import init_logger from vllm.logprobs import Logprob from vllm.outputs import RequestOutput from vllm.parser import ParserManager from vllm.parser.abstract_parser import Parser from vllm.renderers import ChatParams from vllm.renderers.online_renderer import OnlineRenderer from vllm.sampling_params import BeamSearchParams, SamplingParams from vllm.tokenizers import TokenizerLike from vllm.utils.collection_utils import as_list from vllm.utils.mistral import is_mistral_tool_parser logger = init_logger(__name__) def _get_mm_token_counts(engine_input: EngineInput) -> dict[str, int]: """Sum per-modality placeholder tokens from ``mm_placeholders``. Keyed by modality name; ``PlaceholderRange.length`` is the placeholder's prompt token span, so each sum matches the placeholder tokens already counted in ``usage.prompt_tokens``. """ mm_placeholders = cast( MultiModalPlaceholders | None, engine_input.get("mm_placeholders") ) return { modality: sum(p.length for p in ranges) for modality, ranges in (mm_placeholders or {}).items() if ranges } def _make_prompt_tokens_details( enable_prompt_tokens_details: bool, num_cached_tokens: int | None, mm_token_counts: dict[str, int] | None, ) -> PromptTokenUsageInfo | None: """Build ``prompt_tokens_details`` from cached + multimodal token counts.""" if not enable_prompt_tokens_details: return None if num_cached_tokens is None and not mm_token_counts: return None return PromptTokenUsageInfo( cached_tokens=num_cached_tokens, multimodal_tokens=mm_token_counts or None, ) class OpenAIServingChat(GenerateBaseServing): def __init__( self, engine_client: EngineClient, models: OpenAIServingModels, response_role: str, *, online_renderer: "OnlineRenderer", request_logger: RequestLogger | None, chat_template: str | None, chat_template_content_format: ChatTemplateContentFormatOption, trust_request_chat_template: bool = False, return_tokens_as_token_ids: bool = False, reasoning_parser: str = "", enable_auto_tools: bool = False, exclude_tools_when_tool_choice_none: bool = False, tool_parser: str | None = None, enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, enable_log_outputs: bool = False, enable_log_deltas: bool = True, default_chat_template_kwargs: dict[str, Any] | None = None, enable_per_request_metrics: bool = False, ) -> None: super().__init__( engine_client=engine_client, models=models, request_logger=request_logger, return_tokens_as_token_ids=return_tokens_as_token_ids, ) self.online_renderer = online_renderer self.response_role = response_role self.chat_template = chat_template self.chat_template_content_format: Final = chat_template_content_format self.trust_request_chat_template = trust_request_chat_template self.default_chat_template_kwargs = default_chat_template_kwargs or {} self.enable_log_outputs = enable_log_outputs self.enable_log_deltas = enable_log_deltas self.enable_auto_tools: bool = enable_auto_tools self.parser_cls = ParserManager.get_parser( tool_parser_name=tool_parser, reasoning_parser_name=reasoning_parser, enable_auto_tools=enable_auto_tools, model_name=self.model_config.model, is_harmony=self.model_config.hf_config.model_type == "gpt_oss", ) if ( self.parser_cls is not None and is_mistral_tool_parser(self.parser_cls.tool_parser_cls) and self.parser_cls.reasoning_parser_cls is not None ): from vllm.tool_parsers.mistral_tool_parser import MistralToolParser MistralToolParser.model_can_reason = True self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none self.enable_prompt_tokens_details = enable_prompt_tokens_details self.enable_force_include_usage = enable_force_include_usage self.enable_per_request_metrics = enable_per_request_metrics self.default_sampling_params = self.model_config.get_diff_sampling_param() mc = self.model_config self.override_max_tokens = ( self.default_sampling_params.get("max_tokens") if mc.generation_config not in ("auto", "vllm") else getattr(mc, "override_generation_config", {}).get("max_new_tokens") ) # NOTE(woosuk): While OpenAI's chat completion API supports browsing # for some models, currently vLLM doesn't support it. Please use the # Responses API instead. self.supports_browsing = False self.browser_tool = None # NOTE(woosuk): Chat completion API does not support code interpreter. # Please use the Responses API instead. self.supports_code_interpreter = False self.python_tool = None def warmup(self) -> None: self.renderer.warmup( ChatParams( chat_template=self.chat_template, chat_template_content_format=self.chat_template_content_format, chat_template_kwargs=self.default_chat_template_kwargs, ) ) def _effective_chat_template_kwargs( self, request: ChatCompletionRequest ) -> dict[str, Any]: return ( request.build_chat_params( self.chat_template, self.chat_template_content_format, ) .with_defaults(self.default_chat_template_kwargs) .chat_template_kwargs ) async def render_chat_request( self, request: ChatCompletionRequest, ) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse: """ Validate the model and preprocess a chat completion request. Delegates preprocessing logic to OnlineRenderer, adding the engine-aware checks (LoRA model validation, engine health). Returns: A tuple of (conversation, engine_inputs) on success, or an ErrorResponse on failure. """ error_check_ret = await self._check_model(request) if error_check_ret is not None: logger.error("Error with model %s", error_check_ret) return error_check_ret # If the engine is dead, raise the engine's DEAD_ERROR. # This is required for the streaming case, where we return a # success status before we actually start generating text :). if self.engine_client.errored: raise self.engine_client.dead_error return await self.online_renderer.render_chat(request) async def create_chat_completion( self, request: ChatCompletionRequest, raw_request: Request | None = None, ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse: """ Chat Completion API similar to OpenAI's API. See https://platform.openai.com/docs/api-reference/chat/create for the API specification. This API mimics the OpenAI Chat Completion API. """ return await self._with_kv_transfer_rejection_cleanup( self._create_chat_completion(request, raw_request), request, raw_request ) async def _create_chat_completion( self, request: ChatCompletionRequest, raw_request: Request | None = None, ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse: # Streaming response tokenizer = self.renderer.tokenizer assert tokenizer is not None chat_template_kwargs = self._effective_chat_template_kwargs(request) parser: Parser | None = None if self.parser_cls is not None: parser = self.parser_cls( tokenizer, request.tools, chat_template_kwargs=chat_template_kwargs, model_config=self.model_config, ) result = await self.render_chat_request(request) if isinstance(result, ErrorResponse): return result conversation, engine_inputs = result request_id = ( f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}" ) request_metadata = RequestResponseMetadata(request_id=request_id) if raw_request: raw_request.state.request_metadata = request_metadata lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True) model_name = self.models.model_name(lora_request) # Extract data_parallel_rank from header (router can inject it) data_parallel_rank = self._get_data_parallel_rank(raw_request) # Schedule the request and get the result generator. max_model_len = self.model_config.max_model_len generators: list[AsyncGenerator[RequestOutput, None]] = [] mm_token_counts: dict[str, int] | None = None for i, engine_input in enumerate(engine_inputs): prompt_token_ids = self._extract_prompt_components(engine_input).token_ids mm_token_counts = _get_mm_token_counts(engine_input) # If we are creating sub requests for multiple prompts, ensure that they # have unique request ids. sub_request_id = ( request_id if len(engine_inputs) == 1 else f"{request_id}_{i}" ) max_tokens = get_max_tokens( max_model_len, request.max_completion_tokens if request.max_completion_tokens is not None else request.max_tokens, self._extract_prompt_len(engine_input), self.default_sampling_params, self.override_max_tokens, truncate_prompt_tokens=request.truncate_prompt_tokens, ) sampling_params: SamplingParams | BeamSearchParams if request.use_beam_search: sampling_params = request.to_beam_search_params( max_tokens, self.default_sampling_params ) else: sampling_params = request.to_sampling_params( max_tokens, self.default_sampling_params, ) self._log_inputs( sub_request_id, engine_input, params=sampling_params, lora_request=lora_request, ) trace_headers = ( None if raw_request is None else await self._get_trace_headers(raw_request.headers) ) if isinstance(sampling_params, BeamSearchParams): generator = self.beam_search( prompt=engine_input, request_id=sub_request_id, params=sampling_params, lora_request=lora_request, trace_headers=trace_headers, ) else: if not request.include_reasoning: reasoning_ended = True elif request._grammar_from_tool_parser: # The Mistral grammar already includes an optional # `think?` rule that handles both reasoning and # non-reasoning outputs. reasoning_ended = True elif parser is not None and parser.reasoning_parser is not None: reasoning_ended = parser.is_reasoning_end(prompt_token_ids or []) else: reasoning_ended = None generator = self.engine_client.generate( engine_input, sampling_params, sub_request_id, lora_request=lora_request, trace_headers=trace_headers, priority=request.priority, data_parallel_rank=data_parallel_rank, reasoning_ended=reasoning_ended, reasoning_parser_kwargs={ "chat_template_kwargs": chat_template_kwargs, } if parser is not None and parser.reasoning_parser is not None else None, ) generators.append(generator) assert len(generators) == 1 (result_generator,) = generators if request.stream: return self.chat_completion_stream_generator( request, result_generator, request_id, model_name, conversation, tokenizer, request_metadata, chat_template_kwargs=chat_template_kwargs, mm_token_counts=mm_token_counts, ) return await self.chat_completion_full_generator( request, result_generator, request_id, model_name, conversation, tokenizer, request_metadata, parser=parser, mm_token_counts=mm_token_counts, ) def get_chat_request_role(self, request: ChatCompletionRequest) -> str: if request.add_generation_prompt: return self.response_role return request.messages[-1]["role"] async def chat_completion_stream_generator( self, request: ChatCompletionRequest, result_generator: AsyncIterator[RequestOutput], request_id: str, model_name: str, conversation: list[ConversationMessage], tokenizer: TokenizerLike, request_metadata: RequestResponseMetadata, chat_template_kwargs: dict[str, Any] | None = None, mm_token_counts: dict[str, int] | None = None, ) -> AsyncGenerator[str, None]: created_time = int(time.time()) chunk_object_type: Final = "chat.completion.chunk" first_iteration = True # Send response for each token for each request.n (index) num_choices = 1 if request.n is None else request.n previous_num_tokens = [0] * num_choices finish_reason_sent = [False] * num_choices num_prompt_tokens = 0 num_cached_tokens = None tools_streamed = [False] * num_choices if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam): tool_choice_function_name = request.tool_choice.function.name else: tool_choice_function_name = None previous_texts = [""] * num_choices try: if self.parser_cls is not None: if tokenizer is None: raise ValueError( "Tokenizer not available when `skip_tokenizer_init=True`" ) parsers: list[Parser | None] = [ self.parser_cls( tokenizer, request.tools, chat_template_kwargs=chat_template_kwargs, model_config=self.model_config, ) for _ in range(num_choices) ] else: parsers = [None] * num_choices except Exception as e: logger.exception("Error in parser creation.") data = self.create_streaming_error_response(e) yield f"data: {data}\n\n" yield "data: [DONE]\n\n" return stream_options = request.stream_options include_usage, include_continuous_usage = should_include_usage( stream_options, self.enable_force_include_usage ) last_res: RequestOutput | None = None try: async for res in result_generator: last_res = res if res.prompt_token_ids is not None: num_prompt_tokens = len(res.prompt_token_ids) if res.encoder_prompt_token_ids is not None: num_prompt_tokens += len(res.encoder_prompt_token_ids) # We need to do it here, because if there are exceptions in # the result_generator, it needs to be sent as the FIRST # response (by the try...catch). if first_iteration: num_cached_tokens = res.num_cached_tokens # Send first response for each request.n (index) with # the role role = self.get_chat_request_role(request) # ``res.prompt`` is the rendered chat-templated prompt prompt_text = res.prompt if request.return_prompt_text else None # NOTE num_choices defaults to 1 so this usually executes # once per request for i in range(num_choices): choice_data = ChatCompletionResponseStreamChoice( index=i, delta=DeltaMessage( role=role, content="", ), logprobs=None, finish_reason=None, ) # return prompt_token_ids at the first chunk ever chunk = ChatCompletionStreamResponse( id=request_id, object=chunk_object_type, created=created_time, choices=[choice_data], model=model_name, prompt_token_ids=( res.prompt_token_ids if request.return_token_ids else None ), prompt_text=prompt_text, ) # if continuous usage stats are requested, add it if include_continuous_usage: chunk.usage = UsageInfo( prompt_tokens=num_prompt_tokens, completion_tokens=0, total_tokens=num_prompt_tokens, ) data = chunk.model_dump_json(exclude_unset=True) yield f"data: {data}\n\n" # Send response to echo the input portion of the # last message if request.echo: last_msg_content: str | list[dict[str, str]] = "" if ( conversation and "content" in conversation[-1] and conversation[-1].get("role") == role ): last_msg_content = conversation[-1]["content"] or "" if last_msg_content: for i in range(num_choices): choice_data = ChatCompletionResponseStreamChoice( index=i, delta=DeltaMessage(content=last_msg_content), logprobs=None, finish_reason=None, ) chunk = ChatCompletionStreamResponse( id=request_id, object=chunk_object_type, created=created_time, choices=[choice_data], model=model_name, ) if include_continuous_usage: chunk.usage = UsageInfo( prompt_tokens=num_prompt_tokens, completion_tokens=0, total_tokens=num_prompt_tokens, ) data = chunk.model_dump_json(exclude_unset=True) yield f"data: {data}\n\n" first_iteration = False for output in res.outputs: i = output.index parser = parsers[i] if finish_reason_sent[i]: continue if request.logprobs and request.top_logprobs is not None: assert output.logprobs is not None, "Did not output logprobs" logprobs = self._create_chat_logprobs( token_ids=output.token_ids, top_logprobs=output.logprobs, tokenizer=tokenizer, num_output_top_logprobs=request.top_logprobs, return_as_token_id=request.return_tokens_as_token_ids, ) else: logprobs = None delta_text = output.text if ( not delta_text and not output.token_ids and not previous_num_tokens[i] ): # Chunked prefill case, don't return empty chunks continue delta_message: DeltaMessage | None if parser is not None: delta_message = parser.parse_delta( delta_text=delta_text, delta_token_ids=as_list(output.token_ids), request=request, prompt_token_ids=res.prompt_token_ids, finished=output.finish_reason is not None, ) if delta_message is not None and delta_message.tool_calls: tools_streamed[i] = True # handle streaming just a content delta (no parsers) else: delta_message = DeltaMessage(content=delta_text) previous_texts[i] += delta_text # set the previous values for the next iteration previous_num_tokens[i] += len(output.token_ids) # if the message delta is None (e.g. because it was a # "control token" for tool calls or the parser otherwise # wasn't ready to send a token, then # get the next token without streaming a chunk # When reasoning is hidden, suppress per-token # metadata (logprobs, token_ids) on every chunk to # prevent leaking reasoning tokens through decoded # token text in logprob entries or raw token IDs. hide_stream_metadata = ( not request.include_reasoning and parser is not None ) if hide_stream_metadata: logprobs = None if delta_message is None: # NOTE: If return_token_ids is enabled, we still need to # send a chunk with token_ids even if delta_message is None # to ensure all tokens are included in the response if output.finish_reason is None and ( not request.return_token_ids or hide_stream_metadata ): continue delta_message = DeltaMessage() # Log streaming delta if output logging is enabled if self.enable_log_outputs and self.request_logger: delta_content_parts = [] if delta_message.content: delta_content_parts.append(delta_message.content) if delta_message.reasoning: reasoning = delta_message.reasoning delta_content_parts.append(f"[reasoning: {reasoning}]") if delta_message.tool_calls: tool_args = "".join( tc.function.arguments for tc in delta_message.tool_calls if tc.function and tc.function.arguments ) if tool_args: delta_content_parts.append(f"[tool_calls: {tool_args}]") if delta_content_parts and self.enable_log_deltas: delta_content = " ".join(delta_content_parts) self.request_logger.log_outputs( request_id=request_id, outputs=delta_content, output_token_ids=as_list(output.token_ids), finish_reason=output.finish_reason, is_streaming=True, delta=True, ) include_token_ids = ( request.return_token_ids and not hide_stream_metadata ) if output.finish_reason is None: # Send token-by-token response for each request.n choice_data = ChatCompletionResponseStreamChoice( index=i, delta=delta_message, logprobs=logprobs, finish_reason=None, token_ids=( as_list(output.token_ids) if include_token_ids else None ), ) # if the model is finished generating else: # check for error finish reason and abort streaming # finish_reason='error' indicates a retryable error self._raise_if_error(output.finish_reason, request_id) # Send the finish response for each request.n only once # In OpenAI's API, when a tool is called, the # finish_reason is: # "tool_calls" for "auto" or "required" tool calls, # and "stop" for named tool calls. if tools_streamed[i] and not tool_choice_function_name: finish_reason_ = "tool_calls" else: finish_reason_ = ( output.finish_reason if output.finish_reason else "stop" ) choice_data = ChatCompletionResponseStreamChoice( index=i, delta=delta_message, logprobs=logprobs, finish_reason=finish_reason_, stop_reason=output.stop_reason, token_ids=( as_list(output.token_ids) if include_token_ids else None ), ) finish_reason_sent[i] = True choice_data = maybe_filter_parallel_tool_calls(choice_data, request) chunk = ChatCompletionStreamResponse( id=request_id, object=chunk_object_type, created=created_time, choices=[choice_data], model=model_name, ) # Stamp the fingerprint on terminal chunks only (those with # finish_reason set). When ``include_usage`` is on, the # trailing usage chunk below overrides this as the true # final message. if ( not include_usage and self.system_fingerprint is not None and choice_data.finish_reason is not None ): chunk.system_fingerprint = self.system_fingerprint # handle usage stats if requested & if continuous if include_continuous_usage: completion_tokens = previous_num_tokens[i] chunk.usage = UsageInfo( prompt_tokens=num_prompt_tokens, completion_tokens=completion_tokens, total_tokens=num_prompt_tokens + completion_tokens, ) data = chunk.model_dump_json(exclude_unset=True) yield f"data: {data}\n\n" # once the final token is handled, if stream_options.include_usage # is sent, send the usage if include_usage: completion_tokens = sum(previous_num_tokens) final_usage = UsageInfo( prompt_tokens=num_prompt_tokens, completion_tokens=completion_tokens, total_tokens=num_prompt_tokens + completion_tokens, ) final_usage.prompt_tokens_details = _make_prompt_tokens_details( self.enable_prompt_tokens_details, num_cached_tokens, mm_token_counts, ) # In streaming, metrics ride on this final usage chunk, which is # only emitted when usage reporting is enabled (i.e. # ``stream_options.include_usage=true`` or # ``--enable-force-include-usage``). stream_per_request_metrics: PerRequestTimingMetrics | None = None if ( self.enable_per_request_metrics # See note in chat_completion_full_generator: suppress for n>1. and (request.n or 1) == 1 ): last_metrics = last_res.metrics if last_res is not None else None stream_per_request_metrics = build_per_request_timing_metrics( last_metrics, completion_tokens ) final_usage_chunk = ChatCompletionStreamResponse( id=request_id, object=chunk_object_type, created=created_time, choices=[], model=model_name, usage=final_usage, system_fingerprint=self.system_fingerprint, metrics=stream_per_request_metrics, ) final_usage_data = final_usage_chunk.model_dump_json( exclude_unset=True, exclude_none=True ) yield f"data: {final_usage_data}\n\n" # report to FastAPI middleware aggregate usage across all choices num_completion_tokens = sum(previous_num_tokens) request_metadata.final_usage_info = UsageInfo( prompt_tokens=num_prompt_tokens, completion_tokens=num_completion_tokens, total_tokens=num_prompt_tokens + num_completion_tokens, ) # Log complete streaming response if output logging is enabled if self.enable_log_outputs and self.request_logger: # Log the complete response for each choice for i in range(num_choices): full_text = ( previous_texts[i] if previous_texts and i < len(previous_texts) else f"" ) self.request_logger.log_outputs( request_id=request_id, outputs=full_text, output_token_ids=None, # Consider also logging all token IDs finish_reason="streaming_complete", is_streaming=True, delta=False, ) except GenerationError as e: yield f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n" except Exception as e: logger.exception("Error in chat completion stream generator.") data = self.create_streaming_error_response(e) yield f"data: {data}\n\n" # Send the final done message after all response.n are finished yield "data: [DONE]\n\n" async def chat_completion_full_generator( self, request: ChatCompletionRequest, result_generator: AsyncIterator[RequestOutput], request_id: str, model_name: str, conversation: list[ConversationMessage], tokenizer: TokenizerLike, request_metadata: RequestResponseMetadata, parser: Parser | None = None, mm_token_counts: dict[str, int] | None = None, ) -> ErrorResponse | ChatCompletionResponse: created_time = int(time.time()) final_res: RequestOutput | None = None try: async for res in result_generator: final_res = res except asyncio.CancelledError: return self.create_error_response("Client disconnected") if final_res is None: return self.create_error_response( "No output received from the engine.", err_type="InternalServerError", status_code=HTTPStatus.INTERNAL_SERVER_ERROR, ) choices: list[ChatCompletionResponseChoice] = [] role = self.get_chat_request_role(request) tool_parser_cls = ( self.parser_cls.tool_parser_cls if self.parser_cls is not None else None ) for output in final_res.outputs: # check for error finish reason and raise GenerationError # finish_reason='error' indicates a retryable request-level internal error self._raise_if_error(output.finish_reason, request_id) token_ids = output.token_ids out_logprobs = output.logprobs if request.logprobs and request.top_logprobs is not None: assert out_logprobs is not None, "Did not output logprobs" logprobs = self._create_chat_logprobs( token_ids=token_ids, top_logprobs=out_logprobs, num_output_top_logprobs=request.top_logprobs, tokenizer=tokenizer, return_as_token_id=request.return_tokens_as_token_ids, ) else: logprobs = None if parser is not None: reasoning, content, tool_calls = parser.parse( output.text, request, enable_auto_tools=self.enable_auto_tools, model_output_token_ids=token_ids, ) suppress_metadata = not request.include_reasoning and parser is not None if not request.include_reasoning: reasoning = None if suppress_metadata: logprobs = None else: reasoning = None content = output.text tool_calls = [] suppress_metadata = False auto_tools_called = False is_named_tool_choice = ( request.tool_choice is not None and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam ) is_required_tool_choice = request.tool_choice == "required" if (not self.enable_auto_tools or not tool_parser_cls) and ( not is_named_tool_choice and not is_required_tool_choice ): message = ChatMessage(role=role, reasoning=reasoning, content=content) elif is_named_tool_choice or is_required_tool_choice: message = ChatMessage( role=role, reasoning=reasoning, content=content or "", tool_calls=[ ToolCall(id=tc.id or make_tool_call_id(), function=tc) for tc in (tool_calls or []) ], ) # if the request doesn't use tool choice # OR specifies to not use a tool elif not request.tool_choice or request.tool_choice == "none": message = ChatMessage(role=role, reasoning=reasoning, content=content) # handle when there are tools and tool choice is auto elif ( request.tools and (request.tool_choice == "auto" or request.tool_choice is None) and self.enable_auto_tools and tool_parser_cls ): auto_tools_called = tool_calls is not None and len(tool_calls) > 0 if tool_calls: message = ChatMessage( role=role, reasoning=reasoning, content=content, tool_calls=[ ToolCall(id=tc.id or make_tool_call_id(), function=tc) for tc in tool_calls ], ) else: message = ChatMessage( role=role, reasoning=reasoning, content=content, ) # undetermined case that is still important to handle else: logger.error( "Error in chat_completion_full_generator - cannot determine" " if tools should be extracted. Returning a standard chat " "completion." ) message = ChatMessage(role=role, reasoning=reasoning, content=content) # In OpenAI's API, when a tool is called, the finish_reason is: # "tool_calls" for "auto" or "required" tool calls, # and "stop" for named tool calls. is_finish_reason_tool_calls = auto_tools_called or ( request.tool_choice and request.tool_choice == "required" and output.finish_reason == "stop" ) # Encode routed_experts for transport. JSON can't carry raw # bytes, so we write the ndarray as a ``.npy`` byte stream # and base64-encode it. ``pybase64`` is ~3x faster than the # stdlib ``base64`` on large payloads thanks to SIMD. routed_experts_b64 = None if output.routed_experts is not None: buf = io.BytesIO() np.save(buf, output.routed_experts) routed_experts_b64 = base64.b64encode(buf.getvalue()).decode("ascii") choice_data = ChatCompletionResponseChoice( index=output.index, message=message, logprobs=logprobs, finish_reason="tool_calls" if is_finish_reason_tool_calls else output.finish_reason if output.finish_reason else "stop", stop_reason=output.stop_reason, token_ids=( as_list(output.token_ids) if request.return_token_ids and not suppress_metadata else None ), routed_experts=routed_experts_b64, ) choice_data = maybe_filter_parallel_tool_calls(choice_data, request) choices.append(choice_data) if request.echo: last_msg_content: str | list[dict[str, str]] = "" if ( conversation and "content" in conversation[-1] and conversation[-1].get("role") == role ): last_msg_content = conversation[-1]["content"] or "" if isinstance(last_msg_content, list): last_msg_content = "\n".join(msg["text"] for msg in last_msg_content) for choice in choices: full_message = last_msg_content + (choice.message.content or "") choice.message.content = full_message assert final_res.prompt_token_ids is not None num_prompt_tokens = len(final_res.prompt_token_ids) if final_res.encoder_prompt_token_ids is not None: num_prompt_tokens += len(final_res.encoder_prompt_token_ids) num_generated_tokens = sum( len(output.token_ids) for output in final_res.outputs ) usage = UsageInfo( prompt_tokens=num_prompt_tokens, completion_tokens=num_generated_tokens, total_tokens=num_prompt_tokens + num_generated_tokens, ) usage.prompt_tokens_details = _make_prompt_tokens_details( self.enable_prompt_tokens_details, final_res.num_cached_tokens, mm_token_counts, ) request_metadata.final_usage_info = usage per_request_metrics: PerRequestTimingMetrics | None = None if ( self.enable_per_request_metrics # Timing metrics describe a single generation stream. For n>1 the # returned stats belong to only one of the n sequences, so they # cannot be accurately attributed to the request; suppress instead. and (request.n or 1) == 1 ): per_request_metrics = build_per_request_timing_metrics( final_res.metrics, num_generated_tokens ) # ``final_res.prompt`` is the rendered chat-templated prompt text prompt_text = final_res.prompt if request.return_prompt_text else None response = ChatCompletionResponse( id=request_id, created=created_time, model=model_name, choices=choices, usage=usage, system_fingerprint=self.system_fingerprint, prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs), prompt_token_ids=( final_res.prompt_token_ids if request.return_token_ids else None ), prompt_text=prompt_text, kv_transfer_params=final_res.kv_transfer_params, ec_transfer_params=final_res.ec_transfer_params, metrics=per_request_metrics, ) # Log complete response if output logging is enabled if self.enable_log_outputs and self.request_logger: for choice in choices: output_text = "" if choice.message.content: output_text = choice.message.content elif choice.message.tool_calls: # For tool calls, log the function name and arguments tool_call_descriptions = [] for tc in choice.message.tool_calls: # type: ignore function_call: FunctionCall = tc.function # type: ignore tool_call_descriptions.append( f"{function_call.name}({function_call.arguments})" ) tool_calls_str = ", ".join(tool_call_descriptions) output_text = f"[tool_calls: {tool_calls_str}]" if output_text: # Get the corresponding output token IDs output_token_ids = None if choice.index < len(final_res.outputs): output_token_ids = final_res.outputs[choice.index].token_ids self.request_logger.log_outputs( request_id=request_id, outputs=output_text, output_token_ids=output_token_ids, finish_reason=choice.finish_reason, is_streaming=False, delta=False, ) return response def _get_top_logprobs( self, logprobs: dict[int, Logprob], top_logprobs: int | None, tokenizer: TokenizerLike | None, should_return_as_token_id: bool, ) -> list[ChatCompletionLogProb]: return [ ChatCompletionLogProb( token=( token := self._get_decoded_token( p[1], p[0], tokenizer, return_as_token_id=should_return_as_token_id, ) ), logprob=max(p[1].logprob, -9999.0), bytes=list(token.encode("utf-8", errors="replace")), ) for i, p in enumerate(logprobs.items()) if (top_logprobs and i < top_logprobs or top_logprobs == -1) ] def _create_chat_logprobs( self, token_ids: GenericSequence[int], top_logprobs: GenericSequence[dict[int, Logprob] | None], tokenizer: TokenizerLike | None, num_output_top_logprobs: int | None = None, return_as_token_id: bool | None = None, ) -> ChatCompletionLogProbs: """Create OpenAI-style logprobs.""" logprobs_content: list[ChatCompletionLogProbsContent] = [] should_return_as_token_id = ( return_as_token_id if return_as_token_id is not None else self.return_tokens_as_token_ids ) for i, token_id in enumerate(token_ids): step_top_logprobs = top_logprobs[i] if step_top_logprobs is None or step_top_logprobs.get(token_id) is None: if should_return_as_token_id: token = format_token_id_placeholder(token_id) else: if tokenizer is None: raise ValueError( "Unable to get tokenizer because `skip_tokenizer_init=True`" ) token = tokenizer.decode(token_id) logprobs_content.append( ChatCompletionLogProbsContent( token=token, bytes=list(token.encode("utf-8", errors="replace")), ) ) else: step_token = step_top_logprobs[token_id] step_decoded = step_token.decoded_token logprobs_content.append( ChatCompletionLogProbsContent( token=self._get_decoded_token( step_token, token_id, tokenizer, should_return_as_token_id, ), logprob=max(step_token.logprob, -9999.0), bytes=( None if step_decoded is None else list(step_decoded.encode("utf-8", errors="replace")) ), top_logprobs=self._get_top_logprobs( step_top_logprobs, num_output_top_logprobs, tokenizer, should_return_as_token_id, ), ) ) return ChatCompletionLogProbs(content=logprobs_content)