# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Prometheus metrics for the parsers.""" from __future__ import annotations from enum import Enum from itertools import product from typing import cast from prometheus_client import REGISTRY, Counter _model_name: str | None = None _TOOL_CALL_PARSER_INVOCATIONS_TOTAL = "vllm:tool_call_parser_invocations_total" _tool_call_parser_invocations: Counter | None = None class ToolCallOutcome(Enum): TOOL_CALL = "tool_call" NO_TOOL_CALL = "no_tool_call" class RequestType(Enum): CHAT_COMPLETIONS = "chat_completions" RESPONSES = "responses" OTHER = "other" def init_parser_metrics(*, model_name: str) -> None: """Lazily register parser metrics and cache the shared model label.""" global _model_name _model_name = model_name global _tool_call_parser_invocations try: _tool_call_parser_invocations = Counter( name=_TOOL_CALL_PARSER_INVOCATIONS_TOTAL, documentation=( "Total number of ToolParser invocations. " "Non-streaming increments once per choice; " "streaming increments once per delta." ), labelnames=["model_name", "mode", "outcome", "request_type"], ) except ValueError: _tool_call_parser_invocations = cast( Counter, REGISTRY._names_to_collectors[_TOOL_CALL_PARSER_INVOCATIONS_TOTAL], ) for mode, outcome, request_type in product( ("streaming", "non_streaming"), ToolCallOutcome, RequestType, ): _tool_call_parser_invocations.labels( model_name=_model_name, mode=mode, outcome=outcome.value, request_type=request_type.value, ) def record_tool_parser_invocation( *, is_tool_called: bool | Exception, is_streaming: bool, request: object, ) -> None: """Increment the tool-call parser invocation counter when registered. Currently parser failures are treated as no tool calls. TODO: To accurately track parser failures, add a new ToolCallOutcome and more importantly, ensure exceptions are propagated out of the ToolParsers instead of being caught internally. This would require going through ToolParser implementation on a case-by-case basis. """ if _tool_call_parser_invocations is None: return from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.responses.protocol import ResponsesRequest match request: case ChatCompletionRequest(): request_type = RequestType.CHAT_COMPLETIONS case ResponsesRequest(): request_type = RequestType.RESPONSES case _: request_type = RequestType.OTHER match is_tool_called: case bool(): outcome = ( ToolCallOutcome.TOOL_CALL if is_tool_called else ToolCallOutcome.NO_TOOL_CALL ) case _: outcome = ToolCallOutcome.NO_TOOL_CALL _tool_call_parser_invocations.labels( model_name=_model_name, mode="streaming" if is_streaming else "non_streaming", outcome=outcome.value, request_type=request_type.value, ).inc()