chore: import upstream snapshot with attribution

vllm/entrypoints/anthropic/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

vllm/entrypoints/anthropic/api_router.py

@@ -0,0 +1,137 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+
+from http import HTTPStatus
+
+from fastapi import APIRouter, Depends, FastAPI, Request
+from fastapi.responses import JSONResponse, StreamingResponse
+
+from vllm.entrypoints.anthropic.protocol import (
+    AnthropicCountTokensRequest,
+    AnthropicCountTokensResponse,
+    AnthropicError,
+    AnthropicErrorResponse,
+    AnthropicMessagesRequest,
+    AnthropicMessagesResponse,
+)
+from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
+from vllm.entrypoints.serve.utils.api_utils import (
+    load_aware_call,
+    sanitize_message,
+    validate_json_request,
+    with_cancellation,
+)
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+router = APIRouter()
+
+
+def messages(request: Request) -> AnthropicServingMessages:
+    return request.app.state.anthropic_serving_messages
+
+
+def translate_error_response(response: ErrorResponse) -> JSONResponse:
+    anthropic_error = AnthropicErrorResponse(
+        error=AnthropicError(
+            type=response.error.type,
+            message=response.error.message,
+        )
+    )
+    return JSONResponse(
+        status_code=response.error.code, content=anthropic_error.model_dump()
+    )
+
+
+@router.post(
+    "/v1/messages",
+    dependencies=[Depends(validate_json_request)],
+    responses={
+        HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
+        HTTPStatus.BAD_REQUEST.value: {"model": AnthropicErrorResponse},
+        HTTPStatus.NOT_FOUND.value: {"model": AnthropicErrorResponse},
+        HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": AnthropicErrorResponse},
+    },
+)
+@with_cancellation
+@load_aware_call
+async def create_messages(request: AnthropicMessagesRequest, raw_request: Request):
+    handler = messages(raw_request)
+    if handler is None:
+        base_server = raw_request.app.state.serving_tokenization
+        error = base_server.create_error_response(
+            NotImplementedError("The model does not support Messages API")
+        )
+        return translate_error_response(error)
+
+    try:
+        generator = await handler.create_messages(request, raw_request)
+    except Exception as e:
+        logger.exception("Error in create_messages: %s", e)
+        return JSONResponse(
+            status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
+            content=AnthropicErrorResponse(
+                error=AnthropicError(
+                    type="internal_error",
+                    message=sanitize_message(str(e)),
+                )
+            ).model_dump(),
+        )
+
+    if isinstance(generator, ErrorResponse):
+        return translate_error_response(generator)
+
+    elif isinstance(generator, AnthropicMessagesResponse):
+        resp = generator.model_dump(exclude_none=True)
+        logger.debug("Anthropic Messages Response: %s", resp)
+        return JSONResponse(content=resp)
+
+    return StreamingResponse(content=generator, media_type="text/event-stream")
+
+
+@router.post(
+    "/v1/messages/count_tokens",
+    dependencies=[Depends(validate_json_request)],
+    responses={
+        HTTPStatus.OK.value: {"model": AnthropicCountTokensResponse},
+        HTTPStatus.BAD_REQUEST.value: {"model": AnthropicErrorResponse},
+        HTTPStatus.NOT_FOUND.value: {"model": AnthropicErrorResponse},
+        HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": AnthropicErrorResponse},
+    },
+)
+@with_cancellation
+@load_aware_call
+async def count_tokens(request: AnthropicCountTokensRequest, raw_request: Request):
+    handler = messages(raw_request)
+    if handler is None:
+        base_server = raw_request.app.state.serving_tokenization
+        error = base_server.create_error_response(
+            NotImplementedError("The model does not support Messages API")
+        )
+        return translate_error_response(error)
+
+    try:
+        response = await handler.count_tokens(request, raw_request)
+    except Exception as e:
+        logger.exception("Error in count_tokens: %s", e)
+        return JSONResponse(
+            status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
+            content=AnthropicErrorResponse(
+                error=AnthropicError(
+                    type="internal_error",
+                    message=sanitize_message(str(e)),
+                )
+            ).model_dump(),
+        )
+
+    if isinstance(response, ErrorResponse):
+        return translate_error_response(response)
+
+    return JSONResponse(content=response.model_dump(exclude_none=True))
+
+
+def attach_router(app: FastAPI):
+    app.include_router(router)

vllm/entrypoints/anthropic/protocol.py

@@ -0,0 +1,272 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Pydantic models for Anthropic API protocol"""
+
+import time
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field, field_validator, model_validator
+
+
+class AnthropicError(BaseModel):
+    """Error structure for Anthropic API"""
+
+    type: str
+    message: str
+
+
+class AnthropicErrorResponse(BaseModel):
+    """Error response structure for Anthropic API"""
+
+    type: Literal["error"] = "error"
+    error: AnthropicError
+
+
+class AnthropicUsage(BaseModel):
+    """Token usage information"""
+
+    input_tokens: int
+    output_tokens: int
+    cache_creation_input_tokens: int | None = None
+    cache_read_input_tokens: int | None = None
+
+
+class AnthropicContentBlock(BaseModel):
+    """Content block in message"""
+
+    type: Literal[
+        "text",
+        "image",
+        "tool_use",
+        "tool_result",
+        "tool_reference",
+        "thinking",
+        "redacted_thinking",
+    ]
+    text: str | None = None
+    # For image content
+    source: dict[str, Any] | None = None
+    # For tool use/result
+    id: str | None = None
+    tool_use_id: str | None = None
+    name: str | None = None
+    input: dict[str, Any] | None = None
+    content: str | list[dict[str, Any]] | None = None
+    is_error: bool | None = None
+    # For tool_reference content
+    tool_name: str | None = None
+    # For thinking content
+    thinking: str | None = None
+    signature: str | None = None
+    # For redacted thinking content (safety-filtered by the API)
+    data: str | None = None
+
+
+class AnthropicMessage(BaseModel):
+    """Message structure"""
+
+    role: Literal["user", "assistant", "system"]
+    content: str | list[AnthropicContentBlock]
+
+
+class AnthropicTool(BaseModel):
+    """Tool definition"""
+
+    name: str
+    description: str | None = None
+    input_schema: dict[str, Any]
+    strict: bool | None = None
+    defer_loading: bool | None = None
+
+    @field_validator("input_schema")
+    @classmethod
+    def validate_input_schema(cls, v):
+        if not isinstance(v, dict):
+            raise ValueError("input_schema must be a dictionary")
+        if "type" not in v:
+            v["type"] = "object"  # Default to object type
+        return v
+
+
+class AnthropicToolChoice(BaseModel):
+    """Tool Choice definition"""
+
+    type: Literal["auto", "any", "tool", "none"]
+    name: str | None = None
+
+    @model_validator(mode="after")
+    def validate_name_required_for_tool(self) -> "AnthropicToolChoice":
+        if self.type == "tool" and not self.name:
+            raise ValueError("tool_choice.name is required when type is 'tool'")
+        return self
+
+
+class AnthropicJsonOutputFormat(BaseModel):
+    """JSON output format configuration"""
+
+    json_schema: dict[str, Any] | None = Field(default=None, alias="schema")
+    type: Literal["json_schema"] = "json_schema"
+
+
+class AnthropicOutputConfig(BaseModel):
+    """Configuration options for the model's output, such as the output format."""
+
+    effort: Literal["low", "medium", "high", "xhigh", "max"] | None = None
+    format: AnthropicJsonOutputFormat | None = None
+
+
+class AnthropicMessagesRequest(BaseModel):
+    """Anthropic Messages API request"""
+
+    model: str
+    messages: list[AnthropicMessage]
+    max_tokens: int
+    metadata: dict[str, Any] | None = None
+    output_config: AnthropicOutputConfig | None = None
+    stop_sequences: list[str] | None = None
+    stream: bool | None = False
+    system: str | list[AnthropicContentBlock] | None = None
+    temperature: float | None = None
+    tool_choice: AnthropicToolChoice | None = None
+    tools: list[AnthropicTool] | None = None
+    top_k: int | None = None
+    top_p: float | None = None
+
+    # vLLM-specific fields that are not in Anthropic spec
+    kv_transfer_params: dict[str, Any] | None = Field(
+        default=None,
+        description="KVTransfer parameters used for disaggregated serving.",
+    )
+    ec_transfer_params: dict[str, Any] | None = Field(
+        default=None,
+        description=(
+            "ECTransfer parameters used for encoder-cache disaggregated serving."
+        ),
+    )
+    chat_template_kwargs: dict[str, Any] | None = Field(
+        default=None,
+        description=(
+            "Additional keyword args to pass to the chat template renderer. "
+            "Will be accessible by the template."
+        ),
+    )
+
+    @field_validator("model")
+    @classmethod
+    def validate_model(cls, v):
+        if not v:
+            raise ValueError("Model is required")
+        return v
+
+    @field_validator("max_tokens")
+    @classmethod
+    def validate_max_tokens(cls, v):
+        if v <= 0:
+            raise ValueError("max_tokens must be positive")
+        return v
+
+
+class AnthropicDelta(BaseModel):
+    """Delta for streaming responses"""
+
+    type: (
+        Literal["text_delta", "input_json_delta", "thinking_delta", "signature_delta"]
+        | None
+    ) = None
+    text: str | None = None
+    thinking: str | None = None
+    partial_json: str | None = None
+    signature: str | None = None
+
+    # Message delta
+    stop_reason: (
+        Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"] | None
+    ) = None
+    stop_sequence: str | None = None
+
+
+class AnthropicStreamEvent(BaseModel):
+    """Streaming event"""
+
+    type: Literal[
+        "message_start",
+        "message_delta",
+        "message_stop",
+        "content_block_start",
+        "content_block_delta",
+        "content_block_stop",
+        "ping",
+        "error",
+    ]
+    message: "AnthropicMessagesResponse | None" = None
+    delta: AnthropicDelta | None = None
+    content_block: AnthropicContentBlock | None = None
+    index: int | None = None
+    error: AnthropicError | None = None
+    usage: AnthropicUsage | None = None
+
+
+class AnthropicMessagesResponse(BaseModel):
+    """Anthropic Messages API response"""
+
+    id: str
+    type: Literal["message"] = "message"
+    role: Literal["assistant"] = "assistant"
+    content: list[AnthropicContentBlock]
+    model: str
+    stop_reason: (
+        Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"] | None
+    ) = None
+    stop_sequence: str | None = None
+    usage: AnthropicUsage | None = None
+
+    # vLLM-specific fields that are not in Anthropic spec
+    kv_transfer_params: dict[str, Any] | None = Field(
+        default=None, description="KVTransfer parameters."
+    )
+    ec_transfer_params: dict[str, Any] | None = Field(
+        default=None, description="ECTransfer parameters."
+    )
+
+    def model_post_init(self, __context):
+        if not self.id:
+            self.id = f"msg_{int(time.time() * 1000)}"
+
+
+class AnthropicContextManagement(BaseModel):
+    """Context management information for token counting."""
+
+    original_input_tokens: int
+
+
+class AnthropicCountTokensRequest(BaseModel):
+    """Anthropic messages.count_tokens request"""
+
+    model: str
+    messages: list[AnthropicMessage]
+    system: str | list[AnthropicContentBlock] | None = None
+    tool_choice: AnthropicToolChoice | None = None
+    tools: list[AnthropicTool] | None = None
+
+    # vLLM-specific fields that are not in Anthropic spec
+    chat_template_kwargs: dict[str, Any] | None = Field(
+        default=None,
+        description=(
+            "Additional keyword args to pass to the chat template renderer. "
+            "Will be accessible by the template."
+        ),
+    )
+
+    @field_validator("model")
+    @classmethod
+    def validate_model(cls, v):
+        if not v:
+            raise ValueError("Model is required")
+        return v
+
+
+class AnthropicCountTokensResponse(BaseModel):
+    """Anthropic messages.count_tokens response"""
+
+    input_tokens: int
+    context_management: AnthropicContextManagement | None = None

vllm/entrypoints/cli/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

vllm/entrypoints/cli/benchmark/base.py

@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import argparse
+
+from vllm.entrypoints.cli.types import CLISubcommand
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+
+
+class BenchmarkSubcommandBase(CLISubcommand):
+    """The base class of subcommands for `vllm bench`."""
+
+    help: str
+
+    @classmethod
+    def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
+        """Add the CLI arguments to the parser."""
+        raise NotImplementedError
+
+    @staticmethod
+    def cmd(args: argparse.Namespace) -> None:
+        """Run the benchmark.
+
+        Args:
+            args: The arguments to the command.
+        """
+        raise NotImplementedError

vllm/entrypoints/cli/benchmark/latency.py

@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import argparse
+
+from vllm.benchmarks.latency import add_cli_args, main
+from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+
+
+class BenchmarkLatencySubcommand(BenchmarkSubcommandBase):
+    """The `latency` subcommand for `vllm bench`."""
+
+    name = "latency"
+    help = "Benchmark the latency of a single batch of requests."
+
+    @classmethod
+    def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
+        add_cli_args(parser)
+
+    @staticmethod
+    def cmd(args: argparse.Namespace) -> None:
+        main(args)

vllm/entrypoints/cli/benchmark/main.py

@@ -0,0 +1,79 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import argparse
+import sys
+import typing
+
+from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
+from vllm.entrypoints.cli.types import CLISubcommand
+from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
+
+if typing.TYPE_CHECKING:
+    from vllm.utils.argparse_utils import FlexibleArgumentParser
+else:
+    FlexibleArgumentParser = argparse.ArgumentParser
+
+
+def _import_bench_subcommand_modules() -> None:
+    # Imported lazily so `BenchmarkSubcommandBase` subclasses register only
+    # when `vllm bench` is actually invoked.
+    import vllm.entrypoints.cli.benchmark.latency  # noqa: F401
+    import vllm.entrypoints.cli.benchmark.mm_processor  # noqa: F401
+    import vllm.entrypoints.cli.benchmark.serve  # noqa: F401
+    import vllm.entrypoints.cli.benchmark.startup  # noqa: F401
+    import vllm.entrypoints.cli.benchmark.sweep  # noqa: F401
+    import vllm.entrypoints.cli.benchmark.throughput  # noqa: F401
+
+
+class BenchmarkSubcommand(CLISubcommand):
+    """The `bench` subcommand for the vLLM CLI."""
+
+    name = "bench"
+    help = "vLLM bench subcommand."
+
+    @staticmethod
+    def cmd(args: argparse.Namespace) -> None:
+        args.dispatch_function(args)
+
+    def validate(self, args: argparse.Namespace) -> None:
+        pass
+
+    def subparser_init(
+        self, subparsers: argparse._SubParsersAction
+    ) -> FlexibleArgumentParser:
+        bench_parser = subparsers.add_parser(
+            self.name,
+            help=self.help,
+            description=self.help,
+            usage=f"vllm {self.name} <bench_type> [options]",
+        )
+        bench_subparsers = bench_parser.add_subparsers(required=True, dest="bench_type")
+
+        # Only build the nested bench subparsers when the user is actually
+        # invoking `bench`; otherwise we'd drag in imports
+        # unnecessarily on every `vllm --help` and `vllm serve`.
+        # Scan for the first positional arg so global flags (e.g. `-v`)
+        # before the subcommand don't break detection.
+        first_positional = next(
+            (arg for arg in sys.argv[1:] if not arg.startswith("-")), None
+        )
+        if first_positional == self.name:
+            _import_bench_subcommand_modules()
+            for cmd_cls in BenchmarkSubcommandBase.__subclasses__():
+                cmd_subparser = bench_subparsers.add_parser(
+                    cmd_cls.name,
+                    help=cmd_cls.help,
+                    description=cmd_cls.help,
+                    usage=f"vllm {self.name} {cmd_cls.name} [options]",
+                )
+                cmd_subparser.set_defaults(dispatch_function=cmd_cls.cmd)
+                cmd_cls.add_cli_args(cmd_subparser)
+                cmd_subparser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(
+                    subcmd=f"{self.name} {cmd_cls.name}"
+                )
+        return bench_parser
+
+
+def cmd_init() -> list[CLISubcommand]:
+    return [BenchmarkSubcommand()]

vllm/entrypoints/cli/benchmark/mm_processor.py

@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import argparse
+
+from vllm.benchmarks.mm_processor import add_cli_args, main
+from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+
+
+class BenchmarkMMProcessorSubcommand(BenchmarkSubcommandBase):
+    """The `mm-processor` subcommand for `vllm bench`."""
+
+    name = "mm-processor"
+    help = "Benchmark multimodal processor latency across different configurations."
+
+    @classmethod
+    def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
+        add_cli_args(parser)
+
+    @staticmethod
+    def cmd(args: argparse.Namespace) -> None:
+        main(args)

vllm/entrypoints/cli/benchmark/serve.py

@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import argparse
+
+from vllm.benchmarks.serve import add_cli_args, main
+from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+
+
+class BenchmarkServingSubcommand(BenchmarkSubcommandBase):
+    """The `serve` subcommand for `vllm bench`."""
+
+    name = "serve"
+    help = "Benchmark the online serving throughput."
+
+    @classmethod
+    def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
+        add_cli_args(parser)
+
+    @staticmethod
+    def cmd(args: argparse.Namespace) -> None:
+        main(args)

vllm/entrypoints/cli/benchmark/startup.py

@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import argparse
+
+from vllm.benchmarks.startup import add_cli_args, main
+from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+
+
+class BenchmarkStartupSubcommand(BenchmarkSubcommandBase):
+    """The `startup` subcommand for `vllm bench`."""
+
+    name = "startup"
+    help = "Benchmark the startup time of vLLM models."
+
+    @classmethod
+    def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
+        add_cli_args(parser)
+
+    @staticmethod
+    def cmd(args: argparse.Namespace) -> None:
+        main(args)

vllm/entrypoints/cli/benchmark/sweep.py

@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import argparse
+
+from vllm.benchmarks.sweep.cli import add_cli_args, main
+from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+
+
+class BenchmarkSweepSubcommand(BenchmarkSubcommandBase):
+    """The `sweep` subcommand for `vllm bench`."""
+
+    name = "sweep"
+    help = "Benchmark for a parameter sweep."
+
+    @classmethod
+    def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
+        add_cli_args(parser)
+
+    @staticmethod
+    def cmd(args: argparse.Namespace) -> None:
+        main(args)

vllm/entrypoints/cli/benchmark/throughput.py

@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import argparse
+
+from vllm.benchmarks.throughput import add_cli_args, main
+from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+
+
+class BenchmarkThroughputSubcommand(BenchmarkSubcommandBase):
+    """The `throughput` subcommand for `vllm bench`."""
+
+    name = "throughput"
+    help = "Benchmark offline inference throughput."
+
+    @classmethod
+    def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
+        add_cli_args(parser)
+
+    @staticmethod
+    def cmd(args: argparse.Namespace) -> None:
+        main(args)

vllm/entrypoints/cli/collect_env.py

@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import argparse
+import typing
+
+from vllm.collect_env import main as collect_env_main
+from vllm.entrypoints.cli.types import CLISubcommand
+
+if typing.TYPE_CHECKING:
+    from vllm.utils.argparse_utils import FlexibleArgumentParser
+else:
+    FlexibleArgumentParser = argparse.ArgumentParser
+
+
+class CollectEnvSubcommand(CLISubcommand):
+    """The `collect-env` subcommand for the vLLM CLI."""
+
+    name = "collect-env"
+
+    @staticmethod
+    def cmd(args: argparse.Namespace) -> None:
+        """Collect information about the environment."""
+        collect_env_main()
+
+    def subparser_init(
+        self, subparsers: argparse._SubParsersAction
+    ) -> FlexibleArgumentParser:
+        return subparsers.add_parser(
+            "collect-env",
+            help="Start collecting environment information.",
+            description="Start collecting environment information.",
+            usage="vllm collect-env",
+        )
+
+
+def cmd_init() -> list[CLISubcommand]:
+    return [CollectEnvSubcommand()]

vllm/entrypoints/cli/launch.py

@@ -0,0 +1,144 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import argparse
+import signal
+
+import uvloop
+
+from vllm import envs
+from vllm.config import VllmConfig
+from vllm.engine.arg_utils import AsyncEngineArgs
+from vllm.entrypoints.cli.types import CLISubcommand
+from vllm.entrypoints.openai.api_server import (
+    build_and_serve_renderer,
+    setup_server,
+)
+from vllm.entrypoints.openai.cli_args import (
+    make_arg_parser,
+    validate_parsed_serve_args,
+)
+from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
+from vllm.logger import init_logger
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+
+logger = init_logger(__name__)
+
+DESCRIPTION = "Launch individual vLLM components."
+
+
+class LaunchSubcommandBase(CLISubcommand):
+    """The base class of subcommands for `vllm launch`."""
+
+    help: str
+
+    @classmethod
+    def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
+        """Add the CLI arguments to the parser.
+
+        By default, adds the standard vLLM serving arguments.
+        Subclasses can override to add component-specific arguments.
+        """
+        make_arg_parser(parser)
+
+    @staticmethod
+    def cmd(args: argparse.Namespace) -> None:
+        raise NotImplementedError
+
+
+class RenderSubcommand(LaunchSubcommandBase):
+    """The `render` subcommand for `vllm launch`."""
+
+    name = "render"
+    help = "Launch a GPU-less rendering server (preprocessing and postprocessing only)."
+
+    @staticmethod
+    def cmd(args: argparse.Namespace) -> None:
+        uvloop.run(run_launch_fastapi(args))
+
+
+class LaunchSubcommand(CLISubcommand):
+    """The `launch` subcommand for the vLLM CLI.
+
+    Uses nested sub-subcommands so each component can define its own
+    arguments independently (e.g. ``vllm launch render``).
+    """
+
+    name = "launch"
+
+    @staticmethod
+    def cmd(args: argparse.Namespace) -> None:
+        if hasattr(args, "model_tag") and args.model_tag is not None:
+            args.model = args.model_tag
+
+        args.launch_command(args)
+
+    def validate(self, args: argparse.Namespace) -> None:
+        validate_parsed_serve_args(args)
+
+    def subparser_init(
+        self, subparsers: argparse._SubParsersAction
+    ) -> FlexibleArgumentParser:
+        launch_parser = subparsers.add_parser(
+            self.name,
+            help=DESCRIPTION,
+            description=DESCRIPTION,
+            usage=f"vllm {self.name} <component> [options]",
+        )
+        launch_subparsers = launch_parser.add_subparsers(
+            required=True, dest="launch_component"
+        )
+
+        for cmd_cls in LaunchSubcommandBase.__subclasses__():
+            cmd_subparser = launch_subparsers.add_parser(
+                cmd_cls.name,
+                help=cmd_cls.help,
+                description=cmd_cls.help,
+                usage=f"vllm {self.name} {cmd_cls.name} [options]",
+            )
+            cmd_subparser.set_defaults(launch_command=cmd_cls.cmd)
+            cmd_cls.add_cli_args(cmd_subparser)
+            cmd_subparser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(
+                subcmd=f"{self.name} {cmd_cls.name}"
+            )
+
+        return launch_parser
+
+
+def cmd_init() -> list[CLISubcommand]:
+    return [LaunchSubcommand()]
+
+
+async def run_launch_fastapi(args: argparse.Namespace) -> None:
+    """Run the online serving layer with FastAPI (no GPU inference)."""
+
+    # Interrupt initialization if SIGTERM arrives before uvicorn installs
+    # its own signal handlers. Once uvicorn is running it replaces this.
+    def _interrupt_init(*_) -> None:
+        raise KeyboardInterrupt("terminated")
+
+    signal.signal(signal.SIGTERM, _interrupt_init)
+
+    # 1. Socket binding
+    listen_address, sock = setup_server(args, reuse_port=False)
+
+    # 2. Build and serve the API server
+    engine_args = AsyncEngineArgs.from_cli_args(args)
+    model_config = engine_args.create_model_config()
+
+    # Render servers preprocess data only — no inference, no quantized kernels.
+    # Clear quantization so VllmConfig skips quant dtype/capability validation.
+    model_config.quantization = None
+
+    # Render servers never allocate KV cache; suppress the spurious CPU KV
+    # cache space warning from CpuPlatform.check_and_update_config.
+    envs.VLLM_CPU_KVCACHE_SPACE = 0
+
+    vllm_config = VllmConfig(model_config=model_config)
+    shutdown_task = await build_and_serve_renderer(
+        vllm_config, listen_address, sock, args
+    )
+    try:
+        await shutdown_task
+    finally:
+        sock.close()

vllm/entrypoints/cli/main.py

@@ -0,0 +1,101 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""The CLI entrypoints of vLLM
+
+Note that all future modules must be lazily loaded within main
+to avoid certain eager import breakage."""
+
+import importlib.metadata
+import sys
+from importlib.util import find_spec
+
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+
+def main():
+    import vllm.entrypoints.cli.benchmark.main
+    import vllm.entrypoints.cli.collect_env
+    import vllm.entrypoints.cli.launch
+    import vllm.entrypoints.cli.openai
+    import vllm.entrypoints.cli.run_batch
+    import vllm.entrypoints.cli.serve
+    from vllm.entrypoints.serve.utils.api_utils import (
+        VLLM_SUBCMD_PARSER_EPILOG,
+        cli_env_setup,
+    )
+    from vllm.utils.argparse_utils import FlexibleArgumentParser
+
+    CMD_MODULES = [
+        vllm.entrypoints.cli.openai,
+        vllm.entrypoints.cli.serve,
+        vllm.entrypoints.cli.launch,
+        vllm.entrypoints.cli.benchmark.main,
+        vllm.entrypoints.cli.collect_env,
+        vllm.entrypoints.cli.run_batch,
+    ]
+
+    cli_env_setup()
+
+    # If `--omni` arg is passed to the CLI, delegate to vLLM Omni's entrypoint handling
+    if "--omni" in sys.argv:
+        # NOTE: Check the spec instead of importing directly here, since things could
+        # fail with ImportError due to mismatched versions if things are moved around.
+        spec = find_spec("vllm_omni")
+        if spec is None:
+            logger.error(
+                "--omni flag requires a valid instance of vllm-omni to be installed."
+            )
+            sys.exit(1)
+
+        from vllm_omni.entrypoints.cli.main import main as omni_main
+
+        logger.info("Delegating entrypoint handling to vllm-omni")
+        omni_main()
+    else:
+        # For 'vllm bench *': use CPU instead of UnspecifiedPlatform by default
+        if len(sys.argv) > 1 and sys.argv[1] == "bench":
+            logger.debug(
+                "Bench command detected, must ensure current platform is not "
+                "UnspecifiedPlatform to avoid device type inference error"
+            )
+            from vllm import platforms
+
+            if platforms.current_platform.is_unspecified():
+                from vllm.platforms.cpu import CpuPlatform
+
+                platforms.current_platform = CpuPlatform()
+                logger.info(
+                    "Unspecified platform detected, switching to CPU Platform instead."
+                )
+
+        parser = FlexibleArgumentParser(
+            description="vLLM CLI",
+            epilog=VLLM_SUBCMD_PARSER_EPILOG.format(subcmd="[subcommand]"),
+        )
+        parser.add_argument(
+            "-v",
+            "--version",
+            action="version",
+            version=importlib.metadata.version("vllm"),
+        )
+        subparsers = parser.add_subparsers(required=False, dest="subparser")
+        cmds = {}
+        for cmd_module in CMD_MODULES:
+            new_cmds = cmd_module.cmd_init()
+            for cmd in new_cmds:
+                cmd.subparser_init(subparsers).set_defaults(dispatch_function=cmd.cmd)
+                cmds[cmd.name] = cmd
+        args = parser.parse_args()
+        if args.subparser in cmds:
+            cmds[args.subparser].validate(args)
+
+        if hasattr(args, "dispatch_function"):
+            args.dispatch_function(args)
+        else:
+            parser.print_help()
+
+
+if __name__ == "__main__":
+    main()

vllm/entrypoints/cli/openai.py

@@ -0,0 +1,312 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import argparse
+import os
+import signal
+import sys
+import time
+from typing import TYPE_CHECKING
+
+from openai import OpenAI
+from openai.types.chat import ChatCompletionMessageParam
+
+from vllm.entrypoints.cli.types import CLISubcommand
+
+if TYPE_CHECKING:
+    from vllm.utils.argparse_utils import FlexibleArgumentParser
+else:
+    FlexibleArgumentParser = argparse.ArgumentParser
+
+
+def _register_signal_handlers():
+    def signal_handler(sig, frame):
+        sys.exit(0)
+
+    signal.signal(signal.SIGINT, signal_handler)
+    signal.signal(signal.SIGTSTP, signal_handler)
+
+
+def _interactive_cli(args: argparse.Namespace) -> tuple[str, OpenAI]:
+    _register_signal_handlers()
+
+    base_url = args.url
+    api_key = args.api_key or os.environ.get("OPENAI_API_KEY", "EMPTY")
+    openai_client = OpenAI(api_key=api_key, base_url=base_url)
+
+    if args.model_name:
+        model_name = args.model_name
+    else:
+        available_models = openai_client.models.list()
+        model_name = available_models.data[0].id
+
+    print(f"Using model: {model_name}")
+
+    return model_name, openai_client
+
+
+def _print_chat_stream(stream, stats: bool = False) -> str:
+    output = ""
+    start = time.perf_counter()
+    ttft: float | None = None
+    completion_tokens = 0
+    for chunk in stream:
+        if chunk.usage is not None:
+            completion_tokens = chunk.usage.completion_tokens
+        if not chunk.choices:
+            continue
+        delta = chunk.choices[0].delta
+        if delta.content:
+            if ttft is None:
+                ttft = time.perf_counter() - start
+            output += delta.content
+            print(delta.content, end="", flush=True)
+    print()
+    if stats:
+        _print_metrics(start, ttft, completion_tokens)
+    return output
+
+
+def _print_metrics(start: float, ttft: float | None, completion_tokens: int) -> None:
+    total_time = time.perf_counter() - start
+    if ttft is None or total_time <= 0:
+        return
+    print(f"{'TTFT:':<5} {ttft * 1000:.2f} ms")
+    print(
+        f"{'TPS:':<5} {completion_tokens / total_time:.2f} tokens/s "
+        f"({completion_tokens} tokens in {total_time:.2f}s)"
+    )
+
+
+def _print_completion_stream(stream, stats: bool = False) -> str:
+    output = ""
+    start = time.perf_counter()
+    ttft: float | None = None
+    completion_tokens = 0
+    for chunk in stream:
+        if chunk.usage is not None:
+            completion_tokens = chunk.usage.completion_tokens
+        if not chunk.choices:
+            continue
+        text = chunk.choices[0].text
+        if text:
+            if ttft is None:
+                ttft = time.perf_counter() - start
+            output += text
+            print(text, end="", flush=True)
+    print()
+    if stats:
+        _print_metrics(start, ttft, completion_tokens)
+    return output
+
+
+def chat(system_prompt: str | None, model_name: str, client: OpenAI) -> None:
+    conversation: list[ChatCompletionMessageParam] = []
+    if system_prompt is not None:
+        conversation.append({"role": "system", "content": system_prompt})
+
+    print("Please enter a message for the chat model:")
+    while True:
+        try:
+            input_message = input("> ")
+        except EOFError:
+            break
+        conversation.append({"role": "user", "content": input_message})
+
+        stream = client.chat.completions.create(
+            model=model_name, messages=conversation, stream=True
+        )
+        output = _print_chat_stream(stream)
+        conversation.append({"role": "assistant", "content": output})
+
+
+def _add_query_options(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
+    parser.add_argument(
+        "--url",
+        type=str,
+        default="http://localhost:8000/v1",
+        help="url of the running OpenAI-Compatible RESTful API server",
+    )
+    parser.add_argument(
+        "--model-name",
+        type=str,
+        default=None,
+        help=(
+            "The model name used in prompt completion, default to "
+            "the first model in list models API call."
+        ),
+    )
+    parser.add_argument(
+        "--api-key",
+        type=str,
+        default=None,
+        help=(
+            "API key for OpenAI services. If provided, this api key "
+            "will overwrite the api key obtained through environment variables."
+            " It is important to note that this option only applies to the "
+            "OpenAI-compatible API endpoints and NOT other endpoints that may "
+            "be present in the server. See the security guide in the vLLM docs "
+            "for more details."
+        ),
+    )
+    return parser
+
+
+class ChatCommand(CLISubcommand):
+    """The `chat` subcommand for the vLLM CLI."""
+
+    name = "chat"
+
+    @staticmethod
+    def cmd(args: argparse.Namespace) -> None:
+        model_name, client = _interactive_cli(args)
+        system_prompt = args.system_prompt
+        stats = args.stats
+        conversation: list[ChatCompletionMessageParam] = []
+
+        if system_prompt is not None:
+            conversation.append({"role": "system", "content": system_prompt})
+
+        create_kwargs = {"model": model_name, "stream": True}
+        if stats:
+            create_kwargs["stream_options"] = {"include_usage": True}
+
+        if args.quick:
+            conversation.append({"role": "user", "content": args.quick})
+
+            stream = client.chat.completions.create(
+                messages=conversation, **create_kwargs
+            )
+            output = _print_chat_stream(stream, stats)
+            conversation.append({"role": "assistant", "content": output})
+            return
+
+        print("Please enter a message for the chat model:")
+        while True:
+            try:
+                input_message = input("> ")
+            except EOFError:
+                break
+            conversation.append({"role": "user", "content": input_message})
+
+            stream = client.chat.completions.create(
+                messages=conversation, **create_kwargs
+            )
+            output = _print_chat_stream(stream, stats)
+            conversation.append({"role": "assistant", "content": output})
+
+    @staticmethod
+    def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
+        """Add CLI arguments for the chat command."""
+        _add_query_options(parser)
+        parser.add_argument(
+            "--system-prompt",
+            type=str,
+            default=None,
+            help=(
+                "The system prompt to be added to the chat template, "
+                "used for models that support system prompts."
+            ),
+        )
+        parser.add_argument(
+            "-q",
+            "--quick",
+            type=str,
+            metavar="MESSAGE",
+            help=("Send a single prompt as MESSAGE and print the response, then exit."),
+        )
+        parser.add_argument(
+            "--stats",
+            action="store_true",
+            help="Print TTFT and TPS statistics after each response.",
+        )
+        return parser
+
+    def subparser_init(
+        self, subparsers: argparse._SubParsersAction
+    ) -> FlexibleArgumentParser:
+        parser = subparsers.add_parser(
+            "chat",
+            help="Generate chat completions via the running API server.",
+            description="Generate chat completions via the running API server.",
+            usage="vllm chat [options]",
+        )
+        return ChatCommand.add_cli_args(parser)
+
+
+class CompleteCommand(CLISubcommand):
+    """The `complete` subcommand for the vLLM CLI."""
+
+    name = "complete"
+
+    @staticmethod
+    def cmd(args: argparse.Namespace) -> None:
+        model_name, client = _interactive_cli(args)
+        stats = args.stats
+
+        kwargs = {
+            "model": model_name,
+            "stream": True,
+        }
+        if args.max_tokens:
+            kwargs["max_tokens"] = args.max_tokens
+        if stats:
+            kwargs["stream_options"] = {"include_usage": True}
+
+        if args.quick:
+            stream = client.completions.create(prompt=args.quick, **kwargs)
+            _print_completion_stream(stream, stats)
+            return
+
+        print("Please enter prompt to complete:")
+        while True:
+            try:
+                input_prompt = input("> ")
+            except EOFError:
+                break
+            stream = client.completions.create(prompt=input_prompt, **kwargs)
+            _print_completion_stream(stream, stats)
+
+    @staticmethod
+    def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
+        """Add CLI arguments for the complete command."""
+        _add_query_options(parser)
+        parser.add_argument(
+            "--max-tokens",
+            type=int,
+            help="Maximum number of tokens to generate per output sequence.",
+        )
+        parser.add_argument(
+            "-q",
+            "--quick",
+            type=str,
+            metavar="PROMPT",
+            help="Send a single prompt and print the completion output, then exit.",
+        )
+        parser.add_argument(
+            "--stats",
+            action="store_true",
+            help="Print TTFT and TPS statistics after each response.",
+        )
+        return parser
+
+    def subparser_init(
+        self, subparsers: argparse._SubParsersAction
+    ) -> FlexibleArgumentParser:
+        parser = subparsers.add_parser(
+            "complete",
+            help=(
+                "Generate text completions based on the given prompt "
+                "via the running API server."
+            ),
+            description=(
+                "Generate text completions based on the given prompt "
+                "via the running API server."
+            ),
+            usage="vllm complete [options]",
+        )
+        return CompleteCommand.add_cli_args(parser)
+
+
+def cmd_init() -> list[CLISubcommand]:
+    return [ChatCommand(), CompleteCommand()]

vllm/entrypoints/cli/run_batch.py

@@ -0,0 +1,68 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import argparse
+import asyncio
+import importlib.metadata
+import typing
+
+from vllm.entrypoints.cli.types import CLISubcommand
+from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
+from vllm.logger import init_logger
+
+if typing.TYPE_CHECKING:
+    from vllm.utils.argparse_utils import FlexibleArgumentParser
+else:
+    FlexibleArgumentParser = argparse.ArgumentParser
+
+logger = init_logger(__name__)
+
+
+class RunBatchSubcommand(CLISubcommand):
+    """The `run-batch` subcommand for vLLM CLI."""
+
+    name = "run-batch"
+
+    @staticmethod
+    def cmd(args: argparse.Namespace) -> None:
+        from vllm.entrypoints.openai.run_batch import main as run_batch_main
+
+        logger.info(
+            "vLLM batch processing API version %s", importlib.metadata.version("vllm")
+        )
+        logger.info("args: %s", args)
+
+        # Start the Prometheus metrics server.
+        # LLMEngine uses the Prometheus client
+        # to publish metrics at the /metrics endpoint.
+        if args.enable_metrics:
+            from prometheus_client import start_http_server
+
+            logger.info("Prometheus metrics enabled")
+            start_http_server(port=args.port, addr=args.url)
+        else:
+            logger.info("Prometheus metrics disabled")
+
+        asyncio.run(run_batch_main(args))
+
+    def subparser_init(
+        self, subparsers: argparse._SubParsersAction
+    ) -> FlexibleArgumentParser:
+        from vllm.entrypoints.openai.run_batch import make_arg_parser
+
+        run_batch_parser = subparsers.add_parser(
+            self.name,
+            help="Run batch prompts and write results to file.",
+            description=(
+                "Run batch prompts using vLLM's OpenAI-compatible API.\n"
+                "Supports local or HTTP input/output files."
+            ),
+            usage="vllm run-batch -i INPUT.jsonl -o OUTPUT.jsonl --model <model>",
+        )
+        run_batch_parser = make_arg_parser(run_batch_parser)
+        run_batch_parser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(subcmd=self.name)
+        return run_batch_parser
+
+
+def cmd_init() -> list[CLISubcommand]:
+    return [RunBatchSubcommand()]

vllm/entrypoints/cli/serve.py

@@ -0,0 +1,394 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import argparse
+import signal
+import time
+
+import uvloop
+
+import vllm
+import vllm.envs as envs
+from vllm.entrypoints.cli.types import CLISubcommand
+from vllm.entrypoints.openai.api_server import run_server, setup_server
+from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
+from vllm.entrypoints.openai.dp_supervisor import (
+    run_dp_supervisor,
+)
+from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
+from vllm.logger import init_logger
+from vllm.usage.usage_lib import UsageContext
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+from vllm.utils.network_utils import get_tcp_uri
+from vllm.v1.engine.utils import CoreEngineProcManager, launch_core_engines
+from vllm.v1.executor import Executor
+from vllm.v1.executor.multiproc_executor import MultiprocExecutor
+from vllm.v1.metrics.prometheus import setup_multiprocess_prometheus
+from vllm.v1.utils import (
+    APIServerProcessManager,
+    RustFrontendProcessManager,
+    wait_for_completion_or_failure,
+)
+
+logger = init_logger(__name__)
+
+DESCRIPTION = """Launch a local OpenAI-compatible API server to serve LLM
+completions via HTTP. Defaults to Qwen/Qwen3-0.6B if no model is specified.
+
+Search by using: `--help=<ConfigGroup>` to explore options by section (e.g.,
+--help=ModelConfig, --help=Frontend)
+  Use `--help=all` to show all available flags at once.
+"""
+
+
+class ServeSubcommand(CLISubcommand):
+    """The `serve` subcommand for the vLLM CLI."""
+
+    name = "serve"
+
+    @staticmethod
+    def cmd(args: argparse.Namespace) -> None:
+        # If model is specified in CLI (as positional arg), it takes precedence
+        if hasattr(args, "model_tag") and args.model_tag is not None:
+            args.model = args.model_tag
+
+        if getattr(args, "grpc", False):
+            from vllm.entrypoints.grpc_server import serve_grpc
+
+            uvloop.run(serve_grpc(args))
+            return
+
+        if args.headless:
+            if args.api_server_count is not None and args.api_server_count > 0:
+                raise ValueError(
+                    f"--api-server-count={args.api_server_count} cannot be "
+                    "used with --headless (no API servers are started in "
+                    "headless mode)."
+                )
+            # Default to 0 in headless mode (no API servers)
+            args.api_server_count = 0
+
+        # Detect LB mode for defaulting api_server_count.
+        # Multi-port: --data-parallel-multi-port-external-lb
+        # External LB: --data-parallel-external-lb or --data-parallel-rank
+        # Hybrid LB: --data-parallel-hybrid-lb or --data-parallel-start-rank
+        is_external_lb = (
+            args.data_parallel_external_lb or args.data_parallel_rank is not None
+        )
+
+        # If --data_parallel_multi_port_external_lb and --data_parallel_hybrid_lb
+        # are unset, default to hybrid if --data-parallel-start-rank is set
+        is_hybrid_lb = is_multi_port = False
+        if (
+            not args.data_parallel_hybrid_lb
+            and not args.data_parallel_multi_port_external_lb
+        ):
+            is_hybrid_lb = args.data_parallel_start_rank is not None
+        else:
+            is_hybrid_lb = args.data_parallel_hybrid_lb
+            is_multi_port = args.data_parallel_multi_port_external_lb
+
+        if sum([is_multi_port, is_external_lb, is_hybrid_lb]) > 1:
+            raise ValueError(
+                "Cannot use more than one data parallel load balancing mode. "
+                "Choose one of: --data-parallel-multi-port-external-lb, "
+                "--data-parallel-external-lb (or --data-parallel-rank), "
+                "--data-parallel-hybrid-lb (or --data-parallel-start-rank)."
+            )
+
+        # Default api_server_count if not explicitly set.
+        # - Multi-port: 1 (supervisor spawns one server per local DP rank)
+        # - Rust frontend: 1 (not applicable as it's multithreaded)
+        # - External LB: 1 (external LB handles distribution)
+        # - Hybrid LB: Use local DP size (internal LB for local ranks only)
+        # - Internal LB: Use full DP size
+        if args.api_server_count is None:
+            if is_multi_port or is_external_lb or envs.VLLM_RUST_FRONTEND_PATH:
+                args.api_server_count = 1
+            elif is_hybrid_lb:
+                args.api_server_count = args.data_parallel_size_local or 1
+                if args.api_server_count > 1:
+                    logger.info(
+                        "Defaulting api_server_count to data_parallel_size_local "
+                        "(%d) for hybrid LB mode.",
+                        args.api_server_count,
+                    )
+            else:
+                args.api_server_count = args.data_parallel_size
+                if args.api_server_count > 1:
+                    logger.info(
+                        "Defaulting api_server_count to data_parallel_size (%d).",
+                        args.api_server_count,
+                    )
+        elif envs.VLLM_RUST_FRONTEND_PATH and args.api_server_count > 1:
+            logger.warning(
+                "Ignoring --api-server-count=%d when using rust front-end process",
+                args.api_server_count,
+            )
+            args.api_server_count = 1
+
+        # Elastic EP currently only supports running with at most one API server.
+        if getattr(args, "enable_elastic_ep", False) and args.api_server_count > 1:
+            logger.warning(
+                "Elastic EP only supports running with with at most one API server. "
+                "Capping api_server_count from %d to 1.",
+                args.api_server_count,
+            )
+            args.api_server_count = 1
+
+        if is_multi_port:
+            run_dp_supervisor(args)
+        elif args.api_server_count < 1:
+            run_headless(args)
+        elif args.api_server_count > 1 or envs.VLLM_RUST_FRONTEND_PATH:
+            run_multi_api_server(args)
+        else:
+            # Single API server (this process).
+            args.api_server_count = None
+            uvloop.run(run_server(args))
+
+    def validate(self, args: argparse.Namespace) -> None:
+        validate_parsed_serve_args(args)
+
+    def subparser_init(
+        self, subparsers: argparse._SubParsersAction
+    ) -> FlexibleArgumentParser:
+        serve_parser = subparsers.add_parser(
+            self.name,
+            help="Launch a local OpenAI-compatible API server to serve LLM "
+            "completions via HTTP.",
+            description=DESCRIPTION,
+            usage="vllm serve [model_tag] [options]",
+        )
+
+        serve_parser = make_arg_parser(serve_parser)
+        serve_parser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(subcmd=self.name)
+        return serve_parser
+
+
+def cmd_init() -> list[CLISubcommand]:
+    return [ServeSubcommand()]
+
+
+def run_headless(args: argparse.Namespace):
+    if args.api_server_count > 1:
+        raise ValueError("api_server_count can't be set in headless mode")
+
+    # Create the EngineConfig.
+    engine_args = vllm.AsyncEngineArgs.from_cli_args(args)
+    usage_context = UsageContext.OPENAI_API_SERVER
+    vllm_config = engine_args.create_engine_config(
+        usage_context=usage_context, headless=True
+    )
+
+    if engine_args.data_parallel_hybrid_lb:
+        raise ValueError("data_parallel_hybrid_lb is not applicable in headless mode")
+
+    parallel_config = vllm_config.parallel_config
+    local_engine_count = parallel_config.data_parallel_size_local
+
+    if local_engine_count <= 0:
+        raise ValueError("data_parallel_size_local must be > 0 in headless mode")
+
+    shutdown_requested = False
+
+    # Catch SIGTERM and SIGINT to allow graceful shutdown.
+    def signal_handler(signum, frame):
+        nonlocal shutdown_requested
+        logger.debug("Received %d signal.", signum)
+        if not shutdown_requested:
+            shutdown_requested = True
+            raise SystemExit
+
+    signal.signal(signal.SIGTERM, signal_handler)
+    signal.signal(signal.SIGINT, signal_handler)
+
+    if parallel_config.node_rank_within_dp > 0:
+        from vllm.version import __version__ as VLLM_VERSION
+
+        # Run headless workers (for multi-node PP/TP).
+        host = parallel_config.master_addr
+        head_node_address = f"{host}:{parallel_config.master_port}"
+        logger.info(
+            "Launching vLLM (v%s) headless multiproc executor, "
+            "with head node address %s for torch.distributed process group.",
+            VLLM_VERSION,
+            head_node_address,
+        )
+
+        executor = MultiprocExecutor(vllm_config, monitor_workers=False)
+        executor.start_worker_monitor(inline=True)
+        return
+
+    host = parallel_config.data_parallel_master_ip
+    port = parallel_config.data_parallel_rpc_port
+    handshake_address = get_tcp_uri(host, port)
+
+    logger.info(
+        "Launching %d data parallel engine(s) in headless mode, "
+        "with head node address %s.",
+        local_engine_count,
+        handshake_address,
+    )
+
+    # Create the engines.
+    engine_manager = CoreEngineProcManager(
+        local_engine_count=local_engine_count,
+        start_index=vllm_config.parallel_config.data_parallel_rank,
+        local_start_index=0,
+        vllm_config=vllm_config,
+        local_client=False,
+        handshake_address=handshake_address,
+        executor_class=Executor.get_class(vllm_config),
+        log_stats=not engine_args.disable_log_stats,
+    )
+
+    try:
+        engine_manager.monitor_engine_liveness()
+    finally:
+        timeout = None
+        if shutdown_requested:
+            timeout = vllm_config.shutdown_timeout
+            logger.info("Waiting up to %d seconds for processes to exit", timeout)
+        engine_manager.shutdown(timeout=timeout)
+        logger.info("Shutting down.")
+
+
+def run_multi_api_server(args: argparse.Namespace):
+    assert not args.headless
+    rust_frontend_path = envs.VLLM_RUST_FRONTEND_PATH
+    num_api_servers: int = args.api_server_count
+    assert num_api_servers > 0
+
+    if rust_frontend_path and num_api_servers > 1:
+        raise ValueError(
+            "VLLM_RUST_FRONTEND_PATH does not support api_server_count > 1"
+        )
+
+    if num_api_servers > 1:
+        setup_multiprocess_prometheus()
+
+    shutdown_requested = False
+
+    # Catch SIGTERM and SIGINT to allow graceful shutdown.
+    def signal_handler(signum, frame):
+        nonlocal shutdown_requested
+        logger.debug("Received %d signal.", signum)
+        if not shutdown_requested:
+            shutdown_requested = True
+            raise SystemExit
+
+    signal.signal(signal.SIGTERM, signal_handler)
+    signal.signal(signal.SIGINT, signal_handler)
+
+    listen_address, sock = setup_server(args, reuse_port=num_api_servers > 1)
+
+    engine_args = vllm.AsyncEngineArgs.from_cli_args(args)
+    engine_args._api_process_count = num_api_servers
+    engine_args._api_process_rank = -1
+
+    usage_context = UsageContext.OPENAI_API_SERVER
+    vllm_config = engine_args.create_engine_config(usage_context=usage_context)
+
+    if num_api_servers > 1 and envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
+        raise ValueError(
+            "VLLM_ALLOW_RUNTIME_LORA_UPDATING cannot be used with api_server_count > 1"
+        )
+
+    executor_class = Executor.get_class(vllm_config)
+    log_stats = not engine_args.disable_log_stats
+
+    parallel_config = vllm_config.parallel_config
+    dp_rank = parallel_config.data_parallel_rank
+    assert parallel_config.local_engines_only or dp_rank == 0
+
+    api_server_manager: APIServerProcessManager | RustFrontendProcessManager | None = (
+        None
+    )
+
+    from vllm.v1.engine.utils import get_engine_zmq_addresses
+
+    # Defer port allocation to the child's bind() to avoid TOCTOU, except
+    # for Rust front-end and Ray DP, which can't see the post-bind rebind
+    # (CLI-arg subprocess / pickled-into-actor snapshot respectively) and
+    # so pre-allocate driver-side -- reintroducing the original race only
+    # there.
+    is_ray_dp = parallel_config.data_parallel_backend == "ray"
+    addresses = get_engine_zmq_addresses(
+        vllm_config,
+        num_api_servers,
+        defer_api_server_ports=not (rust_frontend_path or is_ray_dp),
+    )
+
+    with launch_core_engines(
+        vllm_config, executor_class, log_stats, addresses, num_api_servers
+    ) as (local_engine_manager, coordinator, addresses, tensor_queue):
+        stats_update_address = (
+            coordinator.get_stats_publish_address() if coordinator else None
+        )
+
+        if rust_frontend_path:
+            if parallel_config.local_engines_only:
+                expected_engine_start_index = parallel_config.data_parallel_rank
+                expected_engine_count = parallel_config.data_parallel_size_local
+            else:
+                expected_engine_start_index = 0
+                expected_engine_count = parallel_config.data_parallel_size
+            # Start rust front-end process.
+            api_server_manager = RustFrontendProcessManager(
+                binary_path=rust_frontend_path,
+                sock=sock,
+                args=args,
+                input_address=addresses.inputs[0],
+                output_address=addresses.outputs[0],
+                engine_start_index=expected_engine_start_index,
+                engine_count=expected_engine_count,
+                stats_update_address=stats_update_address,
+            )
+        else:
+            # Start API server(s).
+            api_server_manager = APIServerProcessManager(
+                listen_address=listen_address,
+                sock=sock,
+                args=args,
+                num_servers=num_api_servers,
+                input_addresses=addresses.inputs,
+                output_addresses=addresses.outputs,
+                stats_update_address=stats_update_address,
+                tensor_queue=tensor_queue,
+            )
+
+            if not is_ray_dp:
+                # Forward each child's bound endpoints to the engine handshake
+                # (runs on ``with`` exit). Skipped for Ray DP, where addresses
+                # are pre-allocated above and Ray actors already hold them.
+                actual_inputs, actual_outputs = (
+                    api_server_manager.gather_actual_addresses()
+                )
+                addresses.inputs = actual_inputs
+                addresses.outputs = actual_outputs
+
+    # Wait for API servers.
+    try:
+        wait_for_completion_or_failure(
+            api_server_manager=api_server_manager,
+            engine_manager=local_engine_manager,
+            coordinator=coordinator,
+        )
+    finally:
+        timeout = shutdown_by = None
+        if shutdown_requested:
+            timeout = vllm_config.shutdown_timeout
+            shutdown_by = time.monotonic() + timeout
+            logger.info("Waiting up to %d seconds for processes to exit", timeout)
+
+        def to_timeout(deadline: float | None) -> float | None:
+            return (
+                deadline if deadline is None else max(deadline - time.monotonic(), 0.0)
+            )
+
+        api_server_manager.shutdown(timeout=timeout)
+        if local_engine_manager:
+            local_engine_manager.shutdown(timeout=to_timeout(shutdown_by))
+        if coordinator:
+            coordinator.shutdown(timeout=to_timeout(shutdown_by))

vllm/entrypoints/cli/types.py

@@ -0,0 +1,29 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import argparse
+import typing
+
+if typing.TYPE_CHECKING:
+    from vllm.utils.argparse_utils import FlexibleArgumentParser
+else:
+    FlexibleArgumentParser = argparse.ArgumentParser
+
+
+class CLISubcommand:
+    """Base class for CLI argument handlers."""
+
+    name: str
+
+    @staticmethod
+    def cmd(args: argparse.Namespace) -> None:
+        raise NotImplementedError("Subclasses should implement this method")
+
+    def validate(self, args: argparse.Namespace) -> None:
+        # No validation by default
+        pass
+
+    def subparser_init(
+        self, subparsers: argparse._SubParsersAction
+    ) -> FlexibleArgumentParser:
+        raise NotImplementedError("Subclasses should implement this method")

vllm/entrypoints/generate/api_router.py

@@ -0,0 +1,186 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from typing import TYPE_CHECKING
+
+from fastapi import FastAPI
+
+if TYPE_CHECKING:
+    from argparse import Namespace
+
+    from starlette.datastructures import State
+
+    from vllm.engine.protocol import EngineClient
+    from vllm.entrypoints.serve.utils.request_logger import RequestLogger
+    from vllm.tasks import SupportedTask
+else:
+    RequestLogger = object
+
+
+def register_generate_api_routers(app: FastAPI):
+    from vllm.entrypoints.openai.chat_completion.api_router import (
+        attach_router as register_chat_api_router,
+    )
+
+    register_chat_api_router(app)
+
+    from vllm.entrypoints.openai.responses.api_router import (
+        attach_router as register_responses_api_router,
+    )
+
+    register_responses_api_router(app)
+
+    from vllm.entrypoints.openai.completion.api_router import (
+        attach_router as register_completion_api_router,
+    )
+
+    register_completion_api_router(app)
+
+    from vllm.entrypoints.anthropic.api_router import (
+        attach_router as register_anthropic_api_router,
+    )
+
+    register_anthropic_api_router(app)
+
+    from .generative_scoring.api_router import register_generative_scoring_api_router
+
+    register_generative_scoring_api_router(app)
+
+
+async def init_generate_state(
+    engine_client: "EngineClient",
+    state: "State",
+    args: "Namespace",
+    request_logger: RequestLogger | None,
+    supported_tasks: tuple["SupportedTask", ...],
+):
+    from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
+    from vllm.entrypoints.chat_utils import load_chat_template
+    from vllm.entrypoints.mcp.tool_server import (
+        DemoToolServer,
+        MCPToolServer,
+        ToolServer,
+    )
+    from vllm.entrypoints.openai.chat_completion.batch_serving import (
+        OpenAIServingChatBatch,
+    )
+    from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
+    from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
+    from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
+    from vllm.entrypoints.serve.utils.fingerprint import set_default_fingerprint_mode
+
+    # Applied before any serving class is constructed so that each one picks
+    # up the chosen mode on its first cache miss.
+    set_default_fingerprint_mode(
+        getattr(args, "fingerprint_mode", "full"),
+        getattr(args, "fingerprint_value", None),
+    )
+
+    if args.tool_server == "demo":
+        tool_server: ToolServer | None = DemoToolServer()
+        assert isinstance(tool_server, DemoToolServer)
+        await tool_server.init_and_validate()
+    elif args.tool_server:
+        tool_server = MCPToolServer()
+        await tool_server.add_tool_server(args.tool_server)
+    else:
+        tool_server = None
+    resolved_chat_template = load_chat_template(args.chat_template)
+
+    # Render endpoints are always backed by OnlineRenderer so that
+    # /v1/chat/completions/render and /v1/completions/render work on both
+    # generate-mode and render-only servers. Created in init_app_state.
+
+    state.openai_serving_responses = (
+        OpenAIServingResponses(
+            engine_client,
+            state.openai_serving_models,
+            state.online_renderer,
+            request_logger=request_logger,
+            chat_template=resolved_chat_template,
+            chat_template_content_format=args.chat_template_content_format,
+            return_tokens_as_token_ids=args.return_tokens_as_token_ids,
+            enable_auto_tools=args.enable_auto_tool_choice,
+            tool_parser=args.tool_call_parser,
+            tool_server=tool_server,
+            reasoning_parser=args.structured_outputs_config.reasoning_parser,
+            enable_prompt_tokens_details=args.enable_prompt_tokens_details,
+            enable_force_include_usage=args.enable_force_include_usage,
+            enable_log_outputs=args.enable_log_outputs,
+            default_chat_template_kwargs=args.default_chat_template_kwargs,
+        )
+        if "generate" in supported_tasks
+        else None
+    )
+    _chat_kwargs = dict(
+        engine_client=engine_client,
+        models=state.openai_serving_models,
+        response_role=args.response_role,
+        online_renderer=state.online_renderer,
+        request_logger=request_logger,
+        chat_template=resolved_chat_template,
+        chat_template_content_format=args.chat_template_content_format,
+        default_chat_template_kwargs=args.default_chat_template_kwargs,
+        trust_request_chat_template=args.trust_request_chat_template,
+        return_tokens_as_token_ids=args.return_tokens_as_token_ids,
+        enable_auto_tools=args.enable_auto_tool_choice,
+        exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
+        tool_parser=args.tool_call_parser,
+        reasoning_parser=args.structured_outputs_config.reasoning_parser,
+        enable_prompt_tokens_details=args.enable_prompt_tokens_details,
+        enable_force_include_usage=args.enable_force_include_usage,
+        enable_log_outputs=args.enable_log_outputs,
+        enable_log_deltas=args.enable_log_deltas,
+        enable_per_request_metrics=args.enable_per_request_metrics,
+    )
+    state.openai_serving_chat = (
+        OpenAIServingChat(**_chat_kwargs) if "generate" in supported_tasks else None
+    )
+    state.openai_serving_chat_batch = (
+        OpenAIServingChatBatch(**_chat_kwargs)
+        if "generate" in supported_tasks
+        else None
+    )
+    if state.openai_serving_chat is not None:
+        state.openai_serving_chat.warmup()
+    state.openai_serving_completion = (
+        OpenAIServingCompletion(
+            engine_client,
+            state.openai_serving_models,
+            online_renderer=state.online_renderer,
+            request_logger=request_logger,
+            return_tokens_as_token_ids=args.return_tokens_as_token_ids,
+            enable_prompt_tokens_details=args.enable_prompt_tokens_details,
+            enable_force_include_usage=args.enable_force_include_usage,
+            enable_per_request_metrics=args.enable_per_request_metrics,
+        )
+        if "generate" in supported_tasks
+        else None
+    )
+    state.anthropic_serving_messages = (
+        AnthropicServingMessages(
+            engine_client,
+            state.openai_serving_models,
+            args.response_role,
+            online_renderer=state.online_renderer,
+            request_logger=request_logger,
+            chat_template=resolved_chat_template,
+            chat_template_content_format=args.chat_template_content_format,
+            return_tokens_as_token_ids=args.return_tokens_as_token_ids,
+            enable_auto_tools=args.enable_auto_tool_choice,
+            tool_parser=args.tool_call_parser,
+            reasoning_parser=args.structured_outputs_config.reasoning_parser,
+            enable_prompt_tokens_details=args.enable_prompt_tokens_details,
+            enable_force_include_usage=args.enable_force_include_usage,
+            default_chat_template_kwargs=args.default_chat_template_kwargs,
+        )
+        if "generate" in supported_tasks
+        else None
+    )
+
+    from .generative_scoring.serving import ServingGenerativeScoring
+
+    state.serving_generative_scoring = ServingGenerativeScoring(
+        engine_client,
+        state.openai_serving_models,
+        request_logger=request_logger,
+    )

vllm/entrypoints/generate/base/serving.py

@@ -0,0 +1,317 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import json
+import time
+from collections.abc import Awaitable, Mapping
+from dataclasses import dataclass, field
+from http import HTTPStatus
+from typing import ClassVar, Generic, TypeVar
+
+from fastapi import Request
+from pydantic import ConfigDict
+from starlette.datastructures import Headers
+
+from vllm.engine.protocol import EngineClient
+from vllm.entrypoints.generate.beam_search.online import BeamSearchOnlineMixin
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
+from vllm.entrypoints.openai.completion.protocol import CompletionRequest
+from vllm.entrypoints.openai.engine.protocol import (
+    ErrorResponse,
+    GenerationError,
+    PerRequestTimingMetrics,
+)
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
+from vllm.entrypoints.serve.engine.serving import BaseServing
+from vllm.entrypoints.serve.engine.typing import AnyRequest
+from vllm.entrypoints.serve.utils.request_logger import RequestLogger
+from vllm.inputs import EngineInput
+from vllm.logger import init_logger
+from vllm.logprobs import Logprob, PromptLogprobs
+from vllm.lora.request import LoRARequest
+from vllm.tokenizers import TokenizerLike
+from vllm.tracing import (
+    contains_trace_headers,
+    extract_trace_headers,
+    log_tracing_disabled_warning,
+)
+from vllm.v1.metrics.stats import RequestStateStats
+
+logger = init_logger(__name__)
+
+RequestT = TypeVar("RequestT", bound=AnyRequest)
+_T = TypeVar("_T")
+
+
+def build_per_request_timing_metrics(
+    metrics: RequestStateStats | None,
+    num_generation_tokens: int,
+) -> PerRequestTimingMetrics:
+    """Build per-request timing metrics from ``RequestStateStats``.
+
+    ``generation_time_ms`` is the decode interval only (first output token to
+    last output token); it excludes both queue wait and prefill/TTFT.
+    ``tokens_per_second`` is overall output throughput: all generated tokens
+    over the inference interval (scheduling to last output token), so it counts
+    the prefill/TTFT phase and is not simply the reciprocal of ``mean_itl_ms``.
+    Each field is left ``None`` when the timestamps it depends on are
+    unavailable.
+    """
+    if metrics is None:
+        return PerRequestTimingMetrics()
+
+    queued_ts = metrics.queued_ts
+    scheduled_ts = metrics.scheduled_ts
+    first_token_ts = metrics.first_token_ts
+    last_token_ts = metrics.last_token_ts
+
+    time_to_first_token_ms: float | None = None
+    generation_time_ms: float | None = None
+    queue_time_ms: float | None = None
+    mean_itl_ms: float | None = None
+    tokens_per_second: float | None = None
+
+    if scheduled_ts > 0 and first_token_ts > 0:
+        time_to_first_token_ms = (first_token_ts - scheduled_ts) * 1000
+
+    if first_token_ts > 0 and last_token_ts > 0:
+        generation_time_ms = (last_token_ts - first_token_ts) * 1000
+
+    if queued_ts > 0 and scheduled_ts > 0:
+        queue_time_ms = (scheduled_ts - queued_ts) * 1000
+
+    if first_token_ts > 0 and last_token_ts > 0 and num_generation_tokens > 1:
+        decode_time = last_token_ts - first_token_ts
+        mean_itl_ms = decode_time / (num_generation_tokens - 1) * 1000
+
+    if scheduled_ts > 0 and last_token_ts > 0:
+        inference_time_ms = (last_token_ts - scheduled_ts) * 1000
+        if inference_time_ms > 0:
+            tokens_per_second = num_generation_tokens / inference_time_ms * 1000
+
+    return PerRequestTimingMetrics(
+        time_to_first_token_ms=time_to_first_token_ms,
+        generation_time_ms=generation_time_ms,
+        queue_time_ms=queue_time_ms,
+        mean_itl_ms=mean_itl_ms,
+        tokens_per_second=tokens_per_second,
+    )
+
+
+@dataclass(kw_only=True)
+class ServeContext(Generic[RequestT]):
+    request: RequestT
+    raw_request: Request | None = None
+    model_name: str
+    request_id: str
+    created_time: int = field(default_factory=lambda: int(time.time()))
+    lora_request: LoRARequest | None = None
+    engine_inputs: list[EngineInput] | None = None
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+
+class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin):
+    request_id_prefix: ClassVar[str] = """
+    A short string prepended to every request’s ID.
+    """
+
+    def __init__(
+        self,
+        engine_client: EngineClient,
+        models: OpenAIServingModels,
+        *,
+        request_logger: RequestLogger | None,
+        return_tokens_as_token_ids: bool = False,
+    ):
+        super().__init__(
+            models=models,
+            model_config=engine_client.model_config,
+            request_logger=request_logger,
+        )
+
+        self.engine_client = engine_client
+        self.return_tokens_as_token_ids = return_tokens_as_token_ids
+        self.renderer = engine_client.renderer
+        self.input_processor = engine_client.input_processor
+        vllm_config = getattr(engine_client, "vllm_config", None)
+        kv_transfer_config = getattr(vllm_config, "kv_transfer_config", None)
+        self.has_kv_connector = kv_transfer_config is not None
+
+        # Computed once at startup (cached by ``vllm_config`` identity) and
+        # stamped on non-streaming responses. Streaming chunks deliberately
+        # omit it to avoid per-chunk overhead.
+        from vllm.entrypoints.serve.utils.fingerprint import get_system_fingerprint
+
+        try:
+            self.system_fingerprint: str | None = get_system_fingerprint(
+                engine_client.vllm_config
+            )
+        except Exception:
+            # Never fail server startup over the fingerprint.
+            self.system_fingerprint = None
+
+    def create_streaming_error_response(
+        self,
+        message: str | Exception,
+        err_type: str = "BadRequestError",
+        status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
+        param: str | None = None,
+    ) -> str:
+        json_str = json.dumps(
+            self.create_error_response(
+                message=message,
+                err_type=err_type,
+                status_code=status_code,
+                param=param,
+            ).model_dump()
+        )
+        return json_str
+
+    def _raise_if_error(self, finish_reason: str | None, request_id: str) -> None:
+        """Raise GenerationError if finish_reason indicates an error."""
+        if finish_reason == "error":
+            logger.error(
+                "Request %s failed with an internal error during generation",
+                request_id,
+            )
+            raise GenerationError("Internal server error")
+
+    def _convert_generation_error_to_streaming_response(
+        self, e: GenerationError
+    ) -> str:
+        """Convert GenerationError to streaming error response."""
+        return self.create_streaming_error_response(
+            str(e),
+            err_type="InternalServerError",
+            status_code=e.status_code,
+        )
+
+    async def _get_trace_headers(
+        self,
+        headers: Headers,
+    ) -> Mapping[str, str] | None:
+        is_tracing_enabled = await self.engine_client.is_tracing_enabled()
+
+        if is_tracing_enabled:
+            return extract_trace_headers(headers)
+
+        if contains_trace_headers(headers):
+            log_tracing_disabled_warning()
+
+        return None
+
+    @staticmethod
+    def _get_data_parallel_rank(raw_request: Request | None) -> int | None:
+        """Pulls the data parallel rank from a header, if provided"""
+        if raw_request is None:
+            return None
+
+        rank_str = raw_request.headers.get("X-data-parallel-rank")
+        if rank_str is None:
+            return None
+
+        try:
+            return int(rank_str)
+        except ValueError:
+            return None
+
+    async def _with_kv_transfer_rejection_cleanup(
+        self,
+        awaitable: Awaitable[_T],
+        request: ChatCompletionRequest | CompletionRequest | ResponsesRequest,
+        raw_request: Request | None,
+    ) -> _T:
+        """Wrap a `create_*` coroutine so that, if it raises or returns an
+        ErrorResponse (i.e. the request never reached the engine), the KV
+        connector is notified to free any pinned remote-prefill blocks."""
+        kv_transfer_params = self.has_kv_connector and request.kv_transfer_params
+        if not kv_transfer_params or not kv_transfer_params.get("do_remote_prefill"):
+            return await awaitable
+
+        notify = True
+        try:
+            result = await awaitable
+            if not isinstance(result, ErrorResponse):
+                notify = False
+            return result
+        finally:
+            if notify:
+                try:
+                    await self.engine_client.notify_kv_transfer_request_rejected(
+                        request.request_id,
+                        kv_transfer_params,
+                        data_parallel_rank=self._get_data_parallel_rank(raw_request),
+                    )
+                except Exception:
+                    logger.warning(
+                        "Failed to notify KV connector about rejected request %s",
+                        request.request_id,
+                        exc_info=True,
+                    )
+
+    @staticmethod
+    def _get_decoded_token(
+        logprob: Logprob,
+        token_id: int,
+        tokenizer: TokenizerLike | None,
+        return_as_token_id: bool = False,
+    ) -> str:
+        if return_as_token_id:
+            return format_token_id_placeholder(token_id)
+
+        if logprob.decoded_token is not None:
+            return logprob.decoded_token
+
+        if tokenizer is None:
+            raise ValueError(
+                "Unable to get tokenizer because `skip_tokenizer_init=True`"
+            )
+
+        return tokenizer.decode([token_id])
+
+
+def format_token_id_placeholder(token_id: int) -> str:
+    return f"token_id:{token_id}"
+
+
+def resolve_token_id_placeholder(
+    token: str, tokenizer: TokenizerLike
+) -> tuple[str, list[int] | None]:
+    """Decode a 'token_id:N' placeholder back to a token string and UTF-8 bytes.
+
+    Returns (token, None) unchanged if token is not a placeholder.
+    This is the inverse of format_token_id_placeholder / _get_decoded_token
+    when return_as_token_id=True.
+    """
+    suffix = token.removeprefix("token_id:")
+    if suffix == token:
+        return token, None
+    try:
+        token_id = int(suffix)
+    except ValueError:
+        return token, None
+    token_repr = tokenizer.convert_ids_to_tokens([token_id])[0]
+    if token_repr is None:
+        logger.warning_once(
+            "resolve_token_id_placeholder: token_id %d has no vocab entry; "
+            "substituting empty string",
+            token_id,
+        )
+        return "", None
+    token_str = tokenizer.convert_tokens_to_string([token_repr])
+    return token_str, list(token_str.encode("utf-8", errors="replace"))
+
+
+def clamp_prompt_logprobs(
+    prompt_logprobs: PromptLogprobs | None,
+) -> PromptLogprobs | None:
+    if prompt_logprobs is None:
+        return prompt_logprobs
+
+    for logprob_dict in prompt_logprobs:
+        if logprob_dict is None:
+            continue
+        for logprob_values in logprob_dict.values():
+            if logprob_values.logprob == float("-inf"):
+                logprob_values.logprob = -9999.0
+    return prompt_logprobs

vllm/entrypoints/generate/beam_search/offline.py

@@ -0,0 +1,453 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import itertools
+from collections.abc import Callable, Sequence
+
+import torch
+from tqdm import tqdm
+
+from vllm import RequestOutput, TextPrompt, TokensPrompt
+from vllm.entrypoints.offline_utils import OfflineInferenceMixin
+from vllm.logger import init_logger
+from vllm.lora.request import LoRARequest
+from vllm.pooling_params import PoolingParams
+from vllm.sampling_params import (
+    BeamSearchParams,
+    SamplingParams,
+    StructuredOutputsParams,
+)
+from vllm.tokenizers import TokenizerLike
+from vllm.v1.structured_output.backend_types import StructuredOutputBackend
+from vllm.v1.structured_output.request import get_structured_output_key
+
+from .utils import (
+    BeamSearchInstance,
+    BeamSearchOutput,
+    BeamSearchSequence,
+    create_sort_beams_key_function,
+)
+
+logger = init_logger(__name__)
+
+# Engine-side cap on `SamplingParams.allowed_token_ids`; keep in sync with
+# MAX_NUM_ALLOWED_TOKEN_IDS in vllm/v1/worker/gpu/sample/logit_bias.py.
+_MAX_NUM_ALLOWED_TOKEN_IDS = 1024
+
+
+_bitmask_cache: dict[int, tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = {}
+
+
+def _bitmask_to_token_ids(bitmask_row: torch.Tensor, vocab_size: int) -> list[int]:
+    """Convert a packed int32 bitmask row to a list of allowed token IDs."""
+    if vocab_size not in _bitmask_cache:
+        indices = torch.arange(vocab_size)
+        _bitmask_cache[vocab_size] = (
+            indices,
+            indices >> 5,  # i // 32
+            indices & 31,  # i % 32
+        )
+    indices, word_indices, bit_indices = _bitmask_cache[vocab_size]
+    mask = ((bitmask_row[word_indices] >> bit_indices) & 1).bool()
+    return indices[mask].tolist()
+
+
+class BeamSearchOfflineMixin(OfflineInferenceMixin):
+    """Offline inference for beam search"""
+
+    def beam_search(
+        self,
+        prompts: list[TokensPrompt | TextPrompt],
+        params: BeamSearchParams,
+        lora_request: list[LoRARequest] | LoRARequest | None = None,
+        use_tqdm: bool = False,
+        concurrency_limit: int | None = None,
+    ) -> list[BeamSearchOutput]:
+        """
+        Generate sequences using beam search.
+
+        Args:
+            prompts: A list of prompts. Each prompt can be a string or a list
+                of token IDs.
+            params: The beam search parameters.
+            lora_request: LoRA request to use for generation, if any.
+            use_tqdm: Whether to use tqdm to display the progress bar.
+            concurrency_limit: The maximum number of concurrent requests.
+                If None, the number of concurrent requests is unlimited.
+        """
+        # TODO: how does beam search work together with length penalty,
+        # frequency, penalty, and stopping criteria, etc.?
+        beam_width = params.beam_width
+        max_tokens = params.max_tokens
+        temperature = params.temperature
+        ignore_eos = params.ignore_eos
+        length_penalty = params.length_penalty
+
+        tokenizer = self.renderer.get_tokenizer()
+        eos_token_id = tokenizer.eos_token_id
+        sort_beams_key = create_sort_beams_key_function(eos_token_id, length_penalty)
+
+        engine_inputs = self._preprocess_cmpl(prompts)
+        lora_requests = self._lora_request_to_seq(lora_request, len(engine_inputs))
+
+        if use_tqdm and concurrency_limit is not None:
+            logger.warning(
+                "Progress bar is not supported when using concurrency_limit. "
+                "Disabling progress bar."
+            )
+            use_tqdm = False
+
+        if concurrency_limit is None:
+            concurrency_limit = len(engine_inputs)
+
+        structured_output_backend: StructuredOutputBackend | None = None
+        structured_output_key = None
+        structured_output_bitmask = None
+        if params.structured_outputs is not None:
+            (
+                structured_output_backend,
+                structured_output_key,
+                structured_output_bitmask,
+            ) = self._init_beam_search_structured_output(
+                params.structured_outputs, tokenizer
+            )
+
+        # generate 2 * beam_width candidates at each step
+        # following the huggingface transformers implementation
+        # at https://github.com/huggingface/transformers/blob/e15687fffe5c9d20598a19aeab721ae0a7580f8a/src/transformers/generation/beam_search.py#L534 # noqa
+        base_sampling_params = SamplingParams(
+            logprobs=2 * beam_width,
+            max_tokens=1,
+            temperature=temperature,
+            skip_clone=True,  # Internal beam search, safe to skip clone
+        )
+        instances: list[BeamSearchInstance] = []
+
+        for lora_req, prompt in zip(lora_requests, engine_inputs):
+            if prompt["type"] == "embeds":
+                raise NotImplementedError(
+                    "Embedding prompt not supported for beam search"
+                )
+
+            instances.append(
+                BeamSearchInstance(
+                    prompt,
+                    lora_request=lora_req,
+                    logprobs=None,
+                ),
+            )
+
+        try:
+            for prompt_start in range(0, len(instances), concurrency_limit):
+                instances_batch = instances[
+                    prompt_start : prompt_start + concurrency_limit
+                ]
+
+                token_iter = range(max_tokens)
+                if use_tqdm:
+                    token_iter = tqdm(
+                        token_iter,
+                        desc="Beam search",
+                        unit="token",
+                        unit_scale=False,
+                    )
+                    logger.warning(
+                        "The progress bar shows the upper bound on token "
+                        "steps and may finish early due to stopping "
+                        "conditions. It does not reflect instance-level "
+                        "progress."
+                    )
+                for _ in token_iter:
+                    should_stop = self._beam_search_step(
+                        instances_batch=instances_batch,
+                        base_sampling_params=base_sampling_params,
+                        eos_token_id=eos_token_id,
+                        ignore_eos=ignore_eos,
+                        beam_width=beam_width,
+                        sort_beams_key=sort_beams_key,
+                        structured_output_backend=structured_output_backend,
+                        structured_output_key=structured_output_key,
+                        structured_output_bitmask=structured_output_bitmask,
+                    )
+                    if should_stop:
+                        break
+        finally:
+            if structured_output_backend is not None:
+                structured_output_backend.destroy()
+
+        outputs = []
+        for instance in instances:
+            instance.completed.extend(instance.beams)
+            sorted_completed = sorted(
+                instance.completed, key=sort_beams_key, reverse=True
+            )
+            best_beams = sorted_completed[:beam_width]
+
+            for beam in best_beams:
+                beam.text = tokenizer.decode(beam.tokens)
+
+            outputs.append(BeamSearchOutput(sequences=best_beams))
+
+        return outputs
+
+    def _beam_search_step(
+        self,
+        instances_batch: list[BeamSearchInstance],
+        base_sampling_params: SamplingParams,
+        eos_token_id: int | None,
+        ignore_eos: bool,
+        beam_width: int,
+        sort_beams_key: Callable,
+        structured_output_backend: StructuredOutputBackend | None,
+        structured_output_key: tuple | None,
+        structured_output_bitmask: torch.Tensor | None,
+    ) -> bool:
+        """Run one token step of beam search across a batch of instances.
+
+        Returns True if all beams are exhausted and search should stop.
+        """
+        all_beams: list[BeamSearchSequence] = list(
+            sum((instance.beams for instance in instances_batch), [])
+        )
+        pos = [0] + list(
+            itertools.accumulate(len(instance.beams) for instance in instances_batch)
+        )
+        instance_start_and_end: list[tuple[int, int]] = list(zip(pos[:-1], pos[1:]))
+
+        if len(all_beams) == 0:
+            return True
+
+        if structured_output_backend is not None:
+            assert (
+                structured_output_key is not None
+                and structured_output_bitmask is not None
+            )
+            beam_entries = self._build_beam_sampling_params(
+                all_beams,
+                base_sampling_params,
+                structured_output_backend,
+                structured_output_key,
+                structured_output_bitmask,
+            )
+            active_indices = [
+                i for i, entry in enumerate(beam_entries) if entry is not None
+            ]
+            for i, entry in enumerate(beam_entries):
+                if entry is None:
+                    beam = all_beams[i]
+                    assert beam.orig_prompt["type"] != "enc_dec"
+                    prompt_len = len(beam.orig_prompt["prompt_token_ids"])
+                    if len(beam.tokens) > prompt_len:
+                        for (s, e), inst in zip(
+                            instance_start_and_end,
+                            instances_batch,
+                        ):
+                            if s <= i < e:
+                                inst.completed.append(beam)
+                                break
+
+            if not active_indices:
+                return True
+
+            active_beams = [all_beams[i] for i in active_indices]
+            active_params: Sequence[SamplingParams | PoolingParams] = [
+                beam_entries[i][0]  # type: ignore[index]
+                for i in active_indices
+            ]
+        else:
+            active_indices = list(range(len(all_beams)))
+            active_beams = all_beams
+            active_params = self._params_to_seq(  # type: ignore[assignment]
+                base_sampling_params, len(all_beams)
+            )
+
+        # only runs for one step
+        # we don't need to use tqdm here
+        active_output = self._render_and_run_requests(
+            prompts=(beam.get_prompt() for beam in active_beams),
+            params=active_params,
+            output_type=RequestOutput,
+            lora_requests=[beam.lora_request for beam in active_beams],
+            use_tqdm=False,
+        )
+
+        output: list[RequestOutput | None] = [None] * len(all_beams)
+        for idx, active_idx in enumerate(active_indices):
+            output[active_idx] = active_output[idx]
+
+        # Logprobs are computed from raw logits before
+        # allowed_token_ids masking, so they may contain
+        # tokens outside the grammar's allowed set. This filtering is also
+        # the only grammar enforcement for beams whose allowed set exceeds
+        # the engine-side allowed_token_ids cap.
+        allowed_sets: list[set[int] | None] = [None] * len(all_beams)
+        if structured_output_backend is not None:
+            for i, entry in enumerate(beam_entries):
+                if entry is not None:
+                    allowed_sets[i] = set(entry[1])
+
+        for (start, end), instance in zip(instance_start_and_end, instances_batch):
+            instance_new_beams = []
+            for i in range(start, end):
+                current_beam = all_beams[i]
+                result = output[i]
+
+                if result is None:
+                    continue
+
+                if result.outputs[0].logprobs is not None:
+                    # if logprobs is None, the sequence completed
+                    # due to max-model-len or abortion.
+                    logprobs = result.outputs[0].logprobs[0]
+                    allowed = allowed_sets[i]
+                    for token_id, logprob_obj in logprobs.items():
+                        if allowed is not None and token_id not in allowed:
+                            continue
+                        new_beam = BeamSearchSequence(
+                            current_beam.orig_prompt,
+                            tokens=current_beam.tokens + [token_id],
+                            logprobs=current_beam.logprobs + [logprobs],
+                            lora_request=current_beam.lora_request,
+                            cum_logprob=current_beam.cum_logprob + logprob_obj.logprob,
+                        )
+
+                        if token_id == eos_token_id and not ignore_eos:
+                            instance.completed.append(new_beam)
+                        else:
+                            instance_new_beams.append(new_beam)
+            sorted_beams = sorted(
+                instance_new_beams,
+                key=sort_beams_key,
+                reverse=True,
+            )
+            instance.beams = sorted_beams[:beam_width]
+
+        return False
+
+    def _init_beam_search_structured_output(
+        self,
+        structured_outputs: StructuredOutputsParams,
+        tokenizer: TokenizerLike,
+    ) -> tuple[StructuredOutputBackend, tuple, torch.Tensor]:
+        """Initialize the structured output backend for beam search."""
+        vllm_config = self.llm_engine.vllm_config
+        so_config = vllm_config.structured_outputs_config
+        if so_config is None:
+            raise ValueError(
+                "structured_outputs_config is required for beam search "
+                "with structured outputs"
+            )
+
+        # Resolve the backend name from engine config if not already set.
+        if not structured_outputs._backend:
+            structured_outputs._backend = so_config.backend
+
+        backend_name = structured_outputs._backend
+        vocab_size = self.model_config.get_vocab_size()
+
+        backend: StructuredOutputBackend
+        if backend_name == "xgrammar":
+            from vllm.v1.structured_output.backend_xgrammar import (
+                XgrammarBackend,
+            )
+
+            backend = XgrammarBackend(
+                vllm_config=vllm_config,
+                tokenizer=tokenizer,
+                vocab_size=vocab_size,
+            )
+        elif backend_name == "guidance":
+            from vllm.v1.structured_output.backend_guidance import (
+                GuidanceBackend,
+            )
+
+            backend = GuidanceBackend(
+                vllm_config=vllm_config,
+                tokenizer=tokenizer,
+                vocab_size=vocab_size,
+            )
+        elif backend_name == "outlines":
+            from vllm.v1.structured_output.backend_outlines import (
+                OutlinesBackend,
+            )
+
+            backend = OutlinesBackend(
+                vllm_config=vllm_config,
+                tokenizer=tokenizer,
+                vocab_size=vocab_size,
+            )
+        elif backend_name == "lm-format-enforcer":
+            from vllm.v1.structured_output.backend_lm_format_enforcer import (
+                LMFormatEnforcerBackend,
+            )
+
+            backend = LMFormatEnforcerBackend(
+                vllm_config=vllm_config,
+                tokenizer=tokenizer,
+                vocab_size=vocab_size,
+            )
+        else:
+            raise ValueError(f"Unsupported structured output backend: {backend_name}")
+
+        structured_output_key = get_structured_output_key(structured_outputs)
+        bitmask = backend.allocate_token_bitmask(1)
+
+        return backend, structured_output_key, bitmask
+
+    def _build_beam_sampling_params(
+        self,
+        beams: list[BeamSearchSequence],
+        base_params: SamplingParams,
+        backend: StructuredOutputBackend,
+        structured_output_key: tuple,
+        bitmask: torch.Tensor,
+    ) -> list[tuple[SamplingParams, list[int]] | None]:
+        """Build per-beam SamplingParams and allowed token IDs from grammar.
+
+        Returns None for beams where the grammar has terminated.
+        """
+        vocab_size = self.model_config.get_vocab_size()
+        request_type, grammar_spec = structured_output_key
+        result: list[tuple[SamplingParams, list[int]] | None] = []
+
+        for beam in beams:
+            # Fresh grammar per beam, replaying generated tokens.
+            # Backends don't support cloning grammar state, so
+            # replay is needed to reconstruct the FSM position.
+            grammar = backend.compile_grammar(request_type, grammar_spec)
+            assert beam.orig_prompt["type"] != "enc_dec"
+            prompt_len = len(beam.orig_prompt["prompt_token_ids"])
+            generated_tokens = beam.tokens[prompt_len:]
+
+            if generated_tokens:
+                grammar.accept_tokens("beam", generated_tokens)
+
+            if grammar.is_terminated():
+                result.append(None)
+                continue
+
+            grammar.fill_bitmask(bitmask, 0)
+            allowed_ids = _bitmask_to_token_ids(bitmask[0], vocab_size)
+
+            if not allowed_ids:
+                result.append(None)
+                continue
+
+            # The engine caps the size of allowed_token_ids. While the
+            # grammar still allows more tokens than the cap (e.g. inside
+            # free-form strings), skip the engine-side constraint and rely
+            # on the logprobs filtering in _beam_search_step instead.
+            beam_params = SamplingParams(
+                logprobs=base_params.logprobs,
+                max_tokens=1,
+                temperature=base_params.temperature,
+                allowed_token_ids=(
+                    allowed_ids
+                    if len(allowed_ids) <= _MAX_NUM_ALLOWED_TOKEN_IDS
+                    else None
+                ),
+                skip_clone=True,
+            )
+            result.append((beam_params, allowed_ids))
+
+        return result

vllm/entrypoints/generate/beam_search/online.py

@@ -0,0 +1,220 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import asyncio
+from abc import ABC
+from collections.abc import AsyncGenerator, Mapping
+
+import numpy as np
+
+from vllm import CompletionOutput, RequestOutput
+from vllm.engine.protocol import EngineClient
+from vllm.inputs import EngineInput
+from vllm.lora.request import LoRARequest
+from vllm.renderers import BaseRenderer
+from vllm.sampling_params import BeamSearchParams, SamplingParams
+from vllm.utils import random_uuid
+from vllm.utils.async_utils import collect_from_async_generator
+
+from .utils import BeamSearchSequence, create_sort_beams_key_function
+
+
+class BeamSearchOnlineMixin(ABC):
+    """online serving for beam search"""
+
+    renderer: BaseRenderer
+    engine_client: EngineClient
+
+    async def beam_search(
+        self,
+        prompt: EngineInput,
+        request_id: str,
+        params: BeamSearchParams,
+        lora_request: LoRARequest | None = None,
+        trace_headers: Mapping[str, str] | None = None,
+    ) -> AsyncGenerator[RequestOutput, None]:
+        beam_width = params.beam_width
+        max_tokens = params.max_tokens
+        ignore_eos = params.ignore_eos
+        temperature = params.temperature
+        length_penalty = params.length_penalty
+        include_stop_str_in_output = params.include_stop_str_in_output
+
+        tokenizer = self.renderer.get_tokenizer()
+        eos_token_id = tokenizer.eos_token_id
+        sort_beams_key = create_sort_beams_key_function(eos_token_id, length_penalty)
+
+        if prompt["type"] == "embeds":
+            raise NotImplementedError("Embedding prompt not supported for beam search")
+
+        # Extract prompt tokens and text based on model type
+        decoder_prompt = (
+            prompt if prompt["type"] != "enc_dec" else prompt["decoder_prompt"]
+        )
+        prompt_text = decoder_prompt.get("prompt")
+        prompt_token_ids = decoder_prompt["prompt_token_ids"]
+
+        tokenized_length = len(prompt_token_ids)
+
+        logprobs_num = 2 * beam_width
+        sampling_params = SamplingParams(
+            logprobs=logprobs_num,
+            max_tokens=1,
+            temperature=temperature,
+            detokenize=False,
+        )
+        all_beams = [
+            BeamSearchSequence(
+                orig_prompt=prompt,
+                tokens=prompt_token_ids,
+                cum_logprob=0,
+                logprobs=[],
+                lora_request=lora_request,
+            )
+        ]
+        completed = []
+
+        for _ in range(max_tokens):
+            tasks = []
+            request_id_batch = f"{request_id}-{random_uuid()}"
+
+            for i, beam in enumerate(all_beams):
+                prompt_item = beam.get_prompt()
+                lora_request_item = beam.lora_request
+                request_id_item = f"{request_id_batch}-beam-{i}"
+                task = asyncio.create_task(
+                    collect_from_async_generator(
+                        self.engine_client.generate(
+                            prompt_item,
+                            sampling_params,
+                            request_id_item,
+                            lora_request=lora_request_item,
+                            trace_headers=trace_headers,
+                        )
+                    )
+                )
+                tasks.append(task)
+
+            output = [x[0] for x in await asyncio.gather(*tasks)]
+
+            candidates = []
+            # Iterate through all beam inference results
+            for i, result in enumerate(output):
+                current_beam = all_beams[i]
+
+                # check for error finish reason and abort beam search
+                if result.outputs[0].finish_reason == "error":
+                    # yield error output and terminate beam search
+                    yield RequestOutput(
+                        request_id=request_id,
+                        prompt=prompt_text,
+                        outputs=[
+                            CompletionOutput(
+                                index=0,
+                                text="",
+                                token_ids=[],
+                                cumulative_logprob=None,
+                                logprobs=None,
+                                finish_reason="error",
+                            )
+                        ],
+                        finished=True,
+                        prompt_token_ids=prompt_token_ids,
+                        prompt_logprobs=None,
+                    )
+                    return
+
+                if result.outputs[0].logprobs is not None:
+                    logprobs = result.outputs[0].logprobs[0]
+                    for token_id, logprob_obj in logprobs.items():
+                        candidate_logprob = (
+                            current_beam.cum_logprob + logprob_obj.logprob
+                        )
+                        if token_id == eos_token_id and not ignore_eos:
+                            completed.append(
+                                BeamSearchSequence(
+                                    orig_prompt=prompt,
+                                    tokens=current_beam.tokens + [eos_token_id]
+                                    if include_stop_str_in_output
+                                    else current_beam.tokens,
+                                    logprobs=current_beam.logprobs + [logprobs],
+                                    cum_logprob=candidate_logprob,
+                                    finish_reason="stop",
+                                    stop_reason=eos_token_id,
+                                )
+                            )
+                        else:
+                            candidates.append(
+                                (
+                                    candidate_logprob,
+                                    int(token_id),
+                                    current_beam,
+                                    logprobs,
+                                )
+                            )
+
+            # Processing non-EOS tokens
+            candidate_logprobs = np.fromiter(
+                (candidate[0] for candidate in candidates),
+                dtype=np.float64,
+                count=len(candidates),
+            )
+            if len(candidates) <= beam_width:
+                topn_idx = np.argsort(-candidate_logprobs)
+            else:
+                topn_idx = np.argpartition(
+                    -candidate_logprobs,
+                    beam_width - 1,
+                )[:beam_width]
+                topn_idx = topn_idx[np.argsort(-candidate_logprobs[topn_idx])]
+
+            new_beams = []
+            for idx in topn_idx:
+                cum_logprob, token_id, current_beam, logprobs = candidates[int(idx)]
+                new_beams.append(
+                    BeamSearchSequence(
+                        orig_prompt=prompt,
+                        tokens=current_beam.tokens + [token_id],
+                        logprobs=current_beam.logprobs + [logprobs],
+                        lora_request=current_beam.lora_request,
+                        cum_logprob=cum_logprob,
+                    )
+                )
+
+            all_beams = new_beams
+            if not all_beams:
+                break
+
+        completed.extend(all_beams)
+        sorted_completed = sorted(completed, key=sort_beams_key, reverse=True)
+        best_beams = sorted_completed[:beam_width]
+
+        for beam in best_beams:
+            if beam.tokens[-1] == eos_token_id and not ignore_eos:
+                # Skip the eos token in the text.
+                tokens = beam.tokens[tokenized_length:-1]
+            else:
+                tokens = beam.tokens[tokenized_length:]
+            beam.text = tokenizer.decode(tokens)
+
+        yield RequestOutput(
+            request_id=request_id,
+            prompt=prompt_text,
+            outputs=[
+                CompletionOutput(
+                    text=beam.text,  # type: ignore
+                    cumulative_logprob=beam.cum_logprob,
+                    token_ids=beam.tokens[tokenized_length:],
+                    index=i,
+                    logprobs=beam.logprobs,
+                    finish_reason=beam.finish_reason
+                    if beam.finish_reason is not None
+                    else "length",
+                    stop_reason=beam.stop_reason,
+                )
+                for (i, beam) in enumerate(best_beams)
+            ],
+            finished=True,
+            prompt_token_ids=prompt_token_ids,
+            prompt_logprobs=None,
+        )

vllm/entrypoints/generate/beam_search/utils.py

@@ -0,0 +1,162 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from dataclasses import dataclass
+
+from vllm.inputs import (
+    DecoderOnlyEngineInput,
+    EncoderDecoderInput,
+    MultiModalInput,
+    TokensInput,
+    mm_input,
+    tokens_input,
+)
+from vllm.logprobs import Logprob
+from vllm.lora.request import LoRARequest
+
+
+@dataclass
+class BeamSearchSequence:
+    """A sequence for beam search.
+    It keeps track of the tokens and the log probability of the sequence.
+    The text field is optional and will only be filled when the sequence is
+    about to be returned to the user.
+    """
+
+    orig_prompt: TokensInput | MultiModalInput | EncoderDecoderInput
+
+    # NOTE: Tokens represents decoder tokens in the encoder / decoder case
+    tokens: list[int]
+    logprobs: list[dict[int, Logprob]]
+    lora_request: LoRARequest | None = None
+    cum_logprob: float = 0.0
+    text: str | None = None
+    finish_reason: str | None = None
+    stop_reason: int | str | None = None
+
+    def get_prompt(self):
+        prompt = self.orig_prompt
+
+        if prompt["type"] == "enc_dec":
+            return self._build_encoder_decoder_inputs(prompt)
+
+        # Handle decoder-only inputs
+        prompt_text = prompt.get("prompt")
+        cache_salt = prompt.get("cache_salt")
+
+        if prompt["type"] == "token":
+            return tokens_input(
+                self.tokens,
+                prompt=prompt_text,
+                cache_salt=cache_salt,
+            )
+
+        return mm_input(
+            prompt_token_ids=self.tokens,
+            mm_kwargs=prompt["mm_kwargs"],
+            mm_hashes=prompt["mm_hashes"],
+            mm_placeholders=prompt["mm_placeholders"],
+            prompt=prompt_text,
+            cache_salt=cache_salt,
+        )
+
+    def _build_encoder_decoder_inputs(
+        self, prompt: EncoderDecoderInput
+    ) -> EncoderDecoderInput:
+        """Rebuild the encoder-decoder inputs with the current beam search
+        sequence's tokens.
+
+        FIXME (alex) - the encoder multimodal cache is not properly wired up
+        yet, which means that currently we are running the encoder on every
+        new beam because num_computed_tokens is 0 on each new request. This
+        will be fixed once the cache is correctly implemented.
+        """
+        dec_prompt = prompt["decoder_prompt"]
+
+        # Rebuild decoder prompt with updated tokens,
+        # but keep everything else the same.
+        new_dec_prompt: DecoderOnlyEngineInput
+        if dec_prompt["type"] == "multimodal":
+            new_dec_prompt = mm_input(
+                self.tokens,
+                mm_kwargs=dec_prompt["mm_kwargs"],
+                mm_hashes=dec_prompt["mm_hashes"],
+                mm_placeholders=dec_prompt["mm_placeholders"],
+                prompt=dec_prompt.get("prompt"),
+                cache_salt=dec_prompt.get("cache_salt"),
+            )
+        else:
+            new_dec_prompt = tokens_input(
+                self.tokens,
+                prompt=dec_prompt.get("prompt"),
+                cache_salt=dec_prompt.get("cache_salt"),
+            )
+
+        return EncoderDecoderInput(
+            type="enc_dec",
+            encoder_prompt=prompt["encoder_prompt"],
+            decoder_prompt=new_dec_prompt,
+        )
+
+
+@dataclass
+class BeamSearchOutput:
+    """The output of beam search.
+    It contains the list of the best beam search sequences.
+    The length of the list is equal to the beam width.
+    """
+
+    sequences: list[BeamSearchSequence]
+
+
+class BeamSearchInstance:
+    def __init__(
+        self,
+        prompt: TokensInput | MultiModalInput | EncoderDecoderInput,
+        lora_request: LoRARequest | None = None,
+        logprobs: list[dict[int, Logprob]] | None = None,
+        **kwargs,
+    ):
+        decoder_prompt = (
+            prompt if prompt["type"] != "enc_dec" else prompt["decoder_prompt"]
+        )
+        initial_tokens = decoder_prompt["prompt_token_ids"]
+
+        self.beams: list[BeamSearchSequence] = [
+            BeamSearchSequence(
+                orig_prompt=prompt,
+                tokens=initial_tokens,
+                logprobs=[] if logprobs is None else list(logprobs),
+                lora_request=lora_request,
+                **kwargs,
+            )
+        ]
+        self.completed: list[BeamSearchSequence] = []
+
+
+def get_beam_search_score(
+    tokens: list[int],
+    cumulative_logprob: float,
+    eos_token_id: int,
+    length_penalty: float = 1.0,
+) -> float:
+    """Calculate the beam search score with length penalty.
+
+    Adapted from
+
+    https://github.com/huggingface/transformers/blob/ccb92be23def445f2afdea94c31286f84b89eb5b/src/transformers/generation/beam_search.py#L938
+    """
+    seq_len = len(tokens)
+    if tokens[-1] == eos_token_id:
+        seq_len -= 1
+
+    return cumulative_logprob / (seq_len**length_penalty)
+
+
+def create_sort_beams_key_function(eos_token_id: int, length_penalty: float):
+    def sort_beams_key(x: BeamSearchSequence) -> float:
+        return get_beam_search_score(
+            x.tokens, x.cum_logprob, eos_token_id, length_penalty
+        )
+
+    return sort_beams_key

vllm/entrypoints/generate/factories.py

@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from typing import TYPE_CHECKING
+
+from vllm.config import ModelConfig
+from vllm.tasks import SupportedTask
+
+if TYPE_CHECKING:
+    from vllm.entrypoints.serve.sagemaker.api_router import (
+        EndpointFn,
+        GetHandlerFn,
+        RequestType,
+    )
+
+
+def get_generate_invocation_types(
+    supported_tasks: tuple["SupportedTask", ...],
+    model_config: ModelConfig | None = None,
+):
+    # NOTE: Items defined earlier take higher priority
+    invocation_types: list[tuple[RequestType, tuple[GetHandlerFn, EndpointFn]]] = []
+
+    if "generate" in supported_tasks:
+        from vllm.entrypoints.openai.chat_completion.api_router import (
+            chat,
+            create_chat_completion,
+        )
+        from vllm.entrypoints.openai.chat_completion.protocol import (
+            ChatCompletionRequest,
+        )
+        from vllm.entrypoints.openai.completion.api_router import (
+            completion,
+            create_completion,
+        )
+        from vllm.entrypoints.openai.completion.protocol import CompletionRequest
+
+        invocation_types += [
+            (ChatCompletionRequest, (chat, create_chat_completion)),
+            (CompletionRequest, (completion, create_completion)),
+        ]
+
+    return invocation_types

vllm/entrypoints/generate/generative_scoring/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

vllm/entrypoints/generate/generative_scoring/api_router.py

@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from http import HTTPStatus
+
+from fastapi import APIRouter, Depends, FastAPI, Request
+from fastapi.responses import JSONResponse
+
+from vllm.entrypoints.generate.generative_scoring.serving import (
+    GenerativeScoringResponse,
+    ServingGenerativeScoring,
+)
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
+from vllm.entrypoints.serve.utils.api_utils import (
+    load_aware_call,
+    validate_json_request,
+    with_cancellation,
+)
+from vllm.logger import init_logger
+
+router = APIRouter()
+
+logger = init_logger(__name__)
+
+
+def generative_scoring(request: Request) -> ServingGenerativeScoring | None:
+    return request.app.state.serving_generative_scoring
+
+
+@router.post(
+    "/generative_scoring",
+    dependencies=[Depends(validate_json_request)],
+    responses={
+        HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
+        HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
+    },
+)
+@with_cancellation
+@load_aware_call
+async def create_generative_scoring(raw_request: Request):
+    handler = generative_scoring(raw_request)
+    if handler is None:
+        raise NotImplementedError(
+            "The model does not support the Generative Scoring API"
+        )
+
+    raw_body = await raw_request.json()
+
+    from vllm.entrypoints.generate.generative_scoring.serving import (
+        GenerativeScoringRequest,
+    )
+
+    gen_request = GenerativeScoringRequest(**raw_body)
+    result = await handler.create_generative_scoring(gen_request, raw_request)
+
+    if isinstance(result, ErrorResponse):
+        return JSONResponse(content=result.model_dump(), status_code=result.error.code)
+    elif isinstance(result, GenerativeScoringResponse):
+        return JSONResponse(content=result.model_dump())
+
+    raise ValueError(f"Unexpected response type: {type(result)}")
+
+
+def register_generative_scoring_api_router(app: FastAPI):
+    app.include_router(router)

vllm/entrypoints/generate/generative_scoring/serving.py

@@ -0,0 +1,493 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Generative Scoring implementation for generative models.
+
+This module implements generative scoring functionality that computes the
+probability of specified token IDs appearing as the next token after a
+given query+item prompt. This works on any generative model that produces
+logits (task="generate").
+"""
+
+import asyncio
+import math
+import time
+from collections.abc import AsyncGenerator, Mapping
+from typing import Literal
+
+from fastapi import Request
+from pydantic import Field
+
+from vllm.engine.protocol import EngineClient
+from vllm.entrypoints.openai.engine.protocol import (
+    ErrorResponse,
+    OpenAIBaseModel,
+    UsageInfo,
+)
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
+from vllm.entrypoints.serve.engine.serving import BaseServing
+from vllm.entrypoints.serve.utils.request_logger import RequestLogger
+from vllm.inputs import EngineInput, tokens_input
+from vllm.logger import init_logger
+from vllm.outputs import RequestOutput
+from vllm.sampling_params import SamplingParams
+from vllm.tokenizers import TokenizerLike
+from vllm.tracing import (
+    contains_trace_headers,
+    extract_trace_headers,
+    log_tracing_disabled_warning,
+)
+from vllm.utils import random_uuid
+from vllm.utils.async_utils import merge_async_iterators
+
+logger = init_logger(__name__)
+
+
+# ============================================================================
+# Protocol definitions
+# ============================================================================
+
+
+class GenerativeScoringRequest(OpenAIBaseModel):
+    """Request for computing generative scoring.
+
+    Attributes:
+        model: The model to use for scoring. Optional, follows existing patterns.
+        query: The query text or pre-tokenized query token IDs.
+        items: The item text(s) or pre-tokenized item token IDs.
+        label_token_ids: List of token IDs to compute probabilities for.
+        apply_softmax: Whether to normalize probabilities using softmax over only
+            the label_token_ids (True) or return true model probabilities over
+            the full vocab for those ids (False).
+        item_first: If True, prepend items to query. Otherwise append items to query.
+        add_special_tokens: Whether to add special tokens when tokenizing.
+    """
+
+    model: str | None = None
+    query: str | list[int] = Field(
+        ...,
+        description="The query text or pre-tokenized query token IDs.",
+    )
+    items: list[str] | list[list[int]] = Field(
+        ...,
+        description="List of item texts or pre-tokenized item token IDs.",
+    )
+    label_token_ids: list[int] = Field(
+        ...,
+        description="List of token IDs to compute probabilities for.",
+    )
+    apply_softmax: bool = Field(
+        default=True,
+        description=(
+            "If True, normalize probabilities using softmax over only the "
+            "label_token_ids. If False, return the true model probabilities "
+            "over the full vocab for those ids."
+        ),
+    )
+    item_first: bool = Field(
+        default=False,
+        description="If True, prepend items to query. Otherwise append items to query.",
+    )
+    add_special_tokens: bool = Field(
+        default=True,
+        description="Whether to add special tokens when tokenizing.",
+    )
+    priority: int = Field(
+        default=0,
+        description=(
+            "The priority of the request (lower means earlier handling; default: 0)."
+        ),
+    )
+    request_id: str = Field(
+        default_factory=random_uuid,
+        description="The request_id related to this request.",
+    )
+
+
+class GenerativeScoringItemResult(OpenAIBaseModel):
+    """Result for a single item in the generative scoring response.
+
+    Attributes:
+        index: The index of this item in the input items list.
+        object: Type of object, always "score".
+        score: The probability score for the first label token.
+    """
+
+    index: int
+    object: Literal["score"] = "score"
+    score: float
+
+
+class GenerativeScoringResponse(OpenAIBaseModel):
+    """Response from the generative scoring computation.
+
+    Attributes:
+        id: Unique identifier for this response.
+        object: Type of object, always "list".
+        created: Unix timestamp of when the response was created.
+        model: The model used for scoring.
+        data: List of scoring results, one per input item.
+        usage: Token usage information.
+    """
+
+    id: str = Field(default="")
+    object: Literal["list"] = "list"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    model: str
+    data: list[GenerativeScoringItemResult]
+    usage: UsageInfo
+
+
+# ============================================================================
+# Serving class
+# ============================================================================
+
+
+class ServingGenerativeScoring(BaseServing):
+    """Serving class for generative scoring computation.
+
+    This class handles computing the probability of specified token IDs
+    appearing as the next token after concatenating query and item prompts.
+
+    The key operation is:
+    1. For each item, build a prompt: query + item (or item + query if item_first)
+    2. Run a forward pass to get the next token distribution
+    3. Extract probabilities for the specified label_token_ids
+    4. Normalize either over the full vocab (apply_softmax=False) or
+       over just the label_token_ids (apply_softmax=True)
+    """
+
+    def __init__(
+        self,
+        engine_client: EngineClient,
+        models: OpenAIServingModels,
+        *,
+        request_logger: RequestLogger | None,
+    ) -> None:
+        super().__init__(
+            models=models,
+            model_config=engine_client.model_config,
+            request_logger=request_logger,
+        )
+        self.engine_client = engine_client
+        self.renderer = engine_client.renderer
+
+    async def create_generative_scoring(
+        self,
+        request: GenerativeScoringRequest,
+        raw_request: Request | None = None,
+    ) -> GenerativeScoringResponse | ErrorResponse:
+        """Create generative scoring for the given request.
+
+        Args:
+            request: The GenerativeScoringRequest containing query, items, and
+                label_token_ids.
+            raw_request: The raw FastAPI request object.
+
+        Returns:
+            GenerativeScoringResponse with probabilities for each item, or
+            ErrorResponse if an error occurred.
+        """
+        # Check model
+        error_check_ret = await self._check_model(request)  # type: ignore[arg-type]
+        if error_check_ret is not None:
+            return error_check_ret
+
+        # Check if engine is alive
+        if self.engine_client.errored:
+            raise self.engine_client.dead_error
+
+        # Get tokenizer
+        tokenizer = self.renderer.tokenizer
+        if tokenizer is None:
+            return self.create_error_response(
+                "Tokenizer not available. Cannot process generative scoring request."
+            )
+
+        # Validate label_token_ids
+        vocab_size = self.model_config.get_vocab_size()
+        for token_id in request.label_token_ids:
+            if token_id < 0 or token_id >= vocab_size:
+                return self.create_error_response(
+                    f"label_token_id {token_id} is out of vocabulary range "
+                    f"[0, {vocab_size}). Please provide valid token IDs."
+                )
+
+        if len(request.label_token_ids) == 0:
+            return self.create_error_response(
+                "label_token_ids must contain at least one token ID."
+            )
+
+        # Validate items
+        if len(request.items) == 0:
+            return self.create_error_response("items must contain at least one item.")
+
+        # Note: Mixed item types (string and token list) are validated by
+        # Pydantic at request parsing time, so we don't need to check here.
+
+        try:
+            lora_request = self._maybe_get_adapters(request)  # type: ignore[arg-type]
+        except (ValueError, TypeError, RuntimeError) as e:
+            logger.exception("Error preparing request components")
+            return self.create_error_response(e)
+
+        base_id = self._base_request_id(raw_request, default=request.request_id)
+        request_id = f"generative-scoring-{base_id}"
+        created_time = int(time.time())
+
+        # Build prompts for each item
+        try:
+            engine_inputs, prompt_token_counts = await self._build_prompts(
+                request, tokenizer, self.model_config.max_model_len
+            )
+        except (ValueError, TypeError) as e:
+            logger.exception("Error building prompts")
+            return self.create_error_response(e)
+
+        # Create sampling params for scoring
+        # We use max_tokens=1 with logprob_token_ids to efficiently get
+        # logprobs for only the specified label tokens (not full vocab)
+        # Note: temperature/top_k/top_p don't affect logprobs - they only
+        # affect the sampling distribution. Logprobs are computed from raw
+        # logits via log_softmax before any sampling transformations.
+        sampling_params = SamplingParams(
+            max_tokens=1,
+            logprobs=len(request.label_token_ids),
+            logprob_token_ids=request.label_token_ids,
+            n=1,
+        )
+
+        # Get trace headers
+        trace_headers = (
+            None
+            if raw_request is None
+            else await self._get_trace_headers(raw_request.headers)
+        )
+
+        # Schedule requests for all inputs
+        generators: list[AsyncGenerator[RequestOutput, None]] = []
+        for i, engine_input in enumerate(engine_inputs):
+            request_id_item = f"{request_id}-{i}"
+
+            self._log_inputs(
+                request_id_item,
+                engine_input,
+                params=sampling_params,
+                lora_request=lora_request,
+            )
+
+            generator = self.engine_client.generate(
+                engine_input,
+                sampling_params,
+                request_id_item,
+                lora_request=lora_request,
+                trace_headers=trace_headers,
+                priority=request.priority,
+            )
+            generators.append(generator)
+
+        # Collect results
+        result_generator = merge_async_iterators(*generators)
+        results: list[RequestOutput | None] = [None] * len(engine_inputs)
+
+        try:
+            async for i, res in result_generator:
+                results[i] = res
+        except asyncio.CancelledError:
+            return self.create_error_response("Client disconnected")
+        except Exception as e:
+            logger.exception("Error during generation")
+            return self.create_error_response(e)
+
+        # Process results to extract label token probabilities
+        item_results: list[GenerativeScoringItemResult] = []
+        total_prompt_tokens = 0
+        total_completion_tokens = 0
+
+        for i, result in enumerate(results):
+            if result is None:
+                return self.create_error_response(
+                    f"Failed to generate result for item {i}"
+                )
+
+            # Check for errors
+            if result.outputs and result.outputs[0].finish_reason == "error":
+                return self.create_error_response(f"Generation error for item {i}")
+
+            # Get logprobs from the generated token
+            if not result.outputs or len(result.outputs) == 0:
+                return self.create_error_response(f"No output generated for item {i}")
+
+            output = result.outputs[0]
+            if output.logprobs is None or len(output.logprobs) == 0:
+                return self.create_error_response(
+                    f"No logprobs available for item {i}. "
+                    "This might indicate an issue with logprobs configuration."
+                )
+
+            # The logprobs dict maps token_id -> Logprob object
+            # For logprobs=-1, this contains all vocab tokens
+            logprobs_dict = output.logprobs[0]
+
+            # Extract logprobs for label tokens
+            label_logprobs: dict[int, float] = {}
+            missing_tokens = []
+            for token_id in request.label_token_ids:
+                if token_id in logprobs_dict:
+                    label_logprobs[token_id] = logprobs_dict[token_id].logprob
+                else:
+                    missing_tokens.append(token_id)
+
+            if missing_tokens:
+                return self.create_error_response(
+                    f"Token IDs {missing_tokens} not found in logprobs for item {i}. "
+                    "This might indicate the tokens are outside the model's vocabulary."
+                )
+
+            # Compute probabilities based on apply_softmax setting
+            token_probs = self._compute_probabilities(
+                label_logprobs,
+                apply_softmax=request.apply_softmax,
+            )
+
+            # Use the first label token's probability as the score
+            first_label_id = request.label_token_ids[0]
+            score = token_probs[first_label_id]
+
+            item_results.append(
+                GenerativeScoringItemResult(
+                    index=i,
+                    score=score,
+                )
+            )
+
+            # Update token counts
+            total_prompt_tokens += prompt_token_counts[i]
+            total_completion_tokens += len(output.token_ids)
+
+        # Build response
+        model_name = self.models.model_name(lora_request)
+        response = GenerativeScoringResponse(
+            id=request_id,
+            created=created_time,
+            model=model_name,
+            data=item_results,
+            usage=UsageInfo(
+                prompt_tokens=total_prompt_tokens,
+                total_tokens=total_prompt_tokens + total_completion_tokens,
+                completion_tokens=total_completion_tokens,
+            ),
+        )
+
+        return response
+
+    async def _build_prompts(
+        self,
+        request: GenerativeScoringRequest,
+        tokenizer: TokenizerLike,
+        max_model_len: int,
+    ) -> tuple[list[EngineInput], list[int]]:
+        """Build prompts by concatenating query and items.
+
+        Uses the Renderer's tokenizer to tokenize text inputs, then
+        creates EngineInput via tokens_input() for engine consumption.
+
+        Args:
+            request: The request containing query, items, and settings.
+            tokenizer: The tokenizer to use.
+            max_model_len: Maximum model context length for truncation.
+
+        Returns:
+            Tuple of (list of EngineInput, list of prompt token counts).
+        """
+        # Tokenize query if it's a string
+        if isinstance(request.query, str):
+            query_token_ids = tokenizer.encode(
+                request.query,
+                add_special_tokens=request.add_special_tokens,
+            )
+        else:
+            query_token_ids = request.query
+
+        engine_inputs: list[EngineInput] = []
+        prompt_token_counts: list[int] = []
+
+        for item in request.items:
+            # Tokenize item if it's a string
+            if isinstance(item, str):
+                # Don't add special tokens for items to avoid duplicate BOS/EOS
+                item_token_ids = tokenizer.encode(
+                    item,
+                    add_special_tokens=False,
+                )
+            else:
+                item_token_ids = item
+
+            # Concatenate based on item_first setting
+            if request.item_first:
+                prompt_token_ids = item_token_ids + query_token_ids
+            else:
+                prompt_token_ids = query_token_ids + item_token_ids
+
+            # Truncate to max_model_len - 1 to leave room for 1 output token
+            max_prompt_len = max_model_len - 1
+            if len(prompt_token_ids) > max_prompt_len:
+                prompt_token_ids = prompt_token_ids[:max_prompt_len]
+
+            engine_inputs.append(tokens_input(prompt_token_ids))
+            prompt_token_counts.append(len(prompt_token_ids))
+
+        return engine_inputs, prompt_token_counts
+
+    def _compute_probabilities(
+        self,
+        label_logprobs: dict[int, float],
+        apply_softmax: bool,
+    ) -> dict[int, float]:
+        """Compute probabilities from logprobs.
+
+        Args:
+            label_logprobs: Dictionary mapping token_id to logprob.
+            apply_softmax: If True, normalize over only the label tokens.
+                If False, return true model probabilities (exp(logprob)).
+
+        Returns:
+            Dictionary mapping token_id to probability.
+        """
+        if apply_softmax:
+            # Normalize over only the label tokens (subset softmax)
+            # softmax(gathered_logits) over the subset
+            logprobs_list = list(label_logprobs.values())
+            max_logprob = max(logprobs_list)
+
+            # Compute exp(logprob - max) for numerical stability
+            exp_values = {
+                token_id: math.exp(logprob - max_logprob)
+                for token_id, logprob in label_logprobs.items()
+            }
+            sum_exp = sum(exp_values.values())
+
+            return {
+                token_id: exp_val / sum_exp for token_id, exp_val in exp_values.items()
+            }
+        else:
+            # Return true model probabilities
+            # Since logprobs are already log(softmax(logits)),
+            # we just need to exp() them
+            return {
+                token_id: math.exp(logprob)
+                for token_id, logprob in label_logprobs.items()
+            }
+
+    async def _get_trace_headers(
+        self,
+        headers: Mapping[str, str],
+    ) -> Mapping[str, str] | None:
+        """Extract trace headers from request headers."""
+        if not contains_trace_headers(headers):
+            return None
+
+        if not await self.engine_client.is_tracing_enabled():
+            log_tracing_disabled_warning()
+            return None
+
+        return extract_trace_headers(headers)

vllm/entrypoints/grpc_server.py

@@ -0,0 +1,200 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# mypy: ignore-errors
+"""
+vLLM gRPC Server
+
+Starts a gRPC server backed by AsyncLLM, using the VllmEngineServicer
+from the smg-grpc-servicer package.
+
+Usage:
+    python -m vllm.entrypoints.grpc_server --model <model_path>
+
+Example:
+    python -m vllm.entrypoints.grpc_server \
+        --model meta-llama/Llama-2-7b-hf \
+        --host 0.0.0.0 \
+        --port 50051
+"""
+
+import argparse
+import asyncio
+import signal
+import sys
+import time
+
+try:
+    import grpc
+    from grpc_health.v1 import health_pb2_grpc
+    from grpc_reflection.v1alpha import reflection
+    from smg_grpc_proto import vllm_engine_pb2, vllm_engine_pb2_grpc
+    from smg_grpc_servicer.vllm.health_servicer import VllmHealthServicer
+    from smg_grpc_servicer.vllm.servicer import VllmEngineServicer
+except ImportError as e:
+    raise ImportError(
+        "gRPC mode requires smg-grpc-servicer. "
+        "If not installed, run: pip install vllm[grpc]. "
+        "If already installed, there may be a broken import due to a "
+        "version mismatch — see the chained exception above for details."
+    ) from e
+
+import uvloop
+
+from vllm import envs
+from vllm.engine.arg_utils import AsyncEngineArgs
+from vllm.entrypoints.serve.utils.api_utils import log_version_and_model
+from vllm.logger import init_logger
+from vllm.usage.usage_lib import UsageContext
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+from vllm.v1.engine.async_llm import AsyncLLM
+from vllm.version import __version__ as VLLM_VERSION
+
+logger = init_logger(__name__)
+
+
+async def serve_grpc(args: argparse.Namespace):
+    """
+    Main gRPC serving function.
+
+    Args:
+        args: Parsed command line arguments
+    """
+    log_version_and_model(logger, VLLM_VERSION, args.model)
+    logger.info("vLLM gRPC server args: %s", args)
+
+    start_time = time.time()
+
+    # Create engine args
+    engine_args = AsyncEngineArgs.from_cli_args(args)
+
+    # Build vLLM config
+    vllm_config = engine_args.create_engine_config(
+        usage_context=UsageContext.OPENAI_API_SERVER,
+    )
+
+    # Create AsyncLLM
+    async_llm = AsyncLLM.from_vllm_config(
+        vllm_config=vllm_config,
+        usage_context=UsageContext.OPENAI_API_SERVER,
+        enable_log_requests=args.enable_log_requests,
+        disable_log_stats=args.disable_log_stats,
+    )
+
+    # Create servicer
+    servicer = VllmEngineServicer(async_llm, start_time)
+
+    # Create gRPC server
+    server = grpc.aio.server(
+        options=[
+            ("grpc.max_send_message_length", -1),
+            ("grpc.max_receive_message_length", -1),
+            # Tolerate client keepalive pings every 10s (default 300s is too
+            # strict for non-streaming requests where no DATA frames flow
+            # during generation)
+            ("grpc.http2.min_recv_ping_interval_without_data_ms", 10000),
+            ("grpc.keepalive_permit_without_calls", True),
+        ],
+    )
+
+    # Add servicer to server
+    vllm_engine_pb2_grpc.add_VllmEngineServicer_to_server(servicer, server)
+
+    # Add standard gRPC health service for Kubernetes probes
+    health_servicer = VllmHealthServicer(async_llm)
+    health_pb2_grpc.add_HealthServicer_to_server(health_servicer, server)
+
+    # Enable reflection for grpcurl and other tools
+    service_names = (
+        vllm_engine_pb2.DESCRIPTOR.services_by_name["VllmEngine"].full_name,
+        "grpc.health.v1.Health",
+        reflection.SERVICE_NAME,
+    )
+    reflection.enable_server_reflection(service_names, server)
+
+    # Bind to address
+    host = args.host or "0.0.0.0"
+    address = f"{host}:{args.port}"
+    server.add_insecure_port(address)
+
+    try:
+        # Start server
+        await server.start()
+        logger.info("vLLM gRPC server started on %s", address)
+        logger.info("Server is ready to accept requests")
+
+        # Start periodic stats logging (mirrors the HTTP server's lifespan task)
+        if not args.disable_log_stats:
+
+            async def _force_log():
+                while True:
+                    await asyncio.sleep(envs.VLLM_LOG_STATS_INTERVAL)
+                    await async_llm.do_log_stats()
+
+            stats_task = asyncio.create_task(_force_log())
+        else:
+            stats_task = None
+
+        # Handle shutdown signals
+        loop = asyncio.get_running_loop()
+        stop_event = asyncio.Event()
+
+        def signal_handler():
+            logger.info("Received shutdown signal")
+            stop_event.set()
+
+        for sig in (signal.SIGTERM, signal.SIGINT):
+            loop.add_signal_handler(sig, signal_handler)
+
+        try:
+            await stop_event.wait()
+        except KeyboardInterrupt:
+            logger.info("Interrupted by user")
+    finally:
+        logger.info("Shutting down vLLM gRPC server...")
+        if stats_task is not None:
+            stats_task.cancel()
+        try:
+            health_servicer.set_not_serving()
+        except Exception:  # broad: must not prevent server.stop() / shutdown()
+            logger.warning("Failed to set health status to NOT_SERVING", exc_info=True)
+        await server.stop(grace=5.0)
+        logger.info("gRPC server stopped")
+        async_llm.shutdown()
+        logger.info("AsyncLLM engine stopped")
+        logger.info("Shutdown complete")
+
+
+def main():
+    """Main entry point for python -m vllm.entrypoints.grpc_server."""
+    parser = FlexibleArgumentParser(
+        description="vLLM gRPC Server",
+    )
+
+    # Server args
+    parser.add_argument(
+        "--host",
+        type=str,
+        default="0.0.0.0",
+        help="Host to bind gRPC server to",
+    )
+    parser.add_argument(
+        "--port",
+        type=int,
+        default=50051,
+        help="Port to bind gRPC server to",
+    )
+    parser = AsyncEngineArgs.add_cli_args(parser)
+
+    args = parser.parse_args()
+
+    # Run server
+    try:
+        uvloop.run(serve_grpc(args))
+    except Exception as e:
+        logger.exception("Server failed: %s", e)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

vllm/entrypoints/launcher.py

@@ -0,0 +1,178 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import asyncio
+import signal
+import socket
+from functools import partial
+from typing import Any
+
+import uvicorn
+from fastapi import FastAPI
+
+from vllm import envs
+from vllm.engine.protocol import EngineClient
+from vllm.entrypoints.serve.utils.constants import (
+    H11_MAX_HEADER_COUNT_DEFAULT,
+    H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT,
+)
+from vllm.entrypoints.serve.utils.ssl import SSLCertRefresher
+from vllm.logger import init_logger
+from vllm.utils.network_utils import find_process_using_port
+
+logger = init_logger(__name__)
+
+
+async def serve_http(
+    app: FastAPI,
+    sock: socket.socket | None,
+    enable_ssl_refresh: bool = False,
+    **uvicorn_kwargs: Any,
+):
+    """
+    Start a FastAPI app using Uvicorn, with support for custom Uvicorn config
+    options.  Supports http header limits via h11_max_incomplete_event_size and
+    h11_max_header_count.
+    """
+    logger.info("Available routes are:")
+    # post endpoints
+    for route in app.routes:
+        methods = getattr(route, "methods", None)
+        path = getattr(route, "path", None)
+
+        if methods is None or path is None:
+            continue
+
+        logger.info("Route: %s, Methods: %s", path, ", ".join(methods))
+
+    # other endpoints
+    for route in app.routes:
+        endpoint = getattr(route, "endpoint", None)
+        methods = getattr(route, "methods", None)
+        path = getattr(route, "path", None)
+
+        if endpoint is None or path is None or methods is not None:
+            continue
+
+        logger.info("Route: %s, Endpoint: %s", path, endpoint.__name__)
+
+    # Extract header limit options if present
+    h11_max_incomplete_event_size = uvicorn_kwargs.pop(
+        "h11_max_incomplete_event_size", None
+    )
+    h11_max_header_count = uvicorn_kwargs.pop("h11_max_header_count", None)
+
+    # Set safe defaults if not provided
+    if h11_max_incomplete_event_size is None:
+        h11_max_incomplete_event_size = H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT
+    if h11_max_header_count is None:
+        h11_max_header_count = H11_MAX_HEADER_COUNT_DEFAULT
+
+    config = uvicorn.Config(app, **uvicorn_kwargs)
+    # Set header limits
+    config.h11_max_incomplete_event_size = h11_max_incomplete_event_size
+    config.h11_max_header_count = h11_max_header_count
+    config.load()
+    server = uvicorn.Server(config)
+    app.state.server = server
+
+    loop = asyncio.get_running_loop()
+
+    watchdog_task = loop.create_task(watchdog_loop(server, app.state.engine_client))
+    server_task = loop.create_task(server.serve(sockets=[sock] if sock else None))
+
+    ssl_cert_refresher = (
+        None
+        if not enable_ssl_refresh
+        else SSLCertRefresher(
+            ssl_context=config.ssl,
+            key_path=config.ssl_keyfile,
+            cert_path=config.ssl_certfile,
+            ca_path=config.ssl_ca_certs,
+        )
+    )
+
+    shutdown_event = asyncio.Event()
+
+    def signal_handler() -> None:
+        if shutdown_event.is_set():
+            return
+        logger.info_once("[shutdown] API server: shutdown triggered")
+        shutdown_event.set()
+
+    async def dummy_shutdown() -> None:
+        pass
+
+    loop.add_signal_handler(signal.SIGINT, signal_handler)
+    loop.add_signal_handler(signal.SIGTERM, signal_handler)
+
+    async def handle_shutdown() -> None:
+        await shutdown_event.wait()
+
+        engine_client = app.state.engine_client
+        timeout = engine_client.vllm_config.shutdown_timeout
+        mode = "abort" if timeout == 0 else "drain"
+
+        logger.info(
+            "[shutdown] API server: stopping engine client mode=%s timeout=%ss",
+            mode,
+            timeout,
+        )
+
+        await loop.run_in_executor(
+            None, partial(engine_client.shutdown, timeout=timeout)
+        )
+        logger.info_once("[shutdown] API server: engine client stopped")
+
+        server.should_exit = True
+        logger.info_once("[shutdown] API server: signalling HTTP server shutdown")
+        server_task.cancel()
+        watchdog_task.cancel()
+        if ssl_cert_refresher:
+            ssl_cert_refresher.stop()
+
+    shutdown_task = loop.create_task(handle_shutdown())
+
+    try:
+        await server_task
+        return dummy_shutdown()
+    except asyncio.CancelledError:
+        port = uvicorn_kwargs["port"]
+        process = find_process_using_port(port)
+        if process is not None:
+            logger.warning(
+                "port %s is used by process %s launched with command:\n%s",
+                port,
+                process,
+                " ".join(process.cmdline()),
+            )
+        logger.info_once("[shutdown] API server: shutting down FastAPI HTTP server")
+        return server.shutdown()
+    finally:
+        shutdown_task.cancel()
+        watchdog_task.cancel()
+
+
+async def watchdog_loop(server: uvicorn.Server, engine: EngineClient):
+    """
+    # Watchdog task that runs in the background, checking
+    # for error state in the engine. Needed to trigger shutdown
+    # if an exception arises is StreamingResponse() generator.
+    """
+    VLLM_WATCHDOG_TIME_S = 5.0
+    while True:
+        await asyncio.sleep(VLLM_WATCHDOG_TIME_S)
+        terminate_if_errored(server, engine)
+
+
+def terminate_if_errored(server: uvicorn.Server, engine: EngineClient):
+    """
+    See discussions here on shutting down a uvicorn server
+    https://github.com/encode/uvicorn/discussions/1103
+    In this case we cannot await the server shutdown here
+    because handler must first return to close the connection
+    for this request.
+    """
+    engine_errored = engine.errored and not engine.is_running
+    if not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH and engine_errored:
+        server.should_exit = True

vllm/entrypoints/llm.py

@@ -0,0 +1,914 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from collections.abc import Callable, Sequence
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import cloudpickle
+import torch.nn as nn
+from pydantic import ValidationError
+from tqdm.auto import tqdm
+from typing_extensions import overload
+
+from vllm.config import (
+    AttentionConfig,
+    CompilationConfig,
+    PoolerConfig,
+    ProfilerConfig,
+    StructuredOutputsConfig,
+    is_init_field,
+)
+from vllm.config.compilation import CompilationMode
+from vllm.config.model import (
+    ConvertOption,
+    HfOverrides,
+    ModelDType,
+    RunnerOption,
+    TokenizerMode,
+)
+from vllm.config.quantization import QuantizationConfigArgs
+from vllm.distributed.weight_transfer.base import (
+    WeightTransferInitRequest,
+    WeightTransferUpdateRequest,
+)
+from vllm.engine.arg_utils import EngineArgs
+from vllm.entrypoints.chat_utils import (
+    ChatCompletionMessageParam,
+    ChatTemplateContentFormatOption,
+    load_chat_template,
+)
+from vllm.entrypoints.generate.beam_search.offline import BeamSearchOfflineMixin
+from vllm.entrypoints.pooling.offline import PoolingOfflineMixin
+from vllm.entrypoints.serve.utils.api_utils import log_non_default_args
+from vllm.inputs import PromptType
+from vllm.logger import init_logger
+from vllm.lora.request import LoRARequest
+from vllm.model_executor.layers.quantization import QuantizationMethods
+from vllm.outputs import PoolingRequestOutput, RequestOutput
+from vllm.platforms import current_platform
+from vllm.sampling_params import SamplingParams
+from vllm.tokenizers import TokenizerLike
+from vllm.usage.usage_lib import UsageContext
+from vllm.utils.counter import Counter
+from vllm.v1.engine import PauseMode
+from vllm.v1.engine.llm_engine import LLMEngine
+from vllm.v1.sample.logits_processor import LogitsProcessor
+
+from .offline_utils import _O, _R, OfflineInferenceMixin
+
+if TYPE_CHECKING:
+    from vllm.v1.metrics.reader import Metric
+
+logger = init_logger(__name__)
+
+
+class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
+    """An LLM for generating texts from given prompts and sampling parameters.
+
+    This class includes a tokenizer, a language model (possibly distributed
+    across multiple GPUs), and GPU memory space allocated for intermediate
+    states (aka KV cache). Given a batch of prompts and sampling parameters,
+    this class generates texts from the model, using an intelligent batching
+    mechanism and efficient memory management.
+
+    Args:
+        model: The name or path of a HuggingFace Transformers model.
+        tokenizer: The name or path of a HuggingFace Transformers tokenizer.
+        tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer
+            if available, and "slow" will always use the slow tokenizer.
+        skip_tokenizer_init: If true, skip initialization of tokenizer and
+            detokenizer. Expect valid prompt_token_ids and None for prompt
+            from the input.
+        trust_remote_code: Trust remote code (e.g., from HuggingFace) when
+            downloading the model and tokenizer.
+        allowed_local_media_path: Allowing API requests to read local images
+            or videos from directories specified by the server file system.
+            This is a security risk. Should only be enabled in trusted
+            environments.
+        allowed_media_domains: If set, only media URLs that belong to this
+            domain can be used for multi-modal inputs.
+        tensor_parallel_size: The number of GPUs to use for distributed
+            execution with tensor parallelism.
+        dtype: The data type for the model weights and activations. Currently,
+            we support `float32`, `float16`, and `bfloat16`. If `auto`, we use
+            the `dtype` attribute of the Transformers model's config. However,
+            if the `dtype` in the config is `float32`, we will use `float16` instead.
+        quantization: The method used to quantize the model weights. Currently,
+            we support "awq", "gptq", and "fp8" (experimental).
+            If None, we first check the `quantization_config` attribute in the
+            model config file. If that is None, we assume the model weights are
+            not quantized and use `dtype` to determine the data type of
+            the weights.
+        revision: The specific model version to use. It can be a branch name,
+            a tag name, or a commit id.
+        tokenizer_revision: The specific tokenizer version to use. It can be a
+            branch name, a tag name, or a commit id.
+        chat_template: The chat template to apply.
+        seed: The seed to initialize the random number generator for sampling.
+        gpu_memory_utilization: The ratio (between 0 and 1) of GPU memory to
+            reserve for the model weights, activations, and KV cache. Higher
+            values will increase the KV cache size and thus improve the model's
+            throughput. However, if the value is too high, it may cause out-of-
+            memory (OOM) errors.
+        kv_cache_memory_bytes: Size of KV Cache per GPU in bytes. By default,
+            this is set to None and vllm can automatically infer the kv cache
+            size based on gpu_memory_utilization. However, users may want to
+            manually specify the kv cache memory size. kv_cache_memory_bytes
+            allows more fine-grain control of how much memory gets used when
+            compared with using gpu_memory_utilization. Note that
+            kv_cache_memory_bytes (when not-None) ignores
+            gpu_memory_utilization
+        cpu_offload_gb: The size (GiB) of CPU memory to use for offloading
+            the model weights. This virtually increases the GPU memory space
+            you can use to hold the model weights, at the cost of CPU-GPU data
+            transfer for every forward pass.
+        offload_group_size: Prefetch offloading: Group every N layers
+            together. Offload last `offload_num_in_group` layers of each group.
+            Default is 0 (disabled).
+        offload_num_in_group: Prefetch offloading: Number of layers to
+            offload per group. Default is 1.
+        offload_prefetch_step: Prefetch offloading: Number of layers to
+            prefetch ahead. Higher values hide more latency but use more GPU
+            memory. Default is 1.
+        offload_params: Prefetch offloading: Set of parameter name segments
+            to selectively offload. Only parameters whose names contain one of
+            these segments will be offloaded (e.g., {"gate_up_proj", "down_proj"}
+            for MLP weights, or {"w13_weight", "w2_weight"} for MoE expert
+            weights). If None or empty, all parameters are offloaded.
+        enforce_eager: Whether to enforce eager execution. If True, we will
+            disable CUDA graph and always execute the model in eager mode.
+            If False, we will use CUDA graph and eager execution in hybrid.
+        enable_return_routed_experts: Whether to return routed experts.
+        disable_custom_all_reduce: See
+            [ParallelConfig][vllm.config.ParallelConfig].
+        hf_token: The token to use as HTTP bearer authorization for remote files
+            . If `True`, will use the token generated when running
+            `hf auth login` (stored in `~/.cache/huggingface/token`).
+        hf_overrides: If a dictionary, contains arguments to be forwarded to the
+            HuggingFace config. If a callable, it is called to update the
+            HuggingFace config.
+        mm_processor_kwargs: Arguments to be forwarded to the model's processor
+            for multi-modal data, e.g., image processor. Overrides for the
+            multi-modal processor obtained from `AutoProcessor.from_pretrained`.
+            The available overrides depend on the model that is being run.
+            For example, for Phi-3-Vision: `{"num_crops": 4}`.
+        pooler_config: Initialize non-default pooling config for the pooling model,
+            e.g., `PoolerConfig(seq_pooling_type="MEAN", use_activation=False)`.
+        compilation_config: Either an integer or a dictionary. If it is an
+            integer, it is used as the mode of compilation optimization. If it
+            is a dictionary, it can specify the full compilation configuration.
+        attention_config: Configuration for attention mechanisms. Can be a
+            dictionary or an AttentionConfig instance. If a dictionary, it will
+            be converted to an AttentionConfig. Allows specifying the attention
+            backend and other attention-related settings.
+        spec_method: Top-level alias for `speculative_config["method"]`.
+        spec_model: Top-level alias for `speculative_config["model"]`.
+        spec_tokens: Top-level alias for
+            `speculative_config["num_speculative_tokens"]`.
+        **kwargs: Arguments for [`EngineArgs`][vllm.EngineArgs].
+
+    Note:
+        This class is intended to be used for offline inference. For online
+        serving, use the [AsyncLLMEngine][vllm.AsyncLLMEngine] class instead.
+    """
+
+    def __init__(
+        self,
+        model: str,
+        *,
+        runner: RunnerOption = "auto",
+        convert: ConvertOption = "auto",
+        tokenizer: str | None = None,
+        tokenizer_mode: TokenizerMode | str = "auto",
+        skip_tokenizer_init: bool = False,
+        trust_remote_code: bool = False,
+        allowed_local_media_path: str = "",
+        allowed_media_domains: list[str] | None = None,
+        tensor_parallel_size: int = 1,
+        dtype: ModelDType = "auto",
+        quantization: QuantizationMethods | None = None,
+        revision: str | None = None,
+        tokenizer_revision: str | None = None,
+        chat_template: Path | str | None = None,
+        seed: int = 0,
+        gpu_memory_utilization: float = 0.92,
+        cpu_offload_gb: float = 0,
+        offload_group_size: int = 0,
+        offload_num_in_group: int = 1,
+        offload_prefetch_step: int = 1,
+        offload_params: set[str] | None = None,
+        enforce_eager: bool = False,
+        enable_return_routed_experts: bool = False,
+        disable_custom_all_reduce: bool = False,
+        hf_token: bool | str | None = None,
+        hf_overrides: HfOverrides | None = None,
+        mm_processor_kwargs: dict[str, Any] | None = None,
+        pooler_config: PoolerConfig | None = None,
+        structured_outputs_config: dict[str, Any]
+        | StructuredOutputsConfig
+        | None = None,
+        profiler_config: dict[str, Any] | ProfilerConfig | None = None,
+        attention_config: dict[str, Any] | AttentionConfig | None = None,
+        kv_cache_memory_bytes: int | None = None,
+        compilation_config: int | dict[str, Any] | CompilationConfig | None = None,
+        quantization_config: dict[str, Any] | QuantizationConfigArgs | None = None,
+        logits_processors: list[str | type[LogitsProcessor]] | None = None,
+        spec_method: str | None = None,
+        spec_model: str | None = None,
+        spec_tokens: int | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """LLM constructor."""
+
+        if "swap_space" in kwargs:
+            kwargs.pop("swap_space")
+            import warnings
+
+            warnings.warn(
+                "The 'swap_space' parameter is deprecated and ignored. "
+                "It will be removed in a future version.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+
+        if "disable_log_stats" not in kwargs:
+            kwargs["disable_log_stats"] = True
+
+        if "worker_cls" in kwargs:
+            worker_cls = kwargs["worker_cls"]
+            # if the worker_cls is not qualified string name,
+            # we serialize it using cloudpickle to avoid pickling issues
+            if isinstance(worker_cls, type):
+                kwargs["worker_cls"] = cloudpickle.dumps(worker_cls)
+
+        if "kv_transfer_config" in kwargs and isinstance(
+            kwargs["kv_transfer_config"], dict
+        ):
+            from vllm.config.kv_transfer import KVTransferConfig
+
+            raw_config_dict = kwargs["kv_transfer_config"]
+            try:
+                kwargs["kv_transfer_config"] = KVTransferConfig(**raw_config_dict)
+            except ValidationError as e:
+                logger.error(
+                    "Failed to convert 'kv_transfer_config' dict to "
+                    "KVTransferConfig object. Dict: %s. Error: %s",
+                    raw_config_dict,
+                    e,
+                )
+                # Consider re-raising a more specific vLLM error or ValueError
+                # to provide better context to the user.
+                raise ValueError(f"Invalid 'kv_transfer_config' provided: {e}") from e
+
+        if hf_overrides is None:
+            hf_overrides = {}
+
+        def _make_config(value: Any, cls: type[_R]) -> _R:
+            """Convert dict/None/instance to a config instance."""
+            if value is None:
+                return cls()
+            if isinstance(value, dict):
+                return cls(**{k: v for k, v in value.items() if is_init_field(cls, k)})  # type: ignore[arg-type]
+            return value
+
+        if isinstance(compilation_config, int):
+            compilation_config_instance = CompilationConfig(
+                mode=CompilationMode(compilation_config)
+            )
+        else:
+            compilation_config_instance = _make_config(
+                compilation_config, CompilationConfig
+            )
+
+        structured_outputs_instance = _make_config(
+            structured_outputs_config, StructuredOutputsConfig
+        )
+        profiler_config_instance = _make_config(profiler_config, ProfilerConfig)
+        attention_config_instance = _make_config(attention_config, AttentionConfig)
+
+        # warn about single-process data parallel usage.
+        _dp_size = int(kwargs.get("data_parallel_size", 1))
+        _distributed_executor_backend = kwargs.get("distributed_executor_backend")
+        if (
+            _dp_size > 1
+            and not _distributed_executor_backend == "external_launcher"
+            and not current_platform.is_tpu()
+        ):
+            raise ValueError(
+                f"LLM(data_parallel_size={_dp_size}) is not supported for single-"
+                "process usage and may hang. Please use "
+                "the explicit multi-process data-parallel example at "
+                "'examples/features/data_parallel/data_parallel_offline.py'."
+            )
+
+        engine_args = EngineArgs(
+            model=model,
+            runner=runner,
+            convert=convert,
+            tokenizer=tokenizer,
+            tokenizer_mode=tokenizer_mode,
+            skip_tokenizer_init=skip_tokenizer_init,
+            trust_remote_code=trust_remote_code,
+            allowed_local_media_path=allowed_local_media_path,
+            allowed_media_domains=allowed_media_domains,
+            tensor_parallel_size=tensor_parallel_size,
+            dtype=dtype,
+            quantization=quantization,
+            revision=revision,
+            tokenizer_revision=tokenizer_revision,
+            seed=seed,
+            gpu_memory_utilization=gpu_memory_utilization,
+            kv_cache_memory_bytes=kv_cache_memory_bytes,
+            cpu_offload_gb=cpu_offload_gb,
+            offload_group_size=offload_group_size,
+            offload_num_in_group=offload_num_in_group,
+            offload_prefetch_step=offload_prefetch_step,
+            offload_params=offload_params or set(),
+            enforce_eager=enforce_eager,
+            enable_return_routed_experts=enable_return_routed_experts,
+            disable_custom_all_reduce=disable_custom_all_reduce,
+            hf_token=hf_token,
+            hf_overrides=hf_overrides,
+            mm_processor_kwargs=mm_processor_kwargs,
+            pooler_config=pooler_config,
+            structured_outputs_config=structured_outputs_instance,
+            profiler_config=profiler_config_instance,
+            attention_config=attention_config_instance,
+            compilation_config=compilation_config_instance,
+            quantization_config=quantization_config,
+            logits_processors=logits_processors,
+            spec_method=spec_method,
+            spec_model=spec_model,
+            spec_tokens=spec_tokens,
+            **kwargs,
+        )
+
+        log_non_default_args(engine_args)
+
+        self.llm_engine = LLMEngine.from_engine_args(
+            engine_args=engine_args, usage_context=UsageContext.LLM_CLASS
+        )
+        self.model_config = self.llm_engine.model_config
+        self.engine_class = type(self.llm_engine)
+
+        self.request_counter = Counter()
+        self.default_sampling_params: dict[str, Any] | None = None
+
+        supported_tasks = self.llm_engine.get_supported_tasks()
+        self.supported_tasks = supported_tasks
+
+        self.runner_type = self.model_config.runner_type
+        self.renderer = self.llm_engine.renderer
+        self.chat_template = load_chat_template(chat_template)
+        self.input_processor = self.llm_engine.input_processor
+
+        # The renderer thread pool is only consumed by the async renderer
+        # path; the synchronous `LLM` entrypoint runs multimodal
+        # preprocessing serially. Warn so the setting is not a silent
+        # no-op. See vllm-project/vllm#42901.
+        if self.model_config.renderer_num_workers > 1:
+            logger.warning_once(
+                "`renderer_num_workers=%d` was set, but the offline `LLM` "
+                "entrypoint uses the synchronous renderer path and runs "
+                "multimodal preprocessing serially across prompts. The "
+                "renderer thread pool is only consumed by the async "
+                "renderer path used by `vllm serve` / `AsyncLLM`, so this "
+                "setting has no effect here.",
+                self.model_config.renderer_num_workers,
+            )
+
+        PoolingOfflineMixin.__init__(self)
+
+        # Cache for __repr__ to avoid repeated collective_rpc calls
+        self._cached_repr: str | None = None
+
+    @classmethod
+    def from_engine_args(cls, engine_args: EngineArgs) -> "LLM":
+        """Create an LLM instance from EngineArgs."""
+        return cls(**vars(engine_args))
+
+    def get_tokenizer(self) -> TokenizerLike:
+        return self.llm_engine.get_tokenizer()
+
+    def get_world_size(self, include_dp: bool = True) -> int:
+        """Get the world size from the parallel config.
+
+        Args:
+            include_dp: If True (default), returns the world size including
+                data parallelism (TP * PP * DP). If False, returns the world
+                size without data parallelism (TP * PP).
+
+        Returns:
+            The world size (tensor_parallel_size * pipeline_parallel_size),
+            optionally multiplied by data_parallel_size if include_dp is True.
+        """
+        parallel_config = self.llm_engine.vllm_config.parallel_config
+        if include_dp:
+            return parallel_config.world_size_across_dp
+        return parallel_config.world_size
+
+    def reset_mm_cache(self) -> None:
+        self.renderer.clear_mm_cache()
+        self.llm_engine.reset_mm_cache()
+
+    def get_default_sampling_params(self) -> SamplingParams:
+        if self.default_sampling_params is None:
+            self.default_sampling_params = self.model_config.get_diff_sampling_param()
+        if self.default_sampling_params:
+            return SamplingParams.from_optional(**self.default_sampling_params)
+        return SamplingParams()
+
+    def generate(
+        self,
+        prompts: PromptType | Sequence[PromptType],
+        sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,
+        *,
+        use_tqdm: bool | Callable[..., tqdm] = True,
+        lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
+        priority: list[int] | None = None,
+        tokenization_kwargs: dict[str, Any] | None = None,
+        mm_processor_kwargs: dict[str, Any] | None = None,
+    ) -> list[RequestOutput]:
+        """Generates the completions for the input prompts.
+
+        This class automatically batches the given prompts, considering
+        the memory constraint. For the best performance, put all of your prompts
+        into a single list and pass it to this method.
+
+        Args:
+            prompts: The prompts to the LLM. You may pass a sequence of prompts
+                for batch inference. See [PromptType][vllm.inputs.PromptType]
+                for more details about the format of each prompt.
+            sampling_params: The sampling parameters for text generation. If
+                None, we use the default sampling parameters.
+                When it is a single value, it is applied to every prompt.
+                When it is a list, the list must have the same length as the
+                prompts and it is paired one by one with the prompt.
+            use_tqdm: If `True`, shows a tqdm progress bar.
+                If a callable (e.g., `functools.partial(tqdm, leave=False)`),
+                it is used to create the progress bar.
+                If `False`, no progress bar is created.
+            lora_request: LoRA request to use for generation, if any.
+            priority: The priority of the requests, if any.
+                Only applicable when priority scheduling policy is enabled.
+                If provided, must be a list of integers matching the length
+                of `prompts`, where each priority value corresponds to the prompt
+                at the same index.
+            tokenization_kwargs: Overrides for `tokenizer.encode`.
+            mm_processor_kwargs: Overrides for `processor.__call__`.
+
+        Returns:
+            A list of `RequestOutput` objects containing the
+            generated completions in the same order as the input prompts.
+        """
+        runner_type = self.model_config.runner_type
+        if runner_type != "generate":
+            raise ValueError(
+                "LLM.generate() is only supported for generative models. "
+                "Try passing `--runner generate` to use the model as a "
+                "generative model."
+            )
+
+        if sampling_params is None:
+            sampling_params = self.get_default_sampling_params()
+
+        return self._run_completion(
+            prompts=prompts,
+            params=sampling_params,
+            output_type=RequestOutput,
+            use_tqdm=use_tqdm,
+            lora_request=lora_request,
+            tokenization_kwargs=tokenization_kwargs,
+            priority=priority,
+            mm_processor_kwargs=mm_processor_kwargs,
+        )
+
+    def enqueue(
+        self,
+        prompts: PromptType | Sequence[PromptType],
+        sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,
+        lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
+        priority: list[int] | None = None,
+        use_tqdm: bool | Callable[..., tqdm] = True,
+        tokenization_kwargs: dict[str, Any] | None = None,
+        mm_processor_kwargs: dict[str, Any] | None = None,
+    ) -> list[str]:
+        """Enqueue prompts for generation without waiting for completion.
+
+        This method adds requests to the engine queue but does not start
+        processing them. Use wait_for_completion() to process the queued
+        requests and get results.
+
+        Args:
+            prompts: The prompts to the LLM. See generate() for details.
+            sampling_params: The sampling parameters for text generation.
+            lora_request: LoRA request to use for generation, if any.
+            priority: The priority of the requests, if any.
+            use_tqdm: If True, shows a tqdm progress bar while adding requests.
+            tokenization_kwargs: Overrides for `tokenizer.encode`.
+            mm_processor_kwargs: Overrides for `processor.__call__`.
+
+        Returns:
+            A list of request IDs for the enqueued requests.
+        """
+        runner_type = self.model_config.runner_type
+        if runner_type != "generate":
+            raise ValueError("LLM.enqueue() is only supported for generative models.")
+
+        if sampling_params is None:
+            sampling_params = self.get_default_sampling_params()
+
+        return self._add_completion_requests(
+            prompts=prompts,
+            params=sampling_params,
+            use_tqdm=use_tqdm,
+            lora_request=lora_request,
+            priority=priority,
+            tokenization_kwargs=tokenization_kwargs,
+            mm_processor_kwargs=mm_processor_kwargs,
+        )
+
+    @overload
+    def wait_for_completion(
+        self,
+        *,
+        use_tqdm: bool | Callable[..., tqdm] = True,
+    ) -> list[RequestOutput | PoolingRequestOutput]: ...
+
+    @overload
+    def wait_for_completion(
+        self,
+        output_type: type[_O] | tuple[type[_O], ...],
+        *,
+        use_tqdm: bool | Callable[..., tqdm] = True,
+    ) -> list[_O]: ...
+
+    def wait_for_completion(
+        self,
+        output_type: type[Any] | tuple[type[Any], ...] | None = None,
+        *,
+        use_tqdm: bool | Callable[..., tqdm] = True,
+    ) -> list[Any]:
+        """Wait for all enqueued requests to complete and return results.
+
+        This method processes all requests currently in the engine queue
+        and returns their outputs. Use after enqueue() to get results.
+
+        Args:
+            output_type: The expected output type(s). If not provided, accepts
+                both RequestOutput and PoolingRequestOutput.
+            use_tqdm: If True, shows a tqdm progress bar.
+
+        Returns:
+            A list of output objects for all completed requests.
+        """
+        if output_type is None:
+            output_type = (RequestOutput, PoolingRequestOutput)
+
+        return self._run_engine(output_type, use_tqdm=use_tqdm)
+
+    def collective_rpc(
+        self,
+        method: str | Callable[..., _R],
+        timeout: float | None = None,
+        args: tuple = (),
+        kwargs: dict[str, Any] | None = None,
+    ) -> list[_R]:
+        """
+        Execute an RPC call on all workers.
+
+        Args:
+            method: Name of the worker method to execute, or a callable that
+                is serialized and sent to all workers to execute.
+
+                If the method is a callable, it should accept an additional
+                `self` argument, in addition to the arguments passed in `args`
+                and `kwargs`. The `self` argument will be the worker object.
+            timeout: Maximum time in seconds to wait for execution. Raises a
+                [`TimeoutError`][] on timeout. `None` means wait indefinitely.
+            args: Positional arguments to pass to the worker method.
+            kwargs: Keyword arguments to pass to the worker method.
+
+        Returns:
+            A list containing the results from each worker.
+
+        Note:
+            It is recommended to use this API to only pass control messages,
+            and set up data-plane communication to pass data.
+        """
+
+        return self.llm_engine.collective_rpc(method, timeout, args, kwargs)
+
+    def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]:
+        """
+        Run a function directly on the model inside each worker,
+        returning the result for each of them.
+
+        !!! warning
+            To reduce the overhead of data transfer, avoid returning large
+            arrays or tensors from this method. If you must return them,
+            make sure you move them to CPU first to avoid taking up additional
+            VRAM!
+        """
+        return self.llm_engine.apply_model(func)
+
+    def chat(
+        self,
+        messages: list[ChatCompletionMessageParam]
+        | Sequence[list[ChatCompletionMessageParam]],
+        sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,
+        use_tqdm: bool | Callable[..., tqdm] = True,
+        lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
+        chat_template: str | None = None,
+        chat_template_content_format: ChatTemplateContentFormatOption = "auto",
+        add_generation_prompt: bool = True,
+        continue_final_message: bool = False,
+        tools: list[dict[str, Any]] | None = None,
+        chat_template_kwargs: dict[str, Any] | None = None,
+        tokenization_kwargs: dict[str, Any] | None = None,
+        mm_processor_kwargs: dict[str, Any] | None = None,
+    ) -> list[RequestOutput]:
+        """
+        Generate responses for a chat conversation.
+
+        The chat conversation is converted into a text prompt using the
+        tokenizer and calls the [generate][vllm.LLM.generate] method to generate
+        the responses.
+
+        Multi-modal inputs can be passed in the same way you would pass them
+        to the OpenAI API.
+
+        Args:
+            messages: A sequence of conversations or a single conversation.
+
+                - Each conversation is represented as a list of messages.
+                - Each message is a dictionary with 'role' and 'content' keys.
+
+            sampling_params: The sampling parameters for text generation.
+                If None, we use the default sampling parameters. When it
+                is a single value, it is applied to every prompt. When it
+                is a list, the list must have the same length as the
+                prompts and it is paired one by one with the prompt.
+            use_tqdm: If `True`, shows a tqdm progress bar.
+                If a callable (e.g., `functools.partial(tqdm, leave=False)`),
+                it is used to create the progress bar.
+                If `False`, no progress bar is created.
+            lora_request: LoRA request to use for generation, if any.
+            chat_template: The template to use for structuring the chat.
+                If not provided, the model's default chat template will be used.
+            chat_template_content_format: The format to render message content.
+
+                - "string" will render the content as a string.
+                  Example: `"Who are you?"`
+                - "openai" will render the content as a list of dictionaries,
+                  similar to OpenAI schema.
+                  Example: `[{"type": "text", "text": "Who are you?"}]`
+
+            add_generation_prompt: If True, adds a generation template
+                to each message.
+            continue_final_message: If True, continues the final message in
+                the conversation instead of starting a new one. Cannot be
+                `True` if `add_generation_prompt` is also `True`.
+            chat_template_kwargs: Additional kwargs to pass to the chat
+                template.
+            tokenization_kwargs: Overrides for `tokenizer.encode`.
+            mm_processor_kwargs: Overrides for `processor.__call__`.
+
+        Returns:
+            A list of `RequestOutput` objects containing the generated
+            responses in the same order as the input messages.
+        """
+        model_config = self.model_config
+        runner_type = model_config.runner_type
+        if runner_type != "generate":
+            raise ValueError(
+                "LLM.chat() is only supported for generative models. "
+                "Try passing `--runner generate` to use the model as a "
+                "generative model."
+            )
+
+        if sampling_params is None:
+            sampling_params = self.get_default_sampling_params()
+
+        return self._run_chat(
+            messages=messages,
+            params=sampling_params,
+            output_type=RequestOutput,
+            use_tqdm=use_tqdm,
+            lora_request=lora_request,
+            chat_template=chat_template,
+            chat_template_content_format=chat_template_content_format,
+            chat_template_kwargs=chat_template_kwargs,
+            add_generation_prompt=add_generation_prompt,
+            continue_final_message=continue_final_message,
+            tools=tools,
+            tokenization_kwargs=tokenization_kwargs,
+            mm_processor_kwargs=mm_processor_kwargs,
+        )
+
+    def enqueue_chat(
+        self,
+        messages: list[ChatCompletionMessageParam]
+        | Sequence[list[ChatCompletionMessageParam]],
+        sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,
+        use_tqdm: bool | Callable[..., tqdm] = True,
+        lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
+        priority: list[int] | None = None,
+        chat_template: str | None = None,
+        chat_template_content_format: ChatTemplateContentFormatOption = "auto",
+        add_generation_prompt: bool = True,
+        continue_final_message: bool = False,
+        tools: list[dict[str, Any]] | None = None,
+        chat_template_kwargs: dict[str, Any] | None = None,
+        tokenization_kwargs: dict[str, Any] | None = None,
+        mm_processor_kwargs: dict[str, Any] | None = None,
+    ) -> list[str]:
+        """Enqueue chat conversations for generation without waiting.
+
+        This method renders chat conversations and adds the resulting requests
+        to the engine queue. Use wait_for_completion() to get results. To
+        guarantee that all requests are queued before scheduling starts, pause
+        scheduling with sleep(level=0) before calling this method and resume it
+        with wake_up(tags=["scheduling"]) afterward.
+
+        Args:
+            messages: A sequence of conversations or a single conversation.
+                Each conversation is represented as a list of messages.
+            sampling_params: The sampling parameters for text generation.
+                If None, we use the default sampling parameters.
+            use_tqdm: If `True`, shows a tqdm progress bar while rendering
+                conversations.
+            lora_request: LoRA request to use for generation, if any.
+            priority: The priority of the requests, if any.
+            chat_template: The template to use for structuring the chat.
+            chat_template_content_format: The format to render message content.
+            add_generation_prompt: If True, adds a generation template
+                to each message.
+            continue_final_message: If True, continues the final message in
+                the conversation instead of starting a new one.
+            tools: Tools to make available to the model, if any.
+            chat_template_kwargs: Additional kwargs to pass to the chat
+                template.
+            tokenization_kwargs: Overrides for `tokenizer.encode`.
+            mm_processor_kwargs: Overrides for `processor.__call__`.
+
+        Returns:
+            A list of request IDs for the enqueued requests.
+        """
+        model_config = self.model_config
+        runner_type = model_config.runner_type
+        if runner_type != "generate":
+            raise ValueError(
+                "LLM.enqueue_chat() is only supported for generative models. "
+                "Try passing `--runner generate` to use the model as a "
+                "generative model."
+            )
+
+        if sampling_params is None:
+            sampling_params = self.get_default_sampling_params()
+
+        return self._add_chat_requests(
+            messages=messages,
+            params=sampling_params,
+            use_tqdm=use_tqdm,
+            lora_request=lora_request,
+            priority=priority,
+            chat_template=chat_template,
+            chat_template_content_format=chat_template_content_format,
+            chat_template_kwargs=chat_template_kwargs,
+            add_generation_prompt=add_generation_prompt,
+            continue_final_message=continue_final_message,
+            tools=tools,
+            tokenization_kwargs=tokenization_kwargs,
+            mm_processor_kwargs=mm_processor_kwargs,
+        )
+
+    def start_profile(self, profile_prefix: str | None = None) -> None:
+        """Start profiling with optional custom trace prefix.
+
+        Args:
+            profile_prefix: Optional prefix for the trace file names. If provided,
+                           trace files will be named as "<prefix>_dp<X>_pp<Y>_tp<Z>".
+                           If not provided, default naming will be used.
+        """
+        self.llm_engine.start_profile(profile_prefix)
+
+    def stop_profile(self) -> None:
+        self.llm_engine.stop_profile()
+
+    def reset_prefix_cache(
+        self, reset_running_requests: bool = False, reset_connector: bool = False
+    ) -> bool:
+        return self.llm_engine.reset_prefix_cache(
+            reset_running_requests, reset_connector
+        )
+
+    def sleep(self, level: int = 1, mode: PauseMode = "abort"):
+        """
+        Put the engine to sleep. The engine should not process any requests.
+        The caller should guarantee that no requests are being processed
+        during the sleep period, before `wake_up` is called.
+
+        Args:
+            level: The sleep level.
+                - Level 0: Pause scheduling but continue accepting requests.
+                           Requests are queued but not processed.
+                - Level 1: Offload model weights to CPU, discard KV cache.
+                           The content of kv cache is forgotten. Good for
+                           sleeping and waking up the engine to run the same
+                           model again. Please make sure there's enough CPU
+                           memory to store the model weights.
+                - Level 2: Discard all GPU memory (weights + KV cache).
+                           Good for sleeping and waking up the engine to run
+                           a different model or update the model, where
+                           previous model weights are not needed. It reduces
+                           CPU memory pressure.
+            mode: How to handle any existing requests, can be "abort", "wait",
+                or "keep".
+        """
+        self.llm_engine.sleep(level=level, mode=mode)
+
+    def wake_up(self, tags: list[str] | None = None):
+        """
+        Wake up the engine from sleep mode. See the [sleep][vllm.LLM.sleep]
+        method for more details.
+
+        Args:
+            tags: An optional list of tags to reallocate the engine memory
+                for specific memory allocations. Values must be in
+                `("weights", "kv_cache", "scheduling")`. If None, all memory
+                is reallocated. wake_up should be called with all tags
+                (or None) before the engine is used again.
+                Use tags=["scheduling"] to resume from level 0 sleep.
+        """
+        self.llm_engine.wake_up(tags)
+
+    def get_metrics(self) -> list["Metric"]:
+        """Return a snapshot of aggregated metrics from Prometheus.
+
+        Returns:
+            A `MetricSnapshot` instance capturing the current state
+            of all aggregated metrics from Prometheus.
+
+        Note:
+            This method is only available with the V1 LLM engine.
+        """
+        return self.llm_engine.get_metrics()
+
+    def init_weight_transfer_engine(
+        self, request: WeightTransferInitRequest | dict
+    ) -> None:
+        """
+        Initialize weight transfer for RL training.
+
+        Args:
+            request: Weight transfer initialization request with backend-specific info
+        """
+        init_info_dict = (
+            request["init_info"] if isinstance(request, dict) else request.init_info
+        )
+
+        self.llm_engine.collective_rpc(
+            "init_weight_transfer_engine", kwargs={"init_info": init_info_dict}
+        )
+
+    def start_weight_update(self) -> None:
+        """Start a new weight update."""
+        self.llm_engine.collective_rpc("start_weight_update")
+
+    def start_draft_weight_update(self) -> None:
+        """Start a new weight update targeting the speculative draft model."""
+        self.llm_engine.collective_rpc("start_draft_weight_update")
+
+    def update_weights(self, request: WeightTransferUpdateRequest | dict) -> None:
+        """
+        Update the weights of the model.
+
+        Args:
+            request: Weight update request with backend-specific update info
+        """
+        update_info_dict = (
+            request["update_info"] if isinstance(request, dict) else request.update_info
+        )
+
+        self.llm_engine.collective_rpc(
+            "update_weights", kwargs={"update_info": update_info_dict}
+        )
+
+    def finish_weight_update(self) -> None:
+        """Finish the current weight update."""
+        self.llm_engine.collective_rpc("finish_weight_update")
+
+    def __repr__(self) -> str:
+        """Return a transformers-style hierarchical view of the model."""
+        # Cache the result to avoid repeated collective_rpc calls
+        if self._cached_repr is None:
+            results = self.llm_engine.collective_rpc("get_model_inspection")
+            # In distributed settings, we get results from all workers
+            # Just return the first one (they should all be the same)
+            if results:
+                self._cached_repr = results[0]
+            else:
+                self._cached_repr = f"LLM(model={self.model_config.model!r})"
+        return self._cached_repr

vllm/entrypoints/mcp/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

vllm/entrypoints/mcp/tool.py

@@ -0,0 +1,187 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import json
+import os
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any
+
+from openai.types.responses.response_function_tool_call_output_item import (
+    ResponseFunctionToolCallOutputItem,
+)
+from openai_harmony import Author, Message, Role, TextContent
+
+from vllm.logger import init_logger
+from vllm.utils import random_uuid
+
+if TYPE_CHECKING:
+    # Avoid circular import.
+    from vllm.entrypoints.openai.responses.context import ConversationContext
+
+logger = init_logger(__name__)
+
+MIN_GPT_OSS_VERSION = "0.0.7"
+
+
+def validate_gpt_oss_install():
+    """
+    Check if the gpt-oss is installed and its version is at least 0.0.7.
+    If not, raise an ImportError.
+    """
+    from importlib.metadata import PackageNotFoundError, version
+
+    from packaging.version import InvalidVersion, Version
+
+    try:
+        pkg_version_str = version("gpt_oss")
+        pkg_version = Version(pkg_version_str)
+    except PackageNotFoundError:
+        raise ImportError("Package 'gpt_oss' is not installed.") from None
+    except InvalidVersion as e:
+        raise ImportError(f"Invalid version string for 'gpt_oss': {e}") from None
+
+    if pkg_version < Version(MIN_GPT_OSS_VERSION):
+        raise ImportError(
+            f"gpt_oss >= {MIN_GPT_OSS_VERSION} is required, "
+            f"but {pkg_version} is installed."
+        ) from None
+
+
+class Tool(ABC):
+    @abstractmethod
+    async def get_result(self, context: "ConversationContext") -> Any:
+        pass
+
+    @abstractmethod
+    async def get_result_parsable_context(self, context: "ConversationContext") -> Any:
+        pass
+
+
+class HarmonyBrowserTool(Tool):
+    def __init__(self):
+        self.enabled = True
+        exa_api_key = os.getenv("EXA_API_KEY")
+        if not exa_api_key:
+            self.enabled = False
+            logger.warning_once("EXA_API_KEY is not set, browsing is disabled")
+            return
+
+        try:
+            validate_gpt_oss_install()
+            from gpt_oss.tools.simple_browser import SimpleBrowserTool
+            from gpt_oss.tools.simple_browser.backend import ExaBackend
+        except ImportError as e:
+            self.enabled = False
+            logger.warning_once(
+                "gpt_oss is not installed properly (%s), browsing is disabled", e
+            )
+            return
+
+        browser_backend = ExaBackend(source="web", api_key=exa_api_key)
+        self.browser_tool = SimpleBrowserTool(backend=browser_backend)
+        logger.info_once("Browser tool initialized")
+
+    async def get_result(self, context: "ConversationContext") -> Any:
+        from vllm.entrypoints.openai.responses.context import HarmonyContext
+
+        assert isinstance(context, HarmonyContext)
+        last_msg = context.messages[-1]
+        tool_output_msgs = []
+        async for msg in self.browser_tool.process(last_msg):
+            tool_output_msgs.append(msg)
+        return tool_output_msgs
+
+    async def get_result_parsable_context(self, context: "ConversationContext") -> Any:
+        raise NotImplementedError("Not implemented yet")
+
+    @property
+    def tool_config(self) -> Any:
+        return self.browser_tool.tool_config
+
+
+class HarmonyPythonTool(Tool):
+    def __init__(self):
+        self.enabled = True
+
+        try:
+            validate_gpt_oss_install()
+            from gpt_oss.tools.python_docker.docker_tool import PythonTool
+        except ImportError as e:
+            self.enabled = False
+            logger.warning_once(
+                "gpt_oss is not installed properly (%s), code interpreter is disabled",
+                e,
+            )
+            return
+
+        self.python_tool = PythonTool()
+
+    async def validate(self):
+        if not self.enabled:
+            return
+        try:
+            message = Message(
+                author=Author(role=Role.ASSISTANT),
+                content=[TextContent(text="print('Hello, world!')")],
+                channel="analysis",
+                recipient="python",
+                content_type="code",
+            )
+            msgs = []
+            async for msg in self.python_tool.process(message):
+                msgs.append(msg)
+            assert msgs[0].content[0].text == "Hello, world!\n"
+        except Exception as e:
+            self.enabled = False
+            logger.warning_once(
+                "Code interpreter tool failed to initialize (%s), code "
+                "interpreter is disabled",
+                e,
+            )
+            return
+        logger.info_once("Code interpreter tool initialized")
+
+    async def get_result(self, context: "ConversationContext") -> Any:
+        from vllm.entrypoints.openai.responses.context import HarmonyContext
+
+        assert isinstance(context, HarmonyContext)
+        last_msg = context.messages[-1]
+        tool_output_msgs = []
+        async for msg in self.python_tool.process(last_msg):
+            tool_output_msgs.append(msg)
+        return tool_output_msgs
+
+    async def get_result_parsable_context(self, context: "ConversationContext") -> Any:
+        """
+        This function converts parsable context types to harmony and
+        back so we can use GPTOSS demo python tool
+        """
+        from vllm.entrypoints.openai.responses.context import ParsableContext
+
+        assert isinstance(context, ParsableContext)
+
+        last_msg = context.response_messages[-1]
+        args = json.loads(last_msg.arguments)
+
+        last_msg_harmony = Message(
+            author=Author(role="assistant", name=None),
+            content=[TextContent(text=args["code"])],
+            channel="analysis",
+            recipient="python",
+            content_type="code",
+        )
+
+        tool_output_msgs = []
+        async for msg in self.python_tool.process(last_msg_harmony):
+            processed = ResponseFunctionToolCallOutputItem(
+                id=f"fco_{random_uuid()}",
+                type="function_call_output",
+                call_id=f"call_{random_uuid()}",
+                output=msg.content[0].text,
+                status="completed",
+            )
+            tool_output_msgs.append(processed)
+        return tool_output_msgs
+
+    @property
+    def tool_config(self) -> Any:
+        return self.python_tool.tool_config

vllm/entrypoints/mcp/tool_server.py

@@ -0,0 +1,234 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from abc import ABC, abstractmethod
+from contextlib import AbstractAsyncContextManager, asynccontextmanager
+from typing import TYPE_CHECKING, Any
+
+from openai_harmony import ToolDescription, ToolNamespaceConfig
+
+from vllm.entrypoints.mcp.tool import HarmonyBrowserTool, HarmonyPythonTool, Tool
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+if TYPE_CHECKING:
+    from mcp.types import ListToolsResult
+
+
+async def list_server_and_tools(server_url: str):
+    from mcp import ClientSession
+    from mcp.client.sse import sse_client
+
+    async with (
+        sse_client(url=server_url) as streams,
+        ClientSession(*streams) as session,
+    ):
+        initialize_response = await session.initialize()
+        list_tools_response = await session.list_tools()
+        return initialize_response, list_tools_response
+
+
+def trim_schema(schema: dict) -> dict:
+    # Turn JSON Schema from MCP generated into Harmony's variant.
+    if "title" in schema:
+        del schema["title"]
+    if "default" in schema and schema["default"] is None:
+        del schema["default"]
+    if "anyOf" in schema:
+        # Turn "anyOf": [{"type": "type-1"}, {"type": "type-2"}]
+        # into "type": ["type-1", "type-2"]
+        # if there's more than 1 types, also remove "null" type as Harmony will
+        # just ignore it
+        types = [
+            type_dict["type"]
+            for type_dict in schema["anyOf"]
+            if type_dict["type"] != "null"
+        ]
+        schema["type"] = types
+        del schema["anyOf"]
+    if "properties" in schema:
+        schema["properties"] = {
+            k: trim_schema(v) for k, v in schema["properties"].items()
+        }
+    return schema
+
+
+def post_process_tools_description(
+    list_tools_result: "ListToolsResult",
+) -> "ListToolsResult":
+    # Adapt the MCP tool result for Harmony
+    for tool in list_tools_result.tools:
+        tool.inputSchema = trim_schema(tool.inputSchema)
+
+    # Some tools schema don't need to be part of the prompt (e.g. simple text
+    # in text out for Python)
+    list_tools_result.tools = [
+        tool
+        for tool in list_tools_result.tools
+        if getattr(tool.annotations, "include_in_prompt", True)
+    ]
+
+    return list_tools_result
+
+
+class ToolServer(ABC):
+    @abstractmethod
+    def has_tool(self, tool_name: str) -> bool:
+        """
+        Return True if the tool is supported, False otherwise.
+        """
+        pass
+
+    @abstractmethod
+    def get_tool_description(
+        self, tool_name: str, allowed_tools: list[str] | None = None
+    ) -> ToolNamespaceConfig | None:
+        """
+        Return the tool description for the given tool name.
+        If the tool is not supported, return None.
+        """
+        pass
+
+    @abstractmethod
+    def new_session(
+        self, tool_name: str, session_id: str, headers: dict[str, str] | None = None
+    ) -> AbstractAsyncContextManager[Any]:
+        """
+        Create a session for the tool.
+        """
+        ...
+
+
+class MCPToolServer(ToolServer):
+    def __init__(self):
+        try:
+            import mcp  # noqa: F401
+        except ImportError:
+            raise ImportError(
+                "mcp is not installed. Please run `pip install mcp` to use "
+                "MCPToolServer."
+            ) from None
+        self.harmony_tool_descriptions = {}
+
+    async def add_tool_server(self, server_url: str):
+        tool_urls = server_url.split(",")
+        self.harmony_tool_descriptions = {}
+        self.urls: dict[str, str] = {}
+        for url in tool_urls:
+            url = f"http://{url}/sse"
+            initialize_response, list_tools_response = await list_server_and_tools(url)
+
+            list_tools_response = post_process_tools_description(list_tools_response)
+
+            tool_from_mcp = ToolNamespaceConfig(
+                name=initialize_response.serverInfo.name,
+                description=initialize_response.instructions,
+                tools=[
+                    ToolDescription.new(
+                        name=tool.name,
+                        description=tool.description,
+                        parameters=tool.inputSchema,
+                    )
+                    for tool in list_tools_response.tools
+                ],
+            )
+            self.harmony_tool_descriptions[tool_from_mcp.name] = tool_from_mcp
+            if tool_from_mcp.name not in self.urls:
+                self.urls[tool_from_mcp.name] = url
+            else:
+                logger.warning(
+                    "Tool %s already exists. Ignoring duplicate tool server %s",
+                    tool_from_mcp.name,
+                    url,
+                )
+        logger.info(
+            "MCPToolServer initialized with tools: %s",
+            list(self.harmony_tool_descriptions.keys()),
+        )
+
+    def has_tool(self, tool_name: str):
+        return tool_name in self.harmony_tool_descriptions
+
+    def get_tool_description(
+        self,
+        server_label: str,
+        allowed_tools: list[str] | None = None,
+    ) -> ToolNamespaceConfig | None:
+        cfg = self.harmony_tool_descriptions.get(server_label)
+        if cfg is None:
+            return None
+
+        # No restrictions: all tools from this MCP server
+        if allowed_tools is None:
+            return cfg
+
+        filtered = [t for t in cfg.tools if t.name in allowed_tools]
+
+        if not filtered:
+            return None
+
+        return ToolNamespaceConfig(
+            name=cfg.name,
+            description=cfg.description,
+            tools=filtered,
+        )
+
+    @asynccontextmanager
+    async def new_session(
+        self, tool_name: str, session_id: str, headers: dict[str, str] | None = None
+    ):
+        from mcp import ClientSession
+        from mcp.client.sse import sse_client
+
+        url = self.urls.get(tool_name)
+        request_headers = {"x-session-id": session_id}
+        if headers is not None:
+            request_headers.update(headers)
+        if not url:
+            raise KeyError(f"Tool '{tool_name}' is not supported")
+        async with (
+            sse_client(url=url, headers=request_headers) as streams,
+            ClientSession(*streams) as session,
+        ):
+            await session.initialize()
+            yield session
+
+
+class DemoToolServer(ToolServer):
+    def __init__(self):
+        self.tools: dict[str, Tool] = {}
+
+    async def init_and_validate(self):
+        browser_tool = HarmonyBrowserTool()
+        python_tool = HarmonyPythonTool()
+        await python_tool.validate()
+        if browser_tool.enabled:
+            self.tools["browser"] = browser_tool
+        if python_tool.enabled:
+            self.tools["python"] = python_tool
+        logger.info(
+            "DemoToolServer initialized with tools: %s", list(self.tools.keys())
+        )
+
+    def has_tool(self, tool_name: str) -> bool:
+        return tool_name in self.tools
+
+    def get_tool_description(
+        self, tool_name: str, allowed_tools: list[str] | None = None
+    ) -> ToolNamespaceConfig | None:
+        if tool_name not in self.tools:
+            return None
+        if tool_name == "browser":
+            return ToolNamespaceConfig.browser()
+        elif tool_name == "python":
+            return ToolNamespaceConfig.python()
+        else:
+            raise ValueError(f"Unknown tool {tool_name}")
+
+    @asynccontextmanager
+    async def new_session(
+        self, tool_name: str, session_id: str, headers: dict[str, str] | None = None
+    ):
+        if tool_name not in self.tools:
+            raise KeyError(f"Tool '{tool_name}' is not supported")
+        yield self.tools[tool_name]

vllm/entrypoints/offline_utils.py

@@ -0,0 +1,626 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from collections.abc import Callable, Iterable, Sequence
+from typing import Any
+
+from tqdm import tqdm
+from typing_extensions import TypeVar
+
+from vllm import (
+    PoolingParams,
+    PoolingRequestOutput,
+    PromptType,
+    RequestOutput,
+    SamplingParams,
+)
+from vllm.config import ModelConfig
+from vllm.entrypoints.chat_utils import (
+    ChatCompletionMessageParam,
+    ChatTemplateContentFormatOption,
+)
+from vllm.inputs import EngineInput
+from vllm.logger import init_logger
+from vllm.lora.request import LoRARequest
+from vllm.renderers import BaseRenderer, ChatParams, merge_kwargs
+from vllm.renderers.inputs.preprocess import (
+    conversation_to_seq,
+    parse_model_prompt,
+    prompt_to_seq,
+)
+from vllm.sampling_params import RequestOutputKind
+from vllm.utils.counter import Counter
+from vllm.utils.mistral import is_mistral_tokenizer
+from vllm.utils.tqdm_utils import maybe_tqdm
+from vllm.v1.engine.llm_engine import LLMEngine
+
+logger = init_logger(__name__)
+
+
+_P = TypeVar("_P", bound=SamplingParams | PoolingParams | None)
+_O = TypeVar(
+    "_O",
+    bound=RequestOutput | PoolingRequestOutput,
+    default=RequestOutput | PoolingRequestOutput,
+)
+_R = TypeVar("_R", default=Any)
+
+
+class OfflineInferenceMixin:
+    """Offline inference utils"""
+
+    request_counter: Counter
+    renderer: BaseRenderer
+    llm_engine: "LLMEngine"
+    model_config: ModelConfig
+
+    def _resolve_mm_lora(
+        self,
+        prompt: EngineInput,
+        lora_request: LoRARequest | None,
+    ) -> LoRARequest | None:
+        if prompt["type"] != "multimodal":
+            return lora_request
+
+        lora_config = self.llm_engine.vllm_config.lora_config
+        default_mm_loras = None if lora_config is None else lora_config.default_mm_loras
+        if not default_mm_loras:
+            return lora_request
+
+        prompt_modalities = prompt["mm_placeholders"].keys()
+        intersection = set(prompt_modalities).intersection(default_mm_loras.keys())
+        if not intersection:
+            return lora_request
+
+        if len(intersection) > 1:
+            # TODO: Would be nice to be able to have multiple loras per prompt
+            logger.warning(
+                "Multiple modality specific loras were registered and would be "
+                "used by a single prompt consuming several modalities; "
+                "currently we only support one lora per request; as such, "
+                "lora(s) registered with modalities: %s will be skipped",
+                intersection,
+            )
+            return lora_request
+
+        # Build the LoRA request; the ID of the default mm lora is the
+        # index of the modality name sorted alphabetically + 1.
+        modality_name = intersection.pop()
+        modality_lora_path = default_mm_loras[modality_name]
+        modality_lora_id = sorted(default_mm_loras).index(modality_name) + 1
+
+        # If we have a collision, warn if there is a collision,
+        # but always send the explicitly provided request.
+        if lora_request:
+            if lora_request.lora_int_id != modality_lora_id:
+                logger.warning(
+                    "A modality with a registered lora and a lora_request "
+                    "with a different ID were provided; falling back to the "
+                    "lora_request as we only apply one LoRARequest per prompt"
+                )
+            return lora_request
+
+        return LoRARequest(
+            modality_name,
+            modality_lora_id,
+            modality_lora_path,
+        )
+
+    def _preprocess_cmpl(
+        self,
+        prompts: Sequence[PromptType],
+        tokenization_kwargs: dict[str, Any] | None = None,
+        mm_processor_kwargs: dict[str, Any] | None = None,
+    ) -> Sequence[EngineInput]:
+        """
+        Convert prompt inputs from LLM APIs (other than [LLM.chat][]) into
+        a format that can be passed to `_add_request`.
+
+        Refer to [LLM.generate][] for a complete description of the arguments.
+
+        Returns:
+            A list of `EngineInput` objects ready to be passed into LLMEngine.
+        """
+        renderer = self.renderer
+        model_config = self.model_config
+
+        parsed_prompts = [
+            parse_model_prompt(model_config, prompt) for prompt in prompts
+        ]
+        tok_params = renderer.default_cmpl_tok_params.with_kwargs(
+            **(tokenization_kwargs or {})
+        )
+        prompt_extras = (
+            None
+            if mm_processor_kwargs is None
+            else {"mm_processor_kwargs": mm_processor_kwargs}
+        )
+
+        return renderer.render_cmpl(
+            parsed_prompts,
+            tok_params,
+            prompt_extras=prompt_extras,
+        )
+
+    def _preprocess_cmpl_one(
+        self,
+        prompt: PromptType,
+        tokenization_kwargs: dict[str, Any] | None = None,
+        mm_processor_kwargs: dict[str, Any] | None = None,
+    ) -> EngineInput:
+        (engine_input,) = self._preprocess_cmpl(
+            [prompt],
+            tokenization_kwargs,
+            mm_processor_kwargs=mm_processor_kwargs,
+        )
+        return engine_input
+
+    def _preprocess_chat(
+        self,
+        conversations: Sequence[list[ChatCompletionMessageParam]],
+        chat_template: str | None = None,
+        chat_template_content_format: ChatTemplateContentFormatOption = "auto",
+        chat_template_kwargs: dict[str, Any] | None = None,
+        add_generation_prompt: bool = True,
+        continue_final_message: bool = False,
+        tools: list[dict[str, Any]] | None = None,
+        tokenization_kwargs: dict[str, Any] | None = None,
+        mm_processor_kwargs: dict[str, Any] | None = None,
+    ) -> Sequence[EngineInput]:
+        """
+        Convert a list of conversations into prompts so that they can then
+        be used as input for other LLM APIs.
+
+        Refer to [LLM.chat][] for a complete description of the arguments.
+
+        Returns:
+            A list of `EngineInput` objects ready to be passed into LLMEngine.
+        """
+        renderer = self.renderer
+
+        chat_params = ChatParams(
+            chat_template=chat_template,
+            chat_template_content_format=chat_template_content_format,
+            chat_template_kwargs=merge_kwargs(
+                chat_template_kwargs,
+                dict(
+                    add_generation_prompt=add_generation_prompt,
+                    continue_final_message=continue_final_message,
+                    tools=tools,
+                    tokenize=(
+                        is_mistral_tokenizer(renderer.tokenizer)
+                        or self.model_config.enable_prompt_embeds
+                    ),
+                ),
+            ),
+            mm_processor_kwargs=mm_processor_kwargs,
+        )
+        tok_params = renderer.default_chat_tok_params.with_kwargs(
+            **(tokenization_kwargs or {})
+        )
+        prompt_extras = (
+            None
+            if mm_processor_kwargs is None
+            else {"mm_processor_kwargs": mm_processor_kwargs}
+        )
+
+        _, engine_inputs = renderer.render_chat(
+            conversations,
+            chat_params,
+            tok_params,
+            prompt_extras=prompt_extras,
+        )
+
+        return engine_inputs
+
+    def _preprocess_chat_one(
+        self,
+        conversation: list[ChatCompletionMessageParam],
+        chat_template: str | None = None,
+        chat_template_content_format: ChatTemplateContentFormatOption = "auto",
+        chat_template_kwargs: dict[str, Any] | None = None,
+        add_generation_prompt: bool = True,
+        continue_final_message: bool = False,
+        tools: list[dict[str, Any]] | None = None,
+        tokenization_kwargs: dict[str, Any] | None = None,
+        mm_processor_kwargs: dict[str, Any] | None = None,
+    ) -> EngineInput:
+        (engine_input,) = self._preprocess_chat(
+            [conversation],
+            chat_template=chat_template,
+            chat_template_content_format=chat_template_content_format,
+            chat_template_kwargs=chat_template_kwargs,
+            add_generation_prompt=add_generation_prompt,
+            continue_final_message=continue_final_message,
+            tools=tools,
+            tokenization_kwargs=tokenization_kwargs,
+            mm_processor_kwargs=mm_processor_kwargs,
+        )
+
+        return engine_input
+
+    def _params_to_seq(
+        self,
+        params: _P | Sequence[_P],
+        num_requests: int,
+    ) -> Sequence[_P]:
+        if isinstance(params, Sequence):
+            if len(params) != num_requests:
+                raise ValueError(
+                    f"The lengths of prompts ({num_requests}) "
+                    f"and params ({len(params)}) must be the same."
+                )
+
+            return params
+
+        return [params] * num_requests
+
+    def _lora_request_to_seq(
+        self,
+        lora_request: LoRARequest | None | Sequence[LoRARequest | None],
+        num_requests: int,
+    ) -> Sequence[LoRARequest | None]:
+        if isinstance(lora_request, Sequence):
+            if len(lora_request) != num_requests:
+                raise ValueError(
+                    f"The lengths of prompts ({num_requests}) "
+                    f"and lora_request ({len(lora_request)}) must be the same."
+                )
+
+            return lora_request
+
+        return [lora_request] * num_requests
+
+    def _priority_to_seq(
+        self,
+        priority: list[int] | None,
+        num_requests: int,
+    ) -> Sequence[int]:
+        if priority is not None:
+            if len(priority) != num_requests:
+                raise ValueError(
+                    f"The lengths of prompts ({num_requests}) "
+                    f"and priority ({len(priority)}) must be the same."
+                )
+
+            return priority
+
+        return [0] * num_requests
+
+    def _add_completion_requests(
+        self,
+        prompts: PromptType | Sequence[PromptType],
+        params: SamplingParams
+        | PoolingParams
+        | Sequence[SamplingParams | PoolingParams],
+        *,
+        use_tqdm: bool | Callable[..., tqdm] = True,
+        lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
+        priority: list[int] | None = None,
+        tokenization_kwargs: dict[str, Any] | None = None,
+        mm_processor_kwargs: dict[str, Any] | None = None,
+    ) -> list[str]:
+        seq_prompts = prompt_to_seq(prompts)
+        seq_params = self._params_to_seq(params, len(seq_prompts))
+        seq_lora_requests = self._lora_request_to_seq(lora_request, len(seq_prompts))
+        seq_priority = self._priority_to_seq(priority, len(seq_prompts))
+
+        return self._render_and_add_requests(
+            prompts=(
+                self._preprocess_cmpl_one(
+                    prompt,
+                    tokenization_kwargs,
+                    mm_processor_kwargs=mm_processor_kwargs,
+                )
+                for prompt in maybe_tqdm(
+                    seq_prompts,
+                    use_tqdm=use_tqdm,
+                    desc="Rendering prompts",
+                )
+            ),
+            params=seq_params,
+            lora_requests=seq_lora_requests,
+            priorities=seq_priority,
+        )
+
+    def _run_completion(
+        self,
+        prompts: PromptType | Sequence[PromptType],
+        params: SamplingParams
+        | PoolingParams
+        | Sequence[SamplingParams | PoolingParams],
+        output_type: type[_O],
+        *,
+        use_tqdm: bool | Callable[..., tqdm] = True,
+        lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
+        priority: list[int] | None = None,
+        tokenization_kwargs: dict[str, Any] | None = None,
+        mm_processor_kwargs: dict[str, Any] | None = None,
+    ):
+        self._add_completion_requests(
+            prompts=prompts,
+            params=params,
+            use_tqdm=use_tqdm,
+            lora_request=lora_request,
+            priority=priority,
+            tokenization_kwargs=tokenization_kwargs,
+            mm_processor_kwargs=mm_processor_kwargs,
+        )
+        return self._run_engine(use_tqdm=use_tqdm, output_type=output_type)
+
+    def _run_chat(
+        self,
+        messages: list[ChatCompletionMessageParam]
+        | Sequence[list[ChatCompletionMessageParam]],
+        params: SamplingParams
+        | PoolingParams
+        | Sequence[SamplingParams | PoolingParams],
+        output_type: type[_O],
+        *,
+        use_tqdm: bool | Callable[..., tqdm] = True,
+        lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
+        chat_template: str | None = None,
+        chat_template_content_format: ChatTemplateContentFormatOption = "auto",
+        add_generation_prompt: bool = True,
+        continue_final_message: bool = False,
+        tools: list[dict[str, Any]] | None = None,
+        chat_template_kwargs: dict[str, Any] | None = None,
+        tokenization_kwargs: dict[str, Any] | None = None,
+        mm_processor_kwargs: dict[str, Any] | None = None,
+    ):
+        self._add_chat_requests(
+            messages=messages,
+            params=params,
+            use_tqdm=use_tqdm,
+            lora_request=lora_request,
+            chat_template=chat_template,
+            chat_template_content_format=chat_template_content_format,
+            chat_template_kwargs=chat_template_kwargs,
+            add_generation_prompt=add_generation_prompt,
+            continue_final_message=continue_final_message,
+            tools=tools,
+            tokenization_kwargs=tokenization_kwargs,
+            mm_processor_kwargs=mm_processor_kwargs,
+        )
+        return self._run_engine(output_type=output_type, use_tqdm=use_tqdm)
+
+    def _add_chat_requests(
+        self,
+        messages: list[ChatCompletionMessageParam]
+        | Sequence[list[ChatCompletionMessageParam]],
+        params: SamplingParams
+        | PoolingParams
+        | Sequence[SamplingParams | PoolingParams],
+        *,
+        use_tqdm: bool | Callable[..., tqdm] = True,
+        lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
+        priority: list[int] | None = None,
+        chat_template: str | None = None,
+        chat_template_content_format: ChatTemplateContentFormatOption = "auto",
+        add_generation_prompt: bool = True,
+        continue_final_message: bool = False,
+        tools: list[dict[str, Any]] | None = None,
+        chat_template_kwargs: dict[str, Any] | None = None,
+        tokenization_kwargs: dict[str, Any] | None = None,
+        mm_processor_kwargs: dict[str, Any] | None = None,
+    ) -> list[str]:
+        seq_convs = conversation_to_seq(messages)
+        seq_params = self._params_to_seq(params, len(seq_convs))
+        seq_lora_requests = self._lora_request_to_seq(lora_request, len(seq_convs))
+        seq_priority = self._priority_to_seq(priority, len(seq_convs))
+
+        # When thinking is enabled or tools are provided, and the model
+        # uses special tokens for structured output (e.g. Gemma4's
+        # <|channel>, <|tool_call>, <|"|>), automatically set
+        # skip_special_tokens=False so these tokens are preserved in
+        # output.text for downstream parsing.
+        needs_parsing = (
+            chat_template_kwargs and chat_template_kwargs.get("enable_thinking")
+        ) or tools
+        if needs_parsing:
+            self._adjust_params_for_parsing(seq_params)
+
+        return self._render_and_add_requests(
+            prompts=(
+                self._preprocess_chat_one(
+                    conversation,
+                    chat_template=chat_template,
+                    chat_template_content_format=chat_template_content_format,
+                    chat_template_kwargs=chat_template_kwargs,
+                    add_generation_prompt=add_generation_prompt,
+                    continue_final_message=continue_final_message,
+                    tools=tools,
+                    tokenization_kwargs=tokenization_kwargs,
+                    mm_processor_kwargs=mm_processor_kwargs,
+                )
+                for conversation in maybe_tqdm(
+                    seq_convs,
+                    use_tqdm=use_tqdm,
+                    desc="Rendering conversations",
+                )
+            ),
+            params=seq_params,
+            lora_requests=seq_lora_requests,
+            priorities=seq_priority,
+        )
+
+    def _adjust_params_for_parsing(
+        self, params: Sequence[SamplingParams | PoolingParams]
+    ) -> None:
+        """Set ``skip_special_tokens=False`` when the model encodes
+        structured output syntax as special tokens.
+
+        Models like Gemma4 register thinking delimiters
+        (``<|channel>``/``<channel|>``) and tool call tokens
+        (``<|tool_call>``/``<tool_call|>``/``<|"|>``) as special tokens.
+        The default ``skip_special_tokens=True`` strips them from
+        ``output.text``, breaking parsing of both reasoning blocks and
+        tool calls.
+
+        This is a no-op for models whose structured tokens are regular
+        text tokens (e.g. DeepSeek's ``<think>``/``</think>``).
+        """
+        # The offline API currently lacks a unified rendering pipeline.
+        # Until the planned Renderer refactor is complete, we hardcode
+        # this token preservation logic specifically for Gemma4 models
+        # to avoid regressions on other models.
+        hf_config = getattr(self.model_config, "hf_config", None)
+        architectures = getattr(hf_config, "architectures", [])
+
+        if any("Gemma4" in arch for arch in architectures):
+            tokenizer = self.renderer.get_tokenizer()
+            vocab = tokenizer.get_vocab()
+            special_ids = set(getattr(tokenizer, "all_special_ids", []))
+
+            # Tokens used for thinking delimiters and tool call syntax
+            # that some models (Gemma4) register as special tokens.
+            structured_tokens = (
+                "<|channel>",
+                "<channel|>",  # thinking delimiters
+                "<|tool_call>",
+                "<tool_call|>",  # tool call delimiters
+                '<|"|>',  # string quoting in tool args
+            )
+            needs_special = any(
+                vocab.get(tok) in special_ids
+                for tok in structured_tokens
+                if tok in vocab
+            )
+            if needs_special:
+                for sp in params:
+                    if isinstance(sp, SamplingParams) and sp.skip_special_tokens:
+                        sp.skip_special_tokens = False
+
+    def _render_and_run_requests(
+        self,
+        prompts: Iterable[EngineInput],
+        params: Sequence[SamplingParams | PoolingParams],
+        output_type: type[_O],
+        *,
+        lora_requests: Sequence[LoRARequest | None] | None = None,
+        priorities: Sequence[int] | None = None,
+        use_tqdm: bool | Callable[..., tqdm] = True,
+    ):
+        if isinstance(prompts, (list, tuple)):
+            logger.warning_once(
+                "Rendering all prompts before adding them to the engine "
+                "is less efficient than performing both on the same prompt "
+                "before processing the next prompt. You should instead pass "
+                "a generator that renders one prompt per iteration, as that allows "
+                "engine execution to begin for the first prompt while processing "
+                "the next prompt."
+            )
+
+        self._render_and_add_requests(
+            prompts=prompts,
+            params=params,
+            lora_requests=lora_requests,
+            priorities=priorities,
+        )
+
+        return self._run_engine(output_type, use_tqdm=use_tqdm)
+
+    def _render_and_add_requests(
+        self,
+        prompts: Iterable[EngineInput],
+        params: Sequence[SamplingParams | PoolingParams],
+        *,
+        lora_requests: Sequence[LoRARequest | None] | None = None,
+        priorities: Sequence[int] | None = None,
+    ) -> list[str]:
+        added_request_ids: list[str] = []
+
+        try:
+            for i, prompt in enumerate(prompts):
+                request_id = self._add_request(
+                    prompt,
+                    params[i],
+                    lora_request=self._resolve_mm_lora(
+                        prompt,
+                        None if lora_requests is None else lora_requests[i],
+                    ),
+                    priority=0 if priorities is None else priorities[i],
+                )
+                added_request_ids.append(request_id)
+        except Exception as e:
+            if added_request_ids:
+                self.llm_engine.abort_request(added_request_ids, internal=True)
+            raise e
+
+        return added_request_ids
+
+    def _add_request(
+        self,
+        prompt: EngineInput,
+        params: SamplingParams | PoolingParams,
+        lora_request: LoRARequest | None = None,
+        priority: int = 0,
+    ) -> str:
+        if isinstance(params, SamplingParams):
+            # We only care about the final output
+            params.output_kind = RequestOutputKind.FINAL_ONLY
+
+        request_id = str(next(self.request_counter))
+
+        return self.llm_engine.add_request(
+            request_id,
+            prompt,
+            params,
+            lora_request=lora_request,
+            priority=priority,
+        )
+
+    def _run_engine(
+        self,
+        output_type: type[_O] | tuple[type[_O], ...],
+        *,
+        use_tqdm: bool | Callable[..., tqdm] = True,
+    ) -> list[_O]:
+        # Initialize tqdm.
+        if use_tqdm:
+            num_requests = self.llm_engine.get_num_unfinished_requests()
+            tqdm_func = use_tqdm if callable(use_tqdm) else tqdm
+            pbar = tqdm_func(
+                total=num_requests,
+                desc="Processed prompts",
+                dynamic_ncols=True,
+                postfix=(f"est. speed input: {0:.2f} toks/s, output: {0:.2f} toks/s"),
+            )
+
+        # Run the engine.
+        outputs: list[_O] = []
+        total_in_toks = 0
+        total_out_toks = 0
+        while self.llm_engine.has_unfinished_requests():
+            step_outputs = self.llm_engine.step()
+            for output in step_outputs:
+                assert isinstance(output, output_type)
+                if output.finished:
+                    outputs.append(output)  # type: ignore[arg-type]
+                    if use_tqdm:
+                        if isinstance(output, RequestOutput):
+                            # Calculate tokens only for RequestOutput
+                            n = len(output.outputs)
+                            assert output.prompt_token_ids is not None
+                            total_in_toks += len(output.prompt_token_ids) * n
+                            in_spd = total_in_toks / pbar.format_dict["elapsed"]
+                            total_out_toks += sum(
+                                len(stp.token_ids) for stp in output.outputs
+                            )
+                            out_spd = total_out_toks / pbar.format_dict["elapsed"]
+                            pbar.postfix = (
+                                f"est. speed input: {in_spd:.2f} toks/s, "
+                                f"output: {out_spd:.2f} toks/s"
+                            )
+                            pbar.update(n)
+                        else:
+                            pbar.update(1)
+                        if pbar.n == num_requests:
+                            pbar.refresh()
+
+        if use_tqdm:
+            pbar.close()
+        # Sort the outputs by request ID.
+        # This is necessary because some requests may be finished earlier than
+        # its previous requests.
+        return sorted(outputs, key=lambda x: int(x.request_id))

vllm/entrypoints/openai/api_server.py

@@ -0,0 +1,799 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import asyncio
+import importlib
+import inspect
+import multiprocessing
+import multiprocessing.forkserver as forkserver
+import os
+import signal
+import socket
+import tempfile
+import warnings
+from argparse import Namespace
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from typing import Any, cast
+
+import uvloop
+from fastapi import FastAPI, HTTPException
+from fastapi.exceptions import RequestValidationError
+from fastapi.middleware.cors import CORSMiddleware
+from starlette.datastructures import State
+
+import vllm.envs as envs
+from vllm.config import ModelConfig, VllmConfig
+from vllm.engine.arg_utils import AsyncEngineArgs
+from vllm.engine.protocol import EngineClient
+from vllm.entrypoints.chat_utils import load_chat_template
+from vllm.entrypoints.launcher import serve_http
+from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
+from vllm.entrypoints.openai.engine.protocol import GenerationError
+from vllm.entrypoints.openai.models.protocol import BaseModelPath
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
+from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
+from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
+from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
+from vllm.entrypoints.serve.utils.api_utils import (
+    cli_env_setup,
+    log_non_default_args,
+    log_version_and_model,
+    process_lora_modules,
+)
+from vllm.entrypoints.serve.utils.request_logger import RequestLogger
+from vllm.entrypoints.serve.utils.server_utils import (
+    engine_error_handler,
+    exception_handler,
+    generation_error_handler,
+    get_uvicorn_log_config,
+    http_exception_handler,
+    lifespan,
+    log_response,
+    validation_exception_handler,
+)
+from vllm.exceptions import (
+    VLLMNotFoundError,
+    VLLMUnprocessableEntityError,
+    VLLMValidationError,
+)
+from vllm.logger import init_logger
+from vllm.reasoning import ReasoningParserManager
+from vllm.renderers.online_derenderer import OnlineDerenderer
+from vllm.renderers.online_renderer import OnlineRenderer
+from vllm.tasks import POOLING_TASKS, SupportedTask
+from vllm.tool_parsers import ToolParserManager
+from vllm.tracing import instrument
+from vllm.usage.usage_lib import UsageContext
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+from vllm.utils.network_utils import is_valid_ipv6_address
+from vllm.utils.system_utils import decorate_logs, set_ulimit
+from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
+from vllm.version import __version__ as VLLM_VERSION
+
+prometheus_multiproc_dir: tempfile.TemporaryDirectory
+
+# Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765)
+logger = init_logger("vllm.entrypoints.openai.api_server")
+
+_FALLBACK_SUPPORTED_TASKS: tuple[SupportedTask, ...] = ("generate",)
+
+
+def _attach_endpoint_plugins(
+    app: FastAPI, supported_tasks: tuple["SupportedTask", ...]
+) -> None:
+    """Phase A of endpoint plugin wiring: discover, gate and attach routes.
+
+    Attached last after all core routers. This is so endpoint plugin routes can
+    shadow core routes with the same path (see `EndpointPlugin.attach_router`
+    docstring). No-ops when no plugins are discovered/allowlisted.
+    """
+    from vllm.plugins import load_endpoint_plugins
+
+    endpoint_plugins = load_endpoint_plugins(supported_tasks)
+    for plugin in endpoint_plugins:
+        plugin.attach_router(app)
+    app.state.endpoint_plugins = endpoint_plugins
+
+
+async def _init_endpoint_plugins_state(
+    engine_client: EngineClient | None, state: State, args: Namespace
+) -> None:
+    """Phase B of endpoint plugin wiring: initialize per app plugin state.
+
+    `state.endpoint_plugins` is set by `_attach_endpoint_plugins` (Phase A)
+    in `build_app`. Some `init_app_state` callers (e.g. `run_batch.py`)
+    build their own bare `State` without going through `build_app`. As a result
+    `endpoint_plugins` may be absent and are treated that the same as "none attached".
+
+    `engine_client` is `None` for the CPU only render server which has no
+    engine (see `init_render_app_state`). Plugins must handle a `None`
+    `engine_client` themselves (see `EndpointPlugin.init_state`).
+    """
+    for plugin in getattr(state, "endpoint_plugins", []):
+        await plugin.init_state(engine_client, state, args)
+
+
+@asynccontextmanager
+async def build_async_engine_client(
+    args: Namespace,
+    *,
+    usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
+    client_config: dict[str, Any] | None = None,
+) -> AsyncIterator[EngineClient]:
+    if os.getenv("VLLM_WORKER_MULTIPROC_METHOD") == "forkserver":
+        # The executor is expected to be mp.
+        # Pre-import heavy modules in the forkserver process
+        logger.debug("Setup forkserver with pre-imports")
+        multiprocessing.set_start_method("forkserver")
+        multiprocessing.set_forkserver_preload(["vllm.v1.engine.async_llm"])
+        forkserver.ensure_running()
+        logger.debug("Forkserver setup complete!")
+
+    # Context manager to handle engine_client lifecycle
+    # Ensures everything is shutdown and cleaned up on error/exit
+    engine_args = AsyncEngineArgs.from_cli_args(args)
+    if client_config:
+        engine_args._api_process_count = client_config.get("client_count", 1)
+        engine_args._api_process_rank = client_config.get("client_index", 0)
+
+    async with build_async_engine_client_from_engine_args(
+        engine_args,
+        usage_context=usage_context,
+        client_config=client_config,
+    ) as engine:
+        yield engine
+
+
+@asynccontextmanager
+async def build_async_engine_client_from_engine_args(
+    engine_args: AsyncEngineArgs,
+    *,
+    usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
+    client_config: dict[str, Any] | None = None,
+) -> AsyncIterator[EngineClient]:
+    """
+    Create EngineClient, either:
+        - in-process using the AsyncLLMEngine Directly
+        - multiprocess using AsyncLLMEngine RPC
+
+    Returns the Client or None if the creation failed.
+    """
+
+    # Create the EngineConfig (determines if we can use V1).
+    vllm_config = engine_args.create_engine_config(usage_context=usage_context)
+
+    from vllm.v1.engine.async_llm import AsyncLLM
+
+    async_llm: AsyncLLM | None = None
+
+    # Don't mutate the input client_config
+    client_config = dict(client_config) if client_config else {}
+    client_count = client_config.pop("client_count", 1)
+    client_index = client_config.pop("client_index", 0)
+
+    try:
+        async_llm = AsyncLLM.from_vllm_config(
+            vllm_config=vllm_config,
+            usage_context=usage_context,
+            enable_log_requests=engine_args.enable_log_requests,
+            aggregate_engine_logging=engine_args.aggregate_engine_logging,
+            disable_log_stats=engine_args.disable_log_stats,
+            client_addresses=client_config,
+            client_count=client_count,
+            client_index=client_index,
+        )
+
+        # Don't keep the dummy data in memory
+        assert async_llm is not None
+        await async_llm.reset_mm_cache()
+
+        yield async_llm
+    finally:
+        if async_llm:
+            async_llm.shutdown(timeout=vllm_config.shutdown_timeout)
+
+
+def build_app(
+    args: Namespace,
+    supported_tasks: tuple["SupportedTask", ...] | None = None,
+    model_config: ModelConfig | None = None,
+) -> FastAPI:
+    if supported_tasks is None:
+        warnings.warn(
+            "The 'supported_tasks' parameter was not provided to "
+            "build_app and will be required in a future version. "
+            "Defaulting to ('generate',).",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        supported_tasks = _FALLBACK_SUPPORTED_TASKS
+
+    if args.disable_fastapi_docs:
+        app = FastAPI(
+            openapi_url=None, docs_url=None, redoc_url=None, lifespan=lifespan
+        )
+    elif args.enable_offline_docs:
+        app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)
+    else:
+        app = FastAPI(lifespan=lifespan)
+    app.state.args = args
+
+    from vllm.entrypoints.serve import register_vllm_serve_api_routers
+
+    register_vllm_serve_api_routers(app)
+
+    from vllm.entrypoints.openai.models.api_router import (
+        attach_router as register_models_api_router,
+    )
+
+    register_models_api_router(app)
+
+    from vllm.entrypoints.serve.sagemaker.api_router import (
+        attach_router as register_sagemaker_api_router,
+    )
+
+    register_sagemaker_api_router(app, supported_tasks, model_config)
+
+    if envs.VLLM_SERVER_DEV_MODE:
+        from vllm.entrypoints.serve import register_vllm_dev_api_routers
+
+        register_vllm_dev_api_routers(app)
+
+    if "generate" in supported_tasks:
+        from vllm.entrypoints.generate.api_router import (
+            register_generate_api_routers,
+        )
+
+        register_generate_api_routers(app)
+
+        from vllm.entrypoints.serve.elastic_ep.api_router import (
+            attach_router as elastic_ep_attach_router,
+        )
+
+        elastic_ep_attach_router(app)
+
+    if "generate" in supported_tasks or "render" in supported_tasks:
+        from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers
+
+        register_scale_out_api_routers(app, supported_tasks)
+
+    if "transcription" in supported_tasks or "realtime" in supported_tasks:
+        from vllm.entrypoints.speech_to_text.factories import (
+            register_speech_to_text_api_routers,
+        )
+
+        register_speech_to_text_api_routers(app, supported_tasks)
+
+    if any(task in POOLING_TASKS for task in supported_tasks):
+        from vllm.entrypoints.pooling.factories import register_pooling_api_routers
+
+        register_pooling_api_routers(app, supported_tasks, model_config)
+
+    # Endpoint plugins are attached last so their routes are registered after all core
+    # routers. This runs even for the CPU only render server. A plugin eligible for
+    # the `render` task still gets its routes registered. It receives
+    # `engine_client=None` at Phase B (see `_init_endpoint_plugins_state`).
+    _attach_endpoint_plugins(app, supported_tasks)
+
+    app.root_path = args.root_path
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=args.allowed_origins,
+        allow_credentials=args.allow_credentials,
+        allow_methods=args.allowed_methods,
+        allow_headers=args.allowed_headers,
+    )
+
+    app.exception_handler(HTTPException)(http_exception_handler)
+    app.exception_handler(RequestValidationError)(validation_exception_handler)
+    app.exception_handler(EngineGenerateError)(engine_error_handler)
+    app.exception_handler(EngineDeadError)(engine_error_handler)
+    app.exception_handler(GenerationError)(generation_error_handler)
+    # Register specific exception types so they are handled by
+    # ExceptionMiddleware (inside the Prometheus middleware) rather than
+    # ServerErrorMiddleware (outside it). Without this, these exceptions
+    # propagate through Prometheus as unhandled and get recorded as 5xx
+    # even though they result in 4xx responses to the client.
+    app.exception_handler(VLLMValidationError)(exception_handler)
+    app.exception_handler(VLLMUnprocessableEntityError)(exception_handler)
+    app.exception_handler(VLLMNotFoundError)(exception_handler)
+    app.exception_handler(ValueError)(exception_handler)
+    app.exception_handler(TypeError)(exception_handler)
+    app.exception_handler(OverflowError)(exception_handler)
+    app.exception_handler(NotImplementedError)(exception_handler)
+    app.exception_handler(Exception)(exception_handler)
+
+    # Ensure --api-key option from CLI takes precedence over VLLM_API_KEY
+    if tokens := [key for key in (args.api_key or [envs.VLLM_API_KEY]) if key]:
+        from vllm.entrypoints.serve.utils.server_utils import AuthenticationMiddleware
+
+        app.add_middleware(AuthenticationMiddleware, tokens=tokens)
+
+    if args.enable_request_id_headers:
+        from vllm.entrypoints.serve.utils.server_utils import XRequestIdMiddleware
+
+        app.add_middleware(XRequestIdMiddleware)
+
+    # Add scaling middleware to check for scaling state
+    app.add_middleware(ScalingMiddleware)
+
+    if "realtime" in supported_tasks:
+        # Add WebSocket metrics middleware
+        from vllm.entrypoints.speech_to_text.factories import (
+            add_websocket_metrics_middleware,
+        )
+
+        add_websocket_metrics_middleware(app)
+
+    if envs.VLLM_DEBUG_LOG_API_SERVER_RESPONSE:
+        logger.warning(
+            "CAUTION: Enabling log response in the API Server. "
+            "This can include sensitive information and should be "
+            "avoided in production."
+        )
+        app.middleware("http")(log_response)
+
+    for middleware in args.middleware:
+        module_path, object_name = middleware.rsplit(".", 1)
+        imported = getattr(importlib.import_module(module_path), object_name)
+        if inspect.isclass(imported):
+            app.add_middleware(imported)  # type: ignore[arg-type]
+        elif inspect.iscoroutinefunction(imported):
+            app.middleware("http")(imported)
+        else:
+            raise ValueError(
+                f"Invalid middleware {middleware}. Must be a function or a class."
+            )
+
+    app = sagemaker_standards_bootstrap(app)
+    return app
+
+
+async def init_app_state(
+    engine_client: EngineClient,
+    state: State,
+    args: Namespace,
+    supported_tasks: tuple["SupportedTask", ...] | None = None,
+) -> None:
+    vllm_config = engine_client.vllm_config
+
+    if args.tool_call_parser is not None:
+        from vllm.parser.metrics import init_parser_metrics
+
+        init_parser_metrics(
+            model_name=cast(str, vllm_config.model_config.served_model_name)
+        )
+
+    if supported_tasks is None:
+        warnings.warn(
+            "The 'supported_tasks' parameter was not provided to "
+            "init_app_state and will be required in a future version. "
+            "Please pass 'supported_tasks' explicitly.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        supported_tasks = _FALLBACK_SUPPORTED_TASKS
+
+    if args.served_model_name is not None:
+        served_model_names = args.served_model_name
+    else:
+        served_model_names = [args.model]
+
+    if args.enable_log_requests:
+        request_logger = RequestLogger(max_log_len=args.max_log_len)
+    else:
+        request_logger = None
+
+    base_model_paths = [
+        BaseModelPath(name=name, model_path=args.model) for name in served_model_names
+    ]
+
+    state.engine_client = engine_client
+    state.log_stats = not args.disable_log_stats
+    state.vllm_config = vllm_config
+    state.args = args
+    resolved_chat_template = load_chat_template(args.chat_template)
+
+    # Merge default_mm_loras into the static lora_modules
+    default_mm_loras = (
+        vllm_config.lora_config.default_mm_loras
+        if vllm_config.lora_config is not None
+        else {}
+    )
+    lora_modules = process_lora_modules(args.lora_modules, default_mm_loras)
+
+    state.openai_serving_models = OpenAIServingModels(
+        engine_client=engine_client,
+        base_model_paths=base_model_paths,
+        lora_modules=lora_modules,
+    )
+    await state.openai_serving_models.init_static_loras()
+
+    state.online_renderer = OnlineRenderer(
+        model_config=engine_client.model_config,
+        renderer=engine_client.renderer,
+        request_logger=request_logger,
+        chat_template=resolved_chat_template,
+        chat_template_content_format=args.chat_template_content_format,
+        trust_request_chat_template=args.trust_request_chat_template,
+        enable_auto_tools=args.enable_auto_tool_choice,
+        exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
+        tool_parser=args.tool_call_parser,
+        reasoning_parser=args.structured_outputs_config.reasoning_parser,
+        default_chat_template_kwargs=args.default_chat_template_kwargs,
+        log_error_stack=args.log_error_stack,
+    )
+
+    state.online_derenderer = OnlineDerenderer(
+        model_config=engine_client.model_config,
+        renderer=engine_client.renderer,
+        request_logger=request_logger,
+        chat_template=resolved_chat_template,
+        chat_template_content_format=args.chat_template_content_format,
+        trust_request_chat_template=args.trust_request_chat_template,
+        enable_auto_tools=args.enable_auto_tool_choice,
+        exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
+        tool_parser=args.tool_call_parser,
+        reasoning_parser=args.structured_outputs_config.reasoning_parser,
+        default_chat_template_kwargs=args.default_chat_template_kwargs,
+        log_error_stack=args.log_error_stack,
+    )
+
+    state.serving_tokenization = ServingTokenization(
+        state.openai_serving_models,
+        state.online_renderer,
+        request_logger=request_logger,
+        chat_template=resolved_chat_template,
+        chat_template_content_format=args.chat_template_content_format,
+        default_chat_template_kwargs=args.default_chat_template_kwargs,
+        trust_request_chat_template=args.trust_request_chat_template,
+    )
+
+    if "generate" in supported_tasks:
+        from vllm.entrypoints.generate.api_router import init_generate_state
+
+        await init_generate_state(
+            engine_client, state, args, request_logger, supported_tasks
+        )
+
+        from vllm.entrypoints.scale_out.factories import init_scale_out_state
+
+        init_scale_out_state(state, args, engine_client, request_logger)
+
+    if "transcription" in supported_tasks or "realtime" in supported_tasks:
+        from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state
+
+        init_speech_to_text_state(
+            engine_client, state, args, request_logger, supported_tasks
+        )
+
+    if any(task in POOLING_TASKS for task in supported_tasks):
+        from vllm.entrypoints.pooling.factories import init_pooling_state
+
+        init_pooling_state(engine_client, state, args, request_logger, supported_tasks)
+
+    await _init_endpoint_plugins_state(engine_client, state, args)
+
+    state.enable_server_load_tracking = args.enable_server_load_tracking
+    state.server_load_metrics = 0
+
+
+async def init_render_app_state(
+    vllm_config: VllmConfig,
+    state: State,
+    args: Namespace,
+) -> None:
+    """Initialise FastAPI app state for a CPU-only render server.
+
+    Unlike :func:`init_app_state` this function does not require an
+    :class:`~vllm.engine.protocol.EngineClient`; it bootstraps the
+    preprocessing pipeline (renderer, input_processor)
+    directly from the :class:`~vllm.config.VllmConfig`.
+    """
+    from vllm.entrypoints.chat_utils import load_chat_template
+    from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
+    from vllm.renderers import renderer_from_config
+    from vllm.renderers.online_renderer import OnlineRenderer
+
+    served_model_names = args.served_model_name or [args.model]
+    model_registry = OpenAIModelRegistry(
+        model_config=vllm_config.model_config,
+        base_model_paths=[
+            BaseModelPath(name=name, model_path=args.model)
+            for name in served_model_names
+        ],
+    )
+
+    if args.enable_log_requests:
+        request_logger = RequestLogger(max_log_len=args.max_log_len)
+    else:
+        request_logger = None
+
+    renderer = renderer_from_config(vllm_config)
+    resolved_chat_template = load_chat_template(args.chat_template)
+
+    state.online_renderer = OnlineRenderer(
+        model_config=vllm_config.model_config,
+        renderer=renderer,
+        request_logger=request_logger,
+        chat_template=resolved_chat_template,
+        chat_template_content_format=args.chat_template_content_format,
+        trust_request_chat_template=args.trust_request_chat_template,
+        enable_auto_tools=args.enable_auto_tool_choice,
+        exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
+        tool_parser=args.tool_call_parser,
+        reasoning_parser=args.reasoning_parser,
+        default_chat_template_kwargs=args.default_chat_template_kwargs,
+        log_error_stack=args.log_error_stack,
+    )
+
+    state.online_derenderer = OnlineDerenderer(
+        model_config=vllm_config.model_config,
+        renderer=renderer,
+        request_logger=request_logger,
+        chat_template=resolved_chat_template,
+        chat_template_content_format=args.chat_template_content_format,
+        trust_request_chat_template=args.trust_request_chat_template,
+        enable_auto_tools=args.enable_auto_tool_choice,
+        exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
+        tool_parser=args.tool_call_parser,
+        reasoning_parser=args.reasoning_parser,
+        default_chat_template_kwargs=args.default_chat_template_kwargs,
+        log_error_stack=args.log_error_stack,
+    )
+
+    state.openai_serving_models = model_registry
+    state.serving_tokenization = ServingTokenization(
+        model_registry,
+        state.online_renderer,
+        request_logger=request_logger,
+        chat_template=resolved_chat_template,
+        chat_template_content_format=args.chat_template_content_format,
+        default_chat_template_kwargs=args.default_chat_template_kwargs,
+        trust_request_chat_template=args.trust_request_chat_template,
+    )
+
+    from vllm.entrypoints.scale_out.factories import init_render_state
+
+    init_render_state(state, request_logger)
+
+    state.vllm_config = vllm_config
+    # Disable stats logging — there is no engine to poll.
+    state.log_stats = False
+    state.engine_client = None
+    state.args = args
+    state.enable_server_load_tracking = False
+    state.server_load_metrics = 0
+
+    # No `EngineClient` exists for the render server, so plugins get `None` and
+    # must handle it themselves (see `EndpointPlugin.init_state`).
+    await _init_endpoint_plugins_state(None, state, args)
+
+
+def create_server_socket(
+    addr: tuple[str, int],
+    *,
+    reuse_port: bool,
+) -> socket.socket:
+    family = socket.AF_INET
+    if is_valid_ipv6_address(addr[0]):
+        family = socket.AF_INET6
+
+    sock = socket.socket(family=family, type=socket.SOCK_STREAM)
+    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    if reuse_port:
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+    sock.bind(addr)
+
+    return sock
+
+
+def create_server_unix_socket(path: str) -> socket.socket:
+    sock = socket.socket(family=socket.AF_UNIX, type=socket.SOCK_STREAM)
+    sock.bind(path)
+    return sock
+
+
+def validate_api_server_args(args):
+    valid_tool_parses = ToolParserManager.list_registered()
+    if args.enable_auto_tool_choice and args.tool_call_parser not in valid_tool_parses:
+        raise KeyError(
+            f"invalid tool call parser: {args.tool_call_parser} "
+            f"(chose from {{ {','.join(valid_tool_parses)} }})"
+        )
+
+    valid_reasoning_parsers = ReasoningParserManager.list_registered()
+    if (
+        reasoning_parser := args.structured_outputs_config.reasoning_parser
+    ) and reasoning_parser not in valid_reasoning_parsers:
+        raise KeyError(
+            f"invalid reasoning parser: {reasoning_parser} "
+            f"(chose from {{ {','.join(valid_reasoning_parsers)} }})"
+        )
+
+
+@instrument(span_name="API server setup")
+def setup_server(args, *, reuse_port: bool):
+    """Validate API server args and create the server socket."""
+
+    log_version_and_model(logger, VLLM_VERSION, args.model)
+    log_non_default_args(args)
+
+    if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
+        ToolParserManager.import_tool_parser(args.tool_parser_plugin)
+
+    if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:
+        ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)
+
+    validate_api_server_args(args)
+
+    # workaround to make sure that we bind the port before the engine is set up.
+    # This avoids race conditions with ray.
+    # see https://github.com/vllm-project/vllm/issues/8204
+    if args.uds:
+        sock = create_server_unix_socket(args.uds)
+    else:
+        sock_addr = (args.host or "", args.port)
+        sock = create_server_socket(sock_addr, reuse_port=reuse_port)
+
+    # workaround to avoid footguns where uvicorn drops requests with too
+    # many concurrent requests active
+    set_ulimit()
+
+    if args.uds:
+        listen_address = f"unix:{args.uds}"
+    else:
+        addr, port = sock_addr
+        is_ssl = args.ssl_keyfile and args.ssl_certfile
+        host_part = f"[{addr}]" if is_valid_ipv6_address(addr) else addr or "0.0.0.0"
+        listen_address = f"http{'s' if is_ssl else ''}://{host_part}:{port}"
+    return listen_address, sock
+
+
+async def build_and_serve(
+    engine_client: EngineClient,
+    listen_address: str,
+    sock: socket.socket,
+    args: Namespace,
+    **uvicorn_kwargs,
+) -> asyncio.Task:
+    """Build FastAPI app, initialize state, and start serving.
+
+    Returns the shutdown task for the caller to await.
+    """
+
+    # Get uvicorn log config (from file or with endpoint filter)
+    log_config = get_uvicorn_log_config(args)
+    if log_config is not None:
+        uvicorn_kwargs["log_config"] = log_config
+
+    supported_tasks = await engine_client.get_supported_tasks()
+    model_config = engine_client.model_config
+
+    logger.info("Supported tasks: %s", supported_tasks)
+    app = build_app(args, supported_tasks, model_config)
+    await init_app_state(engine_client, app.state, args, supported_tasks)
+
+    logger.info("Starting vLLM server on %s", listen_address)
+
+    return await serve_http(
+        app,
+        sock=sock,
+        enable_ssl_refresh=args.enable_ssl_refresh,
+        host=args.host,
+        port=args.port,
+        log_level=args.uvicorn_log_level,
+        # NOTE: When the 'disable_uvicorn_access_log' value is True,
+        # no access log will be output.
+        access_log=not args.disable_uvicorn_access_log,
+        timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
+        ssl_keyfile=args.ssl_keyfile,
+        ssl_certfile=args.ssl_certfile,
+        ssl_ca_certs=args.ssl_ca_certs,
+        ssl_cert_reqs=args.ssl_cert_reqs,
+        ssl_ciphers=args.ssl_ciphers,
+        h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
+        h11_max_header_count=args.h11_max_header_count,
+        **uvicorn_kwargs,
+    )
+
+
+async def build_and_serve_renderer(
+    vllm_config: VllmConfig,
+    listen_address: str,
+    sock: socket.socket,
+    args: Namespace,
+    **uvicorn_kwargs,
+) -> asyncio.Task:
+    """Build FastAPI app for a CPU-only render server, initialize state, and
+    start serving.
+
+    Returns the shutdown task for the caller to await.
+    """
+
+    # Get uvicorn log config (from file or with endpoint filter)
+    log_config = get_uvicorn_log_config(args)
+    if log_config is not None:
+        uvicorn_kwargs["log_config"] = log_config
+
+    app = build_app(args, ("render",))
+    await init_render_app_state(vllm_config, app.state, args)
+
+    logger.info("Starting vLLM server on %s", listen_address)
+
+    return await serve_http(
+        app,
+        sock=sock,
+        enable_ssl_refresh=args.enable_ssl_refresh,
+        host=args.host,
+        port=args.port,
+        log_level=args.uvicorn_log_level,
+        # NOTE: When the 'disable_uvicorn_access_log' value is True,
+        # no access log will be output.
+        access_log=not args.disable_uvicorn_access_log,
+        timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
+        ssl_keyfile=args.ssl_keyfile,
+        ssl_certfile=args.ssl_certfile,
+        ssl_ca_certs=args.ssl_ca_certs,
+        ssl_cert_reqs=args.ssl_cert_reqs,
+        ssl_ciphers=args.ssl_ciphers,
+        h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
+        h11_max_header_count=args.h11_max_header_count,
+        **uvicorn_kwargs,
+    )
+
+
+async def run_server(args, **uvicorn_kwargs) -> None:
+    """Run a single-worker API server."""
+
+    decorate_logs("APIServer", skip_if_decorated=True)
+
+    # Interrupt initialization if SIGTERM arrives before uvicorn installs its
+    # own signal handlers. Once uvicorn is running it replaces this.
+    def _interrupt_init(*_) -> None:
+        raise KeyboardInterrupt("terminated")
+
+    signal.signal(signal.SIGTERM, _interrupt_init)
+
+    listen_address, sock = setup_server(args, reuse_port=False)
+    await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
+
+
+async def run_server_worker(
+    listen_address, sock, args, client_config=None, **uvicorn_kwargs
+) -> None:
+    """Run a single API server worker."""
+
+    if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
+        ToolParserManager.import_tool_parser(args.tool_parser_plugin)
+
+    if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:
+        ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)
+
+    async with build_async_engine_client(
+        args,
+        client_config=client_config,
+    ) as engine_client:
+        shutdown_task = await build_and_serve(
+            engine_client, listen_address, sock, args, **uvicorn_kwargs
+        )
+    # NB: Await server shutdown only after the backend context is exited
+    try:
+        await shutdown_task
+    finally:
+        sock.close()
+
+
+if __name__ == "__main__":
+    # NOTE(simon):
+    # This section should be in sync with vllm/entrypoints/cli/main.py for CLI
+    # entrypoints.
+    cli_env_setup()
+    parser = FlexibleArgumentParser(
+        description="vLLM OpenAI-Compatible RESTful API server."
+    )
+    parser = make_arg_parser(parser)
+    args = parser.parse_args()
+    validate_parsed_serve_args(args)
+
+    uvloop.run(run_server(args))

vllm/entrypoints/openai/chat_completion/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

vllm/entrypoints/openai/chat_completion/api_router.py

@@ -0,0 +1,106 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+
+from http import HTTPStatus
+
+from fastapi import APIRouter, Depends, FastAPI, Request
+from fastapi.responses import JSONResponse, StreamingResponse
+
+from vllm.entrypoints.openai.chat_completion.batch_serving import OpenAIServingChatBatch
+from vllm.entrypoints.openai.chat_completion.protocol import (
+    BatchChatCompletionRequest,
+    ChatCompletionRequest,
+    ChatCompletionResponse,
+)
+from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
+from vllm.entrypoints.serve.utils.api_utils import (
+    load_aware_call,
+    validate_json_request,
+    with_cancellation,
+)
+from vllm.entrypoints.serve.utils.orca_metrics import metrics_header
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+router = APIRouter()
+ENDPOINT_LOAD_METRICS_FORMAT_HEADER_LABEL = "endpoint-load-metrics-format"
+
+
+def chat(request: Request) -> OpenAIServingChat | None:
+    return request.app.state.openai_serving_chat
+
+
+def batch_chat(request: Request) -> OpenAIServingChatBatch | None:
+    return request.app.state.openai_serving_chat_batch
+
+
+@router.post(
+    "/v1/chat/completions",
+    dependencies=[Depends(validate_json_request)],
+    responses={
+        HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
+        HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
+        HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
+        HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
+        HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
+    },
+)
+@with_cancellation
+@load_aware_call
+async def create_chat_completion(request: ChatCompletionRequest, raw_request: Request):
+    metrics_header_format = raw_request.headers.get(
+        ENDPOINT_LOAD_METRICS_FORMAT_HEADER_LABEL, ""
+    )
+    handler = chat(raw_request)
+    if handler is None:
+        raise NotImplementedError("The model does not support Chat Completions API")
+
+    generator = await handler.create_chat_completion(request, raw_request)
+
+    if isinstance(generator, ErrorResponse):
+        return JSONResponse(
+            content=generator.model_dump(), status_code=generator.error.code
+        )
+
+    elif isinstance(generator, ChatCompletionResponse):
+        return JSONResponse(
+            content=generator.model_dump(),
+            headers=metrics_header(metrics_header_format),
+        )
+
+    return StreamingResponse(content=generator, media_type="text/event-stream")
+
+
+@router.post(
+    "/v1/chat/completions/batch",
+    dependencies=[Depends(validate_json_request)],
+    responses={
+        HTTPStatus.OK.value: {},
+        HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
+        HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
+        HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
+        HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
+    },
+)
+@with_cancellation
+@load_aware_call
+async def create_batch_chat_completion(
+    request: BatchChatCompletionRequest, raw_request: Request
+):
+    handler = batch_chat(raw_request)
+    if handler is None:
+        raise NotImplementedError("The model does not support Chat Completions API")
+
+    result = await handler.create_batch_chat_completion(request, raw_request)
+
+    if isinstance(result, ErrorResponse):
+        return JSONResponse(content=result.model_dump(), status_code=result.error.code)
+
+    return JSONResponse(content=result.model_dump())
+
+
+def attach_router(app: FastAPI):
+    app.include_router(router)

vllm/entrypoints/openai/chat_completion/batch_serving.py

@@ -0,0 +1,327 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import asyncio
+import time
+from collections.abc import AsyncGenerator
+from http import HTTPStatus
+
+from fastapi import Request
+
+from vllm.entrypoints.chat_utils import ConversationMessage
+from vllm.entrypoints.openai.chat_completion.protocol import (
+    BatchChatCompletionRequest,
+    ChatCompletionResponse,
+    ChatCompletionResponseChoice,
+    ChatMessage,
+)
+from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
+from vllm.entrypoints.openai.engine.protocol import (
+    ErrorResponse,
+    RequestResponseMetadata,
+    UsageInfo,
+)
+from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
+from vllm.inputs import EngineInput
+from vllm.logger import init_logger
+from vllm.outputs import RequestOutput
+from vllm.parser.abstract_parser import Parser
+from vllm.tokenizers import TokenizerLike
+from vllm.utils.async_utils import merge_async_iterators
+from vllm.utils.collection_utils import as_list
+
+logger = init_logger(__name__)
+
+
+class OpenAIServingChatBatch(OpenAIServingChat):
+    """Extends OpenAIServingChat with the /v1/chat/completions/batch endpoint.
+
+    Processes N conversations from a single request concurrently and returns
+    one choice per conversation indexed 0, 1, ..., N-1.
+    """
+
+    async def render_batch_chat_request(
+        self,
+        request: BatchChatCompletionRequest,
+    ) -> tuple[list[list[ConversationMessage]], list[EngineInput]] | ErrorResponse:
+        """Validate the model and preprocess a batched chat completion request.
+
+        Performs engine-aware checks then delegates per-conversation
+        preprocessing to OnlineRenderer, validating the chat template
+        once for the whole batch.
+
+        Returns:
+            A tuple of (all_conversations, engine_prompts) on success — one
+            entry per conversation — or an ErrorResponse on failure.
+        """
+        error_check_ret = await self._check_model(request)
+        if error_check_ret is not None:
+            logger.error("Error with model %s", error_check_ret)
+            return error_check_ret
+
+        if self.engine_client.errored:
+            raise self.engine_client.dead_error
+
+        renderer = self.online_renderer
+
+        if not renderer.use_harmony:
+            # Common case: validate the chat template once for the whole batch.
+            error_check_ret = renderer.validate_chat_template(
+                request_chat_template=request.chat_template,
+                chat_template_kwargs=request.chat_template_kwargs,
+                trust_request_chat_template=renderer.trust_request_chat_template,
+            )
+            if error_check_ret is not None:
+                return error_check_ret
+
+        parser = renderer.parser
+        tool_dicts: list[dict] | None = None
+
+        all_conversations: list[list[ConversationMessage]] = []
+        all_engine_prompts: list[EngineInput] = []
+
+        for messages in request.messages:
+            single_request = request.to_chat_completion_request(messages)
+            if renderer.use_harmony:
+                conversation, engine_prompts = renderer._make_request_with_harmony(
+                    single_request, should_include_tools=tool_dicts is not None
+                )
+            else:
+                conversation, engine_prompts = await renderer.preprocess_chat(
+                    single_request,
+                    messages,
+                    default_template=renderer.chat_template,
+                    default_template_content_format=renderer.chat_template_content_format,
+                    default_template_kwargs=renderer.default_chat_template_kwargs,
+                    tool_dicts=tool_dicts,
+                    parser=parser,
+                )
+            all_conversations.append(conversation)
+            all_engine_prompts.append(engine_prompts[0])
+
+        return all_conversations, all_engine_prompts
+
+    async def create_batch_chat_completion(
+        self,
+        request: BatchChatCompletionRequest,
+        raw_request: Request | None = None,
+    ) -> ChatCompletionResponse | ErrorResponse:
+        """Batch Chat Completion endpoint (/v1/chat/completions/batch).
+
+        Processes N conversations from a single request concurrently and
+        returns one choice per conversation indexed 0, 1, ..., N-1.
+        Streaming, tool use, and beam search are not supported.
+        """
+        tokenizer = self.renderer.tokenizer
+        assert tokenizer is not None
+        single_requests = [
+            request.to_chat_completion_request(messages)
+            for messages in request.messages
+        ]
+
+        parser: Parser | None = None
+        if self.parser_cls is not None:
+            chat_template_kwargs = self._effective_chat_template_kwargs(
+                single_requests[0]
+            )
+            parser = self.parser_cls(
+                tokenizer,
+                None,  # tools
+                chat_template_kwargs=chat_template_kwargs,
+            )
+
+        render_result = await self.render_batch_chat_request(request)
+        if isinstance(render_result, ErrorResponse):
+            return render_result
+        all_conversations, engine_prompts = render_result
+
+        request_id = (
+            f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
+        )
+        request_metadata = RequestResponseMetadata(request_id=request_id)
+        if raw_request:
+            raw_request.state.request_metadata = request_metadata
+
+        lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True)
+        model_name = self.models.model_name(lora_request)
+        data_parallel_rank = self._get_data_parallel_rank(raw_request)
+        max_model_len = self.model_config.max_model_len
+
+        generators: list[AsyncGenerator[RequestOutput, None]] = []
+        for i, engine_prompt in enumerate(engine_prompts):
+            sub_request_id = f"{request_id}_{i}"
+            max_tokens = get_max_tokens(
+                max_model_len,
+                request.max_completion_tokens
+                if request.max_completion_tokens is not None
+                else request.max_tokens,
+                self._extract_prompt_len(engine_prompt),
+                self.default_sampling_params,
+                self.override_max_tokens,
+            )
+            single_request = single_requests[i]
+            sampling_params = single_request.to_sampling_params(
+                max_tokens, self.default_sampling_params
+            )
+            self._log_inputs(
+                sub_request_id,
+                engine_prompt,
+                params=sampling_params,
+                lora_request=lora_request,
+            )
+            trace_headers = (
+                None
+                if raw_request is None
+                else await self._get_trace_headers(raw_request.headers)
+            )
+            generators.append(
+                self.engine_client.generate(
+                    engine_prompt,
+                    sampling_params,
+                    sub_request_id,
+                    lora_request=lora_request,
+                    trace_headers=trace_headers,
+                    priority=request.priority,
+                    data_parallel_rank=data_parallel_rank,
+                    reasoning_ended=None,
+                )
+            )
+
+        return await self.chat_completion_full_generator_batch(
+            request,  # type: ignore[arg-type]
+            generators,
+            request_id,
+            model_name,
+            all_conversations,
+            tokenizer,
+            request_metadata,
+            parser,
+        )
+
+    async def chat_completion_full_generator_batch(
+        self,
+        request: BatchChatCompletionRequest,  # type: ignore[override]
+        generators: list[AsyncGenerator[RequestOutput, None]],
+        request_id: str,
+        model_name: str,
+        all_conversations: list[list[ConversationMessage]],
+        tokenizer: TokenizerLike,
+        request_metadata: RequestResponseMetadata,
+        parser: Parser | None = None,
+    ) -> ErrorResponse | ChatCompletionResponse:
+        """Handle batched (non-streaming) chat completions.
+
+        Fans out N generators (one per conversation in the batch), collects
+        the final output for each, and assembles a single
+        ``ChatCompletionResponse`` whose ``choices`` are indexed 0,...,N-1.
+
+        Tool-use and streaming are rejected upstream by the
+        ``check_batch_mode`` validator, so neither needs to be handled here.
+        """
+        created_time = int(time.time())
+        final_results: dict[int, RequestOutput] = {}
+        try:
+            async for prompt_idx, res in merge_async_iterators(*generators):
+                final_results[prompt_idx] = res
+        except asyncio.CancelledError:
+            return self.create_error_response("Client disconnected")
+
+        choices: list[ChatCompletionResponseChoice] = []
+        total_prompt_tokens = 0
+        total_completion_tokens = 0
+
+        for prompt_idx in range(len(generators)):
+            final_res = final_results.get(prompt_idx)
+            if final_res is None:
+                return self.create_error_response(
+                    f"No output received from the engine for prompt {prompt_idx}.",
+                    err_type="InternalServerError",
+                    status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
+                )
+
+            assert final_res.prompt_token_ids is not None
+            num_prompt_tokens = len(final_res.prompt_token_ids)
+            if final_res.encoder_prompt_token_ids is not None:
+                num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
+            total_prompt_tokens += num_prompt_tokens
+            total_completion_tokens += sum(
+                len(output.token_ids) for output in final_res.outputs
+            )
+
+            for output in final_res.outputs:
+                self._raise_if_error(output.finish_reason, request_id)
+
+                if request.logprobs and request.top_logprobs is not None:
+                    assert output.logprobs is not None, "Did not output logprobs"
+                    logprobs = self._create_chat_logprobs(
+                        token_ids=output.token_ids,
+                        top_logprobs=output.logprobs,
+                        num_output_top_logprobs=request.top_logprobs,
+                        tokenizer=tokenizer,
+                        return_as_token_id=request.return_tokens_as_token_ids,
+                    )
+                else:
+                    logprobs = None
+
+                if parser is not None:
+                    reasoning, content, _ = parser.parse(
+                        output.text,
+                        request=request,  # type: ignore[arg-type]
+                        model_output_token_ids=output.token_ids,
+                    )
+                    if not request.include_reasoning:
+                        reasoning = None
+                else:
+                    reasoning = None
+                    content = output.text
+
+                role = (
+                    self.response_role
+                    if request.add_generation_prompt
+                    else request.messages[prompt_idx][-1]["role"]
+                )
+
+                message = ChatMessage(role=role, reasoning=reasoning, content=content)
+
+                if request.echo:
+                    conversation = all_conversations[prompt_idx]
+                    last_msg_content: str | list[dict[str, str]] = ""
+                    if conversation and "content" in conversation[-1]:
+                        last_msg_content = conversation[-1]["content"] or ""
+                    if isinstance(last_msg_content, list):
+                        last_msg_content = "\n".join(
+                            msg["text"] for msg in last_msg_content
+                        )
+                    message.content = last_msg_content + (message.content or "")
+
+                choice_data = ChatCompletionResponseChoice(
+                    index=prompt_idx,
+                    message=message,
+                    logprobs=logprobs,
+                    finish_reason=output.finish_reason
+                    if output.finish_reason
+                    else "stop",
+                    stop_reason=output.stop_reason,
+                    token_ids=(
+                        as_list(output.token_ids) if request.return_token_ids else None
+                    ),
+                )
+                choices.append(choice_data)
+
+        usage = UsageInfo(
+            prompt_tokens=total_prompt_tokens,
+            completion_tokens=total_completion_tokens,
+            total_tokens=total_prompt_tokens + total_completion_tokens,
+        )
+        request_metadata.final_usage_info = usage
+
+        choices.sort(key=lambda c: c.index)
+
+        return ChatCompletionResponse(
+            id=request_id,
+            created=created_time,
+            model=model_name,
+            choices=choices,
+            usage=usage,
+            system_fingerprint=self.system_fingerprint,
+        )

vllm/entrypoints/openai/cli_args.py

@@ -0,0 +1,420 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+This file contains the command line arguments for the vLLM's
+OpenAI-compatible server. It is kept in a separate file for documentation
+purposes.
+"""
+
+import argparse
+import json
+import ssl
+from collections.abc import Sequence
+from dataclasses import field
+from typing import Any, Literal
+
+import vllm.envs as envs
+from vllm.config import config
+from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
+from vllm.entrypoints.chat_utils import (
+    ChatTemplateContentFormatOption,
+    validate_chat_template,
+)
+from vllm.entrypoints.openai.models.protocol import LoRAModulePath
+from vllm.entrypoints.serve.utils.constants import (
+    H11_MAX_HEADER_COUNT_DEFAULT,
+    H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT,
+)
+from vllm.tool_parsers import ToolParserManager
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+
+
+class LoRAParserAction(argparse.Action):
+    def __call__(
+        self,
+        parser: argparse.ArgumentParser,
+        namespace: argparse.Namespace,
+        values: str | Sequence[str] | None,
+        option_string: str | None = None,
+    ):
+        if values is None:
+            values = []
+        if isinstance(values, str):
+            raise TypeError("Expected values to be a list")
+
+        lora_list: list[LoRAModulePath] = []
+        for item in values:
+            if item in [None, ""]:  # Skip if item is None or empty string
+                continue
+            if "=" in item and "," not in item:  # Old format: name=path
+                name, path = item.split("=")
+                lora_list.append(LoRAModulePath(name, path))
+            else:  # Assume JSON format
+                try:
+                    lora_dict = json.loads(item)
+                    lora = LoRAModulePath(**lora_dict)
+                    lora_list.append(lora)
+                except json.JSONDecodeError:
+                    parser.error(f"Invalid JSON format for --lora-modules: {item}")
+                except TypeError as e:
+                    parser.error(
+                        f"Invalid fields for --lora-modules: {item} - {str(e)}"
+                    )
+        setattr(namespace, self.dest, lora_list)
+
+
+@config
+class BaseFrontendArgs:
+    """Base arguments for the OpenAI-compatible frontend server.
+
+    This base class does not include host, port, and server-specific arguments
+    like SSL, CORS, and HTTP server settings. Those arguments are added by
+    the subclasses.
+    """
+
+    lora_modules: list[LoRAModulePath] | None = None
+    """LoRA modules configurations in either 'name=path' format or JSON format
+    or JSON list format. Example (old format): `'name=path'` Example (new
+    format): `{\"name\": \"name\", \"path\": \"lora_path\",
+    \"base_model_name\": \"id\"}`"""
+    chat_template: str | None = None
+    """The file path to the chat template, or the template in single-line form
+    for the specified model."""
+    chat_template_content_format: ChatTemplateContentFormatOption = "auto"
+    """The format to render message content within a chat template.
+
+    * "string" will render the content as a string. Example: `"Hello World"`
+    * "openai" will render the content as a list of dictionaries, similar to
+      OpenAI schema. Example: `[{"type": "text", "text": "Hello world!"}]`"""
+    trust_request_chat_template: bool = False
+    """Whether to trust the chat template provided in the request. If False,
+    the server will always use the chat template specified by `--chat-template`
+    or the ones from tokenizer."""
+    default_chat_template_kwargs: dict[str, Any] | None = None
+    """Default keyword arguments to pass to the chat template renderer.
+    These will be merged with request-level chat_template_kwargs,
+    with request values taking precedence. Useful for setting default
+    behavior for reasoning models. Example: '{"enable_thinking": false}'
+    to disable thinking mode by default for Qwen3/DeepSeek models."""
+    response_role: str = "assistant"
+    """The role name to return if `request.add_generation_prompt=true`."""
+    return_tokens_as_token_ids: bool = False
+    """When `--max-logprobs` is specified, represents single tokens as
+    strings of the form 'token_id:{token_id}' so that tokens that are not
+    JSON-encodable can be identified."""
+    enable_auto_tool_choice: bool = False
+    """Enable auto tool choice for supported models. Use `--tool-call-parser`
+    to specify which parser to use."""
+    exclude_tools_when_tool_choice_none: bool = False
+    """If specified, exclude tool definitions in prompts when
+    tool_choice='none'."""
+    tool_call_parser: str | None = None
+    """Select the tool call parser depending on the model that you're using.
+    This is used to parse the model-generated tool call into OpenAI API format.
+    Required for `--enable-auto-tool-choice`. You can choose any option from
+    the built-in parsers or register a plugin via `--tool-parser-plugin`."""
+    tool_parser_plugin: str = ""
+    """Special the tool parser plugin write to parse the model-generated tool
+    into OpenAI API format, the name register in this plugin can be used in
+    `--tool-call-parser`."""
+    tool_server: str | None = None
+    """Comma-separated list of host:port pairs (IPv4, IPv6, or hostname).
+    Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for
+    built-in demo tools (browser and Python code interpreter). WARNING:
+    The `demo` Python tool executes model-generated code in Docker without
+    network isolation by default. See the security guide for more
+    information."""
+    log_config_file: str | None = envs.VLLM_LOGGING_CONFIG_PATH
+    """Path to logging config JSON file for both vllm and uvicorn"""
+    max_log_len: int | None = None
+    """Max number of prompt characters or prompt ID numbers being printed in
+    log. The default of None means unlimited."""
+    enable_prompt_tokens_details: bool = False
+    """If set to True, enable prompt_tokens_details in usage."""
+    enable_per_request_metrics: bool = False
+    """If set to True, include per-request timing metrics in API responses."""
+    enable_server_load_tracking: bool = False
+    """If set to True, enable tracking server_load_metrics in the app state."""
+    enable_force_include_usage: bool = False
+    """If set to True, including usage on every request."""
+    enable_tokenizer_info_endpoint: bool = False
+    """Enable the `/tokenizer_info` endpoint. May expose chat
+    templates and other tokenizer configuration."""
+    enable_log_outputs: bool = False
+    """If set to True, log model outputs (generations).
+    Requires `--enable-log-requests`. As with `--enable-log-requests`,
+    information is only logged at INFO level at maximum."""
+    enable_log_deltas: bool = True
+    """If set to False, output deltas will not be logged. Relevant only if 
+    --enable-log-outputs is set.
+    """
+    log_error_stack: bool = envs.VLLM_SERVER_DEV_MODE
+    """If set to True, log the stack trace of error responses"""
+    tokens_only: bool = False
+    """
+    If set to True, only enable the Tokens In<>Out endpoint.
+    This is intended for use in a Disaggregated Everything setup.
+    """
+    fingerprint_mode: Literal["full", "hash", "custom", "none"] = "full"
+    """Controls the ``system_fingerprint`` field on responses.
+
+    - ``full`` (default): ``vllm-<version>[-<parallelism>]-<hash8>``. Encodes
+      server version, non-trivial parallelism degrees (tp/pp/dp/ep), and an
+      8-char config hash.
+    - ``hash``: ``vllm-<version>-<hash8>``. Parallelism stripped.
+    - ``custom``: emits the literal string from ``--fingerprint-value``.
+    - ``none``: the field is omitted (serialized as ``null``).
+    """
+    fingerprint_value: str | None = None
+    """Literal fingerprint string used when ``--fingerprint-mode=custom``."""
+
+    @classmethod
+    def _customize_cli_kwargs(
+        cls,
+        frontend_kwargs: dict[str, Any],
+    ) -> dict[str, Any]:
+        """Customize argparse kwargs before arguments are registered.
+
+        Subclasses should override this and call
+        ``super()._customize_cli_kwargs(frontend_kwargs)`` first.
+        """
+        # Special case: default_chat_template_kwargs needs json.loads type
+        frontend_kwargs["default_chat_template_kwargs"]["type"] = json.loads
+
+        # Special case: LoRA modules need custom parser action and
+        # optional_type(str)
+        frontend_kwargs["lora_modules"]["type"] = optional_type(str)
+        frontend_kwargs["lora_modules"]["action"] = LoRAParserAction
+
+        # Special case: Tool call parser shows built-in options.
+        valid_tool_parsers = list(ToolParserManager.list_registered())
+        parsers_str = ",".join(valid_tool_parsers)
+        frontend_kwargs["tool_call_parser"]["metavar"] = (
+            f"{{{parsers_str}}} or name registered in --tool-parser-plugin"
+        )
+        return frontend_kwargs
+
+    @classmethod
+    def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
+        """Register CLI arguments for this frontend class.
+
+        Subclasses should override ``_customize_cli_kwargs`` instead of
+        this method so that base-class postprocessing is always applied.
+        """
+        from vllm.engine.arg_utils import get_kwargs
+
+        frontend_kwargs = get_kwargs(cls)
+        frontend_kwargs = cls._customize_cli_kwargs(frontend_kwargs)
+
+        group_name = cls.__name__.replace("Args", "")
+        frontend_group = parser.add_argument_group(
+            title=group_name,
+            description=cls.__doc__,
+        )
+        for key, value in frontend_kwargs.items():
+            extra_flags = value.pop("flags", [])
+            frontend_group.add_argument(
+                *extra_flags, f"--{key.replace('_', '-')}", **value
+            )
+
+        return parser
+
+
+@config
+class FrontendArgs(BaseFrontendArgs):
+    """Arguments for the OpenAI-compatible frontend server."""
+
+    host: str | None = None
+    """Host name."""
+    port: int = 8000
+    """Port number."""
+    data_parallel_supervisor_port: int = 9256
+    """HTTP port for aggregated health endpoints in multi-port external LB
+    mode."""
+    dp_supervisor_probe_interval_s: float = 5.0
+    """Seconds between aggregated health probes in multi-port external LB mode."""
+    dp_supervisor_probe_timeout_s: float = 5.0
+    """Seconds to wait between retries when a child health probe fails with a
+    connection error in multi-port external LB mode."""
+    dp_supervisor_probe_failure_threshold: int = 3
+    """Number of consecutive connection-error retries before a child health
+    probe is declared failed in multi-port external LB mode."""
+    uds: str | None = None
+    """Unix domain socket path. If set, host and port arguments are ignored."""
+    uvicorn_log_level: Literal[
+        "critical", "error", "warning", "info", "debug", "trace"
+    ] = "info"
+    """Log level for uvicorn."""
+    disable_uvicorn_access_log: bool = False
+    """Disable uvicorn access log."""
+    disable_access_log_for_endpoints: str | None = None
+    """Comma-separated list of endpoint paths to exclude from uvicorn access
+    logs. This is useful to reduce log noise from high-frequency endpoints
+    like health checks. Example: "/health,/metrics,/ping".
+    When set, access logs for requests to these paths will be suppressed
+    while keeping logs for other endpoints."""
+    allow_credentials: bool = False
+    """Allow credentials."""
+    allowed_origins: list[str] = field(default_factory=lambda: ["*"])
+    """Allowed origins."""
+    allowed_methods: list[str] = field(default_factory=lambda: ["*"])
+    """Allowed methods."""
+    allowed_headers: list[str] = field(default_factory=lambda: ["*"])
+    """Allowed headers."""
+    api_key: list[str] | None = None
+    """If provided, the server will require one of these keys to be presented in
+    the header."""
+    ssl_keyfile: str | None = None
+    """The file path to the SSL key file."""
+    ssl_certfile: str | None = None
+    """The file path to the SSL cert file."""
+    ssl_ca_certs: str | None = None
+    """The CA certificates file."""
+    enable_ssl_refresh: bool = False
+    """Refresh SSL Context when SSL certificate files change"""
+    ssl_cert_reqs: int = int(ssl.CERT_NONE)
+    """Whether client certificate is required (see stdlib ssl module's)."""
+    ssl_ciphers: str | None = None
+    """SSL cipher suites for HTTPS (TLS 1.2 and below only).
+    Example: 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305'"""
+    root_path: str | None = None
+    """FastAPI root_path when app is behind a path based routing proxy."""
+    middleware: list[str] = field(default_factory=lambda: [])
+    """Additional ASGI middleware to apply to the app. We accept multiple
+    --middleware arguments. The value should be an import path. If a function
+    is provided, vLLM will add it to the server using
+    `@app.middleware('http')`. If a class is provided, vLLM will
+    add it to the server using `app.add_middleware()`."""
+    enable_request_id_headers: bool = False
+    """If specified, API server will add X-Request-Id header to responses."""
+    disable_fastapi_docs: bool = False
+    """Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint."""
+    h11_max_incomplete_event_size: int = H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT
+    """Maximum size (bytes) of an incomplete HTTP event (header or body) for
+    h11 parser. Helps mitigate header abuse. Default: 4194304 (4 MB)."""
+    h11_max_header_count: int = H11_MAX_HEADER_COUNT_DEFAULT
+    """Maximum number of HTTP headers allowed in a request for h11 parser.
+    Helps mitigate header abuse. Default: 256."""
+    enable_offline_docs: bool = False
+    """
+    Enable offline FastAPI documentation for air-gapped environments.
+    Uses vendored static assets bundled with vLLM.
+    """
+    enable_flash_late_interaction: bool = True
+    """If set, run pooling score MaxSim on GPU in the API server process.
+    Can significantly improve late-interaction scoring performance."""
+
+    @classmethod
+    def _customize_cli_kwargs(
+        cls,
+        frontend_kwargs: dict[str, Any],
+    ) -> dict[str, Any]:
+        frontend_kwargs = super()._customize_cli_kwargs(frontend_kwargs)
+
+        # Special case: allowed_origins, allowed_methods, allowed_headers all
+        # need json.loads type
+        # Should also remove nargs
+        frontend_kwargs["allowed_origins"]["type"] = json.loads
+        frontend_kwargs["allowed_methods"]["type"] = json.loads
+        frontend_kwargs["allowed_headers"]["type"] = json.loads
+        del frontend_kwargs["allowed_origins"]["nargs"]
+        del frontend_kwargs["allowed_methods"]["nargs"]
+        del frontend_kwargs["allowed_headers"]["nargs"]
+
+        # Special case: Middleware needs to append action
+        frontend_kwargs["middleware"]["action"] = "append"
+        frontend_kwargs["middleware"]["type"] = str
+        if "nargs" in frontend_kwargs["middleware"]:
+            del frontend_kwargs["middleware"]["nargs"]
+        frontend_kwargs["middleware"]["default"] = []
+
+        # Special case: disable_access_log_for_endpoints is a single
+        # comma-separated string, not a list
+        if "nargs" in frontend_kwargs["disable_access_log_for_endpoints"]:
+            del frontend_kwargs["disable_access_log_for_endpoints"]["nargs"]
+
+        return frontend_kwargs
+
+
+def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
+    """Create the CLI argument parser used by the OpenAI API server.
+
+    We rely on the helper methods of `FrontendArgs` and `AsyncEngineArgs` to
+    register all arguments instead of manually enumerating them here. This
+    avoids code duplication and keeps the argument definitions in one place.
+    """
+    parser.add_argument(
+        "model_tag",
+        type=str,
+        nargs="?",
+        help="The model tag to serve (optional if specified in config)",
+    )
+    parser.add_argument(
+        "--headless",
+        action="store_true",
+        default=False,
+        help="Run in headless mode. See multi-node data parallel "
+        "documentation for more details.",
+    )
+    parser.add_argument(
+        "--api-server-count",
+        "-asc",
+        type=int,
+        default=None,
+        help="How many API server processes to run. "
+        "Defaults to data_parallel_size if not specified.",
+    )
+    parser.add_argument(
+        "--config",
+        help="Read CLI options from a config file. "
+        "Must be a YAML with the following options: "
+        "https://docs.vllm.ai/en/latest/configuration/serve_args.html",
+    )
+    parser.add_argument(
+        "--grpc",
+        action="store_true",
+        default=False,
+        help="Launch a gRPC server instead of the HTTP OpenAI-compatible "
+        "server. Requires: pip install vllm[grpc].",
+    )
+    parser = FrontendArgs.add_cli_args(parser)
+    parser = AsyncEngineArgs.add_cli_args(parser)
+
+    return parser
+
+
+def validate_parsed_serve_args(args: argparse.Namespace):
+    """Quick checks for model serve args that raise prior to loading."""
+    if hasattr(args, "subparser") and args.subparser != "serve":
+        return
+
+    # Ensure that the chat template is valid; raises if it likely isn't
+    validate_chat_template(args.chat_template)
+
+    # Enable auto tool needs a tool call parser to be valid
+    if args.enable_auto_tool_choice and not args.tool_call_parser:
+        raise TypeError("Error: --enable-auto-tool-choice requires --tool-call-parser")
+    if args.enable_log_outputs and not args.enable_log_requests:
+        raise TypeError("Error: --enable-log-outputs requires --enable-log-requests")
+
+    if getattr(args, "enable_per_request_metrics", False) and getattr(
+        args, "disable_log_stats", False
+    ):
+        raise ValueError(
+            "Error: --enable-per-request-metrics requires engine statistics "
+            "logging; remove --disable-log-stats to enable per-request metrics."
+        )
+
+    if args.data_parallel_multi_port_external_lb:
+        from vllm.entrypoints.openai.dp_supervisor import (
+            validate_multi_port_external_lb_args,
+        )
+
+        validate_multi_port_external_lb_args(args)
+
+
+def create_parser_for_docs() -> FlexibleArgumentParser:
+    parser_for_docs = FlexibleArgumentParser(
+        prog="-m vllm.entrypoints.openai.api_server"
+    )
+    return make_arg_parser(parser_for_docs)

vllm/entrypoints/openai/completion/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

vllm/entrypoints/openai/completion/api_router.py

@@ -0,0 +1,70 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+
+from http import HTTPStatus
+
+from fastapi import APIRouter, Depends, FastAPI, Request
+from fastapi.responses import JSONResponse, StreamingResponse
+
+from vllm.entrypoints.openai.completion.protocol import (
+    CompletionRequest,
+    CompletionResponse,
+)
+from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
+from vllm.entrypoints.serve.utils.api_utils import (
+    load_aware_call,
+    validate_json_request,
+    with_cancellation,
+)
+from vllm.entrypoints.serve.utils.orca_metrics import metrics_header
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+router = APIRouter()
+ENDPOINT_LOAD_METRICS_FORMAT_HEADER_LABEL = "endpoint-load-metrics-format"
+
+
+def completion(request: Request) -> OpenAIServingCompletion | None:
+    return request.app.state.openai_serving_completion
+
+
+@router.post(
+    "/v1/completions",
+    dependencies=[Depends(validate_json_request)],
+    responses={
+        HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
+        HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
+        HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
+        HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
+    },
+)
+@with_cancellation
+@load_aware_call
+async def create_completion(request: CompletionRequest, raw_request: Request):
+    metrics_header_format = raw_request.headers.get(
+        ENDPOINT_LOAD_METRICS_FORMAT_HEADER_LABEL, ""
+    )
+    handler = completion(raw_request)
+    if handler is None:
+        raise NotImplementedError("The model does not support Completions API")
+
+    generator = await handler.create_completion(request, raw_request)
+
+    if isinstance(generator, ErrorResponse):
+        return JSONResponse(
+            content=generator.model_dump(), status_code=generator.error.code
+        )
+    elif isinstance(generator, CompletionResponse):
+        return JSONResponse(
+            content=generator.model_dump(),
+            headers=metrics_header(metrics_header_format),
+        )
+
+    return StreamingResponse(content=generator, media_type="text/event-stream")
+
+
+def attach_router(app: FastAPI):
+    app.include_router(router)

vllm/entrypoints/openai/completion/protocol.py

@@ -0,0 +1,643 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+# Adapted from
+# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
+import json
+import time
+from typing import Annotated, Any, Literal
+
+from pydantic import Field, model_validator
+
+import vllm.envs as envs
+from vllm.config import ModelConfig
+from vllm.config.utils import replace
+from vllm.entrypoints.openai.engine.protocol import (
+    AnyResponseFormat,
+    LegacyStructuralTagResponseFormat,
+    OpenAIBaseModel,
+    PerRequestTimingMetrics,
+    StreamOptions,
+    StructuralTagResponseFormat,
+    UsageInfo,
+    validate_structural_tag_response_format,
+    validate_structured_outputs_structural_tag,
+)
+from vllm.exceptions import VLLMValidationError
+from vllm.logger import init_logger
+from vllm.logprobs import Logprob
+from vllm.renderers import TokenizeParams
+from vllm.sampling_params import (
+    BeamSearchParams,
+    RepetitionDetectionParams,
+    RequestOutputKind,
+    SamplingParams,
+    StructuredOutputsParams,
+    ThinkingTokenBudget,
+)
+from vllm.utils import random_uuid
+from vllm.utils.collection_utils import is_list_of
+
+logger = init_logger(__name__)
+
+
+_INT64_MIN = -(2**63)
+_INT64_MAX = 2**63 - 1
+
+
+class CompletionRequest(OpenAIBaseModel):
+    # Ordered by official OpenAI API documentation
+    # https://platform.openai.com/docs/api-reference/completions/create
+    model: str | None = None
+    prompt: (
+        list[Annotated[int, Field(ge=0)]]
+        | list[list[Annotated[int, Field(ge=0)]]]
+        | str
+        | list[str]
+        | None
+    ) = None
+    echo: bool | None = False
+    frequency_penalty: float | None = 0.0
+    logit_bias: dict[str, float] | None = None
+    logprobs: int | None = None
+    max_tokens: int | None = 16
+    n: int = 1
+    presence_penalty: float | None = 0.0
+    seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
+    stop: str | list[str] | None = []
+    stream: bool | None = False
+    stream_options: StreamOptions | None = None
+    suffix: str | None = None
+    temperature: float | None = None
+    top_p: float | None = None
+    user: str | None = None
+
+    # --8<-- [start:completion-sampling-params]
+    use_beam_search: bool = False
+    top_k: int | None = None
+    min_p: float | None = None
+    repetition_penalty: float | None = None
+    length_penalty: float = 1.0
+    stop_token_ids: list[int] | None = []
+    include_stop_str_in_output: bool = False
+    ignore_eos: bool = False
+    min_tokens: int = 0
+    skip_special_tokens: bool = True
+    spaces_between_special_tokens: bool = True
+    truncate_prompt_tokens: Annotated[int, Field(ge=-1, le=_INT64_MAX)] | None = None
+    truncation_side: Literal["left", "right"] | None = Field(
+        default=None,
+        description=(
+            "Which side to truncate from when truncate_prompt_tokens is active. "
+            "'right' keeps the first N tokens. "
+            "'left' keeps the last N tokens."
+        ),
+    )
+    allowed_token_ids: list[int] | None = None
+    prompt_logprobs: int | None = None
+    bad_words: list[str] = Field(default_factory=list)
+    # --8<-- [end:completion-sampling-params]
+
+    # --8<-- [start:completion-extra-params]
+    prompt_embeds: bytes | list[bytes] | None = None
+    add_special_tokens: bool = Field(
+        default=True,
+        description=(
+            "If true (the default), special tokens (e.g. BOS) will be added to "
+            "the prompt."
+        ),
+    )
+    response_format: AnyResponseFormat | None = Field(
+        default=None,
+        description=(
+            "Similar to chat completion, this parameter specifies the format "
+            "of output. Only {'type': 'json_object'}, {'type': 'json_schema'}"
+            ", {'type': 'structural_tag'}, or {'type': 'text' } is supported."
+        ),
+    )
+    structured_outputs: StructuredOutputsParams | None = Field(
+        default=None,
+        description="Additional kwargs for structured outputs",
+    )
+    priority: int = Field(
+        default=0,
+        ge=_INT64_MIN,
+        le=_INT64_MAX,
+        description=(
+            "The priority of the request (lower means earlier handling; "
+            "default: 0). Any priority other than 0 will raise an error "
+            "if the served model does not use priority scheduling."
+        ),
+    )
+    request_id: str = Field(
+        default_factory=random_uuid,
+        description=(
+            "The request_id related to this request. If the caller does "
+            "not set it, a random_uuid will be generated. This id is used "
+            "through out the inference process and return in response."
+        ),
+    )
+
+    return_tokens_as_token_ids: bool | None = Field(
+        default=None,
+        description=(
+            "If specified with 'logprobs', tokens are represented "
+            " as strings of the form 'token_id:{token_id}' so that tokens "
+            "that are not JSON-encodable can be identified."
+        ),
+    )
+    return_token_ids: bool | None = Field(
+        default=None,
+        description=(
+            "If specified, the result will include token IDs alongside the "
+            "generated text. In streaming mode, prompt_token_ids is included "
+            "only in the first chunk, and token_ids contains the delta tokens "
+            "for each chunk. This is useful for debugging or when you "
+            "need to map generated text back to input tokens."
+        ),
+    )
+    return_token_offsets: bool | None = Field(
+        default=False,
+        description=(
+            "If true, return char-level (start, end) offsets for each "
+            "token relative to the tokenized source string in the "
+            "`token_offsets` field of the rendered response. Only "
+            "supported on the `/v1/completions/render` and "
+            "`/v1/chat/completions/render` endpoints; ignored on regular "
+            "generation endpoints. Honored only for Fast (Rust-backed) "
+            "tokenizers; otherwise `token_offsets` is null. For chat "
+            "requests, offsets are relative to the templated prompt "
+            "string (after applying the chat template). Multimodal "
+            "inputs and pre-tokenized inputs always yield null."
+        ),
+    )
+
+    cache_salt: str | None = Field(
+        default=None,
+        description=(
+            "If specified, the prefix cache will be salted with the provided "
+            "string to prevent an attacker to guess prompts in multi-user "
+            "environments. The salt should be random, protected from "
+            "access by 3rd parties, and long enough to be "
+            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
+            "to 256 bit)."
+        ),
+    )
+
+    kv_transfer_params: dict[str, Any] | None = Field(
+        default=None,
+        description="KVTransfer parameters used for disaggregated serving.",
+    )
+
+    ec_transfer_params: dict[str, Any] | None = Field(
+        default=None,
+        description=(
+            "ECTransfer parameters used for encoder-cache disaggregated serving."
+        ),
+    )
+
+    vllm_xargs: dict[str, str | int | float] | None = Field(
+        default=None,
+        description=(
+            "Additional request parameters with string or "
+            "numeric values, used by custom extensions."
+        ),
+    )
+
+    repetition_detection: RepetitionDetectionParams | None = Field(
+        default=None,
+        description="Parameters for detecting repetitive N-gram patterns "
+        "in output tokens. If such repetition is detected, generation will "
+        "be ended early. LLMs can sometimes generate repetitive, unhelpful "
+        "token patterns, stopping only when they hit the maximum output length "
+        "(e.g. 'abcdabcdabcd...' or '\\emoji \\emoji \\emoji ...'). This feature "
+        "can detect such behavior and terminate early, saving time and tokens.",
+    )
+
+    thinking_token_budget: ThinkingTokenBudget = Field(
+        default=None,
+        description=(
+            "Maximum number of tokens allowed for thinking operations "
+            "(reasoning models). Non-negative integer sets the limit; "
+            "-1 means unlimited (treated as unset)."
+        ),
+    )
+
+    # --8<-- [end:completion-extra-params]
+
+    def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
+        return TokenizeParams(
+            max_total_tokens=model_config.max_model_len,
+            max_output_tokens=self.max_tokens or 0,
+            truncate_prompt_tokens=self.truncate_prompt_tokens,
+            truncation_side=self.truncation_side,
+            add_special_tokens=self.add_special_tokens,
+            needs_detokenization=bool(self.echo and not self.return_token_ids),
+            max_total_tokens_param="max_model_len",
+            max_output_tokens_param="max_tokens",
+            return_token_offsets=bool(self.return_token_offsets),
+        )
+
+    # Default sampling parameters for completion requests
+    _DEFAULT_SAMPLING_PARAMS: dict = {
+        "repetition_penalty": 1.0,
+        "temperature": 1.0,
+        "top_p": 1.0,
+        "top_k": 0,
+        "min_p": 0.0,
+    }
+
+    def to_beam_search_params(
+        self,
+        max_tokens: int,
+        default_sampling_params: dict | None = None,
+    ) -> BeamSearchParams:
+        if default_sampling_params is None:
+            default_sampling_params = {}
+        n = self.n if self.n is not None else 1
+
+        if (temperature := self.temperature) is None:
+            temperature = default_sampling_params.get("temperature", 1.0)
+
+        return BeamSearchParams(
+            beam_width=n,
+            max_tokens=max_tokens,
+            ignore_eos=self.ignore_eos,
+            temperature=temperature,
+            length_penalty=self.length_penalty,
+            include_stop_str_in_output=self.include_stop_str_in_output,
+        )
+
+    def to_sampling_params(
+        self,
+        max_tokens: int,
+        default_sampling_params: dict | None = None,
+    ) -> SamplingParams:
+        if default_sampling_params is None:
+            default_sampling_params = {}
+
+        # Default parameters
+        if (repetition_penalty := self.repetition_penalty) is None:
+            repetition_penalty = default_sampling_params.get(
+                "repetition_penalty",
+                self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
+            )
+        if (temperature := self.temperature) is None:
+            temperature = default_sampling_params.get(
+                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
+            )
+        if (top_p := self.top_p) is None:
+            top_p = default_sampling_params.get(
+                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
+            )
+        if (top_k := self.top_k) is None:
+            top_k = default_sampling_params.get(
+                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
+            )
+        if (min_p := self.min_p) is None:
+            min_p = default_sampling_params.get(
+                "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
+            )
+
+        # Merge server-default stop_token_ids (e.g., model-specific tokens
+        # like </call> for gpt-oss) with any request-specified ones
+        stop_token_ids = self.stop_token_ids
+        default_stop_ids = default_sampling_params.get("stop_token_ids")
+        if default_stop_ids:
+            if not stop_token_ids:
+                stop_token_ids = list(default_stop_ids)
+            else:
+                stop_token_ids = list(
+                    dict.fromkeys([*stop_token_ids, *default_stop_ids])
+                )
+
+        prompt_logprobs = self.prompt_logprobs
+        if prompt_logprobs is None and self.echo:
+            prompt_logprobs = self.logprobs
+
+        echo_without_generation = self.echo and self.max_tokens == 0
+
+        response_format = self.response_format
+        if response_format is not None:
+            structured_outputs_kwargs = dict[str, Any]()
+
+            # Set structured output params for response format
+            if response_format.type == "json_object":
+                structured_outputs_kwargs["json_object"] = True
+            elif response_format.type == "json_schema":
+                json_schema = response_format.json_schema
+                assert json_schema is not None
+                structured_outputs_kwargs["json"] = json_schema.json_schema
+            elif response_format.type == "structural_tag":
+                structural_tag = response_format
+                assert isinstance(
+                    structural_tag,
+                    (
+                        LegacyStructuralTagResponseFormat,
+                        StructuralTagResponseFormat,
+                    ),
+                )
+                s_tag_obj = structural_tag.model_dump(by_alias=True)
+                structured_outputs_kwargs["structural_tag"] = json.dumps(s_tag_obj)
+
+            # If structured outputs wasn't already enabled,
+            # we must enable it for these features to work
+            if len(structured_outputs_kwargs) > 0:
+                self.structured_outputs = (
+                    StructuredOutputsParams(**structured_outputs_kwargs)
+                    if self.structured_outputs is None
+                    else replace(self.structured_outputs, **structured_outputs_kwargs)
+                )
+
+        extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
+        if self.kv_transfer_params:
+            # Pass in kv_transfer_params via extra_args
+            extra_args["kv_transfer_params"] = self.kv_transfer_params
+        if self.ec_transfer_params:
+            # Pass in ec_transfer_params via extra_args
+            extra_args["ec_transfer_params"] = self.ec_transfer_params
+        return SamplingParams.from_optional(
+            n=self.n,
+            presence_penalty=self.presence_penalty,
+            frequency_penalty=self.frequency_penalty,
+            repetition_penalty=repetition_penalty,
+            temperature=temperature,
+            top_p=top_p,
+            top_k=top_k,
+            min_p=min_p,
+            seed=self.seed,
+            stop=self.stop,
+            stop_token_ids=stop_token_ids,
+            logprobs=self.logprobs,
+            ignore_eos=self.ignore_eos,
+            max_tokens=max_tokens if not echo_without_generation else 1,
+            min_tokens=self.min_tokens,
+            prompt_logprobs=prompt_logprobs,
+            skip_special_tokens=self.skip_special_tokens,
+            spaces_between_special_tokens=self.spaces_between_special_tokens,
+            include_stop_str_in_output=self.include_stop_str_in_output,
+            output_kind=RequestOutputKind.DELTA
+            if self.stream
+            else RequestOutputKind.FINAL_ONLY,
+            structured_outputs=self.structured_outputs,
+            logit_bias=self.logit_bias,
+            allowed_token_ids=self.allowed_token_ids,
+            bad_words=self.bad_words,
+            extra_args=extra_args or None,
+            skip_clone=True,  # Created fresh per request, safe to skip clone
+            repetition_detection=self.repetition_detection,
+            thinking_token_budget=self.thinking_token_budget,
+        )
+
+    @model_validator(mode="before")
+    @classmethod
+    def normalize_null_max_tokens(cls, data):
+        if isinstance(data, dict) and data.get("max_tokens") is None:
+            data = data.copy()
+            data["max_tokens"] = cls.model_fields["max_tokens"].default
+        return data
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_response_format(cls, data):
+        response_format = data.get("response_format")
+        if response_format is None:
+            return data
+
+        rf_type = (
+            response_format.get("type")
+            if isinstance(response_format, dict)
+            else getattr(response_format, "type", None)
+        )
+
+        if rf_type == "json_schema":
+            json_schema = (
+                response_format.get("json_schema")
+                if isinstance(response_format, dict)
+                else getattr(response_format, "json_schema", None)
+            )
+            if json_schema is None:
+                raise VLLMValidationError(
+                    "When response_format type is 'json_schema', the "
+                    "'json_schema' field must be provided.",
+                    parameter="response_format",
+                )
+
+        if rf_type == "structural_tag":
+            validate_structural_tag_response_format(response_format)
+
+        return data
+
+    @model_validator(mode="before")
+    @classmethod
+    def check_structured_outputs_count(cls, data):
+        if data.get("structured_outputs", None) is None:
+            return data
+
+        structured_outputs_kwargs = data["structured_outputs"]
+        # structured_outputs may arrive as a dict (from JSON/raw kwargs) or
+        # as a StructuredOutputsParams dataclass instance.
+        is_dataclass = isinstance(structured_outputs_kwargs, StructuredOutputsParams)
+        count = sum(
+            (
+                getattr(structured_outputs_kwargs, k, None)
+                if is_dataclass
+                else structured_outputs_kwargs.get(k)
+            )
+            is not None
+            for k in ("json", "regex", "choice")
+        )
+        if count > 1:
+            raise VLLMValidationError(
+                "You can only use one kind of constraints for structured "
+                "outputs ('json', 'regex' or 'choice').",
+                parameter="structured_outputs",
+            )
+        validate_structured_outputs_structural_tag(structured_outputs_kwargs)
+        return data
+
+    @model_validator(mode="before")
+    @classmethod
+    def check_logprobs(cls, data):
+        if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
+            if data.get("stream") and (prompt_logprobs > 0 or prompt_logprobs == -1):
+                raise VLLMValidationError(
+                    "`prompt_logprobs` are not available when `stream=True`.",
+                    parameter="prompt_logprobs",
+                )
+
+            if prompt_logprobs < 0 and prompt_logprobs != -1:
+                raise VLLMValidationError(
+                    "`prompt_logprobs` must be a positive value or -1.",
+                    parameter="prompt_logprobs",
+                    value=prompt_logprobs,
+                )
+        if (logprobs := data.get("logprobs")) is not None and logprobs < 0:
+            raise VLLMValidationError(
+                "`logprobs` must be a positive value.",
+                parameter="logprobs",
+                value=logprobs,
+            )
+
+        return data
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_stream_options(cls, data):
+        if data.get("stream_options") and not data.get("stream"):
+            raise VLLMValidationError(
+                "Stream options can only be defined when `stream=True`.",
+                parameter="stream_options",
+            )
+
+        return data
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_prompt_and_prompt_embeds(cls, data):
+        prompt = data.get("prompt")
+        prompt_embeds = data.get("prompt_embeds")
+
+        prompt_is_empty = prompt is None or (isinstance(prompt, str) and prompt == "")
+        embeds_is_empty = prompt_embeds is None or (
+            isinstance(prompt_embeds, list) and len(prompt_embeds) == 0
+        )
+
+        if prompt_is_empty and embeds_is_empty:
+            raise VLLMValidationError(
+                "Either prompt or prompt_embeds must be provided and non-empty.",
+                parameter="prompt",
+            )
+
+        return data
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_prompt_list_length(cls, data):
+        max_prompts = envs.VLLM_MAX_COMPLETION_PROMPTS
+
+        prompt = data.get("prompt")
+        if (
+            isinstance(prompt, list)
+            and len(prompt) > 0
+            and not is_list_of(prompt, int)
+            and len(prompt) > max_prompts
+        ):
+            raise VLLMValidationError(
+                f"prompt list length {len(prompt)} exceeds the maximum "
+                f"allowed count of {max_prompts}. To increase this "
+                "limit, set the VLLM_MAX_COMPLETION_PROMPTS "
+                "environment variable.",
+                parameter="prompt",
+            )
+
+        prompt_embeds = data.get("prompt_embeds")
+        if isinstance(prompt_embeds, list) and len(prompt_embeds) > max_prompts:
+            raise VLLMValidationError(
+                f"prompt_embeds list length {len(prompt_embeds)} exceeds "
+                f"the maximum allowed count of {max_prompts}. To increase "
+                "this limit, set the VLLM_MAX_COMPLETION_PROMPTS "
+                "environment variable.",
+                parameter="prompt_embeds",
+            )
+
+        return data
+
+    @model_validator(mode="before")
+    @classmethod
+    def check_cache_salt_support(cls, data):
+        if data.get("cache_salt") is not None and (
+            not isinstance(data["cache_salt"], str) or not data["cache_salt"]
+        ):
+            raise VLLMValidationError(
+                "Parameter 'cache_salt' must be a non-empty string if provided.",
+                parameter="cache_salt",
+            )
+        return data
+
+
+class CompletionLogProbs(OpenAIBaseModel):
+    text_offset: list[int] = Field(default_factory=list)
+    token_logprobs: list[float | None] = Field(default_factory=list)
+    tokens: list[str] = Field(default_factory=list)
+    top_logprobs: list[dict[str, float] | None] = Field(default_factory=list)
+
+
+class CompletionResponseChoice(OpenAIBaseModel):
+    index: int
+    text: str
+    logprobs: CompletionLogProbs | None = None
+    finish_reason: str | None = None
+    stop_reason: int | str | None = Field(
+        default=None,
+        description=(
+            "The stop string or token id that caused the completion "
+            "to stop, None if the completion finished for some other reason "
+            "including encountering the EOS token"
+        ),
+    )
+    token_ids: list[int] | None = None  # For response
+    prompt_logprobs: list[dict[int, Logprob] | None] | None = None
+    prompt_token_ids: list[int] | None = None  # For prompt
+    # Per-token expert routing decisions, base64-encoded ``.npy`` bytes
+    # (numpy serialization). Shape after decode:
+    #   (num_tokens - 1, num_layers, num_experts_per_tok)  dtype uint8/uint16
+    # ``num_tokens - 1`` because the last sampled token has not been
+    # forwarded yet and therefore has no routing data.
+    # Decode:
+    #   np.load(io.BytesIO(base64.b64decode(s)))
+    # ``None`` if (a) the request was aborted before any forward pass,
+    # or (b) ``enable_return_routed_experts`` is off server-side.
+    routed_experts: str | None = None
+
+
+class CompletionResponse(OpenAIBaseModel):
+    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
+    object: Literal["text_completion"] = "text_completion"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    model: str
+    choices: list[CompletionResponseChoice]
+    service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
+    system_fingerprint: str | None = None
+    usage: UsageInfo
+
+    # vLLM-specific fields that are not in OpenAI spec
+    kv_transfer_params: dict[str, Any] | None = Field(
+        default=None, description="KVTransfer parameters."
+    )
+    ec_transfer_params: dict[str, Any] | None = Field(
+        default=None, description="ECTransfer parameters."
+    )
+    metrics: PerRequestTimingMetrics | None = None
+
+
+class CompletionResponseStreamChoice(OpenAIBaseModel):
+    index: int
+    text: str
+    logprobs: CompletionLogProbs | None = None
+    finish_reason: str | None = None
+    stop_reason: int | str | None = Field(
+        default=None,
+        description=(
+            "The stop string or token id that caused the completion "
+            "to stop, None if the completion finished for some other reason "
+            "including encountering the EOS token"
+        ),
+    )
+    # not part of the OpenAI spec but for tracing the tokens
+    # prompt tokens is put into choice to align with CompletionResponseChoice
+    prompt_token_ids: list[int] | None = None
+    token_ids: list[int] | None = None
+
+
+class CompletionStreamResponse(OpenAIBaseModel):
+    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
+    object: str = "text_completion"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    model: str
+    choices: list[CompletionResponseStreamChoice]
+    usage: UsageInfo | None = Field(default=None)
+    # Set only on the final chunk of a stream to mirror non-streaming responses
+    # without the per-chunk serialization overhead.
+    system_fingerprint: str | None = None
+    metrics: PerRequestTimingMetrics | None = None

vllm/entrypoints/openai/completion/serving.py

@@ -0,0 +1,735 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import asyncio
+import io
+import time
+from collections.abc import AsyncGenerator, AsyncIterator
+from collections.abc import Sequence as GenericSequence
+from typing import cast
+
+import numpy as np
+import pybase64 as base64
+from fastapi import Request
+
+from vllm.engine.protocol import EngineClient
+from vllm.entrypoints.generate.base.serving import (
+    GenerateBaseServing,
+    GenerationError,
+    build_per_request_timing_metrics,
+    clamp_prompt_logprobs,
+    format_token_id_placeholder,
+)
+from vllm.entrypoints.openai.completion.protocol import (
+    CompletionLogProbs,
+    CompletionRequest,
+    CompletionResponse,
+    CompletionResponseChoice,
+    CompletionResponseStreamChoice,
+    CompletionStreamResponse,
+)
+from vllm.entrypoints.openai.engine.protocol import (
+    ErrorResponse,
+    PerRequestTimingMetrics,
+    PromptTokenUsageInfo,
+    RequestResponseMetadata,
+    UsageInfo,
+)
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
+from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
+from vllm.entrypoints.serve.utils.request_logger import RequestLogger
+from vllm.exceptions import VLLMValidationError
+from vllm.inputs import EngineInput
+from vllm.logger import init_logger
+from vllm.logprobs import Logprob
+from vllm.outputs import RequestOutput
+from vllm.renderers.online_renderer import OnlineRenderer
+from vllm.sampling_params import BeamSearchParams, SamplingParams
+from vllm.tokenizers import TokenizerLike
+from vllm.utils.async_utils import merge_async_iterators
+from vllm.utils.collection_utils import as_list
+
+logger = init_logger(__name__)
+
+
+class OpenAIServingCompletion(GenerateBaseServing):
+    def __init__(
+        self,
+        engine_client: EngineClient,
+        models: OpenAIServingModels,
+        *,
+        online_renderer: "OnlineRenderer",
+        request_logger: RequestLogger | None,
+        return_tokens_as_token_ids: bool = False,
+        enable_prompt_tokens_details: bool = False,
+        enable_force_include_usage: bool = False,
+        enable_per_request_metrics: bool = False,
+    ):
+        super().__init__(
+            engine_client=engine_client,
+            models=models,
+            request_logger=request_logger,
+            return_tokens_as_token_ids=return_tokens_as_token_ids,
+        )
+
+        self.online_renderer = online_renderer
+        self.enable_prompt_tokens_details = enable_prompt_tokens_details
+        self.enable_force_include_usage = enable_force_include_usage
+        self.enable_per_request_metrics = enable_per_request_metrics
+
+        self.default_sampling_params = self.model_config.get_diff_sampling_param()
+        mc = self.model_config
+        self.override_max_tokens = (
+            self.default_sampling_params.get("max_tokens")
+            if mc.generation_config not in ("auto", "vllm")
+            else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
+        )
+
+    async def render_completion_request(
+        self,
+        request: CompletionRequest,
+    ) -> list[EngineInput] | ErrorResponse:
+        """
+        Validate the model and preprocess a completion request.
+
+        Delegates preprocessing logic to OnlineRenderer, adding the
+        engine-aware checks (LoRA model validation, engine health).
+
+        Returns:
+            A list of engine_inputs on success, or an ErrorResponse on failure.
+        """
+        error_check_ret = await self._check_model(request)
+        if error_check_ret is not None:
+            return error_check_ret
+
+        # If the engine is dead, raise the engine's DEAD_ERROR.
+        # This is required for the streaming case, where we return a
+        # success status before we actually start generating text :).
+        if self.engine_client.errored:
+            raise self.engine_client.dead_error
+
+        return await self.online_renderer.render_completion(request)
+
+    async def create_completion(
+        self,
+        request: CompletionRequest,
+        raw_request: Request | None = None,
+    ) -> AsyncGenerator[str, None] | CompletionResponse | ErrorResponse:
+        """Completion API similar to OpenAI's API.
+
+        See https://platform.openai.com/docs/api-reference/completions/create
+        for the API specification. This API mimics the OpenAI Completion API.
+
+        NOTE: Currently we do not support the following feature:
+            - suffix (the language models we currently support do not support
+            suffix)
+        """
+        return await self._with_kv_transfer_rejection_cleanup(
+            self._create_completion(request, raw_request), request, raw_request
+        )
+
+    async def _create_completion(
+        self,
+        request: CompletionRequest,
+        raw_request: Request | None = None,
+    ) -> AsyncGenerator[str, None] | CompletionResponse | ErrorResponse:
+        if request.stream and request.use_beam_search:
+            return self.create_error_response(
+                "Streaming is not currently supported with beam search"
+            )
+
+        result = await self.render_completion_request(request)
+        if isinstance(result, ErrorResponse):
+            return result
+
+        engine_inputs = result
+
+        request_id = f"cmpl-{self._base_request_id(raw_request, request.request_id)}"
+        created_time = int(time.time())
+
+        request_metadata = RequestResponseMetadata(request_id=request_id)
+        if raw_request:
+            raw_request.state.request_metadata = request_metadata
+
+        lora_request = self._maybe_get_adapters(request)
+
+        # Extract data_parallel_rank from header (router can inject it)
+        data_parallel_rank = self._get_data_parallel_rank(raw_request)
+
+        # Schedule the request and get the result generator.
+        max_model_len = self.model_config.max_model_len
+        generators: list[AsyncGenerator[RequestOutput, None]] = []
+        for i, engine_input in enumerate(engine_inputs):
+            max_tokens = get_max_tokens(
+                max_model_len,
+                request.max_tokens,
+                self._extract_prompt_len(engine_input),
+                self.default_sampling_params,
+                self.override_max_tokens,
+                truncate_prompt_tokens=request.truncate_prompt_tokens,
+            )
+
+            sampling_params: SamplingParams | BeamSearchParams
+            if request.use_beam_search:
+                sampling_params = request.to_beam_search_params(
+                    max_tokens, self.default_sampling_params
+                )
+            else:
+                sampling_params = request.to_sampling_params(
+                    max_tokens,
+                    self.default_sampling_params,
+                )
+
+            request_id_item = f"{request_id}-{i}"
+
+            self._log_inputs(
+                request_id_item,
+                engine_input,
+                params=sampling_params,
+                lora_request=lora_request,
+            )
+
+            trace_headers = (
+                None
+                if raw_request is None
+                else await self._get_trace_headers(raw_request.headers)
+            )
+
+            if isinstance(sampling_params, BeamSearchParams):
+                generator = self.beam_search(
+                    prompt=engine_input,
+                    request_id=request_id,
+                    params=sampling_params,
+                    lora_request=lora_request,
+                    trace_headers=trace_headers,
+                )
+            else:
+                generator = self.engine_client.generate(
+                    engine_input,
+                    sampling_params,
+                    request_id_item,
+                    lora_request=lora_request,
+                    trace_headers=trace_headers,
+                    priority=request.priority,
+                    data_parallel_rank=data_parallel_rank,
+                )
+
+            generators.append(generator)
+
+        result_generator = merge_async_iterators(*generators)
+
+        model_name = self.models.model_name(lora_request)
+        num_prompts = len(engine_inputs)
+
+        # Streaming response
+        tokenizer = self.renderer.tokenizer
+
+        if request.stream:
+            return self.completion_stream_generator(
+                request,
+                engine_inputs,
+                result_generator,
+                request_id,
+                created_time,
+                model_name,
+                num_prompts=num_prompts,
+                tokenizer=tokenizer,
+                request_metadata=request_metadata,
+            )
+
+        # Non-streaming response
+        final_res_batch: list[RequestOutput | None] = [None] * num_prompts
+        try:
+            async for i, res in result_generator:
+                final_res_batch[i] = res
+
+            for i, final_res in enumerate(final_res_batch):
+                assert final_res is not None
+
+                # The output should contain the input text
+                # We did not pass it into vLLM engine to avoid being redundant
+                # with the inputs token IDs
+                if final_res.prompt is None:
+                    final_res.prompt = self._extract_prompt_text(engine_inputs[i])
+
+            final_res_batch_checked = cast(list[RequestOutput], final_res_batch)
+
+            response = self.request_output_to_completion_response(
+                final_res_batch_checked,
+                request,
+                request_id,
+                created_time,
+                model_name,
+                tokenizer,
+                request_metadata,
+            )
+        except asyncio.CancelledError:
+            return self.create_error_response("Client disconnected")
+
+        # When user requests streaming but we don't stream, we still need to
+        # return a streaming response with a single event.
+        if request.stream:
+            response_json = response.model_dump_json()
+
+            async def fake_stream_generator() -> AsyncGenerator[str, None]:
+                yield f"data: {response_json}\n\n"
+                yield "data: [DONE]\n\n"
+
+            return fake_stream_generator()
+
+        return response
+
+    async def completion_stream_generator(
+        self,
+        request: CompletionRequest,
+        engine_inputs: list[EngineInput],
+        result_generator: AsyncIterator[tuple[int, RequestOutput]],
+        request_id: str,
+        created_time: int,
+        model_name: str,
+        num_prompts: int,
+        tokenizer: TokenizerLike | None,
+        request_metadata: RequestResponseMetadata,
+    ) -> AsyncGenerator[str, None]:
+        num_choices = 1 if request.n is None else request.n
+        previous_text_lens = [0] * num_choices * num_prompts
+        previous_num_tokens = [0] * num_choices * num_prompts
+        has_echoed = [False] * num_choices * num_prompts
+        num_prompt_tokens = [0] * num_prompts
+        num_cached_tokens = None
+        first_iteration = True
+
+        stream_options = request.stream_options
+        include_usage, include_continuous_usage = should_include_usage(
+            stream_options, self.enable_force_include_usage
+        )
+
+        last_res: RequestOutput | None = None
+        try:
+            async for prompt_idx, res in result_generator:
+                last_res = res
+                prompt_token_ids = res.prompt_token_ids
+                prompt_logprobs = res.prompt_logprobs
+
+                if first_iteration:
+                    num_cached_tokens = res.num_cached_tokens
+                    first_iteration = False
+
+                prompt_text = res.prompt
+                if prompt_text is None:
+                    engine_input = engine_inputs[prompt_idx]
+                    prompt_text = self._extract_prompt_text(engine_input)
+
+                # Prompt details are excluded from later streamed outputs
+                if prompt_token_ids is not None:
+                    num_prompt_tokens[prompt_idx] = len(prompt_token_ids)
+
+                delta_token_ids: GenericSequence[int]
+                out_logprobs: GenericSequence[dict[int, Logprob] | None] | None
+
+                for output in res.outputs:
+                    i = output.index + prompt_idx * num_choices
+
+                    # Useful when request.return_token_ids is True
+                    # Returning prompt token IDs shares the same logic
+                    # with the echo implementation.
+                    prompt_token_ids_to_return: list[int] | None = None
+
+                    assert request.max_tokens is not None
+                    if request.echo and not has_echoed[i]:
+                        assert prompt_token_ids is not None
+                        if request.return_token_ids:
+                            prompt_text = ""
+                        assert prompt_text is not None
+                        if request.max_tokens == 0:
+                            # only return the prompt
+                            delta_text = prompt_text
+                            delta_token_ids = prompt_token_ids
+                            out_logprobs = prompt_logprobs
+                        else:
+                            # echo the prompt and first token
+                            delta_text = prompt_text + output.text
+                            delta_token_ids = [
+                                *prompt_token_ids,
+                                *output.token_ids,
+                            ]
+                            out_logprobs = [
+                                *(prompt_logprobs or []),
+                                *(output.logprobs or []),
+                            ]
+                        prompt_token_ids_to_return = prompt_token_ids
+                        has_echoed[i] = True
+                    else:
+                        # return just the delta
+                        delta_text = output.text
+                        delta_token_ids = output.token_ids
+                        out_logprobs = output.logprobs
+
+                        # has_echoed[i] is reused here to indicate whether
+                        # we have already returned the prompt token IDs.
+                        if not has_echoed[i] and request.return_token_ids:
+                            prompt_token_ids_to_return = prompt_token_ids
+                            has_echoed[i] = True
+
+                        if (
+                            not delta_text
+                            and not delta_token_ids
+                            and not previous_num_tokens[i]
+                        ):
+                            # Chunked prefill case, don't return empty chunks
+                            continue
+
+                    if request.logprobs is not None:
+                        assert out_logprobs is not None, "Did not output logprobs"
+                        logprobs = self._create_completion_logprobs(
+                            token_ids=delta_token_ids,
+                            top_logprobs=out_logprobs,
+                            num_output_top_logprobs=request.logprobs,
+                            tokenizer=tokenizer,
+                            initial_text_offset=previous_text_lens[i],
+                            return_as_token_id=request.return_tokens_as_token_ids,
+                        )
+                    else:
+                        logprobs = None
+
+                    previous_text_lens[i] += len(output.text)
+                    previous_num_tokens[i] += len(output.token_ids)
+                    finish_reason = output.finish_reason
+                    stop_reason = output.stop_reason
+
+                    self._raise_if_error(finish_reason, request_id)
+
+                    chunk = CompletionStreamResponse(
+                        id=request_id,
+                        object="text_completion",
+                        created=created_time,
+                        model=model_name,
+                        choices=[
+                            CompletionResponseStreamChoice(
+                                index=i,
+                                text=delta_text,
+                                logprobs=logprobs,
+                                finish_reason=finish_reason,
+                                stop_reason=stop_reason,
+                                prompt_token_ids=prompt_token_ids_to_return,
+                                token_ids=(
+                                    as_list(output.token_ids)
+                                    if request.return_token_ids
+                                    else None
+                                ),
+                            )
+                        ],
+                    )
+                    # Stamp on terminal chunk only when no trailing usage chunk
+                    # will follow (that one is the true final message).
+                    if (
+                        not include_usage
+                        and self.system_fingerprint is not None
+                        and finish_reason is not None
+                    ):
+                        chunk.system_fingerprint = self.system_fingerprint
+                    if include_continuous_usage:
+                        prompt_tokens = num_prompt_tokens[prompt_idx]
+                        completion_tokens = previous_num_tokens[i]
+                        chunk.usage = UsageInfo(
+                            prompt_tokens=prompt_tokens,
+                            completion_tokens=completion_tokens,
+                            total_tokens=prompt_tokens + completion_tokens,
+                        )
+
+                    response_json = chunk.model_dump_json(exclude_unset=True)
+                    yield f"data: {response_json}\n\n"
+
+            total_prompt_tokens = sum(num_prompt_tokens)
+            total_completion_tokens = sum(previous_num_tokens)
+            final_usage_info = UsageInfo(
+                prompt_tokens=total_prompt_tokens,
+                completion_tokens=total_completion_tokens,
+                total_tokens=total_prompt_tokens + total_completion_tokens,
+            )
+
+            if self.enable_prompt_tokens_details and num_cached_tokens is not None:
+                final_usage_info.prompt_tokens_details = PromptTokenUsageInfo(
+                    cached_tokens=num_cached_tokens
+                )
+
+            if include_usage:
+                # In streaming, metrics ride on this final usage chunk, which is
+                # only emitted when usage reporting is enabled (i.e.
+                # ``stream_options.include_usage=true`` or
+                # ``--enable-force-include-usage``).
+                stream_per_request_metrics: PerRequestTimingMetrics | None = None
+                if (
+                    self.enable_per_request_metrics
+                    # See note in request_output_to_completion_response: suppress
+                    # when not attributable to one stream (multi-prompt or n>1).
+                    and num_prompts == 1
+                    and (request.n or 1) == 1
+                ):
+                    last_metrics = last_res.metrics if last_res is not None else None
+                    stream_per_request_metrics = build_per_request_timing_metrics(
+                        last_metrics, total_completion_tokens
+                    )
+
+                final_usage_chunk = CompletionStreamResponse(
+                    id=request_id,
+                    created=created_time,
+                    model=model_name,
+                    choices=[],
+                    usage=final_usage_info,
+                    system_fingerprint=self.system_fingerprint,
+                    metrics=stream_per_request_metrics,
+                )
+                final_usage_data = final_usage_chunk.model_dump_json(
+                    exclude_unset=False, exclude_none=True
+                )
+                yield f"data: {final_usage_data}\n\n"
+
+            # report to FastAPI middleware aggregate usage across all choices
+            request_metadata.final_usage_info = final_usage_info
+
+        except GenerationError as e:
+            yield f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n"
+        except Exception as e:
+            logger.exception("Error in completion stream generator.")
+            data = self.create_streaming_error_response(e)
+            yield f"data: {data}\n\n"
+        yield "data: [DONE]\n\n"
+
+    def request_output_to_completion_response(
+        self,
+        final_res_batch: list[RequestOutput],
+        request: CompletionRequest,
+        request_id: str,
+        created_time: int,
+        model_name: str,
+        tokenizer: TokenizerLike | None,
+        request_metadata: RequestResponseMetadata,
+    ) -> CompletionResponse:
+        choices: list[CompletionResponseChoice] = []
+        num_prompt_tokens = 0
+        num_generated_tokens = 0
+        kv_transfer_params = None
+        ec_transfer_params = None
+        last_final_res = None
+        for final_res in final_res_batch:
+            last_final_res = final_res
+            prompt_token_ids = final_res.prompt_token_ids
+            assert prompt_token_ids is not None
+            prompt_logprobs = clamp_prompt_logprobs(final_res.prompt_logprobs)
+            prompt_text = final_res.prompt
+
+            token_ids: GenericSequence[int]
+            out_logprobs: GenericSequence[dict[int, Logprob] | None] | None
+
+            for output in final_res.outputs:
+                self._raise_if_error(output.finish_reason, request_id)
+
+                assert request.max_tokens is not None
+                if request.echo:
+                    if request.return_token_ids:
+                        prompt_text = ""
+                    assert prompt_text is not None
+                    if request.max_tokens == 0:
+                        token_ids = prompt_token_ids
+                        out_logprobs = prompt_logprobs
+                        output_text = prompt_text
+                    else:
+                        token_ids = [*prompt_token_ids, *output.token_ids]
+
+                        if request.logprobs is None:
+                            out_logprobs = None
+                        else:
+                            assert prompt_logprobs is not None
+                            assert output.logprobs is not None
+                            out_logprobs = [
+                                *prompt_logprobs,
+                                *output.logprobs,
+                            ]
+
+                        output_text = prompt_text + output.text
+                else:
+                    token_ids = output.token_ids
+                    out_logprobs = output.logprobs
+                    output_text = output.text
+
+                if request.logprobs is not None:
+                    assert out_logprobs is not None, "Did not output logprobs"
+                    logprobs = self._create_completion_logprobs(
+                        token_ids=token_ids,
+                        top_logprobs=out_logprobs,
+                        tokenizer=tokenizer,
+                        num_output_top_logprobs=request.logprobs,
+                        return_as_token_id=request.return_tokens_as_token_ids,
+                    )
+                else:
+                    logprobs = None
+
+                # Encode routed_experts for transport. JSON can't carry raw
+                # bytes, so we write the ndarray as a ``.npy`` byte stream
+                # and base64-encode it. ``pybase64`` is ~3x faster than the
+                # stdlib ``base64`` on large payloads thanks to SIMD.
+                routed_experts_b64 = None
+                if output.routed_experts is not None:
+                    buf = io.BytesIO()
+                    np.save(buf, output.routed_experts)
+                    routed_experts_b64 = base64.b64encode(buf.getvalue()).decode(
+                        "ascii"
+                    )
+
+                choice_data = CompletionResponseChoice(
+                    index=len(choices),
+                    text=output_text,
+                    logprobs=logprobs,
+                    finish_reason=output.finish_reason,
+                    stop_reason=output.stop_reason,
+                    prompt_logprobs=final_res.prompt_logprobs,
+                    prompt_token_ids=(
+                        prompt_token_ids if request.return_token_ids else None
+                    ),
+                    token_ids=(
+                        as_list(output.token_ids) if request.return_token_ids else None
+                    ),
+                    routed_experts=routed_experts_b64,
+                )
+                choices.append(choice_data)
+
+                num_generated_tokens += len(output.token_ids)
+
+            num_prompt_tokens += len(prompt_token_ids)
+
+        usage = UsageInfo(
+            prompt_tokens=num_prompt_tokens,
+            completion_tokens=num_generated_tokens,
+            total_tokens=num_prompt_tokens + num_generated_tokens,
+        )
+
+        if (
+            self.enable_prompt_tokens_details
+            and last_final_res
+            and last_final_res.num_cached_tokens is not None
+        ):
+            usage.prompt_tokens_details = PromptTokenUsageInfo(
+                cached_tokens=last_final_res.num_cached_tokens
+            )
+
+        request_metadata.final_usage_info = usage
+
+        per_request_metrics: PerRequestTimingMetrics | None = None
+        if (
+            self.enable_per_request_metrics
+            # Metrics describe a single generation stream, so suppress them when
+            # they cannot be attributed to one: multiple prompts (timestamps
+            # span prompts) or n>1 (stats belong to one of the n sequences).
+            and len(final_res_batch) == 1
+            and (request.n or 1) == 1
+        ):
+            last_metrics = (
+                last_final_res.metrics if last_final_res is not None else None
+            )
+            per_request_metrics = build_per_request_timing_metrics(
+                last_metrics, num_generated_tokens
+            )
+
+        if final_res_batch:
+            kv_transfer_params = final_res_batch[0].kv_transfer_params
+            ec_transfer_params = final_res_batch[0].ec_transfer_params
+
+        return CompletionResponse(
+            id=request_id,
+            created=created_time,
+            model=model_name,
+            choices=choices,
+            usage=usage,
+            system_fingerprint=self.system_fingerprint,
+            kv_transfer_params=kv_transfer_params,
+            ec_transfer_params=ec_transfer_params,
+            metrics=per_request_metrics,
+        )
+
+    def _create_completion_logprobs(
+        self,
+        token_ids: GenericSequence[int],
+        top_logprobs: GenericSequence[dict[int, Logprob] | None],
+        num_output_top_logprobs: int,
+        tokenizer: TokenizerLike | None,
+        initial_text_offset: int = 0,
+        return_as_token_id: bool | None = None,
+    ) -> CompletionLogProbs:
+        """Create logprobs for OpenAI Completion API."""
+        out_text_offset: list[int] = []
+        out_token_logprobs: list[float | None] = []
+        out_tokens: list[str] = []
+        out_top_logprobs: list[dict[str, float] | None] = []
+
+        last_token_len = 0
+
+        should_return_as_token_id = (
+            return_as_token_id
+            if return_as_token_id is not None
+            else self.return_tokens_as_token_ids
+        )
+        for i, token_id in enumerate(token_ids):
+            step_top_logprobs = top_logprobs[i]
+            if step_top_logprobs is None:
+                if should_return_as_token_id:
+                    token = format_token_id_placeholder(token_id)
+                else:
+                    if tokenizer is None:
+                        raise VLLMValidationError(
+                            "Unable to get tokenizer because "
+                            "`skip_tokenizer_init=True`",
+                            parameter="skip_tokenizer_init",
+                            value=True,
+                        )
+
+                    token = tokenizer.decode(token_id)
+
+                out_tokens.append(token)
+                out_token_logprobs.append(None)
+                out_top_logprobs.append(None)
+            else:
+                step_token = step_top_logprobs[token_id]
+
+                token = self._get_decoded_token(
+                    step_token,
+                    token_id,
+                    tokenizer,
+                    return_as_token_id=should_return_as_token_id,
+                )
+                token_logprob = max(step_token.logprob, -9999.0)
+
+                out_tokens.append(token)
+                out_token_logprobs.append(token_logprob)
+
+                # makes sure to add the top num_output_top_logprobs + 1
+                # logprobs, as defined in the openai API
+                # (cf. https://github.com/openai/openai-openapi/blob/
+                # 893ba52242dbd5387a97b96444ee1c742cfce9bd/openapi.yaml#L7153)
+                out_top_logprobs.append(
+                    {
+                        # Convert float("-inf") to the
+                        # JSON-serializable float that OpenAI uses
+                        self._get_decoded_token(
+                            top_lp[1],
+                            top_lp[0],
+                            tokenizer,
+                            return_as_token_id=should_return_as_token_id,
+                        ): max(top_lp[1].logprob, -9999.0)
+                        for i, top_lp in enumerate(step_top_logprobs.items())
+                        if num_output_top_logprobs >= i
+                    }
+                )
+
+            if len(out_text_offset) == 0:
+                out_text_offset.append(initial_text_offset)
+            else:
+                out_text_offset.append(out_text_offset[-1] + last_token_len)
+            last_token_len = len(token)
+
+        return CompletionLogProbs(
+            text_offset=out_text_offset,
+            token_logprobs=out_token_logprobs,
+            tokens=out_tokens,
+            top_logprobs=out_top_logprobs,
+        )

vllm/entrypoints/openai/dp_supervisor.py

@@ -0,0 +1,556 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import contextlib
+import copy
+import multiprocessing
+import os
+import signal
+import time
+from functools import partial
+from http import HTTPStatus
+from multiprocessing.process import BaseProcess
+
+import aiohttp
+import psutil
+import uvicorn
+import uvloop
+from fastapi import FastAPI, Response
+
+import vllm.envs as envs
+from vllm.logger import init_logger
+from vllm.utils.system_utils import (
+    decorate_logs,
+    kill_process_tree,
+    set_process_title,
+)
+
+logger = init_logger(__name__)
+
+CHILD_EXIT_GRACE_S = 5.0
+
+
+def infer_multi_port_external_lb_start_rank(args: argparse.Namespace) -> int:
+    start_rank = getattr(args, "data_parallel_start_rank", None)
+    if start_rank is not None:
+        return start_rank
+
+    node_rank = getattr(args, "node_rank", 0) or 0
+    local_size = getattr(args, "data_parallel_size_local", 0) or 0
+    return node_rank * local_size
+
+
+def validate_multi_port_external_lb_args(args: argparse.Namespace) -> None:
+    if getattr(args, "grpc", False):
+        raise ValueError(
+            "Error: --data-parallel-multi-port-external-lb does not support --grpc"
+        )
+    if args.uds is not None:
+        raise ValueError(
+            "Error: --data-parallel-multi-port-external-lb does not support --uds"
+        )
+    if bool(args.ssl_keyfile) != bool(args.ssl_certfile):
+        raise ValueError(
+            "Error: --ssl-keyfile and --ssl-certfile must be provided together"
+        )
+    if args.api_server_count not in (None, 1):
+        raise ValueError(
+            "Error: --data-parallel-multi-port-external-lb currently requires "
+            "--api-server-count=1"
+        )
+    if args.data_parallel_rank is not None:
+        raise ValueError(
+            "Error: --data-parallel-multi-port-external-lb manages child "
+            "--data-parallel-rank values internally"
+        )
+    if args.data_parallel_external_lb or args.data_parallel_hybrid_lb:
+        raise ValueError(
+            "Error: --data-parallel-multi-port-external-lb cannot be combined with "
+            "--data-parallel-external-lb or --data-parallel-hybrid-lb"
+        )
+    if args.data_parallel_size < 2:
+        raise ValueError(
+            "Error: --data-parallel-multi-port-external-lb requires "
+            "--data-parallel-size > 1"
+        )
+
+    local_size = args.data_parallel_size_local
+    if local_size is None or local_size < 2:
+        raise ValueError(
+            "Error: --data-parallel-multi-port-external-lb requires "
+            "--data-parallel-size-local >= 2"
+        )
+    if local_size > args.data_parallel_size:
+        raise ValueError(
+            "Error: --data-parallel-size-local cannot exceed --data-parallel-size"
+        )
+    if args.data_parallel_size % local_size != 0:
+        raise ValueError(
+            "Error: --data-parallel-size must be divisible by "
+            "--data-parallel-size-local"
+        )
+
+    start_rank = infer_multi_port_external_lb_start_rank(args)
+    if start_rank + local_size > args.data_parallel_size:
+        raise ValueError(
+            "Error: multi-port supervised ranks would exceed --data-parallel-size"
+        )
+
+    supervisor_port = args.data_parallel_supervisor_port
+    child_port_min = args.port
+    child_port_max = args.port + local_size - 1
+    if child_port_min <= supervisor_port <= child_port_max:
+        raise ValueError(
+            f"Error: --data-parallel-supervisor-port {supervisor_port} "
+            f"overlaps with child rank ports {child_port_min}-{child_port_max}"
+        )
+
+
+def _build_vllm_dp_server_args(
+    args: argparse.Namespace, local_rank: int
+) -> argparse.Namespace:
+    child_args = copy.copy(args)
+    child_args.port = args.port + local_rank
+    child_args.data_parallel_rank = (
+        infer_multi_port_external_lb_start_rank(args) + local_rank
+    )
+    child_args.data_parallel_start_rank = None
+    child_args.data_parallel_size_local = 1
+    child_args.data_parallel_external_lb = True
+    child_args.data_parallel_hybrid_lb = False
+    child_args.data_parallel_multi_port_external_lb = False
+    child_args.data_parallel_supervisor_port = None
+    child_args.api_server_count = 1
+    child_args.device_ids = _build_device_ids(args, local_rank)
+    return child_args
+
+
+def _build_device_ids(args: argparse.Namespace, local_rank: int) -> list[int | str]:
+    """Build the --device-ids value for a DP child process.
+
+    The child resolves these against its own inherited device-control env
+    var (e.g. CUDA_VISIBLE_DEVICES), so integer IDs must stay env-relative
+    here rather than being translated to physical IDs.
+    """
+    devices_per_rank = args.tensor_parallel_size * args.pipeline_parallel_size
+    start = local_rank * devices_per_rank
+    stop = start + devices_per_rank
+    device_ids = getattr(args, "device_ids", None)
+    if device_ids is not None:
+        if stop > len(device_ids):
+            raise ValueError(
+                f"--device-ids has {len(device_ids)} entries, but DP rank "
+                f"{local_rank} needs devices [{start}, {stop})"
+            )
+        return device_ids[start:stop]
+    return list(range(start, stop))
+
+
+def _child_base_url(args: argparse.Namespace, port: int) -> str:
+    host = args.host or "127.0.0.1"
+    if host == "0.0.0.0":
+        host = "127.0.0.1"
+    elif host == "::":
+        host = "::1"
+    scheme = "https" if args.ssl_keyfile and args.ssl_certfile else "http"
+    return f"{scheme}://{host}:{port}"
+
+
+def _join_processes_with_timeout(processes: list[BaseProcess], timeout: float) -> None:
+    deadline = time.monotonic() + timeout
+    for process in processes:
+        remaining = deadline - time.monotonic()
+        if remaining <= 0:
+            break
+        process.join(timeout=remaining)
+
+
+async def _probe_endpoint(
+    session: aiohttp.ClientSession,
+    args: argparse.Namespace,
+    port: int,
+    path: str,
+    conn_err_failure_threshold: int = 3,
+    conn_err_retry_delay: float = 5.0,
+) -> bool:
+    """
+    Probe /health endpoint for 200 status.
+
+    If there is a connection error, retry every N seconds.
+    """
+    for iteration in range(conn_err_failure_threshold):
+        try:
+            probe_ssl = None
+            if args.ssl_keyfile and args.ssl_certfile:
+                # Probes target node-local child servers over loopback, so skip
+                # certificate verification to avoid SAN/hostname mismatches for
+                # localhost/127.0.0.1 deployments.
+                probe_ssl = False
+            async with session.get(
+                _child_base_url(args, port) + path, ssl=probe_ssl
+            ) as response:
+                # vLLM returns 503 on EngineDeadError, so we should return
+                # immediately if vLLM responds with a non-200 status code.
+                return response.status == HTTPStatus.OK
+        except (aiohttp.ClientError, asyncio.TimeoutError) as e:
+            # Allow retry of connection errors.
+            logger.debug(
+                "Probe attempt %d/%d failed on port %d: %r",
+                iteration + 1,
+                conn_err_failure_threshold,
+                port,
+                e,
+            )
+
+        if iteration < conn_err_failure_threshold - 1:
+            await asyncio.sleep(conn_err_retry_delay)
+
+    return False
+
+
+def _build_dp_supervisor_app(supervisor: DPSupervisor) -> FastAPI:
+    app = FastAPI(openapi_url=None, docs_url=None, redoc_url=None)
+    app.state.supervisor = supervisor
+
+    def _status_response(ok: bool) -> Response:
+        return Response(
+            status_code=(HTTPStatus.OK if ok else HTTPStatus.SERVICE_UNAVAILABLE)
+        )
+
+    @app.get("/health", include_in_schema=False)
+    async def health() -> Response:
+        return _status_response(app.state.supervisor.is_ready)
+
+    @app.get("/ready", include_in_schema=False)
+    @app.get("/readyz", include_in_schema=False)
+    async def ready() -> Response:
+        return _status_response(app.state.supervisor.is_ready)
+
+    return app
+
+
+def _run_python_vllm_dp_server(child_args: argparse.Namespace) -> None:
+    from vllm.entrypoints.openai.api_server import run_server
+
+    uvloop.run(run_server(child_args))
+
+
+def _run_rust_vllm_dp_server(child_args: argparse.Namespace) -> None:
+    from vllm.entrypoints.cli.serve import run_multi_api_server
+
+    run_multi_api_server(child_args)
+
+
+def _run_vllm_dp_server(child_args: argparse.Namespace) -> None:
+    """
+    Entrypoint function for the vLLM DP Server.
+    """
+    # Create a fresh process group for the vLLM DP Server,
+    # so that CTRL-C is propagated cleanly.
+    os.setpgrp()
+
+    name = f"APIServer_DP{child_args.data_parallel_rank}"
+    set_process_title(name)
+    decorate_logs(name)
+    if envs.VLLM_RUST_FRONTEND_PATH:
+        _run_rust_vllm_dp_server(child_args)
+    else:
+        _run_python_vllm_dp_server(child_args)
+
+
+class DPSupervisor:
+    def __init__(self, args: argparse.Namespace):
+        validate_multi_port_external_lb_args(args)
+        self.args = args
+        self.supervisor_port = args.data_parallel_supervisor_port
+        self.child_ports = [
+            args.port + local_rank
+            for local_rank in range(args.data_parallel_size_local)
+        ]
+        self._is_ready = False
+        self._processes: list[BaseProcess] = []
+        self._shutdown_event = asyncio.Event()
+        self._shutdown_signal = signal.SIGTERM
+
+    @property
+    def is_ready(self) -> bool:
+        return self._is_ready and not self._shutdown_event.is_set()
+
+    async def run(self) -> None:
+        loop = asyncio.get_running_loop()
+        decorate_logs("DPSupervisor")
+
+        # K8s sends SIGTERM for shutdown - begin graceful termination.
+        for sig in (signal.SIGTERM, signal.SIGINT):
+            loop.add_signal_handler(sig, partial(self._handle_signal, sig))
+
+        supervisor_server: uvicorn.Server | None = None
+        supervisor_server_task: asyncio.Task[None] | None = None
+        try:
+            # Launch vLLM DP Servers and begin monitoring them.
+            self._start_children()
+            monitor_task = asyncio.create_task(
+                self._monitor_children(), name="dp-monitor"
+            )
+
+            # Only start the DPSupervisor server once the vLLM DP Servers are
+            # ready. This avoids the supervisor /health endpoint returning 503
+            # to external load balancer probes while the engines initialize.
+            await self._wait_until_ready(monitor_task)
+            if self.is_ready and not monitor_task.done():
+                supervisor_server, supervisor_server_task = await self._start_server()
+
+            await monitor_task
+        finally:
+            self._is_ready = False
+            await self._shutdown_children()
+
+            # Shutdown the DP Supervisor server.
+            if supervisor_server is not None and supervisor_server_task is not None:
+                supervisor_server.should_exit = True
+                await supervisor_server_task
+
+    async def _start_server(self) -> tuple[uvicorn.Server, asyncio.Task[None]]:
+        """
+        Launch the DPSupervisor HTTP server.
+
+        Called only after the vLLM DP Servers are ready so that /health does
+        not return 503 to external probes while the engines are initializing.
+        """
+        app = _build_dp_supervisor_app(self)
+        host = self.args.host or "0.0.0.0"
+        config = uvicorn.Config(
+            app,
+            host=host,
+            port=self.supervisor_port,
+            log_level=self.args.uvicorn_log_level,
+            ssl_keyfile=self.args.ssl_keyfile,
+            ssl_certfile=self.args.ssl_certfile,
+            ssl_ca_certs=self.args.ssl_ca_certs,
+            ssl_cert_reqs=self.args.ssl_cert_reqs,
+            ssl_ciphers=self.args.ssl_ciphers,
+        )
+        supervisor_server = uvicorn.Server(config)
+        supervisor_server_task = asyncio.create_task(
+            supervisor_server.serve(),
+            name="dp-supervisor",
+        )
+        supervisor_server_task.add_done_callback(
+            lambda _task: self._shutdown_event.set()
+        )
+
+        # Ensure DPSupervisor task starts on the event loop.
+        while not supervisor_server.started:
+            if supervisor_server_task.done():
+                supervisor_server_task.result()
+                raise RuntimeError("DPSupervisor exited before startup.")
+            await asyncio.sleep(0.05)
+        logger.info("Started DPSupervisor on %s:%d", host, self.supervisor_port)
+        return supervisor_server, supervisor_server_task
+
+    async def _wait_until_ready(self, monitor_task: asyncio.Task[None]) -> None:
+        """
+        Block until the vLLM DP Servers are ready or shutdown is triggered.
+
+        Returns early if monitoring stops (e.g. a DP Server dies during
+        startup), in which case the supervisor server is never started.
+        """
+        logger.info("Waiting for vLLM DP Servers to become ready.")
+        while not self._is_ready and not self._shutdown_event.is_set():
+            if monitor_task.done():
+                return
+            await asyncio.sleep(0.05)
+
+    def _handle_signal(self, signum: int) -> None:
+        """
+        Signal handler that is added to the event loop.
+
+        This catches the SIGTERM from K8s and begins graceful shutdown,
+        by setting the _shutdown_event(), which is watched by the main
+        coroutine monitoring the vLLM DP Servers.
+        """
+
+        if self._shutdown_event.is_set():
+            return
+
+        self._shutdown_signal = signal.Signals(signum)
+        logger.info(
+            "DPSupervisor received %s, shutting down.",
+            self._shutdown_signal.name,
+        )
+
+        self._shutdown_event.set()
+        self._is_ready = False
+
+    def _start_children(self) -> None:
+        """
+        Launch vLLM DP Servers on separate GPUs.
+        """
+        logger.info("Launching vLLM DP Servers")
+        context = multiprocessing.get_context("spawn")
+        for local_rank in range(self.args.data_parallel_size_local):
+            child_args = _build_vllm_dp_server_args(self.args, local_rank)
+            process = context.Process(
+                target=_run_vllm_dp_server,
+                name=f"APIServer_DPRank_{child_args.data_parallel_rank}",
+                args=(child_args,),
+            )
+            process.start()
+            self._processes.append(process)
+
+    async def _probe_all_children(self) -> None:
+        """
+        Background coroutine: probes all child endpoints on each interval.
+
+        Exits when any server becomes unhealthy after being ready, signalling
+        _monitor_children to initiate shutdown.
+        """
+        timeout = aiohttp.ClientTimeout(total=self.args.dp_supervisor_probe_timeout_s)
+        async with aiohttp.ClientSession(timeout=timeout) as session:
+            while not self._shutdown_event.is_set():
+                threshold = (
+                    self.args.dp_supervisor_probe_failure_threshold
+                    if self._is_ready
+                    else 1
+                )
+                results = await asyncio.gather(
+                    *(
+                        _probe_endpoint(
+                            session,
+                            self.args,
+                            port,
+                            "/health",
+                            conn_err_failure_threshold=threshold,
+                            conn_err_retry_delay=self.args.dp_supervisor_probe_interval_s,
+                        )
+                        for port in self.child_ports
+                    ),
+                    return_exceptions=True,
+                )
+                all_healthy = all(r is True for r in results)
+
+                if all_healthy:
+                    # If all healthy, we are ready to receive requests.
+                    # This conditional avoids a potential race condition
+                    # where shutdown is set, THEN the probe returns true.
+                    if not self._shutdown_event.is_set():
+                        self._is_ready = True
+                elif self._is_ready:
+                    # Once ready, any failure in the probe means vLLM is dead.
+                    num_unhealthy = sum(1 for r in results if r is not True)
+                    logger.info(
+                        "DPSupervisor probe found %s unhealthy DP Servers.",
+                        num_unhealthy,
+                    )
+                    self._is_ready = False
+                    self._shutdown_event.set()
+                    return
+
+                with contextlib.suppress(asyncio.TimeoutError):
+                    logger.debug(
+                        "Waiting for %s seconds before next probe",
+                        self.args.dp_supervisor_probe_interval_s,
+                    )
+                    await asyncio.wait_for(
+                        self._shutdown_event.wait(),
+                        timeout=self.args.dp_supervisor_probe_interval_s,
+                    )
+
+    async def _monitor_children(self) -> None:
+        """
+        Main coroutine task that monitors the children vLLM servers.
+
+        Before the vLLM servers are /ready:
+        - if the pid is dead, we will shut down
+        - if the probe fails, we try again after dp_supervisor_probe_interval_s
+
+        After the vLLM servers are /ready:
+        - if the pid is dead, we will shut down
+        - if the probe fails, we will shut down
+        """
+        probe_task = asyncio.create_task(
+            self._probe_all_children(), name="dp-health-probe"
+        )
+
+        try:
+            while not self._shutdown_event.is_set():
+                # 1. Check for dead processes
+                n_failed = len([p for p in self._processes if not p.is_alive()])
+                if n_failed > 0:
+                    logger.info("DPSupervisor found %s exited DP Servers.", n_failed)
+                    break
+
+                # 2. Check if the probe background task crashed or failed.
+                if probe_task.done():
+                    # Extract exception if it crashed, or log failure
+                    exc = probe_task.exception() if not probe_task.cancelled() else None
+                    logger.info("DPSupervisor probe task stopped. Exception: %s", exc)
+                    break
+
+                # Sleep for probe_interval seconds or until a shutdown.
+                with contextlib.suppress(asyncio.TimeoutError):
+                    logger.debug(
+                        "Waiting for %s seconds before next monitor",
+                        self.args.dp_supervisor_probe_interval_s,
+                    )
+                    await asyncio.wait_for(
+                        self._shutdown_event.wait(),
+                        timeout=self.args.dp_supervisor_probe_interval_s,
+                    )
+
+        finally:
+            # Cleanup probe task if needed.
+            if not probe_task.done():
+                probe_task.cancel()
+                with contextlib.suppress(asyncio.CancelledError):
+                    await probe_task
+
+    async def _shutdown_children(self) -> None:
+        """Terminate the vLLM DP servers."""
+        timeout = self.args.shutdown_timeout + CHILD_EXIT_GRACE_S
+
+        try:
+            logger.info(
+                "DPSupervisor forwarding %s to DP Servers.",
+                self._shutdown_signal.name,
+            )
+            for process in self._processes:
+                pid = process.pid
+                if not process.is_alive() or pid is None:
+                    continue
+                with contextlib.suppress(ProcessLookupError, OSError):
+                    os.kill(pid, self._shutdown_signal)
+
+            try:
+                await asyncio.to_thread(
+                    _join_processes_with_timeout, self._processes, timeout
+                )
+            except asyncio.CancelledError:
+                logger.warning("Shutdown await cancelled")
+                raise
+        finally:
+            for process in self._processes:
+                pid = process.pid
+                if not process.is_alive() or pid is None:
+                    continue
+                logger.warning(
+                    "DP server %s did not exit within %.1fs; force killing.",
+                    process.name,
+                    timeout,
+                )
+                with contextlib.suppress(
+                    ProcessLookupError,
+                    OSError,
+                    psutil.NoSuchProcess,
+                    psutil.AccessDenied,
+                ):
+                    kill_process_tree(pid)
+
+
+def run_dp_supervisor(args: argparse.Namespace) -> None:
+    uvloop.run(DPSupervisor(args).run())

vllm/entrypoints/openai/engine/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

vllm/entrypoints/openai/engine/protocol.py

@@ -0,0 +1,377 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+# Adapted from
+# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
+import time
+from http import HTTPStatus
+from typing import Any, ClassVar, Literal, TypeAlias
+
+import regex as re
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    model_serializer,
+    model_validator,
+)
+
+from vllm.entrypoints.chat_utils import make_tool_call_id
+from vllm.exceptions import VLLMValidationError
+from vllm.logger import init_logger
+from vllm.utils import random_uuid
+from vllm.utils.import_utils import resolve_obj_by_qualname
+
+logger = init_logger(__name__)
+
+
+class OpenAIBaseModel(BaseModel):
+    # OpenAI API does allow extra fields
+    model_config = ConfigDict(extra="allow")
+
+    # Cache class field names
+    field_names: ClassVar[set[str] | None] = None
+
+    @model_validator(mode="wrap")
+    @classmethod
+    def __log_extra_fields__(cls, data, handler):
+        result = handler(data)
+        if not isinstance(data, dict):
+            return result
+        field_names = cls.field_names
+        if field_names is None:
+            # Get all class field names and their potential aliases
+            field_names = set()
+            for field_name, field in cls.model_fields.items():
+                field_names.add(field_name)
+                if alias := getattr(field, "alias", None):
+                    field_names.add(alias)
+            cls.field_names = field_names
+
+        # Compare against both field names and aliases
+        if any(k not in field_names for k in data):
+            logger.debug(
+                "The following fields were present in the request but ignored: %s",
+                data.keys() - field_names,
+            )
+        return result
+
+
+class ErrorInfo(OpenAIBaseModel):
+    message: str
+    type: str
+    param: str | None = None
+    code: int
+
+
+class ErrorResponse(OpenAIBaseModel):
+    error: ErrorInfo
+
+
+class ModelPermission(OpenAIBaseModel):
+    id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}")
+    object: str = "model_permission"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    allow_create_engine: bool = False
+    allow_sampling: bool = True
+    allow_logprobs: bool = True
+    allow_search_indices: bool = False
+    allow_view: bool = True
+    allow_fine_tuning: bool = False
+    organization: str = "*"
+    group: str | None = None
+    is_blocking: bool = False
+
+
+class ModelCard(OpenAIBaseModel):
+    id: str
+    object: str = "model"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    owned_by: str = "vllm"
+    root: str | None = None
+    parent: str | None = None
+    max_model_len: int | None = None
+    permission: list[ModelPermission] = Field(default_factory=list)
+
+
+class ModelList(OpenAIBaseModel):
+    object: str = "list"
+    data: list[ModelCard] = Field(default_factory=list)
+
+
+class PromptTokenUsageInfo(OpenAIBaseModel):
+    cached_tokens: int | None = None
+    multimodal_tokens: dict[str, int] | None = None
+    """Prompt tokens contributed by each input modality, keyed by modality name
+    (e.g. `image`, `audio`, `video`). A breakdown of the multimodal
+    placeholder tokens already counted in `prompt_tokens`; `None` when the
+    request has no multimodal input."""
+
+
+class UsageInfo(OpenAIBaseModel):
+    prompt_tokens: int = 0
+    total_tokens: int = 0
+    completion_tokens: int | None = 0
+    prompt_tokens_details: PromptTokenUsageInfo | None = None
+
+
+class PerRequestTimingMetrics(OpenAIBaseModel):
+    time_to_first_token_ms: float | None = None
+    generation_time_ms: float | None = None
+    queue_time_ms: float | None = None
+    mean_itl_ms: float | None = None
+    tokens_per_second: float | None = None
+
+
+class RequestResponseMetadata(BaseModel):
+    request_id: str
+    final_usage_info: UsageInfo | None = None
+
+
+class JsonSchemaResponseFormat(OpenAIBaseModel):
+    name: str
+    description: str | None = None
+    # schema is the field in openai but that causes conflicts with pydantic so
+    # instead use json_schema with an alias
+    json_schema: dict[str, Any] | None = Field(default=None, alias="schema")
+    strict: bool | None = None
+
+
+class LegacyStructuralTag(OpenAIBaseModel):
+    begin: str
+    # schema is the field, but that causes conflicts with pydantic so
+    # instead use structural_tag_schema with an alias
+    structural_tag_schema: dict[str, Any] | None = Field(default=None, alias="schema")
+    end: str
+
+
+class LegacyStructuralTagResponseFormat(OpenAIBaseModel):
+    type: Literal["structural_tag"]
+    structures: list[LegacyStructuralTag]
+    triggers: list[str]
+
+
+class StructuralTagResponseFormat(OpenAIBaseModel):
+    type: Literal["structural_tag"]
+    format: Any
+
+
+AnyStructuralTagResponseFormat: TypeAlias = (
+    LegacyStructuralTagResponseFormat | StructuralTagResponseFormat
+)
+
+
+class ResponseFormat(OpenAIBaseModel):
+    # type must be "json_schema", "json_object", or "text"
+    type: Literal["text", "json_object", "json_schema"]
+    json_schema: JsonSchemaResponseFormat | None = None
+
+
+AnyResponseFormat: TypeAlias = (
+    ResponseFormat | StructuralTagResponseFormat | LegacyStructuralTagResponseFormat
+)
+
+
+def validate_structural_tag_response_format(
+    response_format: AnyStructuralTagResponseFormat | dict[str, Any],
+) -> None:
+    """Validate structural tags before they are sent to the engine.
+
+    Engine-side validation reports malformed structural tags as generation
+    failures. OpenAI request parsing should classify them as bad requests.
+    """
+    import json
+
+    from pydantic import TypeAdapter, ValidationError
+
+    if isinstance(response_format, dict):
+        try:
+            response_format = TypeAdapter(
+                AnyStructuralTagResponseFormat
+            ).validate_python(response_format)
+        except ValidationError as exc:
+            raise VLLMValidationError(
+                "Invalid response_format structural_tag specification.",
+                parameter="response_format",
+            ) from exc
+
+    try:
+        payload = json.dumps(response_format.model_dump(by_alias=True))
+        validate_structural_tag_payload(payload, parameter="response_format")
+    except (TypeError, ValueError) as exc:
+        raise VLLMValidationError(
+            "Invalid response_format structural_tag specification.",
+            parameter="response_format",
+        ) from exc
+
+
+def validate_structural_tag_payload(payload: Any, *, parameter: str) -> None:
+    from vllm.sampling_params import SamplingParams, StructuredOutputsParams
+    from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar
+
+    if isinstance(payload, str) and not payload:
+        raise VLLMValidationError(
+            f"Invalid {parameter} structural_tag specification.",
+            parameter=parameter,
+        )
+
+    try:
+        validate_xgrammar_grammar(
+            SamplingParams(
+                structured_outputs=StructuredOutputsParams(structural_tag=payload)
+            )
+        )
+    except (TypeError, ValueError) as exc:
+        raise VLLMValidationError(
+            f"Invalid {parameter} structural_tag specification.",
+            parameter=parameter,
+        ) from exc
+
+
+def validate_structured_outputs_structural_tag(
+    structured_outputs: Any,
+) -> None:
+    from vllm.sampling_params import StructuredOutputsParams
+
+    if isinstance(structured_outputs, StructuredOutputsParams):
+        structural_tag = structured_outputs.structural_tag
+    elif isinstance(structured_outputs, dict):
+        structural_tag = structured_outputs.get("structural_tag")
+    else:
+        return
+    if structural_tag is not None:
+        validate_structural_tag_payload(
+            structural_tag,
+            parameter="structured_outputs",
+        )
+
+
+class StreamOptions(OpenAIBaseModel):
+    include_usage: bool | None = False
+    continuous_usage_stats: bool | None = False
+
+
+class FunctionDefinition(OpenAIBaseModel):
+    name: str
+    description: str | None = None
+    parameters: dict[str, Any] | None = None
+    strict: bool | None = None
+    defer_loading: bool | None = None
+
+    @model_serializer(mode="wrap")
+    def _serialize(self, handler):
+        data = handler(self)
+        if self.strict is None:
+            data.pop("strict", None)
+        if self.defer_loading is None:
+            data.pop("defer_loading", None)
+        return data
+
+
+# extra="forbid" is a workaround to have kwargs as a field,
+# see https://github.com/pydantic/pydantic/issues/3125
+class LogitsProcessorConstructor(BaseModel):
+    qualname: str
+    args: list[Any] | None = None
+    kwargs: dict[str, Any] | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+LogitsProcessors = list[str | LogitsProcessorConstructor]
+
+
+def get_logits_processors(
+    processors: LogitsProcessors | None, pattern: str | None
+) -> list[Any] | None:
+    if processors and pattern:
+        logits_processors = []
+        for processor in processors:
+            qualname = processor if isinstance(processor, str) else processor.qualname
+            if not re.match(pattern, qualname):
+                raise ValueError(
+                    f"Logits processor '{qualname}' is not allowed by this "
+                    "server. See --logits-processor-pattern engine argument "
+                    "for more information."
+                )
+            try:
+                logits_processor = resolve_obj_by_qualname(qualname)
+            except Exception as e:
+                raise ValueError(
+                    f"Logits processor '{qualname}' could not be resolved: {e}"
+                ) from e
+            if isinstance(processor, LogitsProcessorConstructor):
+                logits_processor = logits_processor(
+                    *processor.args or [], **processor.kwargs or {}
+                )
+            logits_processors.append(logits_processor)
+        return logits_processors
+    elif processors:
+        raise ValueError(
+            "The `logits_processors` argument is not supported by this "
+            "server. See --logits-processor-pattern engine argument "
+            "for more information."
+        )
+    return None
+
+
+class FunctionCall(OpenAIBaseModel):
+    # Internal field to preserve native tool call ID from tool parser.
+    # Excluded from serialization to maintain OpenAI API compatibility
+    # (function object should only contain 'name' and 'arguments').
+    id: str | None = Field(default=None, exclude=True)
+    name: str
+    arguments: str
+
+
+class ToolCall(OpenAIBaseModel):
+    id: str = Field(default_factory=make_tool_call_id)
+    type: Literal["function"] = "function"
+    function: FunctionCall
+
+
+class DeltaFunctionCall(BaseModel):
+    name: str | None = None
+    arguments: str | None = None
+
+
+# a tool call delta where everything is optional
+class DeltaToolCall(OpenAIBaseModel):
+    id: str | None = None
+    type: Literal["function"] | None = None
+    index: int
+    function: DeltaFunctionCall | None = None
+
+
+class ExtractedToolCallInformation(BaseModel):
+    # indicate if tools were called
+    tools_called: bool
+
+    # extracted tool calls
+    tool_calls: list[ToolCall]
+
+    # content - per OpenAI spec, content AND tool calls can be returned rarely
+    # But some models will do this intentionally
+    content: str | None = None
+
+
+class DeltaMessage(OpenAIBaseModel):
+    role: str | None = None
+    content: str | None = None
+    reasoning: str | None = None
+    tool_calls: list[DeltaToolCall] = Field(default_factory=list)
+
+    @model_serializer(mode="wrap")
+    def _serialize(self, handler):
+        data = handler(self)
+        if len(data.get("tool_calls", [])) == 0:
+            data.pop("tool_calls", None)
+        return data
+
+
+class GenerationError(Exception):
+    """raised when finish_reason indicates internal server error (500)"""
+
+    def __init__(self, message: str = "Internal server error"):
+        super().__init__(message)
+        self.status_code = HTTPStatus.INTERNAL_SERVER_ERROR

vllm/entrypoints/openai/models/api_router.py

@@ -0,0 +1,29 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+
+from fastapi import APIRouter, FastAPI, Request
+from fastapi.responses import JSONResponse
+
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+router = APIRouter()
+
+
+def models(request: Request) -> OpenAIServingModels:
+    return request.app.state.openai_serving_models
+
+
+@router.get("/v1/models")
+async def show_available_models(raw_request: Request):
+    handler = models(raw_request)
+
+    models_ = await handler.show_available_models()
+    return JSONResponse(content=models_.model_dump())
+
+
+def attach_router(app: FastAPI):
+    app.include_router(router)

vllm/entrypoints/openai/models/protocol.py

@@ -0,0 +1,19 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+
+from dataclasses import dataclass
+
+
+@dataclass
+class BaseModelPath:
+    name: str
+    model_path: str
+
+
+@dataclass
+class LoRAModulePath:
+    name: str
+    path: str
+    base_model_name: str | None = None
+    is_3d_lora_weight: bool = False

vllm/entrypoints/openai/models/serving.py

@@ -0,0 +1,344 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from asyncio import Lock
+from collections import defaultdict
+from http import HTTPStatus
+
+from vllm.config import ModelConfig
+from vllm.engine.protocol import EngineClient
+from vllm.entrypoints.openai.engine.protocol import (
+    ErrorResponse,
+    ModelCard,
+    ModelList,
+    ModelPermission,
+)
+from vllm.entrypoints.openai.models.protocol import BaseModelPath, LoRAModulePath
+from vllm.entrypoints.serve.lora.protocol import (
+    LoadLoRAAdapterRequest,
+    UnloadLoRAAdapterRequest,
+)
+from vllm.entrypoints.serve.utils.error_response import create_error_response
+from vllm.exceptions import LoRAAdapterNotFoundError
+from vllm.logger import init_logger
+from vllm.lora.request import LoRARequest
+from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
+from vllm.utils.counter import AtomicCounter
+
+logger = init_logger(__name__)
+
+
+class OpenAIModelRegistry:
+    """Read-only view of the loaded base models with no engine dependency.
+
+    Suitable for CPU-only / render-only contexts that have no engine client
+    and no LoRA support.
+    """
+
+    def __init__(
+        self,
+        model_config: ModelConfig,
+        base_model_paths: list[BaseModelPath],
+    ) -> None:
+        self.model_config = model_config
+        self.base_model_paths = base_model_paths
+        self.lora_requests: dict[str, LoRARequest] = {}
+
+    def model_name(self, lora_request: LoRARequest | None = None) -> str:
+        return self.base_model_paths[0].name
+
+    def is_base_model(self, model_name: str) -> bool:
+        return any(model.name == model_name for model in self.base_model_paths)
+
+    async def check_model(self, model_name: str | None) -> ErrorResponse | None:
+        """Return an ErrorResponse if model_name is not served, else None."""
+        if not model_name or self.is_base_model(model_name):
+            return None
+        return create_error_response(
+            message=f"The model `{model_name}` does not exist.",
+            err_type="NotFoundError",
+            status_code=HTTPStatus.NOT_FOUND,
+            param="model",
+        )
+
+    async def show_available_models(self) -> ModelList:
+        """Show available models (base models only)."""
+        max_model_len = self.model_config.max_model_len
+        return ModelList(
+            data=[
+                ModelCard(
+                    id=base_model.name,
+                    max_model_len=max_model_len,
+                    root=base_model.model_path,
+                    permission=[ModelPermission()],
+                )
+                for base_model in self.base_model_paths
+            ]
+        )
+
+    async def resolve_lora(self, lora_name: str):
+        raise RuntimeError("The OpenAIModelRegistry has no LoRA support.")
+
+
+class OpenAIServingModels:
+    """Shared instance to hold data about the loaded base model(s) and adapters.
+
+    Handles the routes:
+    - /v1/models
+    - /v1/load_lora_adapter
+    - /v1/unload_lora_adapter
+    """
+
+    def __init__(
+        self,
+        engine_client: EngineClient,
+        base_model_paths: list[BaseModelPath],
+        *,
+        lora_modules: list[LoRAModulePath] | None = None,
+    ):
+        super().__init__()
+
+        self.registry = OpenAIModelRegistry(
+            model_config=engine_client.model_config,
+            base_model_paths=base_model_paths,
+        )
+
+        self.engine_client = engine_client
+        self.base_model_paths = base_model_paths
+
+        self.static_lora_modules = lora_modules
+        self.lora_requests: dict[str, LoRARequest] = {}
+        self.lora_id_counter = AtomicCounter(0)
+
+        self.lora_resolvers: list[LoRAResolver] = []
+        for lora_resolver_name in LoRAResolverRegistry.get_supported_resolvers():
+            self.lora_resolvers.append(
+                LoRAResolverRegistry.get_resolver(lora_resolver_name)
+            )
+        self.lora_resolver_lock: dict[str, Lock] = defaultdict(Lock)
+
+        self.model_config = self.engine_client.model_config
+        self.renderer = self.engine_client.renderer
+        self.input_processor = self.engine_client.input_processor
+
+    async def init_static_loras(self):
+        """Loads all static LoRA modules.
+        Raises if any fail to load"""
+        if self.static_lora_modules is None:
+            return
+        for lora in self.static_lora_modules:
+            load_request = LoadLoRAAdapterRequest(
+                lora_path=lora.path,
+                lora_name=lora.name,
+                is_3d_lora_weight=lora.is_3d_lora_weight,
+            )
+            load_result = await self.load_lora_adapter(
+                request=load_request, base_model_name=lora.base_model_name
+            )
+            if isinstance(load_result, ErrorResponse):
+                raise ValueError(load_result.error.message)
+
+    def is_base_model(self, model_name: str) -> bool:
+        return self.registry.is_base_model(model_name)
+
+    def model_name(self, lora_request: LoRARequest | None = None) -> str:
+        if lora_request is not None:
+            return lora_request.lora_name
+        return self.base_model_paths[0].name
+
+    async def show_available_models(self) -> ModelList:
+        """Show available models. This includes the base model and all
+        adapters."""
+        model_list = await self.registry.show_available_models()
+        lora_cards = [
+            ModelCard(
+                id=lora.lora_name,
+                root=lora.path,
+                parent=lora.base_model_name
+                if lora.base_model_name
+                else self.base_model_paths[0].name,
+                permission=[ModelPermission()],
+            )
+            for lora in self.lora_requests.values()
+        ]
+        model_list.data.extend(lora_cards)
+        return model_list
+
+    async def load_lora_adapter(
+        self, request: LoadLoRAAdapterRequest, base_model_name: str | None = None
+    ) -> ErrorResponse | str:
+        lora_name = request.lora_name
+
+        # Ensure atomicity based on the lora name
+        async with self.lora_resolver_lock[lora_name]:
+            error_check_ret = await self._check_load_lora_adapter_request(request)
+            if error_check_ret is not None:
+                return error_check_ret
+
+            lora_path = request.lora_path
+            lora_int_id = (
+                self.lora_requests[lora_name].lora_int_id
+                if lora_name in self.lora_requests
+                else self.lora_id_counter.inc(1)
+            )
+            lora_request = LoRARequest(
+                lora_name=lora_name,
+                lora_int_id=lora_int_id,
+                lora_path=lora_path,
+                load_inplace=request.load_inplace,
+                is_3d_lora_weight=request.is_3d_lora_weight,
+            )
+            if base_model_name is not None and self.is_base_model(base_model_name):
+                lora_request.base_model_name = base_model_name
+
+            # Validate that the adapter can be loaded into the engine
+            # This will also preload it for incoming requests
+            try:
+                await self.engine_client.add_lora(lora_request)
+            except Exception as e:
+                if str(
+                    LoRAAdapterNotFoundError(
+                        lora_request.lora_name, lora_request.lora_path
+                    )
+                ) in str(e):
+                    return create_error_response(
+                        LoRAAdapterNotFoundError(
+                            lora_request.lora_name, lora_request.lora_path
+                        )
+                    )
+                return create_error_response(
+                    message=str(e),
+                    err_type="InternalServerError",
+                    status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
+                )
+
+            self.lora_requests[lora_name] = lora_request
+            logger.info(
+                "Loaded new LoRA adapter: name '%s', path '%s'", lora_name, lora_path
+            )
+            return f"Success: LoRA adapter '{lora_name}' added successfully."
+
+    async def unload_lora_adapter(
+        self, request: UnloadLoRAAdapterRequest
+    ) -> ErrorResponse | str:
+        lora_name = request.lora_name
+
+        # Ensure atomicity based on the lora name
+        async with self.lora_resolver_lock[lora_name]:
+            error_check_ret = await self._check_unload_lora_adapter_request(request)
+            if error_check_ret is not None:
+                return error_check_ret
+
+            # Safe to delete now since we hold the lock
+            del self.lora_requests[lora_name]
+            logger.info("Removed LoRA adapter: name '%s'", lora_name)
+            return f"Success: LoRA adapter '{lora_name}' removed successfully."
+
+    async def _check_load_lora_adapter_request(
+        self, request: LoadLoRAAdapterRequest
+    ) -> ErrorResponse | None:
+        # Check if both 'lora_name' and 'lora_path' are provided
+        if not request.lora_name or not request.lora_path:
+            return create_error_response(
+                message="Both 'lora_name' and 'lora_path' must be provided.",
+                err_type="InvalidUserInput",
+                status_code=HTTPStatus.BAD_REQUEST,
+            )
+
+        # If not loading inplace
+        # Check if the lora adapter with the given name already exists
+        if not request.load_inplace and request.lora_name in self.lora_requests:
+            return create_error_response(
+                message=f"The lora adapter '{request.lora_name}' has already been "
+                "loaded. If you want to load the adapter in place, set 'load_inplace'"
+                " to True.",
+                err_type="InvalidUserInput",
+                status_code=HTTPStatus.BAD_REQUEST,
+            )
+
+        return None
+
+    async def _check_unload_lora_adapter_request(
+        self, request: UnloadLoRAAdapterRequest
+    ) -> ErrorResponse | None:
+        # Check if 'lora_name' is not provided return an error
+        if not request.lora_name:
+            return create_error_response(
+                message="'lora_name' needs to be provided to unload a LoRA adapter.",
+                err_type="InvalidUserInput",
+                status_code=HTTPStatus.BAD_REQUEST,
+            )
+
+        # Check if the lora adapter with the given name exists
+        if request.lora_name not in self.lora_requests:
+            return create_error_response(
+                message=f"The lora adapter '{request.lora_name}' cannot be found.",
+                err_type="NotFoundError",
+                status_code=HTTPStatus.NOT_FOUND,
+            )
+
+        return None
+
+    async def resolve_lora(self, lora_name: str) -> LoRARequest | ErrorResponse:
+        """Attempt to resolve a LoRA adapter using available resolvers.
+
+        Args:
+            lora_name: Name/identifier of the LoRA adapter
+
+        Returns:
+            LoRARequest if found and loaded successfully.
+            ErrorResponse (404) if no resolver finds the adapter.
+            ErrorResponse (400) if adapter(s) are found but none load.
+        """
+        async with self.lora_resolver_lock[lora_name]:
+            # First check if this LoRA is already loaded
+            if lora_name in self.lora_requests:
+                return self.lora_requests[lora_name]
+
+            base_model_name = self.model_config.model
+            unique_id = self.lora_id_counter.inc(1)
+            found_adapter = False
+
+            # Try to resolve using available resolvers
+            for resolver in self.lora_resolvers:
+                lora_request = await resolver.resolve_lora(base_model_name, lora_name)
+
+                if lora_request is not None:
+                    found_adapter = True
+                    lora_request.lora_int_id = unique_id
+
+                    try:
+                        await self.engine_client.add_lora(lora_request)
+                        self.lora_requests[lora_name] = lora_request
+                        logger.info(
+                            "Resolved and loaded LoRA adapter '%s' using %s",
+                            lora_name,
+                            resolver.__class__.__name__,
+                        )
+                        return lora_request
+                    except BaseException as e:
+                        logger.warning(
+                            "Failed to load LoRA '%s' resolved by %s: %s. "
+                            "Trying next resolver.",
+                            lora_name,
+                            resolver.__class__.__name__,
+                            e,
+                        )
+                        continue
+
+            if found_adapter:
+                # An adapter was found, but all attempts to load it failed.
+                return create_error_response(
+                    message=(
+                        f"LoRA adapter '{lora_name}' was found but could not be loaded."
+                    ),
+                    err_type="BadRequestError",
+                    status_code=HTTPStatus.BAD_REQUEST,
+                )
+            else:
+                # No adapter was found
+                return create_error_response(
+                    message=f"LoRA adapter {lora_name} does not exist",
+                    err_type="NotFoundError",
+                    status_code=HTTPStatus.NOT_FOUND,
+                )

vllm/entrypoints/openai/parser/harmony_utils.py

@@ -0,0 +1,461 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import datetime
+from collections.abc import Sequence
+from typing import Any
+
+from openai.types.responses.tool import Tool
+from openai_harmony import (
+    Author,
+    Conversation,
+    DeveloperContent,
+    HarmonyEncodingName,
+    Message,
+    ReasoningEffort,
+    RenderConversationConfig,
+    Role,
+    StreamableParser,
+    SystemContent,
+    TextContent,
+    ToolDescription,
+    load_harmony_encoding,
+)
+
+from vllm import envs
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionToolsParam
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+
+def is_function_recipient(
+    recipient: str,
+    allowed_function_tool_names: frozenset[str] | None = None,
+) -> bool:
+    """Check whether *recipient* refers to a function tool call.
+
+    The optional *allowed_function_tool_names* parameter is used by the
+    Responses API to distinguish bare function-call recipients (missing the
+    ``functions.`` prefix) from MCP tool calls.  When provided, a bare
+    recipient is only treated as a function call if it appears in the set.
+    The Chat Completions path omits this parameter so that all bare
+    recipients are accepted as function calls (the heuristic fallback).
+    """
+    if not recipient:
+        return False
+    if recipient.startswith("<|"):
+        return False
+    if recipient.startswith("functions."):
+        return len(recipient) > len("functions.")
+    if recipient == "assistant":
+        return False
+    if recipient in BUILTIN_TOOL_TO_MCP_SERVER_LABEL:
+        return False
+    first_segment = recipient.split(".", 1)[0]
+    if first_segment in BUILTIN_TOOL_TO_MCP_SERVER_LABEL:
+        return False
+    if allowed_function_tool_names is not None:
+        return recipient in allowed_function_tool_names
+    return True
+
+
+def extract_function_from_recipient(recipient: str) -> str:
+    return recipient.removeprefix("functions.")
+
+
+REASONING_EFFORT = {
+    "high": ReasoningEffort.HIGH,
+    "medium": ReasoningEffort.MEDIUM,
+    "low": ReasoningEffort.LOW,
+}
+
+_harmony_encoding = None
+
+# Builtin tools that should be included in the system message when
+# they are available and requested by the user.
+# Tool args are provided by MCP tool descriptions. Output
+# of the tools are stringified.
+BUILTIN_TOOL_TO_MCP_SERVER_LABEL: dict[str, str] = {
+    "python": "code_interpreter",
+    "browser": "web_search_preview",
+    "container": "container",
+}
+
+# Derive MCP_BUILTIN_TOOLS from the canonical mapping
+MCP_BUILTIN_TOOLS: set[str] = set(BUILTIN_TOOL_TO_MCP_SERVER_LABEL.values())
+
+
+def has_custom_tools(tool_types: set[str]) -> bool:
+    """
+    Checks if the given tool types are custom tools
+    (i.e. any tool other than MCP builtin tools)
+    """
+    return not tool_types.issubset(MCP_BUILTIN_TOOLS)
+
+
+def get_encoding():
+    global _harmony_encoding
+    if _harmony_encoding is None:
+        _harmony_encoding = load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS)
+    return _harmony_encoding
+
+
+def get_system_message(
+    model_identity: str | None = None,
+    reasoning_effort: str | None = None,
+    start_date: str | None = None,
+    browser_description: str | None = None,
+    python_description: str | None = None,
+    container_description: str | None = None,
+    instructions: str | None = None,
+    with_custom_tools: bool = False,
+) -> Message:
+    sys_msg_content = SystemContent.new()
+    if model_identity is not None:
+        sys_msg_content = sys_msg_content.with_model_identity(model_identity)
+    if instructions is not None and envs.VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS:
+        current_identity = sys_msg_content.model_identity
+        new_identity = (
+            f"{current_identity}\n{instructions}" if current_identity else instructions
+        )
+        sys_msg_content = sys_msg_content.with_model_identity(new_identity)
+    if reasoning_effort is not None:
+        if reasoning_effort not in REASONING_EFFORT:
+            supported_values = ", ".join(REASONING_EFFORT)
+            raise ValueError(
+                f"reasoning_effort={reasoning_effort!r} is not supported by "
+                f"Harmony. Supported values are: {supported_values}."
+            )
+        sys_msg_content = sys_msg_content.with_reasoning_effort(
+            REASONING_EFFORT[reasoning_effort]
+        )
+    if start_date is None:
+        # NOTE(woosuk): This brings non-determinism in vLLM.
+        # Set VLLM_SYSTEM_START_DATE to pin it.
+        start_date = envs.VLLM_SYSTEM_START_DATE or datetime.datetime.now().strftime(
+            "%Y-%m-%d"
+        )
+    sys_msg_content = sys_msg_content.with_conversation_start_date(start_date)
+    if browser_description is not None:
+        sys_msg_content = sys_msg_content.with_tools(browser_description)
+    if python_description is not None:
+        sys_msg_content = sys_msg_content.with_tools(python_description)
+    if container_description is not None:
+        sys_msg_content = sys_msg_content.with_tools(container_description)
+    sys_msg = Message.from_role_and_content(Role.SYSTEM, sys_msg_content)
+    return sys_msg
+
+
+def create_tool_definition(tool: ChatCompletionToolsParam | Tool):
+    if isinstance(tool, ChatCompletionToolsParam):
+        return ToolDescription.new(
+            name=tool.function.name,
+            description=tool.function.description or "",
+            parameters=tool.function.parameters,
+        )
+    return ToolDescription.new(
+        name=tool.name,
+        description=tool.description or "",
+        parameters=tool.parameters,
+    )
+
+
+def get_developer_message(
+    instructions: str | None = None,
+    tools: list[Tool | ChatCompletionToolsParam] | None = None,
+) -> Message:
+    dev_msg_content = DeveloperContent.new()
+    if instructions is not None and not envs.VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS:
+        dev_msg_content = dev_msg_content.with_instructions(instructions)
+    if tools is not None:
+        function_tools: list[Tool | ChatCompletionToolsParam] = []
+        for tool in tools:
+            if tool.type in (
+                "web_search_preview",
+                "code_interpreter",
+                "container",
+            ):
+                pass
+
+            elif tool.type == "function":
+                function_tools.append(tool)
+            else:
+                raise ValueError(f"tool type {tool.type} not supported")
+        if function_tools:
+            function_tool_descriptions = [
+                create_tool_definition(tool) for tool in function_tools
+            ]
+            dev_msg_content = dev_msg_content.with_function_tools(
+                function_tool_descriptions
+            )
+    dev_msg = Message.from_role_and_content(Role.DEVELOPER, dev_msg_content)
+    return dev_msg
+
+
+def get_user_message(content: str) -> Message:
+    return Message.from_role_and_content(Role.USER, content)
+
+
+def get_system_or_developer_message(role: str, instructions: str) -> Message:
+    if role == "system" and envs.VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS:
+        return get_system_message(instructions=instructions)
+    return get_developer_message(instructions=instructions)
+
+
+def parse_chat_inputs_to_harmony_messages(chat_msgs: list) -> list[Message]:
+    """
+    Parse a list of messages from request.messages in the Chat Completion API to
+    Harmony messages.
+    """
+    msgs: list[Message] = []
+    tool_id_names: dict[str, str] = {}
+
+    # Collect tool id to name mappings for tool response recipient values
+    for chat_msg in chat_msgs:
+        for tool_call in chat_msg.get("tool_calls", []):
+            tool_id_names[tool_call.get("id")] = tool_call.get("function", {}).get(
+                "name"
+            )
+
+    for chat_msg in chat_msgs:
+        msgs.extend(parse_chat_input_to_harmony_message(chat_msg, tool_id_names))
+
+    return msgs
+
+
+def auto_drop_analysis_messages(msgs: list[Message]) -> list[Message]:
+    """
+    Harmony models expect the analysis messages (representing raw chain of thought) to
+    be dropped after an assistant message to the final channel is produced from the
+    reasoning of those messages.
+
+    The openai-harmony library does this if the very last assistant message is to the
+    final channel, but it does not handle the case where we're in longer multi-turn
+    conversations and the client gave us reasoning content from previous turns of
+    the conversation with multiple assistant messages to the final channel in the
+    conversation.
+
+    So, we find the index of the last assistant message to the final channel and drop
+    all analysis messages that precede it, leaving only the analysis messages that
+    are relevant to the current part of the conversation.
+    """
+    last_assistant_final_index = -1
+    for i in range(len(msgs) - 1, -1, -1):
+        msg = msgs[i]
+        if msg.author.role == "assistant" and msg.channel == "final":
+            last_assistant_final_index = i
+            break
+
+    cleaned_msgs: list[Message] = []
+    for i, msg in enumerate(msgs):
+        if i < last_assistant_final_index and msg.channel == "analysis":
+            continue
+        cleaned_msgs.append(msg)
+
+    return cleaned_msgs
+
+
+def flatten_input_text_content(content: Any) -> str | None:
+    """
+    Extract text parts from a Chat Completion or Responses API content field and
+    flatten them into a single string. Returns None if no text content is found.
+    """
+    if content is None or isinstance(content, str):
+        return content
+    if not isinstance(content, list):
+        return None
+
+    texts: list[str] = []
+    for item in content:
+        if isinstance(item, str):
+            texts.append(item)
+            continue
+        if isinstance(item, dict):
+            text = item.get("text")
+            if text is not None:
+                texts.append(text)
+    return "".join(texts) if texts else None
+
+
+def extract_instructions_from_messages(
+    messages: Sequence[Any],
+) -> tuple[str | None, list[Any]]:
+    """
+    Peel a leading system/developer Chat Completion or Responses message and
+    flatten its instruction text.
+    """
+    remaining_messages = list(messages)
+    if not remaining_messages:
+        return None, remaining_messages
+
+    first_message = remaining_messages[0]
+    if not isinstance(first_message, dict):
+        if hasattr(first_message, "to_dict"):
+            # Handle OpenAI Harmony Message
+            first_message = first_message.to_dict()
+        elif hasattr(first_message, "model_dump"):
+            first_message = first_message.model_dump(exclude_none=True)
+        else:
+            raise ValueError(f"Unknown message type: {type(first_message)}")
+
+    if first_message.get("role") not in (
+        "system",
+        "developer",
+    ):
+        return None, remaining_messages
+
+    instructions = flatten_input_text_content(first_message.get("content"))
+    return instructions, remaining_messages[1:]
+
+
+def build_harmony_preamble(
+    *,
+    instructions: str | None = None,
+    tools: list[Tool | ChatCompletionToolsParam] | None = None,
+    reasoning_effort: str | None = None,
+    browser_description: str | None = None,
+    python_description: str | None = None,
+    container_description: str | None = None,
+    with_custom_tools: bool = False,
+) -> list[Message]:
+    """
+    Build the standard Harmony system/developer prefix for a request.
+    """
+    developer_instructions = system_instructions = None
+    if envs.VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS:
+        system_instructions = instructions
+    else:
+        developer_instructions = instructions
+
+    messages = [
+        get_system_message(
+            reasoning_effort=reasoning_effort,
+            browser_description=browser_description,
+            python_description=python_description,
+            container_description=container_description,
+            instructions=system_instructions,
+            with_custom_tools=with_custom_tools,
+        )
+    ]
+    if developer_instructions or tools:
+        messages.append(
+            get_developer_message(
+                instructions=developer_instructions,
+                tools=tools,
+            )
+        )
+    return messages
+
+
+def parse_chat_input_to_harmony_message(
+    chat_msg, tool_id_names: dict[str, str] | None = None
+) -> list[Message]:
+    """
+    Parse a message from request.messages in the Chat Completion API to
+    Harmony messages.
+    """
+    tool_id_names = tool_id_names or {}
+
+    if not isinstance(chat_msg, dict):
+        # Handle Pydantic models
+        chat_msg = chat_msg.model_dump(exclude_none=True)
+
+    role = chat_msg.get("role")
+    msgs: list[Message] = []
+
+    # Assistant message with tool calls
+    tool_calls = chat_msg.get("tool_calls", [])
+
+    if role == "assistant" and tool_calls:
+        content = flatten_input_text_content(chat_msg.get("content"))
+        if content:
+            commentary_msg = Message.from_role_and_content(Role.ASSISTANT, content)
+            commentary_msg = commentary_msg.with_channel("commentary")
+            msgs.append(commentary_msg)
+
+        reasoning = chat_msg.get("reasoning")
+        if reasoning:
+            analysis_msg = Message.from_role_and_content(Role.ASSISTANT, reasoning)
+            analysis_msg = analysis_msg.with_channel("analysis")
+            msgs.append(analysis_msg)
+
+        for call in tool_calls:
+            func = call.get("function", {})
+            name = func.get("name", "")
+            arguments = func.get("arguments", "") or ""
+            msg = Message.from_role_and_content(Role.ASSISTANT, arguments)
+            msg = msg.with_channel("commentary")
+            msg = msg.with_recipient(f"functions.{name}")
+            # Officially, this should be `<|constrain|>json` but there is not clear
+            # evidence that improves accuracy over `json` and some anecdotes to the
+            # contrary. Further testing of the different content_types is needed.
+            msg = msg.with_content_type("json")
+            msgs.append(msg)
+        return msgs
+
+    # Tool role message (tool output)
+    if role == "tool":
+        tool_call_id = chat_msg.get("tool_call_id", "")
+        name = tool_id_names.get(tool_call_id, "")
+        content = flatten_input_text_content(chat_msg.get("content")) or ""
+
+        msg = (
+            Message.from_author_and_content(
+                Author.new(Role.TOOL, f"functions.{name}"), content
+            )
+            .with_channel("commentary")
+            .with_recipient("assistant")
+        )
+        return [msg]
+
+    # Non-tool reasoning content
+    reasoning = chat_msg.get("reasoning")
+    if role == "assistant" and reasoning:
+        analysis_msg = Message.from_role_and_content(Role.ASSISTANT, reasoning)
+        analysis_msg = analysis_msg.with_channel("analysis")
+        msgs.append(analysis_msg)
+
+    # Default: user/assistant/system messages with content
+    content = chat_msg.get("content") or ""
+    if content is None:
+        content = ""
+    if isinstance(content, str):
+        contents = [TextContent(text=content)]
+    else:
+        # TODO: Support refusal.
+        contents = [TextContent(text=c.get("text", "")) for c in content]
+
+    # Only add assistant messages if they have content, as reasoning or tool calling
+    # assistant messages were already added above.
+    if role == "assistant" and contents and contents[0].text:
+        msg = Message.from_role_and_contents(role, contents)
+        # Send non-tool assistant messages to the final channel
+        msg = msg.with_channel("final")
+        msgs.append(msg)
+    elif role in ("system", "developer"):
+        instructions = flatten_input_text_content(chat_msg.get("content"))
+        if instructions is not None:
+            msg = get_system_or_developer_message(role, instructions)
+            msgs.append(msg)
+    # For user messages, add them directly even if no content.
+    elif role != "assistant":
+        msg = Message.from_role_and_contents(role, contents)
+        msgs.append(msg)
+
+    return msgs
+
+
+def render_for_completion(messages: list[Message]) -> list[int]:
+    messages = auto_drop_analysis_messages(messages)
+    conversation = Conversation.from_messages(messages)
+    token_ids = get_encoding().render_conversation_for_completion(
+        conversation,
+        Role.ASSISTANT,
+        config=RenderConversationConfig(auto_drop_analysis=False),
+    )
+    return token_ids
+
+
+def get_streamable_parser_for_assistant() -> StreamableParser:
+    return StreamableParser(get_encoding(), role=Role.ASSISTANT)

vllm/entrypoints/openai/responses/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

vllm/entrypoints/openai/responses/api_router.py

@@ -0,0 +1,128 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+
+from collections.abc import AsyncGenerator
+from http import HTTPStatus
+
+from fastapi import APIRouter, Depends, FastAPI, Request
+from fastapi.responses import JSONResponse, StreamingResponse
+
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
+from vllm.entrypoints.openai.responses.protocol import (
+    ResponsesRequest,
+    ResponsesResponse,
+    StreamingResponsesResponse,
+)
+from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
+from vllm.entrypoints.serve.utils.api_utils import (
+    load_aware_call,
+    validate_json_request,
+    with_cancellation,
+)
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+router = APIRouter()
+
+
+def responses(request: Request) -> OpenAIServingResponses | None:
+    return request.app.state.openai_serving_responses
+
+
+async def _convert_stream_to_sse_events(
+    generator: AsyncGenerator[StreamingResponsesResponse, None],
+) -> AsyncGenerator[str, None]:
+    """Convert the generator to a stream of events in SSE format"""
+    async for event in generator:
+        event_type = getattr(event, "type", "unknown")
+        # https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#event_stream_format
+        event_data = (
+            f"event: {event_type}\ndata: "
+            f"{event.model_dump_json(indent=None, by_alias=True)}\n\n"
+        )
+        yield event_data
+
+
+@router.post(
+    "/v1/responses",
+    dependencies=[Depends(validate_json_request)],
+    responses={
+        HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
+        HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
+        HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
+        HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
+    },
+)
+@with_cancellation
+@load_aware_call
+async def create_responses(request: ResponsesRequest, raw_request: Request):
+    handler = responses(raw_request)
+    if handler is None:
+        raise NotImplementedError("The model does not support Responses API")
+
+    generator = await handler.create_responses(request, raw_request)
+
+    if isinstance(generator, ErrorResponse):
+        return JSONResponse(
+            content=generator.model_dump(mode="json", by_alias=True),
+            status_code=generator.error.code,
+        )
+    elif isinstance(generator, ResponsesResponse):
+        return JSONResponse(content=generator.model_dump(mode="json", by_alias=True))
+
+    return StreamingResponse(
+        content=_convert_stream_to_sse_events(generator), media_type="text/event-stream"
+    )
+
+
+@router.get("/v1/responses/{response_id}")
+@load_aware_call
+async def retrieve_responses(
+    response_id: str,
+    raw_request: Request,
+    starting_after: int | None = None,
+    stream: bool | None = False,
+):
+    handler = responses(raw_request)
+    if handler is None:
+        raise NotImplementedError("The model does not support Responses API")
+
+    response = await handler.retrieve_responses(
+        response_id,
+        starting_after=starting_after,
+        stream=stream,
+    )
+
+    if isinstance(response, ErrorResponse):
+        return JSONResponse(
+            content=response.model_dump(mode="json", by_alias=True),
+            status_code=response.error.code,
+        )
+    elif isinstance(response, ResponsesResponse):
+        return JSONResponse(content=response.model_dump(mode="json", by_alias=True))
+    return StreamingResponse(
+        content=_convert_stream_to_sse_events(response), media_type="text/event-stream"
+    )
+
+
+@router.post("/v1/responses/{response_id}/cancel")
+@load_aware_call
+async def cancel_responses(response_id: str, raw_request: Request):
+    handler = responses(raw_request)
+    if handler is None:
+        raise NotImplementedError("The model does not support Responses API")
+
+    response = await handler.cancel_responses(response_id)
+
+    if isinstance(response, ErrorResponse):
+        return JSONResponse(
+            content=response.model_dump(mode="json", by_alias=True),
+            status_code=response.error.code,
+        )
+    return JSONResponse(content=response.model_dump(mode="json", by_alias=True))
+
+
+def attach_router(app: FastAPI):
+    app.include_router(router)

vllm/entrypoints/openai/responses/context.py

@@ -0,0 +1,939 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import asyncio
+import contextlib
+import copy
+import json
+import logging
+from abc import ABC, abstractmethod
+from contextlib import AsyncExitStack
+from dataclasses import replace
+from typing import TYPE_CHECKING, Any, Final, Union
+
+from openai.types.responses import ResponseFunctionToolCall, ResponseOutputItem
+from openai.types.responses.response_function_tool_call_output_item import (
+    ResponseFunctionToolCallOutputItem,
+)
+from openai.types.responses.response_output_item import McpCall
+from openai.types.responses.response_output_message import ResponseOutputMessage
+from openai.types.responses.response_output_text import ResponseOutputText
+from openai.types.responses.tool import Mcp
+from openai_harmony import Author, Message, Role, TextContent
+
+from vllm import envs
+from vllm.entrypoints.chat_utils import (
+    ChatTemplateContentFormatOption,
+)
+from vllm.entrypoints.mcp.tool import Tool
+from vllm.entrypoints.mcp.tool_server import ToolServer
+from vllm.entrypoints.openai.engine.protocol import (
+    FunctionCall,
+)
+from vllm.entrypoints.openai.parser.harmony_utils import render_for_completion
+from vllm.entrypoints.openai.responses.protocol import (
+    ResponseInputOutputItem,
+    ResponseRawMessageAndToken,
+    ResponsesRequest,
+)
+from vllm.entrypoints.openai.responses.utils import (
+    build_response_output_items,
+    construct_tool_dicts,
+)
+from vllm.entrypoints.serve.utils.constants import MCP_PREFIX
+from vllm.outputs import RequestOutput
+from vllm.parser.abstract_parser import Parser
+from vllm.tokenizers import TokenizerLike
+from vllm.utils import random_uuid
+
+if TYPE_CHECKING:
+    from mcp.client import ClientSession
+
+logger = logging.getLogger(__name__)
+
+# This is currently needed as the tool type doesn't 1:1 match the
+# tool namespace, which is what is used to look up the
+# connection to the tool server
+_TOOL_NAME_TO_TYPE_MAP = {
+    "browser": "web_search_preview",
+    "python": "code_interpreter",
+    "container": "container",
+}
+
+
+def _map_tool_name_to_tool_type(tool_name: str) -> str:
+    if tool_name not in _TOOL_NAME_TO_TYPE_MAP:
+        available_tools = ", ".join(_TOOL_NAME_TO_TYPE_MAP.keys())
+        raise ValueError(
+            f"Built-in tool name '{tool_name}' not defined in mapping. "
+            f"Available tools: {available_tools}"
+        )
+    return _TOOL_NAME_TO_TYPE_MAP[tool_name]
+
+
+class TurnMetrics:
+    """Tracks token and toolcall details for a single conversation turn."""
+
+    def __init__(
+        self,
+        input_tokens: int = 0,
+        output_tokens: int = 0,
+        cached_input_tokens: int = 0,
+        tool_output_tokens: int = 0,
+    ) -> None:
+        self.input_tokens = input_tokens
+        self.output_tokens = output_tokens
+        self.cached_input_tokens = cached_input_tokens
+        self.tool_output_tokens = tool_output_tokens
+
+    def reset(self) -> None:
+        """Reset counters for a new turn."""
+        self.input_tokens = 0
+        self.output_tokens = 0
+        self.cached_input_tokens = 0
+        self.tool_output_tokens = 0
+
+    def copy(self) -> "TurnMetrics":
+        """Create a copy of this turn's token counts."""
+        return TurnMetrics(
+            self.input_tokens,
+            self.output_tokens,
+            self.cached_input_tokens,
+            self.tool_output_tokens,
+        )
+
+
+class ConversationContext(ABC):
+    response_parser: Parser | None = None
+
+    @abstractmethod
+    def append_output(self, output: RequestOutput) -> None:
+        pass
+
+    @abstractmethod
+    def append_tool_output(self, output) -> None:
+        pass
+
+    @abstractmethod
+    async def call_tool(self) -> list[Message]:
+        pass
+
+    @abstractmethod
+    def need_builtin_tool_call(self) -> bool:
+        pass
+
+    @abstractmethod
+    def render_for_completion(self) -> list[int]:
+        pass
+
+    @abstractmethod
+    async def init_tool_sessions(
+        self,
+        tool_server: ToolServer | None,
+        exit_stack: AsyncExitStack,
+        request_id: str,
+        mcp_tools: dict[str, Mcp],
+    ) -> None:
+        pass
+
+    @abstractmethod
+    async def cleanup_session(self) -> None:
+        raise NotImplementedError("Should not be called.")
+
+
+def _create_json_parse_error_messages(
+    last_msg: Message, e: json.JSONDecodeError
+) -> list[Message]:
+    """
+    Creates an error message when json parse failed.
+    """
+    error_msg = (
+        f"Error parsing tool arguments as JSON: {str(e)}. "
+        "Please ensure the tool call arguments are valid JSON and try again."
+    )
+    content = TextContent(text=error_msg)
+    author = Author(role=Role.TOOL, name=last_msg.recipient)
+    return [
+        Message(
+            author=author,
+            content=[content],
+            recipient=Role.ASSISTANT,
+            channel=last_msg.channel,
+        )
+    ]
+
+
+class SimpleContext(ConversationContext):
+    """This is a context that cannot handle MCP tool calls"""
+
+    def __init__(
+        self,
+        *,
+        response_parser: Parser | None = None,
+        parser_cls: type[Parser] | None = None,
+        tokenizer: TokenizerLike | None = None,
+        request: ResponsesRequest | None = None,
+        chat_template_kwargs: dict[str, Any] | None = None,
+    ):
+        self.last_output = None
+        self.response_parser = response_parser or (
+            parser_cls(
+                tokenizer,
+                request.tools,
+                chat_template_kwargs=chat_template_kwargs,
+            )
+            if parser_cls is not None and tokenizer is not None and request is not None
+            else None
+        )
+
+        # Accumulated final output for streaming mode
+        self._accumulated_text: str = ""
+        self._accumulated_token_ids: list[int] = []
+        self._accumulated_logprobs: list = []
+
+        self.num_prompt_tokens = 0
+        self.num_output_tokens = 0
+        self.num_cached_tokens = 0
+        # todo num_reasoning_tokens is not implemented yet.
+        self.num_reasoning_tokens = 0
+        # not implemented yet for SimpleContext
+        self.all_turn_metrics: list[TurnMetrics] = []
+
+        self.input_messages: list[ResponseRawMessageAndToken] = []
+        self.kv_transfer_params: dict[str, Any] | None = None
+        self.ec_transfer_params: dict[str, Any] | None = None
+
+    def append_output(self, output) -> None:
+        self.last_output = output
+        if not isinstance(output, RequestOutput):
+            raise ValueError("SimpleContext only supports RequestOutput.")
+        self.num_prompt_tokens = len(output.prompt_token_ids or [])
+        self.num_cached_tokens = output.num_cached_tokens or 0
+        self.num_output_tokens += len(output.outputs[0].token_ids or [])
+        if output.kv_transfer_params is not None:
+            self.kv_transfer_params = output.kv_transfer_params
+        if output.ec_transfer_params is not None:
+            self.ec_transfer_params = output.ec_transfer_params
+
+        # Accumulate text, token_ids, and logprobs for streaming mode
+        delta_output = output.outputs[0]
+        self._accumulated_text += delta_output.text
+        self._accumulated_token_ids.extend(delta_output.token_ids)
+        if delta_output.logprobs is not None:
+            self._accumulated_logprobs.extend(delta_output.logprobs)
+
+        if len(self.input_messages) == 0:
+            output_prompt = output.prompt or ""
+            output_prompt_token_ids = output.prompt_token_ids or []
+            self.input_messages.append(
+                ResponseRawMessageAndToken(
+                    message=output_prompt,
+                    tokens=output_prompt_token_ids,
+                )
+            )
+
+    @property
+    def output_messages(self) -> list[ResponseRawMessageAndToken]:
+        """Return consolidated output as a single message.
+
+        In streaming mode, text and tokens are accumulated across many deltas.
+        This property returns them as a single entry rather than one per delta.
+        """
+        if not self._accumulated_text and not self._accumulated_token_ids:
+            return []
+        return [
+            ResponseRawMessageAndToken(
+                message=self._accumulated_text,
+                tokens=list(self._accumulated_token_ids),
+            )
+        ]
+
+    @property
+    def final_output(self) -> RequestOutput | None:
+        """Return the final output, with complete text/token_ids/logprobs."""
+        if self.last_output is not None and self.last_output.outputs:
+            assert isinstance(self.last_output, RequestOutput)
+            final_output = copy.copy(self.last_output)
+            # copy inner item to avoid modify last_output
+            final_output.outputs = [replace(item) for item in self.last_output.outputs]
+            final_output.outputs[0].text = self._accumulated_text
+            final_output.outputs[0].token_ids = tuple(self._accumulated_token_ids)
+            if self._accumulated_logprobs:
+                final_output.outputs[0].logprobs = self._accumulated_logprobs
+            return final_output
+        return self.last_output
+
+    def append_tool_output(self, output) -> None:
+        raise NotImplementedError("Should not be called.")
+
+    def need_builtin_tool_call(self) -> bool:
+        return False
+
+    async def call_tool(self) -> list[Message]:
+        raise NotImplementedError("Should not be called.")
+
+    def render_for_completion(self) -> list[int]:
+        raise NotImplementedError("Should not be called.")
+
+    async def init_tool_sessions(
+        self,
+        tool_server: ToolServer | None,
+        exit_stack: AsyncExitStack,
+        request_id: str,
+        mcp_tools: dict[str, Mcp],
+    ) -> None:
+        pass
+
+    async def cleanup_session(self) -> None:
+        raise NotImplementedError("Should not be called.")
+
+
+class ParsableContext(ConversationContext):
+    def __init__(
+        self,
+        *,
+        response_messages: list[ResponseInputOutputItem],
+        tokenizer: TokenizerLike,
+        parser_cls: type[Parser] | None,
+        request: ResponsesRequest,
+        available_tools: list[str] | None,
+        chat_template: str | None,
+        chat_template_content_format: ChatTemplateContentFormatOption,
+        response_parser: Parser | None = None,
+        enable_auto_tools: bool = False,
+    ):
+        self.num_prompt_tokens = 0
+        self.num_output_tokens = 0
+        self.num_cached_tokens = 0
+        self.num_reasoning_tokens = 0
+        # not implemented yet for ParsableContext
+        self.all_turn_metrics: list[TurnMetrics] = []
+
+        self.response_messages: list[ResponseInputOutputItem] = response_messages
+        self.num_init_messages = len(response_messages)
+        self.finish_reason: str | None = None
+        self.enable_auto_tools = enable_auto_tools
+
+        self.response_parser = response_parser or (
+            parser_cls(tokenizer, request.tools) if parser_cls is not None else None
+        )
+        self.parser_cls = parser_cls
+        self.request = request
+
+        self.available_tools = available_tools or []
+        self._tool_sessions: dict[str, ClientSession | Tool] = {}
+        self.called_tools: set[str] = set()
+
+        self.tool_dicts = construct_tool_dicts(request.tools, request.tool_choice)
+        self.chat_template = chat_template
+        self.chat_template_content_format: Final = chat_template_content_format
+
+        self.input_messages: list[ResponseRawMessageAndToken] = []
+        self.output_messages: list[ResponseRawMessageAndToken] = []
+        self._accumulated_token_ids: list[int] = []
+        self.kv_transfer_params: dict[str, Any] | None = None
+        self.ec_transfer_params: dict[str, Any] | None = None
+
+    def append_output(self, output: RequestOutput) -> None:
+        self.num_prompt_tokens = len(output.prompt_token_ids or [])
+        self.num_cached_tokens = output.num_cached_tokens or 0
+        self.num_output_tokens += len(output.outputs[0].token_ids or [])
+        if output.kv_transfer_params is not None:
+            self.kv_transfer_params = output.kv_transfer_params
+
+        if output.ec_transfer_params is not None:
+            self.ec_transfer_params = output.ec_transfer_params
+
+        completion = output.outputs[0]
+        self.finish_reason = completion.finish_reason
+
+        if self.response_parser is not None:
+            reasoning, content, tool_calls = self.response_parser.parse(
+                completion.text,
+                self.request,
+                enable_auto_tools=self.enable_auto_tools,
+                model_output_token_ids=completion.token_ids,
+            )
+            if not self.request.include_reasoning:
+                reasoning = None
+            self.response_messages.extend(
+                build_response_output_items(
+                    reasoning=reasoning,
+                    content=content,
+                    tool_calls=tool_calls,
+                    tools=self.request.tools,
+                )
+            )
+        elif completion.text:
+            self.response_messages.append(
+                ResponseOutputMessage(
+                    type="message",
+                    id=f"msg_{random_uuid()}",
+                    status="completed",
+                    role="assistant",
+                    content=[
+                        ResponseOutputText(
+                            annotations=[],
+                            type="output_text",
+                            text=completion.text,
+                            logprobs=None,
+                        )
+                    ],
+                )
+            )
+
+        self._accumulated_token_ids.extend(completion.token_ids or [])
+
+        if self.request.enable_response_messages:
+            output_prompt = output.prompt or ""
+            output_prompt_token_ids = output.prompt_token_ids or []
+            if len(self.input_messages) == 0:
+                self.input_messages.append(
+                    ResponseRawMessageAndToken(
+                        message=output_prompt,
+                        tokens=output_prompt_token_ids,
+                    )
+                )
+            else:
+                self.output_messages.append(
+                    ResponseRawMessageAndToken(
+                        message=output_prompt,
+                        tokens=output_prompt_token_ids,
+                    )
+                )
+            self.output_messages.append(
+                ResponseRawMessageAndToken(
+                    message=completion.text,
+                    tokens=completion.token_ids,
+                )
+            )
+
+    def append_tool_output(self, output: list[ResponseInputOutputItem]) -> None:
+        self.response_messages.extend(output)
+
+    def need_builtin_tool_call(self) -> bool:
+        """Return true if the last message is a builtin tool call
+        that the request has enabled."""
+        last_message = self.response_messages[-1]
+        if last_message.type != "function_call":
+            return False
+        if last_message.name in ("code_interpreter", "python"):
+            return "python" in self.available_tools
+        if last_message.name == "web_search_preview":
+            return "browser" in self.available_tools
+        if last_message.name.startswith("container"):
+            return "container" in self.available_tools
+        return False
+
+    async def call_python_tool(
+        self, tool_session: Union["ClientSession", Tool], last_msg: FunctionCall
+    ) -> list[ResponseInputOutputItem]:
+        self.called_tools.add("python")
+        if isinstance(tool_session, Tool):
+            return await tool_session.get_result_parsable_context(self)
+        args = json.loads(last_msg.arguments)
+        param = {
+            "code": args["code"],
+        }
+        result = await tool_session.call_tool("python", param)
+        result_str = result.content[0].text
+
+        message = ResponseFunctionToolCallOutputItem(
+            id=f"mcpo_{random_uuid()}",
+            type="function_call_output",
+            call_id=f"call_{random_uuid()}",
+            output=result_str,
+            status="completed",
+        )
+
+        return [message]
+
+    async def call_search_tool(
+        self, tool_session: Union["ClientSession", Tool], last_msg: FunctionCall
+    ) -> list[ResponseInputOutputItem]:
+        self.called_tools.add("browser")
+        if isinstance(tool_session, Tool):
+            return await tool_session.get_result_parsable_context(self)
+        if envs.VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY:
+            try:
+                args = json.loads(last_msg.arguments)
+            except json.JSONDecodeError as e:
+                return _create_json_parse_error_messages(last_msg, e)
+        else:
+            args = json.loads(last_msg.arguments)
+        result = await tool_session.call_tool("search", args)
+        result_str = result.content[0].text
+
+        message = ResponseFunctionToolCallOutputItem(
+            id=f"fco_{random_uuid()}",
+            type="function_call_output",
+            call_id=f"call_{random_uuid()}",
+            output=result_str,
+            status="completed",
+        )
+
+        return [message]
+
+    async def call_container_tool(
+        self, tool_session: Union["ClientSession", Tool], last_msg: Message
+    ) -> list[Message]:
+        """
+        Call container tool. Expect this to be run in a stateful docker
+        with command line terminal.
+        The official container tool would at least
+        expect the following format:
+        - for tool name: exec
+            - args:
+                {
+                    "cmd":List[str] "command to execute",
+                    "workdir":optional[str] "current working directory",
+                    "env":optional[object/dict] "environment variables",
+                    "session_name":optional[str] "session name",
+                    "timeout":optional[int] "timeout in seconds",
+                    "user":optional[str] "user name",
+                }
+        """
+        self.called_tools.add("container")
+        if isinstance(tool_session, Tool):
+            return await tool_session.get_result_parsable_context(self)
+        # tool_name = last_msg.recipient.split(".")[1].split(" ")[0]
+        if envs.VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY:
+            try:
+                args = json.loads(last_msg.arguments)
+            except json.JSONDecodeError as e:
+                return _create_json_parse_error_messages(last_msg, e)
+        else:
+            args = json.loads(last_msg.arguments)
+        result = await tool_session.call_tool("exec", args)
+        result_str = result.content[0].text
+
+        message = ResponseFunctionToolCallOutputItem(
+            id=f"fco_{random_uuid()}",
+            type="function_call_output",
+            call_id=f"call_{random_uuid()}",
+            output=result_str,
+            status="completed",
+        )
+
+        return [message]
+
+    async def call_tool(self) -> list[ResponseInputOutputItem]:
+        if not self.response_messages:
+            return []
+        last_msg = self.response_messages[-1]
+        # change this to a mcp_ function call
+        last_msg.id = f"{MCP_PREFIX}{random_uuid()}"
+        self.response_messages[-1] = last_msg
+        if last_msg.name == "code_interpreter":
+            return await self.call_python_tool(self._tool_sessions["python"], last_msg)
+        elif last_msg.name == "web_search_preview":
+            return await self.call_search_tool(self._tool_sessions["browser"], last_msg)
+        elif last_msg.name.startswith("container"):
+            return await self.call_container_tool(
+                self._tool_sessions["container"], last_msg
+            )
+        return []
+
+    def make_response_output_items(self) -> list[ResponseOutputItem]:
+        response_messages = self.response_messages[self.num_init_messages :]
+        output_messages: list[ResponseOutputItem] = []
+        for message in response_messages:
+            if not isinstance(message, ResponseFunctionToolCallOutputItem):
+                output_messages.append(message)
+            else:
+                if len(output_messages) == 0:
+                    raise ValueError(
+                        "Cannot have a FunctionToolCallOutput before FunctionToolCall."
+                    )
+                if isinstance(output_messages[-1], ResponseFunctionToolCall):
+                    output_messages[-1] = McpCall(
+                        id=f"{MCP_PREFIX}{random_uuid()}",
+                        arguments=output_messages[-1].arguments,
+                        name=output_messages[-1].name,
+                        server_label=output_messages[-1].name,
+                        type="mcp_call",
+                        status="completed",
+                        output=message.output,
+                    )
+        return output_messages
+
+    def render_for_completion(self):
+        raise NotImplementedError("Should not be called.")
+
+    async def init_tool_sessions(
+        self,
+        tool_server: ToolServer | None,
+        exit_stack: AsyncExitStack,
+        request_id: str,
+        mcp_tools: dict[str, Mcp],
+    ):
+        if tool_server:
+            for tool_name in self.available_tools:
+                if tool_name in self._tool_sessions:
+                    continue
+
+                tool_type = _map_tool_name_to_tool_type(tool_name)
+                headers = (
+                    mcp_tools[tool_type].headers if tool_type in mcp_tools else None
+                )
+                tool_session = await exit_stack.enter_async_context(
+                    tool_server.new_session(tool_name, request_id, headers)
+                )
+                self._tool_sessions[tool_name] = tool_session
+                exit_stack.push_async_exit(self.cleanup_session)
+
+    async def cleanup_session(self, *args, **kwargs) -> None:
+        """Can be used as coro to used in __aexit__"""
+
+        async def cleanup_tool_session(tool_session):
+            if not isinstance(tool_session, Tool):
+                logger.info(
+                    "Cleaning up tool session for %s", tool_session._client_info
+                )
+                with contextlib.suppress(Exception):
+                    await tool_session.call_tool("cleanup_session", {})
+
+        await asyncio.gather(
+            *(
+                cleanup_tool_session(self._tool_sessions[tool])
+                for tool in self.called_tools
+            )
+        )
+
+
+class HarmonyContext(ConversationContext):
+    def __init__(
+        self,
+        messages: list,
+        available_tools: list[str],
+        function_tool_names: frozenset[str],
+        response_parser: Parser | None = None,
+    ):
+        from vllm.parser.harmony import HarmonyParser, Segment
+
+        assert isinstance(response_parser, HarmonyParser)
+
+        self._messages = messages
+        self.response_parser: HarmonyParser = response_parser
+        self.finish_reason: str | None = None
+        self.available_tools = available_tools
+        self.function_tool_names = function_tool_names
+        self._tool_sessions: dict[str, ClientSession | Tool] = {}
+        self.called_tools: set[str] = set()
+
+        self.num_init_messages = len(messages)
+        self.num_prompt_tokens = 0
+        self.num_output_tokens = 0
+        self.num_cached_tokens = 0
+        self.num_reasoning_tokens = 0
+        self.num_tool_output_tokens = 0
+
+        self.last_append_segments: list[Segment] = []
+        self.last_append_flush_status: bool = False
+
+        # Turn tracking - replaces multiple individual tracking variables
+        self.current_turn_metrics = TurnMetrics()
+        # Track metrics for all turns
+        self.all_turn_metrics: list[TurnMetrics] = []
+        self.is_first_turn = True
+        self.first_tok_of_message = True
+        self.kv_transfer_params: dict[str, Any] | None = None
+        self.ec_transfer_params: dict[str, Any] | None = None
+
+    def append_output(self, output: RequestOutput) -> None:
+        if self.first_tok_of_message:
+            self.finish_reason = None
+            self._update_prefill_token_usage(output)
+
+        output_token_ids = output.outputs[0].token_ids
+        result = self.response_parser.process_chunk(output_token_ids)
+        segments = result.segments
+        self.num_reasoning_tokens += result.reasoning_token_count
+
+        self.first_tok_of_message = output.finished
+        self._update_decode_token_usage(output)
+        if output.kv_transfer_params is not None:
+            self.kv_transfer_params = output.kv_transfer_params
+        if output.ec_transfer_params is not None:
+            self.ec_transfer_params = output.ec_transfer_params
+
+        if output.finished:
+            self.finish_reason = output.outputs[0].finish_reason
+            flushed_segments = self.response_parser.flush()
+            if flushed_segments:
+                segments.extend(flushed_segments)
+            self.last_append_flush_status = len(flushed_segments) > 0
+            self.all_turn_metrics.append(self.current_turn_metrics.copy())
+            self.current_turn_metrics.reset()
+
+        self.last_append_segments = segments
+        self._messages.extend(
+            segment.completed_message
+            for segment in segments
+            if segment.completed_message is not None
+        )
+
+    def append_tool_output(self, output: list[Message]) -> None:
+        output_msgs = output
+        self._messages.extend(output_msgs)
+
+    def _update_prefill_token_usage(self, output: RequestOutput) -> None:
+        """Update token usage statistics for the prefill phase of generation.
+
+        The prefill phase processes the input prompt tokens. This method:
+        1. Counts the prompt tokens for this turn
+        2. Calculates tool output tokens for multi-turn conversations
+        3. Updates cached token counts
+        4. Tracks state for next turn calculations
+
+        Tool output tokens are calculated as:
+        current_prompt_tokens - last_turn_prompt_tokens -
+        last_turn_output_tokens
+        This represents tokens added between turns (typically tool responses).
+
+        Args:
+            output: The RequestOutput containing prompt token information
+        """
+        if output.prompt_token_ids is not None:
+            this_turn_input_tokens = len(output.prompt_token_ids)
+        else:
+            this_turn_input_tokens = 0
+            logger.error("RequestOutput appended contains no prompt_token_ids.")
+
+        # Update current turn input tokens
+        self.current_turn_metrics.input_tokens = this_turn_input_tokens
+        self.num_prompt_tokens += this_turn_input_tokens
+
+        # Calculate tool tokens (except on first turn)
+        if self.is_first_turn:
+            self.is_first_turn = False
+        else:
+            previous_turn = self.all_turn_metrics[-1]
+            # start counting tool after first turn
+            # tool tokens = this turn prefill - last turn prefill -
+            # last turn decode
+            this_turn_tool_tokens = (
+                self.current_turn_metrics.input_tokens
+                - previous_turn.input_tokens
+                - previous_turn.output_tokens
+            )
+
+            # Handle negative tool token counts (shouldn't happen in normal
+            # cases)
+            if this_turn_tool_tokens < 0:
+                logger.error(
+                    "Negative tool output tokens calculated: %d "
+                    "(current_input=%d, previous_input=%d, "
+                    "previous_output=%d). Setting to 0.",
+                    this_turn_tool_tokens,
+                    self.current_turn_metrics.input_tokens,
+                    previous_turn.input_tokens,
+                    previous_turn.output_tokens,
+                )
+                this_turn_tool_tokens = 0
+
+            self.num_tool_output_tokens += this_turn_tool_tokens
+            self.current_turn_metrics.tool_output_tokens = this_turn_tool_tokens
+
+        # Update cached tokens
+        num_cached_token = output.num_cached_tokens
+        if num_cached_token is not None:
+            self.num_cached_tokens += num_cached_token
+            self.current_turn_metrics.cached_input_tokens = num_cached_token
+
+    def _update_decode_token_usage(self, output: RequestOutput) -> int:
+        """Update token usage statistics for the decode phase of generation.
+
+        The decode phase processes the generated output tokens. This method:
+        1. Counts output tokens from all completion outputs
+        2. Updates the total output token count
+        3. Tracks tokens generated in the current turn
+
+        In streaming mode, this is called for each token generated.
+        In non-streaming mode, this is called once with all output tokens.
+
+        Args:
+            output: The RequestOutput containing generated token information
+
+        Returns:
+            int: Number of output tokens processed in this call
+        """
+        updated_output_token_count = 0
+        if output.outputs:
+            for completion_output in output.outputs:
+                # only keep last round
+                updated_output_token_count += len(completion_output.token_ids)
+            self.num_output_tokens += updated_output_token_count
+            self.current_turn_metrics.output_tokens += updated_output_token_count
+        return updated_output_token_count
+
+    @property
+    def messages(self) -> list:
+        return self._messages
+
+    def need_builtin_tool_call(self) -> bool:
+        last_msg = self.messages[-1]
+        recipient = last_msg.recipient
+        if recipient is None:
+            return False
+        if recipient.startswith("browser."):
+            return "browser" in self.available_tools
+        if recipient.startswith("python"):
+            return "python" in self.available_tools
+        if recipient.startswith("container."):
+            return "container" in self.available_tools
+        return False
+
+    async def call_tool(self) -> list[Message]:
+        if not self.messages:
+            return []
+        last_msg = self.messages[-1]
+        recipient = last_msg.recipient
+        if recipient is not None:
+            if recipient.startswith("browser."):
+                return await self.call_search_tool(
+                    self._tool_sessions["browser"], last_msg
+                )
+            elif recipient.startswith("python"):
+                return await self.call_python_tool(
+                    self._tool_sessions["python"], last_msg
+                )
+            elif recipient.startswith("container."):
+                return await self.call_container_tool(
+                    self._tool_sessions["container"], last_msg
+                )
+        raise ValueError("No tool call found")
+
+    def render_for_completion(self) -> list[int]:
+        return render_for_completion(self.messages)
+
+    async def call_search_tool(
+        self, tool_session: Union["ClientSession", Tool], last_msg: Message
+    ) -> list[Message]:
+        self.called_tools.add("browser")
+        if isinstance(tool_session, Tool):
+            return await tool_session.get_result(self)
+        tool_name = last_msg.recipient.split(".")[1]
+        if envs.VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY:
+            try:
+                args = json.loads(last_msg.content[0].text)
+            except json.JSONDecodeError as e:
+                return _create_json_parse_error_messages(last_msg, e)
+        else:
+            args = json.loads(last_msg.content[0].text)
+        result = await tool_session.call_tool(tool_name, args)
+        result_str = result.content[0].text
+        content = TextContent(text=result_str)
+        author = Author(role=Role.TOOL, name=last_msg.recipient)
+        return [
+            Message(
+                author=author,
+                content=[content],
+                recipient=Role.ASSISTANT,
+                channel=last_msg.channel,
+            )
+        ]
+
+    async def call_python_tool(
+        self, tool_session: Union["ClientSession", Tool], last_msg: Message
+    ) -> list[Message]:
+        self.called_tools.add("python")
+        if isinstance(tool_session, Tool):
+            return await tool_session.get_result(self)
+        param = {
+            "code": last_msg.content[0].text,
+        }
+        result = await tool_session.call_tool("python", param)
+        result_str = result.content[0].text
+
+        content = TextContent(text=result_str)
+        author = Author(role=Role.TOOL, name="python")
+
+        return [
+            Message(
+                author=author,
+                content=[content],
+                channel=last_msg.channel,
+                recipient=Role.ASSISTANT,
+            )
+        ]
+
+    async def init_tool_sessions(
+        self,
+        tool_server: ToolServer | None,
+        exit_stack: AsyncExitStack,
+        request_id: str,
+        mcp_tools: dict[str, Mcp],
+    ):
+        if tool_server:
+            for tool_name in self.available_tools:
+                if tool_name not in self._tool_sessions:
+                    tool_type = _map_tool_name_to_tool_type(tool_name)
+                    headers = (
+                        mcp_tools[tool_type].headers if tool_type in mcp_tools else None
+                    )
+                    tool_session = await exit_stack.enter_async_context(
+                        tool_server.new_session(tool_name, request_id, headers)
+                    )
+                    self._tool_sessions[tool_name] = tool_session
+                    exit_stack.push_async_exit(self.cleanup_session)
+
+    async def call_container_tool(
+        self, tool_session: Union["ClientSession", Tool], last_msg: Message
+    ) -> list[Message]:
+        """
+        Call container tool. Expect this to be run in a stateful docker
+        with command line terminal.
+        The official container tool would at least
+        expect the following format:
+        - for tool name: exec
+            - args:
+                {
+                    "cmd":List[str] "command to execute",
+                    "workdir":optional[str] "current working directory",
+                    "env":optional[object/dict] "environment variables",
+                    "session_name":optional[str] "session name",
+                    "timeout":optional[int] "timeout in seconds",
+                    "user":optional[str] "user name",
+                }
+        """
+        self.called_tools.add("container")
+        if isinstance(tool_session, Tool):
+            return await tool_session.get_result(self)
+        tool_name = last_msg.recipient.split(".")[1].split(" ")[0]
+        if envs.VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY:
+            try:
+                args = json.loads(last_msg.content[0].text)
+            except json.JSONDecodeError as e:
+                return _create_json_parse_error_messages(last_msg, e)
+        else:
+            args = json.loads(last_msg.content[0].text)
+        result = await tool_session.call_tool(tool_name, args)
+        result_str = result.content[0].text
+        content = TextContent(text=result_str)
+        author = Author(role=Role.TOOL, name=last_msg.recipient)
+        return [
+            Message(
+                author=author,
+                content=[content],
+                recipient=Role.ASSISTANT,
+                channel=last_msg.channel,
+            )
+        ]
+
+    async def cleanup_session(self, *args, **kwargs) -> None:
+        """Can be used as coro to used in __aexit__"""
+
+        async def cleanup_tool_session(tool_session):
+            if not isinstance(tool_session, Tool):
+                logger.info(
+                    "Cleaning up tool session for %s", tool_session._client_info
+                )
+                with contextlib.suppress(Exception):
+                    await tool_session.call_tool("cleanup_session", {})
+
+        await asyncio.gather(
+            *(
+                cleanup_tool_session(self._tool_sessions[tool])
+                for tool in self.called_tools
+            )
+        )

vllm/entrypoints/openai/responses/harmony.py

@@ -0,0 +1,479 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+Harmony ↔ Responses API conversion utilities.
+
+Handles two directions:
+  1. Response Input → Harmony Messages  (input parsing)
+  2. Harmony Messages → Response Output Items  (output parsing)
+"""
+
+import json
+
+from openai.types.responses import (
+    ResponseFunctionToolCall,
+    ResponseOutputItem,
+    ResponseOutputMessage,
+    ResponseOutputText,
+    ResponseReasoningItem,
+)
+from openai.types.responses.response_function_web_search import (
+    ActionFind,
+    ActionOpenPage,
+    ActionSearch,
+    ResponseFunctionWebSearch,
+)
+from openai.types.responses.response_output_item import McpCall
+from openai.types.responses.response_reasoning_item import (
+    Content as ResponseReasoningTextContent,
+)
+from openai_harmony import Author, Message, Role, TextContent
+
+from vllm.entrypoints.openai.parser.harmony_utils import (
+    BUILTIN_TOOL_TO_MCP_SERVER_LABEL,
+    extract_function_from_recipient,
+    flatten_input_text_content,
+    get_system_or_developer_message,
+    is_function_recipient,
+)
+from vllm.entrypoints.openai.responses.protocol import (
+    ResponseInputOutputItem,
+    ResponsesRequest,
+)
+from vllm.logger import init_logger
+from vllm.utils import random_uuid
+
+logger = init_logger(__name__)
+
+# ---------------------------------------------------------------------------
+# 1. Private helpers for input parsing
+# ---------------------------------------------------------------------------
+
+
+def _parse_harmony_format_message(chat_msg: dict) -> Message:
+    """Reconstruct a Message from Harmony-format dict,
+    preserving channel, recipient, and content_type."""
+    author_dict = chat_msg["author"]
+    role = author_dict.get("role")
+    name = author_dict.get("name")
+
+    raw_content = chat_msg.get("content", "")
+    if isinstance(raw_content, list):
+        # TODO: Support refusal and non-text content types.
+        contents = [TextContent(text=c.get("text", "")) for c in raw_content]
+    elif isinstance(raw_content, str):
+        contents = [TextContent(text=raw_content)]
+    else:
+        contents = [TextContent(text="")]
+
+    if name:
+        msg = Message(author=Author.new(Role(role), name), content=contents)
+    else:
+        msg = Message.from_role_and_contents(Role(role), contents)
+
+    channel = chat_msg.get("channel")
+    if channel:
+        msg = msg.with_channel(channel)
+    recipient = chat_msg.get("recipient")
+    if recipient:
+        msg = msg.with_recipient(recipient)
+    content_type = chat_msg.get("content_type")
+    if content_type:
+        msg = msg.with_content_type(content_type)
+
+    return msg
+
+
+def _parse_chat_format_message(chat_msg: dict) -> list[Message]:
+    """Parse an OpenAI chat-format dict into Harmony messages."""
+    role = chat_msg.get("role")
+    if role is None:
+        raise ValueError(f"Message has no 'role' key: {chat_msg}")
+
+    # Assistant message with tool calls
+    tool_calls = chat_msg.get("tool_calls")
+    if role == "assistant" and tool_calls:
+        msgs: list[Message] = []
+        for call in tool_calls:
+            func = call.get("function", {})
+            name = func.get("name", "")
+            arguments = func.get("arguments", "") or ""
+            msg = Message.from_role_and_content(Role.ASSISTANT, arguments)
+            msg = msg.with_channel("commentary")
+            msg = msg.with_recipient(f"functions.{name}")
+            msg = msg.with_content_type("json")
+            msgs.append(msg)
+        return msgs
+
+    # Tool role message (tool output)
+    if role == "tool":
+        name = chat_msg.get("name", "")
+        if name and not name.startswith("functions."):
+            name = f"functions.{name}"
+        content = flatten_input_text_content(chat_msg.get("content")) or ""
+        # NOTE: .with_recipient("assistant") is required on tool messages
+        # to match parse_chat_input_to_harmony_message behavior and ensure
+        # proper routing in the Harmony protocol.
+        msg = (
+            Message.from_author_and_content(Author.new(Role.TOOL, name), content)
+            .with_channel("commentary")
+            .with_recipient("assistant")
+        )
+        return [msg]
+
+    # System/developer messages into proper DeveloperContent
+    if role in ("system", "developer"):
+        text = flatten_input_text_content(chat_msg.get("content"))
+        if text:
+            msg = get_system_or_developer_message(role, text)
+            return [msg]
+        return []
+
+    # Default: user/assistant messages
+    content = chat_msg.get("content", "")
+    if isinstance(content, str):
+        contents = [TextContent(text=content)]
+    else:
+        # TODO: Support refusal.
+        contents = [TextContent(text=c.get("text", "")) for c in content]
+    msg = Message.from_role_and_contents(role, contents)
+    return [msg]
+
+
+# ---------------------------------------------------------------------------
+# 2. Public input parsing functions
+# ---------------------------------------------------------------------------
+
+
+def response_input_to_harmony(
+    response_msg: ResponseInputOutputItem,
+    prev_responses: list[ResponseOutputItem | ResponseReasoningItem],
+) -> Message | None:
+    """Convert a single ResponseInputOutputItem into a Harmony Message.
+
+    Returns None for reasoning items with empty or absent content so
+    the caller can skip them.
+    """
+    if not isinstance(response_msg, dict):
+        response_msg = response_msg.model_dump()
+    if "type" not in response_msg or response_msg["type"] == "message":
+        role = response_msg["role"]
+        content = response_msg["content"]
+        if role in ("system", "developer"):
+            text = flatten_input_text_content(content)
+            if text:
+                msg = get_system_or_developer_message(role, text)
+            else:
+                # Empty content — skip, no message emitted.
+                return None
+        elif isinstance(content, str):
+            msg = Message.from_role_and_content(role, content)
+        else:
+            contents = [TextContent(text=c.get("text", "")) for c in content]
+            msg = Message.from_role_and_contents(role, contents)
+        if role == "assistant":
+            msg = msg.with_channel("final")
+    elif response_msg["type"] == "function_call_output":
+        call_id = response_msg["call_id"]
+        call_response: ResponseFunctionToolCall | None = None
+        for prev_response in reversed(prev_responses):
+            if (
+                isinstance(prev_response, ResponseFunctionToolCall)
+                and prev_response.call_id == call_id
+            ):
+                call_response = prev_response
+                break
+        if call_response is None:
+            raise ValueError(f"No call message found for {call_id}")
+        msg = Message.from_author_and_content(
+            Author.new(Role.TOOL, f"functions.{call_response.name}"),
+            response_msg["output"],
+        )
+        msg = msg.with_channel("commentary")
+        msg = msg.with_recipient("assistant")
+    elif response_msg["type"] == "reasoning":
+        content = response_msg.get("content")
+        if content and len(content) >= 1:
+            reasoning_text = "\n".join(item["text"] for item in content)
+            msg = Message.from_role_and_content(Role.ASSISTANT, reasoning_text)
+            msg = msg.with_channel("analysis")
+        else:
+            return None
+    elif response_msg["type"] == "function_call":
+        msg = Message.from_role_and_content(Role.ASSISTANT, response_msg["arguments"])
+        msg = msg.with_channel("commentary")
+        msg = msg.with_recipient(f"functions.{response_msg['name']}")
+        msg = msg.with_content_type("json")
+    else:
+        raise ValueError(f"Unknown input type: {response_msg['type']}")
+    return msg
+
+
+def response_previous_input_to_harmony(chat_msg) -> list[Message]:
+    """Parse a message from request.previous_input_messages
+    into Harmony messages.
+
+    Supports both OpenAI chat format ({"role": "..."}) and
+    Harmony format ({"author": {"role": "..."}}).
+    """
+    if not isinstance(chat_msg, dict):
+        chat_msg = chat_msg.model_dump(exclude_none=True)
+
+    if "author" in chat_msg and isinstance(chat_msg.get("author"), dict):
+        return [_parse_harmony_format_message(chat_msg)]
+
+    return _parse_chat_format_message(chat_msg)
+
+
+def construct_harmony_previous_input_messages(
+    request: ResponsesRequest,
+) -> list[Message]:
+    """Build a Harmony message list from request.previous_input_messages.
+
+    Filters out system/developer messages to match OpenAI behavior where
+    instructions are always taken from the most recent Responses API request.
+    """
+    messages: list[Message] = []
+    if request.previous_input_messages:
+        for message in request.previous_input_messages:
+            # Handle both Message objects and dictionary inputs
+            if isinstance(message, Message):
+                message_role = message.author.role
+                if message_role == Role.SYSTEM or message_role == Role.DEVELOPER:
+                    continue
+                messages.append(message)
+            else:
+                harmony_messages = response_previous_input_to_harmony(message)
+                for harmony_msg in harmony_messages:
+                    message_role = harmony_msg.author.role
+                    if message_role == Role.SYSTEM or message_role == Role.DEVELOPER:
+                        continue
+                    messages.append(harmony_msg)
+    return messages
+
+
+# ---------------------------------------------------------------------------
+# 3. Private helpers for output parsing
+# ---------------------------------------------------------------------------
+
+
+def _parse_browser_tool_call(message: Message, recipient: str) -> ResponseOutputItem:
+    """Parse browser tool calls (search, open, find) into web search items."""
+    if len(message.content) != 1:
+        raise ValueError("Invalid number of contents in browser message")
+    content = message.content[0]
+
+    # Parse JSON args (with retry detection)
+    try:
+        browser_call = json.loads(content.text)
+    except json.JSONDecodeError:
+        logger.warning(
+            "Invalid JSON in browser tool call, using error placeholder: %s",
+            content.text,
+        )
+        json_retry_output_message = (
+            f"Invalid JSON args, caught and retried: {content.text}"
+        )
+        browser_call = {
+            "query": json_retry_output_message,
+            "url": json_retry_output_message,
+            "pattern": json_retry_output_message,
+        }
+
+    # Create appropriate action based on recipient
+    if recipient == "browser.search":
+        action = ActionSearch(
+            query=f"cursor:{browser_call.get('query', '')}", type="search"
+        )
+    elif recipient == "browser.open":
+        action = ActionOpenPage(
+            url=f"cursor:{browser_call.get('url', '')}", type="open_page"
+        )
+    elif recipient == "browser.find":
+        action = ActionFind(
+            pattern=browser_call.get("pattern", ""),
+            url=f"cursor:{browser_call.get('url', '')}",
+            type="find",
+        )
+    else:
+        raise ValueError(f"Unknown browser action: {recipient}")
+
+    return ResponseFunctionWebSearch(
+        id=f"ws_{random_uuid()}",
+        action=action,
+        status="completed",
+        type="web_search_call",
+    )
+
+
+def _parse_function_call(
+    message: Message, recipient: str, incomplete: bool = False
+) -> list[ResponseOutputItem]:
+    """Parse function calls into function tool call items."""
+    function_name = extract_function_from_recipient(recipient)
+    output_items = []
+    for content in message.content:
+        random_id = random_uuid()
+        response_item = ResponseFunctionToolCall(
+            arguments=content.text,
+            call_id=f"call_{random_id}",
+            type="function_call",
+            name=function_name,
+            id=f"fc_{random_id}",
+            status="incomplete" if incomplete else "completed",
+        )
+        output_items.append(response_item)
+    return output_items
+
+
+def _parse_reasoning(message: Message) -> list[ResponseOutputItem]:
+    """Parse reasoning/analysis content into reasoning items."""
+    output_items = []
+    for content in message.content:
+        reasoning_item = ResponseReasoningItem(
+            id=f"rs_{random_uuid()}",
+            summary=[],
+            type="reasoning",
+            content=[
+                ResponseReasoningTextContent(text=content.text, type="reasoning_text")
+            ],
+            status=None,
+        )
+        output_items.append(reasoning_item)
+    return output_items
+
+
+def _parse_final_message(
+    message: Message, incomplete: bool = False
+) -> ResponseOutputItem:
+    """Parse final channel messages into output message items."""
+    contents = []
+    for content in message.content:
+        output_text = ResponseOutputText(
+            text=content.text,
+            annotations=[],  # TODO
+            type="output_text",
+            logprobs=None,  # TODO
+        )
+        contents.append(output_text)
+    return ResponseOutputMessage(
+        id=f"msg_{random_uuid()}",
+        content=contents,
+        role=message.author.role,
+        status="incomplete" if incomplete else "completed",
+        type="message",
+    )
+
+
+def _parse_mcp_recipient(recipient: str) -> tuple[str, str]:
+    """Parse MCP recipient into (server_label, tool_name).
+
+    For dotted recipients like "repo_browser.list":
+        - server_label: "repo_browser" (namespace/server)
+        - tool_name: "list" (specific tool)
+
+    For simple recipients like "filesystem":
+        - server_label: "filesystem"
+        - tool_name: "filesystem"
+    """
+    if "." in recipient:
+        server_label = recipient.split(".")[0]
+        tool_name = recipient.split(".")[-1]
+    else:
+        server_label = recipient
+        tool_name = recipient
+    return server_label, tool_name
+
+
+def _parse_mcp_call(
+    message: Message, recipient: str, incomplete: bool = False
+) -> list[ResponseOutputItem]:
+    """Parse MCP calls into MCP call items."""
+    # Handle built-in tools that need server_label mapping
+    if recipient in BUILTIN_TOOL_TO_MCP_SERVER_LABEL:
+        server_label = BUILTIN_TOOL_TO_MCP_SERVER_LABEL[recipient]
+        tool_name = recipient
+    else:
+        server_label, tool_name = _parse_mcp_recipient(recipient)
+
+    output_items = []
+    for content in message.content:
+        response_item = McpCall(
+            arguments=content.text,
+            type="mcp_call",
+            name=tool_name,
+            server_label=server_label,
+            id=f"mcp_{random_uuid()}",
+            status="incomplete" if incomplete else "completed",
+        )
+        output_items.append(response_item)
+    return output_items
+
+
+def _parse_message_no_recipient(
+    message: Message,
+    incomplete: bool = False,
+) -> list[ResponseOutputItem]:
+    """Parse a Harmony message with no recipient based on its channel."""
+    if message.channel == "analysis":
+        return _parse_reasoning(message)
+
+    if message.channel in ("commentary", "final"):
+        # Per Harmony format, preambles (commentary with no recipient) and
+        # final channel content are both intended to be shown to end-users.
+        # See: https://cookbook.openai.com/articles/openai-harmony
+        return [_parse_final_message(message, incomplete=incomplete)]
+
+    raise ValueError(f"Unknown channel: {message.channel}")
+
+
+# ---------------------------------------------------------------------------
+# 4. Public output parsing functions
+# ---------------------------------------------------------------------------
+
+
+def harmony_to_response_output(
+    message: Message,
+    function_tool_names: frozenset[str],
+    incomplete: bool = False,
+) -> list[ResponseOutputItem]:
+    """Parse a Harmony message into a list of output response items.
+
+    This is the main dispatcher that routes based on channel and recipient.
+    """
+    if message.author.role != "assistant":
+        # This is a message from a tool to the assistant (e.g., search result).
+        # Don't include it in the final output for now. This aligns with
+        # OpenAI's behavior on models like o4-mini.
+        return []
+
+    output_items: list[ResponseOutputItem] = []
+    recipient = message.recipient
+
+    if recipient is not None:
+        # Browser tool calls (browser.search, browser.open, browser.find)
+        if recipient.startswith("browser."):
+            if not incomplete:
+                output_items.append(_parse_browser_tool_call(message, recipient))
+
+        # Function calls (with or without "functions." prefix)
+        elif is_function_recipient(recipient, function_tool_names):
+            output_items.extend(
+                _parse_function_call(message, recipient, incomplete=incomplete)
+            )
+
+        # Built-in MCP tools (python, browser, container)
+        elif recipient in BUILTIN_TOOL_TO_MCP_SERVER_LABEL:
+            output_items.extend(_parse_reasoning(message))
+
+        # All other recipients are MCP calls
+        else:
+            output_items.extend(
+                _parse_mcp_call(message, recipient, incomplete=incomplete)
+            )
+
+    # No recipient - handle based on channel for non-tool messages
+    else:
+        output_items.extend(_parse_message_no_recipient(message, incomplete=incomplete))
+
+    return output_items

vllm/entrypoints/openai/responses/protocol.py

@@ -0,0 +1,859 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+# Adapted from
+# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
+import time
+from typing import Any, Literal, TypeAlias
+
+from openai.types.responses import (
+    ResponseCodeInterpreterCallCodeDeltaEvent,
+    ResponseCodeInterpreterCallCodeDoneEvent,
+    ResponseCodeInterpreterCallCompletedEvent,
+    ResponseCodeInterpreterCallInProgressEvent,
+    ResponseCodeInterpreterCallInterpretingEvent,
+    ResponseContentPartAddedEvent,
+    ResponseContentPartDoneEvent,
+    ResponseFunctionToolCall,
+    ResponseInputItemParam,
+    ResponseMcpCallArgumentsDeltaEvent,
+    ResponseMcpCallArgumentsDoneEvent,
+    ResponseMcpCallCompletedEvent,
+    ResponseMcpCallInProgressEvent,
+    ResponseOutputItem,
+    ResponseOutputItemAddedEvent,
+    ResponseOutputItemDoneEvent,
+    ResponseOutputMessage,
+    ResponsePrompt,
+    ResponseReasoningItem,
+    ResponseReasoningTextDeltaEvent,
+    ResponseReasoningTextDoneEvent,
+    ResponseStatus,
+    ResponseTextConfig,
+    ResponseWebSearchCallCompletedEvent,
+    ResponseWebSearchCallInProgressEvent,
+    ResponseWebSearchCallSearchingEvent,
+)
+from openai.types.responses import (
+    ResponseCompletedEvent as OpenAIResponseCompletedEvent,
+)
+from openai.types.responses import ResponseCreatedEvent as OpenAIResponseCreatedEvent
+from openai.types.responses import (
+    ResponseInProgressEvent as OpenAIResponseInProgressEvent,
+)
+from openai.types.responses.response import IncompleteDetails, ToolChoice
+from openai.types.responses.response_reasoning_item import (
+    Content as ResponseReasoningTextContent,
+)
+from openai.types.responses.tool import Tool
+from openai.types.shared import Metadata, Reasoning
+from openai_harmony import Message as OpenAIHarmonyMessage
+from pydantic import (
+    Field,
+    ValidationError,
+    field_serializer,
+    model_validator,
+)
+
+from vllm.config import ModelConfig
+from vllm.entrypoints.chat_utils import (
+    ChatCompletionMessageParam,
+    ChatTemplateContentFormatOption,
+)
+from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel
+from vllm.exceptions import VLLMValidationError
+from vllm.logger import init_logger
+from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
+from vllm.sampling_params import (
+    RequestOutputKind,
+    SamplingParams,
+    StructuredOutputsParams,
+)
+from vllm.utils import random_uuid
+
+logger = init_logger(__name__)
+
+_INT64_MIN = -(2**63)
+_INT64_MAX = 2**63 - 1
+
+
+class InputTokensDetails(OpenAIBaseModel):
+    cached_tokens: int
+    input_tokens_per_turn: list[int] = Field(default_factory=list)
+    cached_tokens_per_turn: list[int] = Field(default_factory=list)
+
+
+class OutputTokensDetails(OpenAIBaseModel):
+    reasoning_tokens: int = 0
+    tool_output_tokens: int = 0
+    output_tokens_per_turn: list[int] = Field(default_factory=list)
+    tool_output_tokens_per_turn: list[int] = Field(default_factory=list)
+
+
+class ResponseUsage(OpenAIBaseModel):
+    input_tokens: int
+    input_tokens_details: InputTokensDetails
+    output_tokens: int
+    output_tokens_details: OutputTokensDetails
+    total_tokens: int
+
+
+def serialize_message(msg):
+    """
+    Serializes a single message
+    """
+    if isinstance(msg, dict):
+        return msg
+    elif hasattr(msg, "to_dict"):
+        return msg.to_dict()
+    else:
+        # fallback to pydantic dump
+        return msg.model_dump(mode="json", by_alias=True)
+
+
+def serialize_messages(msgs):
+    """
+    Serializes multiple messages
+    """
+    return [serialize_message(msg) for msg in msgs] if msgs else None
+
+
+class ResponseRawMessageAndToken(OpenAIBaseModel):
+    """Class to show the raw message.
+    If message / tokens diverge, tokens is the source of truth"""
+
+    message: str
+    tokens: list[int]
+    type: Literal["raw_message_tokens"] = "raw_message_tokens"
+
+
+ResponseInputOutputMessage: TypeAlias = (
+    list[ChatCompletionMessageParam] | list[ResponseRawMessageAndToken]
+)
+ResponseInputOutputItem: TypeAlias = ResponseInputItemParam | ResponseOutputItem
+
+
+class ResponsesRequest(OpenAIBaseModel):
+    # Ordered by official OpenAI API documentation
+    # https://platform.openai.com/docs/api-reference/responses/create
+    background: bool | None = False
+    include: (
+        list[
+            Literal[
+                "code_interpreter_call.outputs",
+                "computer_call_output.output.image_url",
+                "file_search_call.results",
+                "message.input_image.image_url",
+                "message.output_text.logprobs",
+                "reasoning.encrypted_content",
+            ],
+        ]
+        | None
+    ) = None
+    input: str | list[ResponseInputOutputItem]
+    instructions: str | None = None
+    max_output_tokens: int | None = None
+    max_tool_calls: int | None = None
+    metadata: Metadata | None = None
+    model: str | None = None
+    logit_bias: dict[str, float] | None = None
+    parallel_tool_calls: bool | None = True
+    previous_response_id: str | None = None
+    prompt: ResponsePrompt | None = None
+    reasoning: Reasoning | None = None
+    include_reasoning: bool = Field(
+        default=True,
+        description=(
+            "Whether to include reasoning content in the response. "
+            "When false, reasoning tokens are still generated but "
+            "excluded from the output. This reduces network traffic "
+            "without affecting model inference."
+        ),
+    )
+    service_tier: Literal["auto", "default", "flex", "scale", "priority"] = "auto"
+    store: bool | None = True
+    stream: bool | None = False
+    temperature: float | None = None
+    text: ResponseTextConfig | None = None
+    tool_choice: ToolChoice = "auto"
+    tools: list[Tool] = Field(default_factory=list)
+    top_logprobs: int | None = 0
+    top_p: float | None = None
+    top_k: int | None = None
+    truncation: Literal["auto", "disabled"] | None = "disabled"
+    user: str | None = None
+    skip_special_tokens: bool = True
+    include_stop_str_in_output: bool = False
+    presence_penalty: float | None = Field(
+        default=None,
+        ge=-2.0,
+        le=2.0,
+        description=(
+            "The presence penalty that was used to penalize new tokens based on "
+            "whether they appear in the text so far."
+        ),
+    )
+    frequency_penalty: float | None = Field(
+        default=None,
+        ge=-2.0,
+        le=2.0,
+        description=(
+            "The frequency penalty that was used to penalize new tokens based on "
+            "their frequency in the text so far."
+        ),
+    )
+    prompt_cache_key: str | None = Field(
+        default=None,
+        description=(
+            "A key that was used to read from or write to the prompt cache."
+            "Note: This field has not been implemented yet "
+            "and vLLM will ignore it."
+        ),
+    )
+
+    # --8<-- [start:responses-extra-params]
+    request_id: str = Field(
+        default_factory=lambda: f"resp_{random_uuid()}",
+        description=(
+            "The request_id related to this request. If the caller does "
+            "not set it, a random_uuid will be generated. This id is used "
+            "through out the inference process and return in response."
+        ),
+    )
+    media_io_kwargs: dict[str, dict[str, Any]] | None = Field(
+        default=None,
+        description=(
+            "Additional kwargs to pass to the media IO connectors, "
+            "keyed by modality. Merged with engine-level media_io_kwargs."
+        ),
+    )
+    mm_processor_kwargs: dict[str, Any] | None = Field(
+        default=None,
+        description=("Additional kwargs to pass to the HF processor."),
+    )
+    priority: int = Field(
+        default=0,
+        ge=_INT64_MIN,
+        le=_INT64_MAX,
+        description=(
+            "The priority of the request (lower means earlier handling; "
+            "default: 0). Any priority other than 0 will raise an error "
+            "if the served model does not use priority scheduling."
+        ),
+    )
+    cache_salt: str | None = Field(
+        default=None,
+        description=(
+            "If specified, the prefix cache will be salted with the provided "
+            "string to prevent an attacker to guess prompts in multi-user "
+            "environments. The salt should be random, protected from "
+            "access by 3rd parties, and long enough to be "
+            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
+            "to 256 bit)."
+        ),
+    )
+
+    enable_response_messages: bool = Field(
+        default=False,
+        description=(
+            "Dictates whether or not to return messages as part of the "
+            "response object. Currently only supported for non-background."
+        ),
+    )
+    # similar to input_messages / output_messages in ResponsesResponse
+    # we take in previous_input_messages (ie in harmony format)
+    # this cannot be used in conjunction with previous_response_id
+    # TODO: consider supporting non harmony messages as well
+    previous_input_messages: list[OpenAIHarmonyMessage | dict] | None = None
+    structured_outputs: StructuredOutputsParams | None = Field(
+        default=None,
+        description="Additional kwargs for structured outputs",
+    )
+
+    repetition_penalty: float | None = None
+    seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
+    stop: str | list[str] | None = []
+    ignore_eos: bool = False
+    vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
+        default=None,
+        description=(
+            "Additional request parameters with (list of) string or "
+            "numeric values, used by custom extensions."
+        ),
+    )
+    kv_transfer_params: dict[str, Any] | None = Field(
+        default=None,
+        description="KVTransfer parameters used for disaggregated serving.",
+    )
+    ec_transfer_params: dict[str, Any] | None = Field(
+        default=None,
+        description=(
+            "ECTransfer parameters used for encoder-cache disaggregated serving."
+        ),
+    )
+    chat_template_kwargs: dict[str, Any] | None = Field(
+        default=None,
+        description=(
+            "Additional keyword args to pass to the chat template renderer. "
+            "Will be accessible by the template."
+        ),
+    )
+    # --8<-- [end:responses-extra-params]
+
+    def build_chat_params(
+        self,
+        default_template: str | None,
+        default_template_content_format: ChatTemplateContentFormatOption,
+    ) -> ChatParams:
+        from .utils import should_continue_final_message
+
+        # Check if we should continue the final message (partial completion)
+        # This enables Anthropic-style partial message completion where the
+        # user provides an incomplete assistant message to continue from.
+        continue_final = should_continue_final_message(self.input)
+
+        reasoning = self.reasoning
+        reasoning_effort = None if reasoning is None else reasoning.effort
+
+        extra_kwargs: dict[str, Any] = dict(
+            add_generation_prompt=not continue_final,
+            continue_final_message=continue_final,
+            reasoning_effort=reasoning_effort,
+        )
+
+        # When reasoning is requested, activate thinking for models whose
+        # chat templates require explicit opt-in (e.g., Gemma4 defaults
+        # enable_thinking to false). For templates that don't declare the
+        # variable, resolve_chat_template_kwargs filters it out harmlessly.
+        user_kwargs = self.chat_template_kwargs or {}
+        if reasoning_effort is not None and "enable_thinking" not in user_kwargs:
+            extra_kwargs["enable_thinking"] = reasoning_effort != "none"
+
+        return ChatParams(
+            chat_template=default_template,
+            chat_template_content_format=default_template_content_format,
+            chat_template_kwargs=merge_kwargs(
+                self.chat_template_kwargs,
+                extra_kwargs,
+            ),
+            media_io_kwargs=self.media_io_kwargs,
+        )
+
+    def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
+        return TokenizeParams(
+            max_total_tokens=model_config.max_model_len,
+            max_output_tokens=self.max_output_tokens or 0,
+            truncate_prompt_tokens=-1 if self.truncation != "disabled" else None,
+            max_total_tokens_param="max_model_len",
+            max_output_tokens_param="max_output_tokens",
+        )
+
+    _DEFAULT_SAMPLING_PARAMS = {
+        "temperature": 1.0,
+        "top_p": 1.0,
+        "top_k": 0,
+    }
+
+    def to_sampling_params(
+        self,
+        default_max_tokens: int,
+        default_sampling_params: dict | None = None,
+    ) -> SamplingParams:
+        if self.max_output_tokens is None:
+            max_tokens = default_max_tokens
+        else:
+            max_tokens = min(self.max_output_tokens, default_max_tokens)
+
+        default_sampling_params = default_sampling_params or {}
+        if (temperature := self.temperature) is None:
+            temperature = default_sampling_params.get(
+                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
+            )
+        if (top_p := self.top_p) is None:
+            top_p = default_sampling_params.get(
+                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
+            )
+        if (top_k := self.top_k) is None:
+            top_k = default_sampling_params.get(
+                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
+            )
+
+        if (repetition_penalty := self.repetition_penalty) is None:
+            repetition_penalty = default_sampling_params.get("repetition_penalty", 1.0)
+
+        if (presence_penalty := self.presence_penalty) is None:
+            presence_penalty = default_sampling_params.get("presence_penalty", 0.0)
+
+        if (frequency_penalty := self.frequency_penalty) is None:
+            frequency_penalty = default_sampling_params.get("frequency_penalty", 0.0)
+
+        # Structured output
+        structured_outputs = self.structured_outputs
+
+        # Also check text.format for OpenAI-style json_schema
+        if self.text is not None and self.text.format is not None:
+            if structured_outputs is not None:
+                raise VLLMValidationError(
+                    "Cannot specify both structured_outputs and text.format",
+                    parameter="structured_outputs",
+                )
+            response_format = self.text.format
+            if (
+                response_format.type == "json_schema"
+                and response_format.schema_ is not None
+            ):
+                structured_outputs = StructuredOutputsParams(
+                    json=response_format.schema_  # type: ignore[call-arg]
+                    # --follow-imports skip hides the class definition but also hides
+                    # multiple third party conflicts, so best of both evils
+                )
+
+        stop = self.stop if self.stop else []
+        if isinstance(stop, str):
+            stop = [stop]
+
+        extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
+        if self.kv_transfer_params:
+            extra_args["kv_transfer_params"] = self.kv_transfer_params
+        if self.ec_transfer_params:
+            extra_args["ec_transfer_params"] = self.ec_transfer_params
+
+        return SamplingParams.from_optional(
+            temperature=temperature,
+            top_p=top_p,
+            top_k=top_k,
+            max_tokens=max_tokens,
+            logprobs=self.top_logprobs if self.is_include_output_logprobs() else None,
+            stop=stop,
+            frequency_penalty=frequency_penalty,
+            presence_penalty=presence_penalty,
+            repetition_penalty=repetition_penalty,
+            seed=self.seed,
+            ignore_eos=self.ignore_eos,
+            output_kind=(
+                RequestOutputKind.DELTA if self.stream else RequestOutputKind.FINAL_ONLY
+            ),
+            structured_outputs=structured_outputs,
+            logit_bias=self.logit_bias,
+            extra_args=extra_args,
+            skip_clone=True,  # Created fresh per request, safe to skip clone
+            skip_special_tokens=self.skip_special_tokens,
+            include_stop_str_in_output=self.include_stop_str_in_output,
+        )
+
+    def is_include_output_logprobs(self) -> bool:
+        """Check if the request includes output logprobs."""
+        if self.include is None:
+            return False
+        return (
+            isinstance(self.include, list)
+            and "message.output_text.logprobs" in self.include
+        )
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_background(cls, data):
+        if not data.get("background"):
+            return data
+        if not data.get("store", True):
+            raise VLLMValidationError(
+                "background can only be used when `store` is true",
+                parameter="background",
+            )
+        return data
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_prompt(cls, data):
+        if data.get("prompt") is not None:
+            raise VLLMValidationError(
+                "prompt template is not supported", parameter="prompt"
+            )
+        return data
+
+    @model_validator(mode="before")
+    @classmethod
+    def check_cache_salt_support(cls, data):
+        if data.get("cache_salt") is not None and (
+            not isinstance(data["cache_salt"], str) or not data["cache_salt"]
+        ):
+            raise VLLMValidationError(
+                "Parameter 'cache_salt' must be a non-empty string if provided.",
+                parameter="cache_salt",
+            )
+        return data
+
+    @model_validator(mode="before")
+    @classmethod
+    def input_item_parsing(cls, data):
+        """Parse input items that are missing required fields or that Pydantic
+        cannot disambiguate in a Union of TypedDict / BaseModel types.
+
+        Specifically handles:
+        - function_call -> ResponseFunctionToolCall
+        - reasoning     -> ResponseReasoningItem (auto-generates id)
+        - message(role=assistant) -> ResponseOutputMessage (auto-generates
+          id/status and annotations)
+
+        Invalid structures are left for Pydantic to reject.
+        """
+        input_data = data.get("input")
+
+        # Early return for None, strings, or bytes
+        if input_data is None or isinstance(input_data, (str, bytes)):
+            return data
+
+        # Convert iterators (like ValidatorIterator) to list
+        if not isinstance(input_data, list):
+            try:
+                input_data = list(input_data)
+            except TypeError:
+                # Not iterable, leave as-is for Pydantic to handle
+                return data
+
+        processed_input = []
+        for item in input_data:
+            if not isinstance(item, dict):
+                processed_input.append(item)
+                continue
+
+            item_type = item.get("type")
+
+            if item_type == "function_call":
+                try:
+                    processed_input.append(ResponseFunctionToolCall(**item))
+                except ValidationError:
+                    logger.debug(
+                        "Failed to parse function_call to ResponseFunctionToolCall, "
+                        "leaving for Pydantic validation"
+                    )
+                    processed_input.append(item)
+
+            elif item_type == "reasoning":
+                if "id" not in item:
+                    item = {**item, "id": f"rs_{random_uuid()}"}
+                try:
+                    processed_input.append(ResponseReasoningItem(**item))
+                except ValidationError:
+                    logger.debug(
+                        "Failed to parse reasoning to ResponseReasoningItem, "
+                        "leaving for Pydantic validation"
+                    )
+                    processed_input.append(item)
+
+            elif item_type == "message" and item.get("role") == "assistant":
+                content = item.get("content")
+                if not isinstance(content, list):
+                    # String content is a valid EasyInputMessageParam,
+                    # do not coerce it to ResponseOutputMessage
+                    processed_input.append(item)
+                    continue
+
+                original_item = item
+                item = dict(item)
+                if "id" not in item:
+                    item["id"] = f"msg_{random_uuid()}"
+                if "status" not in item:
+                    item["status"] = "completed"
+                # ResponseOutputText requires annotations
+                new_content = []
+                for c in content:
+                    if (
+                        isinstance(c, dict)
+                        and c.get("type") == "output_text"
+                        and "annotations" not in c
+                    ):
+                        c = {**c, "annotations": []}
+                    new_content.append(c)
+                item["content"] = new_content
+                try:
+                    processed_input.append(ResponseOutputMessage(**item))
+                except ValidationError:
+                    logger.debug(
+                        "Failed to parse assistant message to ResponseOutputMessage, "
+                        "leaving for Pydantic validation"
+                    )
+                    processed_input.append(original_item)
+
+            else:
+                processed_input.append(item)
+
+        data["input"] = processed_input
+        return data
+
+    @model_validator(mode="before")
+    @classmethod
+    def check_tool_usage(cls, data):
+        if not isinstance(data, dict):
+            return data
+
+        tools = data.get("tools")
+        tool_choice = data.get("tool_choice", "auto")
+        has_tools = tools is not None and len(tools) > 0
+        is_named_tool_choice = (
+            isinstance(tool_choice, dict) and tool_choice.get("type") == "function"
+        )
+
+        if not has_tools:
+            if tool_choice in ("auto", "none"):
+                data["tool_choice"] = "none"
+            elif tool_choice == "required":
+                raise VLLMValidationError(
+                    "Tool choice 'required' must be specified with 'tools' parameter.",
+                    parameter="tool_choice",
+                )
+            elif is_named_tool_choice:
+                raise VLLMValidationError(
+                    "Tool choice 'function' not found in 'tools' parameter.",
+                    parameter="tool_choice",
+                )
+        elif is_named_tool_choice and tools is not None:
+            tool_name = tool_choice.get("name")
+            tool_names = set()
+            for tool in tools:
+                if isinstance(tool, dict):
+                    if tool.get("type") == "namespace":
+                        namespace = tool.get("name")
+                        for namespaced_tool in tool.get("tools", []):
+                            namespaced_name = namespaced_tool.get("name")
+                            tool_names.add(namespaced_name)
+                            tool_names.add(f"{namespace}__{namespaced_name}")
+                    else:
+                        tool_names.add(tool.get("name"))
+                else:
+                    tool_names.add(getattr(tool, "name", None))
+            if not tool_name or tool_name not in tool_names:
+                raise VLLMValidationError(
+                    "Tool choice 'function' not found in 'tools' parameter.",
+                    parameter="tool_choice",
+                )
+
+        return data
+
+
+class ResponsesResponse(OpenAIBaseModel):
+    id: str = Field(default_factory=lambda: f"resp_{random_uuid()}")
+    created_at: int = Field(default_factory=lambda: int(time.time()))
+    # error: Optional[ResponseError] = None
+    incomplete_details: IncompleteDetails | None = None
+    instructions: str | None = None
+    metadata: Metadata | None = None
+    model: str
+    object: Literal["response"] = "response"
+    output: list[ResponseOutputItem]
+    parallel_tool_calls: bool
+    temperature: float
+    tool_choice: ToolChoice
+    tools: list[Tool]
+    top_p: float
+    background: bool
+    max_output_tokens: int
+    max_tool_calls: int | None = None
+    previous_response_id: str | None = None
+    prompt: ResponsePrompt | None = None
+    reasoning: Reasoning | None = None
+    service_tier: Literal["auto", "default", "flex", "scale", "priority"]
+    status: ResponseStatus
+    text: ResponseTextConfig | None = None
+    top_logprobs: int | None = None
+    truncation: Literal["auto", "disabled"]
+    usage: ResponseUsage | None = None
+    user: str | None = None
+
+    presence_penalty: float | None = Field(
+        default=None,
+        ge=-2.0,
+        le=2.0,
+        description=(
+            "The presence penalty that was used to penalize new tokens based on "
+            "whether they appear in the text so far."
+        ),
+    )
+    frequency_penalty: float | None = Field(
+        default=None,
+        ge=-2.0,
+        le=2.0,
+        description=(
+            "The frequency penalty that was used to penalize new tokens based on "
+            "their frequency in the text so far."
+        ),
+    )
+
+    # vLLM-specific fields that are not in OpenAI spec
+    kv_transfer_params: dict[str, Any] | None = Field(
+        default=None, description="KVTransfer parameters."
+    )
+    ec_transfer_params: dict[str, Any] | None = Field(
+        default=None, description="ECTransfer parameters."
+    )
+
+    # --8<-- [start:responses-response-extra-params]
+    # These are populated when enable_response_messages is set to True
+    # NOTE: custom serialization is needed
+    # see serialize_input_messages and serialize_output_messages
+    input_messages: ResponseInputOutputMessage | None = Field(
+        default=None,
+        description=(
+            "If enable_response_messages, we can show raw token input to model."
+        ),
+    )
+    output_messages: ResponseInputOutputMessage | None = Field(
+        default=None,
+        description=(
+            "If enable_response_messages, we can show raw token output of model."
+        ),
+    )
+    # --8<-- [end:responses-response-extra-params]
+
+    # NOTE: openAI harmony doesn't serialize TextContent properly,
+    # TODO: this fixes for TextContent, but need to verify for tools etc
+    # https://github.com/openai/harmony/issues/78
+    @field_serializer("output_messages", when_used="json")
+    def serialize_output_messages(self, msgs, _info):
+        return serialize_messages(msgs)
+
+    # NOTE: openAI harmony doesn't serialize TextContent properly, this fixes it
+    # https://github.com/openai/harmony/issues/78
+    @field_serializer("input_messages", when_used="json")
+    def serialize_input_messages(self, msgs, _info):
+        return serialize_messages(msgs)
+
+    @classmethod
+    def from_request(
+        cls,
+        request: ResponsesRequest,
+        sampling_params: SamplingParams,
+        model_name: str,
+        created_time: int,
+        output: list[ResponseOutputItem],
+        status: ResponseStatus,
+        usage: ResponseUsage | None = None,
+        input_messages: ResponseInputOutputMessage | None = None,
+        output_messages: ResponseInputOutputMessage | None = None,
+        kv_transfer_params: dict[str, Any] | None = None,
+        ec_transfer_params: dict[str, Any] | None = None,
+    ) -> "ResponsesResponse":
+        incomplete_details: IncompleteDetails | None = None
+        if status == "incomplete":
+            incomplete_details = IncompleteDetails(reason="max_output_tokens")
+        # TODO: implement the other reason for incomplete_details,
+        # which is content_filter
+        # incomplete_details = IncompleteDetails(reason='content_filter')
+        return cls(
+            id=request.request_id,
+            created_at=created_time,
+            incomplete_details=incomplete_details,
+            instructions=request.instructions,
+            metadata=request.metadata,
+            model=model_name,
+            output=output,
+            input_messages=input_messages,
+            output_messages=output_messages,
+            parallel_tool_calls=request.parallel_tool_calls,
+            temperature=sampling_params.temperature,
+            tool_choice=request.tool_choice,
+            tools=request.tools,
+            top_p=sampling_params.top_p,
+            background=request.background,
+            max_output_tokens=sampling_params.max_tokens,
+            max_tool_calls=request.max_tool_calls,
+            previous_response_id=request.previous_response_id,
+            prompt=request.prompt,
+            reasoning=request.reasoning,
+            service_tier=request.service_tier,
+            presence_penalty=sampling_params.presence_penalty,
+            frequency_penalty=sampling_params.frequency_penalty,
+            status=status,
+            text=request.text,
+            top_logprobs=sampling_params.logprobs,
+            truncation=request.truncation,
+            user=request.user,
+            usage=usage,
+            kv_transfer_params=kv_transfer_params,
+            ec_transfer_params=ec_transfer_params,
+        )
+
+
+# TODO: this code can be removed once
+# https://github.com/openai/openai-python/issues/2634 has been resolved
+class ResponseReasoningPartDoneEvent(OpenAIBaseModel):
+    content_index: int
+    """The index of the content part that is done."""
+
+    item_id: str
+    """The ID of the output item that the content part was added to."""
+
+    output_index: int
+    """The index of the output item that the content part was added to."""
+
+    part: ResponseReasoningTextContent
+    """The content part that is done."""
+
+    sequence_number: int
+    """The sequence number of this event."""
+
+    type: Literal["response.reasoning_part.done"]
+    """The type of the event. Always `response.reasoning_part.done`."""
+
+
+# TODO: this code can be removed once
+# https://github.com/openai/openai-python/issues/2634 has been resolved
+class ResponseReasoningPartAddedEvent(OpenAIBaseModel):
+    content_index: int
+    """The index of the content part that is done."""
+
+    item_id: str
+    """The ID of the output item that the content part was added to."""
+
+    output_index: int
+    """The index of the output item that the content part was added to."""
+
+    part: ResponseReasoningTextContent
+    """The content part that is done."""
+
+    sequence_number: int
+    """The sequence number of this event."""
+
+    type: Literal["response.reasoning_part.added"]
+    """The type of the event. Always `response.reasoning_part.added`."""
+
+
+# vLLM Streaming Events
+# Note: we override the response type with the vLLM ResponsesResponse type
+class ResponseCompletedEvent(OpenAIResponseCompletedEvent):
+    response: ResponsesResponse  # type: ignore[override]
+
+
+class ResponseCreatedEvent(OpenAIResponseCreatedEvent):
+    response: ResponsesResponse  # type: ignore[override]
+
+
+class ResponseInProgressEvent(OpenAIResponseInProgressEvent):
+    response: ResponsesResponse  # type: ignore[override]
+
+
+StreamingResponsesResponse: TypeAlias = (
+    ResponseCreatedEvent
+    | ResponseInProgressEvent
+    | ResponseCompletedEvent
+    | ResponseOutputItemAddedEvent
+    | ResponseOutputItemDoneEvent
+    | ResponseContentPartAddedEvent
+    | ResponseContentPartDoneEvent
+    | ResponseReasoningTextDeltaEvent
+    | ResponseReasoningTextDoneEvent
+    | ResponseReasoningPartAddedEvent
+    | ResponseReasoningPartDoneEvent
+    | ResponseCodeInterpreterCallInProgressEvent
+    | ResponseCodeInterpreterCallCodeDeltaEvent
+    | ResponseWebSearchCallInProgressEvent
+    | ResponseWebSearchCallSearchingEvent
+    | ResponseWebSearchCallCompletedEvent
+    | ResponseCodeInterpreterCallCodeDoneEvent
+    | ResponseCodeInterpreterCallInterpretingEvent
+    | ResponseCodeInterpreterCallCompletedEvent
+    | ResponseMcpCallArgumentsDeltaEvent
+    | ResponseMcpCallArgumentsDoneEvent
+    | ResponseMcpCallInProgressEvent
+    | ResponseMcpCallCompletedEvent
+)

vllm/entrypoints/openai/responses/utils.py

@@ -0,0 +1,389 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from collections.abc import Iterable
+from typing import Any
+
+from openai.types.chat import (
+    ChatCompletionAssistantMessageParam,
+    ChatCompletionMessageToolCallParam,
+    ChatCompletionToolMessageParam,
+)
+from openai.types.chat.chat_completion_message_tool_call_param import (
+    Function as FunctionCallTool,
+)
+from openai.types.responses import (
+    ResponseFunctionToolCall,
+    ResponseOutputItem,
+    ResponseOutputMessage,
+    ResponseOutputText,
+    ResponseReasoningItem,
+)
+from openai.types.responses.response import ToolChoice
+from openai.types.responses.response_function_tool_call_output_item import (
+    ResponseFunctionToolCallOutputItem,
+)
+from openai.types.responses.response_output_text import Logprob
+from openai.types.responses.response_reasoning_item import (
+    Content as ResponseReasoningTextContent,
+)
+from openai.types.responses.tool import Tool
+
+from vllm import envs
+from vllm.entrypoints.chat_utils import make_tool_call_id
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionMessageParam
+from vllm.entrypoints.openai.engine.protocol import FunctionCall
+from vllm.entrypoints.openai.responses.protocol import ResponseInputOutputItem
+from vllm.logger import init_logger
+from vllm.tool_parsers.utils import (
+    build_responses_tool_call_name_map,
+    flat_namespace_tool_name,
+    iter_response_function_tool_dicts,
+    resolve_responses_tool_call_name,
+)
+from vllm.utils import random_uuid
+
+logger = init_logger(__name__)
+
+
+def build_response_output_items(
+    reasoning: str | None,
+    content: str | None,
+    tool_calls: list[FunctionCall] | None,
+    logprobs: list[Logprob] | None = None,
+    tools: list[Tool] | None = None,
+) -> list[ResponseOutputItem]:
+    outputs: list[ResponseOutputItem] = []
+    tool_call_name_map = build_responses_tool_call_name_map(tools)
+
+    if reasoning:
+        outputs.append(
+            ResponseReasoningItem(
+                id=f"rs_{random_uuid()}",
+                summary=[],
+                type="reasoning",
+                content=[
+                    ResponseReasoningTextContent(text=reasoning, type="reasoning_text")
+                ],
+                status=None,
+            )
+        )
+
+    if content:
+        outputs.append(
+            ResponseOutputMessage(
+                id=f"msg_{random_uuid()}",
+                content=[
+                    ResponseOutputText(
+                        text=content,
+                        annotations=[],
+                        type="output_text",
+                        logprobs=logprobs,
+                    )
+                ],
+                role="assistant",
+                status="completed",
+                type="message",
+            )
+        )
+
+    if tool_calls:
+        for idx, tool_call in enumerate(tool_calls):
+            call_name = resolve_responses_tool_call_name(
+                tool_call.name, tool_call_name_map=tool_call_name_map
+            )
+            outputs.append(
+                ResponseFunctionToolCall(
+                    id=f"fc_{random_uuid()}",
+                    call_id=tool_call.id
+                    or make_tool_call_id(func_name=tool_call.name, idx=idx),
+                    type="function_call",
+                    status="completed",
+                    name=call_name.name,
+                    namespace=call_name.namespace,
+                    arguments=tool_call.arguments,
+                )
+            )
+
+    return outputs
+
+
+def should_continue_final_message(
+    request_input: str | list[ResponseInputOutputItem],
+) -> bool:
+    """
+    Determine if the last input message is a partial assistant message
+    that should be continued rather than starting a new generation.
+
+    This enables partial message completion similar to Anthropic's Messages API,
+    where users can provide an incomplete assistant message and have the model
+    continue from where it left off.
+
+    A message is considered partial if:
+    1. It's a ResponseOutputMessage or ResponseReasoningItem
+    2. Its status is "in_progress" or "incomplete"
+
+    Args:
+        request_input: The input to the Responses API request
+
+    Returns:
+        True if the final message should be continued, False otherwise
+    """
+    if isinstance(request_input, str):
+        # Simple string input is always a user message
+        return False
+
+    if not request_input:
+        return False
+
+    last_item = request_input[-1]
+
+    # Check if the last item is a partial assistant message
+    if isinstance(last_item, ResponseOutputMessage):
+        return last_item.status in ("in_progress", "incomplete")
+
+    # Check if the last item is a partial reasoning item
+    if isinstance(last_item, ResponseReasoningItem):
+        return last_item.status in ("in_progress", "incomplete")
+
+    if isinstance(last_item, dict):
+        # only support partial completion for messages for now
+        if last_item.get("type", "message") not in ("message", "reasoning"):
+            return False
+        return last_item.get("status") in ("in_progress", "incomplete")
+
+    return False
+
+
+def construct_input_messages(
+    *,
+    request_instructions: str | None = None,
+    request_input: str | list[ResponseInputOutputItem],
+    prev_msg: list[ChatCompletionMessageParam] | None = None,
+    prev_response_output: list[ResponseOutputItem] | None = None,
+):
+    messages: list[ChatCompletionMessageParam] = []
+    if request_instructions:
+        messages.append(
+            {
+                "role": "system",
+                "content": request_instructions,
+            }
+        )
+
+    # Prepend the conversation history.
+    if prev_msg is not None:
+        # Filter out system messages from previous conversation -- per the
+        # OpenAI spec, instructions should NOT carry over across responses.
+        # The current request's instructions (if any) were already added above.
+        messages.extend(m for m in prev_msg if m.get("role") != "system")
+    if prev_response_output is not None:
+        # Add the previous output.
+        for output_item in prev_response_output:
+            # NOTE: We skip the reasoning output.
+            if isinstance(output_item, ResponseOutputMessage):
+                for content in output_item.content:
+                    messages.append(
+                        {
+                            "role": "assistant",
+                            "content": content.text,
+                        }
+                    )
+
+    # Append the new input.
+    # Responses API supports simple text inputs without chat format.
+    if isinstance(request_input, str):
+        messages.append({"role": "user", "content": request_input})
+    else:
+        input_messages = construct_chat_messages_with_tool_call(request_input)
+        messages.extend(input_messages)
+    return messages
+
+
+def construct_chat_messages_with_tool_call(
+    input_messages: list[ResponseInputOutputItem],
+) -> list[ChatCompletionMessageParam]:
+    """Build chat messages from response items.
+
+    Some chat messages span multiple response items (e.g., reasoning + tool calls).
+    """
+    messages: list[ChatCompletionMessageParam] = []
+    for item in input_messages:
+        message = _construct_message_from_response_item(
+            item, prev_msg=messages[-1] if messages else None
+        )
+        if message is not None:
+            messages.append(message)
+
+    return messages
+
+
+def _construct_message_from_response_item(
+    item: ResponseInputOutputItem,
+    prev_msg: ChatCompletionMessageParam | None = None,
+) -> ChatCompletionMessageParam | None:
+    """
+    Returns a new message or None. If `None`, `prev_msg` might be updated.
+    If `prev_msg` is `None`, a new message is always returned.
+    """
+    prev_assistant_msg = (
+        prev_msg if prev_msg and prev_msg.get("role") == "assistant" else None
+    )
+
+    if isinstance(item, ResponseFunctionToolCall):
+        tool_name = item.name
+        if item.namespace:
+            tool_name = flat_namespace_tool_name(item.namespace, item.name)
+        tool_call = ChatCompletionMessageToolCallParam(
+            id=item.call_id,
+            function=FunctionCallTool(
+                name=tool_name,
+                arguments=item.arguments,
+            ),
+            type="function",
+        )
+        if prev_assistant_msg:
+            tool_calls = prev_assistant_msg.get("tool_calls")
+            if tool_calls is None:
+                prev_assistant_msg["tool_calls"] = [tool_call]
+                return None
+            if isinstance(tool_calls, list):
+                tool_calls.append(tool_call)
+                return None
+            if isinstance(tool_calls, Iterable) and not isinstance(
+                tool_calls, (dict, str)
+            ):
+                tool_calls = list(tool_calls)
+                tool_calls.append(tool_call)
+                prev_assistant_msg["tool_calls"] = tool_calls
+                return None
+            logger.warning(
+                "Previous assistant message has unknown tool_calls format. "
+                "Tool call merging is skipped and a new assistant message is created. "
+                "Item %s",
+                item.id,
+            )
+        return ChatCompletionAssistantMessageParam(
+            role="assistant",
+            tool_calls=[tool_call],
+        )
+    elif isinstance(item, ResponseReasoningItem):
+        reasoning = ""
+        if item.encrypted_content:
+            raise ValueError("Encrypted content is not supported.")
+        elif item.content and len(item.content) >= 1:
+            reasoning = item.content[0].text
+        elif len(item.summary) >= 1:
+            reasoning = item.summary[0].text
+            logger.warning(
+                "Using summary text as reasoning content for item %s. "
+                "Please use content instead of summary for "
+                "reasoning items.",
+                item.id,
+            )
+
+        if prev_assistant_msg:
+            previous_reasoning = prev_assistant_msg.get("reasoning")
+            if previous_reasoning is None:
+                prev_assistant_msg["reasoning"] = reasoning
+                return None
+        return {
+            "role": "assistant",
+            "reasoning": reasoning,
+        }
+    elif isinstance(item, ResponseOutputMessage):
+        output_text = item.content[0].text
+        if prev_assistant_msg:
+            previous_content = prev_assistant_msg.get("content")
+            if previous_content is None:
+                prev_assistant_msg["content"] = output_text
+                return None
+        return {
+            "role": "assistant",
+            "content": output_text,
+        }
+    elif isinstance(item, ResponseFunctionToolCallOutputItem):
+        return ChatCompletionToolMessageParam(
+            role="tool",
+            content=item.output,
+            tool_call_id=item.call_id,
+        )
+    elif isinstance(item, dict) and item.get("type") == "function_call_output":
+        # Append the function call output as a tool message.
+        return ChatCompletionToolMessageParam(
+            role="tool",
+            content=item.get("output"),
+            tool_call_id=item.get("call_id"),
+        )
+    elif isinstance(item, dict) and item.get("role") == "assistant":
+        content = item.get("content")
+        text: str | None = None
+        if isinstance(content, str):
+            text = content
+        elif isinstance(content, list) and content:
+            text = content[0].get("text")
+        if text is not None:
+            if prev_assistant_msg:
+                previous_content = prev_assistant_msg.get("content")
+                if previous_content is None:
+                    prev_assistant_msg["content"] = text
+                    return None
+            return {"role": "assistant", "content": text}
+    return item  # type: ignore[arg-type]
+
+
+def extract_function_tool_names(tools: list[Tool]) -> frozenset[str]:
+    names = []
+    for tool in tools:
+        if tool.type == "function":
+            names.append(tool.name)
+        elif tool.type == "namespace":
+            names.extend(
+                flat_namespace_tool_name(tool.name, namespaced_tool.name)
+                for namespaced_tool in tool.tools
+                if namespaced_tool.type == "function"
+            )
+    return frozenset(names)
+
+
+def extract_tool_types(tools: list[Tool]) -> set[str]:
+    """
+    Extracts the tool types from the given tools.
+    """
+    tool_types: set[str] = set()
+    for tool in tools:
+        if tool.type == "mcp":
+            # Allow the MCP Tool type to enable built in tools if the
+            # server_label is allowlisted in
+            # envs.VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS
+            if tool.server_label in envs.VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS:
+                tool_types.add(tool.server_label)
+        else:
+            tool_types.add(tool.type)
+    return tool_types
+
+
+def convert_tool_responses_to_completions_format(tool: dict) -> dict:
+    """
+    Convert a flat tool schema:
+        {"type": "function", "name": "...", "description": "...", "parameters": {...}}
+    into:
+        {"type": "function", "function": {...}}
+    """
+    return {
+        "type": "function",
+        "function": tool,
+    }
+
+
+def construct_tool_dicts(
+    tools: list[Tool], tool_choice: ToolChoice
+) -> list[dict[str, Any]] | None:
+    if not tools or (tool_choice == "none"):
+        tool_dicts = None
+    else:
+        tool_dicts = [
+            convert_tool_responses_to_completions_format(tool)
+            for tool in iter_response_function_tool_dicts(tools)
+        ]
+    return tool_dicts

vllm/entrypoints/openai/run_batch.py

@@ -0,0 +1,877 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import asyncio
+import contextlib
+import json
+import sys
+import tempfile
+from argparse import Namespace
+from collections.abc import Awaitable, Callable
+from http import HTTPStatus
+from io import BytesIO, StringIO
+from typing import Any, TypeAlias
+from urllib.parse import urlparse
+
+import aiohttp
+import pybase64 as base64
+import pydantic
+import torch
+from fastapi import UploadFile
+from prometheus_client import start_http_server
+from pydantic import Field, TypeAdapter, field_validator, model_validator
+from pydantic_core.core_schema import ValidationInfo
+from starlette.datastructures import State
+from starlette.responses import JSONResponse
+from tqdm import tqdm
+from urllib3.util import parse_url
+
+import vllm.envs as envs
+from vllm.config import config
+from vllm.connections import global_http_connection
+from vllm.engine.arg_utils import AsyncEngineArgs
+from vllm.engine.protocol import EngineClient
+from vllm.entrypoints.openai.api_server import init_app_state
+from vllm.entrypoints.openai.chat_completion.protocol import (
+    ChatCompletionRequest,
+    ChatCompletionResponse,
+)
+from vllm.entrypoints.openai.cli_args import BaseFrontendArgs
+from vllm.entrypoints.openai.engine.protocol import (
+    ErrorInfo,
+    ErrorResponse,
+    OpenAIBaseModel,
+)
+from vllm.entrypoints.pooling.embed.protocol import (
+    EmbeddingRequest,
+    EmbeddingResponse,
+)
+from vllm.entrypoints.pooling.scoring.protocol import (
+    RerankRequest,
+    RerankResponse,
+    ScoreRequest,
+    ScoreResponse,
+)
+from vllm.entrypoints.serve.utils.error_response import create_error_response
+from vllm.entrypoints.speech_to_text.transcription.protocol import (
+    TranscriptionRequest,
+    TranscriptionResponse,
+    TranscriptionResponseVerbose,
+)
+from vllm.entrypoints.speech_to_text.translation.protocol import (
+    TranslationRequest,
+    TranslationResponse,
+    TranslationResponseVerbose,
+)
+from vllm.exceptions import VLLMValidationError
+from vllm.logger import init_logger
+from vllm.reasoning import ReasoningParserManager
+from vllm.utils import random_uuid
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+from vllm.version import __version__ as VLLM_VERSION
+
+logger = init_logger(__name__)
+
+
+class BatchTranscriptionRequest(TranscriptionRequest):
+    """
+    Batch transcription request that uses file_url instead of file.
+
+    This class extends TranscriptionRequest but replaces the file field
+    with file_url to support batch processing from audio files written in JSON format.
+    """
+
+    file_url: str = Field(
+        ...,
+        description=(
+            "Either a URL of the audio or a data URL with base64 encoded audio data. "
+        ),
+    )
+
+    # Override file to be optional and unused for batch processing
+    file: UploadFile | None = Field(default=None, exclude=True)  # type: ignore[assignment]
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_no_file(cls, data: Any):
+        """Ensure file field is not provided in batch requests."""
+        if isinstance(data, dict) and "file" in data:
+            raise VLLMValidationError(
+                "The 'file' field is not supported in batch requests. "
+                "Use 'file_url' instead.",
+                parameter="file",
+            )
+        return data
+
+
+class BatchTranslationRequest(TranslationRequest):
+    """
+    Batch translation request that uses file_url instead of file.
+
+    This class extends TranslationRequest but replaces the file field
+    with file_url to support batch processing from audio files written in JSON format.
+    """
+
+    file_url: str = Field(
+        ...,
+        description=(
+            "Either a URL of the audio or a data URL with base64 encoded audio data. "
+        ),
+    )
+
+    # Override file to be optional and unused for batch processing
+    file: UploadFile | None = Field(default=None, exclude=True)  # type: ignore[assignment]
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_no_file(cls, data: Any):
+        """Ensure file field is not provided in batch requests."""
+        if isinstance(data, dict) and "file" in data:
+            raise VLLMValidationError(
+                "The 'file' field is not supported in batch requests. "
+                "Use 'file_url' instead.",
+                parameter="file",
+            )
+        return data
+
+
+BatchRequestInputBody: TypeAlias = (
+    ChatCompletionRequest
+    | EmbeddingRequest
+    | ScoreRequest
+    | RerankRequest
+    | BatchTranscriptionRequest
+    | BatchTranslationRequest
+)
+
+
+class BatchRequestInput(OpenAIBaseModel):
+    """
+    The per-line object of the batch input file.
+
+    NOTE: Currently only the `/v1/chat/completions` endpoint is supported.
+    """
+
+    # A developer-provided per-request id that will be used to match outputs to
+    # inputs. Must be unique for each request in a batch.
+    custom_id: str
+
+    # The HTTP method to be used for the request. Currently only POST is
+    # supported.
+    method: str
+
+    # The OpenAI API relative URL to be used for the request. Currently
+    # /v1/chat/completions is supported.
+    url: str
+
+    # The parameters of the request.
+    body: BatchRequestInputBody
+
+    @field_validator("body", mode="plain")
+    @classmethod
+    def check_type_for_url(cls, value: Any, info: ValidationInfo):
+        # Use url to disambiguate models
+        url: str = info.data["url"]
+        if url == "/v1/chat/completions":
+            return ChatCompletionRequest.model_validate(value)
+        if url == "/v1/embeddings":
+            return TypeAdapter(EmbeddingRequest).validate_python(value)
+        if url.endswith("/score"):
+            return TypeAdapter(ScoreRequest).validate_python(value)
+        if url.endswith("/rerank"):
+            return RerankRequest.model_validate(value)
+        if url == "/v1/audio/transcriptions":
+            return BatchTranscriptionRequest.model_validate(value)
+        if url == "/v1/audio/translations":
+            return BatchTranslationRequest.model_validate(value)
+        return TypeAdapter(BatchRequestInputBody).validate_python(value)
+
+
+AllResponse: TypeAlias = (
+    ChatCompletionResponse
+    | EmbeddingResponse
+    | ScoreResponse
+    | RerankResponse
+    | TranscriptionResponse
+    | TranscriptionResponseVerbose
+    | TranslationResponse
+    | TranslationResponseVerbose
+)
+
+
+class BatchResponseData(OpenAIBaseModel):
+    # HTTP status code of the response.
+    status_code: int = 200
+
+    # An unique identifier for the API request.
+    request_id: str
+
+    # The body of the response.
+    body: AllResponse | None = None
+
+
+class BatchRequestOutput(OpenAIBaseModel):
+    """
+    The per-line object of the batch output and error files
+    """
+
+    id: str
+
+    # A developer-provided per-request id that will be used to match outputs to
+    # inputs.
+    custom_id: str
+
+    response: BatchResponseData | None
+
+    # For requests that failed with a non-HTTP error, this will contain more
+    # information on the cause of the failure.
+    error: Any | None
+
+
+@config
+class BatchFrontendArgs(BaseFrontendArgs):
+    """Arguments for the batch runner frontend."""
+
+    input_file: str | None = None
+    """The path or url to a single input file. Currently supports local file
+    paths, or the http protocol (http or https). If a URL is specified,
+    the file should be available via HTTP GET."""
+    output_file: str | None = None
+    """The path or url to a single output file. Currently supports
+    local file paths, or web (http or https) urls. If a URL is specified,
+    the file should be available via HTTP PUT."""
+    output_tmp_dir: str | None = None
+    """The directory to store the output file before uploading it
+    to the output URL."""
+    enable_metrics: bool = False
+    """Enable Prometheus metrics"""
+    host: str | None = None
+    """Host name for the Prometheus metrics server
+    (only needed if enable-metrics is set)."""
+    port: int = 8000
+    """Port number for the Prometheus metrics server
+    (only needed if enable-metrics is set)."""
+    url: str = "0.0.0.0"
+    """[DEPRECATED] Host name for the Prometheus metrics server
+    (only needed if enable-metrics is set). Use --host instead."""
+
+    @classmethod
+    def _customize_cli_kwargs(
+        cls,
+        frontend_kwargs: dict[str, Any],
+    ) -> dict[str, Any]:
+        frontend_kwargs = super()._customize_cli_kwargs(frontend_kwargs)
+
+        frontend_kwargs["input_file"]["flags"] = ["-i"]
+        frontend_kwargs["input_file"]["required"] = True
+        frontend_kwargs["output_file"]["flags"] = ["-o"]
+        frontend_kwargs["output_file"]["required"] = True
+
+        frontend_kwargs["enable_metrics"]["action"] = "store_true"
+
+        frontend_kwargs["url"]["deprecated"] = True
+        return frontend_kwargs
+
+
+def make_arg_parser(parser: FlexibleArgumentParser):
+    parser = BatchFrontendArgs.add_cli_args(parser)
+    parser = AsyncEngineArgs.add_cli_args(parser)
+    return parser
+
+
+def parse_args():
+    parser = FlexibleArgumentParser(description="vLLM OpenAI-Compatible batch runner.")
+    args = make_arg_parser(parser).parse_args()
+
+    # Backward compatibility: If --url is set, use it for host
+    url_explicit = any(arg == "--url" or arg.startswith("--url=") for arg in sys.argv)
+    host_explicit = any(
+        arg == "--host" or arg.startswith("--host=") for arg in sys.argv
+    )
+    if url_explicit and hasattr(args, "url") and not host_explicit:
+        args.host = args.url
+        logger.warning_once(
+            "Using --url for metrics is deprecated. Please use --host instead."
+        )
+
+    return args
+
+
+# explicitly use pure text format, with a newline at the end
+# this makes it impossible to see the animation in the progress bar
+# but will avoid messing up with ray or multiprocessing, which wraps
+# each line of output with some prefix.
+_BAR_FORMAT = "{desc}: {percentage:3.0f}% Completed | {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]\n"  # noqa: E501
+
+
+class BatchProgressTracker:
+    def __init__(self):
+        self._total = 0
+        self._pbar: tqdm | None = None
+
+    def submitted(self):
+        self._total += 1
+
+    def completed(self):
+        if self._pbar:
+            self._pbar.update()
+
+    def pbar(self) -> tqdm:
+        enable_tqdm = (
+            not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0
+        )
+        self._pbar = tqdm(
+            total=self._total,
+            unit="req",
+            desc="Running batch",
+            mininterval=5,
+            disable=not enable_tqdm,
+            bar_format=_BAR_FORMAT,
+        )
+        return self._pbar
+
+
+async def read_file(path_or_url: str) -> str:
+    if path_or_url.startswith("http://") or path_or_url.startswith("https://"):
+        async with aiohttp.ClientSession() as session, session.get(path_or_url) as resp:
+            resp.raise_for_status()
+            return await resp.text()
+    else:
+        with open(path_or_url, encoding="utf-8") as f:
+            return f.read()
+
+
+async def write_local_file(
+    output_path: str, batch_outputs: list[BatchRequestOutput]
+) -> None:
+    """
+    Write the responses to a local file.
+    output_path: The path to write the responses to.
+    batch_outputs: The list of batch outputs to write.
+    """
+    # We should make this async, but as long as run_batch runs as a
+    # standalone program, blocking the event loop won't affect performance.
+    with open(output_path, "w", encoding="utf-8") as f:
+        for o in batch_outputs:
+            print(o.model_dump_json(), file=f)
+
+
+async def upload_data(output_url: str, data_or_file: str, from_file: bool) -> None:
+    """
+    Upload a local file to a URL.
+    output_url: The URL to upload the file to.
+    data_or_file: Either the data to upload or the path to the file to upload.
+    from_file: If True, data_or_file is the path to the file to upload.
+    """
+    # Timeout is a common issue when uploading large files.
+    # We retry max_retries times before giving up.
+    max_retries = 5
+    # Number of seconds to wait before retrying.
+    delay = 5
+
+    for attempt in range(1, max_retries + 1):
+        try:
+            # We increase the timeout to 1000 seconds to allow
+            # for large files (default is 300).
+            async with aiohttp.ClientSession(
+                timeout=aiohttp.ClientTimeout(total=1000)
+            ) as session:
+                if from_file:
+                    with open(data_or_file, "rb") as file:
+                        async with session.put(output_url, data=file) as response:
+                            if response.status != 200:
+                                raise Exception(
+                                    f"Failed to upload file.\n"
+                                    f"Status: {response.status}\n"
+                                    f"Response: {response.text()}"
+                                )
+                else:
+                    async with session.put(output_url, data=data_or_file) as response:
+                        if response.status != 200:
+                            raise Exception(
+                                f"Failed to upload data.\n"
+                                f"Status: {response.status}\n"
+                                f"Response: {response.text()}"
+                            )
+
+        except Exception as e:
+            if attempt < max_retries:
+                logger.error(
+                    "Failed to upload data (attempt %d). Error message: %s.\nRetrying in %d seconds...",  # noqa: E501
+                    attempt,
+                    e,
+                    delay,
+                )
+                await asyncio.sleep(delay)
+            else:
+                raise Exception(
+                    f"Failed to upload data (attempt {attempt}). Error message: {str(e)}."  # noqa: E501
+                ) from e
+
+
+async def write_file(
+    path_or_url: str, batch_outputs: list[BatchRequestOutput], output_tmp_dir: str
+) -> None:
+    """
+    Write batch_outputs to a file or upload to a URL.
+    path_or_url: The path or URL to write batch_outputs to.
+    batch_outputs: The list of batch outputs to write.
+    output_tmp_dir: The directory to store the output file before uploading it
+    to the output URL.
+    """
+    if path_or_url.startswith("http://") or path_or_url.startswith("https://"):
+        if output_tmp_dir is None:
+            logger.info("Writing outputs to memory buffer")
+            output_buffer = StringIO()
+            for o in batch_outputs:
+                print(o.model_dump_json(), file=output_buffer)
+            output_buffer.seek(0)
+            logger.info("Uploading outputs to %s", path_or_url)
+            await upload_data(
+                path_or_url,
+                output_buffer.read().strip().encode("utf-8"),
+                from_file=False,
+            )
+        else:
+            # Write responses to a temporary file and then upload it to the URL.
+            with tempfile.NamedTemporaryFile(
+                mode="w",
+                encoding="utf-8",
+                dir=output_tmp_dir,
+                prefix="tmp_batch_output_",
+                suffix=".jsonl",
+            ) as f:
+                logger.info("Writing outputs to temporary local file %s", f.name)
+                await write_local_file(f.name, batch_outputs)
+                logger.info("Uploading outputs to %s", path_or_url)
+                await upload_data(path_or_url, f.name, from_file=True)
+    else:
+        logger.info("Writing outputs to local file %s", path_or_url)
+        await write_local_file(path_or_url, batch_outputs)
+
+
+async def download_bytes_from_url(
+    url: str,
+    allowed_media_domains: list[str] | None = None,
+) -> bytes:
+    """
+    Download data from a URL or decode from a data URL.
+
+    Args:
+        url: Either an HTTP/HTTPS URL or a data URL (data:...;base64,...)
+        allowed_media_domains: If set, only HTTP/HTTPS URLs whose hostname
+            is in this list are permitted. data: URLs are not subject to
+            this restriction.
+
+    Returns:
+        Data as bytes
+    """
+    parsed = urlparse(url)
+
+    # Handle data URLs (base64 encoded) - not subject to domain restrictions
+    if parsed.scheme == "data":
+        # Format: data:...;base64,<base64_data>
+        if "," in url:
+            header, data = url.split(",", 1)
+            if "base64" in header:
+                return base64.b64decode(data)
+            else:
+                raise ValueError(f"Unsupported data URL encoding: {header}")
+        else:
+            raise ValueError(f"Invalid data URL format: {url}")
+
+    # Handle HTTP/HTTPS URLs
+    elif parsed.scheme in ("http", "https"):
+        if allowed_media_domains is not None:
+            url_spec = parse_url(url)
+            if url_spec.hostname not in allowed_media_domains:
+                raise ValueError(
+                    f"The URL must be from one of the allowed domains: "
+                    f"{allowed_media_domains}. Input URL domain: "
+                    f"{url_spec.hostname}"
+                )
+            # Use the normalized URL to prevent parsing discrepancies
+            # between urllib3 and aiohttp (e.g. backslash-@ attacks).
+            url = url_spec.url
+
+        return await global_http_connection.async_get_bytes(
+            url, allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS
+        )
+
+    else:
+        raise ValueError(
+            f"Unsupported URL scheme: {parsed.scheme}. "
+            "Supported schemes: http, https, data"
+        )
+
+
+def make_error_request_output(
+    request: BatchRequestInput, error_msg: str
+) -> BatchRequestOutput:
+    batch_output = BatchRequestOutput(
+        id=f"vllm-{random_uuid()}",
+        custom_id=request.custom_id,
+        response=BatchResponseData(
+            status_code=HTTPStatus.BAD_REQUEST,
+            request_id=f"vllm-batch-{random_uuid()}",
+        ),
+        error=error_msg,
+    )
+    return batch_output
+
+
+async def make_async_error_request_output(
+    request: BatchRequestInput, error_msg: str
+) -> BatchRequestOutput:
+    return make_error_request_output(request, error_msg)
+
+
+async def run_request(
+    serving_engine_func: Callable,
+    request: BatchRequestInput,
+    tracker: BatchProgressTracker,
+) -> BatchRequestOutput:
+    try:
+        response = await serving_engine_func(request.body)
+    except Exception as e:
+        response = create_error_response(e)
+
+    if isinstance(response, JSONResponse):
+        with contextlib.suppress(pydantic.ValidationError):
+            response = TypeAdapter(AllResponse | ErrorResponse).validate_python(
+                json.loads(response.body)
+            )
+
+    if isinstance(response, AllResponse):
+        batch_output = BatchRequestOutput(
+            id=f"vllm-{random_uuid()}",
+            custom_id=request.custom_id,
+            response=BatchResponseData(
+                body=response, request_id=f"vllm-batch-{random_uuid()}"
+            ),
+            error=None,
+        )
+    elif isinstance(response, ErrorResponse):
+        batch_output = BatchRequestOutput(
+            id=f"vllm-{random_uuid()}",
+            custom_id=request.custom_id,
+            response=BatchResponseData(
+                status_code=response.error.code,
+                request_id=f"vllm-batch-{random_uuid()}",
+            ),
+            error=response,
+        )
+    else:
+        batch_output = make_error_request_output(
+            request, error_msg="Request must not be sent in stream mode"
+        )
+
+    tracker.completed()
+    return batch_output
+
+
+WrapperFn: TypeAlias = Callable[[Callable], Callable]
+
+
+def handle_endpoint_request(
+    request: BatchRequestInput,
+    tracker: BatchProgressTracker,
+    url_matcher: Callable[[str], bool],
+    handler_getter: Callable[[], Callable | None],
+    wrapper_fn: WrapperFn | None = None,
+) -> Awaitable[BatchRequestOutput] | None:
+    """
+    Generic handler for endpoint requests.
+
+    Args:
+        request: The batch request input
+        tracker: Progress tracker for the batch
+        url_matcher: Function that takes a URL and returns True if it matches
+        handler_getter: Function that returns the handler function or None
+        wrapper_fn: Optional function to wrap the handler (e.g., for transcriptions)
+
+    Returns:
+        Awaitable[BatchRequestOutput] if the request was handled,
+        None if URL didn't match
+    """
+    if not url_matcher(request.url):
+        return None
+
+    handler_fn = handler_getter()
+    if handler_fn is None:
+        error_msg = f"Model does not support endpoint: {request.url}"
+        return make_async_error_request_output(request, error_msg=error_msg)
+
+    # Apply wrapper if provided (e.g., for transcriptions/translations)
+    if wrapper_fn is not None:
+        handler_fn = wrapper_fn(handler_fn)
+
+    tracker.submitted()
+    return run_request(handler_fn, request, tracker)
+
+
+def make_transcription_wrapper(
+    is_translation: bool,
+    allowed_media_domains: list[str] | None = None,
+) -> WrapperFn:
+    """
+    Factory function to create a wrapper for transcription/translation handlers.
+    The wrapper converts BatchTranscriptionRequest or BatchTranslationRequest
+    to TranscriptionRequest or TranslationRequest and calls the appropriate handler.
+
+    Args:
+        is_translation: If True, process as translation; otherwise process
+            as transcription
+        allowed_media_domains: If set, only URLs from these domains are
+            permitted for HTTP/HTTPS fetches.
+
+    Returns:
+        A function that takes a handler and returns a wrapped handler
+    """
+
+    def wrapper(handler_fn: Callable):
+        async def transcription_wrapper(
+            batch_request_body: (BatchTranscriptionRequest | BatchTranslationRequest),
+        ) -> (
+            TranscriptionResponse
+            | TranscriptionResponseVerbose
+            | TranslationResponse
+            | TranslationResponseVerbose
+            | ErrorResponse
+        ):
+            try:
+                # Download data from URL
+                audio_data = await download_bytes_from_url(
+                    batch_request_body.file_url,
+                    allowed_media_domains=allowed_media_domains,
+                )
+
+                # Create a mock file from the downloaded audio data
+                mock_file = UploadFile(
+                    file=BytesIO(audio_data),
+                    filename="audio.bin",
+                )
+
+                # Convert batch request to regular request
+                # by copying all fields except file_url and setting file to mock_file
+                request_dict = batch_request_body.model_dump(exclude={"file_url"})
+                request_dict["file"] = mock_file
+
+                if is_translation:
+                    # Create TranslationRequest from BatchTranslationRequest
+                    translation_request = TranslationRequest.model_validate(
+                        request_dict
+                    )
+                    return await handler_fn(audio_data, translation_request)
+                else:
+                    # Create TranscriptionRequest from BatchTranscriptionRequest
+                    transcription_request = TranscriptionRequest.model_validate(
+                        request_dict
+                    )
+                    return await handler_fn(audio_data, transcription_request)
+            except Exception as e:
+                operation = "translation" if is_translation else "transcription"
+                return ErrorResponse(
+                    error=ErrorInfo(
+                        message=f"Failed to process {operation}: {str(e)}",
+                        type="BadRequestError",
+                        code=HTTPStatus.BAD_REQUEST.value,
+                    )
+                )
+
+        return transcription_wrapper
+
+    return wrapper
+
+
+async def build_endpoint_registry(
+    engine_client: EngineClient,
+    args: Namespace,
+) -> dict[str, dict[str, Any]]:
+    """
+    Build the endpoint registry with all serving objects and handler configurations.
+
+    Args:
+        engine_client: The engine client
+        args: Command line arguments
+
+    Returns:
+        Dictionary mapping endpoint keys to their configurations
+    """
+    supported_tasks = await engine_client.get_supported_tasks()
+    logger.info("Supported tasks: %s", supported_tasks)
+
+    # Create a state object to hold serving objects
+    state = State()
+
+    # Initialize all serving objects using init_app_state
+    # This provides full functionality including chat template processing,
+    # LoRA support, tool servers, etc.
+    await init_app_state(engine_client, state, args, supported_tasks)
+
+    # Get serving objects from state (defaulting to None if not set)
+    openai_serving_chat = getattr(state, "openai_serving_chat", None)
+    openai_serving_transcription = getattr(state, "openai_serving_transcription", None)
+    openai_serving_translation = getattr(state, "openai_serving_translation", None)
+    serving_embedding = getattr(state, "serving_embedding", None)
+    serving_scores = getattr(state, "serving_scores", None)
+
+    allowed_media_domains = getattr(args, "allowed_media_domains", None)
+
+    # Registry of endpoint configurations
+    endpoint_registry: dict[str, dict[str, Any]] = {
+        "completions": {
+            "url_matcher": lambda url: url == "/v1/chat/completions",
+            "handler_getter": lambda: (
+                openai_serving_chat.create_chat_completion
+                if openai_serving_chat is not None
+                else None
+            ),
+            "wrapper_fn": None,
+        },
+        "embeddings": {
+            "url_matcher": lambda url: url == "/v1/embeddings",
+            "handler_getter": lambda: (
+                serving_embedding if serving_embedding is not None else None
+            ),
+            "wrapper_fn": None,
+        },
+        "score": {
+            "url_matcher": lambda url: url.endswith("/score"),
+            "handler_getter": lambda: (
+                serving_scores if serving_scores is not None else None
+            ),
+            "wrapper_fn": None,
+        },
+        "rerank": {
+            "url_matcher": lambda url: url.endswith("/rerank"),
+            "handler_getter": lambda: (
+                serving_scores if serving_scores is not None else None
+            ),
+            "wrapper_fn": None,
+        },
+        "transcriptions": {
+            "url_matcher": lambda url: url == "/v1/audio/transcriptions",
+            "handler_getter": lambda: (
+                openai_serving_transcription.create_transcription
+                if openai_serving_transcription is not None
+                else None
+            ),
+            "wrapper_fn": make_transcription_wrapper(
+                is_translation=False,
+                allowed_media_domains=allowed_media_domains,
+            ),
+        },
+        "translations": {
+            "url_matcher": lambda url: url == "/v1/audio/translations",
+            "handler_getter": lambda: (
+                openai_serving_translation.create_translation
+                if openai_serving_translation is not None
+                else None
+            ),
+            "wrapper_fn": make_transcription_wrapper(
+                is_translation=True,
+                allowed_media_domains=allowed_media_domains,
+            ),
+        },
+    }
+
+    return endpoint_registry
+
+
+def validate_run_batch_args(args):
+    valid_reasoning_parsers = ReasoningParserManager.list_registered()
+    if (
+        reasoning_parser := args.structured_outputs_config.reasoning_parser
+    ) and reasoning_parser not in valid_reasoning_parsers:
+        raise KeyError(
+            f"invalid reasoning parser: {reasoning_parser} "
+            f"(chose from {{ {','.join(valid_reasoning_parsers)} }})"
+        )
+
+
+async def run_batch(
+    engine_client: EngineClient,
+    args: Namespace,
+) -> None:
+    endpoint_registry = await build_endpoint_registry(
+        engine_client=engine_client,
+        args=args,
+    )
+
+    tracker = BatchProgressTracker()
+    logger.info("Reading batch from %s...", args.input_file)
+
+    # Submit all requests in the file to the engine "concurrently".
+    response_futures: list[Awaitable[BatchRequestOutput]] = []
+    for request_json in (await read_file(args.input_file)).strip().split("\n"):
+        # Skip empty lines.
+        request_json = request_json.strip()
+        if not request_json:
+            continue
+
+        request = BatchRequestInput.model_validate_json(request_json)
+
+        # Use the last segment of the URL as the endpoint key.
+        # More advanced URL matching is done in url_matcher of endpoint_registry.
+        endpoint_key = request.url.split("/")[-1]
+
+        result = None
+        if endpoint_key in endpoint_registry:
+            endpoint_config = endpoint_registry[endpoint_key]
+            result = handle_endpoint_request(
+                request,
+                tracker,
+                url_matcher=endpoint_config["url_matcher"],
+                handler_getter=endpoint_config["handler_getter"],
+                wrapper_fn=endpoint_config["wrapper_fn"],
+            )
+
+        if result is not None:
+            response_futures.append(result)
+        else:
+            response_futures.append(
+                make_async_error_request_output(
+                    request,
+                    error_msg=f"URL {request.url} was used. "
+                    "Supported endpoints: /v1/chat/completions, /v1/embeddings,"
+                    " /v1/audio/transcriptions, /v1/audio/translations, /score, "
+                    " /rerank. See vllm/entrypoints/openai/api_server.py "
+                    "for supported score/rerank versions.",
+                )
+            )
+
+    with tracker.pbar():
+        responses = await asyncio.gather(*response_futures)
+
+    await write_file(args.output_file, responses, args.output_tmp_dir)
+
+
+async def main(args: Namespace):
+    from vllm.entrypoints.openai.api_server import build_async_engine_client
+    from vllm.usage.usage_lib import UsageContext
+
+    validate_run_batch_args(args)
+
+    async with build_async_engine_client(
+        args,
+        usage_context=UsageContext.OPENAI_BATCH_RUNNER,
+    ) as engine_client:
+        await run_batch(engine_client, args)
+
+
+if __name__ == "__main__":
+    args = parse_args()
+
+    logger.info("vLLM batch processing API version %s", VLLM_VERSION)
+    logger.info("args: %s", args)
+
+    # Start the Prometheus metrics server. LLMEngine uses the Prometheus client
+    # to publish metrics at the /metrics endpoint.
+    if args.enable_metrics:
+        logger.info("Prometheus metrics enabled")
+        start_http_server(port=args.port, addr=args.host)
+    else:
+        logger.info("Prometheus metrics disabled")
+
+    asyncio.run(main(args))

vllm/entrypoints/pooling/base/io_processor.py

@@ -0,0 +1,253 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from collections.abc import Sequence
+from typing import Any, Final
+
+from vllm import PoolingParams, PoolingRequestOutput, PromptType
+from vllm.config import VllmConfig
+from vllm.entrypoints.chat_utils import (
+    ChatCompletionMessageParam,
+    ChatTemplateConfig,
+    ChatTemplateContentFormatOption,
+    ConversationMessage,
+)
+from vllm.entrypoints.serve.engine.typing import RendererChatRequest, RendererRequest
+from vllm.inputs import EngineInput, SingletonPrompt
+from vllm.renderers import BaseRenderer, TokenizeParams, merge_kwargs
+from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
+from vllm.tool_parsers import ToolParser
+from vllm.utils.mistral import is_mistral_tokenizer
+
+from ..scoring.typing import ScoringData
+from ..typing import (
+    OfflineInputsContext,
+    OfflineOutputsContext,
+    PoolingChatLikeRequest,
+    PoolingCompletionLikeRequest,
+    PoolingServeContext,
+)
+
+
+class PoolingIOProcessor:
+    """Processor for handling preprocessing & postprocessing ops for pooling requests.
+
+    This class manages both online (serving) and offline (batch) processing of pooling
+    requests, handling chat and completion formats.
+    """
+
+    name: str
+
+    def __init__(
+        self,
+        vllm_config: VllmConfig,
+        renderer: BaseRenderer,
+        chat_template_config: ChatTemplateConfig,
+    ):
+        self.vllm_config = vllm_config
+        self.model_config = vllm_config.model_config
+        self.renderer = renderer
+
+        self.chat_template = chat_template_config.chat_template
+        self.chat_template_content_format: Final = (
+            chat_template_config.chat_template_content_format
+        )
+        self.trust_request_chat_template = (
+            chat_template_config.trust_request_chat_template
+        )
+
+    #######################################
+    # online APIs
+
+    def create_pooling_params(self, request):
+        return request.to_pooling_params()
+
+    def pre_process_online(self, ctx: PoolingServeContext):
+        request = ctx.request
+
+        if isinstance(request, PoolingChatLikeRequest):
+            self._validate_chat_template(
+                request_chat_template=request.chat_template,
+                chat_template_kwargs=request.chat_template_kwargs,
+                trust_request_chat_template=self.trust_request_chat_template,
+            )
+            _, engine_inputs = self._preprocess_chat_online(
+                request,
+                request.messages,
+                default_template=self.chat_template,
+                default_template_content_format=self.chat_template_content_format,
+                default_template_kwargs=None,
+            )
+        elif isinstance(request, PoolingCompletionLikeRequest):
+            engine_inputs = self._preprocess_cmpl_online(
+                request,
+                prompt_input=request.input,
+                prompt_embeds=None,
+            )
+        else:
+            raise ValueError(f"Invalid {self.name} request type")
+
+        ctx.engine_inputs = engine_inputs
+
+    def post_process_online(
+        self,
+        ctx: PoolingServeContext,
+    ):
+        pass
+
+    #######################################
+    # offline APIs
+
+    def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
+        assert not isinstance(ctx.prompts, ScoringData) and not (
+            isinstance(ctx.prompts, dict) and "data" in ctx.prompts
+        )
+
+        prompts_seq = prompt_to_seq(ctx.prompts)
+        tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
+            **(ctx.tokenization_kwargs or {})
+        )
+        return self._preprocess_cmpl_offline(prompts=prompts_seq, tok_params=tok_params)
+
+    def post_process_offline(
+        self,
+        ctx: OfflineOutputsContext,
+    ) -> list[PoolingRequestOutput]:
+        return ctx.outputs
+
+    #######################################
+    # helpers
+
+    def _preprocess_cmpl_online(
+        self,
+        request: RendererRequest,
+        prompt_input: str | list[str] | list[int] | list[list[int]] | None,
+        prompt_embeds: bytes | list[bytes] | None,
+    ) -> list[EngineInput]:
+        renderer = self.renderer
+        model_config = self.model_config
+
+        prompts = list[SingletonPrompt | bytes]()
+        if prompt_embeds is not None:  # embeds take higher priority
+            prompts.extend(prompt_to_seq(prompt_embeds))
+        if prompt_input is not None:
+            prompts.extend(prompt_to_seq(prompt_input))
+
+        parsed_prompts = [
+            (
+                prompt
+                if isinstance(prompt, bytes)
+                else parse_model_prompt(model_config, prompt)
+            )
+            for prompt in prompts
+        ]
+        tok_params = request.build_tok_params(model_config)
+
+        return renderer.render_cmpl(
+            parsed_prompts,
+            tok_params,
+            prompt_extras={
+                k: v
+                for k in ("mm_processor_kwargs", "cache_salt")
+                if (v := getattr(request, k, None)) is not None
+            },
+        )
+
+    def _preprocess_chat_online(
+        self,
+        request: RendererChatRequest,
+        messages: list[ChatCompletionMessageParam],
+        default_template: str | None,
+        default_template_content_format: ChatTemplateContentFormatOption,
+        default_template_kwargs: dict[str, Any] | None,
+        tool_dicts: list[dict[str, Any]] | None = None,
+        tool_parser: type[ToolParser] | None = None,
+    ) -> tuple[list[ConversationMessage], list[EngineInput]]:
+        renderer = self.renderer
+
+        default_template_kwargs = merge_kwargs(
+            default_template_kwargs,
+            dict(
+                tools=tool_dicts,
+                tokenize=is_mistral_tokenizer(renderer.tokenizer),
+            ),
+        )
+
+        mm_config = self.model_config.multimodal_config
+
+        tok_params = request.build_tok_params(self.model_config)
+        chat_params = request.build_chat_params(
+            default_template, default_template_content_format
+        ).with_defaults(
+            default_template_kwargs,
+            default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
+        )
+
+        (conversation,), (engine_input,) = renderer.render_chat(
+            [messages],
+            chat_params,
+            tok_params,
+            prompt_extras={
+                k: v
+                for k in ("mm_processor_kwargs", "cache_salt")
+                if (v := getattr(request, k, None)) is not None
+            },
+        )
+
+        return conversation, [engine_input]
+
+    def _preprocess_cmpl_offline(
+        self,
+        prompts: PromptType | Sequence[PromptType],
+        tok_params: TokenizeParams,
+        prompt_extras: dict[str, Any] | None = None,
+    ) -> Sequence[EngineInput]:
+        prompts = prompt_to_seq(prompts)
+        parsed_prompts = [
+            (
+                prompt
+                if isinstance(prompt, bytes)
+                else parse_model_prompt(self.model_config, prompt)
+            )
+            for prompt in prompts
+        ]
+
+        return self.renderer.render_cmpl(
+            parsed_prompts, tok_params, prompt_extras=prompt_extras
+        )
+
+    def _validate_chat_template(
+        self,
+        request_chat_template: str | None,
+        chat_template_kwargs: dict[str, Any] | None,
+        trust_request_chat_template: bool,
+    ):
+        if not trust_request_chat_template and (
+            request_chat_template is not None
+            or (
+                chat_template_kwargs
+                and chat_template_kwargs.get("chat_template") is not None
+            )
+        ):
+            raise ValueError(
+                "Chat template is passed with request, but "
+                "--trust-request-chat-template is not set. "
+                "Refused request with untrusted chat template."
+            )
+        return None
+
+    def _params_to_seq(
+        self,
+        params: PoolingParams | Sequence[PoolingParams],
+        num_requests: int,
+    ) -> Sequence[PoolingParams]:
+        if isinstance(params, Sequence):
+            if len(params) != num_requests:
+                raise ValueError(
+                    f"The lengths of prompts ({num_requests}) "
+                    f"and params ({len(params)}) must be the same."
+                )
+
+            return params
+
+        return [params] * num_requests

vllm/entrypoints/pooling/base/protocol.py

@@ -0,0 +1,322 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+
+from typing import Annotated, Any, Literal
+
+from pydantic import Field, model_validator
+
+from vllm.config import ModelConfig
+from vllm.entrypoints.chat_utils import (
+    ChatCompletionMessageParam,
+    ChatTemplateContentFormatOption,
+)
+from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel
+from vllm.exceptions import VLLMValidationError
+from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
+from vllm.utils import random_uuid
+from vllm.utils.serial_utils import EmbedDType, EncodingFormat, Endianness
+
+
+class PoolingBasicRequestMixin(OpenAIBaseModel):
+    # --8<-- [start:pooling-common-params]
+    model: str | None = None
+    user: str | None = None
+    # --8<-- [end:pooling-common-params]
+
+    # --8<-- [start:pooling-common-extra-params]
+    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
+    truncation_side: Literal["left", "right"] | None = Field(
+        default=None,
+        description=(
+            "Which side to truncate from when truncate_prompt_tokens is active. "
+            "'right' keeps the first N tokens. "
+            "'left' keeps the last N tokens."
+        ),
+    )
+    request_id: str = Field(
+        default_factory=random_uuid,
+        description=(
+            "The request_id related to this request. If the caller does "
+            "not set it, a random_uuid will be generated. This id is used "
+            "through out the inference process and return in response."
+        ),
+    )
+    priority: int = Field(
+        default=0,
+        ge=-(2**63),
+        le=2**63 - 1,
+        description=(
+            "The priority of the request (lower means earlier handling; "
+            "default: 0). Any priority other than 0 will raise an error "
+            "if the served model does not use priority scheduling."
+        ),
+    )
+    mm_processor_kwargs: dict[str, Any] | None = Field(
+        default=None,
+        description="Additional kwargs to pass to the HF processor.",
+    )
+    cache_salt: str | None = Field(
+        default=None,
+        description=(
+            "If specified, the prefix cache will be salted with the provided "
+            "string to prevent an attacker to guess prompts in multi-user "
+            "environments. The salt should be random, protected from "
+            "access by 3rd parties, and long enough to be "
+            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
+            "to 256 bit)."
+        ),
+    )
+    # --8<-- [end:pooling-common-extra-params]
+
+    @model_validator(mode="before")
+    @classmethod
+    def check_cache_salt_support(cls, data):
+        if not isinstance(data, dict):
+            return data
+
+        if data.get("cache_salt") is not None and (
+            not isinstance(data["cache_salt"], str) or not data["cache_salt"]
+        ):
+            raise VLLMValidationError(
+                "Parameter 'cache_salt' must be a non-empty string if provided.",
+                parameter="cache_salt",
+            )
+        return data
+
+    def _build_pooling_tok_params(
+        self,
+        model_config: ModelConfig,
+        *,
+        add_special_tokens: bool,
+        max_total_tokens: int | None,
+        max_output_tokens: int,
+        max_total_tokens_param: str = "max_model_len",
+        max_output_tokens_param: str | None = None,
+    ) -> TokenizeParams:
+        encoder_config = model_config.encoder_config or {}
+        if max_output_tokens_param is None:
+            return TokenizeParams(
+                max_total_tokens=max_total_tokens,
+                max_output_tokens=max_output_tokens,
+                truncate_prompt_tokens=self.truncate_prompt_tokens,
+                truncation_side=self.truncation_side,
+                do_lower_case=encoder_config.get("do_lower_case", False),
+                add_special_tokens=add_special_tokens,
+                max_total_tokens_param=max_total_tokens_param,
+            )
+
+        return TokenizeParams(
+            max_total_tokens=max_total_tokens,
+            max_output_tokens=max_output_tokens,
+            truncate_prompt_tokens=self.truncate_prompt_tokens,
+            truncation_side=self.truncation_side,
+            do_lower_case=encoder_config.get("do_lower_case", False),
+            add_special_tokens=add_special_tokens,
+            max_total_tokens_param=max_total_tokens_param,
+            max_output_tokens_param=max_output_tokens_param,
+        )
+
+
+class PoolingTokenizeParamsMixin:
+    add_special_tokens: bool
+
+    def _build_pooling_tok_params(
+        self,
+        model_config: ModelConfig,
+        *,
+        add_special_tokens: bool,
+        max_total_tokens: int | None,
+        max_output_tokens: int,
+        max_total_tokens_param: str = "max_model_len",
+        max_output_tokens_param: str | None = None,
+    ) -> TokenizeParams:
+        raise NotImplementedError
+
+
+class FixedMaxLenTokenizeParamsMixin(PoolingTokenizeParamsMixin):
+    def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
+        return self._build_pooling_tok_params(
+            model_config,
+            add_special_tokens=self.add_special_tokens,
+            max_total_tokens=model_config.max_model_len,
+            max_output_tokens=0,
+        )
+
+
+class EmbeddingTokenizeParamsMixin(PoolingTokenizeParamsMixin):
+    def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
+        default_max_total_tokens = model_config.max_model_len
+        max_total_tokens: int | None = default_max_total_tokens
+        max_output_tokens = 0
+
+        pooler_config = model_config.pooler_config
+        if pooler_config is not None:
+            if pooler_config.enable_chunked_processing:
+                max_total_tokens = None
+            else:
+                max_embed_len = pooler_config.max_embed_len or default_max_total_tokens
+                max_output_tokens = default_max_total_tokens - max_embed_len
+
+        return self._build_pooling_tok_params(
+            model_config,
+            add_special_tokens=self.add_special_tokens,
+            max_total_tokens=max_total_tokens,
+            max_output_tokens=max_output_tokens,
+            max_output_tokens_param="max_model_len - max_embed_len",
+        )
+
+
+class CompletionRequestMixin(OpenAIBaseModel):
+    # --8<-- [start:completion-params]
+    input: list[int] | list[list[int]] | str | list[str]
+    # --8<-- [end:completion-params]
+
+    # --8<-- [start:completion-extra-params]
+    add_special_tokens: bool = Field(
+        default=True,
+        description=(
+            "If true (the default), special tokens (e.g. BOS) will be added to "
+            "the prompt."
+        ),
+    )
+    # --8<-- [end:completion-extra-params]
+
+
+class ChatRequestOptionsMixin(OpenAIBaseModel):
+    # --8<-- [start:chat-extra-params]
+    add_generation_prompt: bool = Field(
+        default=False,
+        description=(
+            "If true, the generation prompt will be added to the chat template. "
+            "This is a parameter used by chat template in tokenizer config of the "
+            "model."
+        ),
+    )
+    continue_final_message: bool = Field(
+        default=False,
+        description=(
+            "If this is set, the chat will be formatted so that the final "
+            "message in the chat is open-ended, without any EOS tokens. The "
+            "model will continue this message rather than starting a new one. "
+            'This allows you to "prefill" part of the model\'s response for it. '
+            "Cannot be used at the same time as `add_generation_prompt`."
+        ),
+    )
+    add_special_tokens: bool = Field(
+        default=False,
+        description=(
+            "If true, special tokens (e.g. BOS) will be added to the prompt "
+            "on top of what is added by the chat template. "
+            "For most models, the chat template takes care of adding the "
+            "special tokens so this should be set to false (as is the "
+            "default)."
+        ),
+    )
+    chat_template: str | None = Field(
+        default=None,
+        description=(
+            "A Jinja template to use for this conversion. "
+            "As of transformers v4.44, default chat template is no longer "
+            "allowed, so you must provide a chat template if the tokenizer "
+            "does not define one."
+        ),
+    )
+    chat_template_kwargs: dict[str, Any] | None = Field(
+        default=None,
+        description=(
+            "Additional keyword args to pass to the template renderer. "
+            "Will be accessible by the chat template."
+        ),
+    )
+    media_io_kwargs: dict[str, dict[str, Any]] | None = Field(
+        default=None,
+        description=(
+            "Additional kwargs to pass to the media IO connectors, "
+            "keyed by modality. Merged with engine-level media_io_kwargs."
+        ),
+    )
+    # --8<-- [end:chat-extra-params]
+
+    @model_validator(mode="before")
+    @classmethod
+    def check_generation_prompt(cls, data):
+        if data.get("continue_final_message") and data.get("add_generation_prompt"):
+            raise VLLMValidationError(
+                "Cannot set both `continue_final_message` and "
+                "`add_generation_prompt` to True.",
+            )
+        return data
+
+    def build_chat_params(
+        self,
+        default_template: str | None,
+        default_template_content_format: ChatTemplateContentFormatOption,
+    ) -> ChatParams:
+        return ChatParams(
+            chat_template=self.chat_template or default_template,
+            chat_template_content_format=default_template_content_format,
+            chat_template_kwargs=merge_kwargs(
+                self.chat_template_kwargs,
+                dict(
+                    add_generation_prompt=self.add_generation_prompt,
+                    continue_final_message=self.continue_final_message,
+                ),
+            ),
+            media_io_kwargs=self.media_io_kwargs,
+        )
+
+
+class ChatRequestMixin(ChatRequestOptionsMixin):
+    # --8<-- [start:chat-params]
+    messages: list[ChatCompletionMessageParam]
+    # --8<-- [end:chat-params]
+
+
+class EncodingRequestMixin(OpenAIBaseModel):
+    # --8<-- [start:encoding-params]
+    encoding_format: EncodingFormat = "float"
+    # --8<-- [end:encoding-params]
+
+    # --8<-- [start:encoding-extra-params]
+    embed_dtype: EmbedDType = Field(
+        default="float32",
+        description=(
+            "What dtype to use for encoding. Default to using float32 for base64 "
+            "encoding to match the OpenAI python client behavior. "
+            "This parameter will affect base64 and binary_response."
+        ),
+    )
+    endianness: Endianness = Field(
+        default="native",
+        description=(
+            "What endianness to use for encoding. Default to using native for "
+            "base64 encoding to match the OpenAI python client behavior."
+            "This parameter will affect base64 and binary_response."
+        ),
+    )
+    # --8<-- [end:encoding-extra-params]
+
+
+class EmbedRequestMixin(EncodingRequestMixin):
+    # --8<-- [start:embed-params]
+    dimensions: int | None = None
+    # --8<-- [end:embed-params]
+
+    # --8<-- [start:embed-extra-params]
+    use_activation: bool | None = Field(
+        default=None,
+        description="Whether to use activation for the pooler outputs. "
+        "`None` uses the pooler's default, which is `True` in most cases.",
+    )
+    # --8<-- [end:embed-extra-params]
+
+
+class ClassifyRequestMixin(OpenAIBaseModel):
+    # --8<-- [start:classify-extra-params]
+    use_activation: bool | None = Field(
+        default=None,
+        description="Whether to use activation for the pooler outputs. "
+        "`None` uses the pooler's default, which is `True` in most cases.",
+    )
+    # --8<-- [end:classify-extra-params]

vllm/entrypoints/pooling/base/serving.py

@@ -0,0 +1,283 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from abc import ABC, abstractmethod
+from collections.abc import AsyncGenerator, Mapping
+from concurrent.futures import Executor
+from http import HTTPStatus
+from typing import ClassVar
+
+import torch
+from fastapi import Request
+from fastapi.responses import Response
+from starlette.datastructures import Headers
+
+from vllm import PoolingRequestOutput, envs
+from vllm.config import VllmConfig
+from vllm.engine.protocol import EngineClient
+from vllm.entrypoints.chat_utils import ChatTemplateConfig
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
+from vllm.entrypoints.serve.engine.serving import BaseServing
+from vllm.entrypoints.serve.engine.typing import AnyRequest
+from vllm.entrypoints.serve.utils.request_logger import RequestLogger
+from vllm.lora.request import LoRARequest
+from vllm.renderers.base import BaseRenderer
+from vllm.tracing import (
+    contains_trace_headers,
+    extract_trace_headers,
+    log_tracing_disabled_warning,
+)
+from vllm.utils.async_utils import make_async, merge_async_iterators
+
+from ..typing import AnyPoolingRequest, PoolingServeContext
+from .io_processor import PoolingIOProcessor
+
+
+class PoolingBaseServing(ABC, BaseServing):
+    request_id_prefix: ClassVar[str]
+
+    def __init__(
+        self,
+        engine_client: EngineClient,
+        models: OpenAIServingModels,
+        *,
+        request_logger: RequestLogger | None,
+        chat_template_config: ChatTemplateConfig,
+        return_tokens_as_token_ids: bool = False,
+        log_error_stack: bool = False,
+    ):
+        super().__init__(
+            models=models,
+            model_config=models.model_config,
+            request_logger=request_logger,
+        )
+
+        self.engine_client = engine_client
+        self.renderer = engine_client.renderer
+        self.vllm_config = engine_client.vllm_config
+        self.max_model_len = self.model_config.max_model_len
+        self.return_tokens_as_token_ids = return_tokens_as_token_ids
+        self.log_error_stack = log_error_stack
+        self.chat_template_config = chat_template_config
+
+        # Shared thread pool executor for preprocessing and postprocessing.
+        self._executor: Executor = self.renderer._executor
+        self._preprocessing_async = make_async(
+            self._preprocessing, executor=self._executor
+        )
+        self._postprocessing_async = make_async(
+            self._postprocessing, executor=self._executor
+        )
+
+    async def __call__(
+        self,
+        request: AnyPoolingRequest,
+        raw_request: Request | None = None,
+    ) -> Response:
+        io_processor = self.get_io_processor(request)
+        ctx = await self._init_ctx(io_processor, request, raw_request)
+        await self._preprocessing_async(io_processor, ctx)
+        await self._prepare_generators(ctx)
+        await self._collect_batch(ctx)
+        return await self._postprocessing_async(io_processor, ctx)
+
+    @abstractmethod
+    def get_io_processor(self, request: AnyPoolingRequest) -> PoolingIOProcessor:
+        raise NotImplementedError
+
+    @torch.inference_mode()
+    def _preprocessing(
+        self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
+    ):
+        return io_processor.pre_process_online(ctx)
+
+    @torch.inference_mode()
+    def _postprocessing(
+        self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
+    ):
+        io_processor.post_process_online(ctx)
+        return self._build_response(ctx)
+
+    async def _init_ctx(
+        self,
+        io_processor: PoolingIOProcessor,
+        request: AnyPoolingRequest,
+        raw_request: Request | None = None,
+    ):
+        model_name = self.models.model_name()
+        request_id = f"{self.request_id_prefix}-{self._base_request_id(raw_request)}"
+        await self._check_model(request)
+
+        pooling_params = io_processor.create_pooling_params(request)
+        ctx = PoolingServeContext(
+            request=request,
+            raw_request=raw_request,
+            model_name=model_name,
+            pooling_params=pooling_params,
+            request_id=request_id,
+        )
+
+        self._validate_request(ctx)
+        ctx.lora_request = self._maybe_get_adapters(ctx.request)
+        return ctx
+
+    async def _prepare_generators(
+        self,
+        ctx: PoolingServeContext,
+    ):
+        if ctx.engine_inputs is None:
+            raise ValueError("Engine prompts not available")
+
+        generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
+
+        trace_headers = (
+            None
+            if ctx.raw_request is None
+            else await self._get_trace_headers(ctx.raw_request.headers)
+        )
+
+        assert ctx.pooling_params is not None
+        pooling_params = ctx.pooling_params
+
+        if isinstance(pooling_params, list):
+            for params in pooling_params:
+                params.verify(self.model_config)
+        else:
+            pooling_params.verify(self.model_config)
+
+        for i, engine_input in enumerate(ctx.engine_inputs):
+            prompt_request_id = (
+                f"{ctx.request_id}-{i}"
+                if ctx.prompt_request_ids is None
+                else ctx.prompt_request_ids[i]
+            )
+
+            params = (
+                pooling_params[i]
+                if isinstance(pooling_params, list)
+                else pooling_params
+            )
+
+            self._log_inputs(
+                prompt_request_id,
+                engine_input,
+                params=params,
+                lora_request=ctx.lora_request,
+            )
+
+            generator = self.engine_client.encode(
+                engine_input,
+                params,
+                prompt_request_id,
+                lora_request=ctx.lora_request,
+                trace_headers=trace_headers,
+                priority=getattr(ctx.request, "priority", 0),
+            )
+
+            generators.append(generator)
+
+        ctx.result_generator = merge_async_iterators(*generators)
+
+    async def _collect_batch(
+        self,
+        ctx: PoolingServeContext,
+    ):
+        if ctx.engine_inputs is None:
+            raise ValueError("Engine prompts not available")
+
+        if ctx.result_generator is None:
+            raise ValueError("Result generator not available")
+
+        num_inputs = len(ctx.engine_inputs)
+        final_res_batch: list[PoolingRequestOutput | None]
+        final_res_batch = [None] * num_inputs
+
+        async for i, res in ctx.result_generator:
+            final_res_batch[i] = res
+
+        if None in final_res_batch:
+            raise ValueError("Failed to generate results for all prompts")
+
+        ctx.final_res_batch = [res for res in final_res_batch if res is not None]
+
+    @abstractmethod
+    def _build_response(
+        self,
+        ctx: PoolingServeContext,
+    ) -> Response:
+        raise NotImplementedError
+
+    async def _check_model(
+        self,
+        request: AnyRequest | AnyPoolingRequest,
+    ) -> ErrorResponse | None:
+        if self._is_model_supported(request.model):
+            return None
+        if request.model in self.models.lora_requests:
+            return None
+        if (
+            envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING
+            and request.model
+            and (load_result := await self.models.resolve_lora(request.model))
+        ):
+            if isinstance(load_result, LoRARequest):
+                return None
+            if (
+                isinstance(load_result, ErrorResponse)
+                and load_result.error.code == HTTPStatus.BAD_REQUEST.value
+            ):
+                raise ValueError(load_result.error.message)
+        return None
+
+    def _validate_request(self, ctx: PoolingServeContext) -> None:
+        truncate_prompt_tokens = getattr(ctx.request, "truncate_prompt_tokens", None)
+
+        if (
+            truncate_prompt_tokens is not None
+            and truncate_prompt_tokens > self.max_model_len
+        ):
+            raise ValueError(
+                "truncate_prompt_tokens value is "
+                "greater than max_model_len."
+                " Please request a smaller truncation size."
+            )
+
+        return None
+
+    async def _get_trace_headers(
+        self,
+        headers: Headers,
+    ) -> Mapping[str, str] | None:
+        is_tracing_enabled = await self.engine_client.is_tracing_enabled()
+
+        if is_tracing_enabled:
+            return extract_trace_headers(headers)
+
+        if contains_trace_headers(headers):
+            log_tracing_disabled_warning()
+
+        return None
+
+
+class PoolingServing(PoolingBaseServing, ABC):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.io_processor = self.init_io_processor(
+            vllm_config=self.vllm_config,
+            renderer=self.renderer,
+            chat_template_config=self.chat_template_config,
+        )
+
+    @abstractmethod
+    def init_io_processor(
+        self,
+        vllm_config: VllmConfig,
+        renderer: BaseRenderer,
+        chat_template_config: ChatTemplateConfig,
+    ) -> PoolingIOProcessor:
+        raise NotImplementedError
+
+    def get_io_processor(self, request: AnyPoolingRequest) -> PoolingIOProcessor:
+        return self.io_processor

vllm/entrypoints/pooling/classify/api_router.py

@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from fastapi import APIRouter, Depends, Request
+from fastapi.responses import Response
+
+from vllm.entrypoints.serve.utils.api_utils import (
+    load_aware_call,
+    validate_json_request,
+    with_cancellation,
+)
+
+from .protocol import ClassificationRequest
+from .serving import ServingClassification
+
+router = APIRouter()
+
+
+def classify(request: Request) -> ServingClassification | None:
+    return request.app.state.serving_classification
+
+
+@router.post("/classify", dependencies=[Depends(validate_json_request)])
+@with_cancellation
+@load_aware_call
+async def create_classify(
+    request: ClassificationRequest, raw_request: Request
+) -> Response:
+    handler = classify(raw_request)
+    if handler is None:
+        raise NotImplementedError("The model does not support Classification API")
+
+    return await handler(request, raw_request)

vllm/entrypoints/pooling/classify/io_processor.py

@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from ..base.io_processor import PoolingIOProcessor
+
+
+class ClassifyIOProcessor(PoolingIOProcessor):
+    name = "classify"
+
+
+class TokenClassifyIOProcessor(PoolingIOProcessor):
+    name = "token_classify"

vllm/entrypoints/pooling/classify/protocol.py

@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import time
+from typing import TypeAlias
+
+from pydantic import Field
+
+from vllm import PoolingParams
+from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
+from vllm.logger import init_logger
+from vllm.utils import random_uuid
+
+from ..base.protocol import (
+    ChatRequestMixin,
+    ClassifyRequestMixin,
+    CompletionRequestMixin,
+    FixedMaxLenTokenizeParamsMixin,
+    PoolingBasicRequestMixin,
+)
+
+logger = init_logger(__name__)
+
+
+class ClassificationCompletionRequest(
+    PoolingBasicRequestMixin,
+    CompletionRequestMixin,
+    ClassifyRequestMixin,
+    FixedMaxLenTokenizeParamsMixin,
+):
+    def to_pooling_params(self):
+        return PoolingParams(
+            task="classify",
+            use_activation=self.use_activation,
+        )
+
+
+class ClassificationChatRequest(
+    PoolingBasicRequestMixin,
+    ChatRequestMixin,
+    ClassifyRequestMixin,
+    FixedMaxLenTokenizeParamsMixin,
+):
+    def to_pooling_params(self):
+        return PoolingParams(
+            task="classify",
+            use_activation=self.use_activation,
+        )
+
+
+ClassificationRequest: TypeAlias = (
+    ClassificationCompletionRequest | ClassificationChatRequest
+)
+
+
+class ClassificationData(OpenAIBaseModel):
+    index: int
+    label: str | None
+    probs: list[float]
+    num_classes: int
+
+
+class ClassificationResponse(OpenAIBaseModel):
+    id: str = Field(default_factory=lambda: f"classify-{random_uuid()}")
+    object: str = "list"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    model: str
+    data: list[ClassificationData]
+    usage: UsageInfo

vllm/entrypoints/pooling/classify/serving.py

@@ -0,0 +1,72 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from typing import TypeAlias
+
+import numpy as np
+from fastapi.responses import JSONResponse
+
+from vllm.entrypoints.openai.engine.protocol import UsageInfo
+from vllm.logger import init_logger
+from vllm.outputs import ClassificationOutput
+
+from ..base.serving import PoolingServing
+from ..typing import PoolingServeContext
+from .io_processor import ClassifyIOProcessor
+from .protocol import (
+    ClassificationData,
+    ClassificationRequest,
+    ClassificationResponse,
+)
+
+logger = init_logger(__name__)
+
+
+ClassificationServeContext: TypeAlias = PoolingServeContext[ClassificationRequest]
+
+
+class ServingClassification(PoolingServing):
+    request_id_prefix = "classify"
+
+    def init_io_processor(self, *args, **kwargs) -> ClassifyIOProcessor:
+        return ClassifyIOProcessor(*args, **kwargs)
+
+    def _build_response(
+        self,
+        ctx: ClassificationServeContext,
+    ) -> JSONResponse:
+        id2label = getattr(self.model_config.hf_config, "id2label", {})
+        num_prompt_tokens = 0
+        items: list[ClassificationData] = []
+        for idx, final_res in enumerate(ctx.final_res_batch):
+            classify_res = ClassificationOutput.from_base(final_res.outputs)
+
+            probs = classify_res.probs
+            predicted_index = int(np.argmax(probs))
+            label = id2label.get(predicted_index)
+
+            item = ClassificationData(
+                index=idx,
+                label=label,
+                probs=probs,
+                num_classes=len(probs),
+            )
+
+            items.append(item)
+            prompt_token_ids = final_res.prompt_token_ids
+            num_prompt_tokens += len(prompt_token_ids)
+
+        usage = UsageInfo(
+            prompt_tokens=num_prompt_tokens,
+            total_tokens=num_prompt_tokens,
+        )
+
+        response = ClassificationResponse(
+            id=ctx.request_id,
+            created=ctx.created_time,
+            model=ctx.model_name,
+            data=items,
+            usage=usage,
+        )
+
+        return JSONResponse(content=response.model_dump())

vllm/entrypoints/pooling/embed/api_router.py

@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from http import HTTPStatus
+
+from fastapi import APIRouter, Depends, Request
+
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
+from vllm.entrypoints.serve.utils.api_utils import (
+    load_aware_call,
+    validate_json_request,
+    with_cancellation,
+)
+
+from .protocol import CohereEmbedRequest, EmbeddingRequest
+from .serving import ServingEmbedding
+
+router = APIRouter()
+
+
+def embedding(request: Request) -> ServingEmbedding | None:
+    return request.app.state.serving_embedding
+
+
+@router.post(
+    "/v1/embeddings",
+    dependencies=[Depends(validate_json_request)],
+    responses={
+        HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
+        HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
+    },
+)
+@with_cancellation
+@load_aware_call
+async def create_embedding(
+    request: EmbeddingRequest,
+    raw_request: Request,
+):
+    handler = embedding(raw_request)
+    if handler is None:
+        raise NotImplementedError("The model does not support Embeddings API")
+
+    return await handler(request, raw_request)
+
+
+@router.post(
+    "/v2/embed",
+    dependencies=[Depends(validate_json_request)],
+    responses={
+        HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
+        HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
+    },
+)
+@with_cancellation
+@load_aware_call
+async def create_cohere_embedding(
+    request: CohereEmbedRequest,
+    raw_request: Request,
+):
+    handler = embedding(raw_request)
+    if handler is None:
+        raise NotImplementedError("The model does not support Embeddings API")
+
+    return await handler(request, raw_request)

vllm/entrypoints/pooling/embed/io_processor.py

@@ -0,0 +1,680 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Any, Literal, cast
+
+import torch
+from openai.types.chat import (
+    ChatCompletionContentPartImageParam,
+    ChatCompletionContentPartTextParam,
+)
+from openai.types.chat.chat_completion_content_part_image_param import ImageURL
+
+from vllm import PoolingParams
+from vllm.entrypoints.chat_utils import (
+    ChatCompletionContentPartParam,
+    ChatCompletionMessageParam,
+    CustomChatCompletionMessageParam,
+)
+from vllm.inputs import EngineInput, tokens_input
+from vllm.logger import init_logger
+from vllm.outputs import PoolingOutput, PoolingRequestOutput
+from vllm.renderers import merge_kwargs
+from vllm.renderers.hf import resolve_chat_template
+from vllm.utils.collection_utils import chunk_list
+from vllm.utils.mistral import is_mistral_tokenizer
+
+from ..base.io_processor import PoolingIOProcessor
+from ..scoring.io_processor import JinaRankingIOProcessorMixin
+from ..typing import (
+    ChunkedEmbeddingMetadata,
+    OfflineInputsContext,
+    PoolingChatLikeRequest,
+    PoolingCompletionLikeRequest,
+    PoolingServeContext,
+)
+from .protocol import (
+    CohereEmbedContent,
+    CohereEmbedInput,
+    CohereEmbedRequest,
+    EmbeddingBatchChatInputRequest,
+    EmbeddingBatchChatRequest,
+    EmbeddingChatInputRequest,
+    EmbeddingChatRequest,
+    EmbeddingCompletionRequest,
+)
+
+logger = init_logger(__name__)
+
+
+@dataclass
+class _ChunkedPromptAggregator:
+    weighted_sum: torch.Tensor | None = None
+    total_weight: int = 0
+
+
+class EmbedIOProcessor(PoolingIOProcessor):
+    name = "embed"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        assert self.model_config.pooler_config is not None
+
+        self.pooler_config = self.model_config.pooler_config
+        self.enable_chunked_processing = self.pooler_config.enable_chunked_processing
+
+        # Load task instructions from HF config or sentence-transformers config
+        self.task_instructions: dict[str, str] | None = self._load_task_instructions(
+            self.model_config.hf_config
+        ) or self._load_st_prompts(self.model_config.model, self.model_config.revision)
+        if self.task_instructions:
+            logger.info(
+                "Loaded prompt prefixes for input_type: %s",
+                list(self.task_instructions.keys()),
+            )
+
+    def pre_process_online(self, ctx: PoolingServeContext):
+        if isinstance(ctx.request, CohereEmbedRequest):
+            self._pre_process_cohere_online(ctx)
+        elif isinstance(
+            ctx.request,
+            (
+                EmbeddingChatRequest,
+                EmbeddingBatchChatRequest,
+                EmbeddingChatInputRequest,
+                EmbeddingBatchChatInputRequest,
+            ),
+        ):
+            self._pre_process_openai_chat_online(ctx)
+        else:
+            super().pre_process_online(ctx)
+
+        if self.enable_chunked_processing:
+            self._pre_process_chunked(ctx)
+
+    def post_process_online(
+        self,
+        ctx: PoolingServeContext,
+    ):
+        if ctx.final_res_batch is None:
+            raise ValueError("Final response batch not available")
+
+        if not self.enable_chunked_processing:
+            self._enforce_cohere_max_tokens(ctx)
+            return super().post_process_online(ctx)
+
+        self._post_process_chunked(ctx)
+        self._enforce_cohere_max_tokens(ctx)
+
+    #################################################################
+    # Long Text Embedding with Chunked Processing
+    # PTAL: examples/pooling/embed/openai_embedding_long_text
+    #################################################################
+
+    def _pre_process_chunked(self, ctx: PoolingServeContext) -> None:
+        if ctx.engine_inputs is None:
+            raise ValueError("Engine prompts not available")
+
+        ctx.original_engine_inputs = ctx.engine_inputs
+        request_id = ctx.request_id
+        max_model_len = self.model_config.max_model_len
+        chunked_engine_inputs: list[EngineInput] = []
+        prompt_request_ids: list[str] = []
+        chunked_embedding_metadata: list[ChunkedEmbeddingMetadata] = []
+        for prompt_idx, engine_input in enumerate(ctx.engine_inputs):
+            token_ids = engine_input.get("prompt_token_ids", None)
+            if token_ids is None:
+                raise NotImplementedError(
+                    "Long Text Embedding with Chunked Processing does "
+                    "not support EmbedsPrompt and EncoderDecoderInput."
+                )
+
+            prompt_token_ids = cast(list[int], token_ids)
+
+            for chunk_idx, chunk_tokens in enumerate(
+                chunk_list(prompt_token_ids, max_model_len)
+            ):
+                chunked_engine_inputs.append(
+                    tokens_input(prompt_token_ids=chunk_tokens)
+                )
+                prompt_request_ids.append(
+                    f"{request_id}-prompt-{prompt_idx}-chunk-{chunk_idx}"
+                )
+                chunked_embedding_metadata.append(
+                    ChunkedEmbeddingMetadata(
+                        prompt_index=prompt_idx,
+                        chunk_index=chunk_idx,
+                    )
+                )
+
+        ctx.engine_inputs = chunked_engine_inputs
+        ctx.prompt_request_ids = prompt_request_ids
+        ctx.chunked_embedding_metadata = chunked_embedding_metadata
+
+        return None
+
+    def _post_process_chunked(self, ctx: PoolingServeContext) -> None:
+        # Online aggregation for chunked requests to
+        # minimize memory usage
+        # Track aggregation state for each prompt
+        if ctx.chunked_embedding_metadata is None:
+            raise ValueError("Chunked embedding metadata not available")
+        if len(ctx.chunked_embedding_metadata) != len(ctx.final_res_batch):
+            raise ValueError(
+                "Chunked embedding metadata count does not match result count"
+            )
+
+        prompt_aggregators: dict[int, _ChunkedPromptAggregator] = {}
+        for result, chunk_metadata in zip(
+            ctx.final_res_batch, ctx.chunked_embedding_metadata
+        ):
+            prompt_idx = chunk_metadata.prompt_index
+            aggregator = prompt_aggregators.setdefault(
+                prompt_idx, _ChunkedPromptAggregator()
+            )
+
+            # MEAN pooling with online weighted averaging
+            # Ensure result is PoolingRequestOutput
+            # for embedding processing
+            if not isinstance(result, PoolingRequestOutput):
+                raise ValueError(
+                    f"Expected PoolingRequestOutput for "
+                    f"chunked embedding, got "
+                    f"{type(result).__name__}"
+                )
+            if result.prompt_token_ids is None:
+                raise ValueError(
+                    "prompt_token_ids cannot be None for chunked processing"
+                )
+
+            weight = len(result.prompt_token_ids)
+            embedding_data = result.outputs.data
+            weighted_embedding = embedding_data.to(dtype=torch.float32) * weight
+
+            if aggregator.weighted_sum is None:
+                # First chunk
+                aggregator.weighted_sum = weighted_embedding
+            else:
+                # Accumulate
+                aggregator.weighted_sum += weighted_embedding
+
+            aggregator.total_weight += weight
+
+        if ctx.original_engine_inputs is None:
+            raise ValueError("Original engine inputs not available")
+
+        original_engine_inputs = ctx.original_engine_inputs
+        num_prompts = len(original_engine_inputs)
+
+        # Finalize aggregated results
+        final_res_batch: list[PoolingRequestOutput] = []
+        for prompt_idx in range(num_prompts):
+            if prompt_idx in prompt_aggregators:
+                # Finalize MEAN aggregation for this chunked prompt
+                aggregator = prompt_aggregators[prompt_idx]
+
+                weighted_sum = aggregator.weighted_sum
+                total_weight = aggregator.total_weight
+
+                if (
+                    weighted_sum is not None
+                    and isinstance(weighted_sum, torch.Tensor)
+                    and isinstance(total_weight, (int, float))
+                    and total_weight > 0
+                ):
+                    # Compute final mean embedding
+                    final_embedding = weighted_sum / total_weight
+
+                    # Create a PoolingRequestOutput
+                    # for the aggregated result
+                    pooling_output_data = PoolingOutput(data=final_embedding)
+
+                    # Get original prompt token IDs for this prompt
+                    original_prompt = original_engine_inputs[prompt_idx]
+                    token_ids = original_prompt.get("prompt_token_ids", None)
+                    if token_ids is None:
+                        raise NotImplementedError(
+                            "Long Text Embedding with Chunked Processing does "
+                            "not support EmbedsPrompt and EncoderDecoderInput."
+                        )
+
+                    original_token_ids = cast(list[int], token_ids)
+                    pooling_request_output = PoolingRequestOutput(
+                        request_id=f"{ctx.request_id}-prompt-{prompt_idx}",
+                        prompt_token_ids=original_token_ids,
+                        outputs=pooling_output_data,
+                        num_cached_tokens=0,
+                        finished=True,
+                    )
+
+                    final_res_batch.append(pooling_request_output)
+                else:
+                    raise ValueError(
+                        f"Failed to aggregate chunks for prompt {prompt_idx}"
+                    )
+            else:
+                raise ValueError(f"Result not found for prompt {prompt_idx}")
+
+        ctx.final_res_batch = final_res_batch
+
+        return None
+
+    #################################################################
+    # Cohere Request Preprocessing & Postprocessing
+    #################################################################
+
+    @staticmethod
+    def _load_task_instructions(hf_config: Any) -> dict[str, str] | None:
+        """Extract ``task_instructions`` from the HF model config."""
+        ti = getattr(hf_config, "task_instructions", None)
+        if not isinstance(ti, dict) or not ti:
+            return None
+        return {k: v for k, v in ti.items() if isinstance(v, str)}
+
+    @staticmethod
+    def _load_st_prompts(
+        model: str | Any,
+        revision: str | None,
+    ) -> dict[str, str] | None:
+        """Load ``task_instructions`` from ``config_sentence_transformers.json``."""
+        from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
+
+        try:
+            cfg = get_hf_file_to_dict(
+                "config_sentence_transformers.json", str(model), revision
+            )
+        except (ValueError, OSError):
+            return None
+
+        if cfg is None:
+            return None
+        prompts = cfg.get("prompts")
+        if not isinstance(prompts, dict) or not prompts:
+            return None
+        return {k: v for k, v in prompts.items() if isinstance(v, str)}
+
+    @staticmethod
+    def _mixed_input_to_messages(
+        inp: CohereEmbedInput,
+        *,
+        task_prefix: str | None = None,
+    ) -> list[ChatCompletionMessageParam]:
+        """Build chat messages from a mixed text+image input.
+
+        When *task_prefix* is given, it is used as the system prompt.
+        """
+        messages: list[ChatCompletionMessageParam] = []
+        if task_prefix is not None:
+            messages.append(
+                CustomChatCompletionMessageParam(
+                    role="system",
+                    content=[
+                        ChatCompletionContentPartTextParam(
+                            type="text", text=task_prefix
+                        )
+                    ],
+                )
+            )
+
+        parts: list[ChatCompletionContentPartParam] = []
+        for item in inp.content:
+            if item.type == "text" and item.text is not None:
+                parts.append(
+                    ChatCompletionContentPartTextParam(type="text", text=item.text)
+                )
+            elif item.type == "image_url" and item.image_url is not None:
+                parts.append(
+                    ChatCompletionContentPartImageParam(
+                        type="image_url",
+                        image_url=ImageURL(url=item.image_url["url"]),
+                    )
+                )
+        messages.append(CustomChatCompletionMessageParam(role="user", content=parts))
+        return messages
+
+    @staticmethod
+    def _check_cohere_max_tokens(
+        outputs: list[PoolingRequestOutput],
+        max_tokens_check: int | None,
+    ) -> None:
+        """Raise if any output exceeds *max_tokens_check* tokens.
+
+        Used to enforce ``truncate=NONE`` with an explicit ``max_tokens``:
+        the pipeline runs without truncation and we reject afterwards.
+        """
+        if max_tokens_check is None:
+            return
+        for out in outputs:
+            n = len(out.prompt_token_ids)
+            if n > max_tokens_check:
+                raise ValueError(
+                    f"Input of {n} tokens exceeds max_tokens={max_tokens_check} "
+                    "with truncate=NONE. Set truncate to END or START to "
+                    "allow truncation."
+                )
+
+    @staticmethod
+    def _resolve_cohere_truncation(
+        request: CohereEmbedRequest,
+    ) -> tuple[int | None, Literal["left", "right"] | None]:
+        """Return ``(truncate_prompt_tokens, truncation_side)``."""
+        if request.truncate == "NONE":
+            return None, None
+        if request.truncate == "START":
+            tokens = request.max_tokens if request.max_tokens is not None else -1
+            return tokens, "left"
+        if request.max_tokens is not None:
+            return request.max_tokens, None
+        return -1, None
+
+    def create_pooling_params(self, request):
+        if isinstance(request, CohereEmbedRequest):
+            return PoolingParams(
+                task="embed",
+                dimensions=request.output_dimension,
+            )
+        return super().create_pooling_params(request)
+
+    def _pre_process_openai_chat_online(
+        self,
+        ctx: PoolingServeContext[
+            EmbeddingChatRequest
+            | EmbeddingBatchChatRequest
+            | EmbeddingChatInputRequest
+            | EmbeddingBatchChatInputRequest
+        ],
+    ) -> None:
+        request = ctx.request
+        self._validate_chat_template(
+            request_chat_template=request.chat_template,
+            chat_template_kwargs=request.chat_template_kwargs,
+            trust_request_chat_template=self.trust_request_chat_template,
+        )
+
+        if isinstance(
+            request, (EmbeddingBatchChatRequest, EmbeddingBatchChatInputRequest)
+        ):
+            all_messages = request.messages
+        else:
+            all_messages = [request.messages]
+        ctx.engine_inputs = self._batch_render_openai_chat(request, all_messages)
+
+    def _batch_render_openai_chat(
+        self,
+        request: (
+            EmbeddingChatRequest
+            | EmbeddingBatchChatRequest
+            | EmbeddingChatInputRequest
+            | EmbeddingBatchChatInputRequest
+        ),
+        all_messages: Sequence[list[ChatCompletionMessageParam]],
+    ) -> list[EngineInput]:
+        renderer = self.renderer
+        mm_config = self.model_config.multimodal_config
+
+        tok_params = request.build_tok_params(self.model_config)
+        chat_params = request.build_chat_params(
+            self.chat_template,
+            self.chat_template_content_format,
+        ).with_defaults(
+            merge_kwargs(
+                None,
+                dict(
+                    tools=None,
+                    tokenize=is_mistral_tokenizer(renderer.tokenizer),
+                ),
+            ),
+            default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
+        )
+
+        _, engine_inputs = renderer.render_chat(
+            all_messages,
+            chat_params,
+            tok_params,
+            prompt_extras={
+                k: v
+                for k in ("mm_processor_kwargs", "cache_salt")
+                if (v := getattr(request, k, None)) is not None
+            },
+        )
+        return engine_inputs
+
+    def _pre_process_cohere_online(self, ctx: PoolingServeContext) -> None:
+        """Convert a ``CohereEmbedRequest`` into engine prompts.
+
+        If a model has a chat template the task instruction are rendered
+        as a system prompt. Otherwise they are just prepended to the input text.
+
+        Images and mixed inputs are always batch-rendered through the chat
+        template in one ``render_chat`` call.
+        """
+        request = ctx.request
+        assert isinstance(request, CohereEmbedRequest)
+
+        if request.texts is None and request.images is None and request.inputs is None:
+            raise ValueError("One of texts, images, or inputs must be provided")
+
+        truncate_prompt_tokens, truncation_side = self._resolve_cohere_truncation(
+            request
+        )
+        input_type = request.input_type
+        self._validate_input_type(input_type)
+
+        if request.images is not None:
+            input: list[CohereEmbedInput] = [
+                CohereEmbedInput(
+                    content=[
+                        CohereEmbedContent(type="image_url", image_url={"url": uri})
+                    ]
+                )
+                for uri in request.images
+            ]
+        elif request.inputs is not None:
+            input = request.inputs
+        else:
+            texts = request.texts or []
+            task_prefix = self._get_task_instruction_prefix(input_type)
+
+            if task_prefix is None:
+                ctx.engine_inputs = self._preprocess_cohere_text_completion(
+                    request,
+                    texts,
+                    truncate_prompt_tokens,
+                    truncation_side,
+                )
+                return
+
+            all_messages = [
+                self._mixed_input_to_messages(
+                    CohereEmbedInput(
+                        content=[CohereEmbedContent(type="text", text=text)]
+                    ),
+                    task_prefix=task_prefix,
+                )
+                for text in texts
+            ]
+            if self._has_chat_template():
+                ctx.engine_inputs = self._batch_render_chat(
+                    request,
+                    all_messages,
+                    truncate_prompt_tokens,
+                    truncation_side,
+                )
+            else:
+                ctx.engine_inputs = self._preprocess_cohere_text_completion(
+                    request,
+                    self._apply_task_instruction(texts, input_type),
+                    truncate_prompt_tokens,
+                    truncation_side,
+                )
+            return
+
+        task_prefix = self._get_task_instruction_prefix(input_type)
+        all_messages = [
+            self._mixed_input_to_messages(inp, task_prefix=task_prefix) for inp in input
+        ]
+        ctx.engine_inputs = self._batch_render_chat(
+            request, all_messages, truncate_prompt_tokens, truncation_side
+        )
+
+    def _has_chat_template(self) -> bool:
+        return (
+            resolve_chat_template(
+                self.renderer.tokenizer,
+                chat_template=self.chat_template,
+                tools=None,
+                model_config=self.model_config,
+            )
+            is not None
+        )
+
+    def _preprocess_cohere_text_completion(
+        self,
+        request: CohereEmbedRequest,
+        texts: list[str],
+        truncate_prompt_tokens: int | None,
+        truncation_side: Literal["left", "right"] | None,
+    ) -> list[EngineInput]:
+        proxy = EmbeddingCompletionRequest(
+            model=request.model,
+            input=texts,
+            dimensions=request.output_dimension,
+            encoding_format="float",
+            truncate_prompt_tokens=truncate_prompt_tokens,
+            truncation_side=truncation_side,
+        )
+        return self._preprocess_cmpl_online(
+            proxy, prompt_input=proxy.input, prompt_embeds=None
+        )
+
+    def _batch_render_chat(
+        self,
+        request: CohereEmbedRequest,
+        all_messages: Sequence[list[ChatCompletionMessageParam]],
+        truncate_prompt_tokens: int | None,
+        truncation_side: Literal["left", "right"] | None,
+    ) -> list[EngineInput]:
+        """Batch-render multiple conversations through the chat template."""
+        if not all_messages:
+            return []
+
+        proxy = EmbeddingChatRequest(
+            model=request.model,
+            messages=list(all_messages[0]),
+            dimensions=request.output_dimension,
+            encoding_format="float",
+            truncate_prompt_tokens=truncate_prompt_tokens,
+            truncation_side=truncation_side,
+        )
+
+        renderer = self.renderer
+        mm_config = self.model_config.multimodal_config
+
+        tok_params = proxy.build_tok_params(self.model_config)
+        chat_params = proxy.build_chat_params(
+            self.chat_template,
+            self.chat_template_content_format,
+        ).with_defaults(
+            merge_kwargs(
+                None,
+                dict(
+                    tools=None,
+                    tokenize=is_mistral_tokenizer(renderer.tokenizer),
+                ),
+            ),
+            default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
+        )
+
+        _, engine_inputs = renderer.render_chat(all_messages, chat_params, tok_params)
+        return engine_inputs
+
+    def _validate_input_type(self, input_type: str | None) -> None:
+        """Raise if *input_type* is not supported by this model."""
+        if input_type is None:
+            return
+        if self.task_instructions is None:
+            raise ValueError(
+                f"Unsupported input_type {input_type!r}. "
+                "This model does not define any input_type task instructions."
+            )
+        if input_type not in self.task_instructions:
+            supported = ", ".join(sorted(self.task_instructions))
+            raise ValueError(
+                f"Unsupported input_type {input_type!r}. Supported values: {supported}"
+            )
+
+    def _apply_task_instruction(
+        self,
+        texts: list[str],
+        input_type: str | None,
+    ) -> list[str]:
+        """Prepend the task-instruction prefix for *input_type*.
+
+        Returns *texts* unchanged when no matching prefix is configured.
+        """
+        prefix = self._get_task_instruction_prefix(input_type)
+        if not prefix:
+            return texts
+        return [prefix + t for t in texts]
+
+    def _get_task_instruction_prefix(self, input_type: str | None) -> str | None:
+        """Return the task-instruction prefix for *input_type*, or ``None``."""
+        if not self.task_instructions or input_type is None:
+            return None
+        return self.task_instructions.get(input_type) or None
+
+    def _enforce_cohere_max_tokens(self, ctx: PoolingServeContext) -> None:
+        if isinstance(ctx.request, CohereEmbedRequest):
+            request = ctx.request
+            if request.truncate == "NONE" and request.max_tokens is not None:
+                self._check_cohere_max_tokens(ctx.final_res_batch, request.max_tokens)
+
+
+class TokenEmbedIOProcessor(PoolingIOProcessor):
+    name = "token_embed"
+
+
+class JinaRankingTokenEmbedIOProcessor(
+    TokenEmbedIOProcessor, JinaRankingIOProcessorMixin
+):
+    def pre_process_online(self, ctx: PoolingServeContext):
+        request = ctx.request
+        if isinstance(request, PoolingCompletionLikeRequest):
+            prompts = request.input
+            if not isinstance(prompts, Sequence) or len(prompts) < 2:
+                raise ValueError("The JinaForRanking model requires at least 2 inputs.")
+
+            text_prompts = self.ensure_str(prompts)
+
+            # The JinaForRanking model concatenates docs first, then query.
+            # Let's stay consistent with this novel design.
+            prompt_input = self.format_docs_prompts_func(
+                query=text_prompts[-1], docs=text_prompts[:-1]
+            )
+
+            engine_inputs = self._preprocess_cmpl_online(
+                request,
+                prompt_input=prompt_input,
+                prompt_embeds=None,
+            )
+        elif isinstance(request, PoolingChatLikeRequest):
+            raise ValueError("The JinaForRanking does not support chat Request.")
+        else:
+            raise ValueError(f"Invalid {self.name} request type")
+
+        ctx.engine_inputs = engine_inputs
+
+    def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
+        if not isinstance(ctx.prompts, Sequence) or len(ctx.prompts) < 2:
+            raise ValueError("The JinaForRanking model requires at least 2 inputs.")
+
+        text_prompts = self.ensure_str(ctx.prompts)
+
+        # The JinaForRanking model concatenates docs first, then query.
+        # Let's stay consistent with this novel design.
+        ctx.prompts = self.format_docs_prompts_func(
+            query=text_prompts[-1], docs=text_prompts[:-1]
+        )
+
+        return super().pre_process_offline(ctx)

vllm/entrypoints/pooling/embed/protocol.py

@@ -0,0 +1,354 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Embedding API protocol models for OpenAI and Cohere formats.
+
+OpenAI: https://platform.openai.com/docs/api-reference/embeddings
+Cohere: https://docs.cohere.com/reference/embed
+"""
+
+import builtins
+import struct
+import time
+from collections.abc import Sequence
+from typing import Annotated, Any, Literal, TypeAlias
+
+import pybase64 as base64
+from pydantic import BaseModel, Field, model_validator
+
+from vllm import PoolingParams
+from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
+from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
+from vllm.utils import random_uuid
+
+from ..base.protocol import (
+    ChatRequestMixin,
+    ChatRequestOptionsMixin,
+    CompletionRequestMixin,
+    EmbeddingTokenizeParamsMixin,
+    EmbedRequestMixin,
+    PoolingBasicRequestMixin,
+)
+
+
+class EmbeddingCompletionRequest(
+    PoolingBasicRequestMixin,
+    CompletionRequestMixin,
+    EmbedRequestMixin,
+    EmbeddingTokenizeParamsMixin,
+):
+    def to_pooling_params(self):
+        return PoolingParams(
+            task="embed",
+            dimensions=self.dimensions,
+            use_activation=self.use_activation,
+        )
+
+
+def _is_chat_message(value: Any) -> bool:
+    return isinstance(value, dict) and isinstance(value.get("role"), str)
+
+
+def _is_chat_messages(value: Any) -> bool:
+    return (
+        isinstance(value, list)
+        and bool(value)
+        and all(_is_chat_message(item) for item in value)
+    )
+
+
+def _is_batched_chat_messages(value: Any) -> bool:
+    return (
+        isinstance(value, list)
+        and bool(value)
+        and all(_is_chat_messages(item) for item in value)
+    )
+
+
+class EmbeddingChatRequest(
+    PoolingBasicRequestMixin,
+    ChatRequestMixin,
+    EmbedRequestMixin,
+    EmbeddingTokenizeParamsMixin,
+):
+    """OpenAI embeddings request with one top-level chat conversation."""
+
+    def to_pooling_params(self):
+        return PoolingParams(
+            task="embed",
+            dimensions=self.dimensions,
+            use_activation=self.use_activation,
+        )
+
+
+class EmbeddingBatchChatRequest(
+    PoolingBasicRequestMixin,
+    ChatRequestOptionsMixin,
+    EmbedRequestMixin,
+    EmbeddingTokenizeParamsMixin,
+):
+    """OpenAI embeddings request with batched top-level chat conversations.
+
+    Mirrors ``BatchChatCompletionRequest`` by keeping batched conversations in
+    ``messages`` instead of introducing a separate batch-specific field.
+    """
+
+    messages: list[Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]] = (
+        Field(..., min_length=1)
+    )
+
+    def to_pooling_params(self):
+        return PoolingParams(
+            task="embed",
+            dimensions=self.dimensions,
+            use_activation=self.use_activation,
+        )
+
+
+class EmbeddingChatInputRequest(
+    EmbeddingChatRequest,
+):
+    """OpenAI embeddings request with one chat conversation in ``input``."""
+
+    input: list[ChatCompletionMessageParam]
+
+    @model_validator(mode="before")
+    @classmethod
+    def normalize_input_messages(cls, data):
+        if not isinstance(data, dict):
+            return data
+
+        if "messages" in data or "input" not in data:
+            return data
+
+        input_data = data["input"]
+        if not _is_chat_messages(input_data):
+            return data
+
+        normalized = dict(data)
+        normalized["messages"] = input_data
+        return normalized
+
+
+class EmbeddingBatchChatInputRequest(EmbeddingBatchChatRequest):
+    """OpenAI embeddings request with batched chat conversations in ``input``."""
+
+    input: list[Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]] = (
+        Field(..., min_length=1)
+    )
+
+    @model_validator(mode="before")
+    @classmethod
+    def normalize_input_messages(cls, data):
+        if not isinstance(data, dict):
+            return data
+
+        if "messages" in data or "input" not in data:
+            return data
+
+        input_data = data["input"]
+        if not _is_batched_chat_messages(input_data):
+            return data
+
+        normalized = dict(data)
+        normalized["messages"] = input_data
+        return normalized
+
+
+EmbeddingRequest: TypeAlias = (
+    EmbeddingCompletionRequest
+    | EmbeddingChatRequest
+    | EmbeddingBatchChatRequest
+    | EmbeddingChatInputRequest
+    | EmbeddingBatchChatInputRequest
+)
+
+
+# ---------------------------------------------------------------------------
+# OpenAI /v1/embeddings — response models
+# ---------------------------------------------------------------------------
+
+
+class EmbeddingResponseData(OpenAIBaseModel):
+    index: int
+    object: str = "embedding"
+    embedding: list[float] | str
+
+
+class EmbeddingResponse(OpenAIBaseModel):
+    id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
+    object: str = "list"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    model: str | None = None
+    data: list[EmbeddingResponseData]
+    usage: UsageInfo
+
+
+class EmbeddingBytesResponse(OpenAIBaseModel):
+    content: list[bytes]
+    headers: dict[str, str] | None = None
+    media_type: str = "application/octet-stream"
+
+
+# ---------------------------------------------------------------------------
+# Cohere /v2/embed — request models
+# ---------------------------------------------------------------------------
+
+CohereEmbeddingType = Literal[
+    "float",
+    "binary",
+    "ubinary",
+    "base64",
+]
+CohereTruncate = Literal["NONE", "START", "END"]
+
+
+class CohereEmbedContent(BaseModel):
+    type: Literal["text", "image_url"]
+    text: str | None = None
+    image_url: dict[str, str] | None = None
+
+    @model_validator(mode="after")
+    def validate_content_payload(self):
+        if self.type == "text":
+            if self.text is None:
+                raise ValueError("CohereEmbedContent with type='text' requires text")
+        elif not self.image_url or not self.image_url.get("url"):
+            raise ValueError(
+                "CohereEmbedContent with type='image_url' requires image_url.url"
+            )
+        return self
+
+
+class CohereEmbedInput(BaseModel):
+    content: list[CohereEmbedContent]
+
+
+class CohereEmbedRequest(BaseModel):
+    model: str | None = None
+    input_type: str | None = None
+    texts: list[str] | None = None
+    images: list[str] | None = None
+    inputs: list[CohereEmbedInput] | None = None
+    output_dimension: int | None = None
+    embedding_types: list[CohereEmbeddingType] | None = None
+    truncate: CohereTruncate = "END"
+    max_tokens: int | None = None
+    priority: int = 0
+
+    @model_validator(mode="after")
+    def validate_input_fields(self):
+        input_fields = (self.texts, self.images, self.inputs)
+        provided_fields = [field for field in input_fields if field is not None]
+        if len(provided_fields) != 1 or not provided_fields[0]:
+            raise ValueError(
+                "Exactly one of texts, images, or inputs must be provided, "
+                "and it must be non-empty"
+            )
+        return self
+
+
+# ---------------------------------------------------------------------------
+# Cohere /v2/embed — response models
+# ---------------------------------------------------------------------------
+
+
+class CohereApiVersion(BaseModel):
+    version: str = "2"
+
+
+class CohereBilledUnits(BaseModel):
+    input_tokens: int | None = None
+    image_tokens: int | None = None
+
+
+class CohereMeta(BaseModel):
+    api_version: CohereApiVersion = Field(default_factory=CohereApiVersion)
+    billed_units: CohereBilledUnits | None = None
+
+
+class CohereEmbedByTypeEmbeddings(BaseModel):
+    # The field name ``float`` shadows the builtin type, so the annotation
+    # must use ``builtins.float`` to avoid a self-referential type error.
+    float: list[list[builtins.float]] | None = None
+    binary: list[list[int]] | None = None
+    ubinary: list[list[int]] | None = None
+    base64: list[str] | None = None
+
+
+class CohereEmbedResponse(BaseModel):
+    id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
+    embeddings: CohereEmbedByTypeEmbeddings
+    texts: list[str] | None = None
+    meta: CohereMeta | None = None
+    response_type: Literal["embeddings_by_type"] = "embeddings_by_type"
+
+
+# ---------------------------------------------------------------------------
+# Cohere embedding type conversion helpers
+# ---------------------------------------------------------------------------
+
+_UNSIGNED_TO_SIGNED_DIFF = 1 << 7  # 128
+
+
+def _pack_binary_embeddings(
+    float_embeddings: list[list[float]],
+    signed: bool,
+) -> list[list[int]]:
+    """Bit-pack float embeddings: positive -> 1, negative -> 0.
+
+    Each bit is shifted left by ``7 - idx%8``, and every 8 bits are packed
+    into one byte.
+    """
+    result: list[list[int]] = []
+    for embedding in float_embeddings:
+        dim = len(embedding)
+        if dim % 8 != 0:
+            raise ValueError(
+                "Embedding dimension must be a multiple of 8 for binary "
+                f"embedding types, but got {dim}."
+            )
+        packed_len = dim // 8
+        packed: list[int] = []
+        byte_val = 0
+        for idx, value in enumerate(embedding):
+            bit = 1 if value >= 0 else 0
+            byte_val += bit << (7 - idx % 8)
+            if (idx + 1) % 8 == 0:
+                if signed:
+                    byte_val -= _UNSIGNED_TO_SIGNED_DIFF
+                packed.append(byte_val)
+                byte_val = 0
+        assert len(packed) == packed_len
+        result.append(packed)
+    return result
+
+
+def _encode_base64_embeddings(
+    float_embeddings: list[list[float]],
+) -> list[str]:
+    """Encode float embeddings as base64 (little-endian float32)."""
+    result: list[str] = []
+    for embedding in float_embeddings:
+        buf = struct.pack(f"<{len(embedding)}f", *embedding)
+        result.append(base64.b64encode(buf).decode("utf-8"))
+    return result
+
+
+def build_typed_embeddings(
+    float_embeddings: list[list[float]],
+    embedding_types: Sequence[str],
+) -> CohereEmbedByTypeEmbeddings:
+    """Convert float embeddings to all requested Cohere embedding types."""
+    result = CohereEmbedByTypeEmbeddings()
+
+    for emb_type in embedding_types:
+        if emb_type == "float":
+            result.float = float_embeddings
+        elif emb_type == "binary":
+            result.binary = _pack_binary_embeddings(float_embeddings, signed=True)
+        elif emb_type == "ubinary":
+            result.ubinary = _pack_binary_embeddings(float_embeddings, signed=False)
+        elif emb_type == "base64":
+            result.base64 = _encode_base64_embeddings(float_embeddings)
+
+    return result

vllm/entrypoints/pooling/embed/serving.py

@@ -0,0 +1,214 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from typing import TypeAlias, cast
+
+from fastapi.responses import JSONResponse, Response, StreamingResponse
+from typing_extensions import assert_never
+
+from vllm.logger import init_logger
+from vllm.outputs import PoolingRequestOutput
+from vllm.utils.serial_utils import EmbedDType, Endianness
+
+from ..base.serving import PoolingServing
+from ..typing import PoolingServeContext
+from ..utils import (
+    BytesEncodingFormat,
+    JsonEncodingFormat,
+    build_pooling_bytes_streaming_response,
+    encode_pooling_output_float,
+    encode_pooling_output_float_or_ndarray,
+    get_json_response_cls,
+    get_pooling_output_encoder,
+    get_pooling_usage,
+)
+from .io_processor import EmbedIOProcessor
+from .protocol import (
+    CohereBilledUnits,
+    CohereEmbedRequest,
+    CohereEmbedResponse,
+    CohereMeta,
+    EmbeddingRequest,
+    EmbeddingResponse,
+    EmbeddingResponseData,
+    build_typed_embeddings,
+)
+
+logger = init_logger(__name__)
+
+
+EmbeddingServeContext: TypeAlias = PoolingServeContext[EmbeddingRequest]
+
+
+class ServingEmbedding(PoolingServing):
+    """Embedding API supporting both OpenAI and Cohere formats."""
+
+    request_id_prefix = "embd"
+    io_processor: EmbedIOProcessor
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.json_response_cls = get_json_response_cls()
+
+    def init_io_processor(self, *args, **kwargs) -> EmbedIOProcessor:
+        return EmbedIOProcessor(*args, **kwargs)
+
+    def _build_response(
+        self,
+        ctx: PoolingServeContext,
+    ) -> Response:
+        if isinstance(ctx.request, CohereEmbedRequest):
+            return self._build_cohere_response_from_ctx(ctx)
+        return self._build_openai_response(ctx)
+
+    def _build_openai_response(
+        self,
+        ctx: EmbeddingServeContext,
+    ) -> JSONResponse | StreamingResponse:
+        encoding_format = ctx.request.encoding_format
+        embed_dtype = ctx.request.embed_dtype
+        endianness = ctx.request.endianness
+
+        if encoding_format == "float" or encoding_format == "base64":
+            return self._openai_json_response(
+                ctx.final_res_batch,
+                ctx.request_id,
+                ctx.created_time,
+                ctx.model_name,
+                encoding_format,
+                embed_dtype,
+                endianness,
+            )
+
+        if encoding_format == "bytes" or encoding_format == "bytes_only":
+            return self._openai_bytes_response(
+                ctx.final_res_batch,
+                ctx.request_id,
+                ctx.created_time,
+                ctx.model_name,
+                encoding_format,
+                embed_dtype,
+                endianness,
+            )
+
+        assert_never(encoding_format)
+
+    def _openai_json_response(
+        self,
+        final_res_batch: list[PoolingRequestOutput],
+        request_id: str,
+        created_time: int,
+        model_name: str,
+        encoding_format: JsonEncodingFormat,
+        embed_dtype: EmbedDType,
+        endianness: Endianness,
+    ) -> JSONResponse:
+        use_ndarray_response = (
+            encoding_format == "float"
+            and self.json_response_cls.__name__ == "ORJSONResponse"
+        )
+        if use_ndarray_response:
+            ndarray_items: list[dict[str, object]] = []
+
+            for idx, final_res in enumerate(final_res_batch):
+                item_dict = EmbeddingResponseData(
+                    index=idx,
+                    embedding=[],
+                ).model_dump()
+                item_dict["embedding"] = encode_pooling_output_float_or_ndarray(
+                    final_res
+                )
+                ndarray_items.append(item_dict)
+            ndarray_response = EmbeddingResponse(
+                id=request_id,
+                created=created_time,
+                model=model_name,
+                data=[],  # type: ignore[arg-type]
+                usage=get_pooling_usage(final_res_batch),
+            ).model_dump()
+            ndarray_response["data"] = ndarray_items
+
+            return self.json_response_cls(content=ndarray_response)
+
+        encode_fn = get_pooling_output_encoder(
+            encoding_format=encoding_format,
+            embed_dtype=embed_dtype,
+            endianness=endianness,
+        )
+
+        items: list[EmbeddingResponseData] = []
+
+        for idx, final_res in enumerate(final_res_batch):
+            item = EmbeddingResponseData(
+                index=idx,
+                embedding=cast(list[float] | str, encode_fn(final_res)),
+            )
+
+            items.append(item)
+
+        response = EmbeddingResponse(
+            id=request_id,
+            created=created_time,
+            model=model_name,
+            data=items,
+            usage=get_pooling_usage(final_res_batch),
+        )
+        return self.json_response_cls(content=response.model_dump())
+
+    def _openai_bytes_response(
+        self,
+        final_res_batch: list[PoolingRequestOutput],
+        request_id: str,
+        created_time: int,
+        model_name: str,
+        encoding_format: BytesEncodingFormat,
+        embed_dtype: EmbedDType,
+        endianness: Endianness,
+    ) -> StreamingResponse:
+        return build_pooling_bytes_streaming_response(
+            pooling_outputs=final_res_batch,
+            request_id=request_id,
+            created_time=created_time,
+            model_name=model_name,
+            encoding_format=encoding_format,
+            embed_dtype=embed_dtype,
+            endianness=endianness,
+        )
+
+    def _build_cohere_response_from_ctx(
+        self,
+        ctx: PoolingServeContext,
+    ) -> JSONResponse:
+        request = ctx.request
+        assert isinstance(request, CohereEmbedRequest)
+
+        all_floats = [
+            cast(list[float], encode_pooling_output_float(out))
+            for out in ctx.final_res_batch
+        ]
+        total_tokens = get_pooling_usage(ctx.final_res_batch).prompt_tokens
+
+        has_image_input = request.images is not None or any(
+            content.type == "image_url"
+            for input_item in request.inputs or []
+            for content in input_item.content
+        )
+        image_tokens = total_tokens if has_image_input else 0
+        texts_echo = request.texts
+
+        embedding_types = request.embedding_types or ["float"]
+        embeddings_obj = build_typed_embeddings(all_floats, embedding_types)
+
+        input_tokens = total_tokens - image_tokens
+        response = CohereEmbedResponse(
+            id=ctx.request_id,
+            embeddings=embeddings_obj,
+            texts=texts_echo,
+            meta=CohereMeta(
+                billed_units=CohereBilledUnits(
+                    input_tokens=input_tokens,
+                    image_tokens=image_tokens,
+                ),
+            ),
+        )
+        return self.json_response_cls(content=response.model_dump(exclude_none=True))

vllm/entrypoints/pooling/factories.py

@@ -0,0 +1,265 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from typing import TYPE_CHECKING
+
+from fastapi import FastAPI
+
+from vllm.config import ModelConfig, VllmConfig
+from vllm.entrypoints.chat_utils import ChatTemplateConfig
+from vllm.logger import init_logger
+from vllm.plugins.io_processors import has_io_processor
+from vllm.renderers import BaseRenderer
+from vllm.tasks import POOLING_TASKS, SCORE_TYPE_MAP, SupportedTask
+
+from .base.io_processor import PoolingIOProcessor
+from .utils import enable_scoring_api
+
+if TYPE_CHECKING:
+    from argparse import Namespace
+
+    from starlette.datastructures import State
+
+    from vllm.engine.protocol import EngineClient
+    from vllm.entrypoints.serve.sagemaker.api_router import (
+        EndpointFn,
+        GetHandlerFn,
+        RequestType,
+    )
+    from vllm.entrypoints.serve.utils.request_logger import RequestLogger
+
+else:
+    RequestLogger = object
+
+
+logger = init_logger(__name__)
+
+
+def init_pooling_io_processors(
+    supported_tasks: tuple[SupportedTask, ...],
+    vllm_config: VllmConfig,
+    renderer: BaseRenderer,
+    chat_template_config: ChatTemplateConfig,
+) -> dict[str, PoolingIOProcessor]:
+    model_config = vllm_config.model_config
+    processors: dict[str, type[PoolingIOProcessor]] = {}
+    pooling_task = model_config.get_pooling_task(supported_tasks)
+
+    if pooling_task == "classify":
+        from .classify.io_processor import ClassifyIOProcessor
+
+        processors["classify"] = ClassifyIOProcessor
+
+    if pooling_task == "token_classify":
+        from .classify.io_processor import TokenClassifyIOProcessor
+
+        processors["token_classify"] = TokenClassifyIOProcessor
+
+    if pooling_task == "embed":
+        from .embed.io_processor import EmbedIOProcessor
+
+        processors["embed"] = EmbedIOProcessor
+
+    if pooling_task == "token_embed":
+        from .embed.io_processor import TokenEmbedIOProcessor
+
+        processors["token_embed"] = TokenEmbedIOProcessor
+
+    if has_io_processor(
+        vllm_config,
+        model_config.io_processor_plugin,
+    ):
+        from .pooling.io_processor import PluginWithIOProcessorPlugins
+
+        processors["plugin"] = PluginWithIOProcessorPlugins
+    elif pooling_task == "plugin":
+        from .pooling.io_processor import PluginWithoutIOProcessorPlugins
+
+        processors["plugin"] = PluginWithoutIOProcessorPlugins
+
+    if enable_scoring_api(supported_tasks, model_config):
+        from .scoring.io_processor import ScoringIOProcessors
+
+        score_type: str | None = SCORE_TYPE_MAP.get(pooling_task, None)  # type: ignore[arg-type]
+        if score_type is not None and score_type in ScoringIOProcessors:
+            processors[score_type] = ScoringIOProcessors[score_type]
+
+    if model_config.architecture == "JinaForRanking":
+        from .embed.io_processor import JinaRankingTokenEmbedIOProcessor
+        from .scoring.io_processor import ScoringIOProcessors
+
+        processors["token_embed"] = JinaRankingTokenEmbedIOProcessor
+        processors["late-interaction"] = ScoringIOProcessors["jina-reranking-scoring"]
+
+    return {
+        task: processor_cls(
+            vllm_config=vllm_config,
+            renderer=renderer,
+            chat_template_config=chat_template_config,
+        )
+        for task, processor_cls in processors.items()
+    }
+
+
+def register_pooling_api_routers(
+    app: FastAPI,
+    supported_tasks: tuple["SupportedTask", ...],
+    model_config: ModelConfig | None = None,
+):
+    if model_config is None:
+        return
+
+    pooling_task = model_config.get_pooling_task(supported_tasks)
+
+    if pooling_task is not None:
+        from .pooling.api_router import router as pooling_router
+
+        app.include_router(pooling_router)
+
+    if "classify" in supported_tasks:
+        from .classify.api_router import (
+            router as classify_router,
+        )
+
+        app.include_router(classify_router)
+
+    if "embed" in supported_tasks:
+        from .embed.api_router import router as embed_router
+
+        app.include_router(embed_router)
+
+    if enable_scoring_api(supported_tasks, model_config):
+        from .scoring.api_router import router as score_router
+
+        app.include_router(score_router)
+
+
+def init_pooling_state(
+    engine_client: "EngineClient",
+    state: "State",
+    args: "Namespace",
+    request_logger: RequestLogger | None,
+    supported_tasks: tuple["SupportedTask", ...],
+):
+    model_config = engine_client.model_config
+    if model_config is None:
+        return
+
+    from vllm.entrypoints.chat_utils import load_chat_template
+    from vllm.tasks import POOLING_TASKS
+
+    from .classify.serving import ServingClassification
+    from .embed.serving import ServingEmbedding
+    from .pooling.serving import ServingPooling
+    from .scoring.serving import ServingScores
+
+    resolved_chat_template = load_chat_template(args.chat_template)
+    pooling_task = model_config.get_pooling_task(supported_tasks)
+
+    chat_template_config = ChatTemplateConfig(
+        chat_template=resolved_chat_template,
+        chat_template_content_format=args.chat_template_content_format,
+        trust_request_chat_template=args.trust_request_chat_template,
+    )
+
+    state.serving_pooling = (
+        (
+            ServingPooling(
+                engine_client,
+                state.openai_serving_models,
+                supported_tasks=supported_tasks,
+                request_logger=request_logger,
+                chat_template_config=chat_template_config,
+            )
+        )
+        if any(t in supported_tasks for t in POOLING_TASKS)
+        else None
+    )
+    state.serving_embedding = (
+        ServingEmbedding(
+            engine_client,
+            state.openai_serving_models,
+            request_logger=request_logger,
+            chat_template_config=chat_template_config,
+        )
+        if pooling_task == "embed"
+        else None
+    )
+    state.serving_classification = (
+        ServingClassification(
+            engine_client,
+            state.openai_serving_models,
+            request_logger=request_logger,
+            chat_template_config=chat_template_config,
+        )
+        if pooling_task == "classify"
+        else None
+    )
+    state.serving_scores = (
+        ServingScores(
+            engine_client,
+            state.openai_serving_models,
+            supported_tasks=supported_tasks,
+            request_logger=request_logger,
+            chat_template_config=chat_template_config,
+            enable_flash_late_interaction=getattr(
+                args, "enable_flash_late_interaction", True
+            ),
+        )
+        if enable_scoring_api(supported_tasks, model_config)
+        else None
+    )
+
+
+def get_pooling_invocation_types(
+    supported_tasks: tuple["SupportedTask", ...],
+    model_config: ModelConfig | None = None,
+):
+    # NOTE: Items defined earlier take higher priority
+    invocation_types: list[tuple[RequestType, tuple[GetHandlerFn, EndpointFn]]] = []
+
+    if model_config is None:
+        return invocation_types
+
+    pooling_task = model_config.get_pooling_task(supported_tasks)
+
+    if pooling_task == "embed":
+        from .embed.api_router import create_embedding, embedding
+        from .embed.protocol import EmbeddingRequest
+
+        invocation_types += [
+            (EmbeddingRequest, (embedding, create_embedding)),
+        ]
+
+    if pooling_task == "classify":
+        from .classify.api_router import classify, create_classify
+        from .classify.protocol import ClassificationRequest
+
+        invocation_types += [
+            (ClassificationRequest, (classify, create_classify)),
+        ]
+
+    if enable_scoring_api(supported_tasks, model_config):
+        from .scoring.api_router import do_rerank, rerank
+        from .scoring.protocol import RerankRequest
+
+        invocation_types += [
+            (RerankRequest, (rerank, do_rerank)),
+        ]
+
+        from .scoring.api_router import create_score, score
+        from .scoring.protocol import ScoreRequest
+
+        invocation_types += [
+            (ScoreRequest, (score, create_score)),
+        ]
+
+    if any(task in POOLING_TASKS for task in supported_tasks):
+        from .pooling.api_router import create_pooling, pooling
+        from .pooling.protocol import PoolingRequest
+
+        invocation_types += [
+            (PoolingRequest, (pooling, create_pooling)),
+        ]
+
+    return invocation_types

vllm/entrypoints/pooling/offline.py

@@ -0,0 +1,402 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from collections.abc import Callable, Sequence
+from typing import Any
+
+from tqdm.auto import tqdm
+
+from vllm.entrypoints.chat_utils import ChatTemplateConfig
+from vllm.entrypoints.offline_utils import OfflineInferenceMixin
+from vllm.inputs import DataPrompt, PromptType
+from vllm.logger import init_logger
+from vllm.lora.request import LoRARequest
+from vllm.outputs import (
+    ClassificationRequestOutput,
+    EmbeddingRequestOutput,
+    PoolingRequestOutput,
+    ScoringRequestOutput,
+)
+from vllm.pooling_params import PoolingParams
+from vllm.tasks import SCORE_TYPE_MAP, PoolingTask, SupportedTask
+
+from .factories import init_pooling_io_processors
+from .scoring.io_processor import ScoringIOProcessor
+from .scoring.typing import ScoreInput
+from .typing import OfflineInputsContext, OfflineOutputsContext
+
+logger = init_logger(__name__)
+
+
+class PoolingOfflineMixin(OfflineInferenceMixin):
+    """Offline inference for pooling models"""
+
+    runner_type: str
+    chat_template: str | None
+    supported_tasks: tuple[SupportedTask, ...]
+
+    def __init__(self):
+        self.pooling_task = self.model_config.get_pooling_task(self.supported_tasks)
+        if self.pooling_task is not None:
+            logger.info("Supported pooling task: %s", self.pooling_task)
+
+        self.chat_template_config = ChatTemplateConfig(chat_template=self.chat_template)
+        self.pooling_io_processors = init_pooling_io_processors(
+            supported_tasks=self.supported_tasks,
+            vllm_config=self.llm_engine.vllm_config,
+            renderer=self.renderer,
+            chat_template_config=self.chat_template_config,
+        )
+
+    def encode(
+        self,
+        prompts: PromptType | Sequence[PromptType] | DataPrompt,
+        pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
+        *,
+        use_tqdm: bool | Callable[..., tqdm] = True,
+        lora_request: list[LoRARequest] | LoRARequest | None = None,
+        pooling_task: PoolingTask | None = None,
+        tokenization_kwargs: dict[str, Any] | None = None,
+    ) -> list[PoolingRequestOutput]:
+        """Apply pooling to the hidden states corresponding to the input
+        prompts.
+
+        This class automatically batches the given prompts, considering
+        the memory constraint. For the best performance, put all of your prompts
+        into a single list and pass it to this method.
+
+        Args:
+            prompts: The prompts to the LLM. You may pass a sequence of prompts
+                for batch inference. See [PromptType][vllm.inputs.PromptType]
+                for more details about the format of each prompt.
+            pooling_params: The pooling parameters for pooling. If None, we
+                use the default pooling parameters.
+            use_tqdm: If `True`, shows a tqdm progress bar.
+                If a callable (e.g., `functools.partial(tqdm, leave=False)`),
+                it is used to create the progress bar.
+                If `False`, no progress bar is created.
+            lora_request: LoRA request to use for generation, if any.
+            pooling_task: Override the pooling task to use.
+            tokenization_kwargs: Overrides for `tokenizer.encode`.
+
+        Returns:
+            A list of `PoolingRequestOutput` objects containing the
+            pooled hidden states in the same order as the input prompts.
+        """
+
+        if isinstance(prompts, dict) and "data" in prompts and pooling_task != "plugin":
+            raise ValueError(
+                "The 'data' field is only supported for the 'plugin' pooling task."
+            )
+        self._verify_pooling_task(pooling_task)
+        assert pooling_task is not None and pooling_task in self.pooling_io_processors
+
+        io_processor = self.pooling_io_processors[pooling_task]
+
+        if pooling_params is None:
+            pooling_params = PoolingParams()
+
+        ctx = OfflineInputsContext(
+            prompts=prompts,
+            pooling_params=pooling_params,
+            tokenization_kwargs=tokenization_kwargs,
+        )
+
+        engine_inputs = io_processor.pre_process_offline(ctx)
+        n_inputs = len(engine_inputs)
+        assert ctx.pooling_params is not None
+
+        params_seq = self._params_to_seq(ctx.pooling_params, n_inputs)
+
+        for param in params_seq:
+            if param.task is None:
+                param.task = pooling_task
+            elif pooling_task == "plugin":
+                # `plugin` task uses io_processor.parse_request to verify inputs.
+                # We actually allow plugin to overwrite pooling_task.
+                pass
+            elif param.task != pooling_task:
+                msg = f"You cannot overwrite {param.task=!r} with {pooling_task=!r}!"
+                raise ValueError(msg)
+
+        seq_lora_requests = self._lora_request_to_seq(lora_request, n_inputs)
+        seq_priority = self._priority_to_seq(None, n_inputs)
+
+        self._render_and_add_requests(
+            prompts=engine_inputs,
+            params=params_seq,
+            lora_requests=seq_lora_requests,
+            priorities=seq_priority,
+        )
+
+        outputs = self._run_engine(use_tqdm=use_tqdm, output_type=PoolingRequestOutput)
+        outputs = io_processor.post_process_offline(
+            ctx=OfflineOutputsContext(outputs=outputs)
+        )
+        return outputs
+
+    def _verify_pooling_task(self, pooling_task: PoolingTask | None):
+        if self.runner_type != "pooling":
+            raise ValueError(
+                "LLM.encode() is only supported for pooling models. "
+                "Try passing `--runner pooling` to use the model as a "
+                "pooling model."
+            )
+
+        if pooling_task is None:
+            raise ValueError(
+                """
+                pooling_task required for `LLM.encode`.
+                Please use one of the more specific methods or set the pooling_task when using `LLM.encode`:
+                  - For embeddings, use `LLM.embed(...)` or `pooling_task="embed"`.
+                  - For classification logits, use `LLM.classify(...)` or `pooling_task="classify"`.
+                  - For similarity scores, use `LLM.score(...)`.
+                  - For rewards, `pooling_task="classify"` or `pooling_task="token_classify"`.
+                  - For token classification, use `pooling_task="token_classify"`.
+                  - For multi-vector retrieval, use `pooling_task="token_embed"`.
+                """  # noqa: E501
+            )
+
+        if (
+            pooling_task in ("embed", "token_embed")
+            and pooling_task not in self.supported_tasks
+        ):
+            raise ValueError(
+                "Embedding API is not supported by this model. "
+                "Try converting the model using `--convert embed`."
+            )
+
+        if (
+            pooling_task in ("classify", "token_classify")
+            and pooling_task not in self.supported_tasks
+        ):
+            raise ValueError(
+                "Classification API is not supported by this model. "
+                "Try converting the model using `--convert classify`."
+            )
+
+        # plugin task uses io_processor.parse_request to verify inputs
+        if pooling_task != "plugin" and pooling_task != self.pooling_task:
+            if pooling_task not in self.supported_tasks:
+                raise ValueError(
+                    f"Unsupported task: {pooling_task!r} "
+                    f"Supported tasks: {self.supported_tasks}"
+                )
+            else:
+                raise ValueError(
+                    f"Try switching the model's pooling_task "
+                    f'via `PoolerConfig(task="{pooling_task}")`'
+                )
+
+        if pooling_task == "plugin" and "plugin" not in self.pooling_io_processors:
+            raise ValueError(
+                "No IOProcessor plugin installed. Please refer "
+                "to the documentation and to the "
+                "'prithvi_geospatial_mae_io_processor' "
+                "offline inference example for more details."
+            )
+
+    def embed(
+        self,
+        prompts: PromptType | Sequence[PromptType],
+        *,
+        use_tqdm: bool | Callable[..., tqdm] = True,
+        pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
+        lora_request: list[LoRARequest] | LoRARequest | None = None,
+        tokenization_kwargs: dict[str, Any] | None = None,
+    ) -> list[EmbeddingRequestOutput]:
+        """
+        Generate an embedding vector for each prompt.
+
+        This class automatically batches the given prompts, considering
+        the memory constraint. For the best performance, put all of your prompts
+        into a single list and pass it to this method.
+
+        Args:
+            prompts: The prompts to the LLM. You may pass a sequence of prompts
+                for batch inference. See [PromptType][vllm.inputs.PromptType]
+                for more details about the format of each prompt.
+            pooling_params: The pooling parameters for pooling. If None, we
+                use the default pooling parameters.
+            use_tqdm: If `True`, shows a tqdm progress bar.
+                If a callable (e.g., `functools.partial(tqdm, leave=False)`),
+                it is used to create the progress bar.
+                If `False`, no progress bar is created.
+            lora_request: LoRA request to use for generation, if any.
+            tokenization_kwargs: Overrides for `tokenizer.encode`.
+
+        Returns:
+            A list of `EmbeddingRequestOutput` objects containing the
+            embedding vectors in the same order as the input prompts.
+        """
+
+        items = self.encode(
+            prompts,
+            use_tqdm=use_tqdm,
+            pooling_params=pooling_params,
+            lora_request=lora_request,
+            pooling_task="embed",
+            tokenization_kwargs=tokenization_kwargs,
+        )
+
+        return [EmbeddingRequestOutput.from_base(item) for item in items]
+
+    def classify(
+        self,
+        prompts: PromptType | Sequence[PromptType],
+        *,
+        pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
+        use_tqdm: bool | Callable[..., tqdm] = True,
+        lora_request: list[LoRARequest] | LoRARequest | None = None,
+        tokenization_kwargs: dict[str, Any] | None = None,
+    ) -> list[ClassificationRequestOutput]:
+        """
+        Generate class logits for each prompt.
+
+        This class automatically batches the given prompts, considering
+        the memory constraint. For the best performance, put all of your prompts
+        into a single list and pass it to this method.
+
+        Args:
+            prompts: The prompts to the LLM. You may pass a sequence of prompts
+                for batch inference. See [PromptType][vllm.inputs.PromptType]
+                for more details about the format of each prompt.
+            pooling_params: The pooling parameters for pooling. If None, we
+                use the default pooling parameters.
+            use_tqdm: If `True`, shows a tqdm progress bar.
+                If a callable (e.g., `functools.partial(tqdm, leave=False)`),
+                it is used to create the progress bar.
+                If `False`, no progress bar is created.
+            lora_request: LoRA request to use for generation, if any.
+            tokenization_kwargs: Overrides for `tokenizer.encode`.
+
+        Returns:
+            A list of `ClassificationRequestOutput` objects containing the
+            embedding vectors in the same order as the input prompts.
+        """
+
+        items = self.encode(
+            prompts,
+            use_tqdm=use_tqdm,
+            pooling_params=pooling_params,
+            lora_request=lora_request,
+            pooling_task="classify",
+            tokenization_kwargs=tokenization_kwargs,
+        )
+
+        return [ClassificationRequestOutput.from_base(item) for item in items]
+
+    def score(
+        self,
+        data_1: ScoreInput | list[ScoreInput],
+        data_2: ScoreInput | list[ScoreInput],
+        /,
+        *,
+        use_tqdm: bool | Callable[..., tqdm] = True,
+        pooling_params: PoolingParams | None = None,
+        lora_request: list[LoRARequest] | LoRARequest | None = None,
+        tokenization_kwargs: dict[str, Any] | None = None,
+        chat_template: str | None = None,
+    ) -> list[ScoringRequestOutput]:
+        """Generate similarity scores for all pairs `<text,text_pair>` or
+          `<multi-modal data, multi-modal data pair>`.
+
+        The inputs can be `1 -> 1`, `1 -> N` or `N -> N`.
+        In the `1 - N` case the `data_1` input will be replicated `N`
+        times to pair with the `data_2` inputs.
+        The input pairs are used to build a list of prompts for the
+        cross encoder model. This class automatically batches the prompts,
+        considering the memory constraint. For the best performance, put all
+        of your inputs into a single list and pass it to this method.
+
+        Supports both text and multi-modal data (images, etc.) when used with
+        appropriate multi-modal models. For multi-modal inputs, ensure the
+        prompt structure matches the model's expected input format.
+
+        Args:
+            data_1: Can be a single prompt, a list of prompts or
+                `ScoreMultiModalParam`, which can contain either text or
+                multi-modal data. When a list, it must have the same length as
+                the `data_2` list.
+            data_2: The data to pair with the query to form the input to
+                the LLM. Can be text or multi-modal data. See [PromptType]
+                [vllm.inputs.PromptType] for more details about the format of
+                each prompt.
+            pooling_params: The pooling parameters for pooling. If None, we
+                use the default pooling parameters.
+            use_tqdm: If `True`, shows a tqdm progress bar.
+                If a callable (e.g., `functools.partial(tqdm, leave=False)`),
+                it is used to create the progress bar.
+                If `False`, no progress bar is created.
+            lora_request: LoRA request to use for generation, if any.
+            chat_template: The chat template to use for the scoring. If None, we
+                use the model's default chat template.
+            tokenization_kwargs: Overrides for `tokenizer.encode`.
+        Returns:
+            A list of `ScoringRequestOutput` objects containing the
+            generated scores in the same order as the input prompts.
+        """
+
+        if self.runner_type != "pooling":
+            raise ValueError(
+                "LLM.score() is only supported for pooling models. "
+                "Try passing `--runner pooling` to use the model as a "
+                "pooling model."
+            )
+
+        score_type: str | None = SCORE_TYPE_MAP.get(self.pooling_task, None)  # type: ignore[arg-type]
+        if (
+            score_type == "cross-encoder"
+            and getattr(self.model_config.hf_config, "num_labels", 0) != 1
+        ):
+            raise ValueError("Scoring API is only enabled for num_labels == 1.")
+
+        if score_type is None or score_type not in self.pooling_io_processors:
+            raise ValueError("This model does not support the Scoring API.")
+
+        io_processor = self.pooling_io_processors[score_type]
+        assert isinstance(io_processor, ScoringIOProcessor)
+
+        pooling_task = io_processor.pooling_task
+        scoring_data = io_processor.valid_inputs(data_1, data_2)
+        n_queries = len(scoring_data.data_1)
+
+        if pooling_params is None:
+            pooling_params = PoolingParams()
+
+        ctx = OfflineInputsContext(
+            prompts=scoring_data,
+            pooling_params=pooling_params,
+            tokenization_kwargs=tokenization_kwargs,
+            chat_template=chat_template,
+            n_queries=n_queries,
+        )
+
+        engine_inputs = io_processor.pre_process_offline(ctx)
+        n_inputs = len(engine_inputs)
+
+        seq_lora_requests = self._lora_request_to_seq(lora_request, n_inputs)
+        params_seq = self._params_to_seq(ctx.pooling_params, n_inputs)
+
+        for param in params_seq:
+            if param.task is None:
+                param.task = pooling_task
+            elif param.task != pooling_task:
+                msg = f"You cannot overwrite {param.task=!r} with {pooling_task=!r}!"
+                raise ValueError(msg)
+
+        seq_priority = self._priority_to_seq(None, n_inputs)
+
+        self._render_and_add_requests(
+            prompts=engine_inputs,
+            params=params_seq,
+            lora_requests=seq_lora_requests,
+            priorities=seq_priority,
+        )
+
+        outputs = self._run_engine(use_tqdm=use_tqdm, output_type=PoolingRequestOutput)
+        outputs = io_processor.post_process_offline(
+            ctx=OfflineOutputsContext(outputs=outputs, n_queries=n_queries),
+        )
+
+        return [ScoringRequestOutput.from_base(item) for item in outputs]

vllm/entrypoints/pooling/pooling/api_router.py

@@ -0,0 +1,39 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from http import HTTPStatus
+
+from fastapi import APIRouter, Depends, Request
+
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
+from vllm.entrypoints.serve.utils.api_utils import (
+    load_aware_call,
+    validate_json_request,
+    with_cancellation,
+)
+
+from .protocol import PoolingRequest
+from .serving import ServingPooling
+
+router = APIRouter()
+
+
+def pooling(request: Request) -> ServingPooling | None:
+    return request.app.state.serving_pooling
+
+
+@router.post(
+    "/pooling",
+    dependencies=[Depends(validate_json_request)],
+    responses={
+        HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
+        HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
+    },
+)
+@with_cancellation
+@load_aware_call
+async def create_pooling(request: PoolingRequest, raw_request: Request):
+    handler = pooling(raw_request)
+    if handler is None:
+        raise NotImplementedError("The model does not support Pooling API")
+
+    return await handler(request, raw_request)

vllm/entrypoints/pooling/pooling/io_processor.py

@@ -0,0 +1,158 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from collections.abc import Sequence
+from typing import Any
+
+from vllm import PoolingParams, PoolingRequestOutput
+from vllm.inputs import EngineInput
+from vllm.logger import init_logger
+from vllm.plugins.io_processors import get_io_processor
+from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
+
+from ..base.io_processor import PoolingIOProcessor
+from ..typing import OfflineInputsContext, OfflineOutputsContext, PoolingServeContext
+from .protocol import IOProcessorRequest, IOProcessorResponse
+
+logger = init_logger(__name__)
+
+
+class PluginWithoutIOProcessorPlugins(PoolingIOProcessor):
+    # Some models, such as Terratorch (tests/models/test_terratorch.py),
+    # use plugin tasks in the pooler but do not use IO Processor plugins.
+    name = "plugin"
+
+
+class PluginWithIOProcessorPlugins(PoolingIOProcessor):
+    """IO Processor plugins are a feature that allows pre- and post-processing
+    of the model input and output for pooling models."""
+
+    name = "plugin"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        io_processor = get_io_processor(
+            self.vllm_config,
+            self.renderer,
+            self.model_config.io_processor_plugin,
+        )
+
+        assert io_processor is not None
+        self.io_processor = io_processor
+
+    #######################################
+    # online APIs
+
+    def pre_process_online(self, ctx: PoolingServeContext):
+        assert isinstance(ctx.request, IOProcessorRequest)
+
+        validated_prompt = self.io_processor.parse_data(ctx.request.data)
+
+        raw_prompts = self.io_processor.pre_process(
+            prompt=validated_prompt, request_id=ctx.request_id
+        )
+
+        parsed_prompts = [
+            (
+                prompt
+                if isinstance(prompt, bytes)
+                else parse_model_prompt(self.model_config, prompt)
+            )
+            for prompt in prompt_to_seq(raw_prompts)
+        ]
+
+        tok_params = ctx.request.build_tok_params(self.model_config)
+
+        ctx.engine_inputs = self.renderer.render_cmpl(
+            parsed_prompts,
+            tok_params,
+            prompt_extras={
+                k: v
+                for k in ("mm_processor_kwargs", "cache_salt")
+                if (v := getattr(ctx.request, k, None)) is not None
+            },
+        )
+
+        pooling_params = self.io_processor.merge_pooling_params()
+        if pooling_params.task is None:
+            pooling_params.task = "plugin"
+        ctx.pooling_params = pooling_params
+
+    def post_process_online(
+        self,
+        ctx: PoolingServeContext,
+    ):
+        output = self.io_processor.post_process(
+            ctx.final_res_batch,
+            request_id=ctx.request_id,
+        )
+
+        if callable(
+            output_to_response := getattr(self.io_processor, "output_to_response", None)
+        ):
+            logger.warning_once(
+                "`IOProcessor.output_to_response` is deprecated. To ensure "
+                "consistency between offline and online APIs, "
+                "`IOProcessorResponse` will become a transparent wrapper "
+                "around output data from v0.19 onwards.",
+            )
+
+            if hasattr(output, "request_id") and output.request_id is None:
+                output.request_id = ctx.request_id  # type: ignore
+
+            ctx.response = output_to_response(output)  # type: ignore
+        else:
+            ctx.response = IOProcessorResponse(request_id=ctx.request_id, data=output)
+
+    #######################################
+    # offline APIs
+
+    def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
+        assert isinstance(ctx.prompts, dict) and "data" in ctx.prompts
+        assert ctx.pooling_params is not None
+
+        # Validate the request data is valid for the loaded plugin
+        prompt_data = ctx.prompts.get("data")
+        if prompt_data is None:
+            raise ValueError(
+                "The 'data' field of the prompt is expected to contain "
+                "the prompt data and it cannot be None. "
+                "Refer to the documentation of the IOProcessor "
+                "in use for more details."
+            )
+        validated_prompt = self.io_processor.parse_data(prompt_data)
+
+        # obtain the actual model prompts from the pre-processor
+        prompts = self.io_processor.pre_process(prompt=validated_prompt)
+        prompts_seq = prompt_to_seq(prompts)
+
+        params_seq: list[PoolingParams] = [
+            self.io_processor.merge_pooling_params(param)
+            for param in self._params_to_seq(
+                ctx.pooling_params,
+                len(prompts_seq),
+            )
+        ]
+        for p in params_seq:
+            if p.task is None:
+                p.task = "plugin"
+
+        ctx.pooling_params = params_seq
+        ctx.prompts = prompts_seq
+        return super().pre_process_offline(ctx)
+
+    def post_process_offline(
+        self,
+        ctx: OfflineOutputsContext,
+    ) -> list[PoolingRequestOutput]:
+        processed_outputs = self.io_processor.post_process(ctx.outputs)
+
+        return [
+            PoolingRequestOutput[Any](
+                request_id="",
+                outputs=processed_outputs,
+                num_cached_tokens=getattr(processed_outputs, "num_cached_tokens", 0),
+                prompt_token_ids=[],
+                finished=True,
+            )
+        ]

vllm/entrypoints/pooling/pooling/protocol.py

@@ -0,0 +1,118 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import time
+from typing import Generic, TypeAlias, TypeVar
+
+from pydantic import Field
+
+from vllm import PoolingParams
+from vllm.config import ModelConfig
+from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
+from vllm.renderers import TokenizeParams
+from vllm.tasks import PoolingTask
+from vllm.utils import random_uuid
+
+from ..base.protocol import (
+    ChatRequestMixin,
+    ClassifyRequestMixin,
+    CompletionRequestMixin,
+    EmbedRequestMixin,
+    EncodingRequestMixin,
+    FixedMaxLenTokenizeParamsMixin,
+    PoolingBasicRequestMixin,
+)
+
+
+class PoolingCompletionRequest(
+    PoolingBasicRequestMixin,
+    CompletionRequestMixin,
+    EmbedRequestMixin,
+    ClassifyRequestMixin,
+    FixedMaxLenTokenizeParamsMixin,
+):
+    task: PoolingTask | None = None
+
+    def to_pooling_params(self):
+        return PoolingParams(
+            task=self.task,
+            use_activation=self.use_activation,
+            dimensions=self.dimensions,
+        )
+
+
+class PoolingChatRequest(
+    PoolingBasicRequestMixin,
+    ChatRequestMixin,
+    EmbedRequestMixin,
+    ClassifyRequestMixin,
+    FixedMaxLenTokenizeParamsMixin,
+):
+    task: PoolingTask | None = None
+
+    def to_pooling_params(self):
+        return PoolingParams(
+            task=self.task,
+            use_activation=self.use_activation,
+            dimensions=self.dimensions,
+        )
+
+
+T = TypeVar("T")
+
+
+class IOProcessorRequest(PoolingBasicRequestMixin, EncodingRequestMixin, Generic[T]):
+    data: T
+    task: PoolingTask = "plugin"
+
+    def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
+        return self._build_pooling_tok_params(
+            model_config,
+            add_special_tokens=not model_config.is_encoder_decoder,
+            max_total_tokens=model_config.max_model_len,
+            max_output_tokens=0,
+        )
+
+    def to_pooling_params(self):
+        return PoolingParams(
+            task=self.task,
+        )
+
+
+class IOProcessorResponse(OpenAIBaseModel, Generic[T]):
+    request_id: str | None = None
+    """
+    The request_id associated with this response
+    """
+    created_at: int = Field(default_factory=lambda: int(time.time()))
+
+    data: T
+    """
+    When using plugins IOProcessor plugins, the actual output is generated
+    by the plugin itself. Hence, we use a generic type for the response data
+    """
+
+
+PoolingRequest: TypeAlias = (
+    PoolingCompletionRequest | PoolingChatRequest | IOProcessorRequest
+)
+
+
+class PoolingResponseData(OpenAIBaseModel):
+    index: int
+    object: str = "pooling"
+    data: list[list[float]] | list[float] | str
+
+
+class PoolingResponse(OpenAIBaseModel):
+    id: str = Field(default_factory=lambda: f"pool-{random_uuid()}")
+    object: str = "list"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    model: str
+    data: list[PoolingResponseData]
+    usage: UsageInfo
+
+
+class PoolingBytesResponse(OpenAIBaseModel):
+    content: list[bytes]
+    headers: dict[str, str] | None = None
+    media_type: str = "application/octet-stream"

vllm/entrypoints/pooling/pooling/serving.py

@@ -0,0 +1,184 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from fastapi.responses import JSONResponse, Response, StreamingResponse
+from typing_extensions import assert_never
+
+from vllm.logger import init_logger
+from vllm.outputs import PoolingRequestOutput
+from vllm.tasks import SupportedTask
+from vllm.utils.serial_utils import EmbedDType, Endianness
+
+from ..base.io_processor import PoolingIOProcessor
+from ..base.serving import PoolingBaseServing
+from ..factories import init_pooling_io_processors
+from ..typing import AnyPoolingRequest, PoolingServeContext
+from ..utils import (
+    BytesEncodingFormat,
+    JsonEncodingFormat,
+    build_pooling_bytes_streaming_response,
+    get_json_response_cls,
+    get_pooling_output_encoder,
+    get_pooling_usage,
+)
+from .protocol import (
+    IOProcessorRequest,
+    PoolingRequest,
+    PoolingResponse,
+    PoolingResponseData,
+)
+
+logger = init_logger(__name__)
+
+
+class ServingPooling(PoolingBaseServing):
+    request_id_prefix = "pooling"
+
+    def __init__(
+        self,
+        *args,
+        supported_tasks: tuple[SupportedTask, ...],
+        **kwargs,
+    ):
+        super().__init__(*args, **kwargs)
+
+        self.supported_tasks = supported_tasks
+        self.pooling_task = self.model_config.get_pooling_task(supported_tasks)
+        self.io_processors = init_pooling_io_processors(
+            supported_tasks=supported_tasks,
+            vllm_config=self.vllm_config,
+            renderer=self.renderer,
+            chat_template_config=self.chat_template_config,
+        )
+        self.json_response_cls = get_json_response_cls()
+
+    def get_io_processor(self, request: AnyPoolingRequest) -> PoolingIOProcessor:
+        assert isinstance(request, PoolingRequest)
+        pooling_task = self._verify_pooling_task(request)
+        return self.io_processors[pooling_task]
+
+    def _verify_pooling_task(self, request: PoolingRequest) -> str:
+        if getattr(request, "dimensions", None) is not None:
+            raise ValueError("dimensions is currently not supported")
+
+        if request.task is None:
+            request.task = self.pooling_task
+
+        if isinstance(request, IOProcessorRequest):
+            request.task = "plugin"
+
+        assert request.task is not None
+        pooling_task = request.task
+
+        # plugin task uses io_processor.parse_request to verify inputs
+        if pooling_task != "plugin" and pooling_task != self.pooling_task:
+            if pooling_task not in self.supported_tasks:
+                raise ValueError(
+                    f"Unsupported task: {pooling_task!r} "
+                    f"Supported tasks: {self.supported_tasks}"
+                )
+            else:
+                raise ValueError(
+                    "Try switching the model's pooling_task "
+                    f"via --pooler-config.task {request.task}."
+                )
+
+        if pooling_task == "plugin" and "plugin" not in self.io_processors:
+            raise ValueError(
+                "No IOProcessor plugin installed. Please refer "
+                "to the documentation and to the "
+                "'prithvi_geospatial_mae_io_processor' "
+                "offline inference example for more details."
+            )
+
+        return pooling_task
+
+    def _build_response(
+        self,
+        ctx: PoolingServeContext,
+    ) -> Response:
+        if ctx.response is not None:
+            # for IOProcessorResponse
+            return self.json_response_cls(content=ctx.response.model_dump())
+
+        encoding_format = ctx.request.encoding_format
+        embed_dtype = ctx.request.embed_dtype
+        endianness = ctx.request.endianness
+
+        if encoding_format == "float" or encoding_format == "base64":
+            return self.request_output_to_pooling_json_response(
+                ctx.final_res_batch,
+                ctx.request_id,
+                ctx.created_time,
+                ctx.model_name,
+                encoding_format,
+                embed_dtype,
+                endianness,
+            )
+
+        if encoding_format == "bytes" or encoding_format == "bytes_only":
+            return self.request_output_to_pooling_bytes_response(
+                ctx.final_res_batch,
+                ctx.request_id,
+                ctx.created_time,
+                ctx.model_name,
+                encoding_format,
+                embed_dtype,
+                endianness,
+            )
+
+        assert_never(encoding_format)
+
+    def request_output_to_pooling_json_response(
+        self,
+        final_res_batch: list[PoolingRequestOutput],
+        request_id: str,
+        created_time: int,
+        model_name: str,
+        encoding_format: JsonEncodingFormat,
+        embed_dtype: EmbedDType,
+        endianness: Endianness,
+    ) -> JSONResponse:
+        encode_fn = get_pooling_output_encoder(
+            encoding_format=encoding_format,
+            embed_dtype=embed_dtype,
+            endianness=endianness,
+        )
+
+        items: list[PoolingResponseData] = []
+
+        for idx, final_res in enumerate(final_res_batch):
+            item = PoolingResponseData(
+                index=idx,
+                data=encode_fn(final_res),
+            )
+
+            items.append(item)
+
+        response = PoolingResponse(
+            id=request_id,
+            created=created_time,
+            model=model_name,
+            data=items,
+            usage=get_pooling_usage(final_res_batch),
+        )
+        return self.json_response_cls(content=response.model_dump())
+
+    def request_output_to_pooling_bytes_response(
+        self,
+        final_res_batch: list[PoolingRequestOutput],
+        request_id: str,
+        created_time: int,
+        model_name: str,
+        encoding_format: BytesEncodingFormat,
+        embed_dtype: EmbedDType,
+        endianness: Endianness,
+    ) -> StreamingResponse:
+        return build_pooling_bytes_streaming_response(
+            pooling_outputs=final_res_batch,
+            request_id=request_id,
+            created_time=created_time,
+            model_name=model_name,
+            encoding_format=encoding_format,
+            embed_dtype=embed_dtype,
+            endianness=endianness,
+        )

vllm/entrypoints/pooling/scoring/api_router.py

@@ -0,0 +1,115 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from http import HTTPStatus
+
+from fastapi import APIRouter, Depends, Request
+
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
+from vllm.entrypoints.serve.utils.api_utils import (
+    load_aware_call,
+    validate_json_request,
+    with_cancellation,
+)
+from vllm.logger import init_logger
+
+from .protocol import RerankRequest, ScoreRequest
+from .serving import ServingScores
+
+router = APIRouter()
+
+logger = init_logger(__name__)
+
+
+def score(request: Request) -> ServingScores | None:
+    return request.app.state.serving_scores
+
+
+def rerank(request: Request) -> ServingScores | None:
+    return request.app.state.serving_scores
+
+
+@router.post(
+    "/score",
+    dependencies=[Depends(validate_json_request)],
+    responses={
+        HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
+        HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
+    },
+)
+@with_cancellation
+@load_aware_call
+async def create_score(request: ScoreRequest, raw_request: Request):
+    handler = score(raw_request)
+    if handler is None:
+        raise NotImplementedError("The model does not support Score API")
+
+    return await handler(request, raw_request)
+
+
+@router.post(
+    "/v1/score",
+    dependencies=[Depends(validate_json_request)],
+    responses={
+        HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
+        HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
+    },
+)
+@with_cancellation
+@load_aware_call
+async def create_score_v1(request: ScoreRequest, raw_request: Request):
+    logger.warning(
+        "To indicate that Score API is not part of standard OpenAI API, we "
+        "have moved it to `/score`. Please update your client accordingly."
+    )
+
+    return await create_score(request, raw_request)
+
+
+@router.post(
+    "/rerank",
+    dependencies=[Depends(validate_json_request)],
+    responses={
+        HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
+        HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
+    },
+)
+@with_cancellation
+@load_aware_call
+async def do_rerank(request: RerankRequest, raw_request: Request):
+    handler = rerank(raw_request)
+    if handler is None:
+        raise NotImplementedError("The model does not support Rerank (Score) API")
+
+    return await handler(request, raw_request)
+
+
+@router.post(
+    "/v1/rerank",
+    dependencies=[Depends(validate_json_request)],
+    responses={
+        HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
+        HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
+    },
+)
+@with_cancellation
+async def do_rerank_v1(request: RerankRequest, raw_request: Request):
+    logger.warning_once(
+        "To indicate that the rerank API is not part of the standard OpenAI"
+        " API, we have located it at `/rerank`. Please update your client "
+        "accordingly. (Note: Conforms to JinaAI rerank API)"
+    )
+
+    return await do_rerank(request, raw_request)
+
+
+@router.post(
+    "/v2/rerank",
+    dependencies=[Depends(validate_json_request)],
+    responses={
+        HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
+        HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
+    },
+)
+@with_cancellation
+async def do_rerank_v2(request: RerankRequest, raw_request: Request):
+    return await do_rerank(request, raw_request)

vllm/entrypoints/pooling/scoring/io_processor.py

@@ -0,0 +1,802 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import time
+from collections.abc import Sequence
+from typing import Any, TypeAlias
+
+import torch.nn.functional as F
+
+from vllm import PoolingParams, PoolingRequestOutput, TokensPrompt
+from vllm.inputs import EngineInput
+from vllm.renderers import TokenizeParams
+from vllm.renderers.hf import safe_apply_chat_template
+from vllm.renderers.inputs.preprocess import extract_target_prompt
+from vllm.tasks import PoolingTask
+from vllm.utils.mistral import is_mistral_tokenizer
+
+from ...chat_utils import ChatTemplateResolutionError
+from ..base.io_processor import PoolingIOProcessor
+from ..typing import (
+    OfflineInputsContext,
+    OfflineOutputsContext,
+    PoolingServeContext,
+)
+from .protocol import RerankRequest, ScoreRequest, ScoringRequest
+from .typing import ScoreData, ScoreInput, ScoringData
+from .utils import (
+    compress_token_type_ids,
+    compute_maxsim_score,
+    get_num_special_tokens_for_pair,
+    parse_score_data,
+    score_data_to_prompts,
+    truncate_text_to_tokens,
+    validate_score_input,
+)
+
+ScoringServeContext: TypeAlias = PoolingServeContext[ScoringRequest]
+
+
+def _apply_post_tokenization_to_token_type_ids(
+    tokenizer: Any,
+    tok_params: TokenizeParams,
+    token_type_ids: list[int],
+) -> list[int]:
+    pad_length = tok_params.pad_prompt_tokens
+    if pad_length is not None and pad_length < 0:
+        pad_length = tok_params.max_input_tokens
+
+    if pad_length is not None and pad_length > len(token_type_ids):
+        pad_token_type_id = token_type_ids[-1] if token_type_ids else 0
+        token_type_ids = token_type_ids + [pad_token_type_id] * (
+            pad_length - len(token_type_ids)
+        )
+
+    max_length = tok_params.truncate_prompt_tokens
+    if max_length is not None and max_length < 0:
+        max_length = tok_params.max_input_tokens
+
+    if max_length is None or max_length >= len(token_type_ids):
+        return token_type_ids
+    if max_length == 0:
+        return token_type_ids[:0]
+
+    side = tok_params.truncation_side or (
+        tokenizer.truncation_side if tokenizer is not None else None
+    )
+    if side == "left":
+        return token_type_ids[-max_length:]
+
+    return token_type_ids[:max_length]
+
+
+class ScoringIOProcessor(PoolingIOProcessor):
+    name: str
+    pooling_task: PoolingTask
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.tokenizer = self.renderer.get_tokenizer()
+        self.architecture = self.model_config.architecture
+        self.is_multimodal_model = self.model_config.is_multimodal_model
+        self.pad_token_id = self.tokenizer.pad_token_id
+
+    def create_pooling_params(self, request):
+        return request.to_pooling_params(self.pooling_task)
+
+    def _validate_token_limit(self, value: int, name: str) -> None:
+        if value < 0:
+            raise ValueError(f"{name} must be a non-negative integer")
+        if value >= self.model_config.max_model_len:
+            raise ValueError(
+                f"{name} ({value}) must be less "
+                f"than max_model_len ({self.model_config.max_model_len})."
+            )
+
+    def _get_token_limits(
+        self,
+        request: ScoringRequest | None = None,
+        pooling_params: PoolingParams | None = None,
+    ) -> tuple[int, int]:
+        """Extract and validate token limits from request or pooling_params."""
+        if request is not None:
+            max_tokens_per_query = getattr(request, "max_tokens_per_query", 0)
+            max_tokens_per_doc = getattr(request, "max_tokens_per_doc", 0)
+        else:
+            extra = (
+                (pooling_params.extra_kwargs or {})
+                if pooling_params is not None
+                else {}
+            )
+            max_tokens_per_query = extra.get("max_tokens_per_query", 0)
+            max_tokens_per_doc = extra.get("max_tokens_per_doc", 0)
+
+        if max_tokens_per_query != 0:
+            self._validate_token_limit(max_tokens_per_query, "max_tokens_per_query")
+        if max_tokens_per_doc != 0:
+            self._validate_token_limit(max_tokens_per_doc, "max_tokens_per_doc")
+        return max_tokens_per_query, max_tokens_per_doc
+
+    def _truncate_scoring_data(
+        self,
+        scoring_data: ScoringData,
+        max_tokens_per_query: int = 0,
+        max_tokens_per_doc: int = 0,
+    ) -> ScoringData:
+        """Truncate query/document texts to token limits."""
+        data_1 = scoring_data.data_1
+        data_2 = scoring_data.data_2
+        if max_tokens_per_query > 0:
+            data_1 = [
+                truncate_text_to_tokens(d, self.tokenizer, max_tokens_per_query)
+                if isinstance(d, str)
+                else d
+                for d in data_1
+            ]
+        if max_tokens_per_doc > 0:
+            data_2 = [
+                truncate_text_to_tokens(d, self.tokenizer, max_tokens_per_doc)
+                if isinstance(d, str)
+                else d
+                for d in data_2
+            ]
+        return ScoringData(data_1=data_1, data_2=data_2)
+
+    def valid_inputs(
+        self,
+        data_1: ScoreInput | list[ScoreInput],
+        data_2: ScoreInput | list[ScoreInput],
+    ) -> ScoringData:
+        scoring_data = validate_score_input(
+            data_1,
+            data_2,
+            is_multimodal_model=self.is_multimodal_model,
+            architecture=self.architecture,
+        )
+        return scoring_data
+
+
+class BiEncoderIOProcessor(ScoringIOProcessor):
+    name = "bi-encoder"
+    pooling_task: PoolingTask = "embed"
+
+    #######################################
+    # online APIs
+
+    def pre_process_online(self, ctx: ScoringServeContext):
+        request = ctx.request
+
+        if isinstance(request, ScoreRequest):
+            data_1 = request.data_1
+            data_2 = request.data_2
+        elif isinstance(request, RerankRequest):
+            data_1 = request.query
+            data_2 = request.documents
+        else:
+            raise ValueError(f"Invalid {self.name} request type")
+
+        scoring_data = self.valid_inputs(data_1, data_2)
+
+        max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
+            request=request
+        )
+        if max_tokens_per_query > 0 or max_tokens_per_doc > 0:
+            scoring_data = self._truncate_scoring_data(
+                scoring_data, max_tokens_per_query, max_tokens_per_doc
+            )
+
+        tok_params = request.build_tok_params(self.model_config)
+        engine_inputs = self._pre_process(
+            scoring_data,
+            tok_params,
+            prompt_extras={
+                k: v
+                for k in ("mm_processor_kwargs", "cache_salt", "chat_template_kwargs")
+                if (v := getattr(request, k, None)) is not None
+            },
+        )
+
+        ctx.engine_inputs = engine_inputs
+        ctx.n_queries = len(scoring_data.data_1)
+
+    def post_process_online(
+        self,
+        ctx: ScoringServeContext,
+    ):
+        assert ctx.final_res_batch is not None
+        assert isinstance(ctx.n_queries, int)
+
+        ctx.final_res_batch = self._post_process(
+            outputs=ctx.final_res_batch, n_queries=ctx.n_queries
+        )
+
+    #######################################
+    # offline APIs
+
+    def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
+        assert isinstance(ctx.prompts, ScoringData)
+        assert not isinstance(ctx.pooling_params, Sequence)
+
+        tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
+            **(ctx.tokenization_kwargs or {})
+        )
+
+        max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
+            pooling_params=ctx.pooling_params
+        )
+
+        scoring_data = ctx.prompts
+        if max_tokens_per_query > 0 or max_tokens_per_doc > 0:
+            scoring_data = self._truncate_scoring_data(
+                scoring_data, max_tokens_per_query, max_tokens_per_doc
+            )
+
+        return self._pre_process(scoring_data, tok_params)
+
+    def post_process_offline(
+        self,
+        ctx: OfflineOutputsContext,
+    ) -> list[PoolingRequestOutput]:
+        assert ctx.n_queries is not None
+        return self._post_process(outputs=ctx.outputs, n_queries=ctx.n_queries)
+
+    #######################################
+    # helpers
+
+    def _pre_process(
+        self,
+        scoring_data: ScoringData,
+        tok_params: TokenizeParams,
+        prompt_extras: dict[str, Any] | None = None,
+    ) -> Sequence[EngineInput]:
+        data_1 = score_data_to_prompts(scoring_data.data_1, "query", self.model_config)
+        data_2 = score_data_to_prompts(
+            scoring_data.data_2, "document", self.model_config
+        )
+
+        return self._preprocess_cmpl_offline(
+            prompts=data_1 + data_2, tok_params=tok_params, prompt_extras=prompt_extras
+        )
+
+    def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
+        emb_data_1 = outputs[:n_queries]
+        emb_data_2 = outputs[n_queries:]
+
+        if len(emb_data_1) == 1:
+            emb_data_1 = emb_data_1 * len(emb_data_2)
+
+        final_res_batch: list[PoolingRequestOutput] = []
+        for emb_1, emb_2 in zip(emb_data_1, emb_data_2):
+            pair_score = F.cosine_similarity(
+                emb_1.outputs.data.float(), emb_2.outputs.data.float(), dim=0
+            )
+
+            padding: list[int] = []
+            if self.pad_token_id is not None:
+                padding = [self.pad_token_id]
+
+            tokens = emb_1.prompt_token_ids + padding + emb_2.prompt_token_ids
+
+            final_res_batch.append(
+                PoolingRequestOutput(
+                    request_id=f"{emb_1.request_id}_{emb_2.request_id}",
+                    outputs=pair_score,
+                    prompt_token_ids=tokens,
+                    num_cached_tokens=emb_1.num_cached_tokens + emb_2.num_cached_tokens,
+                    finished=True,
+                )
+            )
+        return final_res_batch
+
+
+class LateInteractionIOProcessor(BiEncoderIOProcessor):
+    name = "late-interaction"
+    pooling_task: PoolingTask = "token_embed"
+
+    def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
+        # Split into query and document embeddings
+        emb_data_1 = outputs[:n_queries]
+        emb_data_2 = outputs[n_queries:]
+
+        # Expand queries if 1:N scoring
+        if len(emb_data_1) == 1:
+            emb_data_1 = emb_data_1 * len(emb_data_2)
+
+        final_res_batch: list[PoolingRequestOutput] = []
+        padding: list[int] = []
+        if (pad_token_id := self.pad_token_id) is not None:
+            padding = [pad_token_id]
+
+        # Compute MaxSim scores
+        for emb_1, emb_2 in zip(emb_data_1, emb_data_2):
+            # emb_1.outputs.data: [query_len, dim]
+            # emb_2.outputs.data: [doc_len, dim]
+            q_emb = emb_1.outputs.data
+            d_emb = emb_2.outputs.data
+
+            maxsim_score = compute_maxsim_score(q_emb, d_emb)
+
+            tokens = emb_1.prompt_token_ids + padding + emb_2.prompt_token_ids
+
+            final_res_batch.append(
+                PoolingRequestOutput(
+                    request_id=f"{emb_1.request_id}_{emb_2.request_id}",
+                    outputs=maxsim_score,
+                    prompt_token_ids=tokens,
+                    num_cached_tokens=emb_1.num_cached_tokens + emb_2.num_cached_tokens,
+                    finished=True,
+                )
+            )
+        return final_res_batch
+
+
+class FlashLateInteractionIOProcessor(LateInteractionIOProcessor):
+    name = "flash-late-interaction"
+
+    def post_process_online(
+        self,
+        ctx: ScoringServeContext,
+    ):
+        assert ctx.query_final_res_batch is not None
+        assert ctx.final_res_batch is not None
+        assert isinstance(ctx.n_queries, int)
+
+        # Expand queries if 1:N scoring
+        if len(ctx.query_final_res_batch) == 1:
+            ctx.query_final_res_batch = ctx.query_final_res_batch * len(
+                ctx.final_res_batch
+            )
+
+        final_res_batch: list[PoolingRequestOutput] = []
+        for d1, d2 in zip(ctx.query_final_res_batch, ctx.final_res_batch):
+            padding: list[int] = []
+            if (pad_token_id := self.pad_token_id) is not None:
+                padding = [pad_token_id]
+
+            tokens = d1.prompt_token_ids + padding + d2.prompt_token_ids
+
+            final_res_batch.append(
+                PoolingRequestOutput(
+                    request_id=f"{d1.request_id}_{d2.request_id}",
+                    outputs=d2.outputs,
+                    prompt_token_ids=tokens,
+                    num_cached_tokens=d1.num_cached_tokens + d2.num_cached_tokens,
+                    finished=True,
+                )
+            )
+        ctx.final_res_batch = final_res_batch
+
+
+class CrossEncoderIOProcessor(ScoringIOProcessor):
+    name = "cross-encoder"
+    pooling_task: PoolingTask = "classify"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        if is_mistral_tokenizer(self.tokenizer):
+            raise ValueError("MistralTokenizer not supported for cross-encoding")
+
+        from vllm.model_executor.model_loader import get_model_cls
+        from vllm.model_executor.models.interfaces import supports_score_template
+
+        model = get_model_cls(self.model_config)
+        self.supports_score_template = supports_score_template(model)
+        self.model = model if self.supports_score_template else None
+        self.use_sep_token = self.model_config.use_sep_token
+
+    #######################################
+    # online APIs
+
+    def pre_process_online(self, ctx: ScoringServeContext):
+        request = ctx.request
+
+        if isinstance(request, ScoreRequest):
+            data_1 = request.data_1
+            data_2 = request.data_2
+        elif isinstance(request, RerankRequest):
+            data_1 = request.query
+            data_2 = request.documents
+        else:
+            raise ValueError(f"Invalid {self.name} request type")
+
+        scoring_data = self.valid_inputs(data_1, data_2)
+
+        max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
+            request=request
+        )
+
+        tok_params = request.build_tok_params(self.model_config)
+        pooling_params = self.create_pooling_params(request)
+
+        engine_inputs, pooling_params_list = self._pre_process(
+            scoring_data,
+            tok_params,
+            pooling_params,
+            chat_template=self.chat_template,
+            max_tokens_per_query=max_tokens_per_query,
+            max_tokens_per_doc=max_tokens_per_doc,
+            prompt_extras={
+                k: v
+                for k in ("mm_processor_kwargs", "cache_salt", "chat_template_kwargs")
+                if (v := getattr(request, k, None)) is not None
+            },
+        )
+
+        ctx.engine_inputs = engine_inputs
+        ctx.pooling_params = pooling_params_list
+
+    #######################################
+    # offline APIs
+
+    def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
+        assert isinstance(ctx.prompts, ScoringData)
+        assert not isinstance(ctx.pooling_params, Sequence)
+
+        tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
+            **(ctx.tokenization_kwargs or {})
+        )
+
+        max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
+            pooling_params=ctx.pooling_params
+        )
+
+        prompt_extras = ctx.pooling_params.extra_kwargs if ctx.pooling_params else None
+        engine_inputs, pooling_params_list = self._pre_process(
+            ctx.prompts,
+            tok_params,
+            ctx.pooling_params,
+            ctx.chat_template,
+            max_tokens_per_query=max_tokens_per_query,
+            max_tokens_per_doc=max_tokens_per_doc,
+            prompt_extras=prompt_extras,
+        )
+        ctx.pooling_params = pooling_params_list
+        return engine_inputs
+
+    #######################################
+    # helpers
+
+    def _pre_process(
+        self,
+        scoring_data: ScoringData,
+        tok_params: TokenizeParams,
+        pooling_params: PoolingParams | None,
+        chat_template: str | None = None,
+        max_tokens_per_query: int = 0,
+        max_tokens_per_doc: int = 0,
+        prompt_extras: dict[str, Any] | None = None,
+    ) -> tuple[Sequence[EngineInput], list[PoolingParams]]:
+        arrival_time = time.time()
+        engine_prompt_extras = (
+            {
+                k: v
+                for k in ("mm_processor_kwargs", "cache_salt")
+                if (v := prompt_extras.get(k)) is not None
+            }
+            if prompt_extras
+            else None
+        )
+
+        data_1 = scoring_data.data_1
+        data_2 = scoring_data.data_2
+
+        if len(data_1) == 1:
+            data_1 = data_1 * len(data_2)
+
+        if pooling_params is None:
+            pooling_params = PoolingParams(task="classify")
+
+        pooling_params_list = list[PoolingParams]()
+        engine_inputs = list[EngineInput]()
+        for q, d in zip(data_1, data_2):
+            _, engine_prompt = self.get_score_prompt(
+                data_1=q,
+                data_2=d,
+                encode_kwargs=tok_params.get_encode_kwargs(),
+                chat_template=chat_template,
+                max_tokens_per_query=max_tokens_per_query,
+                max_tokens_per_doc=max_tokens_per_doc,
+                chat_template_kwargs=prompt_extras.get("chat_template_kwargs")
+                if prompt_extras
+                else None,
+            )
+
+            token_type_ids = engine_prompt.pop("token_type_ids", None)
+            tok_params.apply_post_tokenization(self.tokenizer, engine_prompt)
+
+            if token_type_ids is not None:
+                params = pooling_params.clone()
+                compressed = compress_token_type_ids(
+                    _apply_post_tokenization_to_token_type_ids(
+                        self.tokenizer, tok_params, token_type_ids
+                    )
+                )
+                params.extra_kwargs = {
+                    **(params.extra_kwargs or {}),
+                    "compressed_token_type_ids": compressed,
+                }
+                pooling_params_list.append(params)
+            else:
+                pooling_params_list.append(pooling_params)
+
+            if engine_prompt_extras:
+                target_prompt = extract_target_prompt(self.model_config, engine_prompt)
+                target_prompt.update(engine_prompt_extras)
+            engine_inputs.append(
+                self.renderer.process_for_engine(engine_prompt, arrival_time)
+            )
+        return engine_inputs, pooling_params_list
+
+    def get_score_prompt(
+        self,
+        data_1: ScoreData,
+        data_2: ScoreData,
+        encode_kwargs: dict[str, Any],
+        chat_template: str | None = None,
+        max_tokens_per_query: int = 0,
+        max_tokens_per_doc: int = 0,
+        chat_template_kwargs: dict[str, Any] | None = None,
+    ):
+        model_config = self.model_config
+        tokenizer = self.tokenizer
+
+        prompt_1, prompt_2, mm_data, mm_uuids = parse_score_data(
+            data_1,
+            data_2,
+            model_config,
+        )
+
+        # Apply truncation before defining closures
+        if max_tokens_per_query > 0 and isinstance(prompt_1, str):
+            prompt_1 = truncate_text_to_tokens(
+                prompt_1, tokenizer, max_tokens_per_query
+            )
+        if max_tokens_per_doc > 0 and isinstance(prompt_2, str):
+            prompt_2 = truncate_text_to_tokens(prompt_2, tokenizer, max_tokens_per_doc)
+
+        def default_tokenizer_encode():
+            local_kwargs = encode_kwargs.copy()
+
+            if self.supports_score_template:
+                assert self.model is not None
+                full_prompt = self.model.get_score_template(prompt_1, prompt_2)
+                if full_prompt is None:
+                    raise ValueError("Get empty score template from model")
+
+                prompt_inputs = tokenizer(full_prompt, **local_kwargs)
+            else:
+                if self.use_sep_token:
+                    # cross_encoder models defaults to using separating token.
+                    if max_tokens_per_doc > 0 and isinstance(prompt_2, str):
+                        query_tokens = tokenizer.encode(
+                            prompt_1, add_special_tokens=False
+                        )
+                        num_special = get_num_special_tokens_for_pair(tokenizer)
+                        doc_limit_max_length = (
+                            len(query_tokens) + max_tokens_per_doc + num_special
+                        )
+                        existing_max_length = local_kwargs.get("max_length")
+                        if existing_max_length is not None:
+                            effective_max_length = min(
+                                doc_limit_max_length, existing_max_length
+                            )
+                        else:
+                            effective_max_length = doc_limit_max_length
+                        local_kwargs["truncation"] = "only_second"
+                        local_kwargs["max_length"] = effective_max_length
+
+                    prompt_inputs = tokenizer(
+                        text=prompt_1, text_pair=prompt_2, **local_kwargs
+                    )
+                    full_prompt = tokenizer.decode(prompt_inputs["input_ids"])
+                else:
+                    # `llm as reranker` defaults to not using separating token.
+                    if max_tokens_per_doc > 0 and isinstance(prompt_2, str):
+                        query_ids = tokenizer.encode(prompt_1, add_special_tokens=False)
+                        doc_ids = tokenizer.encode(prompt_2, add_special_tokens=False)
+                        doc_ids = doc_ids[:max_tokens_per_doc]
+                        input_ids = query_ids + doc_ids
+                        full_prompt = tokenizer.decode(input_ids)
+                        prompt_inputs = {"input_ids": input_ids}
+                    else:
+                        full_prompt = prompt_1 + prompt_2
+                        prompt_inputs = tokenizer(text=full_prompt, **local_kwargs)
+            return full_prompt, prompt_inputs
+
+        # FIXME: For now, we only apply a template when one is explicitly provided.
+        # We cannot rely on the tokenizer's chat template because many models
+        # inherit junk templates from their base LLM, which breaks both the models
+        # and the tests that use them.
+        if chat_template is None:
+            full_prompt, prompt_inputs = default_tokenizer_encode()
+        else:
+            # FIXME:
+            # Try applying a score template from the CLI arg or tokenizer_config.json
+            # If that fails because there is no such template,
+            # fall back to the default implementation.
+            try:
+                _safe_kwargs = chat_template_kwargs or {}
+                _reserved = {"chat_template", "tools", "tokenize"}
+                _unexpected = _reserved & _safe_kwargs.keys()
+                if _unexpected:
+                    raise ValueError(
+                        "chat_template_kwargs contains reserved keys that "
+                        f"conflict with fixed scorer arguments: {_unexpected}"
+                    )
+                full_prompt = safe_apply_chat_template(
+                    model_config,
+                    tokenizer,
+                    [
+                        {"role": "query", "content": prompt_1},
+                        {"role": "document", "content": prompt_2},
+                    ],
+                    chat_template=chat_template,
+                    tools=None,
+                    tokenize=False,
+                    **_safe_kwargs,
+                )
+                prompt_inputs = tokenizer(full_prompt, **encode_kwargs)
+            except ChatTemplateResolutionError:
+                full_prompt, prompt_inputs = default_tokenizer_encode()
+
+        engine_prompt = TokensPrompt(prompt_token_ids=prompt_inputs["input_ids"])
+
+        if (token_type_ids := prompt_inputs.get("token_type_ids")) is not None:
+            engine_prompt["token_type_ids"] = token_type_ids
+
+        if self.model is not None:
+            self.model.post_process_tokens(engine_prompt)
+
+        if mm_data is not None:
+            engine_prompt["multi_modal_data"] = mm_data
+        if mm_uuids is not None:
+            engine_prompt["multi_modal_uuids"] = mm_uuids
+
+        return full_prompt, engine_prompt
+
+
+class JinaRankingIOProcessorMixin:
+    @staticmethod
+    def format_docs_prompts_func(
+        query: str,
+        docs: list[str],
+        special_tokens: dict[str, str] | None = None,
+        instruction: str | None = None,
+        no_thinking: bool = True,
+    ) -> str:
+        # TODO: Try converting the code below into a chat template.
+
+        default_special_tokens = {
+            "query_embed_token": "<|rerank_token|>",
+            "doc_embed_token": "<|embed_token|>",
+        }
+        if special_tokens is None:
+            special_tokens = default_special_tokens
+
+        def sanitize_input(text: str) -> str:
+            for token in special_tokens.values():
+                text = text.replace(token, "")
+            return text
+
+        query = sanitize_input(query)
+        docs = [sanitize_input(doc) for doc in docs]
+
+        prefix = (
+            "<|im_start|>system\n"
+            "You are a search relevance expert who can determine a ranking of the passages based on how relevant they are to the query. "  # noqa: E501
+            "If the query is a question, how relevant a passage is depends on how well it answers the question. "  # noqa: E501
+            "If not, try to analyze the intent of the query and assess how well each passage satisfies the intent. "  # noqa: E501
+            "If an instruction is provided, you should follow the instruction when determining the ranking."  # noqa: E501
+            "<|im_end|>\n<|im_start|>user\n"
+        )
+        suffix = "<|im_end|>\n<|im_start|>assistant\n"
+        if no_thinking:
+            suffix += "<think>\n\n</think>\n\n"
+
+        doc_emb_token = special_tokens["doc_embed_token"]
+        query_emb_token = special_tokens["query_embed_token"]
+
+        prompt = (
+            f"I will provide you with {len(docs)} passages, each indicated by a numerical identifier. "  # noqa: E501
+            f"Rank the passages based on their relevance to query: {query}\n"
+        )
+
+        if instruction:
+            instruction = sanitize_input(instruction)
+            prompt += f"<instruct>\n{instruction}\n</instruct>\n"
+
+        doc_prompts = [
+            f'<passage id="{i}">\n{doc}{doc_emb_token}\n</passage>'
+            for i, doc in enumerate(docs)
+        ]
+        prompt += "\n".join(doc_prompts) + "\n"
+        prompt += f"<query>\n{query}{query_emb_token}\n</query>"
+
+        return prefix + prompt + suffix
+
+    @staticmethod
+    def ensure_str(data: Sequence[Any]) -> list[str]:
+        text: list[str] = []
+        for prompt in data:
+            if not isinstance(prompt, str):
+                raise ValueError(
+                    "The JinaForRanking model only supports text as input."
+                )
+            text.append(prompt)
+        return text
+
+
+class JinaRankingIOProcessor(LateInteractionIOProcessor, JinaRankingIOProcessorMixin):
+    name = "jina-reranking-scoring"
+    pooling_task: PoolingTask = "token_embed"
+
+    def _pre_process(
+        self,
+        scoring_data: ScoringData,
+        tok_params: TokenizeParams,
+        prompt_extras: dict[str, Any] | None = None,
+    ) -> Sequence[EngineInput]:
+        queries = self.ensure_str(scoring_data.data_1)
+        docs = self.ensure_str(scoring_data.data_2)
+        chat_template_kwargs = (
+            prompt_extras.get("chat_template_kwargs") if prompt_extras else None
+        )
+        instruction = (
+            chat_template_kwargs.get("instruction") if chat_template_kwargs else None
+        )
+
+        if len(queries) == 1:
+            prompts = [
+                self.format_docs_prompts_func(
+                    query=queries[0], docs=docs, instruction=instruction
+                )
+            ]
+        else:
+            prompts = [
+                self.format_docs_prompts_func(
+                    query=q, docs=[d], instruction=instruction
+                )
+                for q, d in zip(queries, docs)
+            ]
+
+        return self._preprocess_cmpl_offline(
+            prompts=prompts, tok_params=tok_params, prompt_extras=prompt_extras
+        )
+
+    def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
+        final_res_batch: list[PoolingRequestOutput] = []
+
+        for i in range(len(outputs)):
+            embeds = outputs[i].outputs.data.float()
+
+            # The JinaForRanking model concatenates docs first, then query.
+            # Let's stay consistent with this novel design.
+            query_embeds = embeds[-1]
+            doc_embeds = embeds[:-1]
+
+            scores = F.cosine_similarity(query_embeds, doc_embeds)
+
+            for score in scores:
+                final_res_batch.append(
+                    PoolingRequestOutput(
+                        request_id=outputs[i].request_id,
+                        outputs=score,
+                        prompt_token_ids=outputs[i].prompt_token_ids,
+                        num_cached_tokens=outputs[i].num_cached_tokens,
+                        finished=True,
+                    )
+                )
+        return final_res_batch
+
+
+ScoringIOProcessors: dict[str, type[ScoringIOProcessor]] = {
+    p.name: p
+    for p in [
+        BiEncoderIOProcessor,
+        LateInteractionIOProcessor,
+        JinaRankingIOProcessor,
+        FlashLateInteractionIOProcessor,
+        CrossEncoderIOProcessor,
+    ]
+}

vllm/entrypoints/pooling/scoring/protocol.py

@@ -0,0 +1,187 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import time
+from typing import Any, TypeAlias
+
+from pydantic import BaseModel, Field, model_validator
+
+from vllm import PoolingParams
+from vllm.config import ModelConfig
+from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
+from vllm.renderers import TokenizeParams
+from vllm.tasks import PoolingTask
+from vllm.utils import random_uuid
+
+from ..base.protocol import ClassifyRequestMixin, PoolingBasicRequestMixin
+from .typing import ScoreContentPartParam, ScoreInput
+
+
+class ScoringRequestMixin(PoolingBasicRequestMixin, ClassifyRequestMixin):
+    # --8<-- [start:scoring-common-params]
+    max_tokens_per_query: int = Field(
+        default=0,
+        description=(
+            "Maximum number of tokens per query. Queries longer than "
+            "this will be truncated to this length. 0 means no "
+            "query-level truncation is applied."
+        ),
+    )
+    max_tokens_per_doc: int = Field(
+        default=0,
+        description=(
+            "Maximum number of tokens per document. Documents longer than "
+            "this will be truncated to this length. 0 means no "
+            "document-level truncation is applied (only truncate_prompt_tokens "
+            "applies to the combined query+document)."
+        ),
+    )
+    instruction: str | None = Field(
+        default=None,
+        description=(
+            "Task instruction prepended to each scored pair via the chat "
+            "template. Equivalent to passing "
+            "chat_template_kwargs={'instruction': ...}."
+        ),
+    )
+    chat_template_kwargs: dict[str, Any] | None = Field(
+        default=None,
+        description=(
+            "Additional keyword args to pass to the chat template renderer. "
+            "Will be accessible by the score/rerank chat template."
+        ),
+    )
+    # --8<-- [end:scoring-common-params]
+
+    @model_validator(mode="after")
+    def _merge_instruction_into_kwargs(self) -> "ScoringRequestMixin":
+        """Fold the top-level `instruction` field into `chat_template_kwargs`.
+
+        This allows callers to use either the convenience field or the generic
+        dict.  Explicit keys inside `chat_template_kwargs` take precedence over
+        the top-level `instruction` field.
+        """
+        if self.instruction is not None:
+            merged = dict(self.chat_template_kwargs or {})
+            merged.setdefault("instruction", self.instruction)
+            self.chat_template_kwargs = merged
+        return self
+
+    def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
+        return self._build_pooling_tok_params(
+            model_config,
+            add_special_tokens=True,
+            max_total_tokens=model_config.max_model_len,
+            max_output_tokens=0,
+        )
+
+    def to_pooling_params(self, task: PoolingTask = "classify"):
+        return PoolingParams(
+            task=task,
+            use_activation=self.use_activation,
+        )
+
+
+class ScoreDataRequest(ScoringRequestMixin):
+    data_1: ScoreInput | list[ScoreInput]
+    data_2: ScoreInput | list[ScoreInput]
+
+
+class ScoreQueriesDocumentsRequest(ScoringRequestMixin):
+    # --8<-- [start:score-request-params]
+    queries: ScoreInput | list[ScoreInput]
+    documents: ScoreInput | list[ScoreInput]
+    # --8<-- [end:score-request-params]
+
+    @property
+    def data_1(self):
+        return self.queries
+
+    @property
+    def data_2(self):
+        return self.documents
+
+
+class ScoreQueriesItemsRequest(ScoringRequestMixin):
+    queries: ScoreInput | list[ScoreInput]
+    items: ScoreInput | list[ScoreInput]
+
+    @property
+    def data_1(self):
+        return self.queries
+
+    @property
+    def data_2(self):
+        return self.items
+
+
+class ScoreTextRequest(ScoringRequestMixin):
+    text_1: ScoreInput | list[ScoreInput]
+    text_2: ScoreInput | list[ScoreInput]
+
+    @property
+    def data_1(self):
+        return self.text_1
+
+    @property
+    def data_2(self):
+        return self.text_2
+
+
+ScoreRequest: TypeAlias = (
+    ScoreQueriesDocumentsRequest
+    | ScoreQueriesItemsRequest
+    | ScoreDataRequest
+    | ScoreTextRequest
+)
+
+
+class RerankRequest(ScoringRequestMixin):
+    # --8<-- [start:rerank-request-params]
+    query: ScoreInput
+    documents: ScoreInput | list[ScoreInput]
+    top_n: int = Field(default=0, ge=0)
+    # --8<-- [end:rerank-request-params]
+
+
+ScoringRequest: TypeAlias = ScoreRequest | RerankRequest
+
+
+class RerankDocument(BaseModel):
+    text: str | None = None
+    multi_modal: list[ScoreContentPartParam] | None = None
+
+
+class RerankResult(BaseModel):
+    index: int
+    document: RerankDocument
+    relevance_score: float
+
+
+class RerankUsage(BaseModel):
+    prompt_tokens: int
+    total_tokens: int
+
+
+class RerankResponse(OpenAIBaseModel):
+    id: str
+    model: str
+    usage: RerankUsage
+    results: list[RerankResult]
+
+
+class ScoreResponseData(OpenAIBaseModel):
+    index: int
+    object: str = "score"
+    score: float
+
+
+class ScoreResponse(OpenAIBaseModel):
+    id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
+    object: str = "list"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    model: str
+    data: list[ScoreResponseData]
+    usage: UsageInfo
+
+
+ScoringResponse: TypeAlias = RerankResponse | ScoreResponse

vllm/entrypoints/pooling/scoring/serving.py

@@ -0,0 +1,287 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from fastapi.responses import JSONResponse, Response
+
+from vllm import PoolingParams
+from vllm.engine.protocol import EngineClient
+from vllm.entrypoints.openai.engine.protocol import UsageInfo
+from vllm.logger import init_logger
+from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput
+from vllm.tasks import SCORE_TYPE_MAP, SupportedTask
+from vllm.v1.pool.late_interaction import (
+    build_late_interaction_doc_params,
+    build_late_interaction_query_params,
+)
+
+from ..base.io_processor import PoolingIOProcessor
+from ..base.serving import PoolingServing
+from .io_processor import ScoringIOProcessors, ScoringServeContext
+from .protocol import (
+    RerankDocument,
+    RerankRequest,
+    RerankResponse,
+    RerankResult,
+    RerankUsage,
+    ScoreRequest,
+    ScoreResponse,
+    ScoreResponseData,
+)
+from .typing import ScoreInput
+
+logger = init_logger(__name__)
+
+
+class ServingScores(PoolingServing):
+    request_id_prefix = "score"
+
+    def __init__(
+        self,
+        engine_client: EngineClient,
+        *args,
+        supported_tasks: tuple[SupportedTask, ...],
+        enable_flash_late_interaction: bool = True,
+        **kwargs,
+    ):
+        pooling_task = engine_client.model_config.get_pooling_task(supported_tasks)
+        score_type = SCORE_TYPE_MAP.get(pooling_task, None)  # type: ignore[arg-type]
+        assert score_type is not None
+
+        self.io_processor_name: str = score_type
+        self.enable_flash_late_interaction = (
+            self.io_processor_name == "late-interaction"
+            and enable_flash_late_interaction
+        )
+
+        if self.enable_flash_late_interaction:
+            self.io_processor_name = "flash-late-interaction"
+
+        if engine_client.model_config.architecture == "JinaForRanking":
+            self.io_processor_name = "jina-reranking-scoring"
+            self.enable_flash_late_interaction = False
+
+        super().__init__(engine_client, *args, **kwargs)
+
+    def init_io_processor(self, *args, **kwargs) -> PoolingIOProcessor:
+        return ScoringIOProcessors[self.io_processor_name](*args, **kwargs)
+
+    async def __call__(self, *args, **kwargs) -> Response:
+        if not self.enable_flash_late_interaction:
+            return await super().__call__(*args, **kwargs)
+
+        return await self.flash_late_interaction(*args, **kwargs)
+
+    def _build_response(
+        self,
+        ctx: ScoringServeContext,
+    ) -> JSONResponse:
+        final_res_batch = ctx.final_res_batch
+        request_id = ctx.request_id
+        created_time = ctx.created_time
+        model_name = self.models.model_name()
+
+        if isinstance(ctx.request, ScoreRequest):
+            return self._request_output_to_score_response(
+                final_res_batch,
+                request_id,
+                created_time,
+                model_name,
+            )
+        elif isinstance(ctx.request, RerankRequest):
+            return self._request_output_to_rerank_response(
+                final_res_batch,
+                request_id,
+                model_name,
+                ctx.request.documents,
+                ctx.request.top_n if ctx.request.top_n > 0 else len(final_res_batch),
+            )
+        else:
+            raise ValueError(f"Invalid {self.request_id_prefix} request type")
+
+    def _request_output_to_score_response(
+        self,
+        final_res_batch: list[PoolingRequestOutput],
+        request_id: str,
+        created_time: int,
+        model_name: str,
+    ) -> JSONResponse:
+        items: list[ScoreResponseData] = []
+        num_prompt_tokens = 0
+
+        for idx, final_res in enumerate(final_res_batch):
+            classify_res = ScoringRequestOutput.from_base(final_res)
+
+            item = ScoreResponseData(
+                index=idx,
+                score=classify_res.outputs.score,
+            )
+            prompt_token_ids = final_res.prompt_token_ids
+
+            items.append(item)
+            num_prompt_tokens += len(prompt_token_ids)
+
+        usage = UsageInfo(
+            prompt_tokens=num_prompt_tokens,
+            total_tokens=num_prompt_tokens,
+        )
+
+        response = ScoreResponse(
+            id=request_id,
+            created=created_time,
+            model=model_name,
+            data=items,
+            usage=usage,
+        )
+
+        return JSONResponse(content=response.model_dump())
+
+    def _request_output_to_rerank_response(
+        self,
+        final_res_batch: list[PoolingRequestOutput],
+        request_id: str,
+        model_name: str,
+        documents: ScoreInput | list[ScoreInput],
+        top_n: int,
+    ) -> JSONResponse:
+        if not isinstance(documents, list):
+            documents = [documents]
+
+        results: list[RerankResult] = []
+        num_prompt_tokens = 0
+        for idx, final_res in enumerate(final_res_batch):
+            classify_res = ScoringRequestOutput.from_base(final_res)
+
+            document = documents[idx]
+            if isinstance(document, str):
+                rerank_document = RerankDocument(text=document)
+            else:
+                rerank_document = RerankDocument(
+                    multi_modal=document.get("content", [])
+                )
+
+            result = RerankResult(
+                index=idx,
+                document=rerank_document,
+                relevance_score=classify_res.outputs.score,
+            )
+            results.append(result)
+            prompt_token_ids = final_res.prompt_token_ids
+            num_prompt_tokens += len(prompt_token_ids)
+
+        # sort by relevance, then return the top n if set
+        results.sort(key=lambda x: x.relevance_score, reverse=True)
+        if top_n < len(documents):
+            results = results[:top_n]
+
+        response = RerankResponse(
+            id=request_id,
+            model=model_name,
+            results=results,
+            usage=RerankUsage(
+                total_tokens=num_prompt_tokens, prompt_tokens=num_prompt_tokens
+            ),
+        )
+
+        return JSONResponse(content=response.model_dump())
+
+    ###################################################################################
+    ### Run pooling score MaxSim on worker side (GPU) in the API server process
+    ### Can significantly improve late-interaction scoring performance.
+
+    async def flash_late_interaction(self, *args, **kwargs) -> Response:
+        ctx = await self._init_ctx(self.io_processor, *args, **kwargs)
+        await self._preprocessing_async(self.io_processor, ctx)
+
+        # stage 1: encode queries and cache token embeddings on workers.
+        await self._flash_late_interaction_encode_queries(ctx)
+        # stage 2: encode docs and return scalar scores from workers.
+        await self._flash_late_interaction_encode_docs(ctx)
+
+        return await self._postprocessing_async(self.io_processor, ctx)
+
+    async def _flash_late_interaction_encode_queries(self, ctx: ScoringServeContext):
+        assert ctx.n_queries is not None
+        assert ctx.engine_inputs is not None
+        assert isinstance(ctx.pooling_params, PoolingParams)
+
+        n_queries = ctx.n_queries
+        n_docs = len(ctx.engine_inputs) - n_queries
+        query_engine_inputs = ctx.engine_inputs[:n_queries]
+
+        query_keys = [f"{ctx.request_id}-query-{i}" for i in range(n_queries)]
+        query_uses = [n_docs if n_queries == 1 else 1] * n_queries
+
+        query_pooling_params_list = []
+        for i in range(n_queries):
+            pooling_params = ctx.pooling_params.clone()
+            pooling_params.late_interaction_params = (
+                build_late_interaction_query_params(
+                    query_key=query_keys[i],
+                    query_uses=query_uses[i],
+                )
+            )
+            query_pooling_params_list.append(pooling_params)
+
+        assert (
+            n_queries
+            == len(query_pooling_params_list)
+            == len(query_engine_inputs)
+            == len(query_keys)
+        )
+
+        query_ctx = ScoringServeContext(
+            request=ctx.request,
+            raw_request=ctx.raw_request,
+            model_name=ctx.model_name,
+            request_id=ctx.request_id,
+            pooling_params=query_pooling_params_list,
+            prompt_request_ids=query_keys,
+            engine_inputs=query_engine_inputs,
+        )
+
+        await self._prepare_generators(query_ctx)
+        await self._collect_batch(query_ctx)
+        ctx.query_final_res_batch = query_ctx.final_res_batch
+
+    async def _flash_late_interaction_encode_docs(self, ctx: ScoringServeContext):
+        assert ctx.n_queries is not None
+        assert ctx.engine_inputs is not None
+        assert isinstance(ctx.pooling_params, PoolingParams)
+
+        n_queries = ctx.n_queries
+        n_docs = len(ctx.engine_inputs) - n_queries
+        doc_engine_inputs = ctx.engine_inputs[n_queries:]
+
+        query_keys = [f"{ctx.request_id}-query-{i}" for i in range(n_queries)]
+        doc_keys = [f"{ctx.request_id}-doc-{i}" for i in range(n_docs)]
+
+        doc_pooling_params_list = []
+        for i in range(n_docs):
+            query_idx = 0 if n_queries == 1 else i
+            pooling_params = ctx.pooling_params.clone()
+            pooling_params.late_interaction_params = build_late_interaction_doc_params(
+                query_key=query_keys[query_idx]
+            )
+            doc_pooling_params_list.append(pooling_params)
+
+        assert (
+            n_docs
+            == len(doc_pooling_params_list)
+            == len(doc_engine_inputs)
+            == len(doc_keys)
+        )
+
+        doc_ctx = ScoringServeContext(
+            request=ctx.request,
+            raw_request=ctx.raw_request,
+            model_name=ctx.model_name,
+            request_id=ctx.request_id,
+            pooling_params=doc_pooling_params_list,
+            prompt_request_ids=doc_keys,
+            engine_inputs=doc_engine_inputs,
+        )
+
+        await self._prepare_generators(doc_ctx)
+        await self._collect_batch(doc_ctx)
+
+        ctx.final_res_batch = doc_ctx.final_res_batch

vllm/entrypoints/pooling/scoring/typing.py

@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from dataclasses import dataclass
+from typing import TypeAlias
+
+from typing_extensions import Required, TypedDict
+
+from vllm.entrypoints.chat_utils import (
+    ChatCompletionContentPartImageEmbedsParam,
+    ChatCompletionContentPartImageParam,
+    ChatCompletionContentPartTextParam,
+    ChatCompletionContentPartVideoParam,
+)
+
+ScoreContentPartParam: TypeAlias = (
+    ChatCompletionContentPartImageParam
+    | ChatCompletionContentPartImageEmbedsParam
+    | ChatCompletionContentPartTextParam
+    | ChatCompletionContentPartVideoParam
+)
+
+
+class ScoreMultiModalParam(TypedDict, total=False):
+    """
+    A specialized parameter type for scoring multimodal content
+
+    The reasons why don't reuse `CustomChatCompletionMessageParam` directly:
+    1. Score tasks don't need the 'role' field (user/assistant/system) that's required in chat completions
+    2. Including chat-specific fields would confuse users about their purpose in scoring
+    3. This is a more focused interface that only exposes what's needed for scoring
+    """  # noqa: E501
+
+    content: Required[list[ScoreContentPartParam]]
+    """The multimodal contents"""
+
+
+# Raw input data with content key in ScoreMultiModalParam.
+ScoreInput = str | ScoreMultiModalParam
+# Score data without content key.
+ScoreData = str | list[ScoreContentPartParam]
+
+
+@dataclass
+class ScoringData:
+    data_1: list[ScoreData]
+    data_2: list[ScoreData]