chore: import upstream snapshot with attribution

python/ray/serve/llm/__init__.py

@@ -0,0 +1,439 @@
+from typing import TYPE_CHECKING, Optional, Type
+
+from ray._common.deprecation import Deprecated
+from ray.llm._internal.serve.core.configs.llm_config import (
+    CloudMirrorConfig as _CloudMirrorConfig,
+    LLMConfig as _LLMConfig,
+    LoraConfig as _LoraConfig,
+    ModelLoadingConfig as _ModelLoadingConfig,
+)
+from ray.llm._internal.serve.core.ingress.builder import (
+    LLMServingArgs as _LLMServingArgs,
+)
+from ray.llm._internal.serve.core.ingress.ingress import (
+    OpenAiIngress as _OpenAiIngress,
+)
+
+# For backward compatibility
+from ray.llm._internal.serve.core.server.llm_server import (
+    LLMServer as _LLMServer,
+)
+from ray.util.annotations import PublicAPI
+
+if TYPE_CHECKING:
+    from ray.serve.deployment import Application
+
+
+##########
+# Models
+##########
+
+
+@PublicAPI(stability="beta")
+class LLMConfig(_LLMConfig):
+    """The configuration for starting an LLM deployment."""
+
+    pass
+
+
+@PublicAPI(stability="beta")
+class LLMServingArgs(_LLMServingArgs):
+    """The configuration for starting an LLM deployment application."""
+
+    pass
+
+
+@PublicAPI(stability="beta")
+class ModelLoadingConfig(_ModelLoadingConfig):
+    """The configuration for loading an LLM model."""
+
+    pass
+
+
+@PublicAPI(stability="beta")
+class CloudMirrorConfig(_CloudMirrorConfig):
+    """The configuration for mirroring an LLM model from cloud storage."""
+
+    pass
+
+
+@PublicAPI(stability="beta")
+class LoraConfig(_LoraConfig):
+    """The configuration for loading an LLM model with LoRA."""
+
+    pass
+
+
+#############
+# Deployments
+#############
+
+
+@Deprecated(
+    old="ray.serve.llm.LLMServer", new="ray.serve.llm.deployment.LLMServer", error=False
+)
+class LLMServer(_LLMServer):
+    pass
+
+
+@Deprecated(
+    old="ray.serve.llm.LLMRouter",
+    new="ray.serve.llm.ingress.OpenAIIngress",
+    error=False,
+)
+class LLMRouter(_OpenAiIngress):
+    pass
+
+
+##########
+# Builders
+##########
+
+
+@PublicAPI(stability="beta")
+def build_llm_deployment(
+    llm_config: "LLMConfig",
+    *,
+    name_prefix: Optional[str] = None,
+    bind_kwargs: Optional[dict] = None,
+    override_serve_options: Optional[dict] = None,
+    deployment_cls: Optional[Type[LLMServer]] = None,
+) -> "Application":
+    """Helper to build a single vllm deployment from the given llm config.
+
+    Examples:
+        .. testcode::
+            :skipif: True
+
+            from ray import serve
+            from ray.serve.llm import LLMConfig, build_llm_deployment
+
+            # Configure the model
+            llm_config = LLMConfig(
+                model_loading_config=dict(
+                    model_id="llama-3.1-8b",
+                    model_source="meta-llama/Llama-3.1-8b-instruct",
+                ),
+                deployment_config=dict(
+                    autoscaling_config=dict(
+                        min_replicas=1,
+                        max_replicas=2,
+                    )
+                ),
+                accelerator_type="A10G",
+            )
+
+            # Build the deployment
+            llm_app = build_llm_deployment(llm_config)
+
+            # Deploy the application
+            model_handle = serve.run(llm_app)
+
+            # Querying the model handle
+            import asyncio
+            model_handle = model_handle.options(stream=True)
+            async def query_model(model_handle):
+                from ray.serve.llm.openai_api_models import ChatCompletionRequest
+
+                request = ChatCompletionRequest(
+                    model="qwen-0.5b",
+                    messages=[
+                        {
+                            "role": "user",
+                            "content": "Hello, world!"
+                        }
+                    ]
+                )
+
+                resp = model_handle.chat.remote(request)
+                async for message in resp:
+                    print("message: ", message)
+
+            asyncio.run(query_model(model_handle))
+
+    Args:
+        llm_config: The llm config to build vllm deployment.
+        name_prefix: Optional prefix to be used for the deployment name.
+        bind_kwargs: Optional kwargs to pass to the deployment.
+        override_serve_options: Optional serve options to override the original serve options based on the llm_config.
+        deployment_cls: Optional deployment class to use.
+
+    Returns:
+        The configured Ray Serve Application for vllm deployment.
+    """
+    from ray.llm._internal.serve.core.server.builder import (
+        build_llm_deployment,
+    )
+
+    return build_llm_deployment(
+        llm_config=llm_config,
+        name_prefix=name_prefix,
+        bind_kwargs=bind_kwargs,
+        override_serve_options=override_serve_options,
+        deployment_cls=deployment_cls,
+    )
+
+
+@PublicAPI(stability="beta")
+def build_openai_app(llm_serving_args: dict) -> "Application":
+    """Helper to build an OpenAI compatible app with the llm deployment setup from
+    the given llm serving args. This is the main entry point for users to create a
+    Serve application serving LLMs.
+
+
+    Examples:
+        .. code-block:: python
+            :caption: Example usage in code.
+
+            from ray import serve
+            from ray.serve.llm import LLMConfig, LLMServingArgs, build_openai_app
+
+            llm_config1 = LLMConfig(
+                model_loading_config=dict(
+                    model_id="qwen-0.5b",
+                    model_source="Qwen/Qwen2.5-0.5B-Instruct",
+                ),
+                deployment_config=dict(
+                    autoscaling_config=dict(
+                        min_replicas=1, max_replicas=2,
+                    )
+                ),
+                accelerator_type="A10G",
+            )
+
+            llm_config2 = LLMConfig(
+                model_loading_config=dict(
+                    model_id="qwen-1.5b",
+                    model_source="Qwen/Qwen2.5-1.5B-Instruct",
+                ),
+                deployment_config=dict(
+                    autoscaling_config=dict(
+                        min_replicas=1, max_replicas=2,
+                    )
+                ),
+                accelerator_type="A10G",
+            )
+
+            # Deploy the application
+            llm_app = build_openai_app(
+                LLMServingArgs(
+                    llm_configs=[
+                        llm_config1,
+                        llm_config2,
+                    ]
+                )
+            )
+            serve.run(llm_app)
+
+
+            # Querying the model via openai client
+            from openai import OpenAI
+
+            # Initialize client
+            client = OpenAI(base_url="http://localhost:8000/v1", api_key="fake-key")
+
+            # Basic completion
+            response = client.chat.completions.create(
+                model="qwen-0.5b",
+                messages=[{"role": "user", "content": "Hello!"}]
+            )
+
+        .. code-block:: yaml
+            :caption: Example usage in YAML.
+
+            # config.yaml
+            applications:
+            - args:
+                llm_configs:
+                    - model_loading_config:
+                        model_id: qwen-0.5b
+                        model_source: Qwen/Qwen2.5-0.5B-Instruct
+                      accelerator_type: A10G
+                      deployment_config:
+                        autoscaling_config:
+                            min_replicas: 1
+                            max_replicas: 2
+                    - model_loading_config:
+                        model_id: qwen-1.5b
+                        model_source: Qwen/Qwen2.5-1.5B-Instruct
+                      accelerator_type: A10G
+                      deployment_config:
+                        autoscaling_config:
+                            min_replicas: 1
+                            max_replicas: 2
+              import_path: ray.serve.llm:build_openai_app
+              name: llm_app
+              route_prefix: "/"
+
+
+    Args:
+        llm_serving_args: A dict that conforms to the LLMServingArgs pydantic model.
+
+    Returns:
+        The configured Ray Serve Application router.
+    """
+    from ray.llm._internal.serve.core.ingress.builder import (
+        build_openai_app,
+    )
+
+    return build_openai_app(builder_config=llm_serving_args)
+
+
+@PublicAPI(stability="alpha")
+def build_pd_openai_app(pd_serving_args: dict) -> "Application":
+    """Build a deployable application utilizing P/D disaggregation.
+
+
+    Examples:
+        .. code-block:: python
+            :caption: Example usage in code.
+
+            from ray import serve
+            from ray.serve.llm import LLMConfig, build_pd_openai_app
+
+            config = LLMConfig(
+                model_loading_config=dict(
+                    model_id="qwen-0.5b",
+                    model_source="Qwen/Qwen2.5-0.5B-Instruct",
+                ),
+                deployment_config=dict(
+                    autoscaling_config=dict(
+                        min_replicas=1, max_replicas=2,
+                    )
+                ),
+                accelerator_type="A10G",
+            )
+
+            # Deploy the application
+            llm_app = build_pd_openai_app(
+                dict(
+                    prefill_config=config,
+                    decode_config=config,
+                )
+            )
+
+            serve.run(llm_app)
+
+
+            # Querying the model via openai client
+            from openai import OpenAI
+
+            # Initialize client
+            client = OpenAI(base_url="http://localhost:8000/v1", api_key="fake-key")
+
+            # Basic completion
+            response = client.chat.completions.create(
+                model="qwen-0.5b",
+                messages=[{"role": "user", "content": "Hello!"}]
+            )
+
+        .. code-block:: yaml
+            :caption: Example usage in YAML.
+
+            # config.yaml
+            applications:
+            - args:
+                prefill_config:
+                    model_loading_config:
+                        model_id: qwen-0.5b
+                        model_source: Qwen/Qwen2.5-0.5B-Instruct
+                    accelerator_type: A10G
+                    deployment_config:
+                        autoscaling_config:
+                            min_replicas: 1
+                            max_replicas: 2
+                decode_config:
+                    model_loading_config:
+                    model_id: qwen-1.5b
+                    model_source: Qwen/Qwen2.5-1.5B-Instruct
+                    accelerator_type: A10G
+                    deployment_config:
+                    autoscaling_config:
+                        min_replicas: 1
+                        max_replicas: 2
+              import_path: ray.serve.llm:build_pd_openai_app
+              name: llm_app
+              route_prefix: "/"
+
+
+    Args:
+        pd_serving_args: The dictionary containing prefill and decode configs. See PDServingArgs for more details.
+
+    Returns:
+        The configured Ray Serve Application router.
+    """
+    from ray.llm._internal.serve.serving_patterns.prefill_decode.builder import (
+        build_pd_openai_app,
+    )
+
+    return build_pd_openai_app(pd_serving_args=pd_serving_args)
+
+
+@PublicAPI(stability="alpha")
+def build_dp_deployment(
+    llm_config: "LLMConfig",
+    *,
+    name_prefix: Optional[str] = None,
+    bind_kwargs: Optional[dict] = None,
+    override_serve_options: Optional[dict] = None,
+    deployment_cls: Optional[Type] = None,
+) -> "Application":
+    """Build a data parallel attention LLM deployment.
+
+    Args:
+        llm_config: The LLM configuration.
+        name_prefix: The prefix to add to the deployment name.
+        bind_kwargs: Optional extra kwargs to pass to the deployment constructor.
+        override_serve_options: The optional serve options to override the
+            default options.
+        deployment_cls: Optional deployment class to use. Defaults to DPServer.
+
+    Returns:
+        The Ray Serve Application for the data parallel attention LLM deployment.
+    """
+    from ray.llm._internal.serve.serving_patterns.data_parallel.builder import (
+        build_dp_deployment,
+    )
+
+    return build_dp_deployment(
+        llm_config=llm_config,
+        name_prefix=name_prefix,
+        bind_kwargs=bind_kwargs,
+        override_serve_options=override_serve_options,
+        deployment_cls=deployment_cls,
+    )
+
+
+@PublicAPI(stability="alpha")
+def build_dp_openai_app(dp_serving_args: dict) -> "Application":
+    """Build an OpenAI compatible app with the DP attention deployment
+    setup from the given builder configuration.
+
+    Args:
+        dp_serving_args: The configuration for the builder. It has to conform
+            to the DPOpenAiServingArgs pydantic model.
+
+    Returns:
+        The configured Ray Serve Application.
+    """
+    from ray.llm._internal.serve.serving_patterns.data_parallel.builder import (
+        build_dp_openai_app,
+    )
+
+    return build_dp_openai_app(builder_config=dp_serving_args)
+
+
+__all__ = [
+    "LLMConfig",
+    "LLMServingArgs",
+    "ModelLoadingConfig",
+    "CloudMirrorConfig",
+    "LoraConfig",
+    "build_llm_deployment",
+    "build_openai_app",
+    "build_pd_openai_app",
+    "build_dp_deployment",
+    "build_dp_openai_app",
+    "LLMServer",
+    "LLMRouter",
+]

python/ray/serve/llm/deployment.py

@@ -0,0 +1,171 @@
+import warnings
+
+from ray.llm._internal.serve.core.server.llm_server import (
+    LLMServer as InternalLLMServer,
+)
+from ray.llm._internal.serve.serving_patterns.data_parallel.dp_server import (
+    DPServer as _DPServer,
+)
+from ray.llm._internal.serve.serving_patterns.prefill_decode.pd_server import (
+    PDDecodeServer as _PDDecodeServer,
+    PDPrefillServer as _PDPrefillServer,
+    PDProxyServer as _PDProxyServer,  # TODO(Kourosh): Deprecated, remove in Ray 2.58.
+)
+from ray.util.annotations import PublicAPI
+
+#############
+# Deployments
+#############
+
+
+@PublicAPI(stability="beta")
+class LLMServer(InternalLLMServer):
+    """The implementation of the vLLM engine deployment.
+
+    To build a Deployment object you should use `build_llm_deployment` function.
+    We also expose a lower level API for more control over the deployment class
+    through `serve.deployment` function.
+
+    Examples:
+        .. testcode::
+            :skipif: True
+
+            from ray import serve
+            from ray.serve.llm import LLMConfig
+            from ray.serve.llm.deployment import LLMServer
+
+            # Configure the model
+            llm_config = LLMConfig(
+                model_loading_config=dict(
+                    served_model_name="llama-3.1-8b",
+                    model_source="meta-llama/Llama-3.1-8b-instruct",
+                ),
+                deployment_config=dict(
+                    autoscaling_config=dict(
+                        min_replicas=1,
+                        max_replicas=8,
+                    )
+                ),
+            )
+
+            # Build the deployment directly
+            serve_options = LLMServer.get_deployment_options(llm_config)
+            llm_app = serve.deployment(LLMServer).options(
+                **serve_options).bind(llm_config)
+
+            model_handle = serve.run(llm_app)
+
+            # Query the model via `chat` api
+            from ray.serve.llm.openai_api_models import ChatCompletionRequest
+            request = ChatCompletionRequest(
+                model="llama-3.1-8b",
+                messages=[
+                    {
+                        "role": "user",
+                        "content": "Hello, world!"
+                    }
+                ]
+            )
+            response = ray.get(model_handle.chat(request))
+            print(response)
+    """
+
+    pass
+
+
+@PublicAPI(stability="beta")
+class PDDecodeServer(_PDDecodeServer):
+    """Decode-side LLM server for prefill-decode disaggregation.
+
+    This deployment owns a real engine (decode config) and holds a handle
+    to the prefill deployment. For chat/completions it runs remote prefill
+    first, then local decode.
+
+    Use ``build_pd_openai_app`` to construct the full 3-tier PD graph.
+    """
+
+    pass
+
+
+@PublicAPI(stability="beta")
+class PDPrefillServer(_PDPrefillServer):
+    """Prefill-side LLM server for prefill-decode disaggregation.
+
+    A standard LLMServer with an additional ``prewarm_prefill`` method
+    used during the optional pre-warm handshake.
+    """
+
+    pass
+
+
+# TODO(Kourosh): Deprecated, remove in Ray 2.58.
+class PDProxyServer(_PDProxyServer):
+    """A proxy server for prefill-decode disaggregation.
+
+    .. deprecated::
+        ``PDProxyServer`` is deprecated. Use ``PDDecodeServer`` instead.
+        This class will be removed in a future release.
+    """
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        warnings.warn(
+            "PDProxyServer is deprecated and will be removed in Ray 2.58. "
+            "Use PDDecodeServer (decode orchestrator) and PDPrefillServer instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+
+
+@PublicAPI(stability="beta")
+class DPServer(_DPServer):
+    """Data Parallel LLM Server.
+
+    This class is used to serve data parallel attention (DP Attention)
+    deployment paradigm, where the attention layers are replicated and
+    the MoE layers are sharded. DP Attention is typically used for models
+    like DeepSeek-V3.
+
+    To build a Deployment object you should use `build_dp_deployment` function.
+    We also expose a lower level API for more control over the deployment class
+    through `serve.deployment` function.
+
+    Examples:
+        .. testcode::
+            :skipif: True
+
+            from ray import serve
+            from ray.serve.llm import LLMConfig, build_dp_deployment
+
+            # Configure the model
+            llm_config = LLMConfig(
+                model_loading_config=dict(
+                    model_id="Qwen/Qwen2.5-0.5B-Instruct",
+                ),
+                engine_kwargs=dict(
+                    data_parallel_size=2,
+                    tensor_parallel_size=1,
+                ),
+                experimental_configs=dict(
+                    dp_size_per_node=2,
+                ),
+                accelerator_type="A10G",
+            )
+
+            # Build the deployment
+            dp_app = build_dp_deployment(llm_config)
+
+            # Deploy the application
+            model_handle = serve.run(dp_app)
+    """
+
+    pass
+
+
+__all__ = [
+    "LLMServer",
+    "PDDecodeServer",
+    "PDPrefillServer",
+    "PDProxyServer",  # TODO(Kourosh): Deprecate in Ray 2.56, remove in Ray 2.58.
+    "DPServer",
+]

python/ray/serve/llm/gen_config.py

@@ -0,0 +1,16 @@
+"""Stub for the removed Serve LLM config generator."""
+import sys
+
+
+def main():
+    print(
+        "The Serve LLM config generator is no longer supported and this command "
+        "will be removed in a future Ray version. "
+        "See https://recipes.vllm.ai/ for current guidance on serving LLMs.",
+        file=sys.stderr,
+    )
+    raise SystemExit(1)
+
+
+if __name__ == "__main__":
+    main()

python/ray/serve/llm/ingress.py

@@ -0,0 +1,102 @@
+from ray.llm._internal.serve.core.ingress.ingress import (
+    OpenAiIngress as _OpenAiIngress,
+    make_fastapi_ingress,
+)
+from ray.util.annotations import PublicAPI
+
+
+@PublicAPI(stability="beta")
+class OpenAiIngress(_OpenAiIngress):
+
+    """The implementation of the OpenAI compatible model router.
+
+    This deployment creates the following endpoints:
+      - /v1/chat/completions: Chat interface (OpenAI-style)
+      - /v1/completions: Text completion
+      - /v1/models: List available models
+      - /v1/models/{model}: Model information
+      - /v1/embeddings: Text embeddings
+      - /v1/audio/transcriptions: Audio transcription
+      - /v1/score: Text scoring
+
+
+    Examples:
+        .. testcode::
+            :skipif: True
+
+
+            from ray import serve
+            from ray.llm._internal.serve.core.configs.openai_api_models import (
+                to_model_metadata,
+            )
+            from ray.serve.llm import LLMConfig
+            from ray.serve.llm.deployment import LLMServer
+            from ray.serve.llm.ingress import OpenAiIngress, make_fastapi_ingress
+
+            llm_config1 = LLMConfig(
+                model_loading_config=dict(
+                    model_id="qwen-0.5b",
+                    model_source="Qwen/Qwen2.5-0.5B-Instruct",
+                ),
+                deployment_config=dict(
+                    autoscaling_config=dict(
+                        min_replicas=1, max_replicas=2,
+                    )
+                ),
+                accelerator_type="A10G",
+            )
+
+            llm_config2 = LLMConfig(
+                model_loading_config=dict(
+                    model_id="qwen-1.5b",
+                    model_source="Qwen/Qwen2.5-1.5B-Instruct",
+                ),
+                deployment_config=dict(
+                    autoscaling_config=dict(
+                        min_replicas=1, max_replicas=2,
+                    )
+                ),
+                accelerator_type="A10G",
+            )
+
+            # deployment #1
+            server_options1 = LLMServer.get_deployment_options(llm_config1)
+            server_deployment1 = serve.deployment(LLMServer).options(
+                **server_options1).bind(llm_config1)
+
+            # deployment #2
+            server_options2 = LLMServer.get_deployment_options(llm_config2)
+            server_deployment2 = serve.deployment(LLMServer).options(
+                **server_options2).bind(llm_config2)
+
+            # ingress: pass dicts keyed by model_id; no remote llm_config fetch.
+            ingress_options = OpenAiIngress.get_deployment_options(
+                llm_configs=[llm_config1, llm_config2])
+            ingress_cls = make_fastapi_ingress(OpenAiIngress)
+            ingress_deployment = (
+                serve.deployment(ingress_cls)
+                .options(**ingress_options)
+                .bind(
+                    llm_deployments={
+                        llm_config1.model_id: server_deployment1,
+                        llm_config2.model_id: server_deployment2,
+                    },
+                    model_cards={
+                        llm_config1.model_id: to_model_metadata(
+                            llm_config1.model_id, llm_config1
+                        ),
+                        llm_config2.model_id: to_model_metadata(
+                            llm_config2.model_id, llm_config2
+                        ),
+                    },
+                )
+            )
+
+            # run
+            serve.run(ingress_deployment, blocking=True)
+    """
+
+    pass
+
+
+__all__ = ["OpenAiIngress", "make_fastapi_ingress"]

python/ray/serve/llm/openai_api_models.py

@@ -0,0 +1,125 @@
+from ray.llm._internal.serve.core.configs.openai_api_models import (
+    ChatCompletionRequest as _ChatCompletionRequest,
+    ChatCompletionResponse as _ChatCompletionResponse,
+    ChatCompletionStreamResponse as _ChatCompletionStreamResponse,
+    CompletionRequest as _CompletionRequest,
+    CompletionResponse as _CompletionResponse,
+    CompletionStreamResponse as _CompletionStreamResponse,
+    EmbeddingRequest as _EmbeddingRequest,
+    EmbeddingResponse as _EmbeddingResponse,
+    ErrorResponse as _ErrorResponse,
+    TranscriptionRequest as _TranscriptionRequest,
+    TranscriptionResponse as _TranscriptionResponse,
+    TranscriptionStreamResponse as _TranscriptionStreamResponse,
+)
+from ray.util.annotations import PublicAPI
+
+
+@PublicAPI(stability="beta")
+class ChatCompletionRequest(_ChatCompletionRequest):
+    """ChatCompletionRequest is the request body for the chat completion API.
+
+    This model is compatible with vLLM's OpenAI API models.
+    """
+
+    pass
+
+
+@PublicAPI(stability="beta")
+class CompletionRequest(_CompletionRequest):
+    """CompletionRequest is the request body for the completion API.
+
+    This model is compatible with vLLM's OpenAI API models.
+    """
+
+    pass
+
+
+@PublicAPI(stability="beta")
+class ChatCompletionStreamResponse(_ChatCompletionStreamResponse):
+    """ChatCompletionStreamResponse is the response body for the chat completion API.
+
+    This model is compatible with vLLM's OpenAI API models.
+    """
+
+    pass
+
+
+@PublicAPI(stability="beta")
+class ChatCompletionResponse(_ChatCompletionResponse):
+    """ChatCompletionResponse is the response body for the chat completion API.
+
+    This model is compatible with vLLM's OpenAI API models.
+    """
+
+    pass
+
+
+@PublicAPI(stability="beta")
+class CompletionStreamResponse(_CompletionStreamResponse):
+    """CompletionStreamResponse is the response body for the completion API.
+
+    This model is compatible with vLLM's OpenAI API models.
+    """
+
+    pass
+
+
+@PublicAPI(stability="beta")
+class CompletionResponse(_CompletionResponse):
+    """CompletionResponse is the response body for the completion API.
+
+    This model is compatible with vLLM's OpenAI API models.
+    """
+
+    pass
+
+
+EmbeddingRequest = _EmbeddingRequest
+
+
+@PublicAPI(stability="beta")
+class EmbeddingResponse(_EmbeddingResponse):
+    """EmbeddingResponse is the response body for the embedding API.
+
+    This model is compatible with vLLM's OpenAI API models.
+    """
+
+    pass
+
+
+@PublicAPI(stability="beta")
+class TranscriptionRequest(_TranscriptionRequest):
+    """TranscriptionRequest is the request body for the transcription API.
+
+    This model is compatible with vLLM's OpenAI API models.
+    """
+
+    pass
+
+
+@PublicAPI(stability="beta")
+class TranscriptionResponse(_TranscriptionResponse):
+    """TranscriptionResponse is the response body for the transcription API.
+
+    This model is compatible with vLLM's OpenAI API models.
+    """
+
+    pass
+
+
+@PublicAPI(stability="beta")
+class TranscriptionStreamResponse(_TranscriptionStreamResponse):
+    """TranscriptionStreamResponse is the response body for the transcription API.
+
+    This model is compatible with vLLM's OpenAI API models.
+    """
+
+    pass
+
+
+@PublicAPI(stability="beta")
+class ErrorResponse(_ErrorResponse):
+    """The returned response in case of an error."""
+
+    pass

python/ray/serve/llm/request_router.py

@@ -0,0 +1,52 @@
+from ray.llm._internal.serve.routing_policies.kv_aware.kv_aware_router import (
+    KVAwareRouter as _KVAwareRouter,
+)
+from ray.llm._internal.serve.routing_policies.prefix_aware.prefix_aware_router import (
+    PrefixCacheAffinityRouter as _PrefixCacheAffinityRouter,
+)
+from ray.util.annotations import PublicAPI
+
+
+@PublicAPI(stability="beta")
+class PrefixCacheAffinityRouter(_PrefixCacheAffinityRouter):
+    """A request router that is aware of the KV cache.
+
+    This router optimizes request routing by considering KV cache locality,
+    directing requests with similar prefixes to the same replica to improve
+    cache hit rates.
+
+    The internal policy is this (it may change in the future):
+
+    1. Mixes between three strategies to balance prefix cache hit rate and load
+    balancing:
+       - When load is balanced (queue length difference < threshold), it
+       selects replicas with the highest prefix match rate for the input text
+       - When load is balanced but match rate is below 10%, it falls back to
+       the smallest tenants (i.e. the replica with the least kv cache)
+       - When load is imbalanced, it uses the default Power of Two selection
+
+    2. Maintains a prefix tree to track which replicas have processed similar
+    inputs:
+       - Inserts prompt text into the prefix tree after routing
+       - Uses this history to inform future routing decisions
+
+    Parameters:
+        imbalanced_threshold: The threshold for considering the load imbalanced.
+        match_rate_threshold: The threshold for considering the match rate.
+        do_eviction: Whether to do eviction.
+        eviction_threshold_chars: Number of characters in the tree to trigger
+            eviction.
+        eviction_target_chars: Number of characters in the tree to target for
+            eviction.
+        eviction_interval_secs: How often (in seconds) to run the eviction
+        policy.
+    """
+
+    pass
+
+
+@PublicAPI(stability="alpha")
+class KVAwareRouter(_KVAwareRouter):
+    """A request router that routes by KV-cache overlap via a KV router actor."""
+
+    pass