chore: import upstream snapshot with attribution

python/ray/llm/_internal/serve/utils/server_utils.py

@@ -0,0 +1,218 @@
+import asyncio
+import threading
+import time
+import traceback
+from functools import partial
+from typing import Awaitable, Callable, TypeVar
+
+from fastapi import HTTPException, status
+from httpx import HTTPStatusError as HTTPXHTTPStatusError
+from pydantic import ValidationError as PydanticValidationError
+
+from ray import serve
+from ray.llm._internal.common.errors import VLLM_FATAL_ERRORS
+from ray.llm._internal.serve.constants import DEFAULT_FATAL_ERROR_COOLDOWN_S
+from ray.llm._internal.serve.core.configs.openai_api_models import (
+    ErrorInfo,
+    ErrorResponse,
+    OpenAIHTTPException,
+)
+from ray.llm._internal.serve.observability.logging import get_logger
+
+logger = get_logger(__name__)
+
+T = TypeVar("T")
+
+
+def _is_fatal_engine_error(e: Exception) -> bool:
+    """Detect fatal engine errors via isinstance check."""
+    if not VLLM_FATAL_ERRORS:
+        return False
+    return isinstance(e, VLLM_FATAL_ERRORS)
+
+
+class _FatalEngineErrorLogHandler:
+    """Rate limits logging for fatal engine errors.
+
+    - First fatal error: logged with full traceback.
+    - Subsequent occurences within ``cooldown_s``: suppressed.
+    - Next fatal error after ``cooldown_s``: emits a summary with suppressed errors.
+    - Fatal error after ``2 * cooldown_s`` of quiet: logs full traceback again.
+    - Non-fatal errors: always logged, unaffected by rate limiting.
+    """
+
+    def __init__(self, cooldown_s: float = DEFAULT_FATAL_ERROR_COOLDOWN_S):
+        self._cooldown_s = cooldown_s
+        self._first_logged = False
+        self._suppressed_count = 0
+        self._last_summary_time = 0.0
+        self._lock = threading.Lock()
+
+    def log(
+        self,
+        e: Exception,
+        request_id: str,
+        status_code: int,
+    ) -> None:
+        """Log the error, rate limiting fatal engine errors."""
+        is_fatal = _is_fatal_engine_error(e)
+
+        if not is_fatal:
+            log_fn = logger.error if status_code >= 500 else logger.warning
+            log_fn(
+                f"Encountered failure while handling request {request_id}",
+                exc_info=e,
+                extra={"ray_serve_extra_fields": {"status_code": status_code}},
+            )
+            return
+
+        with self._lock:
+            now = time.monotonic()
+
+            # If enough quiet time has passed, treat this as a new failure
+            # event. The suppressed count is intentionally dropped since the
+            # original fatal error's full traceback was already emitted.
+            if (
+                self._first_logged
+                and (now - self._last_summary_time) >= 2 * self._cooldown_s
+            ):
+                self._first_logged = False
+                self._suppressed_count = 0
+
+            if not self._first_logged:
+                self._first_logged = True
+                self._last_summary_time = now
+                logger.error(
+                    "Encountered failure while handling request %s",
+                    request_id,
+                    exc_info=e,
+                    extra={"ray_serve_extra_fields": {"status_code": status_code}},
+                )
+                return
+
+            self._suppressed_count += 1
+            elapsed = now - self._last_summary_time
+            if elapsed >= self._cooldown_s:
+                logger.error(
+                    "Suppressed %d fatal engine error(s) in the last %.0fs. "
+                    "Engine is dead, awaiting replica restart.",
+                    self._suppressed_count,
+                    elapsed,
+                )
+                self._suppressed_count = 0
+                self._last_summary_time = now
+
+
+_fatal_error_log_handler = _FatalEngineErrorLogHandler()
+
+
+def make_async(_func: Callable[..., T]) -> Callable[..., Awaitable[T]]:
+    """Take a blocking function, and run it on in an executor thread.
+
+    This function prevents the blocking function from blocking the asyncio event loop.
+    The code in this function needs to be thread safe.
+    """
+
+    def _async_wrapper(*args, **kwargs) -> asyncio.Future:
+        loop = asyncio.get_event_loop()
+        func = partial(_func, *args, **kwargs)
+        return loop.run_in_executor(executor=None, func=func)
+
+    return _async_wrapper
+
+
+def extract_message_from_exception(e: Exception) -> str:
+    # If the exception is a Ray exception, we need to dig through the text to get just
+    # the exception message without the stack trace
+    # This also works for normal exceptions (we will just return everything from
+    # format_exception_only in that case)
+    message_lines = traceback.format_exception_only(type(e), e)[-1].strip().split("\n")
+    message = ""
+    # The stack trace lines will be prefixed with spaces, so we need to start from the bottom
+    # and stop at the last line before a line with a space
+    found_last_line_before_stack_trace = False
+    for line in reversed(message_lines):
+        if not line.startswith(" "):
+            found_last_line_before_stack_trace = True
+        if found_last_line_before_stack_trace and line.startswith(" "):
+            break
+        message = line + "\n" + message
+    message = message.strip()
+    return message
+
+
+def _extract_message(e):
+    if isinstance(e, OpenAIHTTPException) and e.internal_message is not None:
+        internal_message = e.internal_message
+    else:
+        internal_message = extract_message_from_exception(e)
+
+    if isinstance(e, HTTPException):
+        message = e.detail
+    elif isinstance(e, OpenAIHTTPException):
+        message = e.message
+    else:
+        message = internal_message
+
+    return internal_message, message
+
+
+def get_response_for_error(
+    e: Exception,
+    request_id: str,
+) -> ErrorResponse:
+    if isinstance(e, HTTPException):
+        status_code = e.status_code
+    elif isinstance(e, OpenAIHTTPException):
+        status_code = e.status_code
+    elif isinstance(e, PydanticValidationError):
+        status_code = 400
+    elif isinstance(e, HTTPXHTTPStatusError):
+        status_code = e.response.status_code
+    else:
+        # Try to get the status code attribute from exception,
+        # if not present, fallback to generic 500
+        status_code = int(
+            getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR)
+        )
+
+    _fatal_error_log_handler.log(e, request_id, status_code)
+
+    if status_code == status.HTTP_500_INTERNAL_SERVER_ERROR:
+        internal_message = message = "Internal Server Error"
+        exc_type = "InternalServerError"
+    else:
+        internal_message, message = _extract_message(e)
+        exc_type = e.__class__.__name__
+
+    # TODO make this more robust
+    if "(Request ID: " not in message:
+        message += f" (Request ID: {request_id})"
+
+    if "(Request ID: " not in internal_message:
+        internal_message += f" (Request ID: {request_id})"
+
+    error_info = ErrorInfo(
+        message=f"Message: {message}, Internal exception: {internal_message}, original exception: {str(e)}",
+        code=status_code,
+        type=exc_type,
+    )
+    error_response = ErrorResponse(error=error_info)
+    return error_response
+
+
+def get_serve_request_id() -> str:
+    """Get request id from serve request context."""
+    context = serve.context._serve_request_context.get()
+    if context is not None:
+        return context.request_id
+    return ""
+
+
+def get_model_request_id(model: str):
+    return model + "-" + get_serve_request_id()
+
+
+def replace_prefix(model: str) -> str:
+    """Replace -- with / in model name to handle slashes within the URL path segment"""
+    return model.replace("--", "/")