wehub-resource-sync
219 lines
7.4 KiB
Python
219 lines
7.4 KiB
Python
import asyncio
|
|
import threading
|
|
import time
|
|
import traceback
|
|
from functools import partial
|
|
from typing import Awaitable, Callable, TypeVar
|
|
|
|
from fastapi import HTTPException, status
|
|
from httpx import HTTPStatusError as HTTPXHTTPStatusError
|
|
from pydantic import ValidationError as PydanticValidationError
|
|
|
|
from ray import serve
|
|
from ray.llm._internal.common.errors import VLLM_FATAL_ERRORS
|
|
from ray.llm._internal.serve.constants import DEFAULT_FATAL_ERROR_COOLDOWN_S
|
|
from ray.llm._internal.serve.core.configs.openai_api_models import (
|
|
ErrorInfo,
|
|
ErrorResponse,
|
|
OpenAIHTTPException,
|
|
)
|
|
from ray.llm._internal.serve.observability.logging import get_logger
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
T = TypeVar("T")
|
|
|
|
|
|
def _is_fatal_engine_error(e: Exception) -> bool:
|
|
"""Detect fatal engine errors via isinstance check."""
|
|
if not VLLM_FATAL_ERRORS:
|
|
return False
|
|
return isinstance(e, VLLM_FATAL_ERRORS)
|
|
|
|
|
|
class _FatalEngineErrorLogHandler:
|
|
"""Rate limits logging for fatal engine errors.
|
|
|
|
- First fatal error: logged with full traceback.
|
|
- Subsequent occurences within ``cooldown_s``: suppressed.
|
|
- Next fatal error after ``cooldown_s``: emits a summary with suppressed errors.
|
|
- Fatal error after ``2 * cooldown_s`` of quiet: logs full traceback again.
|
|
- Non-fatal errors: always logged, unaffected by rate limiting.
|
|
"""
|
|
|
|
def __init__(self, cooldown_s: float = DEFAULT_FATAL_ERROR_COOLDOWN_S):
|
|
self._cooldown_s = cooldown_s
|
|
self._first_logged = False
|
|
self._suppressed_count = 0
|
|
self._last_summary_time = 0.0
|
|
self._lock = threading.Lock()
|
|
|
|
def log(
|
|
self,
|
|
e: Exception,
|
|
request_id: str,
|
|
status_code: int,
|
|
) -> None:
|
|
"""Log the error, rate limiting fatal engine errors."""
|
|
is_fatal = _is_fatal_engine_error(e)
|
|
|
|
if not is_fatal:
|
|
log_fn = logger.error if status_code >= 500 else logger.warning
|
|
log_fn(
|
|
f"Encountered failure while handling request {request_id}",
|
|
exc_info=e,
|
|
extra={"ray_serve_extra_fields": {"status_code": status_code}},
|
|
)
|
|
return
|
|
|
|
with self._lock:
|
|
now = time.monotonic()
|
|
|
|
# If enough quiet time has passed, treat this as a new failure
|
|
# event. The suppressed count is intentionally dropped since the
|
|
# original fatal error's full traceback was already emitted.
|
|
if (
|
|
self._first_logged
|
|
and (now - self._last_summary_time) >= 2 * self._cooldown_s
|
|
):
|
|
self._first_logged = False
|
|
self._suppressed_count = 0
|
|
|
|
if not self._first_logged:
|
|
self._first_logged = True
|
|
self._last_summary_time = now
|
|
logger.error(
|
|
"Encountered failure while handling request %s",
|
|
request_id,
|
|
exc_info=e,
|
|
extra={"ray_serve_extra_fields": {"status_code": status_code}},
|
|
)
|
|
return
|
|
|
|
self._suppressed_count += 1
|
|
elapsed = now - self._last_summary_time
|
|
if elapsed >= self._cooldown_s:
|
|
logger.error(
|
|
"Suppressed %d fatal engine error(s) in the last %.0fs. "
|
|
"Engine is dead, awaiting replica restart.",
|
|
self._suppressed_count,
|
|
elapsed,
|
|
)
|
|
self._suppressed_count = 0
|
|
self._last_summary_time = now
|
|
|
|
|
|
_fatal_error_log_handler = _FatalEngineErrorLogHandler()
|
|
|
|
|
|
def make_async(_func: Callable[..., T]) -> Callable[..., Awaitable[T]]:
|
|
"""Take a blocking function, and run it on in an executor thread.
|
|
|
|
This function prevents the blocking function from blocking the asyncio event loop.
|
|
The code in this function needs to be thread safe.
|
|
"""
|
|
|
|
def _async_wrapper(*args, **kwargs) -> asyncio.Future:
|
|
loop = asyncio.get_event_loop()
|
|
func = partial(_func, *args, **kwargs)
|
|
return loop.run_in_executor(executor=None, func=func)
|
|
|
|
return _async_wrapper
|
|
|
|
|
|
def extract_message_from_exception(e: Exception) -> str:
|
|
# If the exception is a Ray exception, we need to dig through the text to get just
|
|
# the exception message without the stack trace
|
|
# This also works for normal exceptions (we will just return everything from
|
|
# format_exception_only in that case)
|
|
message_lines = traceback.format_exception_only(type(e), e)[-1].strip().split("\n")
|
|
message = ""
|
|
# The stack trace lines will be prefixed with spaces, so we need to start from the bottom
|
|
# and stop at the last line before a line with a space
|
|
found_last_line_before_stack_trace = False
|
|
for line in reversed(message_lines):
|
|
if not line.startswith(" "):
|
|
found_last_line_before_stack_trace = True
|
|
if found_last_line_before_stack_trace and line.startswith(" "):
|
|
break
|
|
message = line + "\n" + message
|
|
message = message.strip()
|
|
return message
|
|
|
|
|
|
def _extract_message(e):
|
|
if isinstance(e, OpenAIHTTPException) and e.internal_message is not None:
|
|
internal_message = e.internal_message
|
|
else:
|
|
internal_message = extract_message_from_exception(e)
|
|
|
|
if isinstance(e, HTTPException):
|
|
message = e.detail
|
|
elif isinstance(e, OpenAIHTTPException):
|
|
message = e.message
|
|
else:
|
|
message = internal_message
|
|
|
|
return internal_message, message
|
|
|
|
|
|
def get_response_for_error(
|
|
e: Exception,
|
|
request_id: str,
|
|
) -> ErrorResponse:
|
|
if isinstance(e, HTTPException):
|
|
status_code = e.status_code
|
|
elif isinstance(e, OpenAIHTTPException):
|
|
status_code = e.status_code
|
|
elif isinstance(e, PydanticValidationError):
|
|
status_code = 400
|
|
elif isinstance(e, HTTPXHTTPStatusError):
|
|
status_code = e.response.status_code
|
|
else:
|
|
# Try to get the status code attribute from exception,
|
|
# if not present, fallback to generic 500
|
|
status_code = int(
|
|
getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR)
|
|
)
|
|
|
|
_fatal_error_log_handler.log(e, request_id, status_code)
|
|
|
|
if status_code == status.HTTP_500_INTERNAL_SERVER_ERROR:
|
|
internal_message = message = "Internal Server Error"
|
|
exc_type = "InternalServerError"
|
|
else:
|
|
internal_message, message = _extract_message(e)
|
|
exc_type = e.__class__.__name__
|
|
|
|
# TODO make this more robust
|
|
if "(Request ID: " not in message:
|
|
message += f" (Request ID: {request_id})"
|
|
|
|
if "(Request ID: " not in internal_message:
|
|
internal_message += f" (Request ID: {request_id})"
|
|
|
|
error_info = ErrorInfo(
|
|
message=f"Message: {message}, Internal exception: {internal_message}, original exception: {str(e)}",
|
|
code=status_code,
|
|
type=exc_type,
|
|
)
|
|
error_response = ErrorResponse(error=error_info)
|
|
return error_response
|
|
|
|
|
|
def get_serve_request_id() -> str:
|
|
"""Get request id from serve request context."""
|
|
context = serve.context._serve_request_context.get()
|
|
if context is not None:
|
|
return context.request_id
|
|
return ""
|
|
|
|
|
|
def get_model_request_id(model: str):
|
|
return model + "-" + get_serve_request_id()
|
|
|
|
|
|
def replace_prefix(model: str) -> str:
|
|
"""Replace -- with / in model name to handle slashes within the URL path segment"""
|
|
return model.replace("--", "/")
|