wehub-resource-sync
e4dcfc49aa
Tests / Import Check (Python 3.13) (push) Has been cancelled
Tests / Import Check (Python 3.14) (push) Has been cancelled
Tests / Python Tests (Python 3.11) (push) Has been cancelled
Tests / Python Tests (Python 3.12) (push) Has been cancelled
Tests / Python Tests (Python 3.14) (push) Has been cancelled
Tests / Test Summary (push) Has been cancelled
Tests / Lint and Format (push) Has been cancelled
Tests / Web Node Tests (push) Has been cancelled
Tests / Import Check (Python 3.11) (push) Has been cancelled
Tests / Import Check (Python 3.12) (push) Has been cancelled
Tests / Python Tests (Python 3.13) (push) Has been cancelled
279 lines
8.8 KiB
Python
279 lines
8.8 KiB
Python
"""Provider-backed LLM executors (openai + anthropic SDKs, no litellm)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import AsyncGenerator
|
|
import logging
|
|
import os
|
|
from typing import Any
|
|
import uuid
|
|
|
|
from openai import AsyncOpenAI, BadRequestError
|
|
|
|
from deeptutor.services.llm.capabilities import disable_response_format_at_runtime
|
|
from deeptutor.services.llm.openai_http_client import openai_client_kwargs
|
|
from deeptutor.services.llm.provider_registry import find_by_name, strip_provider_prefix
|
|
from deeptutor.services.llm.reasoning_params import default_reasoning_effort_for
|
|
|
|
from .config import get_token_limit_kwargs
|
|
from .utils import extract_response_content
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _is_unsupported_response_format_error(exc: BaseException) -> bool:
|
|
"""Detect whether a BadRequestError stems from an unsupported ``response_format``.
|
|
|
|
Examples seen in the wild:
|
|
- LM Studio + Gemma: ``"'response_format.type' must be 'json_schema' or 'text'"``
|
|
- DashScope + various models: ``"'response_format.type' specified ... not valid: 'json_object' is not supported by this model"``
|
|
"""
|
|
text = str(exc).lower()
|
|
if "response_format" not in text and "response format" not in text:
|
|
return False
|
|
return (
|
|
"json_object" in text
|
|
or "json_schema" in text
|
|
or "not supported" in text
|
|
or "not valid" in text
|
|
or "must be" in text
|
|
)
|
|
|
|
|
|
async def _create_with_format_fallback(
|
|
client: AsyncOpenAI,
|
|
payload: dict[str, Any],
|
|
*,
|
|
binding: str,
|
|
model: str,
|
|
) -> Any:
|
|
"""Run ``client.chat.completions.create`` with auto-fallback on response_format errors.
|
|
|
|
Some local servers (LM Studio + Gemma/Qwen) reject ``response_format``
|
|
with HTTP 400. On a matching :class:`BadRequestError`, drop the offending
|
|
field and retry once, then cache the (binding, model) pair so future calls
|
|
skip ``response_format`` upfront.
|
|
"""
|
|
try:
|
|
return await client.chat.completions.create(**payload)
|
|
except BadRequestError as exc:
|
|
if "response_format" not in payload or not _is_unsupported_response_format_error(exc):
|
|
raise
|
|
logger.warning(
|
|
f"Provider {binding} rejected response_format for model {model} ({exc}); "
|
|
"retrying without it and disabling response_format for this binding+model."
|
|
)
|
|
disable_response_format_at_runtime(binding, model)
|
|
retry_payload = dict(payload)
|
|
retry_payload.pop("response_format", None)
|
|
return await client.chat.completions.create(**retry_payload)
|
|
|
|
|
|
def _build_messages(
|
|
*,
|
|
prompt: str,
|
|
system_prompt: str,
|
|
messages: list[dict[str, Any]] | None,
|
|
) -> list[dict[str, Any]]:
|
|
if messages:
|
|
return messages
|
|
return [
|
|
{"role": "system", "content": system_prompt},
|
|
{"role": "user", "content": prompt},
|
|
]
|
|
|
|
|
|
def _setup_provider_env(provider_name: str, api_key: str | None, api_base: str | None) -> None:
|
|
spec = find_by_name(provider_name)
|
|
if not spec or not api_key:
|
|
return
|
|
if spec.env_key:
|
|
os.environ.setdefault(spec.env_key, api_key)
|
|
effective_base = api_base or spec.default_api_base
|
|
for env_name, env_val in spec.env_extras:
|
|
resolved = env_val.replace("{api_key}", api_key).replace("{api_base}", effective_base or "")
|
|
os.environ.setdefault(env_name, resolved)
|
|
|
|
|
|
def _resolve_model_and_base(
|
|
provider_name: str,
|
|
model: str,
|
|
api_key: str | None,
|
|
base_url: str | None,
|
|
) -> tuple[str, str | None, str | None]:
|
|
"""Resolve the actual model name, base_url, and api_key for the provider.
|
|
|
|
Returns (resolved_model, effective_base_url, effective_api_key).
|
|
"""
|
|
spec = find_by_name(provider_name)
|
|
resolved_model = strip_provider_prefix(model, spec) if spec else model
|
|
effective_base = base_url or (spec.default_api_base if spec else None) or None
|
|
effective_key = api_key
|
|
return resolved_model, effective_base, effective_key
|
|
|
|
|
|
def _coerce_int(value: Any, default: int) -> int:
|
|
try:
|
|
return int(value)
|
|
except (TypeError, ValueError):
|
|
return default
|
|
|
|
|
|
def _coerce_float(value: Any, default: float) -> float:
|
|
try:
|
|
return float(value)
|
|
except (TypeError, ValueError):
|
|
return default
|
|
|
|
|
|
async def sdk_complete(
|
|
*,
|
|
prompt: str,
|
|
system_prompt: str,
|
|
provider_name: str,
|
|
model: str,
|
|
api_key: str | None,
|
|
base_url: str | None,
|
|
messages: list[dict[str, Any]] | None = None,
|
|
api_version: str | None = None,
|
|
extra_headers: dict[str, str] | None = None,
|
|
reasoning_effort: str | None = None,
|
|
**kwargs: Any,
|
|
) -> str:
|
|
"""Non-streaming completion using the openai SDK."""
|
|
_setup_provider_env(provider_name, api_key, base_url)
|
|
resolved_model, effective_base, effective_key = _resolve_model_and_base(
|
|
provider_name,
|
|
model,
|
|
api_key,
|
|
base_url,
|
|
)
|
|
|
|
default_headers: dict[str, str] = {"x-session-affinity": uuid.uuid4().hex}
|
|
if extra_headers:
|
|
default_headers.update(extra_headers)
|
|
|
|
client = AsyncOpenAI(
|
|
api_key=effective_key or "no-key",
|
|
base_url=effective_base,
|
|
default_headers=default_headers,
|
|
max_retries=0,
|
|
**openai_client_kwargs(),
|
|
)
|
|
|
|
max_tokens_val = _coerce_int(kwargs.pop("max_tokens", 4096), 4096)
|
|
temperature_val = _coerce_float(kwargs.pop("temperature", 0.7), 0.7)
|
|
|
|
payload: dict[str, Any] = {
|
|
"model": resolved_model,
|
|
"messages": _build_messages(
|
|
prompt=prompt,
|
|
system_prompt=system_prompt,
|
|
messages=messages,
|
|
),
|
|
"temperature": temperature_val,
|
|
}
|
|
|
|
token_kwargs = get_token_limit_kwargs(resolved_model, max_tokens_val)
|
|
payload.update(token_kwargs)
|
|
|
|
effective_effort = reasoning_effort or default_reasoning_effort_for(
|
|
provider_name, resolved_model
|
|
)
|
|
if effective_effort:
|
|
payload["reasoning_effort"] = effective_effort
|
|
payload.update(kwargs)
|
|
|
|
response = await _create_with_format_fallback(
|
|
client, payload, binding=provider_name or "openai", model=resolved_model
|
|
)
|
|
choices = getattr(response, "choices", None) or []
|
|
if not choices:
|
|
return ""
|
|
message = getattr(choices[0], "message", None)
|
|
if message is None and isinstance(choices[0], dict):
|
|
message = choices[0].get("message")
|
|
return extract_response_content(message)
|
|
|
|
|
|
async def sdk_stream(
|
|
*,
|
|
prompt: str,
|
|
system_prompt: str,
|
|
provider_name: str,
|
|
model: str,
|
|
api_key: str | None,
|
|
base_url: str | None,
|
|
messages: list[dict[str, Any]] | None = None,
|
|
api_version: str | None = None,
|
|
extra_headers: dict[str, str] | None = None,
|
|
reasoning_effort: str | None = None,
|
|
**kwargs: Any,
|
|
) -> AsyncGenerator[str, None]:
|
|
"""Streaming completion using the openai SDK."""
|
|
_setup_provider_env(provider_name, api_key, base_url)
|
|
resolved_model, effective_base, effective_key = _resolve_model_and_base(
|
|
provider_name,
|
|
model,
|
|
api_key,
|
|
base_url,
|
|
)
|
|
|
|
default_headers: dict[str, str] = {"x-session-affinity": uuid.uuid4().hex}
|
|
if extra_headers:
|
|
default_headers.update(extra_headers)
|
|
|
|
client = AsyncOpenAI(
|
|
api_key=effective_key or "no-key",
|
|
base_url=effective_base,
|
|
default_headers=default_headers,
|
|
max_retries=0,
|
|
**openai_client_kwargs(),
|
|
)
|
|
|
|
max_tokens_val = _coerce_int(kwargs.pop("max_tokens", 4096), 4096)
|
|
temperature_val = _coerce_float(kwargs.pop("temperature", 0.7), 0.7)
|
|
|
|
payload: dict[str, Any] = {
|
|
"model": resolved_model,
|
|
"messages": _build_messages(
|
|
prompt=prompt,
|
|
system_prompt=system_prompt,
|
|
messages=messages,
|
|
),
|
|
"temperature": temperature_val,
|
|
"stream": True,
|
|
}
|
|
|
|
token_kwargs = get_token_limit_kwargs(resolved_model, max_tokens_val)
|
|
payload.update(token_kwargs)
|
|
|
|
effective_effort = reasoning_effort or default_reasoning_effort_for(
|
|
provider_name, resolved_model
|
|
)
|
|
if effective_effort:
|
|
payload["reasoning_effort"] = effective_effort
|
|
payload.update(kwargs)
|
|
|
|
stream_response = await _create_with_format_fallback(
|
|
client, payload, binding=provider_name or "openai", model=resolved_model
|
|
)
|
|
async for chunk in stream_response:
|
|
choices = getattr(chunk, "choices", None) or []
|
|
if not choices:
|
|
continue
|
|
choice = choices[0]
|
|
delta = getattr(choice, "delta", None)
|
|
if delta is None and isinstance(choice, dict):
|
|
delta = choice.get("delta")
|
|
if delta is None:
|
|
continue
|
|
raw_content = (
|
|
getattr(delta, "content", None) if not isinstance(delta, dict) else delta.get("content")
|
|
)
|
|
if raw_content is None:
|
|
continue
|
|
content = extract_response_content(delta)
|
|
if content:
|
|
yield content
|