wehub-resource-sync
129 lines
5.5 KiB
Python
129 lines
5.5 KiB
Python
import json
|
|
import os
|
|
import time
|
|
from typing import Any
|
|
|
|
from bfcl_eval.model_handler.api_inference.openai_completion import OpenAICompletionsHandler
|
|
from bfcl_eval.constants.enums import ModelStyle
|
|
from bfcl_eval.model_handler.utils import (
|
|
combine_consecutive_user_prompts,
|
|
retry_with_backoff,
|
|
system_prompt_pre_processing_chat_model,
|
|
)
|
|
from openai import OpenAI, RateLimitError
|
|
from overrides import override
|
|
|
|
|
|
|
|
class DeepSeekAPIHandler(OpenAICompletionsHandler):
|
|
def __init__(
|
|
self,
|
|
model_name,
|
|
temperature,
|
|
registry_name,
|
|
is_fc_model,
|
|
**kwargs,
|
|
) -> None:
|
|
super().__init__(model_name, temperature, registry_name, is_fc_model, **kwargs)
|
|
self.model_style = ModelStyle.OPENAI_COMPLETIONS
|
|
base = "https://api.deepseek.com"
|
|
|
|
self.client = OpenAI(
|
|
base_url=base,
|
|
api_key=os.getenv("DEEPSEEK_API_KEY"),
|
|
)
|
|
|
|
# The deepseek API is unstable at the moment, and will frequently give empty responses, so retry on JSONDecodeError is necessary
|
|
@retry_with_backoff(error_type=[RateLimitError, json.JSONDecodeError], error_message_pattern=r".*Insufficient Balance.*")
|
|
def generate_with_backoff(self, **kwargs):
|
|
"""
|
|
Per the DeepSeek API documentation:
|
|
https://api-docs.deepseek.com/quick_start/rate_limit
|
|
|
|
DeepSeek API does NOT constrain user's rate limit. We will try out best to serve every request.
|
|
But please note that when our servers are under high traffic pressure, you may receive 429 (Rate Limit Reached) or 503 (Server Overloaded). When this happens, please wait for a while and retry.
|
|
|
|
Thus, backoff is still useful for handling 429 and 503 errors.
|
|
"""
|
|
start_time = time.time()
|
|
api_response = self.client.chat.completions.create(**kwargs)
|
|
end_time = time.time()
|
|
|
|
return api_response, end_time - start_time
|
|
|
|
@override
|
|
def _query_FC(self, inference_data: dict):
|
|
message: list[dict] = inference_data["message"]
|
|
tools = inference_data["tools"]
|
|
inference_data["inference_input_log"] = {"message": repr(message), "tools": tools}
|
|
|
|
if len(tools) > 0:
|
|
return self.generate_with_backoff(
|
|
model=self.model_name,
|
|
messages=message,
|
|
tools=tools,
|
|
temperature=self.temperature,
|
|
)
|
|
else:
|
|
return self.generate_with_backoff(
|
|
model=self.model_name,
|
|
messages=message,
|
|
temperature=self.temperature,
|
|
)
|
|
|
|
@override
|
|
def _query_prompting(self, inference_data: dict):
|
|
"""
|
|
This method is intended to be used by the `DeepSeek-R1` models. If used for other models, you will need to modify the code accordingly.
|
|
|
|
Reasoning models don't support temperature parameter
|
|
https://api-docs.deepseek.com/guides/reasoning_model
|
|
|
|
`DeepSeek-R1` should use `deepseek-reasoner` as the model name in the API
|
|
https://api-docs.deepseek.com/quick_start/pricing
|
|
"""
|
|
message: list[dict] = inference_data["message"]
|
|
inference_data["inference_input_log"] = {"message": repr(message)}
|
|
|
|
return self.generate_with_backoff(
|
|
model=self.model_name,
|
|
messages=message,
|
|
)
|
|
|
|
@override
|
|
def _pre_query_processing_prompting(self, test_entry: dict) -> dict:
|
|
functions: list = test_entry["function"]
|
|
test_entry_id: str = test_entry["id"]
|
|
|
|
test_entry["question"][0] = system_prompt_pre_processing_chat_model(
|
|
test_entry["question"][0], functions, test_entry_id
|
|
)
|
|
|
|
# 'deepseek-reasoner does not support successive user messages, so we need to combine them
|
|
for round_idx in range(len(test_entry["question"])):
|
|
test_entry["question"][round_idx] = combine_consecutive_user_prompts(
|
|
test_entry["question"][round_idx]
|
|
)
|
|
|
|
return {"message": []}
|
|
|
|
@override
|
|
def _parse_query_response_prompting(self, api_response: Any) -> dict:
|
|
"""
|
|
DeepSeek does not take reasoning content in next turn chat history, for both prompting and function calling mode.
|
|
Error: Error code: 400 - {'error': {'message': 'The reasoning_content is an intermediate result for display purposes only and will not be included in the context for inference. Please remove the reasoning_content from your message to reduce network traffic.', 'type': 'invalid_request_error', 'param': None, 'code': 'invalid_request_error'}}
|
|
"""
|
|
response_data = super()._parse_query_response_prompting(api_response)
|
|
self._add_reasoning_content_if_available_prompting(api_response, response_data)
|
|
return response_data
|
|
|
|
@override
|
|
def _parse_query_response_FC(self, api_response: Any) -> dict:
|
|
"""
|
|
DeepSeek does not take reasoning content in next turn chat history, for both prompting and function calling mode.
|
|
Error: Error code: 400 - {'error': {'message': 'The reasoning_content is an intermediate result for display purposes only and will not be included in the context for inference. Please remove the reasoning_content from your message to reduce network traffic.', 'type': 'invalid_request_error', 'param': None, 'code': 'invalid_request_error'}}
|
|
"""
|
|
response_data = super()._parse_query_response_FC(api_response)
|
|
self._add_reasoning_content_if_available_FC(api_response, response_data)
|
|
return response_data
|