chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,618 @@
|
||||
#include "conv_template.h"
|
||||
|
||||
#include <tvm/ffi/function.h>
|
||||
|
||||
#include "../support/json_parser.h"
|
||||
#include "image_utils.h"
|
||||
|
||||
namespace mlc {
|
||||
namespace llm {
|
||||
namespace json_ffi {
|
||||
|
||||
using namespace mlc::llm;
|
||||
|
||||
/****************** Model vision config ******************/
|
||||
|
||||
ModelVisionConfig ModelVisionConfig::FromJSON(const tvm::ffi::json::Object& json_obj) {
|
||||
ModelVisionConfig config;
|
||||
|
||||
Result<int64_t> hidden_size_res = json::LookupWithResultReturn<int64_t>(json_obj, "hidden_size");
|
||||
if (hidden_size_res.IsOk()) {
|
||||
config.hidden_size = static_cast<int>(hidden_size_res.Unwrap());
|
||||
}
|
||||
|
||||
Result<int64_t> image_size_res = json::LookupWithResultReturn<int64_t>(json_obj, "image_size");
|
||||
if (image_size_res.IsOk()) {
|
||||
config.image_size = static_cast<int>(image_size_res.Unwrap());
|
||||
}
|
||||
|
||||
Result<int64_t> intermediate_size_res =
|
||||
json::LookupWithResultReturn<int64_t>(json_obj, "intermediate_size");
|
||||
if (intermediate_size_res.IsOk()) {
|
||||
config.intermediate_size = static_cast<int>(intermediate_size_res.Unwrap());
|
||||
}
|
||||
|
||||
Result<int64_t> num_attention_heads_res =
|
||||
json::LookupWithResultReturn<int64_t>(json_obj, "num_attention_heads");
|
||||
if (num_attention_heads_res.IsOk()) {
|
||||
config.num_attention_heads = static_cast<int>(num_attention_heads_res.Unwrap());
|
||||
}
|
||||
|
||||
Result<int64_t> num_hidden_layers_res =
|
||||
json::LookupWithResultReturn<int64_t>(json_obj, "num_hidden_layers");
|
||||
if (num_hidden_layers_res.IsOk()) {
|
||||
config.num_hidden_layers = static_cast<int>(num_hidden_layers_res.Unwrap());
|
||||
}
|
||||
|
||||
Result<int64_t> patch_size_res = json::LookupWithResultReturn<int64_t>(json_obj, "patch_size");
|
||||
if (patch_size_res.IsOk()) {
|
||||
config.patch_size = static_cast<int>(patch_size_res.Unwrap());
|
||||
}
|
||||
|
||||
Result<int64_t> projection_dim_res =
|
||||
json::LookupWithResultReturn<int64_t>(json_obj, "projection_dim");
|
||||
if (projection_dim_res.IsOk()) {
|
||||
config.projection_dim = static_cast<int>(projection_dim_res.Unwrap());
|
||||
}
|
||||
|
||||
Result<int64_t> vocab_size_res = json::LookupWithResultReturn<int64_t>(json_obj, "vocab_size");
|
||||
if (vocab_size_res.IsOk()) {
|
||||
config.vocab_size = static_cast<int>(vocab_size_res.Unwrap());
|
||||
}
|
||||
|
||||
Result<std::string> dtype_res = json::LookupWithResultReturn<std::string>(json_obj, "dtype");
|
||||
if (dtype_res.IsOk()) {
|
||||
config.dtype = dtype_res.Unwrap();
|
||||
}
|
||||
|
||||
Result<int64_t> num_channels_res =
|
||||
json::LookupWithResultReturn<int64_t>(json_obj, "num_channels");
|
||||
if (num_channels_res.IsOk()) {
|
||||
config.num_channels = static_cast<int>(num_channels_res.Unwrap());
|
||||
}
|
||||
|
||||
Result<double> layer_norm_eps_res =
|
||||
json::LookupWithResultReturn<double>(json_obj, "layer_norm_eps");
|
||||
if (layer_norm_eps_res.IsOk()) {
|
||||
config.layer_norm_eps = layer_norm_eps_res.Unwrap();
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
/****************** Model config ******************/
|
||||
|
||||
ModelConfig ModelConfig::FromJSON(const tvm::ffi::json::Object& json_obj) {
|
||||
ModelConfig config;
|
||||
|
||||
Result<int64_t> vocab_size_res = json::LookupWithResultReturn<int64_t>(json_obj, "vocab_size");
|
||||
if (vocab_size_res.IsOk()) {
|
||||
config.vocab_size = static_cast<int>(vocab_size_res.Unwrap());
|
||||
}
|
||||
|
||||
Result<int64_t> context_window_size_res =
|
||||
json::LookupWithResultReturn<int64_t>(json_obj, "context_window_size");
|
||||
if (context_window_size_res.IsOk()) {
|
||||
config.context_window_size = static_cast<int>(context_window_size_res.Unwrap());
|
||||
}
|
||||
|
||||
Result<int64_t> sliding_window_size_res =
|
||||
json::LookupWithResultReturn<int64_t>(json_obj, "sliding_window_size");
|
||||
if (sliding_window_size_res.IsOk()) {
|
||||
config.sliding_window_size = static_cast<int>(sliding_window_size_res.Unwrap());
|
||||
}
|
||||
|
||||
Result<int64_t> prefill_chunk_size_res =
|
||||
json::LookupWithResultReturn<int64_t>(json_obj, "prefill_chunk_size");
|
||||
if (prefill_chunk_size_res.IsOk()) {
|
||||
config.prefill_chunk_size = static_cast<int>(prefill_chunk_size_res.Unwrap());
|
||||
}
|
||||
|
||||
Result<int64_t> tensor_parallel_shards_res =
|
||||
json::LookupWithResultReturn<int64_t>(json_obj, "tensor_parallel_shards");
|
||||
if (tensor_parallel_shards_res.IsOk()) {
|
||||
config.tensor_parallel_shards = static_cast<int>(tensor_parallel_shards_res.Unwrap());
|
||||
}
|
||||
|
||||
Result<int64_t> pipeline_parallel_stages_res =
|
||||
json::LookupWithResultReturn<int64_t>(json_obj, "pipeline_parallel_stages");
|
||||
if (pipeline_parallel_stages_res.IsOk()) {
|
||||
config.pipeline_parallel_stages = static_cast<int>(pipeline_parallel_stages_res.Unwrap());
|
||||
}
|
||||
|
||||
Result<int64_t> max_batch_size_res =
|
||||
json::LookupWithResultReturn<int64_t>(json_obj, "max_batch_size");
|
||||
if (max_batch_size_res.IsOk()) {
|
||||
config.max_batch_size = static_cast<int>(max_batch_size_res.Unwrap());
|
||||
}
|
||||
|
||||
if (json_obj.count("vision_config")) {
|
||||
const tvm::ffi::json::Object& vision_config_obj =
|
||||
json_obj.at("vision_config").cast<tvm::ffi::json::Object>();
|
||||
config.vision_config = ModelVisionConfig::FromJSON(vision_config_obj);
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
/****************** Conversation template ******************/
|
||||
|
||||
std::unordered_map<MessagePlaceholders, std::string> PLACEHOLDERS = {
|
||||
{MessagePlaceholders::SYSTEM, "{system_message}"},
|
||||
{MessagePlaceholders::USER, "{user_message}"},
|
||||
{MessagePlaceholders::ASSISTANT, "{assistant_message}"},
|
||||
{MessagePlaceholders::TOOL, "{tool_message}"},
|
||||
{MessagePlaceholders::FUNCTION, "{function_string}"}};
|
||||
|
||||
MessagePlaceholders MessagePlaceholderFromString(const std::string& role) {
|
||||
static const std::unordered_map<std::string, MessagePlaceholders> enum_map = {
|
||||
{"system", MessagePlaceholders::SYSTEM}, {"user", MessagePlaceholders::USER},
|
||||
{"assistant", MessagePlaceholders::ASSISTANT}, {"tool", MessagePlaceholders::TOOL},
|
||||
{"function", MessagePlaceholders::FUNCTION},
|
||||
};
|
||||
|
||||
return enum_map.at(role);
|
||||
}
|
||||
|
||||
Conversation::Conversation()
|
||||
: role_templates({{"user", PLACEHOLDERS[MessagePlaceholders::USER]},
|
||||
{"assistant", PLACEHOLDERS[MessagePlaceholders::ASSISTANT]},
|
||||
{"tool", PLACEHOLDERS[MessagePlaceholders::TOOL]}}) {}
|
||||
|
||||
std::string Conversation::GetSystemText(const std::string& system_msg) const {
|
||||
std::string system_text = this->system_template;
|
||||
static std::string system_placeholder = PLACEHOLDERS[MessagePlaceholders::SYSTEM];
|
||||
size_t pos = system_text.find(system_placeholder);
|
||||
if (pos != std::string::npos) {
|
||||
system_text.replace(pos, system_placeholder.length(), system_msg);
|
||||
}
|
||||
return system_text;
|
||||
}
|
||||
|
||||
std::string Conversation::GetRoleText(const std::string& role, const std::string& content,
|
||||
const std::optional<std::string>& fn_call_string) const {
|
||||
std::string role_text = this->role_templates.at(role);
|
||||
std::string placeholder = PLACEHOLDERS[MessagePlaceholderFromString(role)];
|
||||
size_t pos = role_text.find(placeholder);
|
||||
if (pos != std::string::npos) {
|
||||
role_text.replace(pos, placeholder.length(), content);
|
||||
}
|
||||
if (fn_call_string) {
|
||||
// replace placeholder[FUNCTION] with function_string
|
||||
// this assumes function calling is used for a single request scenario only
|
||||
pos = role_text.find(PLACEHOLDERS[MessagePlaceholders::FUNCTION]);
|
||||
if (pos != std::string::npos) {
|
||||
role_text.replace(pos, PLACEHOLDERS[MessagePlaceholders::FUNCTION].length(),
|
||||
fn_call_string.value());
|
||||
}
|
||||
}
|
||||
return role_text;
|
||||
}
|
||||
|
||||
/// Try to detect if function calling is needed, if so, return the function calling string
|
||||
Result<std::optional<std::string>> TryGetFunctionCallingString(
|
||||
const Conversation& conv, const ChatCompletionRequest& request) {
|
||||
using TResult = Result<std::optional<std::string>>;
|
||||
if (!request.tools.has_value() ||
|
||||
(request.tool_choice.has_value() && request.tool_choice.value() == "none")) {
|
||||
return TResult::Ok(std::nullopt);
|
||||
}
|
||||
std::vector<ChatTool> tools_ = request.tools.value();
|
||||
std::string tool_choice_ = request.tool_choice.value();
|
||||
|
||||
// TODO: support with tool choice as dict
|
||||
for (const auto& tool : tools_) {
|
||||
if (tool.function.name == tool_choice_) {
|
||||
tvm::ffi::json::Value function_str(tool.function.AsJSON());
|
||||
return TResult::Ok(tvm::ffi::json::Stringify(function_str));
|
||||
}
|
||||
}
|
||||
|
||||
if (tool_choice_ != "auto") {
|
||||
return TResult::Error("Invalid tool_choice value in the request: " + tool_choice_);
|
||||
}
|
||||
|
||||
tvm::ffi::json::Array function_list;
|
||||
for (const auto& tool : tools_) {
|
||||
function_list.push_back(tool.function.AsJSON());
|
||||
}
|
||||
|
||||
tvm::ffi::json::Value function_list_json(function_list);
|
||||
return TResult::Ok(tvm::ffi::json::Stringify(function_list_json));
|
||||
};
|
||||
|
||||
Result<std::vector<Data>> CreatePrompt(const Conversation& conv,
|
||||
const ChatCompletionRequest& request,
|
||||
const ModelConfig& config, DLDevice device) {
|
||||
using TResult = Result<std::vector<Data>>;
|
||||
|
||||
Result<std::optional<std::string>> fn_call_str_tmp = TryGetFunctionCallingString(conv, request);
|
||||
if (fn_call_str_tmp.IsErr()) {
|
||||
return TResult::Error(fn_call_str_tmp.UnwrapErr());
|
||||
}
|
||||
std::optional<std::string> fn_call_string = fn_call_str_tmp.Unwrap();
|
||||
|
||||
// Handle system message
|
||||
// concz
|
||||
bool has_custom_system = false;
|
||||
std::string custom_system_inputs;
|
||||
|
||||
auto f_populate_system_message = [&](const std::vector<ChatCompletionMessage>& msg_vec) {
|
||||
for (ChatCompletionMessage msg : msg_vec) {
|
||||
if (msg.role == "system") {
|
||||
TVM_FFI_ICHECK(msg.content.IsText()) << "System message must be text";
|
||||
custom_system_inputs += msg.content.Text();
|
||||
has_custom_system = true;
|
||||
}
|
||||
}
|
||||
};
|
||||
// go through messages in template and passed in.
|
||||
f_populate_system_message(conv.messages);
|
||||
f_populate_system_message(request.messages);
|
||||
|
||||
// pending text records the text to be put into data
|
||||
// we lazily accumulate the pending text
|
||||
// to reduce amount of segments in the Data vector
|
||||
std::string pending_text =
|
||||
conv.GetSystemText(has_custom_system ? custom_system_inputs : conv.system_message);
|
||||
|
||||
// Get the message strings
|
||||
std::vector<Data> message_list;
|
||||
size_t non_system_msg_count = 0;
|
||||
|
||||
// returns error if error happens
|
||||
auto f_process_messages =
|
||||
[&](const std::vector<ChatCompletionMessage>& msg_vec) -> std::optional<TResult> {
|
||||
for (size_t i = 0; i < msg_vec.size(); ++i) {
|
||||
const ChatCompletionMessage& msg = msg_vec[i];
|
||||
// skip system message as it is already processed
|
||||
if (msg.role == "system") continue;
|
||||
|
||||
auto role_it = conv.roles.find(msg.role);
|
||||
if (role_it == conv.roles.end()) {
|
||||
return TResult::Error("Role \"" + msg.role + "\" is not supported");
|
||||
}
|
||||
const std::string& role_name = role_it->second;
|
||||
// skip when content is empty
|
||||
if (msg.content.IsNull()) {
|
||||
pending_text += role_name + conv.role_empty_sep;
|
||||
continue;
|
||||
}
|
||||
++non_system_msg_count;
|
||||
// assistant uses conv.seps[1] if there are two seps
|
||||
int sep_offset = msg.role == "assistant" ? 1 : 0;
|
||||
const std::string& seperator = conv.seps[sep_offset % conv.seps.size()];
|
||||
// setup role prefix
|
||||
std::string role_prefix = "";
|
||||
// Do not append role prefix if this is the first message and there is already a system
|
||||
// message
|
||||
if (conv.add_role_after_system_message || pending_text.empty() || non_system_msg_count != 1) {
|
||||
role_prefix = role_name + conv.role_content_sep;
|
||||
}
|
||||
pending_text += role_prefix;
|
||||
|
||||
if (msg.content.IsParts()) {
|
||||
for (const auto& item : msg.content.Parts()) {
|
||||
auto it_type = item.find("type");
|
||||
if (it_type == item.end()) {
|
||||
return TResult::Error("The content of a message does not have \"type\" field");
|
||||
}
|
||||
if (it_type->second == "text") {
|
||||
auto it_text = item.find("text");
|
||||
if (it_text == item.end()) {
|
||||
return TResult::Error(
|
||||
"The text type content of a message does not have \"text\" field");
|
||||
}
|
||||
// replace placeholder[ROLE] with input message from role
|
||||
pending_text += conv.GetRoleText(msg.role, it_text->second, fn_call_string);
|
||||
} else if (it_type->second == "image_url") {
|
||||
if (item.find("image_url") == item.end()) {
|
||||
return TResult::Error("Content should have an image_url field");
|
||||
}
|
||||
std::string image_url =
|
||||
item.at("image_url"); // TODO(mlc-team): According to OpenAI API reference this
|
||||
// should be a map, with a "url" key containing the URL, but
|
||||
// we are just assuming this as the URL for now
|
||||
std::string base64_image = image_url.substr(image_url.find(",") + 1);
|
||||
Result<Tensor> image_data_res = LoadImageFromBase64(base64_image);
|
||||
if (image_data_res.IsErr()) {
|
||||
return TResult::Error(image_data_res.UnwrapErr());
|
||||
}
|
||||
if (!config.vision_config.has_value()) {
|
||||
return TResult::Error("Vision config is required for image input");
|
||||
}
|
||||
int image_size = config.vision_config.value().image_size;
|
||||
int patch_size = config.vision_config.value().patch_size;
|
||||
|
||||
int embed_size = (image_size * image_size) / (patch_size * patch_size);
|
||||
|
||||
Tensor image_data = image_data_res.Unwrap();
|
||||
std::vector<int64_t> new_shape = {1, image_size, image_size, 3};
|
||||
Tensor image_tensor = image_data.CreateView(new_shape, image_data.DataType());
|
||||
// TODO: Not sure if commenting will affect other functions. But
|
||||
// python part will do clip preprocessing. auto image_tensor =
|
||||
// ClipPreprocessor(image_data_res.Unwrap(), image_size, device);
|
||||
// lazily commit text data
|
||||
if (pending_text.length() != 0) {
|
||||
message_list.push_back(TextData(pending_text));
|
||||
pending_text = "";
|
||||
}
|
||||
message_list.push_back(ImageData(image_tensor, embed_size));
|
||||
} else {
|
||||
return TResult::Error("Unsupported content type: " + it_type->second);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
TVM_FFI_ICHECK(msg.content.IsText());
|
||||
pending_text += conv.GetRoleText(msg.role, msg.content.Text(), fn_call_string);
|
||||
}
|
||||
pending_text += seperator;
|
||||
}
|
||||
return std::nullopt;
|
||||
};
|
||||
|
||||
// Optionally strip `<think>...</think>` blocks from historical assistant
|
||||
// messages (those before the last user message), mirroring Qwen3's HF chat
|
||||
// template. See mlc-ai/mlc-llm#3482.
|
||||
const std::vector<ChatCompletionMessage>* conv_messages_ptr = &conv.messages;
|
||||
const std::vector<ChatCompletionMessage>* request_messages_ptr = &request.messages;
|
||||
std::vector<ChatCompletionMessage> stripped_conv_messages;
|
||||
std::vector<ChatCompletionMessage> stripped_request_messages;
|
||||
if (conv.strip_reasoning_in_history) {
|
||||
const size_t conv_size = conv.messages.size();
|
||||
int64_t last_user_idx = -1;
|
||||
for (size_t i = 0; i < conv_size; ++i) {
|
||||
if (conv.messages[i].role == "user") last_user_idx = static_cast<int64_t>(i);
|
||||
}
|
||||
for (size_t i = 0; i < request.messages.size(); ++i) {
|
||||
if (request.messages[i].role == "user") last_user_idx = static_cast<int64_t>(conv_size + i);
|
||||
}
|
||||
const std::string kCloseTag = "</think>";
|
||||
auto strip_range = [&](const std::vector<ChatCompletionMessage>& in, size_t offset,
|
||||
std::vector<ChatCompletionMessage>& out) {
|
||||
out.reserve(in.size());
|
||||
for (size_t i = 0; i < in.size(); ++i) {
|
||||
ChatCompletionMessage msg = in[i];
|
||||
const int64_t global_idx = static_cast<int64_t>(offset + i);
|
||||
if (msg.role == "assistant" && global_idx < last_user_idx && msg.content.IsText()) {
|
||||
const std::string& text = msg.content.Text();
|
||||
const size_t close_pos = text.rfind(kCloseTag);
|
||||
if (close_pos != std::string::npos) {
|
||||
size_t start = close_pos + kCloseTag.size();
|
||||
while (start < text.size() && text[start] == '\n') ++start;
|
||||
msg.content = ChatCompletionMessageContent(text.substr(start));
|
||||
}
|
||||
}
|
||||
out.push_back(std::move(msg));
|
||||
}
|
||||
};
|
||||
strip_range(conv.messages, 0, stripped_conv_messages);
|
||||
strip_range(request.messages, conv_size, stripped_request_messages);
|
||||
conv_messages_ptr = &stripped_conv_messages;
|
||||
request_messages_ptr = &stripped_request_messages;
|
||||
}
|
||||
|
||||
if (auto err = f_process_messages(*conv_messages_ptr)) {
|
||||
return err.value();
|
||||
}
|
||||
if (auto err = f_process_messages(*request_messages_ptr)) {
|
||||
return err.value();
|
||||
}
|
||||
// append last assistant begin message
|
||||
ChatCompletionMessage last_assistant_begin;
|
||||
last_assistant_begin.role = "assistant";
|
||||
last_assistant_begin.content = std::nullopt;
|
||||
if (auto err = f_process_messages({last_assistant_begin})) {
|
||||
return err.value();
|
||||
}
|
||||
if (pending_text.length() != 0) {
|
||||
message_list.push_back(TextData(pending_text));
|
||||
}
|
||||
// Handle system_prefix_token_ids
|
||||
if (conv.system_prefix_token_ids.has_value()) {
|
||||
message_list.insert(message_list.begin(), TokenData(conv.system_prefix_token_ids.value()));
|
||||
}
|
||||
return TResult::Ok(message_list);
|
||||
}
|
||||
|
||||
Result<Conversation> Conversation::FromJSON(const tvm::ffi::json::Object& json_obj) {
|
||||
using TResult = Result<Conversation>;
|
||||
Conversation conv;
|
||||
|
||||
Result<std::optional<std::string>> name_res =
|
||||
json::LookupOptionalWithResultReturn<std::string>(json_obj, "name");
|
||||
if (name_res.IsErr()) {
|
||||
return TResult::Error(name_res.UnwrapErr());
|
||||
}
|
||||
conv.name = name_res.Unwrap();
|
||||
|
||||
Result<std::string> system_template_res =
|
||||
json::LookupWithResultReturn<std::string>(json_obj, "system_template");
|
||||
if (system_template_res.IsErr()) {
|
||||
return TResult::Error(system_template_res.UnwrapErr());
|
||||
}
|
||||
conv.system_template = system_template_res.Unwrap();
|
||||
|
||||
Result<std::string> system_message_res =
|
||||
json::LookupWithResultReturn<std::string>(json_obj, "system_message");
|
||||
if (system_message_res.IsErr()) {
|
||||
return TResult::Error(system_message_res.UnwrapErr());
|
||||
}
|
||||
conv.system_message = system_message_res.Unwrap();
|
||||
|
||||
Result<std::optional<tvm::ffi::json::Array>> system_prefix_token_ids_arr_res =
|
||||
json::LookupOptionalWithResultReturn<tvm::ffi::json::Array>(json_obj,
|
||||
"system_prefix_token_ids");
|
||||
if (system_prefix_token_ids_arr_res.IsErr()) {
|
||||
return TResult::Error(system_prefix_token_ids_arr_res.UnwrapErr());
|
||||
}
|
||||
std::optional<tvm::ffi::json::Array> system_prefix_token_ids_arr =
|
||||
system_prefix_token_ids_arr_res.Unwrap();
|
||||
if (system_prefix_token_ids_arr.has_value()) {
|
||||
std::vector<int> system_prefix_token_ids;
|
||||
system_prefix_token_ids.reserve(system_prefix_token_ids_arr.value().size());
|
||||
for (const auto& token_id : system_prefix_token_ids_arr.value()) {
|
||||
if (!token_id.try_cast<int64_t>().has_value()) {
|
||||
return TResult::Error("A system prefix token id is not integer.");
|
||||
}
|
||||
system_prefix_token_ids.push_back(static_cast<int>(token_id.cast<int64_t>()));
|
||||
}
|
||||
conv.system_prefix_token_ids = std::move(system_prefix_token_ids);
|
||||
}
|
||||
|
||||
Result<bool> add_role_after_system_message_res =
|
||||
json::LookupWithResultReturn<bool>(json_obj, "add_role_after_system_message");
|
||||
if (add_role_after_system_message_res.IsErr()) {
|
||||
return TResult::Error(add_role_after_system_message_res.UnwrapErr());
|
||||
}
|
||||
conv.add_role_after_system_message = add_role_after_system_message_res.Unwrap();
|
||||
|
||||
Result<tvm::ffi::json::Object> roles_object_res =
|
||||
json::LookupWithResultReturn<tvm::ffi::json::Object>(json_obj, "roles");
|
||||
if (roles_object_res.IsErr()) {
|
||||
return TResult::Error(roles_object_res.UnwrapErr());
|
||||
}
|
||||
for (const auto& role : roles_object_res.Unwrap()) {
|
||||
if (!role.second.try_cast<std::string>().has_value()) {
|
||||
return TResult::Error("A role value in the conversation template is not a string.");
|
||||
}
|
||||
conv.roles[role.first.cast<tvm::ffi::String>()] = role.second.cast<std::string>();
|
||||
}
|
||||
|
||||
Result<std::optional<tvm::ffi::json::Object>> role_templates_object_res =
|
||||
json::LookupOptionalWithResultReturn<tvm::ffi::json::Object>(json_obj, "role_templates");
|
||||
if (role_templates_object_res.IsErr()) {
|
||||
return TResult::Error(role_templates_object_res.UnwrapErr());
|
||||
}
|
||||
std::optional<tvm::ffi::json::Object> role_templates_object = role_templates_object_res.Unwrap();
|
||||
if (role_templates_object.has_value()) {
|
||||
for (const auto& [role, msg] : role_templates_object.value()) {
|
||||
if (!msg.try_cast<std::string>().has_value()) {
|
||||
return TResult::Error("A value in \"role_templates\" is not a string.");
|
||||
}
|
||||
conv.role_templates[role.cast<tvm::ffi::String>()] = msg.cast<std::string>();
|
||||
}
|
||||
}
|
||||
|
||||
Result<tvm::ffi::json::Array> messages_arr_res =
|
||||
json::LookupWithResultReturn<tvm::ffi::json::Array>(json_obj, "messages");
|
||||
if (messages_arr_res.IsErr()) {
|
||||
return TResult::Error(messages_arr_res.UnwrapErr());
|
||||
}
|
||||
for (const auto& message : messages_arr_res.Unwrap()) {
|
||||
if (!message.try_cast<tvm::ffi::json::Array>().has_value() ||
|
||||
message.cast<tvm::ffi::json::Array>().size() != 2) {
|
||||
return TResult::Error(
|
||||
"A message in the conversation template is not an array of [role, content].");
|
||||
}
|
||||
tvm::ffi::json::Array message_arr = message.cast<tvm::ffi::json::Array>();
|
||||
if (!message_arr[0].try_cast<std::string>().has_value()) {
|
||||
return TResult::Error("The role of a message in the conversation template is not a string.");
|
||||
}
|
||||
std::string role = message_arr[0].cast<std::string>();
|
||||
// content can be a string or an array of objects
|
||||
if (message_arr[1].try_cast<std::string>().has_value()) {
|
||||
ChatCompletionMessage msg;
|
||||
msg.role = role;
|
||||
msg.content = message_arr[1].cast<std::string>();
|
||||
conv.messages.push_back(msg);
|
||||
continue;
|
||||
} else if (message_arr[1].try_cast<tvm::ffi::json::Array>().has_value()) {
|
||||
tvm::ffi::json::Array content_arr = message_arr[1].cast<tvm::ffi::json::Array>();
|
||||
std::vector<std::unordered_map<std::string, std::string>> content;
|
||||
content.reserve(content_arr.size());
|
||||
for (const auto& item : content_arr) {
|
||||
if (!item.try_cast<tvm::ffi::json::Object>().has_value()) {
|
||||
return TResult::Error("The content of conversation template message is not an object");
|
||||
}
|
||||
std::unordered_map<std::string, std::string> item_map;
|
||||
for (const auto& [key, value] : item.cast<tvm::ffi::json::Object>()) {
|
||||
item_map[key.cast<tvm::ffi::String>()] = tvm::ffi::json::Stringify(value);
|
||||
}
|
||||
content.push_back(std::move(item_map));
|
||||
}
|
||||
ChatCompletionMessage msg;
|
||||
msg.role = role;
|
||||
msg.content = content;
|
||||
conv.messages.push_back(msg);
|
||||
continue;
|
||||
} else {
|
||||
return TResult::Error(
|
||||
"The content of a message in the conversation template is not a string or an array.");
|
||||
}
|
||||
}
|
||||
|
||||
Result<tvm::ffi::json::Array> seps_arr_res =
|
||||
json::LookupWithResultReturn<tvm::ffi::json::Array>(json_obj, "seps");
|
||||
if (seps_arr_res.IsErr()) {
|
||||
return TResult::Error(seps_arr_res.UnwrapErr());
|
||||
}
|
||||
std::vector<std::string> seps;
|
||||
for (const auto& sep : seps_arr_res.Unwrap()) {
|
||||
if (!sep.try_cast<std::string>().has_value()) {
|
||||
return TResult::Error("A separator (\"seps\") of the conversation template is not a string");
|
||||
}
|
||||
conv.seps.push_back(sep.cast<std::string>());
|
||||
}
|
||||
|
||||
Result<std::string> role_content_sep_res =
|
||||
json::LookupWithResultReturn<std::string>(json_obj, "role_content_sep");
|
||||
if (role_content_sep_res.IsErr()) {
|
||||
return TResult::Error(role_content_sep_res.UnwrapErr());
|
||||
}
|
||||
conv.role_content_sep = role_content_sep_res.Unwrap();
|
||||
|
||||
Result<std::string> role_empty_sep_res =
|
||||
json::LookupWithResultReturn<std::string>(json_obj, "role_empty_sep");
|
||||
if (role_empty_sep_res.IsErr()) {
|
||||
return TResult::Error(role_empty_sep_res.UnwrapErr());
|
||||
}
|
||||
conv.role_empty_sep = role_empty_sep_res.Unwrap();
|
||||
|
||||
Result<tvm::ffi::json::Array> stop_str_arr_res =
|
||||
json::LookupWithResultReturn<tvm::ffi::json::Array>(json_obj, "stop_str");
|
||||
if (stop_str_arr_res.IsErr()) {
|
||||
return TResult::Error(stop_str_arr_res.UnwrapErr());
|
||||
}
|
||||
for (const auto& stop : stop_str_arr_res.Unwrap()) {
|
||||
if (!stop.try_cast<std::string>().has_value()) {
|
||||
return TResult::Error(
|
||||
"A stop string (\"stop_str\") of the conversation template is not a string.");
|
||||
}
|
||||
conv.stop_str.push_back(stop.cast<std::string>());
|
||||
}
|
||||
|
||||
Result<tvm::ffi::json::Array> stop_token_ids_arr_res =
|
||||
json::LookupWithResultReturn<tvm::ffi::json::Array>(json_obj, "stop_token_ids");
|
||||
if (stop_token_ids_arr_res.IsErr()) {
|
||||
return TResult::Error(stop_token_ids_arr_res.UnwrapErr());
|
||||
}
|
||||
for (const auto& stop : stop_token_ids_arr_res.Unwrap()) {
|
||||
if (!stop.try_cast<int64_t>().has_value()) {
|
||||
return TResult::Error(
|
||||
"A stop token id (\"stop_token_ids\") of the conversation template is not an integer.");
|
||||
}
|
||||
conv.stop_token_ids.push_back(static_cast<int>(stop.cast<int64_t>()));
|
||||
}
|
||||
|
||||
Result<std::optional<bool>> strip_reasoning_res =
|
||||
json::LookupOptionalWithResultReturn<bool>(json_obj, "strip_reasoning_in_history");
|
||||
if (strip_reasoning_res.IsErr()) {
|
||||
return TResult::Error(strip_reasoning_res.UnwrapErr());
|
||||
}
|
||||
conv.strip_reasoning_in_history = strip_reasoning_res.Unwrap().value_or(false);
|
||||
|
||||
return TResult::Ok(conv);
|
||||
}
|
||||
|
||||
Result<Conversation> Conversation::FromJSON(const std::string& json_str) {
|
||||
Result<tvm::ffi::json::Object> json_obj = json::ParseToJSONObjectWithResultReturn(json_str);
|
||||
if (json_obj.IsErr()) {
|
||||
return Result<Conversation>::Error(json_obj.UnwrapErr());
|
||||
}
|
||||
return Conversation::FromJSON(json_obj.Unwrap());
|
||||
}
|
||||
|
||||
} // namespace json_ffi
|
||||
} // namespace llm
|
||||
} // namespace mlc
|
||||
@@ -0,0 +1,161 @@
|
||||
#ifndef MLC_LLM_JSON_FFI_CONV_TEMPLATE_H
|
||||
#define MLC_LLM_JSON_FFI_CONV_TEMPLATE_H
|
||||
|
||||
#include <tvm/ffi/extra/json.h>
|
||||
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <typeinfo>
|
||||
#include <variant>
|
||||
#include <vector>
|
||||
|
||||
#include "../serve/data.h"
|
||||
#include "../support/result.h"
|
||||
#include "openai_api_protocol.h"
|
||||
|
||||
using namespace mlc::llm::serve;
|
||||
|
||||
namespace mlc {
|
||||
namespace llm {
|
||||
namespace json_ffi {
|
||||
|
||||
/****************** Model vision config ******************/
|
||||
|
||||
/*! \brief Defines the Vision config of the model (if present) */
|
||||
class ModelVisionConfig {
|
||||
public:
|
||||
int hidden_size;
|
||||
int image_size;
|
||||
int intermediate_size;
|
||||
int num_attention_heads;
|
||||
int num_hidden_layers;
|
||||
int patch_size;
|
||||
int projection_dim;
|
||||
int vocab_size;
|
||||
std::string dtype;
|
||||
int num_channels;
|
||||
double layer_norm_eps;
|
||||
|
||||
static ModelVisionConfig FromJSON(const tvm::ffi::json::Object& json_obj);
|
||||
};
|
||||
|
||||
/****************** Model config ******************/
|
||||
|
||||
/*! \brief Defines the config of the model.
|
||||
Populated from "model_config" field in mlc-chat-config.json */
|
||||
class ModelConfig {
|
||||
public:
|
||||
int vocab_size;
|
||||
int context_window_size;
|
||||
int sliding_window_size;
|
||||
int prefill_chunk_size;
|
||||
int tensor_parallel_shards;
|
||||
int pipeline_parallel_stages;
|
||||
int max_batch_size;
|
||||
std::optional<ModelVisionConfig> vision_config = std::nullopt;
|
||||
|
||||
static ModelConfig FromJSON(const tvm::ffi::json::Object& json_obj);
|
||||
};
|
||||
|
||||
/****************** Conversation template ******************/
|
||||
|
||||
enum class MessagePlaceholders { SYSTEM, USER, ASSISTANT, TOOL, FUNCTION };
|
||||
|
||||
MessagePlaceholders MessagePlaceholderFromString(const std::string& role);
|
||||
|
||||
/**
|
||||
* @brief A struct that specifies the convention template of conversation
|
||||
* and contains the conversation history.
|
||||
*/
|
||||
struct Conversation {
|
||||
// Optional name of the template.
|
||||
std::optional<std::string> name = std::nullopt;
|
||||
|
||||
// The system prompt template, it optionally contains the system
|
||||
// message placeholder, and the placeholder will be replaced with
|
||||
// the system message below.
|
||||
std::string system_template;
|
||||
|
||||
// The content of the system prompt (without the template format).
|
||||
std::string system_message;
|
||||
|
||||
// The system token ids to be prepended at the beginning of tokenized
|
||||
// generated prompt.
|
||||
std::optional<std::vector<int>> system_prefix_token_ids = std::nullopt;
|
||||
|
||||
// Whether or not to append user role and separator after the system message.
|
||||
// This is mainly for [INST] [/INST] style prompt format
|
||||
bool add_role_after_system_message = true;
|
||||
|
||||
// The conversation roles
|
||||
std::unordered_map<std::string, std::string> roles;
|
||||
|
||||
// The roles prompt template, it optionally contains the defaults
|
||||
// message placeholders and will be replaced by actual content
|
||||
std::unordered_map<std::string, std::string> role_templates;
|
||||
|
||||
// The conversation history messages.
|
||||
// Each message is a pair of strings, denoting "(role, content)".
|
||||
// The content can be None.
|
||||
std::vector<ChatCompletionMessage> messages;
|
||||
|
||||
// The separators between messages when concatenating into a single prompt.
|
||||
// List size should be either 1 or 2.
|
||||
// - When size is 1, the separator will be used between adjacent messages.
|
||||
// - When size is 2, seps[0] is used after user message, and
|
||||
// seps[1] is used after assistant message.
|
||||
std::vector<std::string> seps;
|
||||
|
||||
// The separator between the role and the content in a message.
|
||||
std::string role_content_sep;
|
||||
|
||||
// The separator between the role and empty contents.
|
||||
std::string role_empty_sep;
|
||||
|
||||
// The stop criteria
|
||||
std::vector<std::string> stop_str;
|
||||
std::vector<int> stop_token_ids;
|
||||
|
||||
// When true, strip `<think>...</think>` blocks (and any trailing whitespace)
|
||||
// from historical assistant messages before rendering the prompt, mirroring
|
||||
// Qwen3's official HF chat template. Only assistant messages that appear
|
||||
// before the last user message are affected.
|
||||
bool strip_reasoning_in_history = false;
|
||||
|
||||
Conversation();
|
||||
|
||||
/*!
|
||||
* \brief Get the system text(with the prompt template) given the system prompt message
|
||||
* \param system_msg The system prompt message.
|
||||
* \return The created system text.
|
||||
*/
|
||||
std::string GetSystemText(const std::string& system_msg) const;
|
||||
|
||||
/*!
|
||||
* \brief replace the content from role by the correct role text in template
|
||||
* \param role The input role
|
||||
* \param content The input content from the role
|
||||
* \param fn_call_str The function calling string if any.
|
||||
* \return The created text.
|
||||
*/
|
||||
std::string GetRoleText(const std::string& role, const std::string& content,
|
||||
const std::optional<std::string>& fn_call_str) const;
|
||||
|
||||
/*! \brief Create a Conversation instance from the given JSON object. */
|
||||
static Result<Conversation> FromJSON(const tvm::ffi::json::Object& json);
|
||||
/*! \brief Parse and create a Conversation instance from the given JSON string. */
|
||||
static Result<Conversation> FromJSON(const std::string& json_str);
|
||||
};
|
||||
|
||||
/*! \brief Create the list of prompts from the messages based on the conversation template. */
|
||||
Result<std::vector<Data>> CreatePrompt(const Conversation& conv,
|
||||
const ChatCompletionRequest& request,
|
||||
const ModelConfig& config, DLDevice device);
|
||||
|
||||
} // namespace json_ffi
|
||||
} // namespace llm
|
||||
} // namespace mlc
|
||||
|
||||
#endif // MLC_LLM_JSON_FFI_CONV_TEMPLATE_H
|
||||
@@ -0,0 +1,160 @@
|
||||
#include "image_utils.h"
|
||||
|
||||
#include <tvm/support/io.h>
|
||||
|
||||
#include "../../3rdparty/tvm/src/support/base64.h"
|
||||
#define STB_IMAGE_IMPLEMENTATION
|
||||
#include "stb_image.h"
|
||||
|
||||
namespace mlc {
|
||||
namespace llm {
|
||||
namespace json_ffi {
|
||||
|
||||
using namespace tvm::runtime;
|
||||
|
||||
class MemoryBufferStream : public tvm::support::Stream {
|
||||
public:
|
||||
using Stream::Read;
|
||||
using Stream::Write;
|
||||
|
||||
MemoryBufferStream(const char* data, size_t size) : data_(data), size_(size), pos_(0) {}
|
||||
|
||||
size_t Read(void* ptr, size_t size) override {
|
||||
size_t remaining = size_ - pos_;
|
||||
if (size > remaining) {
|
||||
size = remaining;
|
||||
}
|
||||
if (size == 0) {
|
||||
return 0;
|
||||
}
|
||||
std::memcpy(ptr, data_ + pos_, size);
|
||||
pos_ += size;
|
||||
return size;
|
||||
}
|
||||
|
||||
size_t Write(const void* ptr, size_t size) override {
|
||||
TVM_FFI_THROW(InternalError) << "MemoryBufferStream does not support write";
|
||||
return 0;
|
||||
}
|
||||
|
||||
private:
|
||||
const char* data_;
|
||||
size_t size_;
|
||||
size_t pos_;
|
||||
};
|
||||
|
||||
size_t Base64DecodedSize(const std::string& base64_str) {
|
||||
size_t len = base64_str.size();
|
||||
size_t padding = 0;
|
||||
if (base64_str[len - 1] == '=') {
|
||||
padding++;
|
||||
}
|
||||
if (base64_str[len - 2] == '=') {
|
||||
padding++;
|
||||
}
|
||||
return 3 * len / 4 - padding;
|
||||
}
|
||||
|
||||
Result<Tensor> LoadImageFromBase64(const std::string& base64_str) {
|
||||
using TResult = Result<Tensor>;
|
||||
MemoryBufferStream stream(base64_str.c_str(), base64_str.size());
|
||||
tvm::support::Base64InStream base64_stream(&stream);
|
||||
size_t decoded_size = Base64DecodedSize(base64_str);
|
||||
std::vector<unsigned char> decoded(decoded_size);
|
||||
base64_stream.InitPosition();
|
||||
base64_stream.Read((void*)decoded.data(), decoded_size);
|
||||
int width, height, num_channels;
|
||||
unsigned char* image_data =
|
||||
stbi_load_from_memory(decoded.data(), decoded_size, &width, &height, &num_channels, 3);
|
||||
if (!image_data) {
|
||||
return TResult::Error(stbi_failure_reason());
|
||||
}
|
||||
auto image_tensor = Tensor::Empty({height, width, 3}, {kDLUInt, 8, 1}, {kDLCPU, 0});
|
||||
image_tensor.CopyFromBytes((void*)image_data, width * height * 3);
|
||||
stbi_image_free(image_data);
|
||||
return TResult::Ok(image_tensor);
|
||||
}
|
||||
|
||||
Tensor ClipPreprocessor(Tensor image_data, int target_size, DLDevice device) {
|
||||
int height = image_data->shape[0];
|
||||
int width = image_data->shape[1];
|
||||
// Resize
|
||||
const int short_side = width < height ? width : height;
|
||||
const int long_side = width > height ? width : height;
|
||||
const int new_short_side = target_size;
|
||||
const int new_long_side = (int)(new_short_side * (long_side / (float)short_side));
|
||||
const int new_width = width < height ? new_short_side : new_long_side;
|
||||
const int new_height = width > height ? new_short_side : new_long_side;
|
||||
|
||||
std::vector<float> processed_image_data(new_width * new_height * 3);
|
||||
|
||||
// Bilinear Interpolation
|
||||
for (int y = 0; y < new_height; y++) {
|
||||
for (int x = 0; x < new_width; x++) {
|
||||
const float x_ratio = float(width - 1) / new_width;
|
||||
const float y_ratio = float(height - 1) / new_height;
|
||||
const int x1 = int(x_ratio * x);
|
||||
const int y1 = int(y_ratio * y);
|
||||
const int x2 = x1 + 1;
|
||||
const int y2 = y1 + 1;
|
||||
const float x_diff = x_ratio * x - x1;
|
||||
const float y_diff = y_ratio * y - y1;
|
||||
for (int c = 0; c < 3; c++) {
|
||||
const uint8_t top_left = ((uint8_t*)image_data->data)[(y1 * width + x1) * 3 + c];
|
||||
const uint8_t top_right = ((uint8_t*)image_data->data)[(y1 * width + x2) * 3 + c];
|
||||
const uint8_t bottom_left = ((uint8_t*)image_data->data)[(y2 * width + x1) * 3 + c];
|
||||
const uint8_t bottom_right = ((uint8_t*)image_data->data)[(y2 * width + x2) * 3 + c];
|
||||
processed_image_data[(y * new_width + x) * 3 + c] =
|
||||
(float)(int(top_left * (1 - x_diff) * (1 - y_diff) + top_right * x_diff * (1 - y_diff) +
|
||||
bottom_left * y_diff * (1 - x_diff) + bottom_right * x_diff * y_diff));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Center crop
|
||||
const int crop_x = (new_width - target_size) / 2;
|
||||
const int crop_y = (new_height - target_size) / 2;
|
||||
std::vector<float> cropped_image_data(target_size * target_size * 3);
|
||||
for (int y = 0; y < target_size; y++) {
|
||||
for (int x = 0; x < target_size; x++) {
|
||||
for (int c = 0; c < 3; c++) {
|
||||
cropped_image_data[(y * target_size + x) * 3 + c] =
|
||||
processed_image_data[((y + crop_y) * new_width + x + crop_x) * 3 + c];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Rescale
|
||||
for (int i = 0; i < target_size * target_size * 3; i++) {
|
||||
cropped_image_data[i] = cropped_image_data[i] / 255.0f;
|
||||
}
|
||||
|
||||
// Normalize
|
||||
const float IMAGE_MEAN[] = {0.48145466f, 0.4578275f, 0.40821073f};
|
||||
const float IMAGE_STD[] = {0.26862954f, 0.26130258f, 0.27577711f};
|
||||
for (int i = 0; i < target_size * target_size * 3; i++) {
|
||||
const int c = i % 3;
|
||||
cropped_image_data[i] = (cropped_image_data[i] - IMAGE_MEAN[c]) / IMAGE_STD[c];
|
||||
}
|
||||
|
||||
std::vector<float> image_data_channel_first(target_size * target_size * 3);
|
||||
for (int y = 0; y < target_size; y++) {
|
||||
for (int x = 0; x < target_size; x++) {
|
||||
for (int c = 0; c < 3; c++) {
|
||||
image_data_channel_first[c * target_size * target_size + y * target_size + x] =
|
||||
cropped_image_data[(y * target_size + x) * 3 + c];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create Tensor
|
||||
auto image_tensor = Tensor::Empty({1, 3, target_size, target_size}, {kDLFloat, 32, 1}, device);
|
||||
image_tensor.CopyFromBytes((void*)image_data_channel_first.data(),
|
||||
target_size * target_size * 3 * sizeof(float));
|
||||
|
||||
return image_tensor;
|
||||
}
|
||||
|
||||
} // namespace json_ffi
|
||||
} // namespace llm
|
||||
} // namespace mlc
|
||||
@@ -0,0 +1,31 @@
|
||||
/*!
|
||||
* Copyright (c) 2023-2025 by Contributors
|
||||
* \file json_ffi/image_utils.h
|
||||
* \brief The header of Image utils for JSON FFI Engine in MLC LLM.
|
||||
*/
|
||||
#ifndef MLC_LLM_JSON_FFI_IMAGE_UTILS_H_
|
||||
#define MLC_LLM_JSON_FFI_IMAGE_UTILS_H_
|
||||
|
||||
#include <tvm/runtime/tensor.h>
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
#include "../support/result.h"
|
||||
|
||||
namespace mlc {
|
||||
namespace llm {
|
||||
namespace json_ffi {
|
||||
|
||||
/*! \brief Load a base64 encoded image string into a CPU Tensor of shape {height, width, 3} */
|
||||
Result<tvm::runtime::Tensor> LoadImageFromBase64(const std::string& base64_str);
|
||||
|
||||
/*! \brief Preprocess the CPU image for CLIP encoder and return an Tensor on the given device */
|
||||
tvm::runtime::Tensor ClipPreprocessor(tvm::runtime::Tensor image_data, int target_size,
|
||||
DLDevice device);
|
||||
|
||||
} // namespace json_ffi
|
||||
} // namespace llm
|
||||
} // namespace mlc
|
||||
|
||||
#endif // MLC_LLM_JSON_FFI_IMAGE_UTILS_H_
|
||||
@@ -0,0 +1,310 @@
|
||||
#include "json_ffi_engine.h"
|
||||
|
||||
#include <tvm/ffi/extra/json.h>
|
||||
#include <tvm/ffi/extra/module.h>
|
||||
#include <tvm/ffi/function.h>
|
||||
#include <tvm/ffi/reflection/registry.h>
|
||||
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
|
||||
#include "../serve/model.h"
|
||||
#include "../support/json_parser.h"
|
||||
#include "../support/module_vtable.h"
|
||||
#include "../support/result.h"
|
||||
|
||||
namespace mlc {
|
||||
namespace llm {
|
||||
namespace json_ffi {
|
||||
|
||||
using namespace tvm::runtime;
|
||||
using tvm::ffi::Object;
|
||||
using tvm::ffi::Shape;
|
||||
|
||||
JSONFFIEngine::JSONFFIEngine() { engine_ = serve::ThreadedEngine::Create(); }
|
||||
|
||||
bool JSONFFIEngine::ChatCompletion(std::string request_json_str, std::string request_id) {
|
||||
bool success = this->AddRequest(request_json_str, request_id);
|
||||
if (!success) {
|
||||
this->StreamBackError(request_id);
|
||||
}
|
||||
return success;
|
||||
}
|
||||
|
||||
void JSONFFIEngine::StreamBackError(std::string request_id) {
|
||||
ChatCompletionMessage delta;
|
||||
delta.content = this->err_;
|
||||
delta.role = "assistant";
|
||||
|
||||
ChatCompletionStreamResponseChoice choice;
|
||||
choice.finish_reason = FinishReason::error;
|
||||
choice.index = 0;
|
||||
choice.delta = delta;
|
||||
|
||||
ChatCompletionStreamResponse response;
|
||||
response.id = request_id;
|
||||
response.choices = std::vector<ChatCompletionStreamResponseChoice>{choice};
|
||||
response.model = "json_ffi"; // TODO: Return model name from engine (or from args)
|
||||
response.system_fingerprint = "";
|
||||
|
||||
tvm::ffi::json::Array response_arr;
|
||||
response_arr.push_back(response.AsJSON());
|
||||
|
||||
// now stream back the final usage block, which is required.
|
||||
// NOTE: always stream back final usage block as it is an
|
||||
// invariant of the system
|
||||
response.choices.clear();
|
||||
tvm::ffi::json::Object dummy_usage;
|
||||
dummy_usage.Set("prompt_tokens", static_cast<int64_t>(0));
|
||||
dummy_usage.Set("completion_tokens", static_cast<int64_t>(0));
|
||||
dummy_usage.Set("total_tokens", static_cast<int64_t>(0));
|
||||
response.usage = tvm::ffi::json::Value(dummy_usage);
|
||||
response_arr.push_back(response.AsJSON());
|
||||
|
||||
std::string stream_back_json = tvm::ffi::json::Stringify(response_arr);
|
||||
this->request_stream_callback_(stream_back_json);
|
||||
}
|
||||
|
||||
bool JSONFFIEngine::AddRequest(std::string request_json_str, std::string request_id) {
|
||||
Result<ChatCompletionRequest> request_res = ChatCompletionRequest::FromJSON(request_json_str);
|
||||
if (request_res.IsErr()) {
|
||||
err_ = request_res.UnwrapErr();
|
||||
return false;
|
||||
}
|
||||
ChatCompletionRequest request = request_res.Unwrap();
|
||||
Array<Data> inputs;
|
||||
Array<String> stop_strs;
|
||||
bool is_special_request =
|
||||
(request.debug_config.has_value() &&
|
||||
request.debug_config.value().special_request != SpecialRequestKind::kNone);
|
||||
// special request does not have to go through prompt construction
|
||||
if (!is_special_request) {
|
||||
// get prompt: note, assistant was appended in the end.
|
||||
Result<std::vector<Data>> inputs_obj =
|
||||
CreatePrompt(this->conv_template_, request, this->model_config_, this->device_);
|
||||
if (inputs_obj.IsErr()) {
|
||||
err_ = inputs_obj.UnwrapErr();
|
||||
return false;
|
||||
}
|
||||
inputs = inputs_obj.Unwrap();
|
||||
|
||||
stop_strs.reserve(this->conv_template_.stop_str.size());
|
||||
for (const std::string& stop_str : this->conv_template_.stop_str) {
|
||||
stop_strs.push_back(stop_str);
|
||||
}
|
||||
if (request.stop.has_value()) {
|
||||
stop_strs.reserve(stop_strs.size() + request.stop.value().size());
|
||||
for (const std::string& stop_str : request.stop.value()) {
|
||||
stop_strs.push_back(stop_str);
|
||||
}
|
||||
}
|
||||
}
|
||||
// create a generation config from request
|
||||
const auto& default_gen_cfg = default_generation_config_;
|
||||
auto gen_cfg = tvm::ffi::make_object<GenerationConfigNode>();
|
||||
gen_cfg->n = request.n;
|
||||
gen_cfg->temperature = request.temperature.value_or(default_gen_cfg->temperature);
|
||||
gen_cfg->top_p = request.top_p.value_or(default_gen_cfg->top_p);
|
||||
gen_cfg->frequency_penalty =
|
||||
request.frequency_penalty.value_or(default_gen_cfg->frequency_penalty);
|
||||
gen_cfg->presence_penalty = request.presence_penalty.value_or(default_gen_cfg->presence_penalty);
|
||||
gen_cfg->logprobs = request.logprobs;
|
||||
gen_cfg->top_logprobs = request.top_logprobs;
|
||||
gen_cfg->logit_bias = request.logit_bias.value_or(default_gen_cfg->logit_bias);
|
||||
gen_cfg->seed = request.seed.value_or(std::random_device{}());
|
||||
gen_cfg->max_tokens = request.max_tokens.value_or(default_gen_cfg->max_tokens);
|
||||
gen_cfg->stop_strs = std::move(stop_strs);
|
||||
gen_cfg->stop_token_ids = conv_template_.stop_token_ids;
|
||||
gen_cfg->response_format = request.response_format.value_or(ResponseFormat());
|
||||
gen_cfg->debug_config = request.debug_config.value_or(DebugConfig());
|
||||
|
||||
Result<GenerationConfig> res_gen_config = GenerationConfig::Validate(GenerationConfig(gen_cfg));
|
||||
if (res_gen_config.IsErr()) {
|
||||
err_ = res_gen_config.UnwrapErr();
|
||||
return false;
|
||||
}
|
||||
|
||||
Request engine_request(request_id, inputs, res_gen_config.Unwrap());
|
||||
|
||||
// setup request state
|
||||
RequestState rstate;
|
||||
rstate.model = request.model.value_or("");
|
||||
rstate.streamer.reserve(gen_cfg->n);
|
||||
for (int i = 0; i < gen_cfg->n; ++i) {
|
||||
rstate.streamer.push_back(TextStreamer(tokenizer_));
|
||||
}
|
||||
request_map_[request_id] = std::move(rstate);
|
||||
|
||||
this->engine_->AddRequest(engine_request);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool JSONFFIEngine::Abort(std::string request_id) {
|
||||
this->engine_->AbortRequest(request_id);
|
||||
auto it = request_map_.find(request_id);
|
||||
if (it != request_map_.end()) {
|
||||
request_map_.erase(it);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string JSONFFIEngine::GetLastError() { return err_; }
|
||||
|
||||
void JSONFFIEngine::ExitBackgroundLoop() { this->engine_->ExitBackgroundLoop(); }
|
||||
|
||||
JSONFFIEngine::~JSONFFIEngine() { this->ExitBackgroundLoop(); }
|
||||
|
||||
class JSONFFIEngineImpl : public JSONFFIEngine, public ffi::ModuleObj {
|
||||
public:
|
||||
TVM_MODULE_VTABLE_BEGIN("mlc.json_ffi");
|
||||
TVM_MODULE_VTABLE_ENTRY("init_background_engine", &JSONFFIEngineImpl::InitBackgroundEngine);
|
||||
TVM_MODULE_VTABLE_ENTRY("reload", &JSONFFIEngineImpl::Reload);
|
||||
TVM_MODULE_VTABLE_ENTRY("unload", &JSONFFIEngineImpl::Unload);
|
||||
TVM_MODULE_VTABLE_ENTRY("reset", &JSONFFIEngineImpl::Reset);
|
||||
TVM_MODULE_VTABLE_ENTRY("chat_completion", &JSONFFIEngineImpl::ChatCompletion);
|
||||
TVM_MODULE_VTABLE_ENTRY("abort", &JSONFFIEngineImpl::Abort);
|
||||
TVM_MODULE_VTABLE_ENTRY("get_last_error", &JSONFFIEngineImpl::GetLastError);
|
||||
TVM_MODULE_VTABLE_ENTRY("run_background_loop", &JSONFFIEngineImpl::RunBackgroundLoop);
|
||||
TVM_MODULE_VTABLE_ENTRY("run_background_stream_back_loop",
|
||||
&JSONFFIEngineImpl::RunBackgroundStreamBackLoop);
|
||||
TVM_MODULE_VTABLE_ENTRY("exit_background_loop", &JSONFFIEngineImpl::ExitBackgroundLoop);
|
||||
TVM_MODULE_VTABLE_END();
|
||||
|
||||
void InitBackgroundEngine(int device_type, int device_id,
|
||||
Optional<Function> request_stream_callback) {
|
||||
DLDevice device{static_cast<DLDeviceType>(device_type), device_id};
|
||||
this->device_ = device;
|
||||
TVM_FFI_ICHECK(request_stream_callback.has_value())
|
||||
<< "JSONFFIEngine requires request stream callback function, but it is not given.";
|
||||
this->request_stream_callback_ = request_stream_callback.value();
|
||||
|
||||
auto frequest_stream_callback_wrapper = [this](ffi::PackedArgs args, ffi::Any* ret) {
|
||||
TVM_FFI_ICHECK_EQ(args.size(), 1);
|
||||
Array<RequestStreamOutput> delta_outputs = args[0].cast<Array<RequestStreamOutput>>();
|
||||
std::string responses = this->GetResponseFromStreamOutput(delta_outputs);
|
||||
this->request_stream_callback_(responses);
|
||||
};
|
||||
|
||||
request_stream_callback = Function(frequest_stream_callback_wrapper);
|
||||
this->engine_->InitThreadedEngine(device, std::move(request_stream_callback), std::nullopt);
|
||||
}
|
||||
|
||||
void Reload(String engine_config_json_str) {
|
||||
this->engine_->Reload(engine_config_json_str);
|
||||
this->default_generation_config_ = this->engine_->GetDefaultGenerationConfig();
|
||||
auto engine_config = this->engine_->GetCompleteEngineConfig();
|
||||
|
||||
// Load conversation template.
|
||||
Result<tvm::ffi::json::Object> model_config_json =
|
||||
serve::Model::LoadModelConfig(engine_config->model);
|
||||
TVM_FFI_ICHECK(model_config_json.IsOk()) << model_config_json.UnwrapErr();
|
||||
const tvm::ffi::json::Object& model_config_json_unwrapped = model_config_json.Unwrap();
|
||||
Result<Conversation> conv_template = Conversation::FromJSON(
|
||||
json::Lookup<tvm::ffi::json::Object>(model_config_json_unwrapped, "conv_template"));
|
||||
TVM_FFI_ICHECK(!conv_template.IsErr())
|
||||
<< "Invalid conversation template JSON: " << conv_template.UnwrapErr();
|
||||
this->conv_template_ = conv_template.Unwrap();
|
||||
this->model_config_ = ModelConfig::FromJSON(
|
||||
json::Lookup<tvm::ffi::json::Object>(model_config_json_unwrapped, "model_config"));
|
||||
this->tokenizer_ = Tokenizer::FromPath(engine_config->model);
|
||||
}
|
||||
|
||||
void Unload() { this->engine_->Unload(); }
|
||||
|
||||
void Reset() { this->engine_->Reset(); }
|
||||
|
||||
void RunBackgroundLoop() { this->engine_->RunBackgroundLoop(); }
|
||||
|
||||
void RunBackgroundStreamBackLoop() { this->engine_->RunBackgroundStreamBackLoop(); }
|
||||
|
||||
String GetResponseFromStreamOutput(Array<RequestStreamOutput> delta_outputs) {
|
||||
tvm::ffi::json::Array json_response_arr;
|
||||
for (const auto& delta_output : delta_outputs) {
|
||||
std::string request_id = delta_output->request_id;
|
||||
auto request_state_it = request_map_.find(request_id);
|
||||
if (request_state_it == request_map_.end()) continue;
|
||||
RequestState& rstate = request_state_it->second;
|
||||
|
||||
// build the final usage messages
|
||||
// invariant, we can always let other messages to come first
|
||||
// then the final usage messages, as final usage is always last
|
||||
if (delta_output->request_final_usage_json_str.has_value()) {
|
||||
ChatCompletionStreamResponse response;
|
||||
response.id = request_id;
|
||||
response.model = rstate.model;
|
||||
response.system_fingerprint = "";
|
||||
std::string usage_json_str = delta_output->request_final_usage_json_str.value();
|
||||
tvm::ffi::String parse_err;
|
||||
auto usage_json = tvm::ffi::json::Parse(usage_json_str, &parse_err);
|
||||
if (!parse_err.empty()) {
|
||||
err_ = parse_err;
|
||||
} else {
|
||||
response.usage = usage_json;
|
||||
}
|
||||
json_response_arr.push_back(response.AsJSON());
|
||||
request_map_.erase(request_state_it);
|
||||
continue;
|
||||
}
|
||||
TVM_FFI_ICHECK_NE(delta_output->group_finish_reason.size(), 0);
|
||||
TVM_FFI_ICHECK_EQ(delta_output->group_delta_token_ids.size(),
|
||||
delta_output->group_finish_reason.size());
|
||||
TVM_FFI_ICHECK_EQ(delta_output->group_delta_token_ids.size(), rstate.streamer.size());
|
||||
|
||||
ChatCompletionStreamResponse response;
|
||||
response.id = request_id;
|
||||
response.model = rstate.model;
|
||||
response.system_fingerprint = "";
|
||||
|
||||
for (size_t i = 0; i < delta_output->group_finish_reason.size(); ++i) {
|
||||
// choice
|
||||
ChatCompletionStreamResponseChoice choice;
|
||||
Optional<String> finish_reason = delta_output->group_finish_reason[i];
|
||||
if (finish_reason.has_value()) {
|
||||
if (finish_reason.value() == "stop") {
|
||||
choice.finish_reason = FinishReason::stop;
|
||||
} else if (finish_reason.value() == "length") {
|
||||
choice.finish_reason = FinishReason::length;
|
||||
} else if (finish_reason.value() == "tool_calls") {
|
||||
choice.finish_reason = FinishReason::tool_calls;
|
||||
} else if (finish_reason.value() == "error") {
|
||||
choice.finish_reason = FinishReason::error;
|
||||
}
|
||||
} else {
|
||||
choice.finish_reason = std::nullopt;
|
||||
}
|
||||
choice.index = static_cast<int>(i);
|
||||
ChatCompletionMessage delta;
|
||||
// Size of delta_output->group_delta_token_ids Array should be 1
|
||||
const Shape& delta_token_ids = delta_output->group_delta_token_ids[i];
|
||||
std::vector<int32_t> delta_token_ids_vec(delta_token_ids.begin(), delta_token_ids.end());
|
||||
std::string content = rstate.streamer[i]->Put(delta_token_ids_vec);
|
||||
if (finish_reason.has_value()) {
|
||||
content += rstate.streamer[i]->Finish();
|
||||
}
|
||||
if (!content.empty()) {
|
||||
delta.content = content;
|
||||
}
|
||||
delta.role = "assistant";
|
||||
choice.delta = delta;
|
||||
if (!choice.delta.content.IsNull() || choice.finish_reason.has_value()) {
|
||||
response.choices.push_back(choice);
|
||||
}
|
||||
}
|
||||
// if it is not the usage block, choices cannot be empty
|
||||
if (!response.choices.empty()) {
|
||||
json_response_arr.push_back(response.AsJSON());
|
||||
}
|
||||
}
|
||||
return tvm::ffi::json::Stringify(json_response_arr);
|
||||
}
|
||||
};
|
||||
|
||||
TVM_FFI_STATIC_INIT_BLOCK() {
|
||||
namespace refl = tvm::ffi::reflection;
|
||||
refl::GlobalDef().def("mlc.json_ffi.CreateJSONFFIEngine",
|
||||
[]() { return ffi::Module(tvm::ffi::make_object<JSONFFIEngineImpl>()); });
|
||||
}
|
||||
|
||||
} // namespace json_ffi
|
||||
} // namespace llm
|
||||
} // namespace mlc
|
||||
@@ -0,0 +1,74 @@
|
||||
/*!
|
||||
* Copyright (c) 2023-2025 by Contributors
|
||||
* \file json_ffi/json_ffi_engine.h
|
||||
* \brief The header of JSON FFI engine in MLC LLM.
|
||||
*/
|
||||
#ifndef MLC_LLM_JSON_FFI_JSON_FFI_ENGINE_H_
|
||||
#define MLC_LLM_JSON_FFI_JSON_FFI_ENGINE_H_
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "../serve/threaded_engine.h"
|
||||
#include "../tokenizers/streamer.h"
|
||||
#include "conv_template.h"
|
||||
#include "openai_api_protocol.h"
|
||||
|
||||
namespace mlc {
|
||||
namespace llm {
|
||||
namespace json_ffi {
|
||||
|
||||
using namespace tvm::runtime;
|
||||
using namespace mlc::llm::serve;
|
||||
|
||||
/*!
|
||||
* \brief // Todo: document this class, fields and member functions
|
||||
*/
|
||||
class JSONFFIEngine {
|
||||
public:
|
||||
JSONFFIEngine();
|
||||
|
||||
~JSONFFIEngine();
|
||||
|
||||
bool ChatCompletion(std::string request_json_str, std::string request_id);
|
||||
|
||||
bool AddRequest(std::string request_json_str, std::string request_id);
|
||||
|
||||
void StreamBackError(std::string request_id);
|
||||
|
||||
bool Abort(std::string request_id);
|
||||
|
||||
std::string GetLastError();
|
||||
|
||||
void ExitBackgroundLoop();
|
||||
|
||||
protected:
|
||||
/*! \brief local request state entry, one per reply stream. */
|
||||
struct RequestState {
|
||||
/*! \brief model to fill in reply. */
|
||||
std::string model;
|
||||
/*! \brief text streamer for each stream */
|
||||
std::vector<TextStreamer> streamer;
|
||||
};
|
||||
|
||||
std::unique_ptr<ThreadedEngine> engine_;
|
||||
std::string err_;
|
||||
Function request_stream_callback_;
|
||||
// tokenizer
|
||||
Tokenizer tokenizer_;
|
||||
// conversation template
|
||||
Conversation conv_template_;
|
||||
// generation config
|
||||
GenerationConfig default_generation_config_;
|
||||
// model config
|
||||
ModelConfig model_config_;
|
||||
// local device
|
||||
DLDevice device_;
|
||||
// request state map
|
||||
std::unordered_map<String, RequestState> request_map_;
|
||||
};
|
||||
|
||||
} // namespace json_ffi
|
||||
} // namespace llm
|
||||
} // namespace mlc
|
||||
|
||||
#endif // MLC_LLM_JSON_FFI_JSON_FFI_ENGINE_H_
|
||||
@@ -0,0 +1,543 @@
|
||||
/*!
|
||||
* Copyright (c) 2023-2025 by Contributors
|
||||
* \file json_ffi/openai_api_protocol.cc
|
||||
* \brief The implementation of OpenAI API Protocol in MLC LLM.
|
||||
*/
|
||||
#include "openai_api_protocol.h"
|
||||
|
||||
#include "../support/json_parser.h"
|
||||
|
||||
namespace mlc {
|
||||
namespace llm {
|
||||
namespace json_ffi {
|
||||
|
||||
Result<ChatFunction> ChatFunction::FromJSON(const tvm::ffi::json::Object& json_obj) {
|
||||
using TResult = Result<ChatFunction>;
|
||||
ChatFunction chat_func;
|
||||
|
||||
// description
|
||||
Result<std::optional<std::string>> description_res =
|
||||
json::LookupOptionalWithResultReturn<std::string>(json_obj, "description");
|
||||
if (description_res.IsErr()) {
|
||||
return TResult::Error(description_res.UnwrapErr());
|
||||
}
|
||||
chat_func.description = description_res.Unwrap();
|
||||
|
||||
// name
|
||||
Result<std::string> name_res = json::LookupWithResultReturn<std::string>(json_obj, "name");
|
||||
if (name_res.IsErr()) {
|
||||
return TResult::Error(name_res.UnwrapErr());
|
||||
}
|
||||
chat_func.name = name_res.Unwrap();
|
||||
|
||||
// parameters
|
||||
Result<tvm::ffi::json::Object> parameters_obj_res =
|
||||
json::LookupWithResultReturn<tvm::ffi::json::Object>(json_obj, "parameters");
|
||||
if (parameters_obj_res.IsErr()) {
|
||||
return TResult::Error(parameters_obj_res.UnwrapErr());
|
||||
}
|
||||
tvm::ffi::json::Object parameters_obj = parameters_obj_res.Unwrap();
|
||||
chat_func.parameters.reserve(parameters_obj.size());
|
||||
for (const auto& [key, value] : parameters_obj) {
|
||||
chat_func.parameters[key.cast<tvm::ffi::String>()] = tvm::ffi::json::Stringify(value);
|
||||
}
|
||||
|
||||
return TResult::Ok(chat_func);
|
||||
}
|
||||
|
||||
tvm::ffi::json::Object ChatFunction::AsJSON() const {
|
||||
tvm::ffi::json::Object obj;
|
||||
if (this->description.has_value()) {
|
||||
obj.Set("description", this->description.value());
|
||||
}
|
||||
obj.Set("name", this->name);
|
||||
tvm::ffi::json::Object parameters_obj;
|
||||
for (const auto& pair : this->parameters) {
|
||||
parameters_obj.Set(pair.first, pair.second);
|
||||
}
|
||||
obj.Set("parameters", parameters_obj);
|
||||
return obj;
|
||||
}
|
||||
|
||||
Result<ChatTool> ChatTool::FromJSON(const tvm::ffi::json::Object& json_obj) {
|
||||
using TResult = Result<ChatTool>;
|
||||
ChatTool chatTool;
|
||||
|
||||
// function
|
||||
Result<tvm::ffi::json::Object> function_obj_res =
|
||||
json::LookupWithResultReturn<tvm::ffi::json::Object>(json_obj, "function");
|
||||
if (function_obj_res.IsErr()) {
|
||||
return TResult::Error(function_obj_res.UnwrapErr());
|
||||
}
|
||||
Result<ChatFunction> function = ChatFunction::FromJSON(function_obj_res.Unwrap());
|
||||
if (function.IsErr()) {
|
||||
return TResult::Error(function.UnwrapErr());
|
||||
}
|
||||
chatTool.function = function.Unwrap();
|
||||
|
||||
return TResult::Ok(chatTool);
|
||||
}
|
||||
|
||||
tvm::ffi::json::Object ChatTool::AsJSON() const {
|
||||
tvm::ffi::json::Object obj;
|
||||
obj.Set("type", "function");
|
||||
obj.Set("function", this->function.AsJSON());
|
||||
return obj;
|
||||
}
|
||||
|
||||
Result<ChatFunctionCall> ChatFunctionCall::FromJSON(const tvm::ffi::json::Object& json_obj) {
|
||||
using TResult = Result<ChatFunctionCall>;
|
||||
ChatFunctionCall chat_func_call;
|
||||
|
||||
// name
|
||||
Result<std::string> name_res = json::LookupWithResultReturn<std::string>(json_obj, "name");
|
||||
if (name_res.IsErr()) {
|
||||
return TResult::Error(name_res.UnwrapErr());
|
||||
}
|
||||
chat_func_call.name = name_res.Unwrap();
|
||||
|
||||
// arguments
|
||||
Result<std::optional<tvm::ffi::json::Object>> arguments_obj_res =
|
||||
json::LookupOptionalWithResultReturn<tvm::ffi::json::Object>(json_obj, "arguments");
|
||||
if (arguments_obj_res.IsErr()) {
|
||||
return TResult::Error(arguments_obj_res.UnwrapErr());
|
||||
}
|
||||
std::optional<tvm::ffi::json::Object> arguments_obj = arguments_obj_res.Unwrap();
|
||||
if (arguments_obj.has_value()) {
|
||||
std::unordered_map<std::string, std::string> arguments;
|
||||
arguments.reserve(arguments_obj.value().size());
|
||||
for (const auto& [key, value] : arguments_obj.value()) {
|
||||
arguments[key.cast<tvm::ffi::String>()] = tvm::ffi::json::Stringify(value);
|
||||
}
|
||||
chat_func_call.arguments = std::move(arguments);
|
||||
}
|
||||
|
||||
return TResult::Ok(chat_func_call);
|
||||
}
|
||||
|
||||
tvm::ffi::json::Object ChatFunctionCall::AsJSON() const {
|
||||
tvm::ffi::json::Object obj;
|
||||
tvm::ffi::json::Object arguments_obj;
|
||||
if (this->arguments.has_value()) {
|
||||
for (const auto& pair : this->arguments.value()) {
|
||||
arguments_obj.Set(pair.first, pair.second);
|
||||
}
|
||||
obj.Set("arguments", arguments_obj);
|
||||
}
|
||||
|
||||
obj.Set("name", this->name);
|
||||
return obj;
|
||||
}
|
||||
|
||||
Result<ChatToolCall> ChatToolCall::FromJSON(const tvm::ffi::json::Object& json_obj) {
|
||||
using TResult = Result<ChatToolCall>;
|
||||
ChatToolCall chat_tool_call;
|
||||
|
||||
// function
|
||||
Result<tvm::ffi::json::Object> function_obj_res =
|
||||
json::LookupWithResultReturn<tvm::ffi::json::Object>(json_obj, "function");
|
||||
if (function_obj_res.IsErr()) {
|
||||
return TResult::Error(function_obj_res.UnwrapErr());
|
||||
}
|
||||
Result<ChatFunctionCall> function_res = ChatFunctionCall::FromJSON(function_obj_res.Unwrap());
|
||||
if (function_res.IsErr()) {
|
||||
return TResult::Error(function_res.UnwrapErr());
|
||||
}
|
||||
chat_tool_call.function = function_res.Unwrap();
|
||||
|
||||
// overwrite default id
|
||||
Result<std::optional<std::string>> id_res =
|
||||
json::LookupOptionalWithResultReturn<std::string>(json_obj, "id");
|
||||
if (id_res.IsErr()) {
|
||||
return TResult::Error(id_res.UnwrapErr());
|
||||
}
|
||||
std::optional<std::string> id = id_res.UnwrapErr();
|
||||
if (id.has_value()) {
|
||||
chat_tool_call.id = id.value();
|
||||
}
|
||||
|
||||
return TResult::Ok(chat_tool_call);
|
||||
}
|
||||
|
||||
tvm::ffi::json::Object ChatToolCall::AsJSON() const {
|
||||
tvm::ffi::json::Object obj;
|
||||
obj.Set("id", this->id);
|
||||
obj.Set("function", this->function.AsJSON());
|
||||
obj.Set("type", "function");
|
||||
return obj;
|
||||
}
|
||||
|
||||
Result<ChatCompletionMessage> ChatCompletionMessage::FromJSON(
|
||||
const tvm::ffi::json::Object& json_obj) {
|
||||
using TResult = Result<ChatCompletionMessage>;
|
||||
ChatCompletionMessage message;
|
||||
ChatCompletionMessageContent content;
|
||||
|
||||
// content
|
||||
if (json_obj.count("content") == 0) {
|
||||
return TResult::Error("ValueError: key \"content\" not found in the chat completion.");
|
||||
}
|
||||
tvm::ffi::json::Value content_val = json_obj.at("content");
|
||||
if (content_val.try_cast<std::string>().has_value()) {
|
||||
content = content_val.cast<std::string>();
|
||||
} else if (content_val == nullptr) {
|
||||
// skip
|
||||
} else {
|
||||
// most complicated case
|
||||
std::vector<std::unordered_map<std::string, std::string>> parts;
|
||||
Result<tvm::ffi::json::Array> content_arr_res =
|
||||
json::LookupWithResultReturn<tvm::ffi::json::Array>(json_obj, "content");
|
||||
if (content_arr_res.IsErr()) {
|
||||
return TResult::Error(content_arr_res.UnwrapErr());
|
||||
}
|
||||
tvm::ffi::json::Array content_arr = content_arr_res.Unwrap();
|
||||
for (const auto& item : content_arr) {
|
||||
if (!item.try_cast<tvm::ffi::json::Object>().has_value()) {
|
||||
return TResult::Error("The content of chat completion message is not an object");
|
||||
}
|
||||
tvm::ffi::json::Object item_obj = item.cast<tvm::ffi::json::Object>();
|
||||
std::unordered_map<std::string, std::string> item_map;
|
||||
for (const auto& [key, value] : item_obj) {
|
||||
item_map[key.cast<tvm::ffi::String>()] = tvm::ffi::json::Stringify(value);
|
||||
}
|
||||
parts.push_back(std::move(item_map));
|
||||
}
|
||||
content = parts;
|
||||
}
|
||||
message.content = content;
|
||||
|
||||
// role
|
||||
Result<std::string> role_str_res = json::LookupWithResultReturn<std::string>(json_obj, "role");
|
||||
if (role_str_res.IsErr()) {
|
||||
return TResult::Error(role_str_res.UnwrapErr());
|
||||
}
|
||||
std::string role_str = role_str_res.Unwrap();
|
||||
if (role_str == "system" || role_str == "user" || role_str == "assistant" || role_str == "tool") {
|
||||
message.role = role_str;
|
||||
} else {
|
||||
return TResult::Error("Invalid role in chat completion message: " + role_str);
|
||||
}
|
||||
|
||||
// name
|
||||
Result<std::optional<std::string>> name_res =
|
||||
json::LookupOptionalWithResultReturn<std::string>(json_obj, "name");
|
||||
if (name_res.IsErr()) {
|
||||
return TResult::Error(name_res.UnwrapErr());
|
||||
}
|
||||
message.name = name_res.Unwrap();
|
||||
|
||||
// tool calls
|
||||
Result<std::optional<tvm::ffi::json::Array>> tool_calls_arr_res =
|
||||
json::LookupOptionalWithResultReturn<tvm::ffi::json::Array>(json_obj, "tool_calls");
|
||||
if (tool_calls_arr_res.IsErr()) {
|
||||
return TResult::Error(tool_calls_arr_res.UnwrapErr());
|
||||
}
|
||||
std::optional<tvm::ffi::json::Array> tool_calls_arr = tool_calls_arr_res.Unwrap();
|
||||
if (tool_calls_arr.has_value()) {
|
||||
std::vector<ChatToolCall> tool_calls;
|
||||
tool_calls.reserve(tool_calls_arr.value().size());
|
||||
for (const auto& item : tool_calls_arr.value()) {
|
||||
if (!item.try_cast<tvm::ffi::json::Object>().has_value()) {
|
||||
return TResult::Error("A tool call item in the chat completion message is not an object");
|
||||
}
|
||||
Result<ChatToolCall> tool_call = ChatToolCall::FromJSON(item.cast<tvm::ffi::json::Object>());
|
||||
if (tool_call.IsErr()) {
|
||||
return TResult::Error(tool_call.UnwrapErr());
|
||||
}
|
||||
tool_calls.push_back(tool_call.Unwrap());
|
||||
}
|
||||
message.tool_calls = tool_calls;
|
||||
}
|
||||
|
||||
// tool call id
|
||||
Result<std::optional<std::string>> tool_call_id_res =
|
||||
json::LookupOptionalWithResultReturn<std::string>(json_obj, "tool_call_id");
|
||||
if (tool_call_id_res.IsErr()) {
|
||||
return TResult::Error(tool_call_id_res.UnwrapErr());
|
||||
}
|
||||
message.tool_call_id = tool_call_id_res.Unwrap();
|
||||
|
||||
return TResult::Ok(message);
|
||||
}
|
||||
|
||||
Result<ChatCompletionRequest> ChatCompletionRequest::FromJSON(const std::string& json_str) {
|
||||
using TResult = Result<ChatCompletionRequest>;
|
||||
Result<tvm::ffi::json::Object> json_obj_res = json::ParseToJSONObjectWithResultReturn(json_str);
|
||||
if (json_obj_res.IsErr()) {
|
||||
return TResult::Error(json_obj_res.UnwrapErr());
|
||||
}
|
||||
tvm::ffi::json::Object json_obj = json_obj_res.Unwrap();
|
||||
ChatCompletionRequest request;
|
||||
|
||||
// messages
|
||||
Result<tvm::ffi::json::Array> messages_arr_res =
|
||||
json::LookupWithResultReturn<tvm::ffi::json::Array>(json_obj, "messages");
|
||||
if (messages_arr_res.IsErr()) {
|
||||
return TResult::Error(messages_arr_res.UnwrapErr());
|
||||
}
|
||||
std::vector<ChatCompletionMessage> messages;
|
||||
tvm::ffi::json::Array messages_arr = messages_arr_res.Unwrap();
|
||||
for (const auto& item : messages_arr) {
|
||||
if (!item.try_cast<tvm::ffi::json::Object>().has_value()) {
|
||||
return TResult::Error("A message in chat completion request is not object");
|
||||
}
|
||||
tvm::ffi::json::Object item_obj = item.cast<tvm::ffi::json::Object>();
|
||||
Result<ChatCompletionMessage> message = ChatCompletionMessage::FromJSON(item_obj);
|
||||
if (message.IsErr()) {
|
||||
return TResult::Error(message.UnwrapErr());
|
||||
}
|
||||
messages.push_back(message.Unwrap());
|
||||
}
|
||||
request.messages = messages;
|
||||
|
||||
// model
|
||||
Result<std::optional<std::string>> model_res =
|
||||
json::LookupOptionalWithResultReturn<std::string>(json_obj, "model");
|
||||
if (model_res.IsErr()) {
|
||||
return TResult::Error(model_res.UnwrapErr());
|
||||
}
|
||||
request.model = model_res.Unwrap();
|
||||
|
||||
// temperature
|
||||
Result<std::optional<double>> temperature_res =
|
||||
json::LookupOptionalWithResultReturn<double>(json_obj, "temperature");
|
||||
if (temperature_res.IsErr()) {
|
||||
return TResult::Error(temperature_res.UnwrapErr());
|
||||
}
|
||||
request.temperature = temperature_res.Unwrap();
|
||||
// top_p
|
||||
Result<std::optional<double>> top_p_res =
|
||||
json::LookupOptionalWithResultReturn<double>(json_obj, "top_p");
|
||||
if (top_p_res.IsErr()) {
|
||||
return TResult::Error(top_p_res.UnwrapErr());
|
||||
}
|
||||
request.top_p = top_p_res.Unwrap();
|
||||
// max_tokens
|
||||
Result<std::optional<int64_t>> max_tokens_res =
|
||||
json::LookupOptionalWithResultReturn<int64_t>(json_obj, "max_tokens");
|
||||
if (max_tokens_res.IsErr()) {
|
||||
return TResult::Error(max_tokens_res.UnwrapErr());
|
||||
}
|
||||
request.max_tokens = max_tokens_res.Unwrap();
|
||||
// n
|
||||
Result<int64_t> n_res = json::LookupOrDefaultWithResultReturn<int64_t>(json_obj, "n", 1);
|
||||
if (n_res.IsErr()) {
|
||||
return TResult::Error(n_res.UnwrapErr());
|
||||
}
|
||||
request.n = n_res.Unwrap();
|
||||
// frequency_penalty
|
||||
Result<std::optional<double>> frequency_penalty_res =
|
||||
json::LookupOptionalWithResultReturn<double>(json_obj, "frequency_penalty");
|
||||
if (frequency_penalty_res.IsErr()) {
|
||||
return TResult::Error(frequency_penalty_res.UnwrapErr());
|
||||
}
|
||||
request.frequency_penalty = frequency_penalty_res.Unwrap();
|
||||
// presence_penalty
|
||||
Result<std::optional<double>> presence_penalty_res =
|
||||
json::LookupOptionalWithResultReturn<double>(json_obj, "presence_penalty");
|
||||
if (presence_penalty_res.IsErr()) {
|
||||
return TResult::Error(presence_penalty_res.UnwrapErr());
|
||||
}
|
||||
request.presence_penalty = presence_penalty_res.Unwrap();
|
||||
// seed
|
||||
Result<std::optional<int64_t>> seed_res =
|
||||
json::LookupOptionalWithResultReturn<int64_t>(json_obj, "seed");
|
||||
if (seed_res.IsErr()) {
|
||||
return TResult::Error(seed_res.UnwrapErr());
|
||||
}
|
||||
request.seed = seed_res.Unwrap();
|
||||
|
||||
// stop strings
|
||||
Result<std::optional<tvm::ffi::json::Array>> stop_strs_res =
|
||||
json::LookupOptionalWithResultReturn<tvm::ffi::json::Array>(json_obj, "stop");
|
||||
if (stop_strs_res.IsErr()) {
|
||||
return TResult::Error(stop_strs_res.UnwrapErr());
|
||||
}
|
||||
std::optional<tvm::ffi::json::Array> stop_strs = stop_strs_res.Unwrap();
|
||||
if (stop_strs.has_value()) {
|
||||
std::vector<std::string> stop;
|
||||
for (const auto& stop_str_value : stop_strs.value()) {
|
||||
if (!stop_str_value.try_cast<std::string>().has_value()) {
|
||||
return TResult::Error("One given value in field \"stop\" is not a string.");
|
||||
}
|
||||
stop.push_back(stop_str_value.cast<std::string>());
|
||||
}
|
||||
request.stop = std::move(stop);
|
||||
}
|
||||
|
||||
// tool_choice
|
||||
Result<std::string> tool_choice_res =
|
||||
json::LookupOrDefaultWithResultReturn<std::string>(json_obj, "tool_choice", "auto");
|
||||
if (tool_choice_res.IsErr()) {
|
||||
return TResult::Error(tool_choice_res.UnwrapErr());
|
||||
}
|
||||
request.tool_choice = tool_choice_res.Unwrap();
|
||||
|
||||
// tools
|
||||
Result<std::optional<tvm::ffi::json::Array>> tools_arr_res =
|
||||
json::LookupOptionalWithResultReturn<tvm::ffi::json::Array>(json_obj, "tools");
|
||||
if (tool_choice_res.IsErr()) {
|
||||
return TResult::Error(tool_choice_res.UnwrapErr());
|
||||
}
|
||||
std::optional<tvm::ffi::json::Array> tools_arr = tools_arr_res.Unwrap();
|
||||
if (tools_arr.has_value()) {
|
||||
std::vector<ChatTool> tools;
|
||||
tools.reserve(tools_arr.value().size());
|
||||
for (const auto& item : tools_arr.value()) {
|
||||
if (!item.try_cast<tvm::ffi::json::Object>().has_value()) {
|
||||
return TResult::Error("A tool of the chat completion request is not an object");
|
||||
}
|
||||
Result<ChatTool> tool = ChatTool::FromJSON(item.cast<tvm::ffi::json::Object>());
|
||||
if (tool.IsErr()) {
|
||||
return TResult::Error(tool.UnwrapErr());
|
||||
}
|
||||
tools.push_back(tool.Unwrap());
|
||||
}
|
||||
request.tools = tools;
|
||||
}
|
||||
|
||||
// response format
|
||||
std::optional<tvm::ffi::json::Object> response_format_obj =
|
||||
json::LookupOptional<tvm::ffi::json::Object>(json_obj, "response_format");
|
||||
if (response_format_obj.has_value()) {
|
||||
Result<ResponseFormat> response_format_res =
|
||||
ResponseFormat::FromJSON(response_format_obj.value());
|
||||
if (response_format_res.IsErr()) {
|
||||
return TResult::Error(response_format_res.UnwrapErr());
|
||||
}
|
||||
request.response_format = response_format_res.Unwrap();
|
||||
}
|
||||
|
||||
// debug_config
|
||||
Result<std::optional<tvm::ffi::json::Object>> debug_config_opt_res =
|
||||
json::LookupOptionalWithResultReturn<tvm::ffi::json::Object>(json_obj, "debug_config");
|
||||
if (debug_config_opt_res.IsErr()) {
|
||||
return TResult::Error(debug_config_opt_res.UnwrapErr());
|
||||
}
|
||||
auto debug_config_opt = debug_config_opt_res.Unwrap();
|
||||
if (debug_config_opt.has_value()) {
|
||||
Result<DebugConfig> debug_config_res = DebugConfig::FromJSON(debug_config_opt.value());
|
||||
if (debug_config_res.IsErr()) {
|
||||
return TResult::Error(debug_config_res.UnwrapErr());
|
||||
}
|
||||
request.debug_config = debug_config_res.Unwrap();
|
||||
}
|
||||
|
||||
// TODO: Other parameters
|
||||
return TResult::Ok(request);
|
||||
}
|
||||
|
||||
tvm::ffi::json::Object ChatCompletionMessage::AsJSON() const {
|
||||
tvm::ffi::json::Object obj;
|
||||
|
||||
if (this->content.IsText()) {
|
||||
obj.Set("content", this->content.Text());
|
||||
} else if (this->content.IsParts()) {
|
||||
tvm::ffi::json::Array content_arr;
|
||||
for (const auto& item : this->content.Parts()) {
|
||||
tvm::ffi::json::Object item_obj;
|
||||
for (const auto& pair : item) {
|
||||
item_obj.Set(pair.first, pair.second);
|
||||
}
|
||||
content_arr.push_back(item_obj);
|
||||
}
|
||||
obj.Set("content", content_arr);
|
||||
}
|
||||
|
||||
obj.Set("role", this->role);
|
||||
|
||||
if (this->name.has_value()) {
|
||||
obj.Set("name", this->name.value());
|
||||
}
|
||||
if (this->tool_call_id.has_value()) {
|
||||
obj.Set("tool_call_id", this->tool_call_id.value());
|
||||
}
|
||||
if (this->tool_calls.has_value()) {
|
||||
tvm::ffi::json::Array tool_calls_arr;
|
||||
for (const auto& tool_call : this->tool_calls.value()) {
|
||||
tool_calls_arr.push_back(tool_call.AsJSON());
|
||||
}
|
||||
obj.Set("tool_calls", tool_calls_arr);
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
tvm::ffi::json::Object ChatCompletionResponseChoice::AsJSON() const {
|
||||
tvm::ffi::json::Object obj;
|
||||
if (!this->finish_reason.has_value()) {
|
||||
obj.Set("finish_reason", nullptr);
|
||||
} else {
|
||||
if (this->finish_reason == FinishReason::stop) {
|
||||
obj.Set("finish_reason", "stop");
|
||||
} else if (this->finish_reason == FinishReason::length) {
|
||||
obj.Set("finish_reason", "length");
|
||||
} else if (this->finish_reason == FinishReason::tool_calls) {
|
||||
obj.Set("finish_reason", "tool_calls");
|
||||
} else if (this->finish_reason == FinishReason::error) {
|
||||
obj.Set("finish_reason", "error");
|
||||
}
|
||||
}
|
||||
obj.Set("index", static_cast<int64_t>(this->index));
|
||||
obj.Set("message", this->message.AsJSON());
|
||||
return obj;
|
||||
}
|
||||
|
||||
tvm::ffi::json::Object ChatCompletionStreamResponseChoice::AsJSON() const {
|
||||
tvm::ffi::json::Object obj;
|
||||
if (!this->finish_reason.has_value()) {
|
||||
obj.Set("finish_reason", nullptr);
|
||||
} else {
|
||||
if (this->finish_reason.value() == FinishReason::stop) {
|
||||
obj.Set("finish_reason", "stop");
|
||||
} else if (this->finish_reason.value() == FinishReason::length) {
|
||||
obj.Set("finish_reason", "length");
|
||||
} else if (this->finish_reason.value() == FinishReason::tool_calls) {
|
||||
obj.Set("finish_reason", "tool_calls");
|
||||
} else if (this->finish_reason.value() == FinishReason::error) {
|
||||
obj.Set("finish_reason", "error");
|
||||
}
|
||||
}
|
||||
|
||||
obj.Set("index", static_cast<int64_t>(this->index));
|
||||
obj.Set("delta", this->delta.AsJSON());
|
||||
return obj;
|
||||
}
|
||||
|
||||
tvm::ffi::json::Object ChatCompletionResponse::AsJSON() const {
|
||||
tvm::ffi::json::Object obj;
|
||||
obj.Set("id", this->id);
|
||||
tvm::ffi::json::Array choices_arr;
|
||||
for (const auto& choice : this->choices) {
|
||||
choices_arr.push_back(choice.AsJSON());
|
||||
}
|
||||
obj.Set("choices", choices_arr);
|
||||
obj.Set("created", static_cast<int64_t>(this->created));
|
||||
obj.Set("model", this->model);
|
||||
obj.Set("system_fingerprint", this->system_fingerprint);
|
||||
obj.Set("object", this->object);
|
||||
return obj;
|
||||
}
|
||||
|
||||
tvm::ffi::json::Object ChatCompletionStreamResponse::AsJSON() const {
|
||||
tvm::ffi::json::Object obj;
|
||||
obj.Set("id", this->id);
|
||||
|
||||
tvm::ffi::json::Array choices_arr;
|
||||
for (const auto& choice : this->choices) {
|
||||
choices_arr.push_back(choice.AsJSON());
|
||||
}
|
||||
obj.Set("choices", choices_arr);
|
||||
|
||||
obj.Set("created", static_cast<int64_t>(this->created));
|
||||
obj.Set("model", this->model);
|
||||
obj.Set("system_fingerprint", this->system_fingerprint);
|
||||
obj.Set("object", this->object);
|
||||
if (usage.has_value()) {
|
||||
obj.Set("usage", usage.value());
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
} // namespace json_ffi
|
||||
} // namespace llm
|
||||
} // namespace mlc
|
||||
@@ -0,0 +1,208 @@
|
||||
/*!
|
||||
* Copyright (c) 2023-2025 by Contributors
|
||||
* \file json_ffi/openai_api_protocol.h
|
||||
* \brief The header of OpenAI API Protocol in MLC LLM.
|
||||
*/
|
||||
#ifndef MLC_LLM_JSON_FFI_OPENAI_API_PROTOCOL_H
|
||||
#define MLC_LLM_JSON_FFI_OPENAI_API_PROTOCOL_H
|
||||
|
||||
#include <tvm/ffi/extra/json.h>
|
||||
|
||||
#include <ctime>
|
||||
#include <optional>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "../serve/config.h"
|
||||
#include "../support/result.h"
|
||||
|
||||
namespace mlc {
|
||||
namespace llm {
|
||||
namespace json_ffi {
|
||||
|
||||
using serve::DebugConfig;
|
||||
using serve::ResponseFormat;
|
||||
|
||||
enum class Type { text, json_object, function };
|
||||
enum class FinishReason { stop, length, tool_calls, error };
|
||||
|
||||
inline std::string GenerateUUID(size_t length) {
|
||||
auto randchar = []() -> char {
|
||||
const char charset[] =
|
||||
"0123456789"
|
||||
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||
"abcdefghijklmnopqrstuvwxyz";
|
||||
const size_t max_index = (sizeof(charset) - 1);
|
||||
return charset[rand() % max_index];
|
||||
};
|
||||
std::string str(length, 0);
|
||||
std::generate_n(str.begin(), length, randchar);
|
||||
return str;
|
||||
}
|
||||
|
||||
class ChatFunction {
|
||||
public:
|
||||
std::optional<std::string> description = std::nullopt;
|
||||
std::string name;
|
||||
// Todo: change to std::vector<std::pair<std::string, std::string>>?
|
||||
std::unordered_map<std::string, std::string>
|
||||
parameters; // Assuming parameters are string key-value pairs
|
||||
|
||||
static Result<ChatFunction> FromJSON(const tvm::ffi::json::Object& json);
|
||||
tvm::ffi::json::Object AsJSON() const;
|
||||
};
|
||||
|
||||
class ChatTool {
|
||||
public:
|
||||
Type type = Type::function;
|
||||
ChatFunction function;
|
||||
|
||||
static Result<ChatTool> FromJSON(const tvm::ffi::json::Object& json);
|
||||
tvm::ffi::json::Object AsJSON() const;
|
||||
};
|
||||
|
||||
class ChatFunctionCall {
|
||||
public:
|
||||
std::string name;
|
||||
std::optional<std::unordered_map<std::string, std::string>> arguments =
|
||||
std::nullopt; // Assuming arguments are string key-value pairs
|
||||
|
||||
static Result<ChatFunctionCall> FromJSON(const tvm::ffi::json::Object& json);
|
||||
tvm::ffi::json::Object AsJSON() const;
|
||||
};
|
||||
|
||||
class ChatToolCall {
|
||||
public:
|
||||
std::string id = "call_" + GenerateUUID(8);
|
||||
Type type = Type::function;
|
||||
ChatFunctionCall function;
|
||||
|
||||
static Result<ChatToolCall> FromJSON(const tvm::ffi::json::Object& json);
|
||||
tvm::ffi::json::Object AsJSON() const;
|
||||
};
|
||||
|
||||
class ChatCompletionMessageContent {
|
||||
public:
|
||||
ChatCompletionMessageContent() = default;
|
||||
|
||||
ChatCompletionMessageContent(std::nullopt_t) {} // NOLINT(*)
|
||||
|
||||
ChatCompletionMessageContent(std::string text) : text_(text) {} // NOLINT(*)
|
||||
|
||||
ChatCompletionMessageContent(
|
||||
std::vector<std::unordered_map<std::string, std::string>> parts) // NOLINT(*)
|
||||
: parts_(parts) {}
|
||||
|
||||
bool IsNull() const { return !IsText() && !IsParts(); }
|
||||
|
||||
bool IsText() const { return text_.operator bool(); }
|
||||
|
||||
bool IsParts() const { return parts_.operator bool(); }
|
||||
|
||||
const std::string& Text() const { return text_.value(); }
|
||||
|
||||
const std::vector<std::unordered_map<std::string, std::string>>& Parts() const {
|
||||
return parts_.value();
|
||||
}
|
||||
|
||||
private:
|
||||
/*! \brief used to store text content */
|
||||
std::optional<std::string> text_;
|
||||
std::optional<std::vector<std::unordered_map<std::string, std::string>>> parts_;
|
||||
};
|
||||
|
||||
class ChatCompletionMessage {
|
||||
public:
|
||||
ChatCompletionMessageContent content =
|
||||
std::nullopt; // Assuming content is a list of string key-value pairs
|
||||
std::string role;
|
||||
std::optional<std::string> name = std::nullopt;
|
||||
std::optional<std::vector<ChatToolCall>> tool_calls = std::nullopt;
|
||||
std::optional<std::string> tool_call_id = std::nullopt;
|
||||
|
||||
static Result<ChatCompletionMessage> FromJSON(const tvm::ffi::json::Object& json);
|
||||
tvm::ffi::json::Object AsJSON() const;
|
||||
};
|
||||
|
||||
class ChatCompletionRequest {
|
||||
public:
|
||||
std::vector<ChatCompletionMessage> messages;
|
||||
std::optional<std::string> model = std::nullopt;
|
||||
std::optional<double> frequency_penalty = std::nullopt;
|
||||
std::optional<double> presence_penalty = std::nullopt;
|
||||
bool logprobs = false;
|
||||
int top_logprobs = 0;
|
||||
std::optional<std::vector<std::pair<int, float>>> logit_bias = std::nullopt;
|
||||
std::optional<int> max_tokens = std::nullopt;
|
||||
int n = 1;
|
||||
std::optional<int> seed = std::nullopt;
|
||||
std::optional<std::vector<std::string>> stop = std::nullopt;
|
||||
bool stream = false;
|
||||
std::optional<double> temperature = std::nullopt;
|
||||
std::optional<double> top_p = std::nullopt;
|
||||
std::optional<std::vector<ChatTool>> tools = std::nullopt;
|
||||
std::optional<std::string> tool_choice = std::nullopt;
|
||||
std::optional<std::string> user = std::nullopt;
|
||||
bool ignore_eos = false;
|
||||
std::optional<ResponseFormat> response_format = std::nullopt;
|
||||
std::optional<DebugConfig> debug_config = std::nullopt;
|
||||
|
||||
/*! \brief Parse and create a ChatCompletionRequest instance from the given JSON string. */
|
||||
static Result<ChatCompletionRequest> FromJSON(const std::string& json_str);
|
||||
|
||||
// TODO: check_penalty_range, check_logit_bias, check_logprobs
|
||||
};
|
||||
|
||||
class ChatCompletionResponseChoice {
|
||||
public:
|
||||
std::optional<FinishReason> finish_reason;
|
||||
int index = 0;
|
||||
ChatCompletionMessage message;
|
||||
// TODO: logprobs
|
||||
|
||||
tvm::ffi::json::Object AsJSON() const;
|
||||
};
|
||||
|
||||
class ChatCompletionStreamResponseChoice {
|
||||
public:
|
||||
std::optional<FinishReason> finish_reason;
|
||||
int index = 0;
|
||||
ChatCompletionMessage delta;
|
||||
// TODO: logprobs
|
||||
|
||||
tvm::ffi::json::Object AsJSON() const;
|
||||
};
|
||||
|
||||
class ChatCompletionResponse {
|
||||
public:
|
||||
std::string id;
|
||||
std::vector<ChatCompletionResponseChoice> choices;
|
||||
int created = static_cast<int>(std::time(nullptr));
|
||||
std::string model;
|
||||
std::string system_fingerprint;
|
||||
std::string object = "chat.completion";
|
||||
// TODO: usage_info
|
||||
|
||||
tvm::ffi::json::Object AsJSON() const;
|
||||
};
|
||||
|
||||
class ChatCompletionStreamResponse {
|
||||
public:
|
||||
std::string id;
|
||||
std::vector<ChatCompletionStreamResponseChoice> choices;
|
||||
int created = static_cast<int>(std::time(nullptr));
|
||||
std::string model;
|
||||
std::string system_fingerprint;
|
||||
std::string object = "chat.completion.chunk";
|
||||
std::optional<tvm::ffi::json::Value> usage;
|
||||
|
||||
tvm::ffi::json::Object AsJSON() const;
|
||||
};
|
||||
|
||||
} // namespace json_ffi
|
||||
} // namespace llm
|
||||
} // namespace mlc
|
||||
|
||||
#endif // MLC_LLM_JSON_FFI_OPENAI_API_PROTOCOL_H
|
||||
Reference in New Issue
Block a user