/*! * Copyright (c) 2023-2025 by Contributors * \file json_ffi/openai_api_protocol.cc * \brief The implementation of OpenAI API Protocol in MLC LLM. */ #include "openai_api_protocol.h" #include "../support/json_parser.h" namespace mlc { namespace llm { namespace json_ffi { Result ChatFunction::FromJSON(const tvm::ffi::json::Object& json_obj) { using TResult = Result; ChatFunction chat_func; // description Result> description_res = json::LookupOptionalWithResultReturn(json_obj, "description"); if (description_res.IsErr()) { return TResult::Error(description_res.UnwrapErr()); } chat_func.description = description_res.Unwrap(); // name Result name_res = json::LookupWithResultReturn(json_obj, "name"); if (name_res.IsErr()) { return TResult::Error(name_res.UnwrapErr()); } chat_func.name = name_res.Unwrap(); // parameters Result parameters_obj_res = json::LookupWithResultReturn(json_obj, "parameters"); if (parameters_obj_res.IsErr()) { return TResult::Error(parameters_obj_res.UnwrapErr()); } tvm::ffi::json::Object parameters_obj = parameters_obj_res.Unwrap(); chat_func.parameters.reserve(parameters_obj.size()); for (const auto& [key, value] : parameters_obj) { chat_func.parameters[key.cast()] = tvm::ffi::json::Stringify(value); } return TResult::Ok(chat_func); } tvm::ffi::json::Object ChatFunction::AsJSON() const { tvm::ffi::json::Object obj; if (this->description.has_value()) { obj.Set("description", this->description.value()); } obj.Set("name", this->name); tvm::ffi::json::Object parameters_obj; for (const auto& pair : this->parameters) { parameters_obj.Set(pair.first, pair.second); } obj.Set("parameters", parameters_obj); return obj; } Result ChatTool::FromJSON(const tvm::ffi::json::Object& json_obj) { using TResult = Result; ChatTool chatTool; // function Result function_obj_res = json::LookupWithResultReturn(json_obj, "function"); if (function_obj_res.IsErr()) { return TResult::Error(function_obj_res.UnwrapErr()); } Result function = ChatFunction::FromJSON(function_obj_res.Unwrap()); if (function.IsErr()) { return TResult::Error(function.UnwrapErr()); } chatTool.function = function.Unwrap(); return TResult::Ok(chatTool); } tvm::ffi::json::Object ChatTool::AsJSON() const { tvm::ffi::json::Object obj; obj.Set("type", "function"); obj.Set("function", this->function.AsJSON()); return obj; } Result ChatFunctionCall::FromJSON(const tvm::ffi::json::Object& json_obj) { using TResult = Result; ChatFunctionCall chat_func_call; // name Result name_res = json::LookupWithResultReturn(json_obj, "name"); if (name_res.IsErr()) { return TResult::Error(name_res.UnwrapErr()); } chat_func_call.name = name_res.Unwrap(); // arguments Result> arguments_obj_res = json::LookupOptionalWithResultReturn(json_obj, "arguments"); if (arguments_obj_res.IsErr()) { return TResult::Error(arguments_obj_res.UnwrapErr()); } std::optional arguments_obj = arguments_obj_res.Unwrap(); if (arguments_obj.has_value()) { std::unordered_map arguments; arguments.reserve(arguments_obj.value().size()); for (const auto& [key, value] : arguments_obj.value()) { arguments[key.cast()] = tvm::ffi::json::Stringify(value); } chat_func_call.arguments = std::move(arguments); } return TResult::Ok(chat_func_call); } tvm::ffi::json::Object ChatFunctionCall::AsJSON() const { tvm::ffi::json::Object obj; tvm::ffi::json::Object arguments_obj; if (this->arguments.has_value()) { for (const auto& pair : this->arguments.value()) { arguments_obj.Set(pair.first, pair.second); } obj.Set("arguments", arguments_obj); } obj.Set("name", this->name); return obj; } Result ChatToolCall::FromJSON(const tvm::ffi::json::Object& json_obj) { using TResult = Result; ChatToolCall chat_tool_call; // function Result function_obj_res = json::LookupWithResultReturn(json_obj, "function"); if (function_obj_res.IsErr()) { return TResult::Error(function_obj_res.UnwrapErr()); } Result function_res = ChatFunctionCall::FromJSON(function_obj_res.Unwrap()); if (function_res.IsErr()) { return TResult::Error(function_res.UnwrapErr()); } chat_tool_call.function = function_res.Unwrap(); // overwrite default id Result> id_res = json::LookupOptionalWithResultReturn(json_obj, "id"); if (id_res.IsErr()) { return TResult::Error(id_res.UnwrapErr()); } std::optional id = id_res.UnwrapErr(); if (id.has_value()) { chat_tool_call.id = id.value(); } return TResult::Ok(chat_tool_call); } tvm::ffi::json::Object ChatToolCall::AsJSON() const { tvm::ffi::json::Object obj; obj.Set("id", this->id); obj.Set("function", this->function.AsJSON()); obj.Set("type", "function"); return obj; } Result ChatCompletionMessage::FromJSON( const tvm::ffi::json::Object& json_obj) { using TResult = Result; ChatCompletionMessage message; ChatCompletionMessageContent content; // content if (json_obj.count("content") == 0) { return TResult::Error("ValueError: key \"content\" not found in the chat completion."); } tvm::ffi::json::Value content_val = json_obj.at("content"); if (content_val.try_cast().has_value()) { content = content_val.cast(); } else if (content_val == nullptr) { // skip } else { // most complicated case std::vector> parts; Result content_arr_res = json::LookupWithResultReturn(json_obj, "content"); if (content_arr_res.IsErr()) { return TResult::Error(content_arr_res.UnwrapErr()); } tvm::ffi::json::Array content_arr = content_arr_res.Unwrap(); for (const auto& item : content_arr) { if (!item.try_cast().has_value()) { return TResult::Error("The content of chat completion message is not an object"); } tvm::ffi::json::Object item_obj = item.cast(); std::unordered_map item_map; for (const auto& [key, value] : item_obj) { item_map[key.cast()] = tvm::ffi::json::Stringify(value); } parts.push_back(std::move(item_map)); } content = parts; } message.content = content; // role Result role_str_res = json::LookupWithResultReturn(json_obj, "role"); if (role_str_res.IsErr()) { return TResult::Error(role_str_res.UnwrapErr()); } std::string role_str = role_str_res.Unwrap(); if (role_str == "system" || role_str == "user" || role_str == "assistant" || role_str == "tool") { message.role = role_str; } else { return TResult::Error("Invalid role in chat completion message: " + role_str); } // name Result> name_res = json::LookupOptionalWithResultReturn(json_obj, "name"); if (name_res.IsErr()) { return TResult::Error(name_res.UnwrapErr()); } message.name = name_res.Unwrap(); // tool calls Result> tool_calls_arr_res = json::LookupOptionalWithResultReturn(json_obj, "tool_calls"); if (tool_calls_arr_res.IsErr()) { return TResult::Error(tool_calls_arr_res.UnwrapErr()); } std::optional tool_calls_arr = tool_calls_arr_res.Unwrap(); if (tool_calls_arr.has_value()) { std::vector tool_calls; tool_calls.reserve(tool_calls_arr.value().size()); for (const auto& item : tool_calls_arr.value()) { if (!item.try_cast().has_value()) { return TResult::Error("A tool call item in the chat completion message is not an object"); } Result tool_call = ChatToolCall::FromJSON(item.cast()); if (tool_call.IsErr()) { return TResult::Error(tool_call.UnwrapErr()); } tool_calls.push_back(tool_call.Unwrap()); } message.tool_calls = tool_calls; } // tool call id Result> tool_call_id_res = json::LookupOptionalWithResultReturn(json_obj, "tool_call_id"); if (tool_call_id_res.IsErr()) { return TResult::Error(tool_call_id_res.UnwrapErr()); } message.tool_call_id = tool_call_id_res.Unwrap(); return TResult::Ok(message); } Result ChatCompletionRequest::FromJSON(const std::string& json_str) { using TResult = Result; Result json_obj_res = json::ParseToJSONObjectWithResultReturn(json_str); if (json_obj_res.IsErr()) { return TResult::Error(json_obj_res.UnwrapErr()); } tvm::ffi::json::Object json_obj = json_obj_res.Unwrap(); ChatCompletionRequest request; // messages Result messages_arr_res = json::LookupWithResultReturn(json_obj, "messages"); if (messages_arr_res.IsErr()) { return TResult::Error(messages_arr_res.UnwrapErr()); } std::vector messages; tvm::ffi::json::Array messages_arr = messages_arr_res.Unwrap(); for (const auto& item : messages_arr) { if (!item.try_cast().has_value()) { return TResult::Error("A message in chat completion request is not object"); } tvm::ffi::json::Object item_obj = item.cast(); Result message = ChatCompletionMessage::FromJSON(item_obj); if (message.IsErr()) { return TResult::Error(message.UnwrapErr()); } messages.push_back(message.Unwrap()); } request.messages = messages; // model Result> model_res = json::LookupOptionalWithResultReturn(json_obj, "model"); if (model_res.IsErr()) { return TResult::Error(model_res.UnwrapErr()); } request.model = model_res.Unwrap(); // temperature Result> temperature_res = json::LookupOptionalWithResultReturn(json_obj, "temperature"); if (temperature_res.IsErr()) { return TResult::Error(temperature_res.UnwrapErr()); } request.temperature = temperature_res.Unwrap(); // top_p Result> top_p_res = json::LookupOptionalWithResultReturn(json_obj, "top_p"); if (top_p_res.IsErr()) { return TResult::Error(top_p_res.UnwrapErr()); } request.top_p = top_p_res.Unwrap(); // max_tokens Result> max_tokens_res = json::LookupOptionalWithResultReturn(json_obj, "max_tokens"); if (max_tokens_res.IsErr()) { return TResult::Error(max_tokens_res.UnwrapErr()); } request.max_tokens = max_tokens_res.Unwrap(); // n Result n_res = json::LookupOrDefaultWithResultReturn(json_obj, "n", 1); if (n_res.IsErr()) { return TResult::Error(n_res.UnwrapErr()); } request.n = n_res.Unwrap(); // frequency_penalty Result> frequency_penalty_res = json::LookupOptionalWithResultReturn(json_obj, "frequency_penalty"); if (frequency_penalty_res.IsErr()) { return TResult::Error(frequency_penalty_res.UnwrapErr()); } request.frequency_penalty = frequency_penalty_res.Unwrap(); // presence_penalty Result> presence_penalty_res = json::LookupOptionalWithResultReturn(json_obj, "presence_penalty"); if (presence_penalty_res.IsErr()) { return TResult::Error(presence_penalty_res.UnwrapErr()); } request.presence_penalty = presence_penalty_res.Unwrap(); // seed Result> seed_res = json::LookupOptionalWithResultReturn(json_obj, "seed"); if (seed_res.IsErr()) { return TResult::Error(seed_res.UnwrapErr()); } request.seed = seed_res.Unwrap(); // stop strings Result> stop_strs_res = json::LookupOptionalWithResultReturn(json_obj, "stop"); if (stop_strs_res.IsErr()) { return TResult::Error(stop_strs_res.UnwrapErr()); } std::optional stop_strs = stop_strs_res.Unwrap(); if (stop_strs.has_value()) { std::vector stop; for (const auto& stop_str_value : stop_strs.value()) { if (!stop_str_value.try_cast().has_value()) { return TResult::Error("One given value in field \"stop\" is not a string."); } stop.push_back(stop_str_value.cast()); } request.stop = std::move(stop); } // tool_choice Result tool_choice_res = json::LookupOrDefaultWithResultReturn(json_obj, "tool_choice", "auto"); if (tool_choice_res.IsErr()) { return TResult::Error(tool_choice_res.UnwrapErr()); } request.tool_choice = tool_choice_res.Unwrap(); // tools Result> tools_arr_res = json::LookupOptionalWithResultReturn(json_obj, "tools"); if (tool_choice_res.IsErr()) { return TResult::Error(tool_choice_res.UnwrapErr()); } std::optional tools_arr = tools_arr_res.Unwrap(); if (tools_arr.has_value()) { std::vector tools; tools.reserve(tools_arr.value().size()); for (const auto& item : tools_arr.value()) { if (!item.try_cast().has_value()) { return TResult::Error("A tool of the chat completion request is not an object"); } Result tool = ChatTool::FromJSON(item.cast()); if (tool.IsErr()) { return TResult::Error(tool.UnwrapErr()); } tools.push_back(tool.Unwrap()); } request.tools = tools; } // response format std::optional response_format_obj = json::LookupOptional(json_obj, "response_format"); if (response_format_obj.has_value()) { Result response_format_res = ResponseFormat::FromJSON(response_format_obj.value()); if (response_format_res.IsErr()) { return TResult::Error(response_format_res.UnwrapErr()); } request.response_format = response_format_res.Unwrap(); } // debug_config Result> debug_config_opt_res = json::LookupOptionalWithResultReturn(json_obj, "debug_config"); if (debug_config_opt_res.IsErr()) { return TResult::Error(debug_config_opt_res.UnwrapErr()); } auto debug_config_opt = debug_config_opt_res.Unwrap(); if (debug_config_opt.has_value()) { Result debug_config_res = DebugConfig::FromJSON(debug_config_opt.value()); if (debug_config_res.IsErr()) { return TResult::Error(debug_config_res.UnwrapErr()); } request.debug_config = debug_config_res.Unwrap(); } // TODO: Other parameters return TResult::Ok(request); } tvm::ffi::json::Object ChatCompletionMessage::AsJSON() const { tvm::ffi::json::Object obj; if (this->content.IsText()) { obj.Set("content", this->content.Text()); } else if (this->content.IsParts()) { tvm::ffi::json::Array content_arr; for (const auto& item : this->content.Parts()) { tvm::ffi::json::Object item_obj; for (const auto& pair : item) { item_obj.Set(pair.first, pair.second); } content_arr.push_back(item_obj); } obj.Set("content", content_arr); } obj.Set("role", this->role); if (this->name.has_value()) { obj.Set("name", this->name.value()); } if (this->tool_call_id.has_value()) { obj.Set("tool_call_id", this->tool_call_id.value()); } if (this->tool_calls.has_value()) { tvm::ffi::json::Array tool_calls_arr; for (const auto& tool_call : this->tool_calls.value()) { tool_calls_arr.push_back(tool_call.AsJSON()); } obj.Set("tool_calls", tool_calls_arr); } return obj; } tvm::ffi::json::Object ChatCompletionResponseChoice::AsJSON() const { tvm::ffi::json::Object obj; if (!this->finish_reason.has_value()) { obj.Set("finish_reason", nullptr); } else { if (this->finish_reason == FinishReason::stop) { obj.Set("finish_reason", "stop"); } else if (this->finish_reason == FinishReason::length) { obj.Set("finish_reason", "length"); } else if (this->finish_reason == FinishReason::tool_calls) { obj.Set("finish_reason", "tool_calls"); } else if (this->finish_reason == FinishReason::error) { obj.Set("finish_reason", "error"); } } obj.Set("index", static_cast(this->index)); obj.Set("message", this->message.AsJSON()); return obj; } tvm::ffi::json::Object ChatCompletionStreamResponseChoice::AsJSON() const { tvm::ffi::json::Object obj; if (!this->finish_reason.has_value()) { obj.Set("finish_reason", nullptr); } else { if (this->finish_reason.value() == FinishReason::stop) { obj.Set("finish_reason", "stop"); } else if (this->finish_reason.value() == FinishReason::length) { obj.Set("finish_reason", "length"); } else if (this->finish_reason.value() == FinishReason::tool_calls) { obj.Set("finish_reason", "tool_calls"); } else if (this->finish_reason.value() == FinishReason::error) { obj.Set("finish_reason", "error"); } } obj.Set("index", static_cast(this->index)); obj.Set("delta", this->delta.AsJSON()); return obj; } tvm::ffi::json::Object ChatCompletionResponse::AsJSON() const { tvm::ffi::json::Object obj; obj.Set("id", this->id); tvm::ffi::json::Array choices_arr; for (const auto& choice : this->choices) { choices_arr.push_back(choice.AsJSON()); } obj.Set("choices", choices_arr); obj.Set("created", static_cast(this->created)); obj.Set("model", this->model); obj.Set("system_fingerprint", this->system_fingerprint); obj.Set("object", this->object); return obj; } tvm::ffi::json::Object ChatCompletionStreamResponse::AsJSON() const { tvm::ffi::json::Object obj; obj.Set("id", this->id); tvm::ffi::json::Array choices_arr; for (const auto& choice : this->choices) { choices_arr.push_back(choice.AsJSON()); } obj.Set("choices", choices_arr); obj.Set("created", static_cast(this->created)); obj.Set("model", this->model); obj.Set("system_fingerprint", this->system_fingerprint); obj.Set("object", this->object); if (usage.has_value()) { obj.Set("usage", usage.value()); } return obj; } } // namespace json_ffi } // namespace llm } // namespace mlc