// // Created by ruoyi.sjd on 2024/12/25. // Copyright (c) 2024 Alibaba Group Holding Limited All rights reserved. // #pragma once #include "../../../transformers/llm/engine/include/llm/llm.hpp" #include "httplib.h" #include "json.hpp" using nlohmann::json; using PromptItem = std::pair; namespace mnncli { class LlmStreamBuffer : public std::streambuf { public: using CallBack = std::function; explicit LlmStreamBuffer(CallBack callback) : callback_(std::move(callback)) {} ~LlmStreamBuffer() override = default; protected: virtual std::streamsize xsputn(const char* s, std::streamsize n) override { if (callback_) { callback_(s, n); } return n; } private: CallBack callback_{}; }; class Utf8StreamProcessor { public: Utf8StreamProcessor(std::function callback) : callback(callback) {} void processStream(const char* str, size_t len) { utf8Buffer.append(str, len); size_t i = 0; std::string completeChars; while (i < utf8Buffer.size()) { int length = utf8CharLength(static_cast(utf8Buffer[i])); if (length == 0 || i + length > utf8Buffer.size()) { break; } completeChars.append(utf8Buffer, i, length); i += length; } utf8Buffer = utf8Buffer.substr(i); if (!completeChars.empty()) { callback(completeChars); } } int utf8CharLength(unsigned char byte) { if ((byte & 0x80) == 0) return 1; if ((byte & 0xE0) == 0xC0) return 2; if ((byte & 0xF0) == 0xE0) return 3; if ((byte & 0xF8) == 0xF0) return 4; return 0; } private: std::string utf8Buffer; std::function callback; }; class MnncliServer { public: const char* html_content = R"""( MNN Frontend

Chat with MNN

MNN-LLM's server API is OpenAI API compatible. You can use other frameworks like OpenWebUI or LobeChat.

⚠️ Model switching is temporarily not supported. Please change models in the app.
Ready to chat
)"""; void Start(MNN::Transformer::Llm* llm, bool is_r1, const std::string& host = "127.0.0.1", int port = 8000); bool is_r1_{false}; private: void Answer(MNN::Transformer::Llm* llm, const json &messages, std::function on_result); void AnswerStreaming(MNN::Transformer::Llm* llm, const json& messages, std::function on_partial); std::mutex llm_mutex_; }; }