// // LLMInferenceEngineWrapper.h // mnn-llm // // Created by wangzhaode on 2023/12/14. // #ifndef LLMInferenceEngineWrapper_h #define LLMInferenceEngineWrapper_h #import #if __has_include() #import #elif __has_include() #import #define UIImage NSImage #endif NS_ASSUME_NONNULL_BEGIN typedef void (^CompletionHandler)(BOOL success); typedef void (^OutputHandler)(NSString * _Nonnull output); // MARK: - Benchmark Related Types /** * Progress type enumeration for structured benchmark reporting */ typedef NS_ENUM(NSInteger, BenchmarkProgressType) { BenchmarkProgressTypeUnknown = 0, BenchmarkProgressTypeInitializing = 1, BenchmarkProgressTypeWarmingUp = 2, BenchmarkProgressTypeRunningTest = 3, BenchmarkProgressTypeProcessingResults = 4, BenchmarkProgressTypeCompleted = 5, BenchmarkProgressTypeStopping = 6 }; /** * Structured progress information for benchmark */ @interface BenchmarkProgressInfo : NSObject @property (nonatomic, assign) NSInteger progress; // 0-100 @property (nonatomic, strong) NSString *statusMessage; // Status description @property (nonatomic, assign) BenchmarkProgressType progressType; @property (nonatomic, assign) NSInteger currentIteration; @property (nonatomic, assign) NSInteger totalIterations; @property (nonatomic, assign) NSInteger nPrompt; @property (nonatomic, assign) NSInteger nGenerate; @property (nonatomic, assign) float runTimeSeconds; @property (nonatomic, assign) float prefillTimeSeconds; @property (nonatomic, assign) float decodeTimeSeconds; @property (nonatomic, assign) float prefillSpeed; @property (nonatomic, assign) float decodeSpeed; @end /** * Benchmark result structure */ @interface BenchmarkResult : NSObject @property (nonatomic, assign) BOOL success; @property (nonatomic, strong, nullable) NSString *errorMessage; @property (nonatomic, strong) NSArray *prefillTimesUs; @property (nonatomic, strong) NSArray *decodeTimesUs; @property (nonatomic, strong) NSArray *sampleTimesUs; @property (nonatomic, assign) NSInteger promptTokens; @property (nonatomic, assign) NSInteger generateTokens; @property (nonatomic, assign) NSInteger repeatCount; @property (nonatomic, assign) BOOL kvCacheEnabled; @end // Benchmark callback blocks typedef void (^BenchmarkProgressCallback)(BenchmarkProgressInfo *progressInfo); typedef void (^BenchmarkErrorCallback)(NSString *error); typedef void (^BenchmarkIterationCompleteCallback)(NSString *detailedStats); typedef void (^BenchmarkCompleteCallback)(BenchmarkResult *result); /** * LLMInferenceEngineWrapper - A high-level Objective-C wrapper for MNN LLM inference engine * * This class provides a convenient interface for integrating MNN's Large Language Model * inference capabilities into iOS applications with enhanced error handling, performance * optimization, and thread safety. */ @interface LLMInferenceEngineWrapper : NSObject /** * Initialize the LLM inference engine with a model path * * @param modelPath The file system path to the model directory * @param completion Completion handler called with success/failure status * @return Initialized instance of LLMInferenceEngineWrapper */ - (instancetype)initWithModelPath:(NSString *)modelPath completion:(CompletionHandler)completion; /** * Process user input and generate streaming LLM response * * @param input The user's input text to process * @param output Callback block that receives streaming output chunks */ - (void)processInput:(NSString *)input withOutput:(OutputHandler)output; /** * Process user input and generate streaming LLM response with optional performance output * * @param input The user's input text to process * @param output Callback block that receives streaming output chunks * @param showPerformance Whether to output performance statistics after response completion */ - (void)processInput:(NSString *)input withOutput:(OutputHandler)output showPerformance:(BOOL)showPerformance; /** * Process multimodal input (text + images) using MNN's MultimodalPrompt API. * * @param promptTemplate Template string containing placeholder tags. * @param images Dictionary mapping placeholder keys to UIImage objects. * @param output Callback block receiving streaming output chunks. * @param showPerformance Whether to show performance stats upon completion. */ - (void)processMultimodalInput:(NSString *)promptTemplate images:(NSDictionary *)images withOutput:(OutputHandler)output showPerformance:(BOOL)showPerformance; /// Update maximum frames extracted for each video. - (void)setVideoMaxFrames:(NSInteger)frames; /// Set audio output enabled/disabled (for Omni models) - (void)setEnableAudioOutput:(BOOL)enable; /// Set talker speaker (for Omni models) - (void)setTalkerSpeaker:(NSString *)speaker; /// Set audio waveform callback for receiving PCM float data - (void)setAudioWaveformCallback:(BOOL (^)(const float *data, size_t size, BOOL isLastChunk))callback; /** * Add chat prompts from an array of dictionaries to the conversation history * * @param array NSArray containing NSDictionary objects with chat messages */ - (void)addPromptsFromArray:(NSArray *)array; /** * Set the configuration for the LLM engine using a JSON string * * @param jsonStr JSON string containing configuration parameters */ - (void)setConfigWithJSONString:(NSString *)jsonStr; /** * Set thinking mode for the LLM engine * * @param enabled Whether to enable thinking mode */ - (void)setThinkingModeEnabled:(BOOL)enabled; /** * Check if model is ready for inference * * @return YES if model is loaded and ready */ - (BOOL)isModelReady; /** * Get current processing status * * @return YES if currently processing an inference request */ - (BOOL)isProcessing; /** * Cancel ongoing inference (if supported) */ - (void)cancelInference; /** * Get chat history count * * @return Number of messages in chat history */ - (NSUInteger)getChatHistoryCount; /** * Clear chat history */ - (void)clearChatHistory; // MARK: - Benchmark Methods /** * Run official benchmark following llm_bench.cpp approach * * @param backend Backend type (0 for CPU) * @param threads Number of threads * @param useMmap Whether to use memory mapping * @param power Power setting * @param precision Precision setting (2 for low precision) * @param memory Memory setting (2 for low memory) * @param dynamicOption Dynamic optimization option * @param nPrompt Number of prompt tokens * @param nGenerate Number of tokens to generate * @param nRepeat Number of repetitions * @param kvCache Whether to use KV cache * @param progressCallback Progress update callback * @param errorCallback Error callback * @param iterationCompleteCallback Iteration completion callback * @param completeCallback Final completion callback */ - (void)runOfficialBenchmarkWithBackend:(NSInteger)backend threads:(NSInteger)threads useMmap:(BOOL)useMmap power:(NSInteger)power precision:(NSInteger)precision memory:(NSInteger)memory dynamicOption:(NSInteger)dynamicOption nPrompt:(NSInteger)nPrompt nGenerate:(NSInteger)nGenerate nRepeat:(NSInteger)nRepeat kvCache:(BOOL)kvCache progressCallback:(BenchmarkProgressCallback _Nullable)progressCallback errorCallback:(BenchmarkErrorCallback _Nullable)errorCallback iterationCompleteCallback:(BenchmarkIterationCompleteCallback _Nullable)iterationCompleteCallback completeCallback:(BenchmarkCompleteCallback _Nullable)completeCallback; /** * Stop running benchmark */ - (void)stopBenchmark; /** * Check if benchmark is currently running * * @return YES if benchmark is running */ - (BOOL)isBenchmarkRunning; /** * Process multiple prompts in a single batch and return their responses. * * This method runs each prompt independently, clears the chat history per prompt, * and collects the generated outputs without streaming UI callbacks. * * @param prompts An array of input prompt strings to process * @param completion Completion block called on the main thread with an array of * response strings in the same order as the input prompts */ - (void)processBatchPrompts:(NSArray *)prompts completion:(void (^)(NSArray *responses))completion; @end NS_ASSUME_NONNULL_END #endif /* LLMInferenceEngineWrapper_h */