chore: import upstream snapshot with attribution

tools/cpp/CMakeLists.txt

@@ -0,0 +1,97 @@
+set(MNN_CPP_TOOLS "")
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "^armv7" OR ARCHS MATCHES "^armv7(;armv7s)?")
+    add_definitions(-DMNN_USE_NEON)
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64" OR ARCHS STREQUAL "arm64")
+    add_definitions(-DMNN_USE_NEON)
+endif()
+add_executable(MNNV2Basic.out ${CMAKE_CURRENT_LIST_DIR}/MNNV2Basic.cpp)
+list(APPEND MNN_CPP_TOOLS MNNV2Basic.out)
+
+if (NOT MNN_SKIPBUILD_GEOMETRY)
+add_executable(GetMNNInfo ${CMAKE_CURRENT_LIST_DIR}/GetMNNInfo.cpp)
+list(APPEND MNN_CPP_TOOLS GetMNNInfo)
+add_executable(ModuleBasic.out ${CMAKE_CURRENT_LIST_DIR}/ModuleBasic.cpp)
+list(APPEND MNN_CPP_TOOLS ModuleBasic.out)
+
+IF(CMAKE_SYSTEM_NAME MATCHES "^Android")
+    add_executable(GpuInterTest.out ${CMAKE_CURRENT_LIST_DIR}/GpuInterTest.cpp )
+    target_link_libraries(GpuInterTest.out android)
+    list(APPEND MNN_CPP_TOOLS GpuInterTest.out)
+ENDIF()
+
+add_executable(OpenCLProgramBuildTest.out ${CMAKE_CURRENT_LIST_DIR}/OpenCLProgramBuildTest.cpp )
+list(APPEND MNN_CPP_TOOLS OpenCLProgramBuildTest.out)
+
+add_executable(SequenceModuleTest.out ${CMAKE_CURRENT_LIST_DIR}/SequenceModuleTest.cpp)
+list(APPEND MNN_CPP_TOOLS SequenceModuleTest.out)
+
+add_executable(mergeInplaceForCPU ${CMAKE_CURRENT_LIST_DIR}/mergeInplaceForCPU.cpp)
+list(APPEND MNN_CPP_TOOLS mergeInplaceForCPU)
+
+add_executable(modelCompare.out ${CMAKE_CURRENT_LIST_DIR}/modelCompare.cpp)
+list(APPEND MNN_CPP_TOOLS modelCompare.out)
+
+if (MNN_USE_SSE)
+    target_compile_options(MNNV2Basic.out PRIVATE -DMNN_USE_SSE)
+endif()
+
+add_executable(mobilenetTest.out ${CMAKE_CURRENT_LIST_DIR}/mobilenetTest.cpp )
+list(APPEND MNN_CPP_TOOLS mobilenetTest.out)
+
+add_executable(backendTest.out ${CMAKE_CURRENT_LIST_DIR}/backendTest.cpp)
+list(APPEND MNN_CPP_TOOLS backendTest.out)
+
+add_executable(testModel.out ${CMAKE_CURRENT_LIST_DIR}/testModel.cpp)
+list(APPEND MNN_CPP_TOOLS testModel.out)
+
+add_executable(compilefornpu ${CMAKE_CURRENT_LIST_DIR}/compilefornpu.cpp ${CMAKE_CURRENT_LIST_DIR}/../../3rd_party/flatbuffers/src/util.cpp)
+list(APPEND MNN_CPP_TOOLS compilefornpu)
+
+add_executable(testModel_expr.out ${CMAKE_CURRENT_LIST_DIR}/testModel_expr.cpp)
+list(APPEND MNN_CPP_TOOLS testModel_expr.out)
+
+add_executable(testModelWithDescribe.out ${CMAKE_CURRENT_LIST_DIR}/testModelWithDescribe.cpp)
+list(APPEND MNN_CPP_TOOLS testModelWithDescribe.out)
+
+add_executable(getPerformance.out ${CMAKE_CURRENT_LIST_DIR}/getPerformance.cpp)
+list(APPEND MNN_CPP_TOOLS getPerformance.out)
+
+add_executable(checkInvalidValue.out ${CMAKE_CURRENT_LIST_DIR}/checkInvalidValue.cpp)
+list(APPEND MNN_CPP_TOOLS checkInvalidValue.out)
+
+add_executable(timeProfile.out ${CMAKE_CURRENT_LIST_DIR}/timeProfile.cpp ${CMAKE_CURRENT_LIST_DIR}/revertMNNModel.cpp ${CMAKE_CURRENT_LIST_DIR}/Profiler.cpp)
+list(APPEND MNN_CPP_TOOLS timeProfile.out)
+
+add_executable(testTrain.out ${CMAKE_CURRENT_LIST_DIR}/testTrain.cpp)
+list(APPEND MNN_CPP_TOOLS testTrain.out)
+
+add_executable(fuseTest ${CMAKE_CURRENT_LIST_DIR}/fuseTest.cpp)
+list(APPEND MNN_CPP_TOOLS fuseTest)
+
+if (MNN_QNN)
+    add_executable(generateIO ${CMAKE_CURRENT_LIST_DIR}/generateIO.cpp)
+    list(APPEND MNN_CPP_TOOLS generateIO)
+    add_executable(MNN2QNNModel ${CMAKE_CURRENT_LIST_DIR}/MNN2QNNModel.cpp)
+    list(APPEND MNN_CPP_TOOLS MNN2QNNModel)
+endif()
+
+endif()
+
+foreach(TARGET ${MNN_CPP_TOOLS})
+    target_link_libraries(${TARGET} ${MNN_DEPS})
+    if (MSVC)
+        target_compile_definitions(${TARGET} PRIVATE "_CRT_SECURE_NO_WARNINGS")
+        if (NOT MNN_BUILD_SHARED_LIBS)
+            foreach (DEPEND ${MNN_DEPS})
+                target_link_options(${TARGET} PRIVATE /WHOLEARCHIVE:$<TARGET_FILE:${DEPEND}>)
+            endforeach ()
+        endif()
+    endif()
+endforeach()
+
+if (NOT WIN32)
+    add_executable(checkDir.out ${CMAKE_CURRENT_LIST_DIR}/checkDir.cpp)
+    add_executable(checkFile.out ${CMAKE_CURRENT_LIST_DIR}/checkFile.cpp)
+    add_executable(winogradExample.out ${CMAKE_CURRENT_LIST_DIR}/winogradExample.cpp)
+    target_link_libraries(winogradExample.out ${MNN_DEPS})
+endif()

tools/cpp/ConfigFile.hpp

@@ -0,0 +1,372 @@
+//
+//  ConfigFile.hpp
+//  MNN
+//
+//  Created by MNN on 2019/01/22.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#ifndef CONFIGFILE_HPP
+#define CONFIGFILE_HPP
+
+#include <fstream>
+#include <iostream>
+#include <vector>
+#include <map>
+#include <sstream>
+#include <string>
+
+/*
+* \brief Generic configuration Class
+* can deal with the file like below:
+*
+*           input_size = 1
+*           input_names = ImageTensor
+*           input_dims = 1x3x320x180
+*           output_size = 1
+*           output_names = strided_slice_61
+*
+*/
+class ConfigFile {
+    // Data
+protected:
+    std::string m_Delimiter;                       //!< separator between key and value
+    std::string m_Comment;                         //!< separator between value and comments
+    std::map<std::string, std::string> m_Contents; //!< extracted keys and values
+
+    typedef std::map<std::string, std::string>::iterator mapi;
+    typedef std::map<std::string, std::string>::const_iterator mapci;
+    // Methods
+public:
+    ConfigFile(std::string filename, std::string delimiter = "=", std::string comment = "#");
+    ConfigFile() = default;
+    template <class T>
+    T Read(
+        const std::string& in_key) const; //!<Search for key and read value or optional default value, call as read<T>
+    template <class T>
+    T Read(const std::string& in_key, const T& in_value) const;
+    template <class T>
+    bool ReadInto(T& out_var, const std::string& in_key) const;
+    template <class T>
+    bool ReadInto(T& out_var, const std::string& in_key, const T& in_value) const;
+    bool FileExist(std::string filename);
+    void ReadFile(std::string filename, std::string delimiter = "=", std::string comment = "#");
+
+    // Check whether key exists in configuration
+    bool KeyExists(const std::string& in_key) const;
+
+    // Modify keys and values
+    template <class T>
+    void Add(const std::string& in_key, const T& in_value);
+    void Remove(const std::string& in_key);
+
+    // Check or change configuration syntax
+    std::string GetDelimiter() const {
+        return m_Delimiter;
+    }
+    std::string GetComment() const {
+        return m_Comment;
+    }
+    std::string SetDelimiter(const std::string& in_s) {
+        std::string old = m_Delimiter;
+        m_Delimiter     = in_s;
+        return old;
+    }
+    std::string SetComment(const std::string& in_s) {
+        std::string old = m_Comment;
+        m_Comment       = in_s;
+        return old;
+    }
+
+    // Write or read configuration
+    friend std::ostream& operator<<(std::ostream& os, const ConfigFile& cf);
+    friend std::istream& operator>>(std::istream& is, ConfigFile& cf);
+
+protected:
+    template <class T>
+    static std::string T_as_string(const T& t);
+    template <class T>
+    static T string_as_T(const std::string& s);
+    static void Trim(std::string& inout_s);
+
+    // Exception types
+public:
+    struct File_not_found {
+        std::string filename;
+        File_not_found(const std::string& filename_ = std::string()) : filename(filename_) {
+        }
+    };
+    struct Key_not_found { // thrown only by T read(key) variant of read()
+        std::string key;
+        Key_not_found(const std::string& key_ = std::string()) : key(key_) {
+        }
+    };
+};
+
+/* static */
+template <class T>
+std::string ConfigFile::T_as_string(const T& t) {
+    // Convert from a T to a string
+    // Type T must support << operator
+    std::ostringstream ost;
+    ost << t;
+    return ost.str();
+}
+
+/* static */
+template <class T>
+T ConfigFile::string_as_T(const std::string& s) {
+    // Convert from a string to a T
+    // Type T must support >> operator
+    T t;
+    std::istringstream ist(s);
+    ist >> t;
+    return t;
+}
+
+/* static */
+template <>
+inline std::string ConfigFile::string_as_T<std::string>(const std::string& s) {
+    // Convert from a string to a string
+    // In other words, do nothing
+    return s;
+}
+
+/* static */
+template <>
+inline bool ConfigFile::string_as_T<bool>(const std::string& s) {
+    // Convert from a string to a bool
+    // Interpret "false", "F", "no", "n", "0" as false
+    // Interpret "true", "T", "yes", "y", "1", "-1", or anything else as true
+    bool b          = true;
+    std::string sup = s;
+    for (std::string::iterator p = sup.begin(); p != sup.end(); ++p)
+        *p = toupper(*p); // make string all caps
+    if (sup == std::string("FALSE") || sup == std::string("F") || sup == std::string("NO") || sup == std::string("N") ||
+        sup == std::string("0") || sup == std::string("NONE"))
+        b = false;
+    return b;
+}
+
+template <class T>
+T ConfigFile::Read(const std::string& key) const {
+    // Read the value corresponding to key
+    mapci p = m_Contents.find(key);
+    if (p == m_Contents.end()) {
+        std::cout << "Key Error!" << std::endl;
+    }
+    return string_as_T<T>(p->second);
+}
+
+template <class T>
+T ConfigFile::Read(const std::string& key, const T& value) const {
+    // Return the value corresponding to key or given default value
+    // if key is not found
+    mapci p = m_Contents.find(key);
+    if (p == m_Contents.end())
+        return value;
+    return string_as_T<T>(p->second);
+}
+
+template <class T>
+bool ConfigFile::ReadInto(T& var, const std::string& key) const {
+    // Get the value corresponding to key and store in var
+    // Return true if key is found
+    // Otherwise leave var untouched
+    mapci p    = m_Contents.find(key);
+    bool found = (p != m_Contents.end());
+    if (found)
+        var = string_as_T<T>(p->second);
+    return found;
+}
+
+template <class T>
+bool ConfigFile::ReadInto(T& var, const std::string& key, const T& value) const {
+    // Get the value corresponding to key and store in var
+    // Return true if key is found
+    // Otherwise set var to given default
+    mapci p    = m_Contents.find(key);
+    bool found = (p != m_Contents.end());
+    if (found)
+        var = string_as_T<T>(p->second);
+    else
+        var = value;
+    return found;
+}
+
+template <class T>
+void ConfigFile::Add(const std::string& in_key, const T& value) {
+    // Add a key with given value
+    std::string v   = T_as_string(value);
+    std::string key = in_key;
+    Trim(key);
+    Trim(v);
+    m_Contents[key] = v;
+    return;
+}
+std::vector<int> stringToDims(const std::string& str, const std::string& pattern) {
+    std::vector<int> res;
+    if (str == "") {
+        return res;
+    }
+    std::string strs = str + pattern;
+    size_t pos       = strs.find(pattern);
+
+    while (pos != strs.npos) {
+        std::string temp = strs.substr(0, pos);
+        std::istringstream tempIs(temp);
+        int p;
+        tempIs >> p;
+        res.push_back(p);
+        strs = strs.substr(pos + 1, strs.size());
+        pos  = strs.find(pattern);
+    }
+
+    return res;
+}
+
+std::vector<std::string> splitNames(const int size, const std::string names) {
+    std::vector<std::string> split(size);
+    std::string rest = names;
+    for (int i = 0; i < size; ++i) {
+        auto position = rest.find(",");
+        split[i]      = rest.substr(0, position);
+        rest          = rest.substr(position + 1, rest.size());
+    }
+    return split;
+}
+
+std::vector<std::vector<int>> splitDims(const int size, const std::string string) {
+    std::vector<std::vector<int>> dims(size);
+    std::string rest = string;
+    for (int i = 0; i < size; ++i) {
+        auto position = rest.find(",");
+        dims[i]       = stringToDims(rest.substr(0, position), "x");
+        rest          = rest.substr(position + 1, rest.size());
+    }
+    return dims;
+}
+
+ConfigFile::ConfigFile(std::string filename, std::string delimiter, std::string comment) : m_Delimiter(delimiter), m_Comment(comment) {
+    // Construct a ConfigFile, getting keys and values from given file
+
+    std::ifstream in(filename.c_str());
+    if (in.fail()) {
+        MNN_ERROR("Can't open %s\n", filename.c_str());
+    }
+    in >> (*this);
+}
+
+bool ConfigFile::KeyExists(const std::string& key) const {
+    // Indicate whether key is found
+    mapci p = m_Contents.find(key);
+    return (p != m_Contents.end());
+}
+
+/* static */
+void ConfigFile::Trim(std::string& inout_s) {
+    // Remove leading and trailing whitespace
+    static const char whitespace[] = " \n\t\v\r\f";
+    inout_s.erase(0, inout_s.find_first_not_of(whitespace));
+    inout_s.erase(inout_s.find_last_not_of(whitespace) + 1U);
+}
+
+std::ostream& operator<<(std::ostream& os, const ConfigFile& cf) {
+    // Save a ConfigFile to os
+    for (ConfigFile::mapci p = cf.m_Contents.begin(); p != cf.m_Contents.end(); ++p) {
+        os << p->first << " " << cf.m_Delimiter << " ";
+        os << p->second << std::endl;
+    }
+    return os;
+}
+
+void ConfigFile::Remove(const std::string& key) {
+    // Remove key and its value
+    m_Contents.erase(m_Contents.find(key));
+    return;
+}
+
+std::istream& operator>>(std::istream& is, ConfigFile& cf) {
+    // Load a ConfigFile from is
+    // Read in keys and values, keeping internal whitespace
+    typedef std::string::size_type pos;
+    const std::string& delim = cf.m_Delimiter; // separator
+    const std::string& comm  = cf.m_Comment;   // comment
+    const pos skip      = delim.length(); // length of separator
+
+    std::string nextline = ""; // might need to read ahead to see where value ends
+
+    while (is || nextline.length() > 0) {
+        // Read an entire line at a time
+        std::string line;
+        if (nextline.length() > 0) {
+            line     = nextline; // we read ahead; use it now
+            nextline = "";
+        } else {
+            std::getline(is, line);
+        }
+
+        // Ignore comments
+        line = line.substr(0, line.find(comm));
+
+        // Parse the line if it contains a delimiter
+        pos delimPos = line.find(delim);
+        if (delimPos < std::string::npos) {
+            // Extract the key
+            std::string key = line.substr(0, delimPos);
+            line.replace(0, delimPos + skip, "");
+
+            // See if value continues on the next line
+            // Stop at blank line, next line with a key, end of stream,
+            // or end of file sentry
+            bool terminate = false;
+            while (!terminate && is) {
+                std::getline(is, nextline);
+                terminate = true;
+
+                std::string nlcopy = nextline;
+                ConfigFile::Trim(nlcopy);
+                if (nlcopy == "")
+                    continue;
+
+                nextline = nextline.substr(0, nextline.find(comm));
+                if (nextline.find(delim) != std::string::npos)
+                    continue;
+
+                nlcopy = nextline;
+                ConfigFile::Trim(nlcopy);
+                if (nlcopy != "")
+                    line += "\n";
+                line += nextline;
+                terminate = false;
+            }
+
+            // Store key and value
+            ConfigFile::Trim(key);
+            ConfigFile::Trim(line);
+            cf.m_Contents[key] = line; // overwrites if key is repeated
+        }
+    }
+
+    return is;
+}
+bool ConfigFile::FileExist(std::string filename) {
+    bool exist = false;
+    std::ifstream in(filename.c_str());
+    if (in)
+        exist = true;
+    return exist;
+}
+
+void ConfigFile::ReadFile(std::string filename, std::string delimiter, std::string comment) {
+    m_Delimiter = delimiter;
+    m_Comment   = comment;
+    std::ifstream in(filename.c_str());
+
+    if (!in) {
+        std::cout << "file is not existed!" << std::endl;
+    }
+
+    in >> (*this);
+}
+#endif // CONFIG_HPP

tools/cpp/ConvertToFullQuant.hpp

@@ -0,0 +1,211 @@
+//
+//  ConvertToFullQuant.hpp
+//  MNN
+//
+//  Created by MNN on 2021/04/01.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#ifndef CONVERTTOFULLQUANT_HPP
+#define CONVERTTOFULLQUANT_HPP
+
+#include <vector>
+#include <map>
+#include <string>
+#include <fstream>
+#include <sstream>
+#include "MNN_generated.h"
+#include "core/IDSTEncoder.hpp"
+
+using namespace MNN;
+
+namespace ConvertToFullQuant {
+
+void ConvertOp(std::unique_ptr<OpT>& op, int opIndex, NetT* net, SubGraphProtoT* subgraph, std::vector<int>& needEraseIndices) {
+    auto opType = op->type;
+    if ((opType != OpType_FloatToInt8) && (opType != OpType_Int8ToFloat) && (opType != OpType_ConvInt8) && (opType != OpType_DepthwiseConvInt8)) {
+        return;
+    }
+
+    auto& tensorNames = subgraph ? subgraph->tensors : net->tensorName;
+    auto& tensorDescribe = subgraph ? subgraph->extraTensorDescribe : net->extraTensorDescribe;
+
+    auto findReferenceOpsAndIndices = [&](int outputIndex) {
+        std::map<OpT*, std::vector<int> > refOps;
+        if (subgraph != nullptr) {
+            for (auto& node : subgraph->nodes) {
+                for (int i = 0; i < node->inputIndexes.size(); i++) {
+                    int index = node->inputIndexes[i];
+                    if (index == outputIndex) {
+                        refOps[node.get()].emplace_back(i);
+                    }
+                }
+            }
+        } else {
+            for (auto& node : net->oplists) {
+                for (int i = 0; i < node->inputIndexes.size(); i++) {
+                    int index = node->inputIndexes[i];
+                    if (index == outputIndex) {
+                        refOps[node.get()].emplace_back(i);
+                    }
+                }
+            }
+        }
+        return refOps;
+    };
+
+    auto inputIndex = op->inputIndexes[0];
+    int outputIndex = op->outputIndexes[0];
+    if ((opType == OpType_FloatToInt8) || (opType == OpType_Int8ToFloat)) {
+        auto params = op->main.AsQuantizedFloatParam();
+        std::unique_ptr<MNN::TensorDescribeT> describe(new MNN::TensorDescribeT);
+        describe->index = inputIndex;
+        std::unique_ptr<MNN::TensorQuantInfoT> qInfo(new MNN::TensorQuantInfoT);
+        qInfo->zero = params->zeroPoint;
+        if (opType == OpType_FloatToInt8) {
+            qInfo->scale = 1. / params->tensorScale[0];
+        } else {
+            qInfo->scale = params->tensorScale[0];
+        }
+        qInfo->min = params->clampMin;
+        qInfo->max = params->clampMax;
+        qInfo->type = MNN::DataType_DT_INT8;
+        describe->quantInfo = std::move(qInfo);
+
+        if (opType == OpType_FloatToInt8) {
+            tensorDescribe.emplace_back(std::move(describe));
+        } else {
+            bool found = false;
+            for (int i = 0; i < tensorDescribe.size(); i++) {
+                if (tensorDescribe[i]->index == inputIndex) {
+                    found = true;
+                    tensorDescribe[i]->index = inputIndex;
+                    tensorDescribe[i]->quantInfo->zero = params->zeroPoint;
+                    tensorDescribe[i]->quantInfo->scale = params->tensorScale[0];
+                    tensorDescribe[i]->quantInfo->min = params->clampMin;
+                    tensorDescribe[i]->quantInfo->max = params->clampMax;
+                    tensorDescribe[i]->quantInfo->type = MNN::DataType_DT_INT8;
+                    break;
+                }
+            }
+            if (!found) {
+                tensorDescribe.emplace_back(std::move(describe));
+            }
+        }
+
+        tensorNames[outputIndex] = "notused";
+        // reference op change input indexes
+        auto referenceOps = findReferenceOpsAndIndices(outputIndex);
+        for (auto& refOps : referenceOps) {
+            for (int i = 0; i < refOps.second.size(); i++) {
+                refOps.first->inputIndexes[refOps.second[i]] = inputIndex;
+            }
+        }
+
+        needEraseIndices.emplace_back(opIndex);
+    }
+
+    if ((opType == OpType_ConvInt8) || (opType == OpType_DepthwiseConvInt8)) {
+        if (opType == OpType_ConvInt8) {
+            op->type = OpType_Convolution;
+        } else {
+            op->type = OpType_ConvolutionDepthwise;
+        }
+        auto conv2D = op->main.AsConvolution2D();
+
+        // encoding
+        if (conv2D->symmetricQuan && (!conv2D->symmetricQuan->weight.empty())) {
+            // full quant support for train quant in NN.cpp
+            if (conv2D->quanParameter && conv2D->quanParameter->buffer.empty()) {
+                auto aMin = conv2D->quanParameter->aMin;
+                auto scaleIn = conv2D->quanParameter->scaleIn;
+                auto scaleOut = conv2D->quanParameter->scaleOut;
+                auto weightScale = conv2D->quanParameter->alpha;
+
+                if (aMin != 0 && scaleIn != 0 && scaleOut != 0 && weightScale.size() > 0) {
+                    auto weight = conv2D->symmetricQuan->weight;
+                    const int kn = conv2D->common->outputCount;
+                    const int ks = weight.size() / kn;
+                    std::vector<float> scales(kn, 1.0f);
+                    std::vector<float> weightFloat;
+
+                    for (int i = 0; i < weight.size(); i++) {
+                        weightFloat.emplace_back(weight[i] * weightScale[i / ks]);
+                    }
+
+                    conv2D->quanParameter = IDSTEncoder::encode(weightFloat.data(), weightScale, ks, kn, false, weight.data(), aMin);
+                    conv2D->quanParameter->scaleIn = scaleIn;
+                    conv2D->quanParameter->scaleOut = scaleOut;
+                    conv2D->symmetricQuan->weight.clear();
+
+                    std::unique_ptr<MNN::TensorDescribeT> describe(new MNN::TensorDescribeT);
+                    describe->index = outputIndex;
+                    std::unique_ptr<MNN::TensorQuantInfoT> qInfo(new MNN::TensorQuantInfoT);
+                    qInfo->zero = conv2D->symmetricQuan->outputZeroPoint;
+                    qInfo->scale = scaleOut;
+                    qInfo->min = conv2D->symmetricQuan->clampMin;
+                    qInfo->max = conv2D->symmetricQuan->clampMax;
+                    qInfo->type = MNN::DataType_DT_INT8;
+                    describe->quantInfo = std::move(qInfo);
+                    tensorDescribe.emplace_back(std::move(describe));
+
+                    return;
+                }
+            }
+        }
+
+        // fake info
+        std::unique_ptr<MNN::TensorDescribeT> describe(new MNN::TensorDescribeT);
+        describe->index = outputIndex;
+        std::unique_ptr<MNN::TensorQuantInfoT> qInfo(new MNN::TensorQuantInfoT);
+        qInfo->zero = 0;
+        qInfo->scale = 0;
+        qInfo->min = -127;
+        qInfo->max = 127;
+        qInfo->type = MNN::DataType_DT_INT8;
+        describe->quantInfo = std::move(qInfo);
+        tensorDescribe.emplace_back(std::move(describe));
+    }
+}
+
+void convert(std::string modelFile) {
+    std::unique_ptr<MNN::NetT> netT;
+    std::ifstream input(modelFile);
+    std::ostringstream outputOs;
+    outputOs << input.rdbuf();
+    netT = MNN::UnPackNet(outputOs.str().c_str());
+    auto net = netT.get();
+
+    std::vector<int> netNeedEraseIndices;
+    for (int i = 0; i < net->oplists.size(); i++) {
+        auto& op = net->oplists[i];
+        ConvertOp(op, i, net, nullptr, netNeedEraseIndices);
+    }
+    std::reverse(netNeedEraseIndices.begin(), netNeedEraseIndices.end());
+    for (int i = 0; i < netNeedEraseIndices.size(); i++) {
+        net->oplists.erase(net->oplists.begin() + netNeedEraseIndices[i]);
+    }
+
+    for (auto& subgraph : net->subgraphs) {
+        std::vector<int> subgraphNeedEraseIndices;
+        for (int i = 0; i < subgraph->nodes.size(); i++) {
+            auto& op = subgraph->nodes[i];
+            ConvertOp(op, i, net, subgraph.get(), subgraphNeedEraseIndices);
+        }
+        std::reverse(subgraphNeedEraseIndices.begin(), subgraphNeedEraseIndices.end());
+        for (int i = 0; i < subgraphNeedEraseIndices.size(); i++) {
+            subgraph->nodes.erase(subgraph->nodes.begin() + subgraphNeedEraseIndices[i]);
+        }
+    }
+
+    flatbuffers::FlatBufferBuilder builderOutput(1024);
+    builderOutput.ForceDefaults(true);
+    auto len = MNN::Net::Pack(builderOutput, net);
+    builderOutput.Finish(len);
+    std::ofstream output(modelFile);
+    output.write((const char*)builderOutput.GetBufferPointer(), builderOutput.GetSize());
+}
+
+} // namespace ConvertToFullQuant
+
+#endif // CONVERTTOFULLQUANT_HPP

tools/cpp/ExprDebug.hpp

@@ -0,0 +1,324 @@
+#include <cmath>
+#include <fstream>
+#include <sstream>
+#include <queue>
+#include <MNN/AutoTime.hpp>
+#include <MNN/expr/ExecutorScope.hpp>
+#define DUMP_NUM_DATA(type)                          \
+    auto data = tensor->host<type>();                \
+    for (int z = 0; z < outside; ++z) {              \
+        for (int x = 0; x < width; ++x) {            \
+            outputOs << data[x + z * width] << "\t"; \
+        }                                            \
+        outputOs << "\n";                            \
+    }
+
+#define DUMP_CHAR_DATA(type)                                           \
+    auto data = tensor->host<type>();                                  \
+    for (int z = 0; z < outside; ++z) {                                \
+        for (int x = 0; x < width; ++x) {                              \
+            outputOs << static_cast<int>(data[x + z * width]) << "\t"; \
+        }                                                              \
+        outputOs << "\n";                                              \
+    }
+
+static void dumpTensor2File(const MNN::Tensor* tensor, const char* file, std::ofstream& orderFile) {
+    orderFile << file << std::endl;
+    std::ofstream outputOs(file);
+    auto type = tensor->getType();
+
+    int dimension = tensor->buffer().dimensions;
+    int width     = 1;
+    if (dimension > 1) {
+        width = tensor->length(dimension - 1);
+    }
+
+    const int outside = tensor->elementSize() / width;
+
+    const auto dataType  = type.code;
+    const auto dataBytes = type.bytes();
+
+    if (dataType == halide_type_float) {
+        DUMP_NUM_DATA(float);
+    }
+    if (dataType == halide_type_int && dataBytes == 4) {
+        DUMP_NUM_DATA(int32_t);
+    }
+    if (dataType == halide_type_uint && dataBytes == 1) {
+        DUMP_CHAR_DATA(uint8_t);
+    }
+    if (dataType == halide_type_int && dataBytes == 1) {
+#ifdef MNN_USE_SSE
+        auto data = tensor->host<uint8_t>();
+        for (int z = 0; z < outside; ++z) {
+            for (int x = 0; x < width; ++x) {
+                outputOs << (static_cast<int>(data[x + z * width]) - 128) << "\t";
+            }
+            outputOs << "\n";
+        }
+#else
+        DUMP_CHAR_DATA(int8_t);
+#endif
+    }
+}
+
+static std::ofstream& getOrderFile() {
+    static std::ofstream gOrderFile("order.txt");
+    return gOrderFile;
+}
+static void _initDebug() {
+    auto& gOrderFile = getOrderFile();
+    MNN::TensorCallBackWithInfo beforeCallBack = [&](const std::vector<MNN::Tensor*>& ntensors, const MNN::OperatorInfo* info) {
+        auto opName = info->name();
+        if (info->type() == "Copy") {
+            return true;
+        }
+        auto opCopyName = opName;
+        for (int j = 0; j < opCopyName.size(); ++j) {
+            if (opCopyName[j] == '/') {
+                opCopyName[j] = '_';
+            }
+        }
+        for (int i = 0; i < ntensors.size(); ++i) {
+            auto ntensor    = ntensors[i];
+            auto outDimType = ntensor->getDimensionType();
+            std::shared_ptr<MNN::Tensor> expectTensor(new MNN::Tensor(ntensor, outDimType));
+            bool res = ntensor->copyToHostTensor(expectTensor.get());
+            if (res) {
+                ntensor = expectTensor.get();
+            }
+            std::ostringstream outputFileName;
+            outputFileName << "output/Input_" << opCopyName << "_" << i;
+            dumpTensor2File(ntensor, outputFileName.str().c_str(), getOrderFile());
+        }
+        return true;
+    };
+    MNN::TensorCallBackWithInfo callBack = [&](const std::vector<MNN::Tensor*>& ntensors,  const MNN::OperatorInfo* info) {
+        auto opName = info->name();
+        if (info->type() == "Copy") {
+            return true;
+        }
+        for (int i = 0; i < ntensors.size(); ++i) {
+            auto ntensor    = ntensors[i];
+            auto outDimType = ntensor->getDimensionType();
+            std::shared_ptr<MNN::Tensor> expectTensor(new MNN::Tensor(ntensor, outDimType));
+            bool res = ntensor->copyToHostTensor(expectTensor.get());
+            if (res) {
+                ntensor = expectTensor.get();
+            }
+            std::ostringstream outputFileName;
+            auto opCopyName = opName;
+            for (int j = 0; j < opCopyName.size(); ++j) {
+                if (opCopyName[j] == '/') {
+                    opCopyName[j] = '_';
+                }
+            }
+            auto tensor = ntensor;
+            if (tensor->dimensions() == 4) {
+                MNN_PRINT("Dimensions: 4, W,H,C,B: %d X %d X %d X %d, OP name %s : %d\n",
+                        tensor->width(), tensor->height(), tensor->channel(), tensor->batch(), opName.c_str(), i);
+            } else {
+                std::ostringstream oss;
+                for (int i = 0; i < tensor->dimensions(); i++) {
+                    oss << (i ? " X " : "") << tensor->length(i);
+                }
+
+                MNN_PRINT("Dimensions: %d, %s, OP name %s : %d\n", tensor->dimensions(), oss.str().c_str(), opName.c_str(), i);
+            }
+
+            outputFileName << "output/" << opCopyName << "_" << i;
+            dumpTensor2File(tensor, outputFileName.str().c_str(), getOrderFile());
+        }
+        return true;
+    };
+    MNN::Express::ExecutorScope::Current()->setCallBack(std::move(beforeCallBack), std::move(callBack));
+}
+
+
+struct TimeTraceInfo {
+    std::map<std::string, std::map<std::string, std::tuple<float, float, int>>> mTypes;
+
+    void begin(const MNN::OperatorInfo* info) {
+        auto tIter = mTypes.find(info->type());
+        if (tIter == mTypes.end()) {
+            std::map<std::string, std::tuple<float, float, int>> _t;
+            mTypes.insert(std::make_pair(info->type(), _t));
+            tIter = mTypes.find(info->type());
+        }
+        mInserIter = tIter->second.find(info->name());
+        if (mInserIter == tIter->second.end()) {
+            tIter->second.insert(std::make_pair(info->name(), std::make_tuple(0.0f, 0.0f, 0)));
+            mInserIter = tIter->second.find(info->name());
+        }
+        mTimer.reset();
+    }
+    void end(const MNN::OperatorInfo* info) {
+        auto timeInMs = (float)mTimer.durationInUs() / 1000.0f;
+        std::get<0>(mInserIter->second) += timeInMs;
+        std::get<1>(mInserIter->second) += info->flops();
+        std::get<2>(mInserIter->second) ++;
+    }
+    void dump(bool dumpPerOp = false) {
+        if (dumpPerOp) {
+            auto cmp = [](const std::tuple<std::string, float, float, int>& first, const std::tuple<std::string, float, float, int>& second) {
+                return std::get<1>(first) > std::get<1>(second);
+            };
+            std::priority_queue<std::tuple<std::string, float, float, int>, std::vector<std::tuple<std::string, float, float, int>>, decltype(cmp)> que(cmp);
+            for (auto& iter : mTypes) {
+                for (auto& t : iter.second) {
+                    auto mergeType = t.first + " ["+iter.first +"]";
+                    auto unit = std::make_tuple(mergeType, std::get<0>(t.second), std::get<1>(t.second), std::get<2>(t.second));
+                    que.push(unit);
+                }
+            }
+            while (!que.empty()) {
+                auto& t = que.top();
+                MNN_PRINT("%s : %.7f ms, FLOP: %.7f, COUNT: %d, Speed: %.7f GFlops\n", std::get<0>(t).c_str(), std::get<1>(t), std::get<2>(t), std::get<3>(t), std::get<2>(t) / std::get<1>(t));
+                que.pop();
+            }
+            return;
+        }
+        float opSummer       = 0.0f;
+        float opFlopsSummber = 0.0f;
+        for (auto& iter : mTypes) {
+            float summer      = 0.0f;
+            float summerflops = 0.0f;
+            int count = 0;
+            for (auto& t : iter.second) {
+                summer += std::get<0>(t.second);
+                summerflops += std::get<1>(t.second);
+                count += std::get<2>(t.second);
+            }
+            MNN_PRINT("%s : %.7f ms, FLOP: %.7f, COUNT: %d, Speed: %.7f GFlops\n", iter.first.c_str(), summer, summerflops, count,
+                          summerflops / summer);
+            opSummer += summer;
+            opFlopsSummber += summerflops;
+        }
+        MNN_PRINT("OP Summer: %.7f ms, Flops: %.7f, Speed: %.7f GFlops\n", opSummer, opFlopsSummber,
+                      opFlopsSummber / opSummer);
+    }
+private:
+    std::map<std::string, std::tuple<float, float, int>>::iterator mInserIter;
+    MNN::Timer mTimer;
+};
+static TimeTraceInfo* gTimeTraceInfo = nullptr;
+static void _initTimeTrace() {
+    static TimeTraceInfo gTime;
+    gTimeTraceInfo = &gTime;
+    MNN::TensorCallBackWithInfo beforeCallBack = [&](const std::vector<MNN::Tensor*>& ntensors, const MNN::OperatorInfo* info) {
+        gTimeTraceInfo->begin(info);
+        return true;
+    };
+    MNN::TensorCallBackWithInfo callBack = [&](const std::vector<MNN::Tensor*>& ntensors,  const MNN::OperatorInfo* info) {
+        for (auto t : ntensors) {
+            t->wait(MNN::Tensor::MAP_TENSOR_READ, true);
+        }
+        gTimeTraceInfo->end(info);
+        return true;
+    };
+    MNN::Express::ExecutorScope::Current()->setCallBack(std::move(beforeCallBack), std::move(callBack));
+}
+
+template<typename T>
+std::tuple<float, float, float> _countTensor(MNN::Tensor* tensor) {
+    auto size = tensor->elementSize();
+    auto ptr =  (T*)tensor->buffer().host;
+    float maxValue = (float)ptr[0];
+    float avgValue = (float)ptr[0];
+    float minValue = (float)ptr[0];
+    for (int i=1; i<size; ++i) {
+        maxValue = fmaxf(maxValue, (float)ptr[i]);
+        minValue = fminf(minValue, (float)ptr[i]);
+        avgValue += (float)ptr[i];
+    }
+    avgValue = avgValue / size;
+    return std::make_tuple(maxValue, minValue, avgValue);
+}
+
+static std::pair<bool, std::tuple<float, float, float>> _countForTensorValid(MNN::Tensor* ntensor) {
+    bool valid = false;
+    std::tuple<float, float, float> res;
+    if (ntensor->elementSize() <= 0) {
+        return std::make_pair(valid, res);
+    }
+    bool validforType = false;
+    if (ntensor->getType().code == halide_type_float || ntensor->getType().code == halide_type_int || ntensor->getType().code == halide_type_uint) {
+        validforType = true;
+    }
+    if (!validforType) {
+        return std::make_pair(valid, res);
+    }
+    valid = true;
+    auto outDimType = ntensor->getDimensionType();
+    std::shared_ptr<MNN::Tensor> expectTensor(new MNN::Tensor(ntensor, outDimType));
+    bool copyRes = ntensor->copyToHostTensor(expectTensor.get());
+    if (copyRes) {
+        ntensor = expectTensor.get();
+    }
+    std::tuple<float, float, float> data;
+    if (ntensor->getType().code == halide_type_float) {
+        data = _countTensor<float>(ntensor);
+    } else if (ntensor->getType().code == halide_type_int) {
+        if (ntensor->getType().bits == 32) {
+            data = _countTensor<int32_t>(ntensor);
+        } else if (ntensor->getType().bits == 8) {
+            data = _countTensor<int8_t>(ntensor);
+        }
+    } else if (ntensor->getType().code == halide_type_uint) {
+        if (ntensor->getType().bits == 32) {
+            data = _countTensor<uint32_t>(ntensor);
+        } else if (ntensor->getType().bits == 8) {
+            data = _countTensor<uint8_t>(ntensor);
+        }
+    }
+    return std::make_pair(valid, data);
+}
+static void _initTensorStatic() {
+    MNN::TensorCallBackWithInfo beforeCallBack = [&](const std::vector<MNN::Tensor*>& ntensors, const MNN::OperatorInfo* info) {
+        auto opName = info->name();
+        if (info->type() == "Copy") {
+            return true;
+        }
+        for (int i = 0; i < ntensors.size(); ++i) {
+            auto ntensor    = ntensors[i];
+            auto res = _countForTensorValid(ntensor);
+            if (!res.first) {
+                continue;
+            }
+            auto data = res.second;
+            MNN_PRINT("%s [Input] %s_%d, type:%d-%d, Max: %f, Min: %f, Avg: %f, [", info->type().c_str(), opName.c_str(), i,  ntensor->getType().code, ntensor->getType().bits, std::get<0>(data), std::get<1>(data), std::get<2>(data));
+            for (int v=0; v<ntensor->dimensions(); ++v) {
+                MNN_PRINT("%d", ntensor->length(v));
+                if (v!=ntensor->dimensions()-1) {
+                    MNN_PRINT(",");
+                }
+            }
+            MNN_PRINT("]\n");
+        }
+        return true;
+    };
+    MNN::TensorCallBackWithInfo callBack = [&](const std::vector<MNN::Tensor*>& ntensors,  const MNN::OperatorInfo* info) {
+        auto opName = info->name();
+        if (info->type() == "Copy") {
+            return true;
+        }
+        for (int i = 0; i < ntensors.size(); ++i) {
+            auto ntensor    = ntensors[i];
+            auto res = _countForTensorValid(ntensor);
+            if (!res.first) {
+                continue;
+            }
+            auto data = res.second;
+            MNN_PRINT("%s [Output] %s_%d, type:%d-%d, Max: %f, Min: %f, Avg: %f, [", info->type().c_str(), opName.c_str(), i,  ntensor->getType().code, ntensor->getType().bits, std::get<0>(data), std::get<1>(data), std::get<2>(data));
+            for (int v=0; v<ntensor->dimensions(); ++v) {
+                MNN_PRINT("%d", ntensor->length(v));
+                if (v!=ntensor->dimensions()-1) {
+                    MNN_PRINT(",");
+                }
+            }
+            MNN_PRINT("]\n");
+        }
+        return true;
+    };
+    MNN::Express::ExecutorScope::Current()->setCallBack(std::move(beforeCallBack), std::move(callBack));
+}

tools/cpp/GetMNNInfo.cpp

@@ -0,0 +1,166 @@
+//
+//  ModuleBasic.cpp
+//  MNN
+//
+//  Created by MNN on 2021/10/15.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#include "MNN_generated.h"
+#include <MNN/expr/ExecutorScope.hpp>
+#include <MNN/expr/Expr.hpp>
+#include <MNN/expr/Module.hpp>
+#include <MNN/expr/ExprCreator.hpp>
+#include <MNN/AutoTime.hpp>
+#include "rapidjson/document.h"
+#include <string>
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <cmath>
+#include <set>
+using namespace MNN::Express;
+using namespace MNN;
+static std::string _getDataType(const halide_type_t& type) {
+    switch (type.code) {
+        case halide_type_float:
+            if (type.bits == 32) {
+                return "float";
+            }
+            if (type.bits == 16) {
+                return "half";
+            }
+            break;
+        case halide_type_uint:
+            if (type.bits == 32) {
+                return "uint32";
+            }
+            if (type.bits == 16) {
+                return "uint16";
+            }
+            if (type.bits == 8) {
+                return "uint8";
+            }
+            break;
+        case halide_type_int:
+            if (type.bits == 32) {
+                return "int32";
+            }
+            if (type.bits == 16) {
+                return "int16";
+            }
+            if (type.bits == 8) {
+                return "int8";
+            }
+            break;
+        default:
+            break;
+    }
+    return "Unknown";
+}
+static std::string _getFormatString(Dimensionformat format) {
+    switch (format) {
+        case MNN::Express::NCHW:
+            return "NCHW";
+        case MNN::Express::NHWC:
+            return "NHWC";
+        case MNN::Express::NC4HW4:
+            return "NC4HW4";
+        default:
+            break;
+    }
+    return "Unknown";
+}
+int main(int argc, char *argv[]) {
+    if (argc < 2) {
+        MNN_ERROR("Usage: ./GetMNNInfo ${test.mnn}\n");
+        return 0;
+    }
+    std::string modelName = argv[1];
+    std::vector<std::string> empty;
+    std::shared_ptr<Module> net(Module::load(empty, empty, argv[1]));
+    if (nullptr == net.get()) {
+        MNN_ERROR("Load MNN from %s Failed\n", argv[1]);
+        return 1;
+    }
+    auto info = net->getInfo();
+    MNN_ASSERT(info->inputNames.size() == info->inputs.size());
+    MNN_PRINT("Model default dimensionFormat is %s\n", _getFormatString(info->defaultFormat).c_str());
+    MNN_PRINT("Model Inputs:\n");
+    for (int i=0; i<info->inputNames.size(); ++i) {
+        auto& varInfo = info->inputs[i];
+        MNN_PRINT("[ %s ]: dimensionFormat: %s, ", info->inputNames[i].c_str(), _getFormatString(varInfo.order).c_str());
+        MNN_PRINT("size: [ ");
+        if (varInfo.dim.size() > 0) {
+            for (int j=0; j<(int)varInfo.dim.size() - 1; ++j) {
+                MNN_PRINT("%d,", varInfo.dim[j]);
+            }
+            MNN_PRINT("%d ", varInfo.dim[(int)varInfo.dim.size() - 1]);
+        }
+        MNN_PRINT("], ");
+        MNN_PRINT("type is %s\n", _getDataType(varInfo.type).c_str());
+    }
+    MNN_PRINT("Model Outputs:\n");
+    for (int i=0; i<info->outputNames.size(); ++i) {
+        MNN_PRINT("[ %s ]\n", info->outputNames[i].c_str());
+    }
+    if (info->version.empty()) {
+        MNN_PRINT("Model Version: < 2.0.0\n");
+    } else {
+        MNN_PRINT("Model Version: %s \n", info->version.c_str());
+    }
+    if (!info->bizCode.empty()) {
+        MNN_PRINT("Model bizCode: %s\n", info->bizCode.c_str());
+    }
+    if (!info->metaData.empty()) {
+        MNN_PRINT("MetaData: Begin \n");
+        for (auto& iter : info->metaData) {
+            MNN_PRINT("[Meta] %s : %s\n", iter.first.c_str(), iter.second.c_str());
+        }
+        MNN_PRINT("MetaData: End \n");
+    }
+    MNN_PRINT("Get Op info to op.txt\n");
+    std::set<std::string> originTypes;
+    {
+        MNN_PRINT("Appen op lists to op.txt\n");
+        std::ifstream is("op.txt");
+        if (!is.fail()) {
+            std::string tmp;
+            while (std::getline(is, tmp, '\n')) {
+                originTypes.insert(tmp);
+            }
+        }
+    }
+    MNN_PRINT("Origin Op: %d\n", originTypes.size());
+
+    // Load origin tyes
+    {
+        std::shared_ptr<MNN::Interpreter> bufferTmp(MNN::Interpreter::createFromFile(modelName.c_str()));
+        auto buffer = bufferTmp->getModelBuffer();
+        auto collect = [&](const flatbuffers::Vector<flatbuffers::Offset<Op>>* oplists) {
+            for (int i=0; i<oplists->size(); ++i) {
+                auto op = oplists->GetAs<Op>(i);
+                originTypes.insert(EnumNameOpType(op->type()));
+            }
+        };
+        auto net = GetNet(buffer.first);
+        collect(net->oplists());
+        if (nullptr != net->subgraphs()) {
+            for (int i=0; i<net->subgraphs()->size(); ++i) {
+                auto graph = net->subgraphs()->GetAs<SubGraphProto>(i);
+                collect(graph->nodes());
+            }
+        }
+    }
+    MNN_PRINT("Current Op: %d\n", originTypes.size());
+
+    {
+        MNN_PRINT("Appen op lists to op.txt\n");
+        std::ofstream os("op.txt");
+        for (auto& s : originTypes) {
+            os << s << std::endl;
+        }
+    }
+    return 0;
+}
+

tools/cpp/GpuInterTest.cpp

@@ -0,0 +1,435 @@
+//
+//  ModuleBasic.cpp
+//  MNN
+//
+//  Created by MNN on 2021/10/15.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#include "MNN_generated.h"
+#include <MNN/expr/Expr.hpp>
+#include <MNN/expr/Module.hpp>
+#include <MNN/expr/ExprCreator.hpp>
+#define MNN_OPEN_TIME_TRACE
+#include <MNN/AutoTime.hpp>
+#include "rapidjson/document.h"
+#include "core/MemoryFormater.h"
+#include <fstream>
+#include <sstream>
+#include <numeric>
+#include "ExprDebug.hpp"
+#include "MNN/MNNSharedContext.h"
+using namespace MNN::Express;
+using namespace MNN;
+
+#ifdef __ANDROID__
+#include <dlfcn.h>
+#include <android/hardware_buffer.h>
+
+/*
+Ref from
+https://android.googlesource.com/platform/external/libchrome/+/refs/tags/aml_res_331314010/base/android/android_hardware_buffer_compat.h
+*/
+using PFAHardwareBuffer_allocate = int (*)(const AHardwareBuffer_Desc* desc,
+                                            AHardwareBuffer** outBuffer);
+using PFAHardwareBuffer_acquire = void (*)(AHardwareBuffer* buffer);
+using PFAHardwareBuffer_describe = void (*)(const AHardwareBuffer* buffer,
+                                            AHardwareBuffer_Desc* outDesc);
+using PFAHardwareBuffer_lock = int (*)(AHardwareBuffer* buffer,
+                                       uint64_t usage,
+                                       int32_t fence,
+                                       const ARect* rect,
+                                       void** outVirtualAddress);
+using PFAHardwareBuffer_recvHandleFromUnixSocket =
+    int (*)(int socketFd, AHardwareBuffer** outBuffer);
+using PFAHardwareBuffer_release = void (*)(AHardwareBuffer* buffer);
+using PFAHardwareBuffer_sendHandleToUnixSocket =
+    int (*)(const AHardwareBuffer* buffer, int socketFd);
+using PFAHardwareBuffer_unlock = int (*)(AHardwareBuffer* buffer,
+                                         int32_t* fence);
+
+class AndroidHardwareBufferCompat {
+ public:
+  bool IsSupportAvailable() const {
+    return mIsSupportAvailable;
+  }
+  AndroidHardwareBufferCompat();
+  int Allocate(const AHardwareBuffer_Desc* desc, AHardwareBuffer** outBuffer);
+  void Acquire(AHardwareBuffer* buffer);
+  void Describe(const AHardwareBuffer* buffer, AHardwareBuffer_Desc* outDesc);
+  int Lock(AHardwareBuffer* buffer,
+           uint64_t usage,
+           int32_t fence,
+           const ARect* rect,
+           void** out_virtual_address);
+  int RecvHandleFromUnixSocket(int socketFd, AHardwareBuffer** outBuffer);
+  void Release(AHardwareBuffer* buffer);
+  int SendHandleToUnixSocket(const AHardwareBuffer* buffer, int socketFd);
+  int Unlock(AHardwareBuffer* buffer, int32_t* fence);
+ private:
+  bool mIsSupportAvailable = true;
+  PFAHardwareBuffer_allocate allocate_;
+  PFAHardwareBuffer_acquire acquire_;
+  PFAHardwareBuffer_describe describe_;
+  PFAHardwareBuffer_lock lock_;
+  PFAHardwareBuffer_recvHandleFromUnixSocket recv_handle_;
+  PFAHardwareBuffer_release release_;
+  PFAHardwareBuffer_sendHandleToUnixSocket send_handle_;
+  PFAHardwareBuffer_unlock unlock_;
+};
+#define DCHECK(x) MNN_ASSERT(x)
+AndroidHardwareBufferCompat::AndroidHardwareBufferCompat() {
+  // TODO(klausw): If the Chromium build requires __ANDROID_API__ >= 26 at some
+  // point in the future, we could directly use the global functions instead of
+  // dynamic loading. However, since this would be incompatible with pre-Oreo
+  // devices, this is unlikely to happen in the foreseeable future, so just
+  // unconditionally use dynamic loading.
+  // cf. base/android/linker/modern_linker_jni.cc
+  void* main_dl_handle = dlopen(nullptr, RTLD_NOW);
+  *reinterpret_cast<void**>(&allocate_) =
+      dlsym(main_dl_handle, "AHardwareBuffer_allocate");
+  if(nullptr == allocate_){
+        mIsSupportAvailable = false;
+  }
+  *reinterpret_cast<void**>(&acquire_) =
+      dlsym(main_dl_handle, "AHardwareBuffer_acquire");
+  if(nullptr == acquire_){
+      mIsSupportAvailable = false;
+  }
+  *reinterpret_cast<void**>(&describe_) =
+      dlsym(main_dl_handle, "AHardwareBuffer_describe");
+  if(nullptr == describe_){
+      mIsSupportAvailable = false;
+  }
+  *reinterpret_cast<void**>(&lock_) =
+      dlsym(main_dl_handle, "AHardwareBuffer_lock");
+  if(nullptr == lock_){
+      mIsSupportAvailable = false;
+  }
+  *reinterpret_cast<void**>(&recv_handle_) =
+      dlsym(main_dl_handle, "AHardwareBuffer_recvHandleFromUnixSocket");
+  if(nullptr == recv_handle_){
+      mIsSupportAvailable = false;
+  }
+  *reinterpret_cast<void**>(&release_) =
+      dlsym(main_dl_handle, "AHardwareBuffer_release");
+  if(nullptr == release_){
+      mIsSupportAvailable = false;
+  }
+  *reinterpret_cast<void**>(&send_handle_) =
+      dlsym(main_dl_handle, "AHardwareBuffer_sendHandleToUnixSocket");
+  if(nullptr == send_handle_){
+      mIsSupportAvailable = false;
+  }
+  *reinterpret_cast<void**>(&unlock_) =
+      dlsym(main_dl_handle, "AHardwareBuffer_unlock");
+  if(nullptr == unlock_){
+      mIsSupportAvailable = false;
+  }
+}
+
+int AndroidHardwareBufferCompat::Allocate(const AHardwareBuffer_Desc* desc,
+                                           AHardwareBuffer** out_buffer) {
+  DCHECK(IsSupportAvailable());
+  return allocate_(desc, out_buffer);
+}
+void AndroidHardwareBufferCompat::Acquire(AHardwareBuffer* buffer) {
+  DCHECK(IsSupportAvailable());
+  acquire_(buffer);
+}
+void AndroidHardwareBufferCompat::Describe(const AHardwareBuffer* buffer,
+                                           AHardwareBuffer_Desc* out_desc) {
+  DCHECK(IsSupportAvailable());
+  describe_(buffer, out_desc);
+}
+int AndroidHardwareBufferCompat::Lock(AHardwareBuffer* buffer,
+                                      uint64_t usage,
+                                      int32_t fence,
+                                      const ARect* rect,
+                                      void** out_virtual_address) {
+  DCHECK(IsSupportAvailable());
+  return lock_(buffer, usage, fence, rect, out_virtual_address);
+}
+int AndroidHardwareBufferCompat::RecvHandleFromUnixSocket(
+    int socket_fd,
+    AHardwareBuffer** out_buffer) {
+  DCHECK(IsSupportAvailable());
+  return recv_handle_(socket_fd, out_buffer);
+}
+void AndroidHardwareBufferCompat::Release(AHardwareBuffer* buffer) {
+  DCHECK(IsSupportAvailable());
+  release_(buffer);
+}
+int AndroidHardwareBufferCompat::SendHandleToUnixSocket(
+    const AHardwareBuffer* buffer,
+    int socket_fd) {
+  DCHECK(IsSupportAvailable());
+  return send_handle_(buffer, socket_fd);
+}
+int AndroidHardwareBufferCompat::Unlock(AHardwareBuffer* buffer,
+                                        int32_t* fence) {
+  DCHECK(IsSupportAvailable());
+  return unlock_(buffer, fence);
+}
+
+static std::shared_ptr<AndroidHardwareBufferCompat> gFunction;
+
+static AHardwareBuffer* creatAHardwareBuffer(int width, int height, void *data){
+    // 创建和初始化硬件缓冲区
+    AHardwareBuffer_Desc bufferDesc = {};
+    bufferDesc.width = width;
+    bufferDesc.height = height;
+    bufferDesc.layers = 1;
+    bufferDesc.format = AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
+    bufferDesc.usage = AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN | AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE;
+
+    AHardwareBuffer* buffer = nullptr;
+    int result = gFunction->Allocate(&bufferDesc, &buffer);
+    if(result != 0) {
+        // Handle allocation error
+        MNN_PRINT("alloc AHardwareBuffer failed   %d\n", result);
+    }
+        
+    if(nullptr != data){
+        void* map = nullptr;
+        ARect rect = { 0, 0, width, height };  // Define the region to lock
+        result = gFunction->Lock(buffer, AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN, -1, &rect, &map);
+        if (result != 0) {
+            // Handle lock failure
+            MNN_PRINT("Handle lock failed\n");
+        }
+        if (map) {
+            // Now write your pixel data to 'data'
+            // For example, fill it with a solid color:
+            memcpy(map, data, width * height * 4); // Assuming RGBA8888 format
+        }
+            
+        gFunction->Unlock(buffer, nullptr);
+    }
+    return buffer;
+}
+static void copyDataFromAHardWareBuffer(AHardwareBuffer* buffer, int width, int height, void *data){
+    int result = 0;
+    if(nullptr != data){
+        void* map = nullptr;
+        ARect rect = { 0, 0, width, height };  // Define the region to lock
+        result = gFunction->Lock(buffer, AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN, -1, &rect, &map);
+        if (result != 0) {
+            MNN_PRINT("Handle lock failed\n");
+        }
+        if (map) {
+            memcpy(data, map, width * height * 4);
+        }
+            
+        gFunction->Unlock(buffer, nullptr);
+    }
+}
+static void ReleaseAHardWareBuffer(AHardwareBuffer* buffer){
+    if(buffer != nullptr){
+        gFunction->Release(buffer);
+    }
+}
+#endif
+
+int main(int argc, char *argv[]) {
+    if (argc < 3) {
+        MNN_ERROR("Usage: ./GpuInterTest.out ${test.mnn} ${Dir} [testMode] [forwardType] [numberThread] [precision | memory]\n");
+        return 0;
+    }
+    std::string modelName = argv[1];
+    std::string directName = argv[2];
+    MNN_PRINT("Test %s from input info: %s\n", modelName.c_str(), directName.c_str());
+    std::map<std::string, float> inputInfo;
+    std::map<std::string, std::vector<int>> inputShape;
+    std::vector<std::string> inputNames;
+    std::vector<std::string> outputNames;
+    int repeatNumber = 1;
+    bool shapeMutable = true;
+    std::vector<VARP> inputs;
+    std::vector<VARP> outputs;
+    if (inputNames.empty()) {
+        rapidjson::Document document;
+        std::ostringstream jsonNameOs;
+        jsonNameOs << directName << "/input.json";
+        std::ifstream fileNames(jsonNameOs.str().c_str());
+        std::ostringstream output;
+        output << fileNames.rdbuf();
+        auto outputStr = output.str();
+        document.Parse(outputStr.c_str());
+        if (document.HasParseError()) {
+            MNN_ERROR("Invalid json\n");
+            return 0;
+        }
+        if (document.HasMember("inputs")) {
+            auto inputsInfo = document["inputs"].GetArray();
+            for (auto iter = inputsInfo.begin(); iter !=inputsInfo.end(); iter++) {
+                auto obj = iter->GetObject();
+                std::string name = obj["name"].GetString();
+                inputNames.emplace_back(name);
+                MNN_PRINT("%s\n", name.c_str());
+                if (obj.HasMember("value")) {
+                    float value = obj["value"].GetFloat();
+                    inputInfo.insert(std::make_pair(name, value));
+                }
+                if (obj.HasMember("shape")) {
+                    auto dims = obj["shape"].GetArray();
+                    std::vector<int> shapes;
+                    for (auto iter = dims.begin(); iter != dims.end(); iter++) {
+                        shapes.emplace_back(iter->GetInt());
+                    }
+                    inputShape.insert(std::make_pair(name, shapes));
+                }
+            }
+        }
+        if (document.HasMember("outputs")) {
+            auto array = document["outputs"].GetArray();
+            for (auto iter = array.begin(); iter !=array.end(); iter++) {
+                std::string name = iter->GetString();
+                MNN_PRINT("output: %s\n", name.c_str());
+                outputNames.emplace_back(name);
+            }
+        }
+        if (document.HasMember("shapeMutable")) {
+            shapeMutable = document["shapeMutable"].GetBool();
+        }
+        if (document.HasMember("repeat")) {
+            repeatNumber = document["repeat"].GetInt();
+        }
+    }
+    int testMode = 0;
+    //testMode = 0 AhardwareBuffer
+    if(argc > 3){
+        testMode = atoi(argv[3]);
+        MNN_PRINT("Use extra forward type: %d(0:AhardwareBuffer)\n", testMode);
+    }
+
+    auto type = MNN_FORWARD_CPU;
+    if (argc > 4) {
+        type = (MNNForwardType)atoi(argv[4]);
+        MNN_PRINT("Use extra forward type: %d\n", type);
+    }
+
+    // Default single thread
+    int modeNum = 1;
+    if (argc > 5) {
+        modeNum = ::atoi(argv[5]);
+    }
+
+    int precision = BackendConfig::Precision_Normal;
+    int memory = BackendConfig::Memory_Normal;
+    if (argc > 6) {
+        int mask = atoi(argv[6]);
+        precision = mask % 4;
+        memory = (mask / 4) % 4;
+    }
+    const char* cacheFileName = ".tempcache";
+    FUNC_PRINT(precision);
+    FUNC_PRINT(memory);
+    FUNC_PRINT_ALL(cacheFileName, s);
+    // create session
+    MNN::ScheduleConfig config;
+    config.type      = type;
+    /*modeNum means gpuMode for GPU usage, Or means numThread for CPU usage.*/
+    config.numThread = modeNum;
+    // If type not fount, let it failed
+    config.backupType = type;
+    BackendConfig backendConfig;
+    backendConfig.precision = static_cast<MNN::BackendConfig::PrecisionMode>(precision);
+    backendConfig.memory = static_cast<MNN::BackendConfig::MemoryMode>(memory);
+    config.backendConfig     = &backendConfig;
+
+    MNN::Express::Module::Config mConfig;
+    mConfig.shapeMutable = shapeMutable;
+#ifdef __ANDROID__
+    gFunction.reset(new AndroidHardwareBufferCompat);
+    std::vector<AHardwareBuffer*> AHardwarePtrInputVec;
+    std::vector<AHardwareBuffer*> AHardwarePtrOutputVec;
+#endif
+
+    std::shared_ptr<Executor::RuntimeManager> rtmgr(Executor::RuntimeManager::createRuntimeManager(config));
+    rtmgr->setCache(cacheFileName);
+    std::shared_ptr<Module> net;
+    {
+        AUTOTIME;
+        net.reset(Module::load(inputNames, outputNames, modelName.c_str(), rtmgr, &mConfig));
+        if (net == nullptr) {
+            MNN_PRINT("Error: can't load module\n");
+            return 0;
+        }
+    }
+    auto mInfo = net->getInfo();
+#ifdef __ANDROID__
+    AHardwarePtrInputVec.resize(mInfo->inputs.size());
+    AHardwarePtrOutputVec.resize(outputNames.size());
+#endif
+    if (inputs.empty()) {
+        inputs.resize(mInfo->inputs.size());
+        for (int i=0; i<inputs.size(); ++i) {
+            inputs[i] = _Input(mInfo->inputs[i].dim, mInfo->inputs[i].order, mInfo->inputs[i].type);
+        }
+        for (int i=0; i<inputs.size(); ++i) {
+            auto inputName = inputNames[i];
+            // Resize
+            auto info = inputs[i]->getInfo();
+            int width = info->dim[3], height = info->dim[2], channel = info->dim[1];
+            auto shapeIter = inputShape.find(inputName);
+            if (shapeIter != inputShape.end()) {
+                auto s = shapeIter->second;
+                inputs[i] = _Input(s, mInfo->inputs[i].order, mInfo->inputs[i].type);
+                width = s[3];
+                height = s[2];
+                channel = s[1];
+            }
+            // set input device ptr
+#ifdef __ANDROID__
+            // OpenGL Texture defaultFormat NC4HW4
+            if(testMode == 0){
+                width = width * ((channel + 3) / 4);
+                AHardwarePtrInputVec[i] = creatAHardwareBuffer(width,height,nullptr);
+                volatile uint64_t value = (uint64_t)AHardwarePtrInputVec[i];
+                inputs[i]->setDevicePtr((void*)value, MNN_MEMORY_AHARDWAREBUFFER);
+            }
+#endif
+        }
+    }
+
+    bool modelError = false;
+    for (int repeat = 0; repeat < repeatNumber; ++repeat) {
+        AUTOTIME;
+        auto outputs = net->onForward(inputs);
+        if (outputs.empty()) {
+            MNN_ERROR("Error in forward\n");
+            return 0;
+        }
+        for (int i=0; i<outputNames.size(); ++i) {
+            auto info = outputs[i]->getInfo();
+            int width = info->dim[3], height = info->dim[2], channel = info->dim[1];
+            // copy output to device ptr
+#ifdef __ANDROID__
+            if(testMode == 0){
+                AHardwarePtrOutputVec[i] = creatAHardwareBuffer(width,height,nullptr);
+                volatile uint64_t value = (uint64_t)AHardwarePtrOutputVec[i];
+                outputs[i]->copyToDevicePtr((void*)value, MNN_MEMORY_AHARDWAREBUFFER);
+            }
+#endif
+        }
+
+        // Print module's memory
+        float memoryInMB = 0.0f;
+        rtmgr->getInfo(Interpreter::MEMORY, &memoryInMB);
+        FUNC_PRINT_ALL(memoryInMB, f);
+    }
+#ifdef __ANDROID__
+    if(testMode == 1){
+        for(int i = 0; i < AHardwarePtrInputVec.size(); ++i){
+            ReleaseAHardWareBuffer(AHardwarePtrInputVec[i]);
+        }
+        for(int i = 0; i < AHardwarePtrOutputVec.size(); ++i){
+            ReleaseAHardWareBuffer(AHardwarePtrOutputVec[i]);
+        }
+    }
+#endif
+    rtmgr->updateCache();
+    return 0;
+}
+

tools/cpp/HelperFuncs.hpp

@@ -0,0 +1,32 @@
+//
+//  HelperFuncs.hpp
+//  MNN
+//
+//  Created by MNN on 2021/07/08.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#ifndef HELPERFUNCS_HPP
+#define HELPERFUNCS_HPP
+
+#include <string>
+#include <fstream>
+#include <sstream>
+#include "MNN_generated.h"
+
+namespace HelperFuncs {
+
+std::string getModelUUID(std::string modelFile) {
+    std::unique_ptr<MNN::NetT> netT;
+    std::ifstream input(modelFile);
+    std::ostringstream outputOs;
+    outputOs << input.rdbuf();
+    netT = MNN::UnPackNet(outputOs.str().c_str());
+    auto net = netT.get();
+
+    return net->mnn_uuid;
+}
+
+} // namespace HelperFuncs
+
+#endif // HELPERFUNCS_HPP

tools/cpp/MNN2QNNModel.cpp

@@ -0,0 +1,558 @@
+#include <MNN/expr/Module.hpp>
+#include <MNN/expr/ExprCreator.hpp>
+#include <MNN/expr/ExecutorScope.hpp>
+#include "core/OpCommonUtils.hpp"
+#include "MNN_generated.h"
+#include <cstdlib>
+#include <iostream>
+#include <sstream>
+#include <fstream>
+#include <rapidjson/document.h>
+#include "rapidjson/prettywriter.h"
+#include "rapidjson/stringbuffer.h"
+#include "core/MNNFileUtils.h"
+#ifndef _WIN32
+#include <sys/utsname.h>
+#endif
+
+using namespace rapidjson;
+
+using namespace MNN::Express;
+using namespace MNN;
+
+static bool generateConfigFile(const std::string & qnnSDKPath, int socID, int dspArch, const std::vector<std::string> & graphNameVec, const std::string & outputDir, std::string & configPath, std::string & subConfigPath) {
+    configPath = MNNFilePathConcat(outputDir, "context_config.json");
+    subConfigPath = MNNFilePathConcat(outputDir, "htp_backend_extensions.json");
+
+    // Write context_config.json
+    rapidjson::Document contextConfigDoc;
+    contextConfigDoc.SetObject();
+    rapidjson::Document::AllocatorType& contextAllocator = contextConfigDoc.GetAllocator();
+    rapidjson::Value backendExtensions(rapidjson::kObjectType);
+    std::string htpBackendExtPath = MNNFilePathConcat(qnnSDKPath, "lib/x86_64-linux-clang/libQnnHtpNetRunExtensions.so");
+    backendExtensions.AddMember("shared_library_path", rapidjson::Value(htpBackendExtPath.c_str(), contextAllocator).Move(), contextAllocator);
+    backendExtensions.AddMember("config_file_path", rapidjson::Value(subConfigPath.c_str(), contextAllocator).Move(), contextAllocator);
+    contextConfigDoc.AddMember("backend_extensions", backendExtensions, contextAllocator);
+    rapidjson::StringBuffer contextBuffer;
+    rapidjson::PrettyWriter<rapidjson::StringBuffer> contextWriter(contextBuffer);
+    contextConfigDoc.Accept(contextWriter);
+    std::ofstream contextConfigOut(configPath);
+    contextConfigOut << contextBuffer.GetString();
+    contextConfigOut.close();
+
+    // Write htp_backend_extensions.json
+    rapidjson::Document htpConfigDoc;
+    htpConfigDoc.SetObject();
+    rapidjson::Document::AllocatorType& htpConfigAllocator = htpConfigDoc.GetAllocator();
+
+    // "graphs" section
+    rapidjson::Value graphs(rapidjson::kArrayType);
+    rapidjson::Value graphObj(rapidjson::kObjectType);
+    graphObj.AddMember("vtcm_mb", 8, htpConfigAllocator);
+    rapidjson::Value names(rapidjson::kArrayType);
+    for (const auto& name : graphNameVec) {
+        names.PushBack(rapidjson::Value(name.c_str(), contextAllocator).Move(), htpConfigAllocator);
+    }
+    graphObj.AddMember("graph_names", names, htpConfigAllocator);
+    graphObj.AddMember("O", 3.0, htpConfigAllocator);
+    graphObj.AddMember("fp16_relaxed_precision", 1, htpConfigAllocator);
+    graphObj.AddMember("weights_packing", true, htpConfigAllocator);
+    graphObj.AddMember("hvx_threads", 4, htpConfigAllocator);
+    graphs.PushBack(graphObj, htpConfigAllocator);
+    htpConfigDoc.AddMember("graphs", graphs, htpConfigAllocator);
+
+    // "devices" section
+    rapidjson::Value devices(rapidjson::kArrayType);
+    rapidjson::Value deviceObj(rapidjson::kObjectType);
+    deviceObj.AddMember("soc_id", socID, htpConfigAllocator);
+    std::string hexagonArchStr = "v" + std::to_string(dspArch);
+    deviceObj.AddMember("dsp_arch", rapidjson::Value(hexagonArchStr.c_str(), contextAllocator).Move(), htpConfigAllocator);
+    rapidjson::Value cores(rapidjson::kArrayType);
+    rapidjson::Value coreObj(rapidjson::kObjectType);
+    coreObj.AddMember("core_id", 0, htpConfigAllocator);
+    coreObj.AddMember("perf_profile", "burst", htpConfigAllocator);
+    coreObj.AddMember("rpc_control_latency", 100, htpConfigAllocator);
+    cores.PushBack(coreObj, htpConfigAllocator);
+    deviceObj.AddMember("cores", cores, htpConfigAllocator);
+    devices.PushBack(deviceObj, htpConfigAllocator);
+    htpConfigDoc.AddMember("devices", devices, htpConfigAllocator);
+
+    // "context" section
+    rapidjson::Value contextObj(rapidjson::kObjectType);
+    contextObj.AddMember("weight_sharing_enabled", true, htpConfigAllocator);
+    htpConfigDoc.AddMember("context", contextObj, htpConfigAllocator);
+
+    rapidjson::StringBuffer htpConfigBuffer;
+    rapidjson::PrettyWriter<rapidjson::StringBuffer> htpConfigWriter(htpConfigBuffer);
+    htpConfigDoc.Accept(htpConfigWriter);
+    std::ofstream htpConfigOut(subConfigPath);
+    htpConfigOut << htpConfigBuffer.GetString();
+    htpConfigOut.close();
+    return true;
+}
+
+static bool parseDims(const std::string& s, std::vector<std::vector<int>>& out) {
+    auto isLegal = [](char c) { 
+        return c == 'x' || c == '_' || (c >= '0' && c <= '9'); 
+    };
+    bool allLegal = std::all_of(s.begin(), s.end(), isLegal);
+    if(!allLegal) {
+        return false;
+    }
+
+    out.clear();
+    std::stringstream ss(s);
+    std::string segment;
+    MNN_PRINT("param dims: %s\n", s.c_str());
+    while (std::getline(ss, segment, '_')) {
+        if (segment.empty()) {
+            MNN_ERROR("%s parse error, format should be like 1x3x512x512_1x256\n", s.c_str());
+            return false;
+        }
+        std::vector<int> dims;
+        std::stringstream inner(segment);
+        std::string token;
+
+        while (std::getline(inner, token, 'x')) {
+            if (token.empty()) {
+                MNN_ERROR("%s parse error, format should be like 1x3x512x512_1x256\n", s.c_str());
+                return false;
+            }
+            int val = std::stoi(token);
+            dims.push_back(val);
+        }
+        if (dims.empty()) {
+            MNN_ERROR("%s parse error, format should be like 1x3x512x512_1x256\n", s.c_str());
+            return false;
+        }
+        out.push_back(dims);
+    }
+    return true;
+}
+
+static bool checkSystem() {
+#ifdef _WIN32
+    // On Windows, skip the system check as this tool is primarily for Linux
+    // but we allow it to compile on Windows for development purposes
+    MNN_PRINT("Warning: This tool is designed for x86_64 Linux systems. Running on Windows may have limitations.\n");
+    return true;
+#else
+    struct utsname buf;
+    if (uname(&buf) != 0) {
+        MNN_ERROR("uname error\n");
+        return false;
+    }
+    if (std::string(buf.sysname) == "Linux" && std::string(buf.machine) == "x86_64") {
+        return true;
+    }
+    MNN_ERROR("This program must be run on a x86_64 Linux system. Current system: %s %s\n", buf.sysname, buf.machine);
+    return false;
+#endif
+}
+
+int main(int argc, const char* argv[]) {
+    if (argc < 6) {
+        MNN_PRINT("This tool generates offline caches for the QNN backend.");
+        MNN_PRINT("Usage: %s <qnnSDKPath> <socId> <hexagonArch> <srcMNNPath> <outputDir> [totalShapeNum] [inputShape1] [inputShape2] ...\n", argv[0]);
+        MNN_PRINT("    <qnnSDKPath>      : Path to the QNN SDK directory.\n");
+        MNN_PRINT("    <socId>           : Target SoC ID.\n");
+        MNN_PRINT("                        Common SoCs: 8Gen2 -> 43, 8Gen3 -> 57, 8 Elite -> 69. For others, please refer to Qualcomm's documentation.\n");
+        MNN_PRINT("    <hexagonArch>     : Hexagon architecture version. This tool requires v73 or higher for weight sharing.\n");
+        MNN_PRINT("                        Common Archs: 8Gen2 -> 73, 8Gen3 -> 75, 8 Elite -> 79. For others, please refer to Qualcomm's documentation.\n");
+        MNN_PRINT("    <srcMNNPath>      : Path to the source MNN model file.\n");
+        MNN_PRINT("    <outputDir>       : Directory to save the generated files, including a MNN model file with the suffix '.mnn' and a QNN serialized artifact with the suffix '.bin'.\n");
+        MNN_PRINT("    [<totalShapeNum>] : Optional. Number of dynamic input shape configurations.\n");
+        MNN_PRINT("    [<inputShapeN>]   : Optional. Input shape configuration. Can be a shape string or a path to a .mnn file.\n");
+        MNN_PRINT("                     Shape string format for multiple inputs: dim1xdim2_dim3xdim4. Example: 1x3x512x512_1x256\n");
+        MNN_PRINT("Examples:\n");
+        MNN_PRINT("  1. Use default shape from the MNN model:\n");
+        MNN_PRINT("     %s /path/to/qnn/sdk 57 75 /path/to/model.mnn /path/to/output\n", argv[0]);
+        MNN_PRINT("  2. Specify two dynamic input shapes:\n");
+        MNN_PRINT("     %s /path/to/qnn/sdk 57 75 /path/to/model.mnn /path/to/output 2 1x3x512x512_1x256 1x3x256x256_1x128\n", argv[0]);
+        MNN_PRINT("     %s /path/to/qnn/sdk 57 75 /path/to/model.mnn /path/to/output 2 input_0.mnn input_1.mnn\n", argv[0]);
+
+        return 1;
+    }
+
+    if (!checkSystem()) {
+        return -1;
+    }
+
+    std::string qnnSdkPath = argv[1];
+    int socId = std::stoi(std::string(argv[2]));
+    int hexagonArch = std::stoi(std::string(argv[3]));
+    const char* srcMNNPath = argv[4];
+    std::string modelBaseName = [](const std::string& path) {
+        std::string filename = path;
+        auto pos = path.find_last_of("/\\");
+        if (pos != std::string::npos) {
+            filename = path.substr(pos + 1);
+        }
+        pos = filename.find_last_of('.');
+        if (pos != std::string::npos) {
+            return filename.substr(0, pos);
+        }
+        return filename;
+    }(srcMNNPath);
+    std::string modelSignature = "_" + std::to_string(socId) + "_" + std::to_string(hexagonArch);
+    std::string outputDir = argv[5];
+    std::string dstMNNPath = MNNFilePathConcat(outputDir, modelBaseName + modelSignature + ".mnn");
+
+    std::vector<std::string> inputNames;
+    std::vector<std::string> outputNames;
+    std::vector<MNN::Express::VARP> inputs;
+    std::vector<MNN::Express::VARP> outputs;
+    std::vector<std::vector<std::vector<int>>> inputShapeLists;
+    bool hasInputsVarp = false;
+    std::vector<std::vector<MNN::Express::VARP>> inputsVarpList;
+
+    int totalShapeType = 1;
+    if(argc > 6) {
+        totalShapeType = std::stoi(argv[6]);
+        std::vector<std::vector<int>> temp;
+        if(parseDims(argv[7], temp)) {
+            inputShapeLists.resize(totalShapeType);
+            for(int i = 0; i < totalShapeType; i++) {
+                // Each inputs shape in model: 128x1x897_1x1x128x128_1x128
+                if(!parseDims(argv[7+i], inputShapeLists[i])) {
+                    return -1;
+                }
+            }
+        } else {
+            inputsVarpList.resize(totalShapeType);
+            for(int i = 0; i < totalShapeType; i++) {
+                inputsVarpList[i] = MNN::Express::Variable::load(argv[7+i]);
+            }
+            inputs = MNN::Express::Variable::load(argv[7]);
+            for (int i=0; i<inputs.size(); ++i) {
+                inputNames.emplace_back(inputs[i]->name());
+            }
+            if(argc > 7+totalShapeType) {
+                outputs = MNN::Express::Variable::load(argv[7+totalShapeType]);
+                for (int i=0; i<outputs.size(); ++i) {
+                    outputNames.emplace_back(outputs[i]->name());
+                }
+            }
+            hasInputsVarp = true;
+        }
+    }
+
+    /**
+    generate qnn .cpp and .bin
+    */
+    std::string totalQnnSo;
+    std::vector<std::string> qnnGraphNames;
+    std::vector<std::vector<MNN::Express::Variable::Info>> outputInfos;
+    std::vector<std::string> qnnModelDirs;
+    std::vector<int> allInputShape;
+
+    MNN_PRINT("Total input shape type size:%d\n", totalShapeType);
+    for(int index = 0; index < totalShapeType; index++)
+    {
+        std::string curQnnModelName = modelBaseName + std::string("_") + std::to_string(index);
+        qnnGraphNames.emplace_back(curQnnModelName);
+        std::string curQnnModelDir = MNNFilePathConcat(outputDir, curQnnModelName);
+        MNN_PRINT("[Temp Product]: Qnn temp product generate at %s\n", curQnnModelDir.c_str());
+        MNNCreateDir(curQnnModelDir.c_str());
+        qnnModelDirs.push_back(curQnnModelDir);
+        if(index < totalShapeType-1) {
+            totalQnnSo += (curQnnModelDir + std::string("/lib/x86_64-linux-clang/lib") + \
+                curQnnModelName + std::string(".so,"));
+        } else {
+            totalQnnSo += (curQnnModelDir + std::string("/lib/x86_64-linux-clang/lib") + \
+                curQnnModelName + std::string(".so "));
+        }
+
+        MNN::ScheduleConfig config;
+        config.type = MNN_CONVERT_QNN;
+        std::shared_ptr<Executor::RuntimeManager> rtmgr(Executor::RuntimeManager::createRuntimeManager(config));
+        rtmgr->setCache(curQnnModelDir.c_str());
+        MNN::Express::Module::Config mConfig;
+        mConfig.shapeMutable = false;
+        std::shared_ptr<MNN::Express::Module> m(MNN::Express::Module::load(inputNames, outputNames, srcMNNPath, rtmgr, &mConfig), MNN::Express::Module::destroy);
+        auto minfo = m->getInfo();
+        if(outputNames.empty()) {
+            outputNames = minfo->outputNames;
+        }
+        if(inputNames.empty()) {
+            inputNames = minfo->inputNames;
+        }
+
+        if(!hasInputsVarp) {
+            inputs.resize(minfo->inputs.size());
+            for (int i=0; i<minfo->inputs.size(); ++i) {
+                auto info = minfo->inputs[i];
+                std::vector<int> inputDims = info.dim;
+                if(!inputShapeLists.empty()) {
+                    inputDims = inputShapeLists[index][i];
+                }
+                MNN_PRINT("input %d shape:", i);
+                for(int d = 0; d < inputDims.size(); d++) {
+                    MNN_PRINT("%d ", inputDims[d]);
+                }
+                MNN_PRINT("\n");
+                auto varp = MNN::Express::_Input(inputDims, info.order, info.type);
+                varp->writeMap<void>();
+                inputs[i] = varp;
+                inputs[i]->setName(inputNames[i]);
+            }
+        } else {
+            inputs = inputsVarpList[index];
+        }
+        outputs = m->onForward(inputs);
+        // sync
+        for(int i = 0; i < outputs.size(); i++) {
+            outputs[i]->readMap<void>();
+        }
+
+        // tar weight
+        std::string binPath = MNNFilePathConcat(curQnnModelDir, curQnnModelName + ".bin");
+        std::string command = "tar -cf " + binPath + " -C " + curQnnModelDir + " $(find " + curQnnModelDir + " -maxdepth 1 -name '*.raw' -printf '%f ') && rm " + curQnnModelDir + "/*.raw";
+        int ret = std::system(command.c_str());
+        if (ret != 0) {
+            MNN_ERROR("Failed to execute command: %s\n", command.c_str());
+        }
+
+        std::string modelLibCmd = qnnSdkPath + "/bin/x86_64-linux-clang/qnn-model-lib-generator" + \
+            " -c " + MNNFilePathConcat(curQnnModelDir, curQnnModelName + ".cpp") + \
+            " -b " + binPath + \
+            " -t x86_64-linux-clang " + \
+            " -o " + curQnnModelDir + "/lib";
+        ret = system(modelLibCmd.c_str());
+        if(ret) {
+            MNN_ERROR("[Error]: qnn-model-lib-generator error!\n");
+            return -1;
+        } else {
+            MNN_PRINT("[Pass]: qnn-model-lib-generator success!\n");
+        }
+
+
+        std::vector<MNN::Express::Variable::Info> inputInfos(inputs.size());
+        for (int i=0; i<inputInfos.size(); ++i) {
+            inputInfos[i] = *inputs[i]->getInfo();
+        }
+        std::vector<int> currInputShape;
+        for (int i = 0; i < inputInfos.size(); i++) {
+            for (int j = 0; j < inputInfos[i].dim.size(); j++) {
+                currInputShape.emplace_back(inputInfos[i].dim[j]);
+            }
+        }
+        allInputShape.insert(allInputShape.end(), currInputShape.begin(), currInputShape.end());
+
+        std::vector<MNN::Express::Variable::Info> outputInfo(outputs.size());
+        for (int i=0; i<outputInfo.size(); ++i) {
+            outputInfo[i] = *outputs[i]->getInfo();
+        }
+        outputInfos.emplace_back(outputInfo);
+        
+    }
+
+    std::string npuArtifactName = modelBaseName + modelSignature + ".bin";
+    std::string npuArtifactPath = MNNFilePathConcat(outputDir, npuArtifactName);
+    {
+        std::string configPath, subConfigPath;
+        if (!generateConfigFile(qnnSdkPath, socId, hexagonArch, qnnGraphNames, outputDir, configPath, subConfigPath)) {
+            MNN_ERROR("[Error]: Failed to generate the config file!\n");
+            return -1;
+        }
+
+        std::string binaryGenCmd = qnnSdkPath + "/bin/x86_64-linux-clang/qnn-context-binary-generator" + \
+            " --model " + totalQnnSo + \
+            " --backend " + qnnSdkPath + "/lib/x86_64-linux-clang/libQnnHtp.so" + \
+            " --binary_file " + modelBaseName + modelSignature + \
+            " --config_file " + configPath + " " + \
+            " --output_dir " + outputDir;
+        auto res = system(binaryGenCmd.c_str());
+        if(res) {
+            MNN_ERROR("[Error]: qnn-context-binary-generator error!\n");
+            return -1;
+        } else {
+            MNN_PRINT("[Pass]: qnn-context-binary-generator success!\n");
+        }
+
+        // Remove intermediate files
+        MNNRemoveFile(configPath.c_str());
+        MNNRemoveFile(subConfigPath.c_str());
+        for (const auto& dir : qnnModelDirs) {
+            std::string cmd = "rm -rf " + dir;
+            int ret = system(cmd.c_str());
+            if (ret != 0) {
+                MNN_PRINT("[Warning]: failed to remove temp dir: %s\n", dir.c_str());
+            }
+        }
+    }
+
+    std::vector<MNN::Express::Variable::Info> inputInfos(inputs.size());
+    for (int i=0; i<inputInfos.size(); ++i) {
+        inputInfos[i] = *inputs[i]->getInfo();
+    }
+
+
+    // Get inputs/outputs index in mnn model
+    std::vector<int> inputIndexes(inputNames.size());
+    std::vector<int> outputIndexes(outputNames.size());
+    {
+        std::shared_ptr<MNN::Interpreter> netC(MNN::Interpreter::createFromFile(srcMNNPath), MNN::Interpreter::destroy);
+        auto bufferPair = netC->getModelBuffer();
+        auto buffer = bufferPair.first;
+        auto length = bufferPair.second;
+        auto net = GetNet(buffer);
+
+        for (int i=0; i<net->tensorName()->size(); ++i) {
+            auto tname = net->tensorName()->GetAsString(i)->str();
+            for (int j=0; j<inputNames.size(); ++j) {
+                if (tname == inputNames[j]) {
+                    inputIndexes[j] = i;
+                    break;
+                }
+            }
+            for (int j=0; j<outputNames.size(); ++j) {
+                if (tname == outputNames[j]) {
+                    outputIndexes[j] = i;
+                    break;
+                }
+            }
+        }
+    }
+
+    std::shared_ptr<MNN::NetT> dstNet(new NetT);
+
+    for (int i=0; i<inputInfos.size(); ++i) {
+        std::unique_ptr<OpT> input(new OpT);
+        input->type = OpType_Input;
+        auto param(new InputT);
+        param->dims = inputInfos[i].dim;
+
+        input->main.type = OpParameter_Input;
+        input->main.value = param;
+        input->name = inputNames[i];
+        input->outputIndexes.push_back(i);
+        dstNet->oplists.emplace_back(std::move(input));
+    }
+
+    /** Fuse to Op*/
+    std::unique_ptr<MNN::OpT> op(new OpT);
+    for(int i = 0; i < inputs.size(); i++) {
+        op->inputIndexes.push_back(i);
+    }
+    for(int i = 0; i < outputs.size(); i++) {
+        op->outputIndexes.push_back(inputs.size() + i);
+    }
+    op->name = "qnn/plugin/op";
+    op->main.Reset();
+    op->type = MNN::OpType_Plugin;
+    op->main.type = MNN::OpParameter_Plugin;
+    op->main.value = new MNN::PluginT;
+    auto extra = op->main.AsPlugin();
+    extra->type = "QNN";
+    std::unique_ptr<MNN::AttributeT> attr(new MNN::AttributeT);
+
+    
+    dstNet->tensorName = inputNames;
+    dstNet->tensorName.insert(dstNet->tensorName.end(), outputNames.begin(), outputNames.end());
+    dstNet->tensorName.push_back(op->name);
+    dstNet->outputName = outputNames;
+
+    attr->key = "allInputShape";
+    attr->list.reset(new ListValueT);
+    attr->list->i.insert(attr->list->i.end(), allInputShape.begin(), allInputShape.end());
+    extra->attr.emplace_back(std::move(attr));
+    attr.reset(new MNN::AttributeT);
+
+    attr->key = "allGraphName";
+    attr->list.reset(new ListValueT);
+    attr->list->s.resize(qnnGraphNames.size());
+    for(int i = 0; i < qnnGraphNames.size(); i++) {
+        attr->list->s[i] = qnnGraphNames[i];
+    }
+    extra->attr.emplace_back(std::move(attr));
+    attr.reset(new MNN::AttributeT);
+
+    attr->key = "path";
+    attr->s = npuArtifactName;
+    extra->attr.emplace_back(std::move(attr));
+    attr.reset(new MNN::AttributeT);
+
+    attr->key = "offset";
+    attr->list.reset(new MNN::ListValueT);
+    attr->list->i.push_back(0);
+    attr->list->i.push_back(0);
+    extra->attr.emplace_back(std::move(attr));
+    attr.reset(new MNN::AttributeT);
+
+    file_t binaryFile = MNNOpenFile(npuArtifactPath.c_str(), MNN_FILE_READ);
+    size_t binarySize = MNNGetFileSize(binaryFile);
+    MNNCloseFile(binaryFile);
+    attr->key = "size";
+    attr->list.reset(new MNN::ListValueT);
+    uint32_t lowSrc = binarySize & 0xFFFFFFFF;
+    uint32_t highSrc = binarySize >> 32;
+    int32_t lowDst, highDst;
+    ::memcpy(&lowDst, &lowSrc, sizeof(int32_t));
+    ::memcpy(&highDst, &highSrc, sizeof(int32_t));
+    attr->list->i.push_back(lowDst);
+    attr->list->i.push_back(highDst);
+    extra->attr.emplace_back(std::move(attr));
+    attr.reset(new MNN::AttributeT);
+
+    attr->key = "inputs";
+    attr->list.reset(new ListValueT);
+    attr->list->s.resize(inputNames.size());
+    for (int i=0; i<inputNames.size(); ++i) {
+        // ::TODO
+        attr->list->s[i] = std::string("t") + std::to_string(inputIndexes[i]);
+    }
+    extra->attr.emplace_back(std::move(attr));
+    attr.reset(new MNN::AttributeT);
+    attr->key = "outputs";
+    attr->list.reset(new ListValueT);
+    attr->list->s.resize(outputNames.size());
+    for (int i=0; i<outputNames.size(); ++i) {
+        // ::TODO
+        attr->list->s[i] = std::string("t") + std::to_string(outputIndexes[i]);
+    }
+    extra->attr.emplace_back(std::move(attr));
+
+
+    for (int i=0; i<outputInfos.size(); ++i) {
+        attr.reset(new MNN::AttributeT);
+        for(int j = 0; j < outputInfos[i].size(); j++) {
+            attr->key = "o_" + std::to_string(i) + std::string("_") +  std::to_string(j);
+            attr->tensor.reset(new BlobT);
+            attr->tensor->dataType = OpCommonUtils::convertDataType(outputInfos[i][j].type);
+            attr->tensor->dims = outputInfos[i][j].dim;
+            switch(outputInfos[i][j].order) {
+                case MNN::Express::NHWC:
+                    attr->tensor->dataFormat = MNN_DATA_FORMAT_NHWC;
+                    break;
+                case MNN::Express::NCHW:
+                    attr->tensor->dataFormat = MNN_DATA_FORMAT_NCHW;
+                    break;
+                case MNN::Express::NC4HW4:
+                    attr->tensor->dataFormat = MNN_DATA_FORMAT_NC4HW4;
+                    break;
+                default:
+                    attr->tensor->dataFormat = MNN_DATA_FORMAT_NCHW;
+                    break;
+            }
+        }
+        extra->attr.emplace_back(std::move(attr));
+    }
+
+    // Compile NPU Module
+    std::unique_ptr<OpT> npuOp;
+    npuOp = std::move(op);
+
+    // Merge to dst
+    dstNet->oplists.emplace_back(std::move(npuOp));
+
+    // Store
+    flatbuffers::FlatBufferBuilder builder;
+    builder.Finish(Net::Pack(builder, dstNet.get()));
+    std::ofstream outputOs(dstMNNPath.c_str(), std::ios::binary);
+    outputOs.write((const char*)builder.GetBufferPointer(), builder.GetSize());
+    outputOs.close();
+
+    MNN_PRINT("[All passed]\n");
+    return 0;
+}

tools/cpp/MNNV2Basic.cpp

@@ -0,0 +1,552 @@
+//
+//  MNNV2Basic.cpp
+//  MNN
+//
+//  Created by MNN on 2019/01/22.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#define MNN_OPEN_TIME_TRACE
+
+#include <stdlib.h>
+#include <algorithm>
+#include <cstring>
+#include <fstream>
+#include <iostream>
+#include <map>
+#include <memory>
+#include <sstream>
+#include <string>
+#if defined(_MSC_VER)
+#include <Windows.h>
+#undef min
+#undef max
+#else
+#include <sys/time.h>
+#endif
+#include <MNN/MNNDefine.h>
+#include <MNN/AutoTime.hpp>
+#include <MNN/Interpreter.hpp>
+#include <MNN/Tensor.hpp>
+#include <core/Backend.hpp>
+#include <core/TensorUtils.hpp>
+#include <MNN_generated.h>
+
+//#define FEED_INPUT_NAME_VALUE
+
+using namespace MNN;
+
+#define DUMP_NUM_DATA(type)                          \
+    auto data = tensor->host<type>();                \
+    for (int z = 0; z < outside; ++z) {              \
+        for (int x = 0; x < width; ++x) {            \
+            outputOs << data[x + z * width] << "\t"; \
+        }                                            \
+        outputOs << "\n";                            \
+    }
+
+#define DUMP_CHAR_DATA(type)                                           \
+    auto data = tensor->host<type>();                                  \
+    for (int z = 0; z < outside; ++z) {                                \
+        for (int x = 0; x < width; ++x) {                              \
+            outputOs << static_cast<int>(data[x + z * width]) << "\t"; \
+        }                                                              \
+        outputOs << "\n";                                              \
+    }
+
+static void dumpTensor2File(const Tensor* tensor, const char* file, std::ofstream& orderFile) {
+    orderFile << file << std::endl;
+    std::ofstream outputOs(file);
+    auto type = tensor->getType();
+
+    int dimension = tensor->buffer().dimensions;
+    int width     = 1;
+    if (dimension > 1) {
+        width = tensor->length(dimension - 1);
+    }
+
+    const int outside = tensor->elementSize() / width;
+
+    const auto dataType  = type.code;
+    const auto dataBytes = type.bytes();
+
+    if (dataType == halide_type_float) {
+        DUMP_NUM_DATA(float);
+    }
+    if (dataType == halide_type_int && dataBytes == 4) {
+        DUMP_NUM_DATA(int32_t);
+    }
+    if (dataType == halide_type_uint && dataBytes == 1) {
+        DUMP_CHAR_DATA(uint8_t);
+    }
+    if (dataType == halide_type_int && dataBytes == 1) {
+#ifdef MNN_USE_SSE
+        auto data = tensor->host<uint8_t>();
+        for (int z = 0; z < outside; ++z) {
+            for (int x = 0; x < width; ++x) {
+                outputOs << (static_cast<int>(data[x + z * width]) - 128) << "\t";
+            }
+            outputOs << "\n";
+        }
+#else
+        DUMP_CHAR_DATA(int8_t);
+#endif
+    }
+}
+
+static void _loadInputFromFile(Tensor* inputTensor, std::string pwd, std::string name) {
+    MNN::Tensor givenTensor(inputTensor, inputTensor->getDimensionType());
+    {
+        int size_w = inputTensor->width();
+        int size_h = inputTensor->height();
+        int bpp    = inputTensor->channel();
+        int batch  = inputTensor->batch();
+        MNN_PRINT("Input size:%d\n", inputTensor->elementSize());
+        inputTensor->printShape();
+
+        std::ostringstream fileName;
+        fileName << pwd << name;
+        std::ifstream input(fileName.str().c_str());
+        FUNC_PRINT_ALL(fileName.str().c_str(), s);
+
+        if (givenTensor.getType().code == halide_type_int) {
+            auto size           = givenTensor.elementSize();
+            const auto bytesLen = givenTensor.getType().bytes();
+            if (bytesLen == 4) {
+                auto inputData = givenTensor.host<int32_t>();
+                double temp;
+                for (int i = 0; i < size; ++i) {
+                    input >> temp;
+                    inputData[i] = temp;
+                }
+            } else if (bytesLen == 1) {
+                auto inputData = givenTensor.host<int8_t>();
+                double pixel      = 0;
+                for (int i = 0; i < size; ++i) {
+                    input >> pixel;
+                    inputData[i] = static_cast<int8_t>(pixel);
+                }
+            }
+        } else if (givenTensor.getType().code == halide_type_uint) {
+            auto size = givenTensor.elementSize();
+            {
+                FUNC_PRINT(givenTensor.getType().bytes());
+                auto inputData = givenTensor.host<uint8_t>();
+                for (int i = 0; i < size; ++i) {
+                    double p;
+                    input >> p;
+                    inputData[i] = (uint8_t)p;
+                }
+            }
+        } else if (givenTensor.getType().code == halide_type_float) {
+            auto inputData = givenTensor.host<float>();
+            auto size      = givenTensor.elementSize();
+            for (int i = 0; i < size; ++i) {
+                input >> inputData[i];
+                // inputData[i] = 1.0f;
+            }
+        }
+        inputTensor->copyFromHostTensor(&givenTensor);
+    }
+}
+
+static inline int64_t getTimeInUs() {
+    uint64_t time;
+#if defined(_MSC_VER)
+    LARGE_INTEGER now, freq;
+    QueryPerformanceCounter(&now);
+    QueryPerformanceFrequency(&freq);
+    uint64_t sec  = now.QuadPart / freq.QuadPart;
+    uint64_t usec = (now.QuadPart % freq.QuadPart) * 1000000 / freq.QuadPart;
+    time          = sec * 1000000 + usec;
+#else
+    struct timeval tv;
+    gettimeofday(&tv, nullptr);
+    time = static_cast<uint64_t>(tv.tv_sec) * 1000000 + tv.tv_usec;
+#endif
+    return time;
+}
+
+static inline std::vector<int> parseIntList(const std::string& str, char delim) {
+    std::vector<int> result;
+    std::ptrdiff_t p1 = 0, p2;
+    while (1) {
+        p2 = str.find(delim, p1);
+        if (p2 != std::string::npos) {
+            result.push_back(atoi(str.substr(p1, p2 - p1).c_str()));
+            p1 = p2 + 1;
+        } else {
+            result.push_back(atoi(str.substr(p1).c_str()));
+            break;
+        }
+    }
+    return result;
+}
+
+static int test_main(int argc, const char* argv[]) {
+    if (argc < 2) {
+        MNN_PRINT("=========================================================================================\n");
+        MNN_PRINT("Arguments: model.MNN runLoops runMask forwardType numberThread precision inputSize cpuIds\n");
+        MNN_PRINT("Example: %s model.MNN 100 0 0 4 0 1x3x224x224 0,1,2,3\n", argv[0]);
+        MNN_PRINT("=========================================================================================\n");
+        return -1;
+    }
+
+    std::string cmd = argv[0];
+    std::string pwd = "./";
+    auto rslash     = cmd.rfind("/");
+    if (rslash != std::string::npos) {
+        pwd = cmd.substr(0, rslash + 1);
+    }
+
+    // read args
+    const char* fileName = argv[1];
+
+    int runTime = 1;
+    if (argc > 2) {
+        runTime = ::atoi(argv[2]);
+    }
+
+    int runMask = 0;
+    if (argc > 3) {
+        runMask = atoi(argv[3]);
+    }
+    int saveOutput = 0;
+    if ((runMask & 1) || (runMask & 2)) {
+        MNN_PRINT("Save AllTensors to output/*.txt\n");
+        saveOutput = 1;
+    }
+    int saveInput = 0;
+    if (runMask & 2) {
+        saveInput = 1;
+    }
+    bool autoBackend = false;
+    if (runMask & 16) {
+        autoBackend = true;
+    }
+    auto type = MNN_FORWARD_CPU;
+    if (argc > 4) {
+        type = (MNNForwardType)atoi(argv[4]);
+        MNN_PRINT("Use extra forward type: %d\n", type);
+    }
+
+    int modeNum = 4;
+    if (argc > 5) {
+        modeNum = ::atoi(argv[5]);
+    }
+    int precision = BackendConfig::Precision_Low;
+    int memory = BackendConfig::Memory_Normal;
+    if (argc > 6) {
+        int mask = atoi(argv[6]);
+        precision = mask % 4;
+        memory = (mask / 4) % 4;
+    }
+    // input dims
+    std::vector<int> inputDims;
+    if (argc > 7) {
+        inputDims = parseIntList(argv[7], 'x');
+    }
+    MNN_PRINT("inputDims: ");
+    for (auto dim : inputDims) {
+        MNN_PRINT("%d ", dim);
+    }
+    MNN_PRINT("\n");
+
+    // CPU IDs
+    std::vector<int> cpuIds;
+    if (argc > 8) {
+        cpuIds = parseIntList(argv[8], ',');
+    }
+    MNN_PRINT("cpuIds: ");
+    for (auto id : cpuIds) {
+        MNN_PRINT("%d ", id);
+    }
+    MNN_PRINT("\n");
+
+    // create net
+    MNN_PRINT("Open Model %s\n", fileName);
+    std::shared_ptr<MNN::Interpreter> net =
+        std::shared_ptr<MNN::Interpreter>(MNN::Interpreter::createFromFile(fileName), MNN::Interpreter::destroy);
+    if (nullptr == net) {
+        return 0;
+    }
+    net->setCacheFile(".tempcache");
+    net->setSessionMode(Interpreter::Session_Debug);
+    if (autoBackend) {
+        net->setSessionMode(Interpreter::Session_Backend_Auto);
+        net->setSessionHint(Interpreter::MAX_TUNING_NUMBER, 15);
+    }
+    if (!inputDims.empty()) {
+        net->setSessionMode(Interpreter::Session_Resize_Defer);
+    }
+    if (runMask & 32) {
+        net->setSessionHint(Interpreter::WINOGRAD_MEMORY_LEVEL, 0);
+    }
+    net->setSessionHint(Interpreter::HintMode::CPU_CORE_IDS, cpuIds.data(), cpuIds.size());
+
+    // create session
+    MNN::ScheduleConfig config;
+    config.type      = type;
+    /*modeNum means gpuMode for GPU usage, Or means numThread for CPU usage.*/
+    config.numThread = modeNum;
+    // If type not fount, let it failed
+    config.backupType = type;
+    BackendConfig backendConfig;
+    // config.path.outputs.push_back("ResizeBilinear_2");
+    // backendConfig.power = BackendConfig::Power_High;
+    backendConfig.precision = static_cast<MNN::BackendConfig::PrecisionMode>(precision);
+    backendConfig.memory = static_cast<MNN::BackendConfig::MemoryMode>(memory);
+    config.backendConfig     = &backendConfig;
+    MNN::Session* session    = NULL;
+    MNN::Tensor* inputTensor = nullptr;
+    {
+        AUTOTIME;
+        session = net->createSession(config);
+        if (nullptr == session) {
+            return 0;
+        }
+        inputTensor = net->getSessionInput(session, NULL);
+        if (!inputDims.empty()) {
+            MNN_PRINT("===========> Resize Again...\n");
+            net->resizeTensor(inputTensor, inputDims);
+            net->resizeSession(session);
+            //Set when size is changed, After resizeSession
+        }
+    }
+    int resizeStatus = 0;
+    net->getSessionInfo(session, MNN::Interpreter::RESIZE_STATUS, &resizeStatus);
+    if (resizeStatus != 0) {
+        MNN_ERROR("Resize error, can't execute MNN\n");
+        return 0;
+    }
+
+    float memoryUsage = 0.0f;
+    net->getSessionInfo(session, MNN::Interpreter::MEMORY, &memoryUsage);
+    float flops = 0.0f;
+    net->getSessionInfo(session, MNN::Interpreter::FLOPS, &flops);
+    int backendType[2];
+    net->getSessionInfo(session, MNN::Interpreter::BACKENDS, backendType);
+    MNN_PRINT("Session Info: memory use %f MB, flops is %f M, backendType is %d\n", memoryUsage, flops, backendType[0]);
+    // Set Other Inputs to Zero
+    auto allInput = net->getSessionInputAll(session);
+    for (auto& iter : allInput) {
+        auto inputTensor = iter.second;
+        auto size = inputTensor->size();
+        if (size <= 0) {
+            continue;
+        }
+        MNN::Tensor tempTensor(inputTensor, inputTensor->getDimensionType());
+        ::memset(tempTensor.host<void>(), 0, tempTensor.size());
+        inputTensor->copyFromHostTensor(&tempTensor);
+    }
+    MNN_PRINT("===========> Session Resize Done.\n");
+    MNN_PRINT("===========> Session Start running...\n");
+    if (type == MNN_FORWARD_CPU || (!autoBackend)) {
+        net->releaseModel();
+    }
+    _loadInputFromFile(inputTensor, pwd, "input_0.txt");
+
+    // input
+    auto dimType = inputTensor->getDimensionType();
+    if (inputTensor->getType().code == halide_type_uint || inputTensor->getType().code == halide_type_int) {
+        dimType = Tensor::TENSORFLOW;
+    }
+
+    std::ofstream orderFileOs;
+    orderFileOs.open(".order");
+    if (saveOutput) {
+        MNN::TensorCallBack beforeCallBack = [&](const std::vector<MNN::Tensor*>& ntensors, const std::string& opName) {
+            if (!saveInput) {
+                return true;
+            }
+            for (int i = 0; i < ntensors.size(); ++i) {
+                auto ntensor      = ntensors[i];
+                if (nullptr == ntensor->host<void>() && 0 == ntensor->deviceId()) {
+                    // Raster Input
+                    continue;
+                }
+                auto outDimType = ntensor->getDimensionType();
+                auto expectTensor = new MNN::Tensor(ntensor, outDimType);
+                ntensor->copyToHostTensor(expectTensor);
+                auto tensor = ntensor;
+                std::ostringstream outputFileName;
+                auto opCopyName = opName;
+                for (int j = 0; j < opCopyName.size(); ++j) {
+                    if (opCopyName[j] == '/') {
+                        opCopyName[j] = '_';
+                    }
+                }
+                MNN_PRINT("Dump %s Input, %d, %d X %d X %d X %d\n", opName.c_str(), i, tensor->width(),
+                          tensor->height(), tensor->channel(), tensor->batch());
+                outputFileName << "output/Input_" << opCopyName << "_" << i;
+                dumpTensor2File(expectTensor, outputFileName.str().c_str(), orderFileOs);
+                delete expectTensor;
+            }
+            return true;
+        };
+        MNN::TensorCallBack callBack = [&](const std::vector<MNN::Tensor*>& ntensors, const std::string& opName) {
+            for (int i = 0; i < ntensors.size(); ++i) {
+                auto ntensor    = ntensors[i];
+                auto outDimType = ntensor->getDimensionType();
+                auto expectTensor = new MNN::Tensor(ntensor, outDimType);
+                ntensor->copyToHostTensor(expectTensor);
+
+                auto tensor = expectTensor;
+
+                std::ostringstream outputFileName;
+                auto opCopyName = opName;
+                for (int j = 0; j < opCopyName.size(); ++j) {
+                    if (opCopyName[j] == '/') {
+                        opCopyName[j] = '_';
+                    }
+                }
+                if (tensor->dimensions() == 4) {
+                    MNN_PRINT("Dimensions: 4, W,H,C,B: %d X %d X %d X %d, OP name %s : %d\n",
+                            tensor->width(), tensor->height(), tensor->channel(), tensor->batch(), opName.c_str(), i);
+                } else {
+                    std::ostringstream oss;
+                    for (int i = 0; i < tensor->dimensions(); i++) {
+                        oss << (i ? " X " : "") << tensor->length(i);
+                    }
+
+                    MNN_PRINT("Dimensions: %d, %s, OP name %s : %d\n", tensor->dimensions(), oss.str().c_str(), opName.c_str(), i);
+                }
+
+                outputFileName << "output/" << opCopyName << "_" << i;
+                dumpTensor2File(expectTensor, outputFileName.str().c_str(), orderFileOs);
+                delete expectTensor;
+            }
+            return true;
+        };
+        net->runSessionWithCallBack(session, beforeCallBack, callBack);
+    } else {
+        net->runSession(session);
+    }
+
+    // save output
+    auto outputTensor = net->getSessionOutput(session, NULL);
+    MNN::Tensor expectTensor(outputTensor, outputTensor->getDimensionType());
+    {
+        outputTensor->copyToHostTensor(&expectTensor);
+        auto outputFile = pwd + "output.txt";
+        if (outputTensor->size() > 0) {
+            dumpTensor2File(&expectTensor, outputFile.c_str(), orderFileOs);
+        } else {
+            MNN_ERROR("output size is 0, can't save\n");
+        }
+    }
+    auto allOutputs = net->getSessionOutputAll(session);
+    for (auto& iter : allOutputs) {
+        MNN_PRINT("output: %s\n", iter.first.c_str());
+        {
+            MNN::Tensor expectTensor2(iter.second, iter.second->getDimensionType());
+            iter.second->copyToHostTensor(&expectTensor2);
+            auto outputFile = pwd + "/output/" +  iter.first + ".txt";
+            if (iter.second->size() > 0) {
+                dumpTensor2File(&expectTensor2, outputFile.c_str(), orderFileOs);
+            }
+        }
+    }
+
+    // benchmark. for CPU, op time means calc duration; for others, op time means schedule duration.
+    {
+        int t = runTime;
+        MNN_PRINT("precision:%d, memory: %d, Run %d time:\n", precision, memory, t);
+        std::map<std::string, std::pair<float, float>> opTimes;
+        std::map<std::string, std::string> opTypes;
+        uint64_t opBegin = 0;
+
+        MNN::TensorCallBackWithInfo beforeCallBack = [&](const std::vector<MNN::Tensor*>& ntensors,
+                                                         const OperatorInfo* info) {
+            if(opTypes.find(info->name()) == opTypes.end()){
+                opTypes.insert(std::make_pair(info->name(), info->type()));
+            }
+            opBegin = getTimeInUs();
+            if (opTimes.find(info->name()) == opTimes.end()) {
+                opTimes.insert(std::make_pair(info->name(), std::make_pair(0.0f, info->flops())));
+            }
+            return true;
+        };
+        MNN::TensorCallBackWithInfo afterCallBack = [&](const std::vector<MNN::Tensor*>& ntensors,
+                                                        const OperatorInfo* info) {
+            auto opEnd = getTimeInUs();
+            float cost = (float)(opEnd - opBegin) / 1000.0f;
+
+            opTimes[info->name()].first += cost;
+            return true;
+        };
+
+        if (t > 0) {
+
+            for (int i = 0; i < 3; ++i) { // warmup
+                {
+                    auto ptr = inputTensor->map(MNN::Tensor::MAP_TENSOR_WRITE, inputTensor->getDimensionType());
+                    inputTensor->unmap(MNN::Tensor::MAP_TENSOR_WRITE, inputTensor->getDimensionType(), ptr);
+                }
+                net->runSession(session);
+                {
+                    auto ptr = outputTensor->map(MNN::Tensor::MAP_TENSOR_READ, outputTensor->getDimensionType());
+                    outputTensor->unmap(MNN::Tensor::MAP_TENSOR_READ, outputTensor->getDimensionType(), ptr);
+                }
+            }
+            float minTime = 0.0f;
+            float maxTime = 0.0f;
+            float sum = 0.0f;
+            for (int i = 0; i < t; ++i) {
+                auto begin = getTimeInUs();
+                {
+                    auto ptr = inputTensor->map(MNN::Tensor::MAP_TENSOR_WRITE, inputTensor->getDimensionType());
+                    inputTensor->unmap(MNN::Tensor::MAP_TENSOR_WRITE, inputTensor->getDimensionType(), ptr);
+                }
+                net->runSessionWithCallBackInfo(session, beforeCallBack, afterCallBack, false);
+                {
+                    auto ptr = outputTensor->map(MNN::Tensor::MAP_TENSOR_READ, outputTensor->getDimensionType());
+                    outputTensor->unmap(MNN::Tensor::MAP_TENSOR_READ, outputTensor->getDimensionType(), ptr);
+                }
+                auto end = getTimeInUs();
+                auto curtime = (end - begin) / 1000.0f;
+                if (0 == i) {
+                    minTime = curtime;
+                    maxTime = curtime;
+                } else {
+                    minTime = ALIMIN(curtime, minTime);
+                    maxTime = ALIMAX(curtime, maxTime);
+                }
+                sum += curtime;
+            }
+            std::vector<std::pair<float, std::pair<std::string, float>>> allOpsTimes;
+            float sumFlops = 0.0f;
+            for (auto& iter : opTimes) {
+                allOpsTimes.push_back(
+                    std::make_pair(iter.second.first, std::make_pair(iter.first, iter.second.second)));
+                sumFlops += iter.second.second;
+            }
+
+            std::sort(allOpsTimes.begin(), allOpsTimes.end());
+            float opSum = 0;
+            for (auto& iter : allOpsTimes) {
+                opSum += iter.first;
+                MNN_PRINT("%*s \t[%s] run %d average cost %f ms, %.3f %%, FlopsRate: %.3f %%\n", 50,
+                    iter.second.first.c_str(),
+                    opTypes[iter.second.first].c_str(),
+                    runTime,
+                    iter.first / (float)runTime,
+                    iter.first / sum * 100.0f,
+                    iter.second.second / sumFlops * 100.0f);
+            }
+            opSum = opSum / runTime;
+            MNN_PRINT("Avg= %f ms, OpSum = %f ms min= %f ms, max= %f ms\n", sum / (float)t, opSum, minTime, maxTime);
+        }
+    }
+    net->updateCacheFile(session);
+    return 0;
+}
+
+int main(int argc, const char* argv[]) {
+    // For Detect Memory Leak, set circle as true
+    bool circle = false;
+    do {
+        test_main(argc, argv);
+    } while (circle);
+    return 0;
+}

tools/cpp/ModuleBasic.cpp

@@ -0,0 +1,561 @@
+//
+//  ModuleBasic.cpp
+//  MNN
+//
+//  Created by MNN on 2021/10/15.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#include "MNN_generated.h"
+#include <MNN/expr/Expr.hpp>
+#include <MNN/expr/ExecutorScope.hpp>
+#include <MNN/expr/Module.hpp>
+#include <MNN/expr/ExprCreator.hpp>
+#define MNN_OPEN_TIME_TRACE
+#include <MNN/AutoTime.hpp>
+#include "rapidjson/document.h"
+#include "core/MemoryFormater.h"
+#include <numeric>
+#include <chrono>
+#include <iostream>
+#include <thread>
+#include "ExprDebug.hpp"
+#include "core/KVMeta.hpp"
+
+using namespace MNN::Express;
+using namespace MNN;
+static bool compareOutput(VARP output, const std::string& directName, const std::string& name, Dimensionformat dataFormat, int order) {
+
+    auto info = output->getInfo();
+    auto ptr = output->readMap<float>();
+    if (info && info->size <= 0) {
+        MNN_PRINT("skip checking value for zero content tensor %s\n", name.c_str());
+        return true;
+    }
+
+    if (nullptr == info || nullptr == ptr) {
+        MNN_ERROR("TESTERROR name:%s, info:%p, ptr:%p. size:%zu\n", name.c_str(), info, ptr, info->size);
+        return false;
+    }
+
+    std::ifstream outputOrigin;
+    // First find key
+    {
+        std::ostringstream outputFileOs;
+        outputFileOs << directName << "/" << name <<".txt";
+        outputOrigin.open(outputFileOs.str().c_str());
+    }
+    // Second find order
+    if (outputOrigin.fail()) {
+        std::ostringstream outputFileOs;
+        outputFileOs << directName << "/" << order <<".txt";
+        outputOrigin.open(outputFileOs.str().c_str());
+    }
+    if (outputOrigin.fail()) {
+        MNN_PRINT("Skip check %s\n", name.c_str());
+        return true;
+    }
+    MNN_PRINT("before compare %s: (", name.c_str());
+    for (int i=0; i<info->dim.size(); ++i) {
+        MNN_PRINT("%d, ", info->dim[i]);
+    }
+    MNN_PRINT(")\n");
+    auto outputPtr = output->readMap<float>();
+    float diffAbsMaxV = 0.0f;
+    float absMaxV = 0.0f;
+#define MNN_IS_INF(x) (fabs(x) == INFINITY)
+#define MNN_IS_NAN(x) ((x) != (x))
+    for (int i=0; i<info->size; ++i) {
+        double targetValue;
+        outputOrigin >> targetValue;
+        if (MNN_IS_INF(outputPtr[i]) || MNN_IS_NAN(outputPtr[i])) {
+            MNN_ERROR("TESTERROR %s value error:%f\n", name.c_str(), outputPtr[i]);
+            return false;
+        }
+        auto diff = fabsf((float)targetValue - outputPtr[i]);
+        absMaxV = fmaxf(absMaxV, targetValue);
+        diffAbsMaxV = fmaxf(diff, diffAbsMaxV);
+    }
+
+    MNN_PRINT("For %s, max = %f, diffmax = %f, diff rate = %f\n", name.c_str(), absMaxV, diffAbsMaxV, diffAbsMaxV / fmaxf(absMaxV, 1e-6));
+    if (absMaxV * 0.01f < diffAbsMaxV || MNN_IS_NAN(absMaxV)) {
+        MNN_ERROR("TESTERROR %s value error : absMaxV:%f - DiffMax %f\n", name.c_str(), absMaxV, diffAbsMaxV);
+        return false;
+    }
+    return true;
+}
+
+static inline std::vector<int> parseIntList(const std::string& str, char delim) {
+    std::vector<int> result;
+    if (str.empty()) {
+        return result;
+    }
+    std::ptrdiff_t p1 = 0, p2;
+    while (1) {
+        p2 = str.find(delim, p1);
+        if (p2 != std::string::npos) {
+            result.push_back(atoi(str.substr(p1, p2 - p1).c_str()));
+            p1 = p2 + 1;
+        } else {
+            result.push_back(atoi(str.substr(p1).c_str()));
+            break;
+        }
+    }
+    return result;
+}
+int main(int argc, char *argv[]) {
+    if (argc < 3) {
+        MNN_PRINT("=======================================================================================================================================\n");
+        MNN_ERROR("Usage: ./ModuleBasic.out ${test.mnn} ${Dir} [runMask] [forwardType] [runLoops] [numberThread] [precision | memory] [cacheFile] [cpuIds] [enableKleidiAI]\n");
+        MNN_PRINT("=======================================================================================================================================\n");
+        return 0;
+    }
+    BackendConfig backendConfigTmp;
+    auto _executor = Executor::newExecutor(MNN_FORWARD_CPU, backendConfigTmp, 1);
+    ExecutorScope _s(_executor);
+
+    std::string modelName = argv[1];
+    std::string directName = argv[2];
+    MNN_PRINT("Test %s from input info: %s\n", modelName.c_str(), directName.c_str());
+    std::map<std::string, float> inputInfo;
+    std::map<std::string, std::vector<int>> inputShape;
+    std::vector<std::string> inputNames;
+    std::vector<std::string> outputNames;
+    bool checkOutput = false;
+    int runMask = 0;
+    if (argc > 3) {
+        runMask = atoi(argv[3]);
+        if (runMask & 1) {
+            _initDebug();
+        }
+        if (runMask & 2) {
+            _initTensorStatic();
+        }
+    }
+    int repeatNumber = 2;
+    bool shapeMutable = true;
+    std::vector<VARP> inputs;
+    std::vector<VARP> outputs;
+    if (runMask & 128) {
+        MNN_PRINT("Use input.mnn and output.mnn for test\n");
+        inputs = MNN::Express::Variable::load((directName + "/input.mnn").c_str());
+        outputs = MNN::Express::Variable::load((directName + "/output.mnn").c_str());
+        if (inputs.size() > 0 && outputs.size() > 0) {
+            MNN_PRINT("Has input.mnn, use input.mnn and output.mnn instead of json\n");
+        }
+        for (auto v : inputs) {
+            inputNames.emplace_back(v->name());
+        }
+        for (auto v : outputs) {
+            outputNames.emplace_back(v->name());
+        }
+        checkOutput = outputs.size() > 0;
+    }
+    // Call Time / Per Second
+    float freq = 0.0f;
+    int cpuDecreaseRate = -1;
+    int kvAdd = 0;
+    if (inputNames.empty()) {
+        rapidjson::Document document;
+        std::ostringstream jsonNameOs;
+        jsonNameOs << directName << "/input.json";
+        std::ifstream fileNames(jsonNameOs.str().c_str());
+        std::ostringstream output;
+        output << fileNames.rdbuf();
+        auto outputStr = output.str();
+        document.Parse(outputStr.c_str());
+        if (document.HasParseError()) {
+            MNN_ERROR("Invalid json\n");
+            return 0;
+        }
+        if (document.HasMember("inputs")) {
+            auto inputsInfo = document["inputs"].GetArray();
+            for (auto iter = inputsInfo.begin(); iter !=inputsInfo.end(); iter++) {
+                auto obj = iter->GetObject();
+                std::string name = obj["name"].GetString();
+                inputNames.emplace_back(name);
+                MNN_PRINT("%s\n", name.c_str());
+                if (obj.HasMember("value")) {
+                    float value = obj["value"].GetFloat();
+                    inputInfo.insert(std::make_pair(name, value));
+                }
+                if (obj.HasMember("shape")) {
+                    auto dims = obj["shape"].GetArray();
+                    std::vector<int> shapes;
+                    for (auto iter = dims.begin(); iter != dims.end(); iter++) {
+                        shapes.emplace_back(iter->GetInt());
+                    }
+                    inputShape.insert(std::make_pair(name, shapes));
+                }
+            }
+        }
+        if (document.HasMember("outputs")) {
+            checkOutput = true;
+            auto array = document["outputs"].GetArray();
+            for (auto iter = array.begin(); iter !=array.end(); iter++) {
+                std::string name = iter->GetString();
+                MNN_PRINT("output: %s\n", name.c_str());
+                outputNames.emplace_back(name);
+            }
+        }
+        if (document.HasMember("shapeMutable")) {
+            shapeMutable = document["shapeMutable"].GetBool();
+        }
+        if (document.HasMember("repeat")) {
+            repeatNumber = document["repeat"].GetInt();
+        }
+        if (document.HasMember("freq")) {
+            freq = document["freq"].GetFloat();
+        }
+        if (document.HasMember("kv_add")) {
+            kvAdd = document["kv_add"].GetInt();
+        }
+        if (document.HasMember("cpu_decrease_rate")) {
+            cpuDecreaseRate = document["cpu_decrease_rate"].GetInt();
+        }
+    }
+    auto type = MNN_FORWARD_CPU;
+    if (argc > 4) {
+        type = (MNNForwardType)atoi(argv[4]);
+        MNN_PRINT("Use extra forward type: %d\n", type);
+    }
+
+    // Default single thread
+    int modeNum = 1;
+    if (argc > 6) {
+        modeNum = ::atoi(argv[6]);
+    }
+
+    int power = BackendConfig::Power_Normal;
+    int precision = BackendConfig::Precision_Normal;
+    int memory = BackendConfig::Memory_Normal;
+    if (argc > 7) {
+        int mask = atoi(argv[7]);
+        precision = mask % 4;
+        memory = (mask / 4) % 4;
+        power = (mask / 16) % 4;
+    }
+    const char* cacheFileName = ".tempcache";
+    if (argc > 8) {
+        cacheFileName = argv[8];
+    }
+    // CPU IDs
+    std::vector<int> cpuIds;
+    if (argc > 9) {
+        cpuIds = parseIntList(argv[9], ',');
+    }
+    MNN_PRINT("cpuIds: ");
+    for (auto id : cpuIds) {
+        MNN_PRINT("%d ", id);
+    }
+    bool enableKleidiAI = false;
+    if (argc > 10) {
+        enableKleidiAI = atoi(argv[10]) > 0 ? true : false;
+    }
+    int mixedRatio = 17;
+    if (argc > 11) {
+        mixedRatio = atoi(argv[11]);
+    }
+    MNN_PRINT("\n");
+    FUNC_PRINT(precision);
+    FUNC_PRINT(memory);
+    FUNC_PRINT(power);
+    FUNC_PRINT_ALL(cacheFileName, s);
+    FUNC_PRINT(enableKleidiAI);
+    FUNC_PRINT(mixedRatio);
+    // create session
+    MNN::ScheduleConfig config;
+    config.type      = type;
+    /*modeNum means gpuMode for GPU usage, Or means numThread for CPU usage.*/
+    config.numThread = modeNum;
+    // If type not fount, let it failed
+    config.backupType = type;
+    BackendConfig backendConfig;
+    // config.path.outputs.push_back("ResizeBilinear_2");
+    backendConfig.power = (BackendConfig::PowerMode)power;
+    backendConfig.precision = static_cast<MNN::BackendConfig::PrecisionMode>(precision);
+    backendConfig.memory = static_cast<MNN::BackendConfig::MemoryMode>(memory);
+    config.backendConfig     = &backendConfig;
+
+    MNN::Express::Module::Config mConfig;
+    if (runMask & 256) {
+        mConfig.dynamic = true;
+    }
+    mConfig.shapeMutable = shapeMutable;
+    std::shared_ptr<Executor::RuntimeManager> rtmgr(Executor::RuntimeManager::createRuntimeManager(config));
+    rtmgr->setCache(cacheFileName);
+    rtmgr->setHint(MNN::Interpreter::INIT_THREAD_NUMBER, 4);
+    rtmgr->setHint(MNN::Interpreter::HintMode::CPU_CORE_IDS, cpuIds.data(), cpuIds.size());
+
+    if (cpuDecreaseRate > 0 && cpuDecreaseRate <= 100) {
+        rtmgr->setHint(Interpreter::CPU_LITTLECORE_DECREASE_RATE, cpuDecreaseRate);
+    }
+    if (runMask & 1) {
+        // Need dump tensor, open debug
+        rtmgr->setMode(Interpreter::Session_Debug);
+    }
+    if (runMask & 2) {
+        // Need tensor static for each op, open debug
+        rtmgr->setMode(Interpreter::Session_Debug);
+    }
+    // For Debug
+    if (false) {
+        int geometryMask = Interpreter::GeometryComputeMask::GEOMETRCOMPUTEMASK_ALL;
+        geometryMask -= Interpreter::GeometryComputeMask::GEOMETRCOMPUTEMASK_FUSEREGION;
+        geometryMask -= Interpreter::GeometryComputeMask::GEOMETRCOMPUTEMASK_OPENCACHE;
+        rtmgr->setHint(Interpreter::GEOMETRY_COMPUTE_MASK, geometryMask);
+    }
+    if (runMask & 4) {
+        // Need time trace for each op, open debug
+        rtmgr->setMode(Interpreter::Session_Debug);
+    }
+    if (runMask & 8) {
+        rtmgr->setMode(Interpreter::Session_Input_Inside);
+    }
+    if (runMask & 16) {
+        rtmgr->setMode(Interpreter::Session_Backend_Auto);
+        rtmgr->setHint(Interpreter::MAX_TUNING_NUMBER, 50);
+    }
+    if (runMask & 32) {
+        mConfig.rearrange = true;
+    }
+    if (runMask & 512) {
+        rtmgr->setHint(Interpreter::WINOGRAD_MEMORY_LEVEL, 0);
+    }
+    if (runMask & 1024) {
+        /*
+        2: INPUT_BLOCK_QUANT
+        1: INPUT_SHARE_ONE_SCALE
+        0: INPUT_CHANNEL_QUANT
+        */
+        rtmgr->setHint(Interpreter::DYNAMIC_QUANT_OPTIONS, 2);
+    }
+
+    if (enableKleidiAI) {
+        rtmgr->setHint(Interpreter::CPU_ENABLE_KLEIDIAI, true);
+    }
+    KVMeta kvMeta;
+    if (kvAdd > 0) {
+        kvMeta.add = kvAdd;
+        rtmgr->setHintPtr(Interpreter::KVCACHE_INFO, &kvMeta);
+    }
+
+    // rtmgr->setHint(Interpreter::CPU_SME2_INSTRUCTIONS, false);
+
+    if (runMask & 2048) {
+        rtmgr->setExternalPath("tmp", Interpreter::EXTERNAL_FEATUREMAP_DIR);
+    }
+    
+    rtmgr->setHint(Interpreter::CPU_SME2_NEON_DIVISION_RATIO, mixedRatio);
+    // set npu model dir, npu model and mnn model in same path
+    size_t pos = modelName.find_last_of("/\\");
+    std::string modelPath;
+    if (pos == std::string::npos) {
+        // current path
+        modelPath = "./";
+    } else {
+        modelPath = modelName.substr(0, pos);
+    }
+    rtmgr->setExternalPath(modelPath, 3);
+    std::shared_ptr<Module> net;
+    {
+        AUTOTIME;
+        net.reset(Module::load(inputNames, outputNames, modelName.c_str(), rtmgr, &mConfig));
+        if (net == nullptr) {
+            MNN_PRINT("Error: can't load module\n");
+            return 0;
+        }
+        if (runMask & 64) {
+            net.reset(Module::clone(net.get()));
+        }
+        if (net == nullptr) {
+            return 0;
+        }
+    }
+    auto mInfo = net->getInfo();
+
+#define LOAD_DATA(TYPE)\
+    if (inputInfo.find(inputName) != inputInfo.end()) {\
+        auto value = inputInfo[inputName];\
+        for (int i=0; i<info->size; ++i) {\
+            ptr[i] = value;\
+        }\
+    } else {\
+        std::ostringstream fileNameOs;\
+        fileNameOs << directName << "/" << inputName << ".txt";\
+        auto fileName = fileNameOs.str();\
+        std::ifstream inputOs(fileName.c_str());\
+        if (inputOs.fail()) {\
+            MNN_ERROR("TESTERROR Can't open %s\n", fileName.c_str());\
+            continue;\
+        }\
+        for (int i=0; i<info->size; ++i) {\
+            double tempValue;\
+            inputOs >> tempValue;\
+            ptr[i] = tempValue;\
+        }\
+    }
+
+    if (inputs.empty()) {
+        inputs.resize(mInfo->inputs.size());
+        for (int i=0; i<inputs.size(); ++i) {
+            inputs[i] = _Input(mInfo->inputs[i].dim, mInfo->inputs[i].order, mInfo->inputs[i].type);
+        }
+        // Load inputs
+        for (int i=0; i<inputs.size(); ++i) {
+            auto inputName = inputNames[i];
+            // Resize
+            auto shapeIter = inputShape.find(inputName);
+            auto order = mInfo->inputs[i].order;
+            if (MNN::Express::Dimensionformat::NC4HW4 == mInfo->inputs[i].order) {
+                order = MNN::Express::Dimensionformat::NCHW;
+            }
+
+            if (shapeIter != inputShape.end()) {
+                auto s = shapeIter->second;
+                inputs[i] = _Input(s, order, mInfo->inputs[i].type);
+            }
+            auto info = inputs[i]->getInfo();
+            if (info->type == halide_type_of<float>()){
+                auto ptr = inputs[i]->writeMap<float>();
+                LOAD_DATA(float)
+            } else {
+                auto floatVar = _Input(info->dim, info->order, halide_type_of<float>());
+                auto ptr = floatVar->writeMap<float>();
+                LOAD_DATA(float)
+                auto temp = _Cast(floatVar, info->type);
+                inputs[i]->input(temp);
+            }
+            if (MNN::Express::Dimensionformat::NC4HW4 == mInfo->inputs[i].order) {
+                inputs[i] = _Convert(inputs[i], MNN::Express::Dimensionformat::NC4HW4);
+            }
+        }
+    }
+#undef LOAD_DATA
+
+    bool modelError = false;
+    for (int repeat = 0; repeat < repeatNumber; ++repeat) {
+        MNN_PRINT("Run for %d time\n", repeat);
+        std::vector<VARP> subInputs = inputs;
+        if (repeat % 2 == 1) {
+            for (int i=0; i<inputs.size(); ++i) {
+                subInputs[i] = _Clone(inputs[i], true);
+            }
+        }
+        kvMeta.add = kvAdd;
+        auto outputs = net->onForward(inputs);
+        kvMeta.sync();
+        if (outputs.empty()) {
+            MNN_ERROR("Error in forward\n");
+            return 0;
+        }
+        for (int i=0; i<outputNames.size(); ++i) {
+            auto name = outputNames[i];
+            auto v = outputs[i];
+            auto info = v->getInfo();
+            if (nullptr == info) {
+                continue;
+            }
+            if (info->order == NC4HW4 && info->dim.size() > 1) {
+                v = _Convert(v, mInfo->defaultFormat);
+            }
+            if (info->type.code != halide_type_float) {
+                v = _Cast<float>(v);
+            }
+            v.fix(VARP::CONSTANT);
+            outputs[i] = v;
+        }
+        if (checkOutput) {
+            for (int i=0; i<outputNames.size(); ++i) {
+                auto output = outputs[i];
+                bool success = compareOutput(output, directName, outputNames[i], mInfo->defaultFormat, i);
+                if (!success) {
+                    modelError = true;
+                    MNN_ERROR("%d run Error for output %s\n", repeat, outputNames[i].c_str());
+                }
+            }
+        }
+        if (0 == repeat) {
+            for (int i=0; i<inputNames.size(); ++i) {
+                inputs[i].fix(VARP::CONSTANT);
+                inputs[i]->setName(inputNames[i]);
+            }
+            for (int i=0; i<outputNames.size(); ++i) {
+                outputs[i].fix(VARP::CONSTANT);
+                outputs[i]->setName(outputNames[i]);
+            }
+            Variable::save(inputs, "output/input.mnn");
+            Variable::save(outputs, "output/output.mnn");
+        }
+        for (int i=0; i<outputNames.size(); ++i) {
+            auto name = outputNames[i];
+            auto v = outputs[i];
+            auto info = v->getInfo();
+            std::ostringstream fileNameOs;
+            fileNameOs << "output/" << repeat <<"_"<< i << ".txt";
+            auto fileName = fileNameOs.str();
+            MNN_PRINT("Write %s output to %s\n", name.c_str(), fileName.c_str());
+            std::ofstream _output(fileName.c_str());
+            auto ptr = v->readMap<float>();
+            for (int v=0; v<info->size; ++v) {
+                _output << ptr[v] << "\n";
+            }
+        }
+        // Print module's memory
+        float memoryInMB = 0.0f;
+        rtmgr->getInfo(Interpreter::MEMORY, &memoryInMB);
+        FUNC_PRINT_ALL(memoryInMB, f);
+    }
+
+    // benchmark. for CPU, op time means calc duration; for others, op time means schedule duration.
+    int runTime = 0;
+    if (argc > 5) {
+        runTime = ::atoi(argv[5]);
+    }
+
+    if (runTime > 0) {
+        kvMeta.remove = kvMeta.previous;
+        int t = runTime;
+        if (runMask & 4) {
+            _initTimeTrace();
+        }
+        float minTime = std::numeric_limits<float>::max();
+        float maxTime = 0.0f;
+        float sum    = 0.0f;
+
+        for (int i = 0; i < t; ++i) {
+            Timer _l;
+            kvMeta.add = kvAdd;
+            auto out = net->onForward(inputs);
+            kvMeta.sync();
+            Variable::compute(out);
+            for (auto o : out) {
+                ((MNN::Tensor*)o->getTensor())->wait(MNN::Tensor::MAP_TENSOR_READ, true);
+            }
+            auto time = _l.durationInUs() / 1000.0f;
+            if (freq > 0.0f) {
+                float remainMs = (1000.0f / freq) - time;
+                if (remainMs > 0.0f) {
+                    std::this_thread::sleep_for(std::chrono::milliseconds((int)remainMs));
+                }
+            }
+            if (maxTime < time) {
+                maxTime = time;
+            }
+            if (minTime > time) {
+                minTime = time;
+            }
+            sum += time;
+        }
+        if (nullptr != gTimeTraceInfo) {
+            MNN_PRINT("Per Op Trace: \n");
+            gTimeTraceInfo->dump(true);
+            MNN_PRINT("Per Type Trace: \n");
+            gTimeTraceInfo->dump(false);
+        }
+        MNN_PRINT("Avg= %f ms, min= %f ms, max= %f ms\n", sum / (float)t, minTime, maxTime);
+    }
+    rtmgr->updateCache();
+    return 0;
+}
+

tools/cpp/OpenCLProgramBuildTest.cpp

@@ -0,0 +1,244 @@
+//
+//  OpenCLProgramBuildTest.cpp
+//  MNN
+//
+//  Created by MNN on 2025/5/15.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#include <fstream>
+#include <string>
+#include <vector>
+#include "CL/cl.h"
+#ifdef _WIN32
+#include <windows.h>
+#include <libloaderapi.h>
+#else
+#include <dlfcn.h>
+#endif
+
+using clGetPlatformIDsFunc = cl_int (CL_API_CALL *)(cl_uint, cl_platform_id *, cl_uint *);
+using clBuildProgramFunc = cl_int (CL_API_CALL *)(cl_program, cl_uint, const cl_device_id *, const char *, void (CL_CALLBACK *pfn_notify)(cl_program, void *), void *);
+using clCreateProgramWithSourceFunc = cl_program (CL_API_CALL *)(cl_context, cl_uint, const char **, const size_t *, cl_int *);
+using clGetProgramBuildInfoFunc = cl_int (CL_API_CALL *)(cl_program, cl_device_id, cl_program_build_info, size_t, void *, size_t *);
+using clCreateContextFunc = cl_context (CL_API_CALL *)(const cl_context_properties *, cl_uint, const cl_device_id *,
+                                                       void(CL_CALLBACK *)( // NOLINT(readability/casting)
+                                                           const char *, const void *, size_t, void *),
+                                                       void *, cl_int *);
+using clGetDeviceIDsFunc = cl_int (CL_API_CALL *)(cl_platform_id, cl_device_type, cl_uint, cl_device_id *, cl_uint *);
+using clGetDeviceInfoFunc = cl_int (CL_API_CALL *)(cl_device_id, cl_device_info, size_t, void *, size_t *);
+using clReleaseProgramFunc = cl_int (CL_API_CALL *)(cl_program program);
+using clReleaseContextFunc = cl_int (CL_API_CALL *)(cl_context);
+using clReleaseDeviceFunc = cl_int (CL_API_CALL *)(cl_device_id);
+
+
+class OpenCLProgramTest {
+public:
+    OpenCLProgramTest(){
+        static const std::vector<std::string> gOpencl_library_paths = {
+
+        #if defined(__APPLE__) || defined(__MACOSX)
+            "libOpenCL.so", "/System/Library/Frameworks/OpenCL.framework/OpenCL"
+        #elif defined(__OHOS__)
+            "/vendor/lib64/chipsetsdk/libhvgr_v200.so",
+            "/vendor/lib64/chipsetsdk/libGLES_mali.so",
+            "/system/lib64/libGLES_mali.so",
+            "libGLES_mali.so",
+            "/vendor/lib64/chipsetsdk/libEGI_imp1.so",
+        #elif defined(__ANDROID__)
+            "libOpenCL.so",
+            "libGLES_mali.so",
+            "libmali.so",
+            "libOpenCL-pixel.so",
+        #if defined(__aarch64__)
+            // Qualcomm Adreno
+            "/system/vendor/lib64/libOpenCL.so",
+            "/system/lib64/libOpenCL.so",
+            // Mali
+            "/system/vendor/lib64/egl/libGLES_mali.so",
+            "/system/lib64/egl/libGLES_mali.so",
+        #else
+            // Qualcomm Adreno
+            "/system/vendor/lib/libOpenCL.so", "/system/lib/libOpenCL.so",
+            // Mali
+            "/system/vendor/lib/egl/libGLES_mali.so", "/system/lib/egl/libGLES_mali.so",
+            // other
+            "/system/vendor/lib/libPVROCL.so", "/data/data/org.pocl.libs/files/lib/libpocl.so"
+        #endif
+        #elif defined(__linux__)
+            "/usr/lib/libOpenCL.so",
+            "/usr/local/lib/libOpenCL.so",
+            "/usr/local/lib/libpocl.so",
+            "/usr/lib64/libOpenCL.so",
+            "/usr/lib32/libOpenCL.so",
+            "libOpenCL.so"
+        #elif defined(_WIN64)
+            "C:/Windows/System32/OpenCL.dll",
+            "C:/Windows/SysWOW64/OpenCL.dll"
+        #elif defined(_WIN32)
+            "C:/Windows/SysWOW64/OpenCL.dll",
+            "C:/Windows/System32/OpenCL.dll"
+        #endif
+        };
+
+        for (const auto &opencl_lib : gOpencl_library_paths) {
+            if (LoadLibraryFromPath(opencl_lib)) {
+                mIsSupportAvailable = true;
+            }
+        }
+        if(mIsSupportAvailable){
+            cl_int err;
+            err = clGetPlatformIDs(1, &platform, NULL);
+            if (err != CL_SUCCESS) {
+                printf("Failed to get platform ID err = %d\n", err);
+                return ;
+            }
+            err = clGetDeviceIDs(platform, CL_DEVICE_TYPE_GPU, 1, &device, NULL);
+            if (err != CL_SUCCESS) {
+                printf("Failed to get device ID err = %d\n", err);
+                return;
+            }
+
+            context = clCreateContext(NULL, 1, &device, NULL, NULL, &err);
+            if (!context || err != CL_SUCCESS) {
+                printf("Failed to create context err = %d\n", err);
+                return;
+            }
+        }
+    }
+    bool LoadLibraryFromPath(const std::string &library_path){
+        #if defined(_WIN32)
+        handle_ = LoadLibraryA(library_path.c_str());
+        if (handle_ == nullptr) {
+            return false;
+        }
+        #define MNN_LOAD_FUNCTION_PTR(func_name) func_name = reinterpret_cast<func_name##Func>(GetProcAddress(static_cast<HMODULE>(handle_), #func_name));
+        #else
+        handle_ = dlopen(library_path.c_str(), RTLD_NOW | RTLD_LOCAL);
+        if (handle_ == nullptr) {
+            return false;
+        }
+        #define MNN_LOAD_FUNCTION_PTR(func_name) func_name = reinterpret_cast<func_name##Func>(dlsym(handle_, #func_name));
+        #endif
+        MNN_LOAD_FUNCTION_PTR(clGetPlatformIDs);
+        MNN_LOAD_FUNCTION_PTR(clBuildProgram);
+        MNN_LOAD_FUNCTION_PTR(clCreateProgramWithSource);
+        MNN_LOAD_FUNCTION_PTR(clGetProgramBuildInfo);
+        MNN_LOAD_FUNCTION_PTR(clCreateContext);
+        MNN_LOAD_FUNCTION_PTR(clGetDeviceIDs);
+        MNN_LOAD_FUNCTION_PTR(clGetDeviceInfo);
+        MNN_LOAD_FUNCTION_PTR(clReleaseProgram);
+        MNN_LOAD_FUNCTION_PTR(clReleaseContext);
+        MNN_LOAD_FUNCTION_PTR(clReleaseDevice);
+        
+        return true;
+    }
+    bool TestProgram(const std::vector<std::string> options){
+        cl_int err;
+        FILE* file = fopen("kernel.cl", "r");
+        if (!file) {
+            printf("Failed to open kernel file: kernel.cl\n");
+            return false;
+        }
+        
+        fseek(file, 0, SEEK_END);
+        size_t fileSize = ftell(file);
+        rewind(file);
+        
+        char* source = (char*)malloc(fileSize + 1);
+        if (!source) {
+            fclose(file);
+            printf("Memory allocation failed for kernel source\n");
+            return false;
+        }
+        
+        fread(source, sizeof(char), fileSize, file);
+        source[fileSize] = '\0';
+        fclose(file);
+        
+        // test program
+        const char *code = source;
+        cl_program program = clCreateProgramWithSource(context, 1, &code, &fileSize, &err);
+        if (!program || err != CL_SUCCESS) {
+            printf("Failed to create program from source\n");
+            return false;
+        }
+        for(int i = 0; i < options.size(); ++i){
+            err = clBuildProgram(program, 1, &device, options[i].c_str(), NULL, NULL);
+            if (err != CL_SUCCESS) {
+                size_t logSize;
+                clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, 0, NULL, &logSize);
+                char *buildLog = (char*)malloc(logSize);
+                clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, logSize, buildLog, NULL);
+                printf("Program build log: ");
+                for (int i = 0; i < logSize; i++) {
+                    printf("%c", buildLog[i]);
+                }
+                clReleaseProgram(program);
+                free(buildLog);
+                return false;
+            }
+        }
+        
+        clReleaseProgram(program);
+        free(source);
+        return true;
+    }
+    bool mIsSupportAvailable = false;
+    ~OpenCLProgramTest(){
+        if(mIsSupportAvailable){
+            clReleaseDevice(device);
+            clReleaseContext(context);
+        }
+        if (handle_ != nullptr) {
+#if defined(_WIN32)
+            FreeLibrary(static_cast<HMODULE>(handle_));
+#else
+            dlclose(handle_);
+#endif
+        }
+    }
+private:
+    void *handle_ = nullptr;
+    clGetPlatformIDsFunc clGetPlatformIDs;
+    clBuildProgramFunc clBuildProgram;
+    clCreateProgramWithSourceFunc clCreateProgramWithSource;
+    clGetProgramBuildInfoFunc clGetProgramBuildInfo;
+    clCreateContextFunc clCreateContext;
+    clGetDeviceIDsFunc clGetDeviceIDs;
+    clGetDeviceInfoFunc clGetDeviceInfo;
+    clReleaseProgramFunc clReleaseProgram;
+    clReleaseContextFunc clReleaseContext;
+    clReleaseDeviceFunc clReleaseDevice;
+    cl_platform_id platform;
+    cl_device_id device;
+    cl_context context;
+};
+
+int main(int argc, char *argv[]) {
+    std::string filename;
+    if(argc > 1){
+        filename = argv[1];
+    }
+    std::vector<std::string> options;
+    std::fstream file("option.txt");
+    if(file.is_open()){
+        std::string line;
+        while (getline(file, line)) { // 按行读取文件内容并输出
+            options.push_back(line);
+        }
+        file.close();
+    }
+    printf("test filename is %s\n", filename.c_str());
+    OpenCLProgramTest BuildTest;
+    if(BuildTest.mIsSupportAvailable){
+        if(BuildTest.TestProgram(options)){
+            return 0;
+        }
+    }else{
+        printf("OpenCL init fail\n");
+        return -1;
+    }
+    return 0;
+}
+

tools/cpp/Profiler.cpp

@@ -0,0 +1,219 @@
+//
+//  Profiler.cpp
+//  MNN
+//
+//  Created by MNN on 2019/01/15.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#include <string.h>
+#include <algorithm>
+#include <string>
+#if defined(_MSC_VER)
+#include <Windows.h>
+#undef min
+#undef max
+#else
+#include <sys/time.h>
+#endif
+#include "Profiler.hpp"
+#include "core/Macro.h"
+
+#define MFLOPS (1e6)
+
+namespace MNN {
+    
+static inline int64_t getTime() {
+    uint64_t time;
+#if defined(_MSC_VER)
+    LARGE_INTEGER now, freq;
+    QueryPerformanceCounter(&now);
+    QueryPerformanceFrequency(&freq);
+    uint64_t sec = now.QuadPart / freq.QuadPart;
+    uint64_t usec = (now.QuadPart % freq.QuadPart) * 1000000 / freq.QuadPart;
+    time = sec * 1000000 + usec;
+#else
+    struct timeval tv;
+    gettimeofday(&tv, nullptr);
+    time = static_cast<uint64_t>(tv.tv_sec) * 1000000 + tv.tv_usec;
+#endif
+    return time;
+}
+
+static std::string toString(float value) {
+    char typeString[100] = {};
+    sprintf(typeString, "%f", value);
+    return std::string(typeString);
+}
+
+static std::string toString(const std::vector<int>& shape) {
+    char content[100] = {};
+    auto current      = content;
+    for (auto s : shape) {
+        current = current + sprintf(current, "%d,", s);
+    }
+    return std::string(current);
+}
+
+Profiler* Profiler::gInstance = nullptr;
+Profiler* Profiler::getInstance() {
+    if (gInstance == nullptr) {
+        gInstance = new Profiler;
+    }
+    return gInstance;
+}
+
+Profiler::Record& Profiler::getTypedRecord(const OperatorInfo* op) {
+    auto typeStr = op->type();
+    auto iter = mMapByType.find(typeStr);
+    if (iter != mMapByType.end()) {
+        return iter->second;
+    }
+
+    // create new
+    mMapByType.insert(std::make_pair(typeStr, Record()));
+    Record& record     = mMapByType.find(typeStr)->second;
+    record.costTime    = 0.0f;
+    record.calledTimes = 0;
+    record.type        = op->type();
+    record.flops       = 0.0f;
+
+    return record;
+}
+
+Profiler::Record& Profiler::getNamedRecord(const OperatorInfo* op) {
+    auto name = op->name();
+    auto iter = mMapByName.find(name);
+    if (iter != mMapByName.end()) {
+        return iter->second;
+    }
+
+    // create new
+    mMapByName.insert(std::make_pair(name, Record()));
+    Record& record     = mMapByName.find(name)->second;
+    record.costTime    = 0.0f;
+    record.name        = op->name();
+    record.type        = op->type();
+    record.flops       = 0.0f;
+
+    return record;
+}
+
+void Profiler::start(const OperatorInfo* info) {
+    mStartTime = getTime();
+    mTotalMFlops += info->flops();
+    auto& typed = getTypedRecord(info);
+    typed.calledTimes++;
+    typed.flops += info->flops();
+    auto& named = getNamedRecord(info);
+    named.flops += info->flops();
+}
+
+void Profiler::end(const OperatorInfo* info) {
+    mEndTime   = getTime();
+    float cost = (float)(mEndTime - mStartTime) / 1000.0f;
+    mMapByType[info->type()].costTime += cost;
+    mMapByName[info->name()].costTime += cost;
+    mTotalTime += cost;
+}
+
+static void printTable(const char* title, const std::vector<std::string>& header,
+                       const std::vector<std::vector<std::string>>& data) {
+    MNN_PRINT("%s\n", title);
+
+    // calc column width
+    std::vector<size_t> maxLength(header.size());
+    for (int i = 0; i < header.size(); ++i) {
+        size_t max = header[i].size();
+        for (auto& row : data) {
+            max = std::max(max, row[i].size());
+        }
+        maxLength[i] = max + 1;
+    }
+
+    // print header
+    for (int i = 0; i < header.size(); ++i) {
+        auto expand = header[i];
+        expand.resize(maxLength[i], ' ');
+        MNN_PRINT("%s\t", expand.c_str());
+    }
+    MNN_PRINT("\n");
+    
+    // print rows
+    for (auto& row : data) {
+        for (int i = 0; i < header.size(); ++i) {
+            auto expand = row[i];
+            expand.resize(maxLength[i], ' ');
+            MNN_PRINT("%s\t", expand.c_str());
+        }
+        MNN_PRINT("\n");
+    }
+}
+    
+void Profiler::printTimeByType(int loops) {
+    // sort by time cost
+    std::vector<std::pair<float, std::string>> sorted;
+    for (auto iter : mMapByType) {
+        sorted.push_back(std::make_pair(iter.second.costTime, iter.first));
+    }
+    std::sort(sorted.begin(), sorted.end());
+    
+    // fill in columns
+    const std::vector<std::string> header = {"Node Type", "Avg(ms)", "%", "Called times", "Flops Rate"};
+    std::vector<std::vector<std::string>> rows;
+    for (auto iter : sorted) {
+        auto record = mMapByType.find(iter.second)->second;
+        std::vector<std::string> columns;
+        columns.push_back(iter.second);
+        columns.push_back(toString(record.costTime / (float)loops));
+        columns.push_back(toString((record.costTime / (float)mTotalTime) * 100));
+        columns.push_back(toString(record.calledTimes / loops));
+        columns.push_back(toString((record.flops / (float)mTotalMFlops) * 100));
+        rows.emplace_back(columns);
+    }
+    printTable("Sort by time cost !", header, rows);
+    float totalAvgTime = mTotalTime / (float)loops;
+    MNN_PRINT("total time : %f ms, total mflops : %f \n", totalAvgTime, mTotalMFlops / loops);
+}
+
+void Profiler::printTimeByName(int loops) {
+    const std::vector<std::string> header = {"Node Name", "Op Type", "Avg(ms)", "%", "Flops Rate"};
+    std::vector<std::vector<std::string>> rows;
+    // sort by name
+    for (auto iter: mMapByName) {
+        auto record = iter.second;
+        std::vector<std::string> columns;
+        columns.push_back(iter.first);
+        columns.push_back(record.type);
+        columns.push_back(toString(record.costTime / (float)loops));
+        columns.push_back(toString((record.costTime / (float)mTotalTime) * 100));
+        columns.push_back(toString((record.flops / (float)mTotalMFlops) * 100));
+        rows.emplace_back(columns);
+    }
+    printTable("Sort by node name !", header, rows);
+}
+void Profiler::printSlowOp(const std::string& type, int topK, float rate) {
+    MNN_PRINT("Print <=%d slowest Op for %s, larger than %.2f\n", topK, type.c_str(), rate * 100.0f);
+    std::vector<std::pair<std::string, float>> result;
+    for (auto& iter : mMapByName) {
+        if (iter.second.type == type || type.empty()) {
+            if (iter.second.flops > 0.0f && iter.second.costTime / mTotalTime >= rate) {
+                result.emplace_back(std::make_pair(iter.second.name, iter.second.costTime / iter.second.flops));
+            }
+        }
+    }
+    if (result.size() < topK) {
+        topK = result.size();
+    }
+    std::partial_sort(result.begin(), result.begin() + topK, result.end(), [&](const std::pair<std::string, float>& left, std::pair<std::string, float>& right) {
+        return left.second > right.second;
+    });
+    for (int i=0; i<topK; ++i) {
+        const auto& record = mMapByName[result[i].first];
+        MNN_PRINT("%s -  %f GFlops, %.2f rate\n", record.name.c_str(), record.flops / record.costTime, record.costTime / mTotalTime * 100.0f);
+    }
+    MNN_PRINT("\n");
+}
+
+
+} // namespace MNN

tools/cpp/Profiler.hpp

@@ -0,0 +1,82 @@
+//
+//  Profiler.hpp
+//  MNN
+//
+//  Created by MNN on 2019/01/15.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#ifndef Profiler_hpp
+#define Profiler_hpp
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <map>
+#include <string>
+#include <vector>
+#include <MNN/Interpreter.hpp>
+#include <MNN/Tensor.hpp>
+
+namespace MNN {
+
+/** Profiler for Ops */
+class Profiler {
+public:
+    /**
+     * @brief get shared instance.
+     */
+    static Profiler* getInstance();
+    /**
+    * @brief start profiler with op, name and inout tensors.
+    * @param op        given op.
+    */
+    void start(const OperatorInfo* info);
+    /**
+     * @brief end profiler with op name and type.
+     * @param name      op name.
+     */
+    void end(const OperatorInfo* info);
+    /**
+     * print profiler time result, grouped by type and sorter by time cost.
+     * @param loops     loop count.
+     */
+    void printTimeByType(int loops = 1);
+    /**
+     * print profiler time result, grouped and sorter by op name.
+     * @param loops     loop count.
+     */
+    void printTimeByName(int loops = 1);
+
+    /**
+     * print op that flops / time is slow
+     */
+    void printSlowOp(const std::string& type, int topk, float limitRate);
+private:
+    ~Profiler() = default;
+
+private:
+    struct Record {
+        std::string name;
+        std::string type;
+        int64_t order;
+        int64_t calledTimes;
+        float costTime;
+        float flops;
+    };
+
+    static Profiler* gInstance;
+    uint64_t mStartTime = 0;
+    uint64_t mEndTime   = 0;
+    float mTotalTime    = 0.0f;
+    float mTotalMFlops  = 0.0f;
+    std::map<std::string, Record> mMapByType;
+    std::map<std::string, Record> mMapByName;
+
+private:
+    Record& getTypedRecord(const OperatorInfo* info);
+    Record& getNamedRecord(const OperatorInfo* info);
+};
+
+} // namespace MNN
+
+#endif /* Profiler_hpp */

tools/cpp/SequenceModuleTest.cpp

@@ -0,0 +1,393 @@
+//
+//  SequenceModuleTest.cpp
+//  MNN
+//
+//  Created by MNN on 2021/10/15.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#include "MNN_generated.h"
+#include <MNN/expr/Expr.hpp>
+#include <MNN/expr/Module.hpp>
+#include <MNN/expr/ExprCreator.hpp>
+#include <MNN/AutoTime.hpp>
+#include "rapidjson/document.h"
+#include <cmath>
+#include "ExprDebug.hpp"
+//#define OPEN_TRACE
+using namespace MNN::Express;
+using namespace MNN;
+
+static bool compareOutput(VARP output, const std::string& directName, const std::string& name, Dimensionformat dataFormat, int order) {
+    auto info = output->getInfo();
+    auto ptr = output->readMap<float>();
+    if (nullptr == info || nullptr == ptr) {
+        MNN_ERROR("TESTERROR ptr / info nullptr\n");
+        return false;
+    }
+    std::string targetFileName;
+    std::ifstream outputOrigin;
+    // First find key
+    {
+        std::ostringstream outputFileOs;
+        outputFileOs << directName << "/" << name <<".txt";
+        targetFileName = outputFileOs.str();
+        outputOrigin.open(targetFileName.c_str());
+    }
+    // Second find order
+    if (outputOrigin.fail()) {
+        std::ostringstream outputFileOs;
+        outputFileOs << directName << "/" << order <<".txt";
+        targetFileName = outputFileOs.str();
+        outputOrigin.open(targetFileName.c_str());
+    }
+    if (info->order == NC4HW4 && info->dim.size() > 1) {
+        output = _Convert(output, dataFormat);
+        info = output->getInfo();
+    }
+    if (info->type.code != halide_type_float) {
+        output = _Cast<float>(output);
+        info = output->getInfo();
+    }
+    MNN_PRINT("%s: (", name.c_str());
+    for (int i=0; i<info->dim.size(); ++i) {
+        MNN_PRINT("%d, ", info->dim[i]);
+    }
+    MNN_PRINT(")\n");
+    auto targetValue = _Input({info->dim}, info->order, info->type);
+    auto targetPtr = targetValue->writeMap<float>();
+    for (int i=0; i<info->size; ++i) {
+        outputOrigin >> targetPtr[i];
+    }
+    auto absMax = _ReduceMax(_Abs(targetValue), {});
+    absMax = _Maximum(absMax, _Scalar<float>(0.0001f));
+    auto diff = _Abs(targetValue - output);
+    auto diffAbsMax = _ReduceMax(diff);
+    auto absMaxV = absMax->readMap<float>()[0];
+    auto diffAbsMaxV = diffAbsMax->readMap<float>()[0];
+    if (absMaxV * 0.01f < diffAbsMaxV || std::isnan(absMaxV)) {
+        MNN_ERROR("TESTERROR from %s value error : absMaxV:%f - DiffMax %f\n", targetFileName.c_str(), absMaxV, diffAbsMaxV);
+        return false;
+    }
+    return true;
+}
+
+#define LOAD_DATA(TYPE)\
+    if (inputInfo.find(inputName) != inputInfo.end()) {\
+        auto value = inputInfo[inputName];\
+        for (int i=0; i<info->size; ++i) {\
+            ptr[i] = value;\
+        }\
+    } else {\
+        std::ostringstream fileNameOs;\
+        fileNameOs << directName << "/" << inputName << ".txt";\
+        auto fileName = fileNameOs.str();\
+        std::ifstream inputOs(fileName.c_str());\
+        if (inputOs.fail()) {\
+            MNN_ERROR("TESTERROR Can't open %s\n", fileName.c_str());\
+            continue;\
+        }\
+        for (int i=0; i<info->size; ++i) {\
+            double tempV; inputOs >> tempV;\
+            ptr[i] = tempV;\
+        }\
+    }
+
+int main(int argc, char *argv[]) {
+    if (argc < 5) {
+        MNN_ERROR("Usage: ./SequenceModuleTest.out ${test.mnn} [forwardType] [shapeMutable] ${testTime} ${Dir} ${Dir1} ......\n");
+        return 0;
+    }
+#ifdef OPEN_TRACE
+    _initTensorStatic();
+#endif
+
+    std::string modelName = argv[1];
+    auto type = (MNNForwardType)atoi(argv[2]);
+    int numberThread = atoi(argv[3]);
+    int testTime = atoi(argv[4]);
+    int offset = 5;
+    MNN_PRINT("Test %s, type = %d\n", modelName.c_str(), type);
+    // create session
+    MNN::ScheduleConfig config;
+    config.type      = type;
+    /*modeNum means gpuMode for GPU usage, Or means numThread for CPU usage.*/
+    // If type not fount, let it failed
+    config.backupType = type;
+    config.numThread = numberThread;
+    BackendConfig backendConfig;
+#ifdef OPEN_TRACE
+    backendConfig.precision = MNN::BackendConfig::Precision_High;
+#endif
+    config.backendConfig     = &backendConfig;
+    
+    MNN::Express::Module::Config mConfig;
+    mConfig.shapeMutable = true;
+    mConfig.rearrange = true;
+    std::shared_ptr<Executor::RuntimeManager> rtmgr(Executor::RuntimeManager::createRuntimeManager(config));
+#ifdef OPEN_TRACE
+    rtmgr->setMode(MNN::Interpreter::Session_Debug);
+#endif
+    std::shared_ptr<Module> net;
+    for (int index = offset; index < argc; ++index) {
+        std::string directName = argv[index];
+        rapidjson::Document document;
+        std::map<std::string, float> inputInfo;
+        std::map<std::string, std::vector<int>> inputShape;
+        std::vector<std::string> inputNames;
+        std::vector<std::string> outputNames;
+        std::ostringstream jsonNameOs;
+        jsonNameOs << argv[index] << "/input.json";
+        std::ifstream fileNames(jsonNameOs.str().c_str());
+        std::ostringstream output;
+        output << fileNames.rdbuf();
+        auto outputStr = output.str();
+        document.Parse(outputStr.c_str());
+        if (document.HasParseError()) {
+            MNN_ERROR("Invalid json\n");
+            continue;
+        }
+        if (document.HasMember("inputs")) {
+            auto inputsInfo = document["inputs"].GetArray();
+            for (auto iter = inputsInfo.begin(); iter !=inputsInfo.end(); iter++) {
+                auto obj = iter->GetObject();
+                std::string name = obj["name"].GetString();
+                inputNames.emplace_back(name);
+                MNN_PRINT("%s\n", name.c_str());
+                if (obj.HasMember("value")) {
+                    float value = obj["value"].GetFloat();
+                    inputInfo.insert(std::make_pair(name, value));
+                }
+                if (obj.HasMember("shape")) {
+                    auto dims = obj["shape"].GetArray();
+                    std::vector<int> shapes;
+                    for (auto iter = dims.begin(); iter != dims.end(); iter++) {
+                        shapes.emplace_back(iter->GetInt());
+                    }
+                    inputShape.insert(std::make_pair(name, shapes));
+                }
+            }
+        }
+        if (document.HasMember("outputs")) {
+            auto array = document["outputs"].GetArray();
+            for (auto iter = array.begin(); iter !=array.end(); iter++) {
+                std::string name = iter->GetString();
+                MNN_PRINT("output: %s\n", name.c_str());
+                outputNames.emplace_back(name);
+            }
+        }
+        if (nullptr == net.get()) {
+            net.reset(Module::load(inputNames, outputNames, modelName.c_str(), rtmgr, &mConfig));
+            if (net == nullptr) {
+                MNN_PRINT("Error: can't load module\n");
+                return 0;
+            }
+            break;
+        }
+    }
+    net->traceOrOptimize(MNN::Interpreter::Session_Resize_Check);
+
+    // First Test
+    auto testCorrect = [&]() {
+        std::vector<bool> correctInputs(argc-offset);
+        for (int index = offset; index < argc; ++index) {
+            MNN_PRINT("Test for %s\n", argv[index]);
+            std::string directName = argv[index];
+            rapidjson::Document document;
+            std::map<std::string, float> inputInfo;
+            std::map<std::string, std::vector<int>> inputShape;
+            std::vector<std::string> inputNames;
+            std::vector<std::string> outputNames;
+            std::ostringstream jsonNameOs;
+            jsonNameOs << argv[index] << "/input.json";
+            std::ifstream fileNames(jsonNameOs.str().c_str());
+            std::ostringstream output;
+            output << fileNames.rdbuf();
+            auto outputStr = output.str();
+            document.Parse(outputStr.c_str());
+            if (document.HasParseError()) {
+                MNN_ERROR("Invalid json\n");
+                continue;
+            }
+            if (document.HasMember("inputs")) {
+                auto inputsInfo = document["inputs"].GetArray();
+                for (auto iter = inputsInfo.begin(); iter !=inputsInfo.end(); iter++) {
+                    auto obj = iter->GetObject();
+                    std::string name = obj["name"].GetString();
+                    inputNames.emplace_back(name);
+                    MNN_PRINT("%s\n", name.c_str());
+                    if (obj.HasMember("value")) {
+                        float value = obj["value"].GetFloat();
+                        inputInfo.insert(std::make_pair(name, value));
+                    }
+                    if (obj.HasMember("shape")) {
+                        auto dims = obj["shape"].GetArray();
+                        std::vector<int> shapes;
+                        for (auto iter = dims.begin(); iter != dims.end(); iter++) {
+                            shapes.emplace_back(iter->GetInt());
+                        }
+                        inputShape.insert(std::make_pair(name, shapes));
+                    }
+                }
+            }
+            if (document.HasMember("outputs")) {
+                auto array = document["outputs"].GetArray();
+                for (auto iter = array.begin(); iter !=array.end(); iter++) {
+                    std::string name = iter->GetString();
+                    MNN_PRINT("output: %s\n", name.c_str());
+                    outputNames.emplace_back(name);
+                }
+            }
+            auto mInfo = net->getInfo();
+
+            std::vector<VARP> inputs(mInfo->inputs.size());
+            for (int i=0; i<inputs.size(); ++i) {
+                inputs[i] = _Input(mInfo->inputs[i].dim, mInfo->inputs[i].order, mInfo->inputs[i].type);
+            }
+            // Load inputs
+            for (int i=0; i<inputs.size(); ++i) {
+                auto inputName = inputNames[i];
+                // Resize
+                auto shapeIter = inputShape.find(inputName);
+                if (shapeIter != inputShape.end()) {
+                    auto s = shapeIter->second;
+                    inputs[i] = _Input(s, mInfo->defaultFormat, mInfo->inputs[i].type);
+                }
+                auto info = inputs[i]->getInfo();
+                if (info->type == halide_type_of<float>()){
+                    auto ptr = inputs[i]->writeMap<float>();
+                    LOAD_DATA(float)
+                } else {
+                    auto floatVar = _Input(info->dim, info->order, halide_type_of<float>());
+                    auto ptr = floatVar->writeMap<float>();
+                    LOAD_DATA(float)
+                    auto temp = _Cast(floatVar, info->type);
+                    inputs[i]->input(temp);
+                }
+                inputs[i] = _Convert(inputs[i], mInfo->inputs[i].order);
+            }
+            bool modelError = false;
+            // Module Branch
+            auto outputs = net->onForward(inputs);
+            for (int i=0; i<outputNames.size(); ++i) {
+                auto output = outputs[i];
+                bool success = compareOutput(output, directName, outputNames[i], mInfo->defaultFormat, i);
+                if (!success) {
+                    modelError = true;
+                    MNN_ERROR("Error for output %s\n", outputNames[i].c_str());
+                }
+            }
+            correctInputs[index-offset] = !modelError;
+        }
+        return correctInputs;
+    };
+    auto correctInfo = testCorrect();
+    MNN_PRINT("Resize optimize for net\n");
+    net->traceOrOptimize(MNN::Interpreter::Session_Resize_Fix);
+    auto optCorrectInfo = testCorrect();
+    for (int i=0; i<correctInfo.size(); ++i) {
+        MNN_PRINT("Correct Json %d: before opt: %d, after opt: %d\n", i, (int)correctInfo[i], (int)optCorrectInfo[i]);
+    }
+    if (testTime > 0) {
+        MNN_PRINT("Test Speed for %d times\n", testTime);
+        // Prepare All Input
+        std::vector<std::vector<MNN::Express::VARP>> allInputs;
+        for (int index = offset; index < argc; ++index) {
+            std::string directName = argv[index];
+            rapidjson::Document document;
+            std::map<std::string, float> inputInfo;
+            std::map<std::string, std::vector<int>> inputShape;
+            std::vector<std::string> inputNames;
+            std::vector<std::string> outputNames;
+            std::ostringstream jsonNameOs;
+            jsonNameOs << argv[index] << "/input.json";
+            std::ifstream fileNames(jsonNameOs.str().c_str());
+            std::ostringstream output;
+            output << fileNames.rdbuf();
+            auto outputStr = output.str();
+            document.Parse(outputStr.c_str());
+            if (document.HasParseError()) {
+                MNN_ERROR("Invalid json\n");
+                continue;
+            }
+            if (document.HasMember("inputs")) {
+                auto inputsInfo = document["inputs"].GetArray();
+                for (auto iter = inputsInfo.begin(); iter !=inputsInfo.end(); iter++) {
+                    auto obj = iter->GetObject();
+                    std::string name = obj["name"].GetString();
+                    inputNames.emplace_back(name);
+                    if (obj.HasMember("value")) {
+                        float value = obj["value"].GetFloat();
+                        inputInfo.insert(std::make_pair(name, value));
+                    }
+                    if (obj.HasMember("shape")) {
+                        auto dims = obj["shape"].GetArray();
+                        std::vector<int> shapes;
+                        for (auto iter = dims.begin(); iter != dims.end(); iter++) {
+                            shapes.emplace_back(iter->GetInt());
+                        }
+                        inputShape.insert(std::make_pair(name, shapes));
+                    }
+                }
+            }
+            if (document.HasMember("outputs")) {
+                auto array = document["outputs"].GetArray();
+                for (auto iter = array.begin(); iter !=array.end(); iter++) {
+                    std::string name = iter->GetString();
+                    outputNames.emplace_back(name);
+                }
+            }
+            auto mInfo = net->getInfo();
+            
+            std::vector<VARP> inputs(mInfo->inputs.size());
+            for (int i=0; i<inputs.size(); ++i) {
+                inputs[i] = _Input(mInfo->inputs[i].dim, mInfo->inputs[i].order, mInfo->inputs[i].type);
+            }
+            // Load inputs
+            for (int i=0; i<inputs.size(); ++i) {
+                auto inputName = inputNames[i];
+                // Resize
+                auto shapeIter = inputShape.find(inputName);
+                if (shapeIter != inputShape.end()) {
+                    auto s = shapeIter->second;
+                    inputs[i] = _Input(s, mInfo->defaultFormat, mInfo->inputs[i].type);
+                }
+                auto info = inputs[i]->getInfo();
+                if (info->type == halide_type_of<float>()){
+                    auto ptr = inputs[i]->writeMap<float>();
+                    LOAD_DATA(float)
+                } else {
+                    auto floatVar = _Input(info->dim, info->order, halide_type_of<float>());
+                    auto ptr = floatVar->writeMap<float>();
+                    LOAD_DATA(float)
+                    auto temp = _Cast(floatVar, info->type);
+                    inputs[i]->input(temp);
+                }
+                inputs[i] = _Convert(inputs[i], mInfo->inputs[i].order);
+                inputs[i].fix(VARP::CONSTANT);
+            }
+            allInputs.emplace_back(inputs);
+        }
+        std::vector<float> times(testTime, 0.0f);
+        for (int t=0; t<testTime; ++t) {
+            Timer _l;
+            for (auto& v : allInputs) {
+                auto output = net->onForward(v);
+                for (auto o : output) {
+                    ((MNN::Tensor*)o->getTensor())->wait(MNN::Tensor::MAP_TENSOR_READ, true);
+                }
+            }
+            times[t] = _l.durationInUs() / 1000.0f;
+        }
+        auto minTime = std::min_element(times.begin(), times.end());
+        auto maxTime = std::max_element(times.begin(), times.end());
+        float sum    = 0.0f;
+        for (auto time : times) {
+            sum += time;
+        }
+        MNN_PRINT("Avg= %f ms, min= %f ms, max= %f ms\n", sum / (float)testTime, *minTime, *maxTime);
+    }
+
+    return 0;
+}
+

tools/cpp/backendTest.cpp

@@ -0,0 +1,328 @@
+//
+//  backendTest.cpp
+//  MNN
+//
+//  Created by MNN on 2019/01/22.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#define MNN_OPEN_TIME_TRACE
+
+#include <math.h>
+#include <stdlib.h>
+#include <cstdlib>
+#include <algorithm>
+#include <cstring>
+#include <fstream>
+#include <iostream>
+#include <map>
+#include <sstream>
+#include <string>
+#include "MNN_generated.h"
+#include <MNN/expr/Module.hpp>
+#include <MNN/AutoTime.hpp>
+#include <MNN/Interpreter.hpp>
+#include <MNN/Tensor.hpp>
+#include "core/TensorUtils.hpp"
+#include "core/Session.hpp"
+#include "rapidjson/document.h"
+typedef std::vector<std::pair<std::string, std::vector<std::string>>> OUTPUTCONFIG;
+
+static OUTPUTCONFIG _getAllOutputs(const MNN::Net* net, const MNN::Session* session) {
+    auto info = session->getPipelineInfo(0);
+    std::vector<std::pair<std::string, std::vector<std::string>>> res;
+    auto tensorName = net->tensorName();
+    auto oplist = net->oplists();
+    if (nullptr == oplist || nullptr == tensorName) {
+        FUNC_PRINT(1);
+        return res;
+    }
+    for (int i=0; i<info.second.size(); ++i) {
+        auto& unit = info.second[i];
+        if (unit.type != MNN::Schedule::SEPARATE) {
+            continue;
+        }
+        auto op = unit.op;
+        if (op->type() == MNN::OpType_Const || op->type() == MNN::OpType_TrainableParam || op->type() == MNN::OpType_Input) {
+            continue;
+        }
+        if (nullptr == op->outputIndexes() || op->outputIndexes()->size() == 0) {
+            continue;
+        }
+        std::vector<std::string> outputNames(op->outputIndexes()->size());
+        for (int v=0; v<op->outputIndexes()->size(); ++v) {
+            auto index = op->outputIndexes()->data()[v];
+            outputNames[v] = tensorName->GetAsString(index)->str();
+        }
+        res.emplace_back(std::make_pair(op->name()->str(), outputNames));
+    }
+    return res;
+}
+static std::vector<std::string> _getAllInputs(const MNN::Net* net) {
+    auto tensorName = net->tensorName();
+    auto oplist = net->oplists();
+    std::vector<std::string> res;
+    if (nullptr == oplist || nullptr == tensorName) {
+        FUNC_PRINT(1);
+        return res;
+    }
+    for (int i=0; i<oplist->size(); ++i) {
+        auto op = oplist->GetAs<MNN::Op>(i);
+        if (op->type() == MNN::OpType_Input) {
+            auto index = op->outputIndexes()->data()[0];
+            res.emplace_back(tensorName->GetAsString(index)->str());
+        }
+    }
+    return res;
+}
+
+template<typename T>
+inline T stringConvert(const char* number) {
+    std::istringstream os(number);
+    T v;
+    os >> v;
+    return v;
+}
+
+using namespace MNN;
+
+static void _zeroInputs(const Interpreter* net, const Session* session) {
+    // Set Other Inputs to Zero
+    auto allInput = net->getSessionInputAll(session);
+    for (auto& iter : allInput) {
+        auto inputTensor = iter.second;
+        auto size = inputTensor->size();
+        if (size <= 0) {
+            continue;
+        }
+        MNN::Tensor tempTensor(inputTensor, inputTensor->getDimensionType());
+        ::memset(tempTensor.host<void>(), 0, tempTensor.size());
+        inputTensor->copyFromHostTensor(&tempTensor);
+    }
+}
+static void compareForwadType(OUTPUTCONFIG outputNames, Interpreter* net, MNNForwardType expectType, MNNForwardType compareType, float tolerance,
+                              const std::map<std::string, std::shared_ptr<Tensor>>& inputs, const std::string& stopOp, BackendConfig::PrecisionMode precision, int modeNum) {
+    auto inputNames = _getAllInputs(MNN::GetNet(net->getModelBuffer().first));
+    for (int v=0; v<outputNames.size(); ++v) {
+        auto outputName = outputNames[v].second;
+        auto opName = outputNames[v].first;
+        MNN::ScheduleConfig expectConfig, compareConfig;
+        BackendConfig backendConfig;
+        backendConfig.precision = precision;
+        expectConfig.type   = expectType;
+        expectConfig.path.inputs = inputNames;
+        expectConfig.path.outputs = outputName;
+        expectConfig.saveTensors = outputName;
+        expectConfig.path.mode = MNN::ScheduleConfig::Path::Tensor;
+
+        compareConfig.type  = compareType;
+        compareConfig.backendConfig = &backendConfig;
+        compareConfig.mode = modeNum;
+        compareConfig.path.inputs = inputNames;
+        compareConfig.path.outputs = outputName;
+        compareConfig.saveTensors = outputName;
+        compareConfig.path.mode = MNN::ScheduleConfig::Path::Tensor;
+        auto expectSession  = net->createSession(expectConfig);
+        auto compareSession = net->createSession(compareConfig);
+        auto realInputs = net->getSessionInputAll(expectSession);
+        _zeroInputs(net, expectSession);
+        _zeroInputs(net, compareSession);
+        for (auto& iter : inputs) {
+            if (realInputs.find(iter.first) == realInputs.end()) {
+                continue;
+            }
+            Tensor* expectInput = net->getSessionInput(expectSession, iter.first.empty() ? NULL : iter.first.c_str());
+            expectInput->copyFromHostTensor(iter.second.get());
+            Tensor* compareInput = net->getSessionInput(compareSession, iter.first.empty() ? NULL : iter.first.c_str());
+            compareInput->copyFromHostTensor(iter.second.get());
+        }
+        net->runSession(expectSession);
+        net->runSession(compareSession);
+        bool allCorrect = true;
+        bool outputValid = false;
+        auto compare = [&]() {
+            for(auto name : outputName) {
+                auto expectTensor = net->getSessionOutput(expectSession, name.c_str());
+                if (nullptr == expectTensor || expectTensor->host<void>() == nullptr) {
+                    MNN_ERROR("Can't compare tensor: %s\n", name.c_str());
+                    continue;
+                }
+                outputValid = true;
+                auto compareTensor = net->getSessionOutput(compareSession, name.c_str());
+                if (nullptr == compareTensor) {
+                    MNN_ERROR("%d [%s] Tensor %s invalid\n", v, opName.c_str(), name.c_str());
+                    allCorrect = false;
+                    break;
+                }
+                auto correct      = TensorUtils::compareTensors(compareTensor, expectTensor, tolerance, true);
+                if (!correct) {
+                    MNN_PRINT("%d [%s] Op outputs %s is error\n", v, opName.c_str(), name.c_str());
+                    allCorrect = false;
+                    break;
+                }
+            }
+        };
+        compare();
+        if (!outputValid) {
+            net->releaseSession(expectSession);
+            net->releaseSession(compareSession);
+            continue;
+        }
+
+        if (allCorrect) {
+            MNN_PRINT("Correct ! Run second pass\n");
+        } else {
+            return;
+        }
+        for (auto& iter : inputs) {
+            if (realInputs.find(iter.first) == realInputs.end()) {
+                continue;
+            }
+            Tensor* compareInput = net->getSessionInput(compareSession, iter.first.empty() ? NULL : iter.first.c_str());
+            compareInput->copyFromHostTensor(iter.second.get());
+        }
+        net->runSession(compareSession);
+        compare();
+        if (allCorrect) {
+            MNN_PRINT("Correct for %d, name=%s\n", v, opName.c_str());
+        } else {
+            return;
+        }
+        net->releaseSession(expectSession);
+        net->releaseSession(compareSession);
+    }
+    MNN_PRINT("Correct !\n");
+}
+
+int main(int argc, const char* argv[]) {
+    // read args
+    std::string cmd = argv[0];
+    std::string pwd = "./";
+    auto rslash     = cmd.rfind("/");
+    if (rslash != std::string::npos) {
+        pwd = cmd.substr(0, rslash + 1);
+    }
+
+    const char* fileName = argv[1];
+
+    auto type = MNN_FORWARD_CPU;
+    if (argc > 2) {
+        type = (MNNForwardType)stringConvert<int>(argv[2]);
+    }
+    MNN_PRINT("Test forward type: %d\n", type);
+
+    float tolerance = 0.05f;
+    if (argc > 3) {
+        tolerance = stringConvert<float>(argv[3]);
+    }
+    MNN_PRINT("Tolerance Rate: %f\n", tolerance);
+
+    // create net
+    MNN_PRINT("Open Model %s\n", fileName);
+    std::shared_ptr<MNN::Interpreter> net =
+        std::shared_ptr<MNN::Interpreter>(MNN::Interpreter::createFromFile(fileName));
+    net->setSessionMode(Interpreter::Session_Debug);
+
+    // create session
+    ScheduleConfig config;
+    config.type  = MNN_FORWARD_CPU;
+    auto session = net->createSession(config);
+
+    std::map<std::string, std::shared_ptr<MNN::Tensor>> inputs;
+    std::vector<std::string> inputNames;
+    do {
+        rapidjson::Document document;
+        std::ostringstream jsonNameOs;
+        jsonNameOs << pwd << "/input.json";
+        std::ifstream fileNames(jsonNameOs.str().c_str());
+        if (fileNames.fail()) {
+            break;
+        }
+        std::ostringstream output;
+        output << fileNames.rdbuf();
+        auto outputStr = output.str();
+        document.Parse(outputStr.c_str());
+        if (document.HasParseError()) {
+            MNN_ERROR("Invalid json\n");
+            break;
+        }
+        if (document.HasMember("inputs")) {
+            auto inputsInfo = document["inputs"].GetArray();
+            for (auto iter = inputsInfo.begin(); iter !=inputsInfo.end(); iter++) {
+                auto obj = iter->GetObject();
+                std::string name = obj["name"].GetString();
+                inputNames.emplace_back(name);
+            }
+        }
+    } while (false);
+    if (!inputNames.empty()) {
+        MNN_PRINT("Find input.json, use inputs:");
+        for (auto& n : inputNames) {
+            MNN_PRINT(" %s, ", n.c_str());
+        }
+        MNN_PRINT("\n");
+        for (auto name : inputNames) {
+            auto inputTensor = net->getSessionInput(session, name.c_str());
+            std::shared_ptr<MNN::Tensor> givenTensor(new Tensor(inputTensor, inputTensor->getDimensionType()));
+            {
+                std::ostringstream fileName;
+                fileName << pwd << name << ".txt";
+                std::ifstream input(fileName.str().c_str());
+                MNN_ASSERT(!input.fail());
+
+                int size_w = inputTensor->width();
+                int size_h = inputTensor->height();
+                int bpp    = inputTensor->channel();
+                int batch  = inputTensor->batch();
+                // auto backend = net->getBackend(session, inputTensor);
+                // MNN_ASSERT(!input.fail());
+                MNN_PRINT("Input: %d,%d,%d,%d\n", size_w, size_h, bpp, batch);
+                auto inputData = givenTensor->host<float>();
+                auto size      = givenTensor->size() / sizeof(float);
+                for (int i = 0; i < size; ++i) {
+                    input >> inputData[i];
+                }
+                inputs.insert(std::make_pair(name, givenTensor));
+            }
+
+        }
+    } else {
+        auto inputTensors = net->getSessionInputAll(session);
+        for (auto& iter : inputTensors) {
+            auto inputTensor = iter.second;
+            std::shared_ptr<MNN::Tensor> givenTensor(new Tensor(inputTensor, inputTensor->getDimensionType()));
+            auto rptr = givenTensor->host<void>();
+            size_t eleSize = TensorUtils::getRawSize(inputTensor);
+            if (inputTensor->getType().code == halide_type_float) {
+                auto ptr = (float*)rptr;
+                for (size_t v=0; v < eleSize; ++v) {
+                    ptr[v] = ((::rand() % 100) - 50) / 1000.0f;
+                }
+            } else {
+                ::memset(rptr, 0, eleSize * inputTensor->getType().bytes());
+            }
+            inputs.insert(std::make_pair(iter.first, givenTensor));
+        }
+    }
+    BackendConfig::PrecisionMode precision = BackendConfig::Precision_Normal;
+    if (argc > 4) {
+        precision = (BackendConfig::PrecisionMode)atoi(argv[4]);
+    }
+    FUNC_PRINT(precision);
+    int modeNum = 1;
+    if(argc > 5) {
+        modeNum = atoi(argv[5]);//set gpu mode
+    }
+    FUNC_PRINT(modeNum);
+    std::string stopOp = "";
+    if (argc > 6) {
+        stopOp = argv[6];
+    }
+    FUNC_PRINT_ALL(stopOp.c_str(), s);
+    auto outputNames = _getAllOutputs(MNN::GetNet(net->getModelBuffer().first), session);
+
+    net->releaseSession(session);
+    compareForwadType(outputNames, net.get(), MNN_FORWARD_CPU, type, tolerance, inputs, stopOp, precision, modeNum);
+
+    return 0;
+}

tools/cpp/checkDir.cpp

@@ -0,0 +1,129 @@
+//
+//  checkDir.cpp
+//  MNN
+//
+//  Created by MNN on 2019/01/22.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#include <assert.h>
+#include <stdio.h>
+#include <cmath>
+#include <fstream>
+#include <string>
+#include <vector>
+
+#if defined(_MSC_VER)
+#include <Windows.h>
+#undef min
+#undef max
+#else
+#include <dirent.h>
+#include <sys/stat.h>
+#endif
+
+using namespace std;
+
+#define NONE "\e[0m"
+#define RED "\e[0;31m"
+#define GREEN "\e[0;32m"
+#define L_GREEN "\e[1;32m"
+#define BLUE "\e[0;34m"
+#define L_BLUE "\e[1;34m"
+#define BOLD "\e[1m"
+
+int main(int argc, char* argv[]) {
+    if (argc < 3) {
+        printf("Usage: ./checkDir.out outputDir1 outputDir2 thredhold [order]\n");
+        return 0;
+    }
+
+    // read args
+    printf("Compare:\n");
+    printf("%s\n", argv[1]);
+    printf("%s\n", argv[2]);
+    float tolerance = 0.001;
+    if (argc > 3) {
+        tolerance = stof(string(argv[3]));
+    }
+    printf("tolerance=%f\n", tolerance);
+
+    std::vector<std::string> compareFiles;
+    if (argc > 4) {
+        printf("Order file is %s\n", argv[4]);
+        std::ifstream orderOs(argv[4]);
+        std::string stringLine;
+        while (getline(orderOs, stringLine, '\n')) {
+            compareFiles.emplace_back(stringLine);
+        }
+    } else {
+#if defined(_MSC_VER)
+        WIN32_FIND_DATA ffd;
+        HANDLE hFind = INVALID_HANDLE_VALUE;
+        hFind = FindFirstFile(argv[1], &ffd);
+        if (INVALID_HANDLE_VALUE == hFind) {
+            printf("Error to open %s\n", argv[1]);
+            return 0;
+        }
+        do {
+            if(INVALID_FILE_ATTRIBUTES != GetFileAttributes(ffd.cFileName) && GetLastError() != ERROR_FILE_NOT_FOUND) {
+                compareFiles.push_back(ffd.cFileName);
+            }
+        } while (FindNextFile(hFind, &ffd) != 0);
+        FindClose(hFind);
+#else
+        // open dir
+        DIR* root = opendir(argv[1]);
+        if (NULL == root) {
+            printf("Error to open %s\n", argv[1]);
+            return 0;
+        }
+        struct dirent* ent;
+        while ((ent = readdir(root)) != NULL) {
+            compareFiles.push_back(ent->d_name);
+        }
+        closedir(root);
+#endif
+    }
+
+    // compare files
+    for (int i=0; i<compareFiles.size(); ++i) {
+        auto& s = compareFiles[i];
+        if (s.size() <= 2) {
+            continue;
+        }
+        std::string empty   = "";
+        std::string oldFile = empty + argv[1] + "/" + s;
+        std::string newFile = empty + argv[2] + "/" + s;
+        std::ifstream oldOs(oldFile.c_str());
+        if (oldOs.fail()) {
+            // printf("Can's open %s\n", oldFile.c_str());
+            continue;
+        }
+        std::ifstream newOs(newFile.c_str());
+        if (newOs.fail()) {
+            // printf("Can's open %s\n", newFile.c_str());
+            continue;
+        }
+
+        int pos      = 0;
+        bool correct = true;
+        float v1, v2;
+        while (oldOs >> v1) {
+            auto& valid   = newOs >> v2;
+            auto absError = fabsf(v1 - v2);
+            if (absError <= tolerance && !isnan(absError)) {
+                pos++;
+                continue;
+            }
+            printf("Error for %d, %s, %d, v1=%.6f, v2=%.6f\n", i, s.c_str(), pos, v1, v2);
+            correct = false;
+            break;
+        }
+        if (correct) {
+            printf("Correct %d: %s\n", i, s.c_str());
+        }
+    }
+
+    return 0;
+}

tools/cpp/checkFile.cpp

@@ -0,0 +1,53 @@
+//
+//  checkFile.cpp
+//  MNN
+//
+//  Created by MNN on 2019/01/22.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <cmath>
+#include <fstream>
+#include <sstream>
+
+using namespace std;
+
+int main(int argc, char* argv[]) {
+    if (argc < 3) {
+        printf("Usage: ./checkFile.out XXX.txt YYY.txt tolerance\n");
+        return 0;
+    }
+
+    // read args
+    const char* file1 = argv[1];
+    const char* file2 = argv[2];
+    float tolerance   = 0.001;
+    if (argc > 3) {
+        std::istringstream ss(argv[3]);
+        ss >> tolerance;
+    }
+
+    // open file
+    std::ifstream input1(file1);
+    assert(!input1.fail());
+    std::ifstream input2(file2);
+    assert(!input2.fail());
+
+    // compare
+    float v1, v2;
+    int pos = 0;
+    while (input1 >> v1) {
+        auto& valid = input2 >> v2;
+        if (::fabsf(v1 - v2) > tolerance) {
+            printf("Error for %d, v1=%.6f, v2=%.6f\n", pos, v1, v2);
+        }
+        pos++;
+        if (!valid) {
+            break;
+        }
+    }
+    return 0;
+}

tools/cpp/checkInvalidValue.cpp

@@ -0,0 +1,113 @@
+//
+//  checkInvalidValue.cpp
+//  MNN
+//
+//  Created by MNN on 2019/10/28.
+//  Copyright © 2018, Alibaba Group Htargeting Limited
+//
+
+#include <assert.h>
+#include <stdio.h>
+#include <cmath>
+#include <fstream>
+#include <string>
+#include <vector>
+#include <stdlib.h>
+
+#if defined(_MSC_VER)
+#include <Windows.h>
+#undef min
+#undef max
+#else
+#include <dirent.h>
+#include <sys/stat.h>
+#endif
+
+using namespace std;
+
+#define NONE "\e[0m"
+#define RED "\e[0;31m"
+#define GREEN "\e[0;32m"
+#define L_GREEN "\e[1;32m"
+#define BLUE "\e[0;34m"
+#define L_BLUE "\e[1;34m"
+#define Btarget "\e[1m"
+
+int main(int argc, char* argv[]) {
+    if (argc < 2) {
+        printf("Usage: ./checkInvalidValue.out outputDir [limit]\n");
+        return 0;
+    }
+
+    // read args
+    printf("Compare:\n");
+    printf("%s\n", argv[1]);
+    float tolerance = 0.001;
+    int limit = 10;
+    if (argc > 2) {
+        limit = atoi(argv[2]);
+    }
+    printf("limit=%d\n", limit);
+
+    std::vector<std::string> compareFiles;
+#if defined(_MSC_VER)
+    WIN32_FIND_DATA ffd;
+    HANDLE hFind = INVALID_HANDLE_VALUE;
+    hFind = FindFirstFile(argv[1], &ffd);
+    if (INVALID_HANDLE_VALUE == hFind) {
+        printf("Error to open %s\n", argv[1]);
+        return 0;
+    }
+    do {
+        if(INVALID_FILE_ATTRIBUTES != GetFileAttributes(ffd.cFileName) && GetLastError() != ERROR_FILE_NOT_FOUND) {
+            compareFiles.push_back(ffd.cFileName);
+        }
+    } while (FindNextFile(hFind, &ffd) != 0);
+    FindClose(hFind);
+#else
+    // open dir
+    DIR* root = opendir(argv[1]);
+    if (NULL == root) {
+        printf("Error to open %s\n", argv[1]);
+        return 0;
+    }
+    struct dirent* ent;
+    while ((ent = readdir(root)) != NULL) {
+        compareFiles.push_back(ent->d_name);
+    }
+    closedir(root);
+#endif
+    auto limitValue = powf(10, limit);
+
+    // compare files
+    for (auto s : compareFiles) {
+        if (s.size() <= 2) {
+            continue;
+        }
+        std::string empty   = "";
+        std::string targetFile = empty + argv[1] + "/" + s;
+        std::ifstream targetOs(targetFile.c_str());
+        if (targetOs.fail()) {
+            // printf("Can's open %s\n", targetFile.c_str());
+            continue;
+        }
+        int pos      = 0;
+        bool correct = true;
+        float v1;
+        while (targetOs >> v1) {
+            auto absValue = fabsf(v1);
+            if (!isnan(v1) && absValue < limitValue) {
+                pos++;
+                continue;
+            }
+            printf(RED "Error for %s, %d, v1=%.6f\n" NONE, s.c_str(), pos, v1);
+            correct = false;
+            break;
+        }
+        if (correct) {
+            printf(GREEN "Correct : %s\n" NONE, s.c_str());
+        }
+    }
+
+    return 0;
+}

tools/cpp/fuseTest.cpp

@@ -0,0 +1,255 @@
+#include <fstream>
+#include <sstream>
+#include "MNN_generated.h"
+#include <MNN/MNNDefine.h>
+#include <MNN/Tensor.hpp>
+#include <MNN/Interpreter.hpp>
+#include "core/Backend.hpp"
+#include "core/Macro.h"
+#include "core/TensorUtils.hpp"
+#include "rapidjson/document.h"
+#include "core/Execution.hpp"
+
+using namespace MNN;
+int main(int argc, const char* argv[]) {
+    if (argc < 3) {
+        MNN_ERROR("Usage: ./fuseTest XXX.spirv XXX.json\n");
+        return 0;
+    }
+    {
+        ScheduleConfig config;
+        std::vector<ScheduleConfig> configs = {config};
+        auto rt = Interpreter::createRuntime(configs);
+    }
+    rapidjson::Document configJson;
+    std::ifstream fileNames(argv[2]);
+    if (fileNames.fail()) {
+        MNN_ERROR("Can' open config file: %s\n", argv[2]);
+        return 0;
+    }
+    {
+        std::ostringstream output;
+        output << fileNames.rdbuf();
+        auto outputStr = output.str();
+        configJson.Parse(outputStr.c_str());
+    }
+    if (configJson.HasParseError()) {
+        MNN_ERROR("Invalid json\n");
+        return 0;
+    }
+
+    auto type    = MNN_FORWARD_VULKAN;
+    auto creator = MNNGetExtraRuntimeCreator(type);
+    if (nullptr == creator) {
+        MNN_ERROR("Don't support %d\n", type);
+        return 0;;
+    }
+    MNN::Backend::Info info;
+    info.type = type;
+    BackendConfig user;
+    user.precision = BackendConfig::Precision_High;
+    info.user = &user;
+    std::shared_ptr<Runtime> runtime(creator->onCreate(info));
+    std::shared_ptr<Backend> bn(runtime->onCreate(&user));
+    
+    // Load Config
+    std::unique_ptr<MNN::OpT> op(new OpT);
+    op->type = OpType_Extra;
+    op->main.type = OpParameter_Extra;
+    op->main.value = new ExtraT;
+    std::vector<std::shared_ptr<MNN::Tensor>> inputs;
+    std::vector<std::shared_ptr<MNN::Tensor>> outputs;
+    if (configJson.HasMember("inputs")) {
+        auto inputArray = configJson["inputs"].GetArray();
+        int pos = 0;
+        for (auto iter = inputArray.Begin(); iter != inputArray.End(); iter++) {
+            std::unique_ptr<AttributeT> attr(new AttributeT);
+            attr->key = "input";
+            attr->list.reset(new ListValueT);
+            attr->i = (*iter)["binding"].GetInt();
+            attr->list->i = {0, pos};
+            attr->b = false;
+
+            op->main.AsExtra()->attr.emplace_back(std::move(attr));
+            halide_type_t type = halide_type_of<float>();
+            std::vector<int> shape;
+            if (iter->HasMember("dims")) {
+                auto dimArray = (*iter)["dims"].GetArray();
+                for (auto shapeIter = dimArray.Begin(); shapeIter != dimArray.End(); shapeIter++) {
+                    shape.emplace_back(shapeIter->GetInt());
+                }
+            }
+            // Create Tensor
+            std::shared_ptr<MNN::Tensor> tensor(Tensor::createDevice(shape, type, Tensor::CAFFE));
+            bn->onAcquireBuffer(tensor.get(), Backend::STATIC);
+            TensorUtils::getDescribeOrigin(tensor.get())->setBackend(bn.get());
+            bool isFloat = std::string((*iter)["type"].GetString()) == "float";
+            if (iter->HasMember("filename")) {
+                auto ptr = tensor->map(MNN::Tensor::MAP_TENSOR_WRITE, MNN::Tensor::CAFFE);
+                {
+                    auto fileName = std::string( (*iter)["filename"].GetString());
+                    FUNC_PRINT_ALL(fileName.c_str(), s);
+                    std::ifstream is(fileName.c_str());
+                    if (is.fail()) {
+                        MNN_ERROR("Can't open data file for %d input\n", pos);
+                    }
+                    auto size = tensor->elementSize();
+                    if (isFloat) {
+                        auto uptr = (float*)ptr;
+                        for (int i=0; i<size; ++i) {
+                            float v;
+                            is >> v;
+                            uptr[i] = v;
+                        }
+                    } else {
+                        auto uptr = (uint32_t*)ptr;
+                        for (int i=0; i<size; ++i) {
+                            float v;
+                            is >> v;
+                            uptr[i] = v;
+                        }
+                    }
+                }
+                tensor->unmap(MNN::Tensor::MAP_TENSOR_WRITE, MNN::Tensor::CAFFE, ptr);
+            }
+            inputs.emplace_back(tensor);
+            pos++;
+        }
+    }
+    if (configJson.HasMember("outputs")) {
+        auto inputArray = configJson["outputs"].GetArray();
+        int pos = 0;
+        for (auto iter = inputArray.Begin(); iter != inputArray.End(); iter++) {
+            std::unique_ptr<AttributeT> attr(new AttributeT);
+            attr->key = "input";
+            attr->list.reset(new ListValueT);
+            attr->i = (*iter)["binding"].GetInt();
+            attr->list->i = {1, pos};
+            attr->b = false;
+
+            op->main.AsExtra()->attr.emplace_back(std::move(attr));
+            halide_type_t type = halide_type_of<float>();
+            std::vector<int> shape;
+            if (iter->HasMember("dims")) {
+                auto dimArray = (*iter)["dims"].GetArray();
+                for (auto shapeIter = dimArray.Begin(); shapeIter != dimArray.End(); shapeIter++) {
+                    shape.emplace_back(shapeIter->GetInt());
+                }
+            }
+            // Create Tensor
+            std::shared_ptr<MNN::Tensor> tensor(Tensor::createDevice(shape, type, Tensor::CAFFE));
+            bn->onAcquireBuffer(tensor.get(), Backend::STATIC);
+            TensorUtils::getDescribeOrigin(tensor.get())->setBackend(bn.get());
+            outputs.emplace_back(tensor);
+            pos++;
+        }
+    }
+    if (configJson.HasMember("uniforms")) {
+        auto inputArray = configJson["uniforms"].GetArray();
+        int pos = 0;
+        for (auto iter = inputArray.Begin(); iter != inputArray.End(); iter++) {
+            std::unique_ptr<AttributeT> attr(new AttributeT);
+            attr->key = "const";
+            attr->list.reset(new ListValueT);
+            attr->i = (*iter)["binding"].GetInt();
+            attr->b = true;
+            attr->tensor.reset(new BlobT);
+            attr->tensor->dataType = DataType_DT_INT32;
+            std::vector<int> shape;
+            int size = 1;
+            if (iter->HasMember("dims")) {
+                auto dimArray = (*iter)["dims"].GetArray();
+                for (auto shapeIter = dimArray.Begin(); shapeIter != dimArray.End(); shapeIter++) {
+                    shape.emplace_back(shapeIter->GetInt());
+                    size *= shapeIter->GetInt();
+                }
+            }
+            attr->tensor->dims = shape;
+            attr->tensor->dataFormat = MNN_DATA_FORMAT_NCHW;
+            if (iter->HasMember("data")) {
+                auto dimArray = (*iter)["data"].GetArray();
+                for (auto shapeIter = dimArray.Begin(); shapeIter != dimArray.End(); shapeIter++) {
+                    attr->tensor->int32s.emplace_back(shapeIter->GetInt());
+                }
+            }
+            op->main.AsExtra()->attr.emplace_back(std::move(attr));
+        }
+    }
+    if (configJson.HasMember("group_size")) {
+        std::vector<int> shape;
+        auto dimArray = configJson["group_size"].GetArray();
+        for (auto shapeIter = dimArray.Begin(); shapeIter != dimArray.End(); shapeIter++) {
+            shape.emplace_back(shapeIter->GetInt());
+        }
+        std::unique_ptr<AttributeT> attr(new AttributeT);
+        attr->key = "group_size";
+        attr->tensor.reset(new BlobT);
+        attr->tensor->int32s = shape;
+        op->main.AsExtra()->attr.emplace_back(std::move(attr));
+    }
+    if (configJson.HasMember("local_size")) {
+        std::vector<int> shape;
+        auto dimArray = configJson["local_size"].GetArray();
+        for (auto shapeIter = dimArray.Begin(); shapeIter != dimArray.End(); shapeIter++) {
+            shape.emplace_back(shapeIter->GetInt());
+        }
+        std::unique_ptr<AttributeT> attr(new AttributeT);
+        attr->key = "local_size";
+        attr->tensor.reset(new BlobT);
+        attr->tensor->int32s = shape;
+        op->main.AsExtra()->attr.emplace_back(std::move(attr));
+    }
+    {
+        std::ifstream is(argv[1]);
+        if (is.fail()) {
+            MNN_ERROR("Can't load spirv\n");
+            return 0;
+        }
+        is.seekg(0, std::ios::end);
+        std::unique_ptr<AttributeT> attr(new AttributeT);
+        attr->key = "spirv";
+        attr->tensor.reset(new BlobT);
+        attr->tensor->int8s.resize(is.tellg());
+        is.seekg(0, std::ios::beg);
+        is.read((char*)attr->tensor->int8s.data(), attr->tensor->int8s.size());
+        op->main.AsExtra()->attr.emplace_back(std::move(attr));
+    }
+    std::vector<Tensor*> inputsW(inputs.size());
+    for (int i=0; i<inputs.size(); ++i) {
+        inputsW[i] = inputs[i].get();
+    }
+    std::vector<Tensor*> outputsW(outputs.size());
+    for (int i=0; i<outputs.size(); ++i) {
+        outputsW[i] = outputs[i].get();
+    }
+    flatbuffers::FlatBufferBuilder builder;
+    builder.Finish(Op::Pack(builder, op.get()));
+    auto opRaw = flatbuffers::GetRoot<Op>(builder.GetBufferPointer());
+    std::shared_ptr<MNN::Execution> exeution(bn->onCreate(inputsW, outputsW, opRaw));
+    bn->onResizeBegin();
+    exeution->onResize(inputsW, outputsW);
+    bn->onResizeEnd();
+    bn->onExecuteBegin();
+    exeution->onExecute(inputsW, outputsW);
+    bn->onExecuteEnd();
+    for (int i=0; i<outputsW.size(); ++i) {
+        auto ptr = outputsW[i]->map(MNN::Tensor::MAP_TENSOR_READ, MNN::Tensor::CAFFE);
+        auto size = outputsW[i]->elementSize();
+        auto iPtr = (int32_t*)ptr;
+        std::ostringstream fileNameOs;
+        fileNameOs << i << ".txt";
+        std::ofstream _o(fileNameOs.str().c_str());
+        for (int v=0; v<size; ++v) {
+            _o << iPtr[v] << "\n";
+        }
+        outputsW[i]->unmap(MNN::Tensor::MAP_TENSOR_READ, MNN::Tensor::CAFFE, ptr);
+    }
+    
+    exeution.reset();
+    inputs.clear();
+    outputs.clear();
+    bn.reset();
+    runtime.reset();
+
+    return 0;
+}

tools/cpp/generateIO.cpp

@@ -0,0 +1,177 @@
+#include <fstream>
+#include <string>
+#include <vector>
+#include <memory>
+#include <cstdlib>
+#include <ctime>
+
+#include <MNN/expr/Module.hpp>
+#include <MNN/expr/ExprCreator.hpp>
+#include <MNN/expr/Executor.hpp>
+#include "core/MNNFileUtils.h"
+
+#include "rapidjson/document.h"
+#include "rapidjson/istreamwrapper.h"
+#include <iostream>
+#include <sstream>
+
+#include <limits>
+
+static void saveInputOutputs(const MNN::Express::Module::Info* info, std::vector<MNN::Express::VARP> inputs,
+                             std::vector<MNN::Express::VARP> outputs, const std::string& outputDir) {
+    MNN_ASSERT(info->inputNames.size() == inputs.size());
+    MNN_ASSERT(info->outputNames.size() == outputs.size());
+    for (int i = 0; i < info->inputNames.size(); ++i) {
+        inputs[i].fix(MNN::Express::VARP::CONSTANT);
+        inputs[i]->setName(info->inputNames[i]);
+    }
+    for (int i = 0; i < info->outputNames.size(); ++i) {
+        outputs[i]->setName(info->outputNames[i]);
+    }
+
+    std::string inputPath = MNNFilePathConcat(outputDir, "input.mnn");
+    std::string outputPath = MNNFilePathConcat(outputDir, "output.mnn");
+    MNN::Express::Variable::save(inputs, inputPath.c_str());
+    MNN::Express::Variable::save(outputs, outputPath.c_str());
+    MNN_PRINT("Successfully generate %s and %s.\n", inputPath.c_str(), outputPath.c_str());
+}
+
+int main(int argc, char* argv[]) {
+    if (argc < 4) {
+        MNN_PRINT("Usage: ./generateLlmIO model inputJson outputDir externalFilePath\n");
+        return 1;
+    }
+
+    std::string ExternalFilePath;
+    std::string modelPath = std::string(argv[1]);
+    std::string inputJson = argv[2];
+    std::string outputDir = argv[3];
+    if (argc >= 5) {
+        ExternalFilePath = argv[4];
+    }
+
+    rapidjson::Document document;
+    std::ifstream fileNames(inputJson.c_str());
+    std::ostringstream output;
+    output << fileNames.rdbuf();
+    auto outputStr = output.str();
+    document.Parse(outputStr.c_str());
+    if (document.HasParseError()) {
+        MNN_ERROR("Invalid json\n");
+        return 0;
+    }
+    int shapeIndex = 0;
+    std::shared_ptr<MNN::Express::Module> net;
+
+    if (document.HasMember("configs")) {
+        if (!(MNNCreateDir(outputDir.c_str()))) {
+            MNN_PRINT("Failed to create dir %s.\n", outputDir.c_str());
+        }
+        auto configsArray = document["configs"].GetArray();
+        for (auto& configObj : configsArray) {
+            std::map<std::string, float> inputInfo;
+            std::map<std::string, std::string> inputType;
+            std::vector<std::string> inputNames;
+            std::vector<std::string> outputNames;
+            std::map<std::string, std::vector<int>> inputShape;
+            std::vector<MNN::Express::VARP> inputs;
+            std::vector<MNN::Express::VARP> outputs;
+            if (configObj.HasMember("inputs")) {
+                auto inputsInfo = configObj["inputs"].GetArray();
+                for (auto iter = inputsInfo.begin(); iter != inputsInfo.end(); iter++) {
+                    auto obj = iter->GetObject();
+                    std::string type = "float";
+                    std::string name = obj["name"].GetString();
+                    inputNames.emplace_back(name);
+                    if (obj.HasMember("type")) {
+                        type = obj["type"].GetString();
+                        inputType.insert(std::make_pair(name, type));
+                    }
+                    if (obj.HasMember("value")) {
+                        float value;
+                        if (type == "int") {
+                            value = (float)obj["value"].GetInt();
+                        } else {
+                            value = obj["value"].GetFloat();
+                        }
+                        inputInfo.insert(std::make_pair(name, value));
+                    }
+                    if (obj.HasMember("shape")) {
+                        auto dims = obj["shape"].GetArray();
+                        std::vector<int> shapes;
+                        for (auto iter = dims.begin(); iter != dims.end(); iter++) {
+                            shapes.emplace_back(iter->GetInt());
+                        }
+                        inputShape.insert(std::make_pair(name, shapes));
+                    }
+                }
+            }
+            if (configObj.HasMember("outputs")) {
+                auto array = configObj["outputs"].GetArray();
+                for (auto iter = array.begin(); iter != array.end(); iter++) {
+                    std::string name = iter->GetString();
+                    outputNames.emplace_back(name);
+                }
+            }
+
+            // Load Model.
+            if (net.get() == nullptr) {
+                MNN::ScheduleConfig config;
+                std::shared_ptr<MNN::Express::Executor::RuntimeManager> rtmgr(
+                    MNN::Express::Executor::RuntimeManager::createRuntimeManager(config));
+                if (ExternalFilePath.length() > 0) {
+                    rtmgr->setExternalFile(ExternalFilePath.c_str());
+                }
+                net.reset(MNN::Express::Module::load(inputNames, outputNames, modelPath.c_str(), rtmgr),
+                          MNN::Express::Module::destroy);
+            }
+
+            auto mInfo = net->getInfo();
+            // create input
+            inputs.resize(mInfo->inputs.size());
+            for (int i = 0; i < inputs.size(); ++i) {
+                inputs[i] = _Input(mInfo->inputs[i].dim, mInfo->inputs[i].order, mInfo->inputs[i].type);
+            }
+            // Load inputs
+            for (int i = 0; i < inputs.size(); ++i) {
+                auto inputName = inputNames[i];
+                std::string type = "float";
+                auto typeIter = inputType.find(inputName);
+                if (typeIter != inputType.end()) {
+                    type = typeIter->second;
+                }
+                // Resize
+                auto shapeIter = inputShape.find(inputName);
+                if (shapeIter != inputShape.end()) {
+                    auto s = shapeIter->second;
+                    inputs[i] = _Input(s, mInfo->inputs[i].order, mInfo->inputs[i].type);
+                }
+                auto info = inputs[i]->getInfo();
+                if (inputInfo.find(inputName) != inputInfo.end()) {
+                    auto value = inputInfo[inputName];
+                    if (type == "int") {
+                        auto ptr = inputs[i]->writeMap<int>();
+                        for (int i = 0; i < info->size; ++i) {
+                            ptr[i] = (int)value;
+                        }
+                    } else {
+                        auto ptr = inputs[i]->writeMap<float>();
+                        for (int i = 0; i < info->size; ++i) {
+                            ptr[i] = (float)value;
+                        }
+                    }
+                }
+            }
+
+            std::string outputDirtmp = MNNFilePathConcat(outputDir, std::to_string(shapeIndex++));
+            if (!(MNNCreateDir(outputDirtmp.c_str()))) {
+                MNN_PRINT("Failed to create dir %s.\n", outputDirtmp.c_str());
+            }
+
+            outputs = net->onForward(inputs);
+            saveInputOutputs(net->getInfo(), inputs, outputs, outputDirtmp);
+        }
+    }
+
+    return 0;
+}

tools/cpp/getLinearInput.hpp

@@ -0,0 +1,248 @@
+#ifndef GET_LINEAR_INPUT_HPP
+#define GET_LINEAR_INPUT_HPP
+
+#include <MNN/expr/Expr.hpp>
+#include <MNN/expr/ExprCreator.hpp>
+#include <MNN/expr/ExecutorScope.hpp>
+#include <MNN/Tensor.hpp>
+#include <mutex>
+#include <fstream>
+#include <cstdlib>
+#include <string>
+
+namespace MNN {
+namespace LinearInput {
+
+static std::ofstream thresholdFile;
+static std::mutex thresholdFileMutex;
+static bool isFirstThreshold = true;
+static std::string currentThresholdFile = "thresholds.json";
+
+static void closeThresholdFile() {
+    std::lock_guard<std::mutex> lock(thresholdFileMutex);
+    
+    if (thresholdFile.is_open()) {
+        thresholdFile << "\n}\n";
+        thresholdFile.close();
+        MNN_PRINT("Threshold file closed: %s\n", currentThresholdFile.c_str());
+    }
+}
+
+static void initThresholdFile(const std::string& filename = "thresholds.json") {
+    std::lock_guard<std::mutex> lock(thresholdFileMutex);
+    
+    if (thresholdFile.is_open()) {
+        thresholdFile.close();
+    }
+    
+    currentThresholdFile = filename;
+    thresholdFile.open(filename, std::ios::out);
+    if (thresholdFile.is_open()) {
+        thresholdFile << "{\n";
+        thresholdFile.flush();
+        isFirstThreshold = true;
+        MNN_PRINT("Initialized threshold file: %s\n", filename.c_str());
+    } else {
+        MNN_ERROR("Failed to open threshold file: %s\n", filename.c_str());
+    }
+}
+
+static void writeThresholdRealtime(const std::string& opName, float thresholdValue) {
+    std::lock_guard<std::mutex> lock(thresholdFileMutex);
+    
+    if (!thresholdFile.is_open()) {
+        initThresholdFile(currentThresholdFile);
+    }
+    
+    if (thresholdFile.is_open()) {
+        if (!isFirstThreshold) {
+            thresholdFile << ",\n";
+        }
+        
+        thresholdFile << "    \"" << opName << "\": " << thresholdValue;
+        thresholdFile.flush();  
+        isFirstThreshold = false;
+        
+        MNN_PRINT("Saved threshold: %s = %f\n", opName.c_str(), thresholdValue);
+    }
+}
+
+static std::ofstream maxValueFile;
+static std::mutex maxValueFileMutex;
+static bool isFirstMaxValue = true;
+static std::string currentMaxValueFile = "max_values.json";
+
+static void closeMaxValueFile() {
+    std::lock_guard<std::mutex> lock(maxValueFileMutex);
+    
+    if (maxValueFile.is_open()) {
+        maxValueFile << "\n}\n";
+        maxValueFile.close();
+        MNN_PRINT("Max value file closed: %s\n", currentMaxValueFile.c_str());
+    }
+}
+
+static void initMaxValueFile(const std::string& filename = "max_values.json") {
+    std::lock_guard<std::mutex> lock(maxValueFileMutex);
+    
+    if (maxValueFile.is_open()) {
+        maxValueFile.close();
+    }
+    
+    currentMaxValueFile = filename;
+    maxValueFile.open(filename, std::ios::out);
+    if (maxValueFile.is_open()) {
+        maxValueFile << "{\n";
+        maxValueFile.flush();
+        isFirstMaxValue = true;
+        MNN_PRINT("Initialized max value file: %s\n", filename.c_str());
+    } else {
+        MNN_ERROR("Failed to open max value file: %s\n", filename.c_str());
+    }
+}
+
+static void writeMaxValueRealtime(const std::string& opName, float maxValueFloat) {
+    std::lock_guard<std::mutex> lock(maxValueFileMutex);
+    
+    if (!maxValueFile.is_open()) {
+        initMaxValueFile(currentMaxValueFile);
+    }
+    
+    if (maxValueFile.is_open()) {
+        if (!isFirstMaxValue) {
+            maxValueFile << ",\n";
+        }
+        
+        maxValueFile << "    \"" << opName << "\": " << maxValueFloat;
+        maxValueFile.flush();  
+        isFirstMaxValue = false;
+        
+        MNN_PRINT("Saved max value: %s = %f\n", opName.c_str(), maxValueFloat);
+    }
+}
+
+static void cleanupAtExit() {
+    closeThresholdFile();
+    closeMaxValueFile();
+}
+
+
+inline void initGetThreshold(const std::string& thresholdFileName, float targetSparsity) {
+    initThresholdFile(thresholdFileName);
+    
+    static bool registered = false;
+    if (!registered) {
+        std::atexit(cleanupAtExit);
+        registered = true;
+    }
+
+    MNN::TensorCallBackWithInfo beforeCallBack = [targetSparsity](const std::vector<MNN::Tensor*>& ntensors, const MNN::OperatorInfo* info) {
+        auto opName = info->name();
+        if (info->type() == "Copy") {
+            return true;
+        }
+        if (opName.find("Linear") == std::string::npos || opName.find("raster") != std::string::npos) {
+            return true;
+        }
+        for (int i = 0; i < ntensors.size(); ++i) {
+            auto ntensor = ntensors[i];
+            auto outDimType = ntensor->getDimensionType();
+            std::shared_ptr<MNN::Tensor> expectTensor(new MNN::Tensor(ntensor, outDimType));
+            bool res = ntensor->copyToHostTensor(expectTensor.get());
+            if (res) {
+                ntensor = expectTensor.get();
+            }
+            {
+                auto ninput = MNN::Express::Variable::create(MNN::Express::Expr::create(ntensor));
+                if (nullptr == ninput->getInfo()) {
+                    MNN_ERROR("Alloc memory or compute size error\n");
+                    return false;
+                }
+                ninput = MNN::Express::_Convert(ninput, MNN::Express::NHWC);
+                ninput = MNN::Express::_Abs(ninput);
+                ninput = MNN::Express::_Reshape(ninput, {-1});
+
+                auto totalNum = ninput->getInfo()->dim[0];
+                int keepNum = totalNum * (1 - targetSparsity);
+
+                auto kv = MNN::Express::_TopKV2(ninput, MNN::Express::_Scalar<int>(keepNum));
+                auto values = kv[0];
+
+                auto threshold = MNN::Express::_Gather(values, MNN::Express::_Scalar<int>(keepNum - 1));
+                auto thresholdValue = threshold->readMap<float>()[0];
+
+                writeThresholdRealtime(opName, thresholdValue);
+            }
+        }
+        return true;
+    };
+
+    MNN::TensorCallBackWithInfo callBack = [](const std::vector<MNN::Tensor*>& ntensors, const MNN::OperatorInfo* info) {
+        return true;
+    };
+
+    MNN::Express::ExecutorScope::Current()->setCallBack(std::move(beforeCallBack), std::move(callBack));
+}
+
+inline void initGetMaxValue(const std::string& maxValueFileName) {
+    initMaxValueFile(maxValueFileName);
+    
+    static bool registered = false;
+    if (!registered) {
+        std::atexit(cleanupAtExit);
+        registered = true;
+    }
+
+    MNN::TensorCallBackWithInfo beforeCallBack = [](const std::vector<MNN::Tensor*>& ntensors, const MNN::OperatorInfo* info) {
+        auto opName = info->name();
+        if (info->type() == "Copy") {
+            return true;
+        }
+        if (opName.find("Linear") == std::string::npos || opName.find("raster") != std::string::npos) {
+            return true;
+        }
+
+        for (int i = 0; i < ntensors.size(); ++i) {
+            auto ntensor = ntensors[i];
+            auto outDimType = ntensor->getDimensionType();
+            std::shared_ptr<MNN::Tensor> expectTensor(new MNN::Tensor(ntensor, outDimType));
+            bool res = ntensor->copyToHostTensor(expectTensor.get());
+            if (res) {
+                ntensor = expectTensor.get();
+            }
+            {
+                auto ninput = MNN::Express::Variable::create(MNN::Express::Expr::create(ntensor));
+                if (nullptr == ninput->getInfo()) {
+                    MNN_ERROR("Alloc memory or compute size error\n");
+                    return false;
+                }
+                ninput = MNN::Express::_Convert(ninput, MNN::Express::NHWC);
+                ninput = MNN::Express::_Abs(ninput);  
+                ninput = MNN::Express::_Reshape(ninput, {-1}); 
+
+                auto kv = MNN::Express::_TopKV2(ninput, MNN::Express::_Scalar<int>(1));     
+                auto maxValues = kv[0];  
+                auto maxValueFloat = maxValues->readMap<float>()[0];
+
+                writeMaxValueRealtime(opName, maxValueFloat);
+            }
+        }
+        return true;
+    };
+
+    MNN::TensorCallBackWithInfo callBack = [](const std::vector<MNN::Tensor*>& ntensors, const MNN::OperatorInfo* info) {
+        return true;
+    };
+
+    MNN::Express::ExecutorScope::Current()->setCallBack(std::move(beforeCallBack), std::move(callBack));
+}
+
+inline void closeAllFiles() {
+    closeThresholdFile();
+    closeMaxValueFile();
+}
+
+} // namespace LinearInput
+} // namespace MNN
+
+#endif // GET_LINEAR_INPUT_HPP

tools/cpp/getPerformance.cpp

@@ -0,0 +1,239 @@
+//
+//  getPerformance.cpp
+//  MNN
+//
+//  Created by MNN on 2019/03/12.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#include <string.h>
+#include <chrono>
+#include <cstdint>
+#include <vector>
+#include <thread>
+#include <MNN/AutoTime.hpp>
+#include <stdlib.h>
+#include <MNN/MNNDefine.h>
+#include "core/Macro.h"
+#ifdef MNN_USE_NEON
+#include <arm_neon.h>
+#endif
+
+class Timer {
+private:
+    std::chrono::high_resolution_clock::time_point inTime, outTime;
+
+public:
+    void startTimer() {
+        inTime = std::chrono::high_resolution_clock::now();
+    }
+
+    // unit ms
+    float getCostTimer() {
+        outTime = std::chrono::high_resolution_clock::now();
+        return (float)(std::chrono::duration_cast<std::chrono::microseconds>(outTime - inTime).count());
+    }
+};
+
+int getCpuCounts() {
+    FILE* fp = fopen("/proc/cpuinfo", "rb");
+    if (fp == nullptr) {
+        MNN_PRINT("fopen error ! \n");
+        return 0;
+    }
+    int cpuCounts = 0;
+    char data[1024];
+    while (!feof(fp)) {
+        char* a = fgets(data, 1024, fp);
+
+        if (a == nullptr) {
+            break;
+        }
+        if (memcmp(data, "processor", 9) == 0) {
+            cpuCounts++;
+        }
+    }
+
+    fclose(fp);
+    fp = nullptr;
+    return cpuCounts;
+}
+
+// 0 max 1 min 2 cur
+void getFreqKhz(int cpuid, std::vector<int>& freqVector) {
+    char path[256];
+    int freqKhz = -1;
+    // max
+    sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/cpuinfo_max_freq", cpuid);
+    FILE* fp = fopen(path, "rb");
+    if (nullptr == fp) {
+        MNN_PRINT("cpuinfo_max_freq fopen error ! \n");
+        freqVector.emplace_back(0);
+    } else {
+        fscanf(fp, "%d", &freqKhz);
+        fclose(fp);
+        freqVector.push_back(freqKhz);
+    }
+
+    // min
+    sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/cpuinfo_min_freq", cpuid);
+    fp = fopen(path, "rb");
+    if (nullptr == fp) {
+        MNN_PRINT("cpuinfo_min_freq fopen error ! \n");
+        freqVector.emplace_back(0);
+    } else {
+        freqKhz = -1;
+        fscanf(fp, "%d", &freqKhz);
+        fclose(fp);
+        freqVector.push_back(freqKhz);
+    }
+
+    // cur
+    // sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/cpuinfo_cur_freq", cpuid);
+    // fp = fopen(path, "rb");
+    // if(nullptr == fp){
+    //     MNN_PRINT("cpuinfo_cur_freq fopen error ! \n");
+    // }else{
+    //     freqKhz = -1;
+    //     fscanf(fp, "%d", &freqKhz);
+    //     fclose(fp);
+    //     freqVector.push_back(freqKhz);
+    // }
+}
+
+void cpuFloatMlaTest(int32_t loopCounts) {
+#ifdef MNN_USE_NEON
+#ifndef __aarch64__
+
+    __asm__ __volatile__(
+        "mov r12, %0\n"
+        "0:                             \n"
+        "vmla.f32   q15, q15, d0[0]        \n"
+        "vmla.f32   q14, q14, d0[1]        \n"
+        "vmla.f32   q13, q13, d1[0]        \n"
+        "vmla.f32   q12, q12, d1[1]        \n"
+        "vmla.f32   q11, q11, d2[0]        \n"
+        "vmla.f32   q10, q10, d2[1]        \n"
+        "vmla.f32   q9, q9, d3[0]        \n"
+        "vmla.f32   q8, q8, d3[1]        \n"
+        "vmla.f32   q7, q7, d4[0]        \n"
+        "vmla.f32   q6, q6, d4[1]        \n"
+        "vmla.f32   q5, q5, d5[0]        \n"
+        "vmla.f32   q4, q4, d5[1]        \n"
+        "vmla.f32   q3, q3, d6[0]        \n"
+        "subs       r12, r12, #1          \n"
+        "bne        0b                  \n"
+        :
+        : "r"(loopCounts)
+        : "cc", "memory", "r12", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q14", "q15"
+    );
+
+#else
+    __asm__ __volatile__(
+        "mov w9, %w0\n"
+        "0:                           \n"
+        "fmla v31.4s, v31.4s, v0.s[0]\n"
+        "fmla v30.4s, v30.4s, v0.s[1]\n"
+        "fmla v29.4s, v29.4s, v0.s[2]\n"
+        "fmla v28.4s, v28.4s, v0.s[3]\n"
+        "fmla v27.4s, v27.4s, v1.s[0]\n"
+        "fmla v26.4s, v26.4s, v1.s[1]\n"
+        "fmla v25.4s, v25.4s, v1.s[2]\n"
+        "fmla v24.4s, v24.4s, v1.s[3]\n"
+        "fmla v23.4s, v23.4s, v3.s[0]\n"
+        "fmla v22.4s, v22.4s, v3.s[1]\n"
+        "fmla v21.4s, v21.4s, v3.s[2]\n"
+        "fmla v20.4s, v20.4s, v3.s[3]\n"
+        "fmla v19.4s, v19.4s, v4.s[0]\n"
+        "fmla v18.4s, v18.4s, v4.s[1]\n"
+        "fmla v17.4s, v17.4s, v4.s[2]\n"
+        "fmla v16.4s, v16.4s, v4.s[3]\n"
+        "fmla v15.4s, v15.4s, v5.s[0]\n"
+        "fmla v14.4s, v14.4s, v5.s[1]\n"
+        "fmla v13.4s, v13.4s, v5.s[2]\n"
+        "fmla v12.4s, v12.4s, v5.s[3]\n"
+        "fmla v11.4s, v11.4s, v6.s[0]\n"
+        "fmla v10.4s, v10.4s, v6.s[1]\n"
+        "fmla v9.4s, v9.4s, v6.s[2]\n"
+        "fmla v8.4s, v8.4s, v6.s[3]\n"
+        "fmla v7.4s, v7.4s, v2.s[0]\n"
+        "subs       w9, w9, #1          \n"
+        "bne        0b                  \n"
+        :
+        : "r"(loopCounts)
+        : "cc", "memory", "w9", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"
+    );
+#endif
+#endif
+}
+
+void cpuFLOPSPerformance() {
+    int32_t loopCounts = 100000000;
+    MNN_PRINT("CPU PERFORMANCE -> loopCounts : %d \n", loopCounts);
+
+    std::vector<int> freqVector;
+    for (int i = 0; i < getCpuCounts(); i++) {
+        freqVector.clear();
+        getFreqKhz(i, freqVector);
+        MNN_PRINT("core %d : max : %d, min : %d \n",i, freqVector.at(0), freqVector.at(1));
+    }
+
+    // warm up
+    cpuFloatMlaTest(loopCounts);
+
+    Timer timeInstance;
+    timeInstance.startTimer();
+    cpuFloatMlaTest(loopCounts);
+#ifdef MNN_USE_NEON
+#ifndef __aarch64__
+    auto number = (double)loopCounts * 13;
+#else 
+    auto number = (double)loopCounts * 25;
+#endif
+#else
+    auto number = 0.0;
+#endif
+    //FUNC_PRINT(number);
+    float costTime_ms = timeInstance.getCostTimer();
+    double costTime_s = (double)(costTime_ms) / 1000000.0f;
+    // MNN_PRINT("cost time : %f \n", costTime_s);
+    double mlaCounts_g = number * 4 / 1000000000.0f;
+    float gflops       = mlaCounts_g / costTime_s;
+    MNN_PRINT(" ======================== float ===============================\n");
+    MNN_PRINT("CPU float gflops : %f\n", gflops);
+}
+
+static void _testMemcpy() {
+    int size = 1024 * 1024;
+    int loop = 10000;
+    std::vector<std::thread> threads;
+    int threadNumber = 2;
+    std::vector<std::vector<int8_t>> tmp(threadNumber);
+    for (int i=0; i<threadNumber; ++i) {
+        tmp[i].resize(size);
+    }
+    MNN::Timer _t;
+    for (int i=0; i<threadNumber; ++i) {
+        threads.emplace_back(std::thread([size, loop, i, &tmp]() {
+            auto t0 = tmp[i].data();
+            for (int i=0; i<loop; ++i) {
+                ::memset(t0, 0, size);
+            }
+        }));
+    }
+    for (auto& t : threads) {
+        t.join();
+    }
+    float timeInS = (float)_t.durationInUs() / 1000.0f / 1000.0f;
+    float speed = (float)size * (float)threads.size() / 1024.0f / 1024.0f / 1024.0f * (float)loop / timeInS;
+    MNN_PRINT("Memcpy speed: %f GB / s\n", speed);
+
+}
+int main(int argc, const char* argv[]) {
+    MNN_PRINT("Start PERFORMANCE !!! \n");
+
+    cpuFLOPSPerformance();
+    _testMemcpy();
+    
+    return 0;
+}

tools/cpp/mergeInplaceForCPU.cpp

@@ -0,0 +1,144 @@
+#include <MNN_generated.h>
+#include <fstream>
+#include <sstream>
+#include <set>
+#include <map>
+#include <MNN/MNNDefine.h>
+using namespace MNN;
+static bool reIndexTensor(std::unique_ptr<MNN::NetT>& net) {
+    auto& mNet = net;
+    std::map<int, int> usefulTensorIndexMap;
+    std::vector<std::string> usefulTensorName;
+
+    std::vector<bool> tensorValid(mNet->tensorName.size(), false);
+    for (auto& op : mNet->oplists) {
+        for (auto index : op->inputIndexes) {
+            if (index < 0) {
+                continue; // optional input, ignore it
+            }
+            tensorValid[index] = true;
+        }
+        for (auto index : op->outputIndexes) {
+            tensorValid[index] = true;
+        }
+    }
+
+    for (int i = 0; i < tensorValid.size(); ++i) {
+        if (tensorValid[i]) {
+            usefulTensorIndexMap.insert(std::make_pair(i, usefulTensorName.size()));
+            usefulTensorName.push_back(mNet->tensorName[i]);
+        }
+    }
+
+    // Re index
+    for (auto& op : mNet->oplists) {
+        for (int i = 0; i < op->inputIndexes.size(); ++i) {
+            if (op->inputIndexes[i] < 0) {
+                continue;
+            }
+            auto iter = usefulTensorIndexMap.find(op->inputIndexes[i]);
+            op->inputIndexes[i] = iter->second;
+        }
+        for (int i = 0; i < op->outputIndexes.size(); ++i) {
+            auto iter = usefulTensorIndexMap.find(op->outputIndexes[i]);
+            op->outputIndexes[i] = iter->second;
+        }
+    }
+
+    mNet->tensorName = usefulTensorName;
+    for (auto iter = mNet->extraTensorDescribe.begin(); iter != mNet->extraTensorDescribe.end();) {
+        auto index = (*iter)->index;
+        if (usefulTensorIndexMap.find(index) == usefulTensorIndexMap.end()) {
+            iter = mNet->extraTensorDescribe.erase(iter);
+            continue;
+        }
+        (*iter)->index = usefulTensorIndexMap.find(index)->second;
+        iter++;
+    }
+    // Check dup name and modify
+    std::set<std::string> names;
+    std::set<std::string> tensorNames;
+    for (int i = 0; i < mNet->oplists.size(); ++i) {
+        auto& op    = mNet->oplists[i];
+        auto opName = op->name;
+        if (opName.empty() || names.find(opName) != names.end()) {
+            std::ostringstream defaultName;
+            defaultName << EnumNameOpType(op->type);
+            defaultName << i;
+            op->name = defaultName.str();
+            MNN_PRINT("%d op name is empty or dup, set to %s\n", i, op->name.c_str());
+            opName = op->name;
+        }
+        names.insert(opName);
+        for (auto output : op->outputIndexes) {
+            auto origin = net->tensorName[output];
+            if (origin.empty() || tensorNames.find(origin) != tensorNames.end()) {
+                std::ostringstream defaultName;
+                defaultName << output;
+                origin                  = defaultName.str();
+                net->tensorName[output] = origin;
+            }
+            tensorNames.insert(origin);
+        }
+    }
+    return true;
+}
+static void mergeInplaceForCPU(MNN::NetT* net) {
+    std::set<MNN::OpType> inplaceOps = {
+        OpType_UnaryOp,
+        OpType_ReLU,
+        OpType_ReLU6,
+        OpType_PReLU,
+        OpType_Scale,
+    };
+    std::vector<int> useCount(net->tensorName.size(), 0);
+    for (auto& op : net->oplists) {
+        for (auto index : op->inputIndexes) {
+            useCount[index]++;
+        }
+    }
+    std::map<int, int> replaceIndex;
+    for (int i=0; i<net->oplists.size(); ++i) {
+        auto op = net->oplists[i].get();
+        for (int j=0; j<op->inputIndexes.size(); ++j) {
+            if (replaceIndex.find(op->inputIndexes[j]) != replaceIndex.end()) {
+                op->inputIndexes[j] = replaceIndex[op->inputIndexes[j]];
+            }
+        }
+        if (inplaceOps.find(op->type) == inplaceOps.end()) {
+            continue;
+        }
+        if (useCount[op->inputIndexes[0]] > 1) {
+            continue;
+        }
+        replaceIndex.insert(std::make_pair(op->outputIndexes[0], op->inputIndexes[0]));
+        op->outputIndexes[0] = op->inputIndexes[0];
+    }
+}
+
+int main(int argc, const char* argv[]) {
+    if (argc < 3) {
+        MNN_ERROR("Usage: ./mergeInplaceForCPU SRC.mnn DST.mnn\n");
+        return 0;
+    }
+    std::unique_ptr<MNN::NetT> net;
+    {
+        std::ifstream inputIs(argv[1]);
+        std::ostringstream inputOs;
+        inputOs << inputIs.rdbuf();
+        auto content = inputOs.str();
+        net.reset(flatbuffers::GetRoot<MNN::Net>(content.c_str())->UnPack());
+    }
+    mergeInplaceForCPU(net.get());
+    reIndexTensor(net);
+    flatbuffers::FlatBufferBuilder builderOutput(1024);
+    builderOutput.ForceDefaults(true);
+    auto len = MNN::Net::Pack(builderOutput, net.get());
+    builderOutput.Finish(len);
+    auto sizeOutput    = builderOutput.GetSize();
+    auto bufferOutput = builderOutput.GetBufferPointer();
+    std::ofstream outputOs(argv[2]);
+    outputOs.write((const char*)bufferOutput, sizeOutput);
+
+    return 0;
+}

tools/cpp/mobilenetTest.cpp

@@ -0,0 +1,143 @@
+//
+//  mobilenetV1Test.cpp
+//  MNN
+//
+//  Created by MNN on 2018/05/14.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#include <stdio.h>
+#include <MNN/ImageProcess.hpp>
+#include <MNN/Interpreter.hpp>
+#define MNN_OPEN_TIME_TRACE
+#include <algorithm>
+#include <fstream>
+#include <functional>
+#include <memory>
+#include <sstream>
+#include <vector>
+#include <MNN/AutoTime.hpp>
+
+#define STB_IMAGE_IMPLEMENTATION
+#include "stb_image.h"
+#include "stb_image_write.h"
+
+using namespace MNN;
+using namespace MNN::CV;
+
+int main(int argc, const char* argv[]) {
+
+    if (argc < 3) {
+        MNN_PRINT("Usage: ./mobilenetTest.out model.mnn input.jpg [forwardType] [precision] [word.txt]\n");
+        return 0;
+    }
+    std::shared_ptr<Interpreter> net(Interpreter::createFromFile(argv[1]));
+    ScheduleConfig config;
+    config.type  = MNN_FORWARD_CPU;
+    config.numThread = 4;
+    if (argc > 3) {
+        config.type = (MNNForwardType)::atoi(argv[3]);
+    }
+
+    MNN::BackendConfig backendConfig;
+    backendConfig.precision = MNN::BackendConfig::Precision_High;
+    if (argc > 4) {
+        backendConfig.precision = (MNN::BackendConfig::PrecisionMode)::atoi(argv[4]);
+    }
+    config.backendConfig = &backendConfig;
+    MNN_PRINT("model:%s, input image:%s, forwardType:%d, precision:%d\n", argv[1], argv[2], config.type, (int)backendConfig.precision);
+    Session* session = net->createSession(config);
+
+    Tensor* inputTensor  = net->getSessionInput(session, NULL);
+    Tensor* outputTensor = net->getSessionOutput(session, NULL);
+
+    Tensor inputTensorUser(inputTensor, Tensor::DimensionType::TENSORFLOW);
+    Tensor outputTensorUser(outputTensor, outputTensor->getDimensionType());
+
+    //image preproccess
+    {
+        int netInputHeight = inputTensorUser.height();
+        int netInputWidth  = inputTensorUser.width();
+
+        int imageChannel, imageWidth, imageHeight;
+        unsigned char* inputImage = stbi_load(argv[2], &imageWidth,
+                                              &imageHeight, &imageChannel, 4);
+
+        Matrix trans;
+        trans.setScale(1.0 / imageWidth, 1.0 / imageHeight);
+        trans.postRotate(0, 0.5f, 0.5f);
+        trans.postScale(netInputWidth, netInputHeight);
+        trans.invert(&trans);
+
+        ImageProcess::Config config;
+        config.filterType = BILINEAR;
+        float mean[3]     = {103.94f, 116.78f, 123.68f};
+        float normals[3]  = {0.017f, 0.017f, 0.017f};
+        ::memcpy(config.mean, mean, sizeof(mean));
+        ::memcpy(config.normal, normals, sizeof(normals));
+        config.sourceFormat = RGBA;
+        config.destFormat = RGB;
+
+        std::shared_ptr<ImageProcess> pretreat(ImageProcess::create(config));
+        pretreat->setMatrix(trans);
+        pretreat->convert(inputImage, imageWidth, imageHeight, 0, &inputTensorUser);
+
+        stbi_image_free(inputImage);
+    }
+
+    //run
+    {
+        AUTOTIME;
+        inputTensor->copyFromHostTensor(&inputTensorUser);
+        net->runSession(session);
+        outputTensor->copyToHostTensor(&outputTensorUser);
+    }
+
+
+    //get predict labels
+    {
+
+        std::vector<std::string> words;
+        if (argc > 5) {
+            std::ifstream inputOs(argv[5]);
+            std::string line;
+            while (std::getline(inputOs, line)) {
+                words.emplace_back(line);
+            }
+        }
+
+        MNN_PRINT("output size:%d\n", outputTensorUser.elementSize());
+        auto type = outputTensorUser.getType();
+
+        auto size = outputTensorUser.elementSize();
+        std::vector<std::pair<int, float>> tempValues(size);
+        if (type.code == halide_type_float) {
+            auto values = outputTensorUser.host<float>();
+            for (int i = 0; i < size; ++i) {
+                tempValues[i] = std::make_pair(i, values[i]);
+            }
+        }
+        if (type.code == halide_type_uint && type.bytes() == 1) {
+            auto values = outputTensorUser.host<uint8_t>();
+            for (int i = 0; i < size; ++i) {
+                tempValues[i] = std::make_pair(i, values[i]);
+            }
+        }
+        // Find Max
+        std::sort(tempValues.begin(), tempValues.end(),
+                  [](std::pair<int, float> a, std::pair<int, float> b) { return a.second > b.second; });
+
+        int length = size > 10 ? 10 : size;
+        if (words.empty()) {
+            for (int i = 0; i < length; ++i) {
+                MNN_PRINT("%d, %f\n", tempValues[i].first, tempValues[i].second);
+            }
+        } else {
+            for (int i = 0; i < length; ++i) {
+                MNN_PRINT("%s: %f\n", words[tempValues[i].first].c_str(), tempValues[i].second);
+            }
+        }
+    }
+
+    return 0;
+}

tools/cpp/modelCompare.cpp

@@ -0,0 +1,266 @@
+//
+//  modelCompare.cpp
+//  MNN
+//
+//  Created by MNN on 2019/01/22.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#define MNN_OPEN_TIME_TRACE
+
+#include <math.h>
+#include <stdlib.h>
+#include <algorithm>
+#include <cstring>
+#include <fstream>
+#include <iostream>
+#include <map>
+#include <sstream>
+#include <string>
+#include <MNN/AutoTime.hpp>
+#include <MNN/Interpreter.hpp>
+#include <MNN/Tensor.hpp>
+#include "core/TensorUtils.hpp"
+#include "rapidjson/document.h"
+
+template<typename T>
+inline T stringConvert(const char* number) {
+    std::istringstream os(number);
+    T v;
+    os >> v;
+    return v;
+}
+
+using namespace MNN;
+
+static void compareNet(Interpreter* net, Interpreter* net2, MNNForwardType expectType, float tolerance,
+                              const std::map<std::string, std::shared_ptr<Tensor>>& inputs, const std::string& stopOp, BackendConfig::PrecisionMode precision, int modeNum) {
+    std::vector<std::shared_ptr<MNN::Tensor>> correctResult;
+    int index;
+    MNN::ScheduleConfig expectConfig;
+    BackendConfig backendConfig;
+    backendConfig.precision = precision;
+    expectConfig.type   = expectType;
+    expectConfig.backendConfig = &backendConfig;
+    expectConfig.mode = modeNum;
+    auto expectSession  = net->createSession(expectConfig);
+    auto compareSession = net2->createSession(expectConfig);
+
+    bool allCorrect = true;
+
+    MNN::TensorCallBackWithInfo beginCallBack = [&](const std::vector<MNN::Tensor*>& t, const OperatorInfo* op) {
+        if (op->name() == stopOp) {
+            return false;
+        }
+        return true;
+    };
+    MNN::TensorCallBackWithInfo saveExpect = [&](const std::vector<MNN::Tensor*>& t, const OperatorInfo* op) {
+        if (op->name() == stopOp) {
+            return false;
+        }
+        if (op->type() == "Raster") {
+            return true;
+        }
+        for (int i=0; i<t.size(); ++i) {
+            auto tensor = t[i];
+            if (tensor->elementSize() <= 0) {
+                return true;
+            }
+            if (tensor->buffer().device == 0 && tensor->buffer().host == nullptr) {
+                return true;
+            }
+            std::shared_ptr<MNN::Tensor> copyTensor(MNN::Tensor::createHostTensorFromDevice(tensor, true));
+            correctResult.emplace_back(copyTensor);
+        }
+        return true;
+    };
+    MNN::TensorCallBackWithInfo compareExpect = [&](const std::vector<MNN::Tensor*>& t, const OperatorInfo* op) {
+        if (op->name() == stopOp) {
+            return false;
+        }
+        if (op->type() == "Raster") {
+            return true;
+        }
+        for (int i=0; i<t.size(); ++i) {
+            auto tensor = t[i];
+            if (tensor->elementSize() <= 0) {
+                return true;
+            }
+            if (tensor->buffer().device == 0 && tensor->buffer().host == nullptr) {
+                return true;
+            }
+            std::shared_ptr<MNN::Tensor> copyTensor(MNN::Tensor::createHostTensorFromDevice(tensor, true));
+            auto expectTensor = correctResult[index++];
+            auto correct      = TensorUtils::compareTensors(copyTensor.get(), expectTensor.get(), tolerance, true);
+            if (!correct) {
+                MNN_PRINT("%s - %d is error\n", op->name().c_str(), i);
+                allCorrect = false;
+            }
+        }
+        return allCorrect;
+    };
+
+    for (auto& iter : inputs) {
+        Tensor* expectInput = net->getSessionInput(expectSession, iter.first.empty() ? NULL : iter.first.c_str());
+        expectInput->copyFromHostTensor(iter.second.get());
+        Tensor* compareInput = net->getSessionInput(compareSession, iter.first.empty() ? NULL : iter.first.c_str());
+        compareInput->copyFromHostTensor(iter.second.get());
+    }
+    correctResult.clear();
+    net->runSessionWithCallBackInfo(expectSession, beginCallBack, saveExpect);
+    index = 0;
+    net2->runSessionWithCallBackInfo(compareSession, beginCallBack, compareExpect);
+    if (allCorrect) {
+        MNN_PRINT("Correct ! Run second pass\n");
+    } else {
+        return;
+    }
+    index = 0;
+    for (auto& iter : inputs) {
+        Tensor* compareInput = net->getSessionInput(compareSession, iter.first.empty() ? NULL : iter.first.c_str());
+        compareInput->copyFromHostTensor(iter.second.get());
+    }
+    net2->runSessionWithCallBackInfo(compareSession, beginCallBack, compareExpect);
+    if (allCorrect) {
+        MNN_PRINT("Correct !\n");
+    }
+}
+
+int main(int argc, const char* argv[]) {
+    if (argc < 3) {
+        MNN_PRINT("Usage: ./modelCompare.out origin.mnn origin_quant.mnn [0.05]");
+    }
+    // read args
+    std::string cmd = argv[0];
+    std::string pwd = "./";
+    auto rslash     = cmd.rfind("/");
+    if (rslash != std::string::npos) {
+        pwd = cmd.substr(0, rslash + 1);
+    }
+
+    const char* fileName = argv[1];
+
+    const char* compareFileName = argv[2];
+
+    float tolerance = 0.05f;
+    if (argc > 3) {
+        tolerance = stringConvert<float>(argv[3]);
+    }
+    MNN_PRINT("Tolerance Rate: %f\n", tolerance);
+
+    // create net
+    MNN_PRINT("Open Model %s, %s\n", fileName, compareFileName);
+    std::shared_ptr<MNN::Interpreter> net =
+        std::shared_ptr<MNN::Interpreter>(MNN::Interpreter::createFromFile(fileName));
+    net->setSessionMode(Interpreter::Session_Debug);
+    std::shared_ptr<MNN::Interpreter> net2 =
+        std::shared_ptr<MNN::Interpreter>(MNN::Interpreter::createFromFile(compareFileName));
+    net2->setSessionMode(Interpreter::Session_Debug);
+
+    // create session for get input info
+    ScheduleConfig config;
+    config.type  = MNN_FORWARD_CPU;
+    auto session = net->createSession(config);
+
+    std::map<std::string, std::shared_ptr<MNN::Tensor>> inputs;
+    std::vector<std::string> inputNames;
+    do {
+        rapidjson::Document document;
+        std::ostringstream jsonNameOs;
+        jsonNameOs << pwd << "/input.json";
+        std::ifstream fileNames(jsonNameOs.str().c_str());
+        if (fileNames.fail()) {
+            break;
+        }
+        std::ostringstream output;
+        output << fileNames.rdbuf();
+        auto outputStr = output.str();
+        document.Parse(outputStr.c_str());
+        if (document.HasParseError()) {
+            MNN_ERROR("Invalid json\n");
+            break;
+        }
+        if (document.HasMember("inputs")) {
+            auto inputsInfo = document["inputs"].GetArray();
+            for (auto iter = inputsInfo.begin(); iter !=inputsInfo.end(); iter++) {
+                auto obj = iter->GetObject();
+                std::string name = obj["name"].GetString();
+                inputNames.emplace_back(name);
+            }
+        }
+    } while (false);
+    if (!inputNames.empty()) {
+        MNN_PRINT("Find input.json, use inputs:");
+        for (auto& n : inputNames) {
+            MNN_PRINT(" %s, ", n.c_str());
+        }
+        MNN_PRINT("\n");
+        for (auto name : inputNames) {
+            auto inputTensor = net->getSessionInput(session, name.c_str());
+            std::shared_ptr<MNN::Tensor> givenTensor(new Tensor(inputTensor, inputTensor->getDimensionType()));
+            {
+                std::ostringstream fileName;
+                fileName << pwd << name << ".txt";
+                std::ifstream input(fileName.str().c_str());
+                MNN_ASSERT(!input.fail());
+
+                int size_w = inputTensor->width();
+                int size_h = inputTensor->height();
+                int bpp    = inputTensor->channel();
+                int batch  = inputTensor->batch();
+                // auto backend = net->getBackend(session, inputTensor);
+                // MNN_ASSERT(!input.fail());
+                MNN_PRINT("Input: %d,%d,%d,%d\n", size_w, size_h, bpp, batch);
+                auto inputData = givenTensor->host<float>();
+                auto size      = givenTensor->size() / sizeof(float);
+                for (int i = 0; i < size; ++i) {
+                    input >> inputData[i];
+                }
+                inputs.insert(std::make_pair(name, givenTensor));
+            }
+
+        }
+    } else {
+        auto inputTensor = net->getSessionInput(session, NULL);
+        std::shared_ptr<MNN::Tensor> givenTensor(new Tensor(inputTensor, inputTensor->getDimensionType()));
+        {
+            std::ostringstream fileName;
+            fileName << pwd << "input_0"
+                     << ".txt";
+            std::ifstream input(fileName.str().c_str());
+
+            int size_w = inputTensor->width();
+            int size_h = inputTensor->height();
+            int bpp    = inputTensor->channel();
+            int batch  = inputTensor->batch();
+            // auto backend = net->getBackend(session, inputTensor);
+            // MNN_ASSERT(!input.fail());
+            MNN_PRINT("Input: %d,%d,%d,%d\n", size_w, size_h, bpp, batch);
+            auto inputData = givenTensor->host<float>();
+            auto size      = givenTensor->size() / sizeof(float);
+            for (int i = 0; i < size; ++i) {
+                input >> inputData[i];
+            }
+            inputs.insert(std::make_pair("", givenTensor));
+        }
+    }
+    net->releaseSession(session);
+    BackendConfig::PrecisionMode precision = BackendConfig::Precision_Normal;
+    if (argc > 4) {
+        precision = (BackendConfig::PrecisionMode)atoi(argv[4]);
+    }
+    FUNC_PRINT(precision);
+    int modeNum = 1;
+    if(argc > 5) {
+        modeNum = atoi(argv[5]);//set gpu mode
+    }
+    FUNC_PRINT(modeNum);
+    std::string stopOp = "";
+    if (argc > 6) {
+        stopOp = argv[6];
+    }
+    FUNC_PRINT_ALL(stopOp.c_str(), s);
+    compareNet(net.get(), net2.get(), MNN_FORWARD_CPU, tolerance, inputs, stopOp, precision, modeNum);
+
+    return 0;
+}

tools/cpp/resfusetest/cumsum.comp

@@ -0,0 +1,53 @@
+#version 440 core
+layout(std430) buffer;
+
+layout(set=0, binding=0) writeonly buffer pointoffsetSum{
+    highp uint data[];
+} uPointoffsetSum;
+
+
+layout(set=0, binding=1) readonly buffer pointoffset{
+    highp uint data[];
+} uPointoffset;
+
+layout(set=0, binding=2) uniform constBuffer {
+    ivec4 point; // point size
+} uConstant;
+
+shared uint local_sum[256];
+
+#define UNIT 3
+#define LOCAL_SIZE 8
+layout (local_size_x = LOCAL_SIZE, local_size_y = 1, local_size_z = 1) in;
+void main()
+{
+	int tId = int(gl_LocalInvocationID.x);
+    int size = uConstant.point.x;
+    int curOffset = 0;
+    uint threadBuffer[UNIT];
+    while (curOffset < size) {
+        int sta = tId * UNIT + curOffset;
+        int fin = min(sta + UNIT, size);
+        for (int i=sta; i<fin; ++i) {
+            int lpos = i - sta;
+            threadBuffer[lpos] = uPointoffset.data[i];
+        }
+        for (int i=sta+1; i<fin; ++i) {
+            int lpos = i - sta;
+            threadBuffer[lpos] = threadBuffer[lpos] + threadBuffer[lpos-1];
+        }
+        local_sum[tId] = threadBuffer[fin-sta-1];
+        barrier();
+        if (fin > sta) {
+            uint sum = 0;
+            for (int i=0; i<tId; ++i) {
+                sum += local_sum[i];
+            }
+            for (int i=sta; i<fin; ++i) {
+                int lpos = i - sta;
+                uPointoffsetSum.data[i] = threadBuffer[lpos] + sum;
+            }
+        }
+        curOffset += LOCAL_SIZE * UNIT;
+    }
+}

tools/cpp/resfusetest/point.txt

@@ -0,0 +1,20 @@
+1
+5
+70
+8
+3
+4
+9
+34
+54
+12
+45
+66
+34
+67
+34
+56
+41
+11
+21
+23

tools/cpp/resfusetest/vulkanTest.json

@@ -0,0 +1,35 @@
+{
+    "inputs":[
+        {
+            "dims":[
+                20
+            ],
+            "binding":1,
+            "type":"uint32",
+            "filename":"point.txt"
+        }
+    ],
+    "outputs":[
+        {
+            "dims":[
+                20
+            ],
+            "type":"uint32",
+            "binding":0
+        }
+    ],
+    "uniforms":[
+        {
+            "dims":[4],
+            "binding":2,
+            "type":"int32",
+            "data":[20, 600, 20, 20]
+        }
+    ],
+    "group_size":[
+        1,1,1
+    ],
+    "local_size":[
+        256, 1, 1
+    ]
+}

tools/cpp/revertMNNModel.cpp

@@ -0,0 +1,245 @@
+//
+//  revertMNNModel.cpp
+//  MNN
+//
+//  Created by MNN on 2019/01/31.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#include <cstdlib>
+#include <random>
+#include <ctime>
+#include <fstream>
+#include <iostream>
+
+#include <string.h>
+#include <stdlib.h>
+#include <MNN/MNNDefine.h>
+#include "revertMNNModel.hpp"
+#include "core/CommonCompute.hpp"
+#include "core/MemoryFormater.h"
+#include "core/IDSTEncoder.hpp"
+#include "core/ConvolutionCommon.hpp"
+
+int SymmetricQuantizeWeight(const float* weight, const int size, int8_t* quantizedWeight, float* scale,
+                            const int channels, float weightClampValue) {
+    const int channelStride     = size / channels;
+    const int quantizedMaxValue = weightClampValue;
+
+    for (int c = 0; c < channels; ++c) {
+        const auto weightChannelStart    = weight + c * channelStride;
+        auto quantizedWeightChannelStart = quantizedWeight + c * channelStride;
+        auto minmaxValue                 = std::minmax_element(weightChannelStart, weightChannelStart + channelStride);
+        const float dataAbsMax           = std::fmax(std::fabs(*minmaxValue.first), std::fabs(*minmaxValue.second));
+
+        float scaleDataToInt8 = 1.0f;
+        if (dataAbsMax == 0) {
+            scale[c] = 0.0f;
+        } else {
+            scale[c]        = dataAbsMax / quantizedMaxValue;
+            scaleDataToInt8 = quantizedMaxValue / dataAbsMax;
+        }
+
+        for (int i = 0; i < channelStride; ++i) {
+            const int32_t quantizedInt8Value = static_cast<int32_t>(roundf(weightChannelStart[i] * scaleDataToInt8));
+            quantizedWeightChannelStart[i] =
+                std::min(quantizedMaxValue, std::max(-quantizedMaxValue, quantizedInt8Value));
+        }
+    }
+
+    return 0;
+}
+
+
+Revert::Revert(const char* originalModelFileName) {
+    std::ifstream inputFile(originalModelFileName, std::ios::binary);
+    inputFile.seekg(0, std::ios::end);
+    const auto size = inputFile.tellg();
+    inputFile.seekg(0, std::ios::beg);
+
+    char* buffer = new char[size];
+    inputFile.read(buffer, size);
+    inputFile.close();
+    mMNNNet = MNN::UnPackNet(buffer);
+    delete[] buffer;
+    MNN_ASSERT(mMNNNet->oplists.size() > 0);
+}
+
+Revert::~Revert() {
+}
+
+void* Revert::getBuffer() const {
+    return reinterpret_cast<void*>(mBuffer.get());
+}
+
+const size_t Revert::getBufferSize() const {
+    return mBufferSize;
+}
+
+void Revert::writeExtraDescribeTensor(float* scale, float* offset) {
+    int opCounts = static_cast<int32_t>(mMNNNet->oplists.size());
+    for (int opIndex = 0; opIndex < opCounts; ++opIndex) {
+        std::unique_ptr<MNN::TensorDescribeT> describe(new MNN::TensorDescribeT);
+        describe->index = opIndex;
+        describe->quantInfo.reset(new MNN::TensorQuantInfoT);
+        describe->quantInfo->scale = *scale;
+        describe->quantInfo->zero = *offset;
+        describe->quantInfo->min = -127;
+        describe->quantInfo->max = 127;
+        describe->quantInfo->type = MNN::DataType_DT_INT8;
+        mMNNNet->extraTensorDescribe.emplace_back(std::move(describe));
+    }
+    for (const auto& op: mMNNNet->oplists) {
+        const auto opType = op->type;
+        if (opType != MNN::OpType_Convolution && opType != MNN::OpType_ConvolutionDepthwise && opType != MNN::OpType_Deconvolution) {
+            continue;
+        }
+        // Conv/ConvDepthwise/Deconv weight quant.
+        const float inputScale = *scale;
+        const float outputScale = *scale;
+        const int outputChannel = static_cast<int32_t>(op->outputIndexes.size());
+
+        auto param = op->main.AsConvolution2D();
+        float* originWeight = param->weight.data();
+        const int channels = param->common->outputCount;
+        param->symmetricQuan.reset(new MNN::QuantizedFloatParamT);
+        param->symmetricQuan->nbits = 8;
+        const int weightSize = static_cast<int32_t>(param->weight.size());
+        param->common->inputCount = weightSize / (channels * param->common->kernelX * param->common->kernelY);
+        std::vector<int8_t> quantizedWeight(weightSize);
+        std::vector<float> quantizedWeightScale(channels);
+
+        if (originWeight[0] == 0.f && originWeight[1] == 0.f) { // Process weight is null.
+            // Initialize originWeight
+            std::uniform_real_distribution<double> u(-200, 200);
+            std::default_random_engine e(time(NULL));
+            for (int i = 0; i < weightSize; ++i) {
+                originWeight[i] = u(e);
+            }
+        }
+
+        SymmetricQuantizeWeight(originWeight, weightSize, quantizedWeight.data(), quantizedWeightScale.data(), channels, 127.0f);
+        param->quanParameter = IDSTEncoder::encode(param->weight.data(), quantizedWeightScale, weightSize/channels, channels, false, quantizedWeight.data(), -127.0f);
+        param->quanParameter->scaleIn = *scale;
+        param->quanParameter->scaleOut = *scale;
+        if (param->common->relu6) {
+            param->common->relu  = true;
+            param->common->relu6 = false;
+        }
+        param->weight.clear();
+    }
+}
+
+void Revert::packMNNNet() {
+    flatbuffers::FlatBufferBuilder builder(1024);
+    auto offset = MNN::Net::Pack(builder, mMNNNet.get());
+    builder.Finish(offset);
+    mBufferSize = builder.GetSize();
+    mBuffer.reset(new uint8_t[mBufferSize], std::default_delete<uint8_t[]>());
+    ::memcpy(mBuffer.get(), builder.GetBufferPointer(), mBufferSize);
+    mMNNNet.reset();
+}
+
+void Revert::initialize(float spasity, int sparseBlockOC, bool rewrite, bool quantizedModel) {
+    if (mMNNNet->bizCode == "benchmark" || rewrite) {
+        randStart();
+        bool useSparse = spasity > 0.5f;
+        for (auto& op : mMNNNet->oplists) {
+            const auto opType = op->type;
+            switch (opType) {
+                case MNN::OpType_Convolution:
+                case MNN::OpType_Deconvolution:
+                case MNN::OpType_ConvolutionDepthwise: {
+                    auto param           = op->main.AsConvolution2D();
+                    auto& convCommon     = param->common;
+                    const int weightReduceStride = convCommon->kernelX * convCommon->kernelY * convCommon->inputCount;
+                    const int oc = convCommon->outputCount / convCommon->group;
+                    param->weight.resize(oc * weightReduceStride);
+                    ::memset(param->weight.data(), 0, param->weight.size() * sizeof(float));
+                    param->bias.resize(convCommon->outputCount);
+                    ::memset(param->bias.data(), 0, param->bias.size() * sizeof(float));
+                    if (useSparse) {
+                        size_t weightNNZElement, weightBlockNumber = 0;
+                        MNN::CommonCompute::fillRandValueAsSparsity(weightNNZElement, weightBlockNumber, param->weight.data(), oc, weightReduceStride, spasity, sparseBlockOC);
+
+                        MNN::AttributeT* arg1(new MNN::AttributeT);
+                        arg1->key = "sparseBlockOC";
+                        arg1->i = sparseBlockOC;
+
+                        MNN::AttributeT* arg2(new MNN::AttributeT);
+                        arg2->key = "sparseBlockKernel";
+                        arg2->i = 1;
+
+                        MNN::AttributeT* arg3(new MNN::AttributeT);
+                        arg3->key = "NNZElement";
+                        arg3->i = weightNNZElement;
+
+                        MNN::AttributeT* arg4(new MNN::AttributeT);
+                        arg4->key = "blockNumber";
+                        arg4->i = weightBlockNumber;
+
+                        flatbuffers::FlatBufferBuilder builder;
+                        std::vector<flatbuffers::Offset<MNN::Attribute>> argsVector;
+                        auto sparseArg1 = MNN::CreateAttribute(builder, arg1);
+                        auto sparseArg2 = MNN::CreateAttribute(builder, arg2);
+                        auto sparseArg3 = MNN::CreateAttribute(builder, arg3);
+                        auto sparseArg4 = MNN::CreateAttribute(builder, arg4);
+
+                        argsVector.emplace_back(sparseArg1);
+                        argsVector.emplace_back(sparseArg2);
+                        argsVector.emplace_back(sparseArg3);
+                        argsVector.emplace_back(sparseArg4);
+
+                        auto sparseArgs = builder.CreateVectorOfSortedTables<MNN::Attribute>(&argsVector);
+                        MNN::SparseAlgo prune_algo_type;
+                        if (sparseBlockOC == 4) {
+                            prune_algo_type = MNN::SparseAlgo_SIMD_OC;
+                        } else {
+                            prune_algo_type = MNN::SparseAlgo_RANDOM;
+                        }
+                        auto sparseCom = MNN::CreateSparseCommon(builder, prune_algo_type, sparseArgs);
+                        builder.Finish(sparseCom);
+                        auto sparseComPtr = flatbuffers::GetRoot<MNN::SparseCommon>(builder.GetBufferPointer())->UnPack();
+                        param->sparseParameter.reset(sparseComPtr);
+                        MNN::CommonCompute::compressFloatWeightToSparse(op.get());
+                    }
+                    break;
+                }
+                case MNN::OpType_Scale: {
+                    auto param = op->main.AsScale();
+                    param->biasData.resize(param->channels);
+                    param->scaleData.resize(param->channels);
+                    fillRandValue(param->scaleData.data(), param->channels);
+                    fillRandValue(param->biasData.data(), param->channels);
+                    break;
+                }
+                default:
+                    break;
+            }
+        }
+    }
+    if (quantizedModel) {
+        int opsize =  mMNNNet->oplists.size();
+        std::vector<float> scale(opsize);
+        for (int i = 0;i < opsize; ++i) {
+            scale[i] = ((i + 1) / (opsize + 100.0f));
+        }
+        float offset = 0;
+        writeExtraDescribeTensor(scale.data(), &offset);
+    }
+    packMNNNet();
+}
+
+void Revert::fillRandValue(float * data, size_t size) {
+    unsigned int seed = 1000;
+    std::mt19937 rng(seed);
+    std::uniform_real_distribution<float> uniform_dist(-2, 2);
+
+    for (size_t i = 0; i < size; i++) {
+        *data = uniform_dist(rng);
+    }
+    return;
+}
+
+void Revert::randStart() {
+}

tools/cpp/revertMNNModel.hpp

@@ -0,0 +1,32 @@
+//
+//  revertMNNModel.hpp
+//  MNN
+//
+//  Created by MNN on 2019/01/31.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#ifndef REVERTMNNMODEL_HPP
+#define REVERTMNNMODEL_HPP
+
+#include "MNN_generated.h"
+
+class Revert {
+public:
+    Revert(const char* originalModelFileName);
+    ~Revert();
+    void* getBuffer() const;
+    const size_t getBufferSize() const;
+    void initialize(float sparsity = 0.0f, int sparseBlockOC = 1, bool rewrite = false, bool quantizedModel = false);
+    static void fillRandValue(float * data, size_t size);
+    void writeExtraDescribeTensor(float* scales, float* offsets);
+private:
+    Revert();
+    std::unique_ptr<MNN::NetT> mMNNNet;
+    size_t mBufferSize;
+    std::shared_ptr<uint8_t> mBuffer;
+    void randStart();
+    void packMNNNet();
+};
+
+#endif // REVERTMNNMODEL_HPP

tools/cpp/testModel.cpp

@@ -0,0 +1,230 @@
+//
+//  testModel.cpp
+//  MNN
+//
+//  Created by MNN on 2019/01/22.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#define MNN_OPEN_TIME_TRACE
+
+#include <MNN/MNNDefine.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <MNN/AutoTime.hpp>
+#include <MNN/Interpreter.hpp>
+#include <MNN/Tensor.hpp>
+#include <fstream>
+#include <map>
+#include <sstream>
+#include "core/Backend.hpp"
+#include "core/Macro.h"
+#include "core/TensorUtils.hpp"
+#define MNN_USER_SET_DEVICE
+#include "MNN/MNNSharedContext.h"
+
+#define NONE "\e[0m"
+#define RED "\e[0;31m"
+#define GREEN "\e[0;32m"
+#define L_GREEN "\e[1;32m"
+#define BLUE "\e[0;34m"
+#define L_BLUE "\e[1;34m"
+#define BOLD "\e[1m"
+
+template<typename T>
+inline T stringConvert(const char* number) {
+    std::istringstream os(number);
+    T v;
+    os >> v;
+    return v;
+}
+
+MNN::Tensor* createTensor(const MNN::Tensor* shape, const char* path) {
+    std::ifstream stream(path);
+    if (stream.fail()) {
+        return NULL;
+    }
+
+    auto result = new MNN::Tensor(shape, shape->getDimensionType());
+    auto data   = result->host<float>();
+    for (int i = 0; i < result->elementSize(); ++i) {
+        double temp = 0.0f;
+        stream >> temp;
+        data[i] = temp;
+    }
+    stream.close();
+    return result;
+}
+
+int main(int argc, const char* argv[]) {
+
+    // check given & expect
+    const char* modelPath  = argv[1];
+    const char* givenName  = argv[2];
+    const char* expectName = argv[3];
+    MNN_PRINT("Testing model %s, input: %s, output: %s\n", modelPath, givenName, expectName);
+
+    // create net
+    auto type = MNN_FORWARD_CPU;
+    if (argc > 4) {
+        type = (MNNForwardType)stringConvert<int>(argv[4]);
+    }
+    auto tolerance = 0.1f;
+    if (argc > 5) {
+        tolerance = stringConvert<float>(argv[5]);
+    }
+    MNN::BackendConfig::PrecisionMode precision = MNN::BackendConfig::Precision_High;
+    if (argc > 6) {
+        precision = (MNN::BackendConfig::PrecisionMode)stringConvert<int>(argv[6]);
+    }
+    
+    std::shared_ptr<MNN::Interpreter> net =
+    std::shared_ptr<MNN::Interpreter>(MNN::Interpreter::createFromFile(modelPath), [](void* net) {
+        MNN::Interpreter::destroy((MNN::Interpreter*)net);
+    });
+    net->setSessionHint(MNN::Interpreter::INIT_THREAD_NUMBER, 8);
+
+    // create session
+    MNN::ScheduleConfig config;
+    config.type = type;
+    MNN::BackendConfig backendConfig;
+    backendConfig.precision = precision;
+    
+    MNNDeviceContext gpuDeviceConfig;
+    // CUDA Backend support user set device_id
+    if(type == MNN_FORWARD_CUDA) {
+        gpuDeviceConfig.deviceId = 0;
+        backendConfig.sharedContext = &gpuDeviceConfig;
+    }
+    // OpenCL Backend support user set platform_size, platform_id, device_id
+    if(type == MNN_FORWARD_OPENCL) {
+        gpuDeviceConfig.platformSize = 1;// GPU Cards number
+        gpuDeviceConfig.platformId = 0;  // Execute on Which GPU Card
+        gpuDeviceConfig.deviceId = 0;    // Execute on Which GPU device
+        backendConfig.sharedContext = &gpuDeviceConfig;
+    }
+    config.backendConfig = &backendConfig;
+    auto session         = net->createSession(config);
+    
+    // input dims
+    std::vector<int> inputDims;
+    if (argc > 7) {
+        std::string inputShape(argv[7]);
+        const char* delim = "x";
+        std::ptrdiff_t p1 = 0, p2;
+        while (1) {
+            p2 = inputShape.find(delim, p1);
+            if (p2 != std::string::npos) {
+                inputDims.push_back(atoi(inputShape.substr(p1, p2 - p1).c_str()));
+                p1 = p2 + 1;
+            } else {
+                inputDims.push_back(atoi(inputShape.substr(p1).c_str()));
+                break;
+            }
+        }
+    }
+    for (auto dim : inputDims) {
+        MNN_PRINT("%d ", dim);
+    }
+    MNN_PRINT("\n");
+    
+    
+    auto allInput = net->getSessionInputAll(session);
+    for (auto& iter : allInput) {
+        auto inputTensor = iter.second;
+        
+        if (!inputDims.empty()) {
+            MNN_PRINT("===========> Resize Tensor...\n");
+            net->resizeTensor(inputTensor, inputDims);
+            net->resizeSession(session);
+        }
+        
+        auto size = inputTensor->size();
+        if (size <= 0) {
+            continue;
+        }
+        
+        void* host = inputTensor->map(MNN::Tensor::MAP_TENSOR_WRITE,  inputTensor->getDimensionType());
+        if(host != nullptr) {
+            // TODO: Find better way to memset zero
+            ::memset(host, 0, MNN::TensorUtils::getRawSize(inputTensor) * inputTensor->getType().bytes());
+        }
+        inputTensor->unmap(MNN::Tensor::MAP_TENSOR_WRITE,  inputTensor->getDimensionType(), host);
+    }
+
+    // write input tensor
+    auto inputTensor = net->getSessionInput(session, NULL);
+    std::shared_ptr<MNN::Tensor> givenTensor(createTensor(inputTensor, givenName), [](void* t) {
+        MNN::Tensor::destroy((MNN::Tensor*)t);
+    });
+    if (!givenTensor) {
+#if defined(_MSC_VER)
+        printf("Failed to open input file %s.\n", givenName);
+#else
+        printf(RED "Failed to open input file %s.\n" NONE, givenName);
+#endif
+        return -1;
+    }
+    // First time
+    void* host = inputTensor->map(MNN::Tensor::MAP_TENSOR_WRITE, givenTensor.get()->getDimensionType());
+    if(host != nullptr) {
+        ::memcpy(host, givenTensor->host<uint8_t>(), givenTensor->size());
+    }
+    inputTensor->unmap(MNN::Tensor::MAP_TENSOR_WRITE,  givenTensor.get()->getDimensionType(), host);
+
+    // infer
+    net->runSession(session);
+    // read expect tensor
+    auto outputTensor = net->getSessionOutput(session, NULL);
+    std::shared_ptr<MNN::Tensor> expectTensor(createTensor(outputTensor, expectName));
+    if (!expectTensor.get()) {
+#if defined(_MSC_VER)
+        printf("Failed to open expect file %s.\n", expectName);
+#else
+        printf(RED "Failed to open expect file %s.\n" NONE, expectName);
+#endif
+        return -1;
+    }
+
+    // compare output with expect
+    bool correct = MNN::TensorUtils::compareTensors(outputTensor, expectTensor.get(), tolerance, true);
+    if (!correct) {
+#if defined(_MSC_VER)
+        printf("Test Failed %s!\n", modelPath);
+#else
+        printf(RED "Test Failed %s!\n" NONE, modelPath);
+#endif
+        return -1;
+    } else {
+        printf("First run pass\n");
+    }
+    // Run Second time
+    void* host1 = inputTensor->map(MNN::Tensor::MAP_TENSOR_WRITE, givenTensor.get()->getDimensionType());
+    if(host1 != nullptr) {
+        ::memcpy(host1, givenTensor->host<uint8_t>(), givenTensor->size());
+    }
+    inputTensor->unmap(MNN::Tensor::MAP_TENSOR_WRITE,  givenTensor.get()->getDimensionType(), host1);
+
+    // infer
+    net->runSession(session);
+    // read expect tensor
+    std::shared_ptr<MNN::Tensor> expectTensor2(createTensor(outputTensor, expectName));
+    correct = MNN::TensorUtils::compareTensors(outputTensor, expectTensor2.get(), tolerance, true);
+
+    if (correct) {
+#if defined(_MSC_VER)
+        printf("Test %s Correct!\n", modelPath);
+#else
+        printf(GREEN BOLD "Test %s Correct!\n" NONE, modelPath);
+#endif
+    } else {
+#if defined(_MSC_VER)
+        printf("Test Failed %s!\n", modelPath);
+#else
+        printf(RED "Test Failed %s!\n" NONE, modelPath);
+#endif
+    }
+    return 0;
+}

tools/cpp/testModelWithDescribe.cpp

@@ -0,0 +1,300 @@
+//
+//  testModelWithDescribe.cpp
+//  MNN
+//
+//  Created by MNN on 2019/01/22.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#define MNN_OPEN_TIME_TRACE
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <cstring>
+#include <fstream>
+#include <map>
+#include <sstream>
+#include <MNN/AutoTime.hpp>
+#include "core/Backend.hpp"
+#include "ConfigFile.hpp"
+#include <MNN/Interpreter.hpp>
+#include <MNN/MNNDefine.h>
+#include "core/Macro.h"
+#include <MNN/Tensor.hpp>
+#include "core/TensorUtils.hpp"
+#include <MNN/expr/Module.hpp>
+#include <MNN/expr/MathOp.hpp>
+#include <MNN/expr/NeuralNetWorkOp.hpp>
+#include <cmath>
+
+#define NONE "\e[0m"
+#define RED "\e[0;31m"
+#define GREEN "\e[0;32m"
+#define L_GREEN "\e[1;32m"
+#define BLUE "\e[0;34m"
+#define L_BLUE "\e[1;34m"
+#define BOLD "\e[1m"
+
+using namespace MNN::Express;
+
+template<typename T>
+inline T stringConvert(const char* number) {
+    std::istringstream os(number);
+    T v;
+    os >> v;
+    return v;
+}
+
+int loadData(const std::string name, void* ptr, int size, halide_type_t dtype) {
+    std::ifstream stream(name.c_str());
+    if (stream.fail()) {
+        return -1;
+    }
+    switch (dtype.code) {
+        case halide_type_float: {
+            auto data = static_cast<float*>(ptr);
+            for (int i = 0; i < size; ++i) {
+                double temp = 0.0f;
+                stream >> temp;
+                data[i] = temp;
+            }
+        } break;
+        case halide_type_int: {
+            MNN_ASSERT(dtype.bits == 32);
+            auto data = static_cast<int32_t*>(ptr);
+            for (int i = 0; i < size; ++i) {
+                int temp = 0;
+                stream >> temp;
+                data[i] = temp;
+            }
+        } break;
+        case halide_type_uint: {
+            MNN_ASSERT(dtype.bits == 8);
+            auto data = static_cast<uint8_t*>(ptr);
+            for (int i = 0; i < size; ++i) {
+                int temp = 0;
+                stream >> temp;
+                data[i] = temp;
+            }
+        } break;
+        default: {
+            stream.close();
+            return -1;
+        }
+    }
+
+    stream.close();
+    return 0;
+}
+
+MNN::Tensor* createTensor(const MNN::Tensor* shape, const std::string name) {
+    auto result = new MNN::Tensor(shape, shape->getDimensionType());
+    result->buffer().type = shape->buffer().type;
+    if (!loadData(name, result->host<void>(), result->elementSize(), result->getType())) {
+        return result;
+    }
+    delete result;
+    return NULL;
+}
+
+VARP createVar(const std::string name, INTS shape, halide_type_t dtype) {
+    int size = 1;
+    for (int dim : shape) {
+        size *= dim;
+    }
+    std::unique_ptr<char[]> data(new char[size * dtype.bytes()]);
+    loadData(name, data.get(), size, dtype);
+    return _Const(data.get(), shape, NHWC, dtype);
+}
+
+template <typename T>
+bool compareVar(VARP var, std::string name) {
+    auto targetValue = createVar(name, var->getInfo()->dim, var->getInfo()->type);
+    auto absMax = _ReduceMax(_Abs(targetValue), {});
+    absMax = _Maximum(absMax, _Scalar<T>(0));
+    auto diff = _Abs(targetValue - var);
+    auto diffAbsMax = _ReduceMax(diff);
+    auto absMaxV = absMax->readMap<T>()[0];
+    auto diffAbsMaxV = diffAbsMax->readMap<T>()[0];
+    // The implemention of isnan in VS2017 isn't accept integer type, so cast all type to double
+#ifdef _MSC_VER
+#define ALI_ISNAN(x) std::isnan(static_cast<long double>(x))
+#else
+#define ALI_ISNAN(x) std::isnan(x)
+#endif
+    if (absMaxV * 0.01f < diffAbsMaxV || ALI_ISNAN(absMaxV)) {
+        std::cout << "TESTERROR " << name << " value error : absMaxV:" << absMaxV << " - DiffMax:" << diffAbsMaxV << std::endl;
+        return false;
+    }
+    return true;
+}
+
+void log_result(bool correct) {
+    if (correct) {
+#if defined(_MSC_VER)
+        std::cout << "Correct!" << std::endl;
+#else
+        std::cout << GREEN << BOLD << "Correct!" << NONE << std::endl;
+#endif
+    }
+}
+
+int main(int argc, const char* argv[]) {
+    // modelName is xxx/xxx/temp.bin ===> xxx/xxx is the root path
+    const char* modelName = argv[1];
+    std::string modelDir  = argv[2];
+    modelDir              = modelDir.substr(0, modelDir.find("config.txt"));
+    std::cout << "model dir: " << modelDir << std::endl;
+
+    // read args
+    auto type = MNN_FORWARD_CPU;
+    if (argc > 3) {
+        type = (MNNForwardType)stringConvert<int>(argv[3]);
+    }
+    auto tolerance = 0.1f;
+    if (argc > 4) {
+        tolerance = stringConvert<float>(argv[4]);
+    }
+    auto precision = MNN::BackendConfig::Precision_High;
+    if (argc > 5) {
+        precision = (MNN::BackendConfig::PrecisionMode)(stringConvert<int>(argv[5]));
+    }
+
+    // input config
+    ConfigFile config(argv[2]);
+    auto numOfInputs = config.Read<int>("input_size");
+    auto numOfOuputs = config.Read<int>("output_size");
+    auto inputNames  = splitNames(numOfInputs, config.Read<std::string>("input_names"));
+    auto inputDims   = splitDims(numOfInputs, config.Read<std::string>("input_dims"));
+    auto expectNames = splitNames(numOfOuputs, config.Read<std::string>("output_names"));
+    bool controlFlow = config.KeyExists("control_flow") && config.Read<bool>("control_flow");
+    auto dataType = halide_type_of<float>();
+    if (config.KeyExists("data_type")) {
+        auto dtype = config.Read<std::string>("data_type");
+        if (dtype == "float") {
+            dataType = halide_type_of<float>();
+        } else if (dtype == "int32_t") {
+            dataType = halide_type_of<int32_t>();
+        } else if (dtype == "uint8_t") {
+            dataType = halide_type_of<uint8_t>();
+        }
+    }
+    // create net & session
+#if defined(_MSC_VER)
+    MNN_PRINT("Testing Model ====> %s\n", modelName);
+#else
+    MNN_PRINT(GREEN "Testing Model ====> %s\n" NONE, modelName);
+#endif
+    if (controlFlow) {
+        std::shared_ptr<Module> model(Module::load(inputNames, expectNames, modelName));
+        std::vector<VARP> inputs;
+        for (int i = 0; i < numOfInputs; i++) {
+            auto inputName = modelDir + inputNames[i] + ".txt";
+            inputs.push_back(createVar(inputName, inputDims[i], dataType));
+        }
+        auto outputs = model->onForward(inputs);
+        bool correct = true;
+        for (int i = 0; i < numOfOuputs; i++) {
+            auto dtype = outputs[i]->getInfo()->type;
+            auto outputName = modelDir + expectNames[i] + ".txt";
+            if (dtype == halide_type_of<int32_t>()) {
+                correct = compareVar<int32_t>(outputs[i], outputName);
+            } else if (dtype == halide_type_of<uint8_t>()) {
+                correct = compareVar<uint8_t>(outputs[i], outputName);
+            } else {
+                correct = compareVar<float>(outputs[i], outputName);
+            }
+            if (!correct) {
+                break;
+            }
+        }
+        log_result(correct);
+    } else {
+        auto net = std::shared_ptr<MNN::Interpreter>(MNN::Interpreter::createFromFile(modelName));
+        MNN::ScheduleConfig schedule;
+        schedule.type = type;
+        MNN::BackendConfig backendConfig;
+        backendConfig.precision = precision;
+
+        schedule.backendConfig = &backendConfig;
+
+        auto session  = net->createSession(schedule);
+
+        // resize
+        for (int i = 0; i < numOfInputs; ++i) {
+            auto inputTensor = net->getSessionInput(session, inputNames[i].c_str());
+            net->resizeTensor(inputTensor, inputDims[i]);
+        }
+        net->resizeSession(session);
+        auto checkFunction = [&]() {
+            // [second] set input-tensor data
+            for (int i = 0; i < numOfInputs; ++i) {
+                auto inputTensor = net->getSessionInput(session, inputNames[i].c_str());
+                auto inputName   = modelDir + inputNames[i] + ".txt";
+                std::cout << "The " << i << " input: " << inputName << std::endl;
+
+                auto givenTensor = createTensor(inputTensor, inputName);
+                if (!givenTensor) {
+#if defined(_MSC_VER)
+                    std::cout << "Failed to open " << inputName << std::endl;
+#else
+                    std::cout << RED << "Failed to open " << inputName << NONE << std::endl;
+#endif
+                    break;
+                }
+                inputTensor->copyFromHostTensor(givenTensor);
+                delete givenTensor;
+            }
+
+            // inference
+            net->runSession(session);
+
+            // get ouput-tensor and compare data
+            bool correct = true;
+            for (int i = 0; i < numOfOuputs; ++i) {
+                auto outputTensor = net->getSessionOutput(session, expectNames[i].c_str());
+                MNN::Tensor* expectTensor = nullptr;
+                std::string expectName;
+                // First Check outputname.txt
+                {
+                    std::ostringstream iStrOs;
+                    iStrOs << expectNames[i];
+                    expectName   = modelDir + iStrOs.str() + ".txt";
+                    expectTensor = createTensor(outputTensor, expectName);
+                }
+                if (!expectTensor) {
+                    // Second check number outputs
+                    std::ostringstream iStrOs;
+                    iStrOs << i;
+                    expectName   = modelDir + iStrOs.str() + ".txt";
+                    expectTensor = createTensor(outputTensor, expectName);
+                }
+                if (!expectTensor) {
+#if defined(_MSC_VER)
+                    std::cout << "Failed to open " << expectName << std::endl;
+#else
+                    std::cout << RED << "Failed to open " << expectName << NONE << std::endl;
+#endif
+                    break;
+                }
+                if (!MNN::TensorUtils::compareTensors(outputTensor, expectTensor, tolerance, true)) {
+                    correct = false;
+                    break;
+                }
+                delete expectTensor;
+            }
+            return correct;
+        };
+        auto correct = checkFunction();
+        if (!correct) {
+            return 0;
+        } else {
+            std::cout << "First Time Pass"<<std::endl;
+        }
+        // Second time
+        correct =  checkFunction();
+        log_result(correct);
+    }
+    return 0;
+}

tools/cpp/testModel_expr.cpp

@@ -0,0 +1,183 @@
+//
+//  testModel_expr.cpp
+//  MNN
+//
+//  Created by MNN on 2021/08/09.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#define MNN_OPEN_TIME_TRACE
+
+#include <MNN/MNNDefine.h>
+#include <math.h>
+#include <algorithm>
+#include <cmath>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <MNN/AutoTime.hpp>
+#include <MNN/expr/Module.hpp>
+#include <MNN/expr/Expr.hpp>
+#include <MNN/expr/ExprCreator.hpp>
+#include <fstream>
+#include <map>
+#include <iostream>
+#include <sstream>
+#include "ExprDebug.hpp"
+#define NONE "\e[0m"
+#define RED "\e[0;31m"
+#define GREEN "\e[0;32m"
+#define L_GREEN "\e[1;32m"
+#define BLUE "\e[0;34m"
+#define L_BLUE "\e[1;34m"
+#define BOLD "\e[1m"
+
+void log_result(bool correct) {
+    if (correct) {
+#if defined(_MSC_VER)
+        std::cout << "Correct!" << std::endl;
+#else
+        std::cout << GREEN << BOLD << "Correct!" << NONE << std::endl;
+#endif
+    }
+}
+
+template<typename T>
+inline T stringConvert(const char* number) {
+    std::istringstream os(number);
+    T v;
+    os >> v;
+    return v;
+}
+
+template <typename T>
+static bool compareImpl(MNN::Express::VARP x, MNN::Express::VARP y, int size, double tolerance) {
+#define _ABS(a) ((a) < 0 ? -(a) : (a))
+#define _MAX(a, b) ((a) > (b) ? (a) : (b))
+    auto px = x->readMap<T>();
+    auto py = y->readMap<T>();
+    // get max if using overall torelance
+    T maxValue = _ABS(py[0]);
+    for (int i = 1; i < size; i++) {
+        maxValue = _MAX(maxValue, _ABS(py[i]));
+    }
+    // compare
+    for (int i = 0; i < size; i++) {
+        T vx = px[i], vy = py[i];
+        if (_ABS(vx - vy) < tolerance * maxValue) {
+            continue;
+        }
+        std::cout << i << ": " << vx << " != " << vy << std::endl;
+        return false;
+    }
+    return true;
+#undef _ABS
+#undef _MAX
+}
+
+static bool compareVar(MNN::Express::VARP x, MNN::Express::VARP y, double tolerance) {
+    auto info = y->getInfo();
+    auto dtype = info->type;
+    auto size = info->size;
+    if (dtype == halide_type_of<int32_t>()) {
+        return compareImpl<int32_t>(x, y, size, tolerance);
+    }
+    if (dtype == halide_type_of<uint8_t>()) {
+        return compareImpl<uint8_t>(x, y, size, tolerance);
+    }
+    return compareImpl<float>(x, y, size, tolerance);
+}
+
+using namespace MNN::Express;
+int main(int argc, const char* argv[]) {
+    if (argc < 4) {
+        MNN_PRINT("Usage: ./testModel_expr.out model.mnn input.mnn output.mnn [type] [tolerance] [precision]\n");
+        return 0;
+    }
+    MNN::ScheduleConfig sdConfig;
+    auto rtMgr = std::shared_ptr<MNN::Express::Executor::RuntimeManager>(MNN::Express::Executor::RuntimeManager::createRuntimeManager(sdConfig), MNN::Express::Executor::RuntimeManager::destroy);
+//#define TEST_DEBUG
+#ifdef TEST_DEBUG
+    _initTensorStatic();
+    //_initDebug();
+    rtMgr->setMode(MNN::Interpreter::Session_Debug);
+#endif
+    // check given & expect
+    const char* modelPath  = argv[1];
+    const char* inputName  = argv[2];
+    const char* outputName = argv[3];
+    MNN_PRINT("Testing model %s, input: %s, output: %s\n", modelPath, inputName, outputName);
+    
+    // create net
+    auto type = MNN_FORWARD_CPU;
+    if (argc > 4) {
+        type = (MNNForwardType)stringConvert<int>(argv[4]);
+    }
+    auto tolerance = 0.1f;
+    if (argc > 5) {
+        tolerance = stringConvert<float>(argv[5]);
+    }
+    MNN::BackendConfig::PrecisionMode precision = MNN::BackendConfig::Precision_High;
+    if (argc > 6) {
+        precision = (MNN::BackendConfig::PrecisionMode)stringConvert<int>(argv[6]);
+    }
+    auto inputVars = Variable::load(inputName);
+    auto outputVars = Variable::load(outputName);
+    std::vector<std::string> inputNames;
+    std::vector<std::string> outputNames;
+    for (auto v : inputVars) {
+        inputNames.emplace_back(v->name());
+    }
+    for (auto v : outputVars) {
+        outputNames.emplace_back(v->name());
+    }
+    if (inputVars.empty()) {
+        MNN_ERROR("Input is Error\n");
+        return 0;
+    }
+    if (outputVars.empty()) {
+        MNN_ERROR("Output is Error\n");
+        return 0;
+    }
+    Module::Config config;
+    config.rearrange = true;
+    rtMgr->setHint(MNN::Interpreter::INIT_THREAD_NUMBER, 4);
+    std::shared_ptr<Module> m(Module::load(inputNames, outputNames, modelPath, rtMgr, &config), [](void* net) {
+        MNN::Express::Module::destroy((MNN::Express::Module*)net);
+    });
+    if (nullptr == m) {
+        MNN_ERROR("Model is Error\n");
+        return 0;
+    }
+    // First
+    auto outputs = m->onForward(inputVars);
+    if (outputs.size() != outputVars.size()) {
+        MNN_ERROR("Number not match\n");
+        return 0;
+    }
+    bool success = true;
+    for (int i=0; i<outputVars.size(); ++i) {
+        success = compareVar(outputs[i], outputVars[i], tolerance);
+        if (!success) {
+            MNN_ERROR("Error for %s\n", outputVars[i]->name().c_str());
+            break;
+        }
+    }
+    if (!success) {
+        return 0;
+    }
+    outputs = m->onForward(inputVars);
+    for (int i=0; i<outputVars.size(); ++i) {
+        success = compareVar(outputs[i], outputVars[i], tolerance);
+        if (!success) {
+            MNN_ERROR("Error for %s\n", outputVars[i]->name().c_str());
+            break;
+        }
+    }
+    if (!success) {
+        MNN_ERROR("Error for test second\n");
+        return 0;
+    }
+    log_result(success);
+    return 0;
+}

tools/cpp/testTrain.cpp

@@ -0,0 +1,235 @@
+//
+//  testTrain.cpp
+//  MNN
+//
+//  Created by MNN on 2021/06/22.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#define MNN_OPEN_TIME_TRACE
+
+#include <MNN/MNNDefine.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <MNN/AutoTime.hpp>
+#include <MNN/Interpreter.hpp>
+#include <MNN/Tensor.hpp>
+#include <MNN/expr/Expr.hpp>
+#include <fstream>
+#include <map>
+#include <cmath>
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <set>
+#include <algorithm>
+#include "rapidjson/document.h"
+#include "rapidjson/stringbuffer.h"
+#include "rapidjson/prettywriter.h"
+
+#define NONE "\e[0m"
+#define RED "\e[0;31m"
+#define GREEN "\e[0;32m"
+#define L_GREEN "\e[1;32m"
+#define BLUE "\e[0;34m"
+#define L_BLUE "\e[1;34m"
+#define BOLD "\e[1m"
+
+template<typename T>
+inline T stringConvert(const char* number) {
+    std::istringstream os(number);
+    T v;
+    os >> v;
+    return v;
+}
+
+MNN::Tensor* createTensor(const MNN::Tensor* shape, const char* path) {
+    std::ifstream stream(path);
+    if (stream.fail()) {
+        return NULL;
+    }
+
+    auto result = new MNN::Tensor(shape, shape->getDimensionType());
+    auto data   = result->host<float>();
+    for (int i = 0; i < result->elementSize(); ++i) {
+        double temp = 0.0f;
+        stream >> temp;
+        data[i] = temp;
+    }
+    stream.close();
+    return result;
+}
+
+int main(int argc, const char* argv[]) {
+    // check given & expect
+    if (argc < 3) {
+        return 0;
+    }
+    const char* jsonPath    = argv[1];
+    const char* dirPath     = argv[2];
+    rapidjson::Document document;
+    {
+        std::ifstream fileNames(jsonPath);
+        std::ostringstream output;
+        output << fileNames.rdbuf();
+        auto outputStr = output.str();
+        document.Parse(outputStr.c_str());
+        if (document.HasParseError()) {
+            MNN_ERROR("Invalid json\n");
+            return 0;
+        }
+    }
+    auto picObj = document.GetObject();
+    float learnRate;
+    if (document.HasMember("LearningRate")) {
+        learnRate = document["LearningRate"].GetFloat();
+    }
+    auto modelPath = std::string(dirPath) + "/" + picObj["Model"].GetString();
+    auto lossName = picObj["Loss"].GetString();
+    auto inputName = picObj["Input"].GetString();
+    auto targetName = picObj["Target"].GetString();
+    auto dataArray = picObj["Data"].GetArray();
+    auto lR = picObj["LR"].GetString();
+    auto decay = picObj["Decay"].GetFloat();
+    
+    // create net
+    auto type = MNN_FORWARD_CPU;
+    MNN::BackendConfig::PrecisionMode precision = MNN::BackendConfig::Precision_Low;
+    std::shared_ptr<MNN::Interpreter> net =
+        std::shared_ptr<MNN::Interpreter>(MNN::Interpreter::createFromFile(modelPath.c_str()));
+
+    // create session
+    MNN::ScheduleConfig config;
+    config.type = type;
+    config.saveTensors.emplace_back(lossName);
+    MNN::BackendConfig backendConfig;
+    backendConfig.precision = precision;
+    config.backendConfig = &backendConfig;
+    auto session         = net->createSession(config);
+    if (nullptr == net->getSessionInput(session, inputName) || nullptr == net->getSessionInput(session, targetName) || nullptr == net->getSessionInput(session, lR) || nullptr == net->getSessionOutput(session, lossName)) {
+        MNN_ERROR("Invalid model for train\n");
+        return 0;
+    }
+    static bool gDebug = false;
+    bool onlyInfer = false;
+    auto lossTensor = net->getSessionOutput(session, lossName);
+    std::vector<float> loss;
+    MNN::TensorCallBack beforeCallBack = [&](const std::vector<MNN::Tensor*>& ntensors, const std::string& opName) {
+        return true;
+    };
+    MNN::TensorCallBack callBack = [&](const std::vector<MNN::Tensor*>& ntensors, const std::string& opName) {
+        for (int i = 0; i < ntensors.size(); ++i) {
+            auto ntensor    = ntensors[i];
+            if (onlyInfer && ntensor == lossTensor) {
+                return false;
+            }
+            if (ntensor->getType().code != halide_type_float) {
+                continue;
+            }
+            if (gDebug) {
+                auto outDimType = ntensor->getDimensionType();
+                auto expectTensor = new MNN::Tensor(ntensor, outDimType);
+                ntensor->copyToHostTensor(expectTensor);
+                auto size = expectTensor->elementSize();
+                float summer = 0.0f;
+                for (int i=0; i<size; ++i) {
+                    summer += expectTensor->host<float>()[i];
+                }
+                delete expectTensor;
+                MNN_PRINT("For op %s, summer=%f\n", opName.c_str(), summer);
+            }
+        }
+        return true;
+    };
+    auto lrTensor = net->getSessionInput(session, lR);
+    std::shared_ptr<MNN::Tensor> userLR(new MNN::Tensor(lrTensor, lrTensor->getDimensionType()));
+    
+    int runTime = 2;
+    for (int i=0; i<runTime; ++i) {
+        onlyInfer = i == (runTime-1);
+        for (auto iter = dataArray.begin(); iter != dataArray.end(); iter++) {
+            auto dataName = std::string(dirPath) + "/" + std::string(iter->GetString());
+            auto varMap = MNN::Express::Variable::load(dataName.c_str());
+            if (varMap.empty()) {
+                continue;
+            }
+            userLR->host<float>()[0] = learnRate;
+            lrTensor->copyFromHostTensor(userLR.get());
+            for (auto v : varMap) {
+                auto target = net->getSessionInput(session, v->name().c_str());
+                if (nullptr == target) {
+                    MNN_ERROR("Invalid data %s\n", v->name().c_str());
+                    continue;
+                }
+                std::shared_ptr<MNN::Tensor> targetUser(new MNN::Tensor(target, target->getDimensionType()));
+                ::memcpy(targetUser->host<void>(), v->readMap<void>(), targetUser->size());
+                target->copyFromHostTensor(targetUser.get());
+            }
+            net->runSessionWithCallBack(session, beforeCallBack, callBack);
+            std::shared_ptr<MNN::Tensor> lossTemp(new MNN::Tensor(lossTensor, lossTensor->getDimensionType()));
+            lossTensor->copyToHostTensor(lossTemp.get());
+            loss.emplace_back(lossTemp->host<float>()[0]);
+        }
+    }
+    bool correct = false;
+    if (loss.size() < 2) {
+        printf("Test Failed, data invalid %s!\n", modelPath.c_str());
+        return 0;
+    }
+    auto firstLoss = loss[0];
+    auto lastLoss = loss[(int)loss.size() - 1];
+    bool validFirst = firstLoss < 0.0f || firstLoss >= 0.0f;
+    bool validLast = lastLoss < 0.0f || lastLoss >= 0.0f;
+    MNN_PRINT("Loss from %f -> %f\n", firstLoss, lastLoss);
+    bool lossValid = lastLoss < firstLoss * decay;
+    if (!lossValid) {
+        MNN_PRINT("Invalid loss decrease\n");
+        return 0;
+    }
+    // Test Update
+    net->updateSessionToModel(session);
+    auto buffer = net->getModelBuffer();
+    config.path.mode = MNN::ScheduleConfig::Path::Tensor;
+    config.path.outputs.emplace_back(lossName);
+    std::shared_ptr<MNN::Interpreter> newNet(MNN::Interpreter::createFromBuffer(buffer.first, buffer.second), MNN::Interpreter::destroy);
+    net.reset();
+    net = newNet;
+    session = net->createSession(config);
+    lossTensor = net->getSessionOutput(session, lossName);
+    onlyInfer = true;
+    lrTensor = net->getSessionInput(session, lR);
+    for (auto iter = dataArray.begin(); iter != dataArray.end(); iter++) {
+        auto dataName = std::string(dirPath) + "/" + std::string(iter->GetString());
+        auto varMap = MNN::Express::Variable::load(dataName.c_str());
+        if (varMap.empty()) {
+            continue;
+        }
+        userLR->host<float>()[0] = learnRate;
+        lrTensor->copyFromHostTensor(userLR.get());
+        for (auto v : varMap) {
+            auto target = net->getSessionInput(session, v->name().c_str());
+            if (nullptr == target) {
+                MNN_ERROR("Invalid data %s\n", v->name().c_str());
+                continue;
+            }
+            std::shared_ptr<MNN::Tensor> targetUser(new MNN::Tensor(target, target->getDimensionType()));
+            ::memcpy(targetUser->host<void>(), v->readMap<void>(), targetUser->size());
+            target->copyFromHostTensor(targetUser.get());
+        }
+        net->runSessionWithCallBack(session, beforeCallBack, callBack);
+        {
+            std::shared_ptr<MNN::Tensor> lossTemp(new MNN::Tensor(lossTensor, lossTensor->getDimensionType()));
+            lossTensor->copyToHostTensor(lossTemp.get());
+            auto newLoss = lossTemp->host<float>()[0];
+            MNN_PRINT("Update and reload, loss from %f -> %f\n", lastLoss, newLoss);
+            if (newLoss > lastLoss + 0.1f) {
+                MNN_ERROR("newLoss not valid\n");
+                return 0;
+            }
+        }
+    }
+    MNN_PRINT("Test %s Correct!\n", modelPath.c_str());
+    return 0;
+}

tools/cpp/timeProfile.cpp

@@ -0,0 +1,185 @@
+//
+//  timeProfile.cpp
+//  MNN
+//
+//  Created by MNN on 2019/01/22.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#define MNN_OPEN_TIME_TRACE
+#include <stdlib.h>
+#include <cstring>
+#include <memory>
+#include <string>
+#include <MNN/AutoTime.hpp>
+#include <MNN/Interpreter.hpp>
+#include <MNN/MNNDefine.h>
+#include "core/Macro.h"
+#include "Profiler.hpp"
+#include <MNN/Tensor.hpp>
+#include "revertMNNModel.hpp"
+
+#define MNN_PRINT_TIME_BY_NAME
+
+using namespace MNN;
+
+static inline std::vector<int> parseIntList(const std::string& str, char delim) {
+    std::vector<int> result;
+    std::ptrdiff_t p1 = 0, p2;
+    while (1) {
+        p2 = str.find(delim, p1);
+        if (p2 != std::string::npos) {
+            result.push_back(atoi(str.substr(p1, p2 - p1).c_str()));
+            p1 = p2 + 1;
+        } else {
+            result.push_back(atoi(str.substr(p1).c_str()));
+            break;
+        }
+    }
+    return result;
+}
+int main(int argc, const char* argv[]) {
+    if (argc < 2) {
+        MNN_PRINT("=========================================================================================\n");
+        MNN_PRINT("Arguments: model.MNN runLoops forwardType inputSize numberThread precision sparsity cpuIds\n");
+        MNN_PRINT("Example: %s model.MNN 100 0 1x3x224x224 4 0 0 0,1,2,3\n", argv[0]);
+        MNN_PRINT("=========================================================================================\n");
+        return -1;
+    }
+
+    std::string cmd = argv[0];
+    std::string pwd = "./";
+    auto rslash     = cmd.rfind("/");
+    if (rslash != std::string::npos) {
+        pwd = cmd.substr(0, rslash + 1);
+    }
+
+    // read args
+    const char* fileName = argv[1];
+    int runTime          = 100;
+    if (argc > 2) {
+        runTime = ::atoi(argv[2]);
+    }
+    auto type = MNN_FORWARD_CPU;
+    if (argc > 3) {
+        type = (MNNForwardType)atoi(argv[3]);
+        printf("Use extra forward type: %d\n", type);
+    }
+
+    // input dims
+    std::vector<int> inputDims;
+    if (argc > 4) {
+        inputDims = parseIntList(argv[4], 'x');
+    }
+    MNN_PRINT("inputDims: ");
+    for (auto dim : inputDims) {
+        MNN_PRINT("%d ", dim);
+    }
+    MNN_PRINT("\n");
+    int threadNumber = 4;
+    if (argc > 5) {
+        threadNumber = ::atoi(argv[5]);
+        MNN_PRINT("Set ThreadNumber = %d\n", threadNumber);
+    }
+
+    auto precision = BackendConfig::PrecisionMode::Precision_Normal;
+    if (argc > 6) {
+        precision = (BackendConfig::PrecisionMode)atoi(argv[6]);
+        printf("Use precision type: %d\n", precision);
+    }
+
+    float sparsity = 0.0f;
+    if(argc > 7) {
+        sparsity = atof(argv[7]);
+    }
+    
+    // CPU IDs
+    std::vector<int> cpuIds;
+    if (argc > 8) {
+        cpuIds = parseIntList(argv[8], ',');
+    }
+    MNN_PRINT("cpuIds: ");
+    for (auto id : cpuIds) {
+        MNN_PRINT("%d ", id);
+    }
+    MNN_PRINT("\n");
+
+
+    // revert MNN model if necessary
+    auto revertor = std::unique_ptr<Revert>(new Revert(fileName));
+    revertor->initialize(sparsity);
+    auto modelBuffer = revertor->getBuffer();
+    auto bufferSize  = revertor->getBufferSize();
+
+    // create net
+    MNN_PRINT("Open Model %s\n", fileName);
+    auto net = std::shared_ptr<Interpreter>(Interpreter::createFromBuffer(modelBuffer, bufferSize));
+    if (nullptr == net) {
+        return 0;
+    }
+    revertor.reset();
+    net->setSessionMode(Interpreter::Session_Debug);
+    net->setSessionHint(Interpreter::HintMode::CPU_CORE_IDS, cpuIds.data(), cpuIds.size());
+
+    // create session
+    MNN::ScheduleConfig config;
+    config.type           = type;
+    config.numThread      = threadNumber;
+    BackendConfig backendConfig;
+    backendConfig.precision = precision;
+    config.backendConfig  = &backendConfig;
+    MNN::Session* session = NULL;
+    session               = net->createSession(config);
+    auto inputTensor      = net->getSessionInput(session, NULL);
+    if (!inputDims.empty()) {
+        net->resizeTensor(inputTensor, inputDims);
+        net->resizeSession(session);
+    }
+    auto allInput = net->getSessionInputAll(session);
+    for (auto& iter : allInput) {
+        auto inputTensor = iter.second;
+        auto size = inputTensor->size();
+        if (size <= 0) {
+            continue;
+        }
+        MNN::Tensor tempTensor(inputTensor, inputTensor->getDimensionType());
+        ::memset(tempTensor.host<void>(), 0, tempTensor.size());
+        inputTensor->copyFromHostTensor(&tempTensor);
+    }
+    net->releaseModel();
+    std::shared_ptr<MNN::Tensor> inputTensorUser(MNN::Tensor::createHostTensorFromDevice(inputTensor, false));
+    auto outputTensor = net->getSessionOutput(session, NULL);
+    if (outputTensor->size() <= 0) {
+        MNN_ERROR("Output not available\n");
+        return 0;
+    }
+    std::shared_ptr<MNN::Tensor> outputTensorUser(MNN::Tensor::createHostTensorFromDevice(outputTensor, false));
+
+    auto profiler      = MNN::Profiler::getInstance();
+    auto beginCallBack = [&](const std::vector<Tensor*>& inputs, const OperatorInfo* info) {
+        profiler->start(info);
+        return true;
+    };
+    auto afterCallBack = [&](const std::vector<Tensor*>& tensors, const OperatorInfo* info) {
+        for (auto o : tensors) {
+            o->wait(MNN::Tensor::MAP_TENSOR_READ, true);
+        }
+        profiler->end(info);
+        return true;
+    };
+
+    AUTOTIME;
+    // just run
+    for (int i = 0; i < runTime; ++i) {
+        inputTensor->copyFromHostTensor(inputTensorUser.get());
+        net->runSessionWithCallBackInfo(session, beginCallBack, afterCallBack);
+        outputTensor->copyToHostTensor(outputTensorUser.get());
+    }
+
+#ifdef MNN_PRINT_TIME_BY_NAME
+    profiler->printTimeByName(runTime);
+#endif
+    profiler->printSlowOp("Convolution", 20, 0.03f);
+    profiler->printTimeByType(runTime);
+    return 0;
+}

tools/cpp/winogradExample.cpp

@@ -0,0 +1,33 @@
+//
+//  winogradExample.cpp
+//  MNN
+//
+//  Created by MNN on 2019/01/22.
+//  Copyright © 2018, Alibaba Group Holding Limited
+//
+
+#include <stdlib.h>
+#include <string.h>
+#include <MNN/MNNDefine.h>
+#include "math/Matrix.hpp"
+#include "math/WingoradGenerater.hpp"
+
+int main(int argc, const char* argv[]) {
+    int unit       = ::atoi(argv[1]);
+    int kernelSize = ::atoi(argv[2]);
+    float interp   = 0.5f;
+    if (argc > 3) {
+        interp = ::atof(argv[3]);
+    }
+    MNN::Math::WinogradGenerater generater(unit, kernelSize, interp);
+    auto a = generater.A();
+    auto b = generater.B();
+    auto g = generater.G();
+    MNN_PRINT("A=\n");
+    MNN::Math::Matrix::print(a.get());
+    MNN_PRINT("B=\n");
+    MNN::Math::Matrix::print(b.get());
+    MNN_PRINT("G=\n");
+    MNN::Math::Matrix::print(g.get());
+    return 0;
+}