#include #include #include #include #include #include #include #include #if defined(_MSC_VER) #include #undef min #undef max #else #include #include #include #endif #include "MNN_generated.h" #include #include #include #include "ExprModels.hpp" using namespace MNN; using namespace MNN::Express; static inline uint64_t getTimeInUs() { uint64_t time; #if defined(_MSC_VER) LARGE_INTEGER now, freq; QueryPerformanceCounter(&now); QueryPerformanceFrequency(&freq); uint64_t sec = now.QuadPart / freq.QuadPart; uint64_t usec = (now.QuadPart % freq.QuadPart) * 1000000 / freq.QuadPart; time = sec * 1000000 + usec; #else struct timeval tv; gettimeofday(&tv, nullptr); time = static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; #endif return time; } static inline std::string forwardType(MNNForwardType type) { switch (type) { case MNN_FORWARD_CPU: return "CPU"; case MNN_FORWARD_VULKAN: return "Vulkan"; case MNN_FORWARD_OPENCL: return "OpenCL"; case MNN_FORWARD_METAL: return "Metal"; default: break; } return "N/A"; } static std::vector splitArgs(const std::string& args, const std::string& delimiter) { std::vector result; size_t pos = 0, nextPos = args.find(delimiter, 0); while (nextPos != std::string::npos) { result.push_back(args.substr(pos, nextPos - pos)); pos = nextPos + delimiter.length(); nextPos = args.find(delimiter, pos); } result.push_back(args.substr(pos, args.length() - pos)); return result; } static void displayStats(const std::string& name, const std::vector& costs) { float max = 0, min = FLT_MAX, sum = 0, avg; for (auto v : costs) { max = max < v ? v : max; min = min > v ? v : min; sum += v; } avg = costs.size() > 0 ? sum / costs.size() : 0; printf("[ - ] %-24s max = %8.3fms min = %8.3fms avg = %8.3fms\n", name.c_str(), max, avg == 0 ? 0 : min, avg); } static std::vector runNet(VARP netOutput, const ScheduleConfig& config, int loop) { std::unique_ptr netTable(new NetT); Variable::save({netOutput}, netTable.get()); flatbuffers::FlatBufferBuilder builder(1024); auto offset = CreateNet(builder, netTable.get()); builder.Finish(offset); const void* buf = builder.GetBufferPointer(); size_t size = builder.GetSize(); std::unique_ptr net(Interpreter::createFromBuffer(buf, size)); net->setSessionMode(MNN::Interpreter::Session_Release); auto session = net->createSession(config); net->releaseModel(); auto inputTensor = net->getSessionInput(session, NULL); std::shared_ptr inputTensorHost(Tensor::createHostTensorFromDevice(inputTensor, false)); int eleSize = inputTensorHost->elementSize(); for (int i = 0; i < eleSize; ++i) { inputTensorHost->host()[i] = 0.0f; } auto outputTensor = net->getSessionOutput(session, NULL); std::shared_ptr outputTensorHost(Tensor::createHostTensorFromDevice(outputTensor, false)); // Warming up... for (int i = 0; i < 3; ++i) { inputTensor->copyFromHostTensor(inputTensorHost.get()); net->runSession(session); outputTensor->copyToHostTensor(outputTensorHost.get()); } std::vector costs; // start run for (int i = 0; i < loop; ++i) { auto timeBegin = getTimeInUs(); inputTensor->copyFromHostTensor(inputTensorHost.get()); net->runSession(session); outputTensor->copyToHostTensor(outputTensorHost.get()); auto timeEnd = getTimeInUs(); costs.push_back((timeEnd - timeBegin) / 1000.0); } return costs; } static void _printHelp() { std::cout << "Usage: " << " model_to_benchmark [loop_count] [forwardtype] [numberThread]" << std::endl; std::cout << "model_to_benchmark: " << std::endl; std::cout << "\t default: run standard models" << std::endl; std::cout << "\t MobileNetV1_{numClass}_{width}_{resolution}, width: {1.0, 0.75, 0.5, 0.25}, resolution: {224, 192, 160, 128}, e.g: MobileNetV1_100_1.0_224" << std::endl; std::cout << "\t MobileNetV2_{numClass}, e.g: MobileNetV2_100" << std::endl; std::cout << "\t ResNet_{numClass}_{layer}, layer: {18, 34, 50, 101, 152}, e.g: ResNet_100_18" << std::endl; std::cout << "\t GoogLeNet_{numClass}, e.g: GoogLeNet_100" << std::endl; std::cout << "\t SqueezeNet_{numClass}, e.g: SqueezeNet_100" << std::endl; std::cout << "\t ShuffleNet_{numClass}_{group}, group: [1, 2, 3, 4, 8], e.g: ShuffleNet_100_4" << std::endl; } static std::vector gDefaultModels = { "MobileNetV1_1000_1.0_224", "MobileNetV2_1000", "GoogLeNet_1000", "ShuffleNet_1000_4", "SqueezeNet_1000", "ResNet_1000_18", "ResNet_1000_50", }; int main(int argc, const char* argv[]) { std::cout << "MNN Expr Models benchmark" << std::endl; size_t loop = 10; MNNForwardType forward = MNN_FORWARD_CPU; size_t numThread = 4; if (argc <= 1) { _printHelp(); return 0; } if (((argc > 1) && (strcmp(argv[1], "help") == 0)) || argc > 5) { _printHelp(); return 0; } std::vector models; if (((argc > 1) && (strcmp(argv[1], "default") == 0)) || argc > 5) { models = gDefaultModels; } else { models = {argv[1]}; } if (argc >= 3) { loop = atoi(argv[2]); } if (argc >= 4) { forward = static_cast(atoi(argv[3])); } if (argc >= 5) { numThread = atoi(argv[4]); } std::cout << "Forward type: **" << forwardType(forward) << "** thread=" << numThread << std::endl; ScheduleConfig config; config.type = forward; config.numThread = numThread; BackendConfig bnConfig; bnConfig.precision = BackendConfig::Precision_Low; bnConfig.power = BackendConfig::Power_High; config.backendConfig = &bnConfig; std::vector costs; // ResNet18 benchmark for (auto model : models) { auto modelArgs = splitArgs(model.c_str(), "_"); auto modelType = modelArgs[0]; int numClass = atoi(modelArgs[1].c_str()); if (modelType == "MobileNetV1") { auto mobileNetWidthType = EnumMobileNetWidthTypeByString(modelArgs[2]); if (mobileNetWidthType < 0) { std::cout << "Not support MobileNetWidthType " << modelArgs[2] << std::endl; std::cout << "Only [1.0, 0.75, 0.5, 0.25] be support" << std::endl; return 1; } auto mobileNetResolutionType = EnumMobileNetResolutionTypeByString(modelArgs[3]); if (mobileNetResolutionType < 0) { std::cout << "Not support MobileNetResolutionType " << modelArgs[3] << std::endl; std::cout << "Only [224, 192, 160, 128] be support" << std::endl; return 1; } costs = runNet(mobileNetV1Expr(mobileNetWidthType, mobileNetResolutionType, numClass), config, loop); } else if (modelType == "MobileNetV2") { costs = runNet(mobileNetV2Expr(numClass), config, loop); } else if (modelType == "ResNet") { auto resNetType = EnumResNetTypeByString(modelArgs[2]); if (resNetType < 0) { std::cout << "Not support ResNet layer " << modelArgs[2] << std::endl; std::cout << "Only [18, 34, 50, 101, 152] be support" << std::endl; return 1; } costs = runNet(resNetExpr(resNetType, numClass), config, loop); } else if (modelType == "GoogLeNet") { costs = runNet(googLeNetExpr(numClass), config, loop); } else if (modelType == "SqueezeNet") { costs = runNet(squeezeNetExpr(numClass), config, loop); } else if (modelType == "ShuffleNet") { int group = atoi(modelArgs[2].c_str()); costs = runNet(shuffleNetExpr(group, numClass), config, loop); } else { std::cout << "Not support Model Type " << modelType << std::endl; continue; } displayStats(model.c_str(), costs); } return 0; }