// // NMSModule.cpp // MNN // // Created by MNN on 2020/09/10. // Copyright © 2018, Alibaba Group Holding Limited // #include "NMSModule.hpp" #include "backend/cpu/CPUNonMaxSuppressionV2.hpp" #include #include #include "MNN_generated.h" //#define MNN_OPEN_TIME_TRACE #include #include namespace MNN { namespace Express { NMSModule* NMSModule::create(const Op* op) { auto module = new NMSModule; module->setType("NMSModule"); if (nullptr != op->name()) { module->setName(op->name()->str()); } return module; } // Return intersection-over-union overlap between boxes i and j static inline float IOU(const float* boxes, int i, int j) { const float yMinI = std::min(boxes[i * 4 + 0], boxes[i * 4 + 2]); const float xMinI = std::min(boxes[i * 4 + 1], boxes[i * 4 + 3]); const float yMaxI = std::max(boxes[i * 4 + 0], boxes[i * 4 + 2]); const float xMaxI = std::max(boxes[i * 4 + 1], boxes[i * 4 + 3]); const float yMinJ = std::min(boxes[j * 4 + 0], boxes[j * 4 + 2]); const float xMinJ = std::min(boxes[j * 4 + 1], boxes[j * 4 + 3]); const float yMaxJ = std::max(boxes[j * 4 + 0], boxes[j * 4 + 2]); const float xMaxJ = std::max(boxes[j * 4 + 1], boxes[j * 4 + 3]); const float areaI = (yMaxI - yMinI) * (xMaxI - xMinI); const float areaJ = (yMaxJ - yMinJ) * (xMaxJ - xMinJ); if (areaI <= 0 || areaJ <= 0) return 0.0; const float intersectionYMin = std::max(yMinI, yMinJ); const float intersectionXMin = std::max(xMinI, xMinJ); const float intersectionYMax = std::min(yMaxI, yMaxJ); const float intersectionXMax = std::min(xMaxI, xMaxJ); const float intersectionArea = std::max(intersectionYMax - intersectionYMin, 0.0) * std::max(intersectionXMax - intersectionXMin, 0.0); return intersectionArea / (areaI + areaJ - intersectionArea); } static void NonMaxSuppressionSingleClasssImpl(const float* boxesPtr, const float* scores, int numBoxes, int maxDetections, float iouThreshold, float scoreThreshold, std::vector* selected) { MNN_ASSERT(iouThreshold >= 0.0f && iouThreshold <= 1.0f); const int outputNum = std::min(maxDetections, numBoxes); std::vector scoresData(numBoxes); std::copy_n(scores, numBoxes, scoresData.begin()); struct Candidate { int boxIndex; float score; }; auto cmp = [](const Candidate bsI, const Candidate bsJ) { return bsI.score < bsJ.score; }; std::priority_queue, decltype(cmp)> candidatePriorityQueue(cmp); for (int i = 0; i < scoresData.size(); ++i) { if (scoresData[i] > scoreThreshold) { candidatePriorityQueue.emplace(Candidate({i, scoresData[i]})); } } // std::vector selectedScores; Candidate nextCandidate; float iou, originalScore; while (selected->size() < outputNum && !candidatePriorityQueue.empty()) { nextCandidate = candidatePriorityQueue.top(); originalScore = nextCandidate.score; candidatePriorityQueue.pop(); // Overlapping boxes are likely to have similar scores, // therefore we iterate through the previously selected boxes backwards // in order to see if `next_candidate` should be suppressed. bool shouldSelect = true; for (int j = (int)selected->size() - 1; j >= 0; --j) { iou = IOU(boxesPtr, nextCandidate.boxIndex, selected->at(j)); if (iou == 0.0) { continue; } if (iou > iouThreshold) { shouldSelect = false; } } if (shouldSelect) { selected->push_back(nextCandidate.boxIndex); // selectedScores.push_back(nextCandidate.score); } } } std::vector NMSModule::onForward(const std::vector& inputs) { const int maxDetections = inputs[2]->readMap()[0]; float iouThreshold = 0, scoreThreshold = std::numeric_limits::lowest(); if (inputs.size() > 3) { iouThreshold = inputs[3]->readMap()[0]; } if (inputs.size() > 4) { scoreThreshold = inputs[4]->readMap()[0]; } auto boxes = inputs[0], score = inputs[1]; auto info = boxes->getInfo(), infoScore = score->getInfo(); MNN_ASSERT(info->dim[info->dim.size() - 1] == 4); int batch = 1, numClass = 1, numBoxes = info->dim[0]; bool onnxFormat = (infoScore->dim.size() > 1); if (onnxFormat) { batch = infoScore->dim[0]; numClass = infoScore->dim[1]; numBoxes = infoScore->dim[2]; } INTS outputData; for (int b = 0; b < batch; ++b) { const auto boxesPtr = boxes->readMap() + b * numBoxes * 4; for (int c = 0; c < numClass; ++c) { std::vector selected; const auto scorePtr = score->readMap() + (b * numClass + c) * numBoxes; NonMaxSuppressionSingleClasssImpl(boxesPtr, scorePtr, numBoxes, maxDetections, iouThreshold, scoreThreshold, &selected); for (int i = 0; i < selected.size(); ++i) { if (onnxFormat) { outputData.push_back(b); outputData.push_back(c); } outputData.push_back(selected[i]); } } } Variable::Info outInfo; outInfo.order = info->order; outInfo.type = halide_type_of(); if (onnxFormat) { outInfo.dim.assign({(int)outputData.size() / 3, 3}); } else { outInfo.dim.assign({(int)outputData.size()}); } outInfo.syncSize(); VARPS outputs; outputs.push_back(Variable::create(Expr::create(std::move(outInfo), outputData.data(), VARP::CONSTANT))); return outputs; } Module* NMSModule::clone(CloneContext* ctx) const { NMSModule* module(new NMSModule); return this->cloneBaseTo(ctx, module); } } }