chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,34 @@
|
||||
#include <opencv2/videoio.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/imgproc.hpp>
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
cv::VideoCapture cap;
|
||||
if (argc > 1) cap.open(argv[1]);
|
||||
else cap.open(0);
|
||||
CV_Assert(cap.isOpened());
|
||||
|
||||
cv::GMat in;
|
||||
cv::GMat vga = cv::gapi::resize(in, cv::Size(), 0.5, 0.5);
|
||||
cv::GMat gray = cv::gapi::BGR2Gray(vga);
|
||||
cv::GMat blurred = cv::gapi::blur(gray, cv::Size(5,5));
|
||||
cv::GMat edges = cv::gapi::Canny(blurred, 32, 128, 3);
|
||||
cv::GMat b,g,r;
|
||||
std::tie(b,g,r) = cv::gapi::split3(vga);
|
||||
cv::GMat out = cv::gapi::merge3(b, g | edges, r);
|
||||
cv::GComputation ac(in, out);
|
||||
|
||||
cv::Mat input_frame;
|
||||
cv::Mat output_frame;
|
||||
CV_Assert(cap.read(input_frame));
|
||||
do
|
||||
{
|
||||
ac.apply(input_frame, output_frame);
|
||||
cv::imshow("output", output_frame);
|
||||
} while (cap.read(input_frame) && cv::waitKey(30) < 0);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
%YAML:1.0
|
||||
|
||||
# Application running time in milliseconds: integer.
|
||||
work_time: 2000
|
||||
|
||||
Pipelines:
|
||||
PL1:
|
||||
source:
|
||||
name: 'Src'
|
||||
latency: 33.0
|
||||
output:
|
||||
dims: [1, 3, 1280, 720]
|
||||
precision: 'U8'
|
||||
|
||||
nodes:
|
||||
- name: 'PP'
|
||||
type: 'Dummy'
|
||||
time: 0.2
|
||||
output:
|
||||
dims: [1, 3, 300, 300]
|
||||
precision: 'U8'
|
||||
|
||||
- name: 'Infer'
|
||||
type: 'Infer'
|
||||
xml: 'face-detection-retail-0004.xml'
|
||||
bin: 'face-detection-retail-0004.bin'
|
||||
device: 'CPU'
|
||||
input_layers:
|
||||
- 'data'
|
||||
output_layers:
|
||||
- 'detection_out'
|
||||
|
||||
edges:
|
||||
- from: 'Src'
|
||||
to: 'PP'
|
||||
- from: 'PP'
|
||||
to: 'Infer'
|
||||
|
||||
# Path to the dump file (*.dot)'
|
||||
dump: 'pl1.dot'
|
||||
|
||||
PL2:
|
||||
source:
|
||||
name: 'Src'
|
||||
latency: 50.0
|
||||
output:
|
||||
dims: [1, 3, 1280, 720]
|
||||
precision: 'U8'
|
||||
|
||||
nodes:
|
||||
- name: 'M1_PP'
|
||||
type: 'Dummy'
|
||||
time: 0.2
|
||||
output:
|
||||
dims: [1, 3, 300, 300]
|
||||
precision: 'U8'
|
||||
|
||||
- name: 'M1_Infer'
|
||||
type: 'Infer'
|
||||
xml: 'face-detection-retail-0004.xml'
|
||||
bin: 'face-detection-retail-0004.bin'
|
||||
device: 'CPU'
|
||||
input_layers:
|
||||
- 'data'
|
||||
output_layers:
|
||||
- 'detection_out'
|
||||
|
||||
- name: 'M2_PP'
|
||||
type: 'Dummy'
|
||||
time: 0.2
|
||||
output:
|
||||
dims: [1, 3, 300, 300]
|
||||
precision: 'U8'
|
||||
|
||||
- name: 'M2_Infer'
|
||||
type: 'Infer'
|
||||
xml: 'face-detection-retail-0004.xml'
|
||||
bin: 'face-detection-retail-0004.bin'
|
||||
device: 'CPU'
|
||||
input_layers:
|
||||
- 'data'
|
||||
output_layers:
|
||||
- 'detection_out'
|
||||
|
||||
- name: 'M3_PP'
|
||||
type: 'Dummy'
|
||||
time: 0.2
|
||||
output:
|
||||
dims: [1, 3, 300, 300]
|
||||
precision: 'U8'
|
||||
|
||||
- name: 'M3_Infer'
|
||||
type: 'Infer'
|
||||
xml: 'face-detection-retail-0004.xml'
|
||||
bin: 'face-detection-retail-0004.bin'
|
||||
device: 'CPU'
|
||||
input_layers:
|
||||
- 'data'
|
||||
output_layers:
|
||||
- 'detection_out'
|
||||
|
||||
- name: 'M4_PP'
|
||||
type: 'Dummy'
|
||||
time: 0.2
|
||||
output:
|
||||
dims: [1, 3, 300, 300]
|
||||
precision: 'U8'
|
||||
|
||||
- name: 'M4_Infer'
|
||||
type: 'Infer'
|
||||
xml: 'face-detection-retail-0004.xml'
|
||||
bin: 'face-detection-retail-0004.bin'
|
||||
device: 'CPU'
|
||||
input_layers:
|
||||
- 'data'
|
||||
output_layers:
|
||||
- 'detection_out'
|
||||
|
||||
- name: 'M5_PP'
|
||||
type: 'Dummy'
|
||||
time: 0.2
|
||||
output:
|
||||
dims: [1, 3, 300, 300]
|
||||
precision: 'U8'
|
||||
|
||||
- name: 'M5_Infer'
|
||||
type: 'Infer'
|
||||
xml: 'face-detection-retail-0004.xml'
|
||||
bin: 'face-detection-retail-0004.bin'
|
||||
device: 'CPU'
|
||||
input_layers:
|
||||
- 'data'
|
||||
output_layers:
|
||||
- 'detection_out'
|
||||
|
||||
edges:
|
||||
- from: 'Src'
|
||||
to: 'M1_PP'
|
||||
- from: 'M1_PP'
|
||||
to: 'M1_Infer'
|
||||
- from: 'M1_Infer'
|
||||
to: 'M2_PP'
|
||||
- from: 'M2_PP'
|
||||
to: 'M2_Infer'
|
||||
- from: 'M2_Infer'
|
||||
to: 'M3_PP'
|
||||
- from: 'M3_PP'
|
||||
to: 'M3_Infer'
|
||||
- from: 'M3_Infer'
|
||||
to: 'M4_PP'
|
||||
- from: 'M4_PP'
|
||||
to: 'M4_Infer'
|
||||
- from: 'M4_Infer'
|
||||
to: 'M5_PP'
|
||||
- from: 'M5_PP'
|
||||
to: 'M5_Infer'
|
||||
|
||||
dump: 'pl2.dot'
|
||||
|
||||
PL3:
|
||||
source:
|
||||
name: 'Src'
|
||||
latency: 33.0
|
||||
output:
|
||||
dims: [1, 3, 1280, 720]
|
||||
precision: 'U8'
|
||||
|
||||
nodes:
|
||||
- name: 'PP'
|
||||
type: 'Dummy'
|
||||
time: 0.2
|
||||
output:
|
||||
dims: [1, 3, 300, 300]
|
||||
precision: 'U8'
|
||||
|
||||
- name: 'Infer'
|
||||
type: 'Infer'
|
||||
xml: 'face-detection-retail-0004.xml'
|
||||
bin: 'face-detection-retail-0004.bin'
|
||||
device: 'CPU'
|
||||
input_layers:
|
||||
- 'data'
|
||||
output_layers:
|
||||
- 'detection_out'
|
||||
|
||||
edges:
|
||||
- from: 'Src'
|
||||
to: 'PP'
|
||||
- from: 'PP'
|
||||
to: 'Infer'
|
||||
|
||||
dump: 'pl3.dot'
|
||||
@@ -0,0 +1,56 @@
|
||||
#include <opencv2/imgproc.hpp> // cv::FONT*, cv::LINE*, cv::FILLED
|
||||
#include <opencv2/highgui.hpp> // imwrite
|
||||
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/render.hpp>
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
if (argc < 2) {
|
||||
std::cerr << "Filename required" << std::endl;
|
||||
return 1;
|
||||
}
|
||||
|
||||
const auto font = cv::FONT_HERSHEY_DUPLEX;
|
||||
const auto blue = cv::Scalar{ 255, 0, 0}; // B/G/R
|
||||
const auto green = cv::Scalar{ 0, 255, 0};
|
||||
const auto coral = cv::Scalar{0x81,0x81,0xF1};
|
||||
const auto white = cv::Scalar{ 255, 255, 255};
|
||||
cv::Mat test(cv::Size(480, 160), CV_8UC3, white);
|
||||
|
||||
namespace draw = cv::gapi::wip::draw;
|
||||
std::vector<draw::Prim> prims;
|
||||
prims.emplace_back(draw::Circle{ // CIRCLE primitive
|
||||
{400,72}, // Position (a cv::Point)
|
||||
32, // Radius
|
||||
coral, // Color
|
||||
cv::FILLED, // Thickness/fill type
|
||||
cv::LINE_8, // Line type
|
||||
0 // Shift
|
||||
});
|
||||
prims.emplace_back(draw::Text{ // TEXT primitive
|
||||
"Hello from G-API!", // Text
|
||||
{64,96}, // Position (a cv::Point)
|
||||
font, // Font
|
||||
1.0, // Scale (size)
|
||||
blue, // Color
|
||||
2, // Thickness
|
||||
cv::LINE_8, // Line type
|
||||
false // Bottom left origin flag
|
||||
});
|
||||
prims.emplace_back(draw::Rect{ // RECTANGLE primitive
|
||||
{16,48,400,72}, // Geometry (a cv::Rect)
|
||||
green, // Color
|
||||
2, // Thickness
|
||||
cv::LINE_8, // Line type
|
||||
0 // Shift
|
||||
});
|
||||
prims.emplace_back(draw::Mosaic{ // MOSAIC primitive
|
||||
{320,96,128,32}, // Geometry (a cv::Rect)
|
||||
16, // Cell size
|
||||
0 // Decimation
|
||||
});
|
||||
draw::render(test, prims);
|
||||
cv::imwrite(argv[1], test);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,733 @@
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <cmath>
|
||||
#include <iostream>
|
||||
#include <limits>
|
||||
#include <numeric>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/imgproc.hpp>
|
||||
#include <opencv2/gapi/cpu/gcpukernel.hpp>
|
||||
#include <opencv2/gapi/infer.hpp>
|
||||
#include <opencv2/gapi/infer/ie.hpp>
|
||||
#include <opencv2/gapi/streaming/cap.hpp>
|
||||
#include <opencv2/gapi/gopaque.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
|
||||
const std::string about =
|
||||
"This is an OpenCV-based version of OMZ MTCNN Face Detection example";
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ input | | Path to the input video file }"
|
||||
"{ mtcnnpm | mtcnn-p.xml | Path to OpenVINO MTCNN P (Proposal) detection model (.xml)}"
|
||||
"{ mtcnnpd | CPU | Target device for the MTCNN P (e.g. CPU, GPU, VPU, ...) }"
|
||||
"{ mtcnnrm | mtcnn-r.xml | Path to OpenVINO MTCNN R (Refinement) detection model (.xml)}"
|
||||
"{ mtcnnrd | CPU | Target device for the MTCNN R (e.g. CPU, GPU, VPU, ...) }"
|
||||
"{ mtcnnom | mtcnn-o.xml | Path to OpenVINO MTCNN O (Output) detection model (.xml)}"
|
||||
"{ mtcnnod | CPU | Target device for the MTCNN O (e.g. CPU, GPU, VPU, ...) }"
|
||||
"{ thrp | 0.6 | MTCNN P confidence threshold}"
|
||||
"{ thrr | 0.7 | MTCNN R confidence threshold}"
|
||||
"{ thro | 0.7 | MTCNN O confidence threshold}"
|
||||
"{ half_scale | false | MTCNN P use half scale pyramid}"
|
||||
"{ queue_capacity | 1 | Streaming executor queue capacity. Calculated automatically if 0}"
|
||||
;
|
||||
|
||||
namespace {
|
||||
std::string weights_path(const std::string& model_path) {
|
||||
const auto EXT_LEN = 4u;
|
||||
const auto sz = model_path.size();
|
||||
CV_Assert(sz > EXT_LEN);
|
||||
|
||||
const auto ext = model_path.substr(sz - EXT_LEN);
|
||||
CV_Assert(cv::toLowerCase(ext) == ".xml");
|
||||
return model_path.substr(0u, sz - EXT_LEN) + ".bin";
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
} // anonymous namespace
|
||||
|
||||
namespace custom {
|
||||
namespace {
|
||||
|
||||
// Define custom structures and operations
|
||||
#define NUM_REGRESSIONS 4
|
||||
#define NUM_PTS 5
|
||||
|
||||
struct BBox {
|
||||
int x1;
|
||||
int y1;
|
||||
int x2;
|
||||
int y2;
|
||||
|
||||
cv::Rect getRect() const { return cv::Rect(x1,
|
||||
y1,
|
||||
x2 - x1,
|
||||
y2 - y1); }
|
||||
|
||||
BBox getSquare() const {
|
||||
BBox bbox;
|
||||
float bboxWidth = static_cast<float>(x2 - x1);
|
||||
float bboxHeight = static_cast<float>(y2 - y1);
|
||||
float side = std::max(bboxWidth, bboxHeight);
|
||||
bbox.x1 = static_cast<int>(static_cast<float>(x1) + (bboxWidth - side) * 0.5f);
|
||||
bbox.y1 = static_cast<int>(static_cast<float>(y1) + (bboxHeight - side) * 0.5f);
|
||||
bbox.x2 = static_cast<int>(static_cast<float>(bbox.x1) + side);
|
||||
bbox.y2 = static_cast<int>(static_cast<float>(bbox.y1) + side);
|
||||
return bbox;
|
||||
}
|
||||
};
|
||||
|
||||
struct Face {
|
||||
BBox bbox;
|
||||
float score;
|
||||
std::array<float, NUM_REGRESSIONS> regression;
|
||||
std::array<float, 2 * NUM_PTS> ptsCoords;
|
||||
|
||||
static void applyRegression(std::vector<Face>& faces, bool addOne = false) {
|
||||
for (auto& face : faces) {
|
||||
float bboxWidth =
|
||||
face.bbox.x2 - face.bbox.x1 + static_cast<float>(addOne);
|
||||
float bboxHeight =
|
||||
face.bbox.y2 - face.bbox.y1 + static_cast<float>(addOne);
|
||||
face.bbox.x1 = static_cast<int>(static_cast<float>(face.bbox.x1) + (face.regression[1] * bboxWidth));
|
||||
face.bbox.y1 = static_cast<int>(static_cast<float>(face.bbox.y1) + (face.regression[0] * bboxHeight));
|
||||
face.bbox.x2 = static_cast<int>(static_cast<float>(face.bbox.x2) + (face.regression[3] * bboxWidth));
|
||||
face.bbox.y2 = static_cast<int>(static_cast<float>(face.bbox.y2) + (face.regression[2] * bboxHeight));
|
||||
}
|
||||
}
|
||||
|
||||
static void bboxes2Squares(std::vector<Face>& faces) {
|
||||
for (auto& face : faces) {
|
||||
face.bbox = face.bbox.getSquare();
|
||||
}
|
||||
}
|
||||
|
||||
static std::vector<Face> runNMS(std::vector<Face>& faces, const float threshold,
|
||||
const bool useMin = false) {
|
||||
std::vector<Face> facesNMS;
|
||||
if (faces.empty()) {
|
||||
return facesNMS;
|
||||
}
|
||||
|
||||
std::sort(faces.begin(), faces.end(), [](const Face& f1, const Face& f2) {
|
||||
return f1.score > f2.score;
|
||||
});
|
||||
|
||||
std::vector<int> indices(faces.size());
|
||||
std::iota(indices.begin(), indices.end(), 0);
|
||||
|
||||
while (indices.size() > 0) {
|
||||
const int idx = indices[0];
|
||||
facesNMS.push_back(faces[idx]);
|
||||
std::vector<int> tmpIndices = indices;
|
||||
indices.clear();
|
||||
const float area1 = static_cast<float>(faces[idx].bbox.x2 - faces[idx].bbox.x1 + 1) *
|
||||
static_cast<float>(faces[idx].bbox.y2 - faces[idx].bbox.y1 + 1);
|
||||
for (size_t i = 1; i < tmpIndices.size(); ++i) {
|
||||
int tmpIdx = tmpIndices[i];
|
||||
const float interX1 = static_cast<float>(std::max(faces[idx].bbox.x1, faces[tmpIdx].bbox.x1));
|
||||
const float interY1 = static_cast<float>(std::max(faces[idx].bbox.y1, faces[tmpIdx].bbox.y1));
|
||||
const float interX2 = static_cast<float>(std::min(faces[idx].bbox.x2, faces[tmpIdx].bbox.x2));
|
||||
const float interY2 = static_cast<float>(std::min(faces[idx].bbox.y2, faces[tmpIdx].bbox.y2));
|
||||
|
||||
const float bboxWidth = std::max(0.0f, (interX2 - interX1 + 1));
|
||||
const float bboxHeight = std::max(0.0f, (interY2 - interY1 + 1));
|
||||
|
||||
const float interArea = bboxWidth * bboxHeight;
|
||||
const float area2 = static_cast<float>(faces[tmpIdx].bbox.x2 - faces[tmpIdx].bbox.x1 + 1) *
|
||||
static_cast<float>(faces[tmpIdx].bbox.y2 - faces[tmpIdx].bbox.y1 + 1);
|
||||
float overlap = 0.0;
|
||||
if (useMin) {
|
||||
overlap = interArea / std::min(area1, area2);
|
||||
} else {
|
||||
overlap = interArea / (area1 + area2 - interArea);
|
||||
}
|
||||
if (overlap <= threshold) {
|
||||
indices.push_back(tmpIdx);
|
||||
}
|
||||
}
|
||||
}
|
||||
return facesNMS;
|
||||
}
|
||||
};
|
||||
|
||||
const float P_NET_WINDOW_SIZE = 12.0f;
|
||||
|
||||
std::vector<Face> buildFaces(const cv::Mat& scores,
|
||||
const cv::Mat& regressions,
|
||||
const float scaleFactor,
|
||||
const float threshold) {
|
||||
|
||||
auto w = scores.size[3];
|
||||
auto h = scores.size[2];
|
||||
auto size = w * h;
|
||||
|
||||
const float* scores_data = scores.ptr<float>();
|
||||
scores_data += size;
|
||||
|
||||
const float* reg_data = regressions.ptr<float>();
|
||||
|
||||
auto out_side = std::max(h, w);
|
||||
auto in_side = 2 * out_side + 11;
|
||||
float stride = 0.0f;
|
||||
if (out_side != 1)
|
||||
{
|
||||
stride = static_cast<float>(in_side - P_NET_WINDOW_SIZE) / static_cast<float>(out_side - 1);
|
||||
}
|
||||
|
||||
std::vector<Face> boxes;
|
||||
|
||||
for (int i = 0; i < size; i++) {
|
||||
if (scores_data[i] >= (threshold)) {
|
||||
float y = static_cast<float>(i / w);
|
||||
float x = static_cast<float>(i - w * y);
|
||||
|
||||
Face faceInfo;
|
||||
BBox& faceBox = faceInfo.bbox;
|
||||
|
||||
faceBox.x1 = std::max(0, static_cast<int>((x * stride) / scaleFactor));
|
||||
faceBox.y1 = std::max(0, static_cast<int>((y * stride) / scaleFactor));
|
||||
faceBox.x2 = static_cast<int>((x * stride + P_NET_WINDOW_SIZE - 1.0f) / scaleFactor);
|
||||
faceBox.y2 = static_cast<int>((y * stride + P_NET_WINDOW_SIZE - 1.0f) / scaleFactor);
|
||||
faceInfo.regression[0] = reg_data[i];
|
||||
faceInfo.regression[1] = reg_data[i + size];
|
||||
faceInfo.regression[2] = reg_data[i + 2 * size];
|
||||
faceInfo.regression[3] = reg_data[i + 3 * size];
|
||||
faceInfo.score = scores_data[i];
|
||||
boxes.push_back(faceInfo);
|
||||
}
|
||||
}
|
||||
|
||||
return boxes;
|
||||
}
|
||||
|
||||
// Define networks for this sample
|
||||
using GMat2 = std::tuple<cv::GMat, cv::GMat>;
|
||||
using GMat3 = std::tuple<cv::GMat, cv::GMat, cv::GMat>;
|
||||
using GMats = cv::GArray<cv::GMat>;
|
||||
using GRects = cv::GArray<cv::Rect>;
|
||||
using GSize = cv::GOpaque<cv::Size>;
|
||||
|
||||
G_API_NET(MTCNNRefinement,
|
||||
<GMat2(cv::GMat)>,
|
||||
"sample.custom.mtcnn_refinement");
|
||||
|
||||
G_API_NET(MTCNNOutput,
|
||||
<GMat3(cv::GMat)>,
|
||||
"sample.custom.mtcnn_output");
|
||||
|
||||
using GFaces = cv::GArray<Face>;
|
||||
G_API_OP(BuildFaces,
|
||||
<GFaces(cv::GMat, cv::GMat, float, float)>,
|
||||
"sample.custom.mtcnn.build_faces") {
|
||||
static cv::GArrayDesc outMeta(const cv::GMatDesc&,
|
||||
const cv::GMatDesc&,
|
||||
const float,
|
||||
const float) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
G_API_OP(RunNMS,
|
||||
<GFaces(GFaces, float, bool)>,
|
||||
"sample.custom.mtcnn.run_nms") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc&,
|
||||
const float, const bool) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
G_API_OP(AccumulatePyramidOutputs,
|
||||
<GFaces(GFaces, GFaces)>,
|
||||
"sample.custom.mtcnn.accumulate_pyramid_outputs") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc&,
|
||||
const cv::GArrayDesc&) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
G_API_OP(ApplyRegression,
|
||||
<GFaces(GFaces, bool)>,
|
||||
"sample.custom.mtcnn.apply_regression") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc&, const bool) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
G_API_OP(BBoxesToSquares,
|
||||
<GFaces(GFaces)>,
|
||||
"sample.custom.mtcnn.bboxes_to_squares") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc&) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
G_API_OP(R_O_NetPreProcGetROIs,
|
||||
<GRects(GFaces, GSize)>,
|
||||
"sample.custom.mtcnn.bboxes_r_o_net_preproc_get_rois") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc&, const cv::GOpaqueDesc&) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
G_API_OP(RNetPostProc,
|
||||
<GFaces(GFaces, GMats, GMats, float)>,
|
||||
"sample.custom.mtcnn.rnet_postproc") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc&,
|
||||
const cv::GArrayDesc&,
|
||||
const cv::GArrayDesc&,
|
||||
const float) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
G_API_OP(ONetPostProc,
|
||||
<GFaces(GFaces, GMats, GMats, GMats, float)>,
|
||||
"sample.custom.mtcnn.onet_postproc") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc&,
|
||||
const cv::GArrayDesc&,
|
||||
const cv::GArrayDesc&,
|
||||
const cv::GArrayDesc&,
|
||||
const float) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
G_API_OP(SwapFaces,
|
||||
<GFaces(GFaces)>,
|
||||
"sample.custom.mtcnn.swap_faces") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc&) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
//Custom kernels implementation
|
||||
GAPI_OCV_KERNEL(OCVBuildFaces, BuildFaces) {
|
||||
static void run(const cv::Mat & in_scores,
|
||||
const cv::Mat & in_regresssions,
|
||||
const float scaleFactor,
|
||||
const float threshold,
|
||||
std::vector<Face> &out_faces) {
|
||||
out_faces = buildFaces(in_scores, in_regresssions, scaleFactor, threshold);
|
||||
}
|
||||
};// GAPI_OCV_KERNEL(BuildFaces)
|
||||
|
||||
GAPI_OCV_KERNEL(OCVRunNMS, RunNMS) {
|
||||
static void run(const std::vector<Face> &in_faces,
|
||||
const float threshold,
|
||||
const bool useMin,
|
||||
std::vector<Face> &out_faces) {
|
||||
std::vector<Face> in_faces_copy = in_faces;
|
||||
out_faces = Face::runNMS(in_faces_copy, threshold, useMin);
|
||||
}
|
||||
};// GAPI_OCV_KERNEL(RunNMS)
|
||||
|
||||
GAPI_OCV_KERNEL(OCVAccumulatePyramidOutputs, AccumulatePyramidOutputs) {
|
||||
static void run(const std::vector<Face> &total_faces,
|
||||
const std::vector<Face> &in_faces,
|
||||
std::vector<Face> &out_faces) {
|
||||
out_faces = total_faces;
|
||||
out_faces.insert(out_faces.end(), in_faces.begin(), in_faces.end());
|
||||
}
|
||||
};// GAPI_OCV_KERNEL(AccumulatePyramidOutputs)
|
||||
|
||||
GAPI_OCV_KERNEL(OCVApplyRegression, ApplyRegression) {
|
||||
static void run(const std::vector<Face> &in_faces,
|
||||
const bool addOne,
|
||||
std::vector<Face> &out_faces) {
|
||||
std::vector<Face> in_faces_copy = in_faces;
|
||||
Face::applyRegression(in_faces_copy, addOne);
|
||||
out_faces.clear();
|
||||
out_faces.insert(out_faces.end(), in_faces_copy.begin(), in_faces_copy.end());
|
||||
}
|
||||
};// GAPI_OCV_KERNEL(ApplyRegression)
|
||||
|
||||
GAPI_OCV_KERNEL(OCVBBoxesToSquares, BBoxesToSquares) {
|
||||
static void run(const std::vector<Face> &in_faces,
|
||||
std::vector<Face> &out_faces) {
|
||||
std::vector<Face> in_faces_copy = in_faces;
|
||||
Face::bboxes2Squares(in_faces_copy);
|
||||
out_faces.clear();
|
||||
out_faces.insert(out_faces.end(), in_faces_copy.begin(), in_faces_copy.end());
|
||||
}
|
||||
};// GAPI_OCV_KERNEL(BBoxesToSquares)
|
||||
|
||||
GAPI_OCV_KERNEL(OCVR_O_NetPreProcGetROIs, R_O_NetPreProcGetROIs) {
|
||||
static void run(const std::vector<Face> &in_faces,
|
||||
const cv::Size & in_image_size,
|
||||
std::vector<cv::Rect> &outs) {
|
||||
outs.clear();
|
||||
for (const auto& face : in_faces) {
|
||||
cv::Rect tmp_rect = face.bbox.getRect();
|
||||
//Compare to transposed sizes width<->height
|
||||
tmp_rect &= cv::Rect(tmp_rect.x, tmp_rect.y, in_image_size.height - tmp_rect.x, in_image_size.width - tmp_rect.y) &
|
||||
cv::Rect(0, 0, in_image_size.height, in_image_size.width);
|
||||
outs.push_back(tmp_rect);
|
||||
}
|
||||
}
|
||||
};// GAPI_OCV_KERNEL(R_O_NetPreProcGetROIs)
|
||||
|
||||
|
||||
GAPI_OCV_KERNEL(OCVRNetPostProc, RNetPostProc) {
|
||||
static void run(const std::vector<Face> &in_faces,
|
||||
const std::vector<cv::Mat> &in_scores,
|
||||
const std::vector<cv::Mat> &in_regresssions,
|
||||
const float threshold,
|
||||
std::vector<Face> &out_faces) {
|
||||
out_faces.clear();
|
||||
for (unsigned int k = 0; k < in_faces.size(); ++k) {
|
||||
const float* scores_data = in_scores[k].ptr<float>();
|
||||
const float* reg_data = in_regresssions[k].ptr<float>();
|
||||
if (scores_data[1] >= threshold) {
|
||||
Face info = in_faces[k];
|
||||
info.score = scores_data[1];
|
||||
std::copy_n(reg_data, NUM_REGRESSIONS, info.regression.begin());
|
||||
out_faces.push_back(info);
|
||||
}
|
||||
}
|
||||
}
|
||||
};// GAPI_OCV_KERNEL(RNetPostProc)
|
||||
|
||||
GAPI_OCV_KERNEL(OCVONetPostProc, ONetPostProc) {
|
||||
static void run(const std::vector<Face> &in_faces,
|
||||
const std::vector<cv::Mat> &in_scores,
|
||||
const std::vector<cv::Mat> &in_regresssions,
|
||||
const std::vector<cv::Mat> &in_landmarks,
|
||||
const float threshold,
|
||||
std::vector<Face> &out_faces) {
|
||||
out_faces.clear();
|
||||
for (unsigned int k = 0; k < in_faces.size(); ++k) {
|
||||
const float* scores_data = in_scores[k].ptr<float>();
|
||||
const float* reg_data = in_regresssions[k].ptr<float>();
|
||||
const float* landmark_data = in_landmarks[k].ptr<float>();
|
||||
if (scores_data[1] >= threshold) {
|
||||
Face info = in_faces[k];
|
||||
info.score = scores_data[1];
|
||||
for (size_t i = 0; i < 4; ++i) {
|
||||
info.regression[i] = reg_data[i];
|
||||
}
|
||||
float w = info.bbox.x2 - info.bbox.x1 + 1.0f;
|
||||
float h = info.bbox.y2 - info.bbox.y1 + 1.0f;
|
||||
|
||||
for (size_t p = 0; p < NUM_PTS; ++p) {
|
||||
info.ptsCoords[2 * p] =
|
||||
info.bbox.x1 + static_cast<float>(landmark_data[NUM_PTS + p]) * w - 1;
|
||||
info.ptsCoords[2 * p + 1] = info.bbox.y1 + static_cast<float>(landmark_data[p]) * h - 1;
|
||||
}
|
||||
|
||||
out_faces.push_back(info);
|
||||
}
|
||||
}
|
||||
}
|
||||
};// GAPI_OCV_KERNEL(ONetPostProc)
|
||||
|
||||
GAPI_OCV_KERNEL(OCVSwapFaces, SwapFaces) {
|
||||
static void run(const std::vector<Face> &in_faces,
|
||||
std::vector<Face> &out_faces) {
|
||||
std::vector<Face> in_faces_copy = in_faces;
|
||||
out_faces.clear();
|
||||
if (!in_faces_copy.empty()) {
|
||||
for (size_t i = 0; i < in_faces_copy.size(); ++i) {
|
||||
std::swap(in_faces_copy[i].bbox.x1, in_faces_copy[i].bbox.y1);
|
||||
std::swap(in_faces_copy[i].bbox.x2, in_faces_copy[i].bbox.y2);
|
||||
for (size_t p = 0; p < NUM_PTS; ++p) {
|
||||
std::swap(in_faces_copy[i].ptsCoords[2 * p], in_faces_copy[i].ptsCoords[2 * p + 1]);
|
||||
}
|
||||
}
|
||||
out_faces = in_faces_copy;
|
||||
}
|
||||
}
|
||||
};// GAPI_OCV_KERNEL(SwapFaces)
|
||||
|
||||
} // anonymous namespace
|
||||
} // namespace custom
|
||||
|
||||
namespace vis {
|
||||
namespace {
|
||||
void bbox(const cv::Mat& m, const cv::Rect& rc) {
|
||||
cv::rectangle(m, rc, cv::Scalar{ 0,255,0 }, 2, cv::LINE_8, 0);
|
||||
};
|
||||
|
||||
using rectPoints = std::pair<cv::Rect, std::vector<cv::Point>>;
|
||||
|
||||
static cv::Mat drawRectsAndPoints(const cv::Mat& img,
|
||||
const std::vector<rectPoints> data) {
|
||||
cv::Mat outImg;
|
||||
img.copyTo(outImg);
|
||||
|
||||
for (const auto& el : data) {
|
||||
vis::bbox(outImg, el.first);
|
||||
auto pts = el.second;
|
||||
for (size_t i = 0; i < pts.size(); ++i) {
|
||||
cv::circle(outImg, pts[i], 3, cv::Scalar(0, 255, 255), 1);
|
||||
}
|
||||
}
|
||||
return outImg;
|
||||
}
|
||||
} // anonymous namespace
|
||||
} // namespace vis
|
||||
|
||||
|
||||
//Infer helper function
|
||||
namespace {
|
||||
static inline std::tuple<cv::GMat, cv::GMat> run_mtcnn_p(cv::GMat &in, const std::string &id) {
|
||||
cv::GInferInputs inputs;
|
||||
inputs["data"] = in;
|
||||
auto outputs = cv::gapi::infer<cv::gapi::Generic>(id, inputs);
|
||||
auto regressions = outputs.at("conv4-2");
|
||||
auto scores = outputs.at("prob1");
|
||||
return std::make_tuple(regressions, scores);
|
||||
}
|
||||
|
||||
static inline std::string get_pnet_level_name(const cv::Size &in_size) {
|
||||
return "MTCNNProposal_" + std::to_string(in_size.width) + "x" + std::to_string(in_size.height);
|
||||
}
|
||||
|
||||
int calculate_scales(const cv::Size &input_size, std::vector<double> &out_scales, std::vector<cv::Size> &out_sizes ) {
|
||||
//calculate multi - scale and limit the maximum side to 1000
|
||||
//pr_scale: limit the maximum side to 1000, < 1.0
|
||||
double pr_scale = 1.0;
|
||||
double h = static_cast<double>(input_size.height);
|
||||
double w = static_cast<double>(input_size.width);
|
||||
if (std::min(w, h) > 1000)
|
||||
{
|
||||
pr_scale = 1000.0 / std::min(h, w);
|
||||
w = w * pr_scale;
|
||||
h = h * pr_scale;
|
||||
}
|
||||
else if (std::max(w, h) < 1000)
|
||||
{
|
||||
w = w * pr_scale;
|
||||
h = h * pr_scale;
|
||||
}
|
||||
//multi - scale
|
||||
out_scales.clear();
|
||||
out_sizes.clear();
|
||||
const double factor = 0.709;
|
||||
int factor_count = 0;
|
||||
double minl = std::min(h, w);
|
||||
while (minl >= 12)
|
||||
{
|
||||
const double current_scale = pr_scale * std::pow(factor, factor_count);
|
||||
cv::Size current_size(static_cast<int>(static_cast<double>(input_size.width) * current_scale),
|
||||
static_cast<int>(static_cast<double>(input_size.height) * current_scale));
|
||||
out_scales.push_back(current_scale);
|
||||
out_sizes.push_back(current_size);
|
||||
minl *= factor;
|
||||
factor_count += 1;
|
||||
}
|
||||
return factor_count;
|
||||
}
|
||||
|
||||
int calculate_half_scales(const cv::Size &input_size, std::vector<double>& out_scales, std::vector<cv::Size>& out_sizes) {
|
||||
double pr_scale = 0.5;
|
||||
const double h = static_cast<double>(input_size.height);
|
||||
const double w = static_cast<double>(input_size.width);
|
||||
//multi - scale
|
||||
out_scales.clear();
|
||||
out_sizes.clear();
|
||||
const double factor = 0.5;
|
||||
int factor_count = 0;
|
||||
double minl = std::min(h, w);
|
||||
while (minl >= 12.0*2.0)
|
||||
{
|
||||
const double current_scale = pr_scale;
|
||||
cv::Size current_size(static_cast<int>(static_cast<double>(input_size.width) * current_scale),
|
||||
static_cast<int>(static_cast<double>(input_size.height) * current_scale));
|
||||
out_scales.push_back(current_scale);
|
||||
out_sizes.push_back(current_size);
|
||||
minl *= factor;
|
||||
factor_count += 1;
|
||||
pr_scale *= 0.5;
|
||||
}
|
||||
return factor_count;
|
||||
}
|
||||
|
||||
const int MAX_PYRAMID_LEVELS = 13;
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
} // anonymous namespace
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
cmd.about(about);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
const auto input_file_name = cmd.get<std::string>("input");
|
||||
const auto model_path_p = cmd.get<std::string>("mtcnnpm");
|
||||
const auto target_dev_p = cmd.get<std::string>("mtcnnpd");
|
||||
const auto conf_thresh_p = cmd.get<float>("thrp");
|
||||
const auto model_path_r = cmd.get<std::string>("mtcnnrm");
|
||||
const auto target_dev_r = cmd.get<std::string>("mtcnnrd");
|
||||
const auto conf_thresh_r = cmd.get<float>("thrr");
|
||||
const auto model_path_o = cmd.get<std::string>("mtcnnom");
|
||||
const auto target_dev_o = cmd.get<std::string>("mtcnnod");
|
||||
const auto conf_thresh_o = cmd.get<float>("thro");
|
||||
const auto use_half_scale = cmd.get<bool>("half_scale");
|
||||
const auto streaming_queue_capacity = cmd.get<unsigned int>("queue_capacity");
|
||||
|
||||
std::vector<cv::Size> level_size;
|
||||
std::vector<double> scales;
|
||||
//MTCNN input size
|
||||
cv::VideoCapture cap;
|
||||
cap.open(input_file_name);
|
||||
if (!cap.isOpened())
|
||||
CV_Assert(false);
|
||||
auto in_rsz = cv::Size{ static_cast<int>(cap.get(cv::CAP_PROP_FRAME_WIDTH)),
|
||||
static_cast<int>(cap.get(cv::CAP_PROP_FRAME_HEIGHT)) };
|
||||
//Calculate scales, number of pyramid levels and sizes for PNet pyramid
|
||||
auto pyramid_levels = use_half_scale ? calculate_half_scales(in_rsz, scales, level_size) :
|
||||
calculate_scales(in_rsz, scales, level_size);
|
||||
CV_Assert(pyramid_levels <= MAX_PYRAMID_LEVELS);
|
||||
|
||||
//Proposal part of MTCNN graph
|
||||
//Preprocessing BGR2RGB + transpose (NCWH is expected instead of NCHW)
|
||||
cv::GMat in_original;
|
||||
cv::GMat in_originalRGB = cv::gapi::BGR2RGB(in_original);
|
||||
cv::GMat in_transposedRGB = cv::gapi::transpose(in_originalRGB);
|
||||
cv::GOpaque<cv::Size> in_sz = cv::gapi::streaming::size(in_original);
|
||||
cv::GMat regressions[MAX_PYRAMID_LEVELS];
|
||||
cv::GMat scores[MAX_PYRAMID_LEVELS];
|
||||
cv::GArray<custom::Face> nms_p_faces[MAX_PYRAMID_LEVELS];
|
||||
cv::GArray<custom::Face> total_faces[MAX_PYRAMID_LEVELS];
|
||||
|
||||
//The very first PNet pyramid layer to init total_faces[0]
|
||||
std::tie(regressions[0], scores[0]) = run_mtcnn_p(in_transposedRGB, get_pnet_level_name(level_size[0]));
|
||||
cv::GArray<custom::Face> faces0 = custom::BuildFaces::on(scores[0], regressions[0], static_cast<float>(scales[0]), conf_thresh_p);
|
||||
cv::GArray<custom::Face> final_p_faces_for_bb2squares = custom::ApplyRegression::on(faces0, true);
|
||||
cv::GArray<custom::Face> final_faces_pnet0 = custom::BBoxesToSquares::on(final_p_faces_for_bb2squares);
|
||||
total_faces[0] = custom::RunNMS::on(final_faces_pnet0, 0.5f, false);
|
||||
//The rest PNet pyramid layers to accumulate all layers result in total_faces[PYRAMID_LEVELS - 1]]
|
||||
for (int i = 1; i < pyramid_levels; ++i)
|
||||
{
|
||||
std::tie(regressions[i], scores[i]) = run_mtcnn_p(in_transposedRGB, get_pnet_level_name(level_size[i]));
|
||||
cv::GArray<custom::Face> faces = custom::BuildFaces::on(scores[i], regressions[i], static_cast<float>(scales[i]), conf_thresh_p);
|
||||
cv::GArray<custom::Face> final_p_faces_for_bb2squares_i = custom::ApplyRegression::on(faces, true);
|
||||
cv::GArray<custom::Face> final_faces_pnet_i = custom::BBoxesToSquares::on(final_p_faces_for_bb2squares_i);
|
||||
nms_p_faces[i] = custom::RunNMS::on(final_faces_pnet_i, 0.5f, false);
|
||||
total_faces[i] = custom::AccumulatePyramidOutputs::on(total_faces[i - 1], nms_p_faces[i]);
|
||||
}
|
||||
|
||||
//Proposal post-processing
|
||||
cv::GArray<custom::Face> final_faces_pnet = custom::RunNMS::on(total_faces[pyramid_levels - 1], 0.7f, true);
|
||||
|
||||
//Refinement part of MTCNN graph
|
||||
cv::GArray<cv::Rect> faces_roi_pnet = custom::R_O_NetPreProcGetROIs::on(final_faces_pnet, in_sz);
|
||||
cv::GArray<cv::GMat> regressionsRNet, scoresRNet;
|
||||
std::tie(regressionsRNet, scoresRNet) = cv::gapi::infer<custom::MTCNNRefinement>(faces_roi_pnet, in_transposedRGB);
|
||||
|
||||
//Refinement post-processing
|
||||
cv::GArray<custom::Face> rnet_post_proc_faces = custom::RNetPostProc::on(final_faces_pnet, scoresRNet, regressionsRNet, conf_thresh_r);
|
||||
cv::GArray<custom::Face> nms07_r_faces_total = custom::RunNMS::on(rnet_post_proc_faces, 0.7f, false);
|
||||
cv::GArray<custom::Face> final_r_faces_for_bb2squares = custom::ApplyRegression::on(nms07_r_faces_total, true);
|
||||
cv::GArray<custom::Face> final_faces_rnet = custom::BBoxesToSquares::on(final_r_faces_for_bb2squares);
|
||||
|
||||
//Output part of MTCNN graph
|
||||
cv::GArray<cv::Rect> faces_roi_rnet = custom::R_O_NetPreProcGetROIs::on(final_faces_rnet, in_sz);
|
||||
cv::GArray<cv::GMat> regressionsONet, scoresONet, landmarksONet;
|
||||
std::tie(regressionsONet, landmarksONet, scoresONet) = cv::gapi::infer<custom::MTCNNOutput>(faces_roi_rnet, in_transposedRGB);
|
||||
|
||||
//Output post-processing
|
||||
cv::GArray<custom::Face> onet_post_proc_faces = custom::ONetPostProc::on(final_faces_rnet, scoresONet, regressionsONet, landmarksONet, conf_thresh_o);
|
||||
cv::GArray<custom::Face> final_o_faces_for_nms07 = custom::ApplyRegression::on(onet_post_proc_faces, true);
|
||||
cv::GArray<custom::Face> nms07_o_faces_total = custom::RunNMS::on(final_o_faces_for_nms07, 0.7f, true);
|
||||
cv::GArray<custom::Face> final_faces_onet = custom::SwapFaces::on(nms07_o_faces_total);
|
||||
|
||||
cv::GComputation graph_mtcnn(cv::GIn(in_original), cv::GOut(cv::gapi::copy(in_original), final_faces_onet));
|
||||
|
||||
// MTCNN Refinement detection network
|
||||
auto mtcnnr_net = cv::gapi::ie::Params<custom::MTCNNRefinement>{
|
||||
model_path_r, // path to topology IR
|
||||
weights_path(model_path_r), // path to weights
|
||||
target_dev_r, // device specifier
|
||||
}.cfgOutputLayers({ "conv5-2", "prob1" }).cfgInputLayers({ "data" });
|
||||
|
||||
// MTCNN Output detection network
|
||||
auto mtcnno_net = cv::gapi::ie::Params<custom::MTCNNOutput>{
|
||||
model_path_o, // path to topology IR
|
||||
weights_path(model_path_o), // path to weights
|
||||
target_dev_o, // device specifier
|
||||
}.cfgOutputLayers({ "conv6-2", "conv6-3", "prob1" }).cfgInputLayers({ "data" });
|
||||
|
||||
auto networks_mtcnn = cv::gapi::networks(mtcnnr_net, mtcnno_net);
|
||||
|
||||
// MTCNN Proposal detection network
|
||||
for (int i = 0; i < pyramid_levels; ++i)
|
||||
{
|
||||
std::string net_id = get_pnet_level_name(level_size[i]);
|
||||
std::vector<size_t> reshape_dims = { 1, 3, (size_t)level_size[i].width, (size_t)level_size[i].height };
|
||||
cv::gapi::ie::Params<cv::gapi::Generic> mtcnnp_net{
|
||||
net_id, // tag
|
||||
model_path_p, // path to topology IR
|
||||
weights_path(model_path_p), // path to weights
|
||||
target_dev_p, // device specifier
|
||||
};
|
||||
mtcnnp_net.cfgInputReshape({ {"data", reshape_dims} });
|
||||
networks_mtcnn += cv::gapi::networks(mtcnnp_net);
|
||||
}
|
||||
|
||||
auto kernels_mtcnn = cv::gapi::kernels< custom::OCVBuildFaces
|
||||
, custom::OCVRunNMS
|
||||
, custom::OCVAccumulatePyramidOutputs
|
||||
, custom::OCVApplyRegression
|
||||
, custom::OCVBBoxesToSquares
|
||||
, custom::OCVR_O_NetPreProcGetROIs
|
||||
, custom::OCVRNetPostProc
|
||||
, custom::OCVONetPostProc
|
||||
, custom::OCVSwapFaces
|
||||
>();
|
||||
auto mtcnn_args = cv::compile_args(networks_mtcnn, kernels_mtcnn);
|
||||
if (streaming_queue_capacity != 0)
|
||||
mtcnn_args += cv::compile_args(cv::gapi::streaming::queue_capacity{ streaming_queue_capacity });
|
||||
auto pipeline_mtcnn = graph_mtcnn.compileStreaming(std::move(mtcnn_args));
|
||||
|
||||
std::cout << "Reading " << input_file_name << std::endl;
|
||||
// Input stream
|
||||
auto in_src = cv::gapi::wip::make_src<cv::gapi::wip::GCaptureSource>(input_file_name);
|
||||
|
||||
// Set the pipeline source & start the pipeline
|
||||
pipeline_mtcnn.setSource(cv::gin(in_src));
|
||||
pipeline_mtcnn.start();
|
||||
|
||||
// Declare the output data & run the processing loop
|
||||
cv::TickMeter tm;
|
||||
cv::Mat image;
|
||||
std::vector<custom::Face> out_faces;
|
||||
|
||||
tm.start();
|
||||
int frames = 0;
|
||||
while (pipeline_mtcnn.pull(cv::gout(image, out_faces))) {
|
||||
frames++;
|
||||
std::cout << "Final Faces Size " << out_faces.size() << std::endl;
|
||||
std::vector<vis::rectPoints> data;
|
||||
// show the image with faces in it
|
||||
for (const auto& out_face : out_faces) {
|
||||
std::vector<cv::Point> pts;
|
||||
for (size_t p = 0; p < NUM_PTS; ++p) {
|
||||
pts.push_back(
|
||||
cv::Point(static_cast<int>(out_face.ptsCoords[2 * p]), static_cast<int>(out_face.ptsCoords[2 * p + 1])));
|
||||
}
|
||||
auto rect = out_face.bbox.getRect();
|
||||
auto d = std::make_pair(rect, pts);
|
||||
data.push_back(d);
|
||||
}
|
||||
// Visualize results on the frame
|
||||
auto resultImg = vis::drawRectsAndPoints(image, data);
|
||||
tm.stop();
|
||||
const auto fps_str = std::to_string(frames / tm.getTimeSec()) + " FPS";
|
||||
cv::putText(resultImg, fps_str, { 0,32 }, cv::FONT_HERSHEY_SIMPLEX, 1.0, { 0,255,0 }, 2);
|
||||
cv::imshow("Out", resultImg);
|
||||
cv::waitKey(1);
|
||||
out_faces.clear();
|
||||
tm.start();
|
||||
}
|
||||
tm.stop();
|
||||
std::cout << "Processed " << frames << " frames"
|
||||
<< " (" << frames / tm.getTimeSec() << " FPS)" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,351 @@
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
#include <cctype>
|
||||
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/infer.hpp>
|
||||
#include <opencv2/gapi/infer/ie.hpp>
|
||||
#include <opencv2/gapi/streaming/cap.hpp>
|
||||
#include <opencv2/gapi/cpu/gcpukernel.hpp>
|
||||
#include <opencv2/highgui.hpp> // CommandLineParser
|
||||
#include <opencv2/gapi/infer/parsers.hpp>
|
||||
|
||||
const std::string about =
|
||||
"This is an OpenCV-based version of Gaze Estimation example";
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ input | | Path to the input video file }"
|
||||
"{ facem | face-detection-retail-0005.xml | Path to OpenVINO face detection model (.xml) }"
|
||||
"{ faced | CPU | Target device for the face detection (e.g. CPU, GPU, ...) }"
|
||||
"{ landm | facial-landmarks-35-adas-0002.xml | Path to OpenVINO landmarks detector model (.xml) }"
|
||||
"{ landd | CPU | Target device for the landmarks detector (e.g. CPU, GPU, ...) }"
|
||||
"{ headm | head-pose-estimation-adas-0001.xml | Path to OpenVINO head pose estimation model (.xml) }"
|
||||
"{ headd | CPU | Target device for the head pose estimation inference (e.g. CPU, GPU, ...) }"
|
||||
"{ gazem | gaze-estimation-adas-0002.xml | Path to OpenVINO gaze vector estimaiton model (.xml) }"
|
||||
"{ gazed | CPU | Target device for the gaze vector estimation inference (e.g. CPU, GPU, ...) }"
|
||||
;
|
||||
|
||||
namespace {
|
||||
std::string weights_path(const std::string &model_path) {
|
||||
const auto EXT_LEN = 4u;
|
||||
const auto sz = model_path.size();
|
||||
CV_Assert(sz > EXT_LEN);
|
||||
|
||||
auto ext = model_path.substr(sz - EXT_LEN);
|
||||
auto lower = [](unsigned char c) {
|
||||
return static_cast<unsigned char>(std::tolower(c));
|
||||
};
|
||||
std::transform(ext.begin(), ext.end(), ext.begin(), lower);
|
||||
CV_Assert(ext == ".xml");
|
||||
return model_path.substr(0u, sz - EXT_LEN) + ".bin";
|
||||
}
|
||||
} // anonymous namespace
|
||||
|
||||
namespace custom {
|
||||
namespace {
|
||||
using GMat3 = std::tuple<cv::GMat,cv::GMat,cv::GMat>;
|
||||
using GMats = cv::GArray<cv::GMat>;
|
||||
using GRects = cv::GArray<cv::Rect>;
|
||||
using GSize = cv::GOpaque<cv::Size>;
|
||||
G_API_NET(Faces, <cv::GMat(cv::GMat)>, "face-detector" );
|
||||
G_API_NET(Landmarks, <cv::GMat(cv::GMat)>, "facial-landmarks");
|
||||
G_API_NET(HeadPose, < GMat3(cv::GMat)>, "head-pose");
|
||||
G_API_NET(Gaze, <cv::GMat(cv::GMat,cv::GMat,cv::GMat)>, "gaze-vector");
|
||||
|
||||
G_API_OP(Size, <GSize(cv::GMat)>, "custom.gapi.size") {
|
||||
static cv::GOpaqueDesc outMeta(const cv::GMatDesc &) {
|
||||
return cv::empty_gopaque_desc();
|
||||
}
|
||||
};
|
||||
|
||||
// Left/Right eye per every face
|
||||
G_API_OP(ParseEyes,
|
||||
<std::tuple<GRects, GRects>(GMats, GRects, GSize)>,
|
||||
"custom.gaze_estimation.parseEyes") {
|
||||
static std::tuple<cv::GArrayDesc, cv::GArrayDesc>
|
||||
outMeta( const cv::GArrayDesc &
|
||||
, const cv::GArrayDesc &
|
||||
, const cv::GOpaqueDesc &) {
|
||||
return std::make_tuple(cv::empty_array_desc(), cv::empty_array_desc());
|
||||
}
|
||||
};
|
||||
|
||||
// Combine three scalars into a 1x3 vector (per every face)
|
||||
G_API_OP(ProcessPoses,
|
||||
<GMats(GMats, GMats, GMats)>,
|
||||
"custom.gaze_estimation.processPoses") {
|
||||
static cv::GArrayDesc outMeta( const cv::GArrayDesc &
|
||||
, const cv::GArrayDesc &
|
||||
, const cv::GArrayDesc &) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
void gazeVectorToGazeAngles(const cv::Point3f& gazeVector,
|
||||
cv::Point2f& gazeAngles) {
|
||||
auto r = cv::norm(gazeVector);
|
||||
|
||||
double v0 = static_cast<double>(gazeVector.x);
|
||||
double v1 = static_cast<double>(gazeVector.y);
|
||||
double v2 = static_cast<double>(gazeVector.z);
|
||||
|
||||
gazeAngles.x = static_cast<float>(180.0 / M_PI * (M_PI_2 + std::atan2(v2, v0)));
|
||||
gazeAngles.y = static_cast<float>(180.0 / M_PI * (M_PI_2 - std::acos(v1 / r)));
|
||||
}
|
||||
|
||||
GAPI_OCV_KERNEL(OCVSize, Size) {
|
||||
static void run(const cv::Mat &in, cv::Size &out) {
|
||||
out = in.size();
|
||||
}
|
||||
};
|
||||
|
||||
cv::Rect eyeBox(const cv::Rect &face_rc,
|
||||
float p1_x, float p1_y, float p2_x, float p2_y,
|
||||
float scale = 1.8f) {
|
||||
const auto &up = face_rc.size();
|
||||
const cv::Point p1 = {
|
||||
static_cast<int>(p1_x*up.width),
|
||||
static_cast<int>(p1_y*up.height)
|
||||
};
|
||||
const cv::Point p2 = {
|
||||
static_cast<int>(p2_x*up.width),
|
||||
static_cast<int>(p2_y*up.height)
|
||||
};
|
||||
cv::Rect result;
|
||||
|
||||
const auto size = static_cast<float>(cv::norm(p1 - p2));
|
||||
const auto midpoint = (p1 + p2) / 2;
|
||||
|
||||
result.width = static_cast<int>(scale * size);
|
||||
result.height = result.width;
|
||||
result.x = face_rc.x + midpoint.x - (result.width / 2);
|
||||
result.y = face_rc.y + midpoint.y - (result.height / 2);
|
||||
// Shift result to the original frame's absolute coordinates
|
||||
return result;
|
||||
}
|
||||
|
||||
GAPI_OCV_KERNEL(OCVParseEyes, ParseEyes) {
|
||||
static void run(const std::vector<cv::Mat> &in_landmarks_per_face,
|
||||
const std::vector<cv::Rect> &in_face_rcs,
|
||||
const cv::Size &frame_size,
|
||||
std::vector<cv::Rect> &out_left_eyes,
|
||||
std::vector<cv::Rect> &out_right_eyes) {
|
||||
const size_t numFaces = in_landmarks_per_face.size();
|
||||
const cv::Rect surface(cv::Point(0,0), frame_size);
|
||||
GAPI_Assert(numFaces == in_face_rcs.size());
|
||||
out_left_eyes.clear();
|
||||
out_right_eyes.clear();
|
||||
out_left_eyes.reserve(numFaces);
|
||||
out_right_eyes.reserve(numFaces);
|
||||
|
||||
for (std::size_t i = 0u; i < numFaces; i++) {
|
||||
const auto &lm = in_landmarks_per_face[i];
|
||||
const auto &rc = in_face_rcs[i];
|
||||
// Left eye is defined by points 0/1 (x2),
|
||||
// Right eye is defined by points 2/3 (x2)
|
||||
const float *data = lm.ptr<float>();
|
||||
out_left_eyes .push_back(surface & eyeBox(rc, data[0], data[1], data[2], data[3]));
|
||||
out_right_eyes.push_back(surface & eyeBox(rc, data[4], data[5], data[6], data[7]));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL(OCVProcessPoses, ProcessPoses) {
|
||||
static void run(const std::vector<cv::Mat> &in_ys,
|
||||
const std::vector<cv::Mat> &in_ps,
|
||||
const std::vector<cv::Mat> &in_rs,
|
||||
std::vector<cv::Mat> &out_poses) {
|
||||
const std::size_t sz = in_ys.size();
|
||||
GAPI_Assert(sz == in_ps.size() && sz == in_rs.size());
|
||||
out_poses.clear();
|
||||
for (std::size_t idx = 0u; idx < sz; idx++) {
|
||||
cv::Mat pose(1, 3, CV_32FC1);
|
||||
float *ptr = pose.ptr<float>();
|
||||
ptr[0] = in_ys[idx].ptr<float>()[0];
|
||||
ptr[1] = in_ps[idx].ptr<float>()[0];
|
||||
ptr[2] = in_rs[idx].ptr<float>()[0];
|
||||
out_poses.push_back(std::move(pose));
|
||||
}
|
||||
}
|
||||
};
|
||||
} // anonymous namespace
|
||||
} // namespace custom
|
||||
|
||||
namespace vis {
|
||||
namespace {
|
||||
cv::Point2f midp(const cv::Rect &rc) {
|
||||
return (rc.tl() + rc.br()) / 2;
|
||||
};
|
||||
void bbox(cv::Mat &m, const cv::Rect &rc) {
|
||||
cv::rectangle(m, rc, cv::Scalar{0,255,0}, 2, cv::LINE_8, 0);
|
||||
};
|
||||
void pose(cv::Mat &m, const cv::Mat &p, const cv::Rect &face_rc) {
|
||||
const auto *posePtr = p.ptr<float>();
|
||||
const auto yaw = static_cast<double>(posePtr[0]);
|
||||
const auto pitch = static_cast<double>(posePtr[1]);
|
||||
const auto roll = static_cast<double>(posePtr[2]);
|
||||
|
||||
const auto sinY = std::sin(yaw * M_PI / 180.0);
|
||||
const auto sinP = std::sin(pitch * M_PI / 180.0);
|
||||
const auto sinR = std::sin(roll * M_PI / 180.0);
|
||||
|
||||
const auto cosY = std::cos(yaw * M_PI / 180.0);
|
||||
const auto cosP = std::cos(pitch * M_PI / 180.0);
|
||||
const auto cosR = std::cos(roll * M_PI / 180.0);
|
||||
|
||||
const auto axisLength = 0.4 * face_rc.width;
|
||||
const auto xCenter = face_rc.x + face_rc.width / 2;
|
||||
const auto yCenter = face_rc.y + face_rc.height / 2;
|
||||
|
||||
const auto center = cv::Point{xCenter, yCenter};
|
||||
const auto axisln = cv::Point2d{axisLength, axisLength};
|
||||
const auto ctr = cv::Matx<double,2,2>(cosR*cosY, sinY*sinP*sinR, 0.f, cosP*sinR);
|
||||
const auto ctt = cv::Matx<double,2,2>(cosR*sinY*sinP, cosY*sinR, 0.f, -cosP*cosR);
|
||||
const auto ctf = cv::Matx<double,2,2>(sinY*cosP, 0.f, 0.f, sinP);
|
||||
|
||||
// center to right
|
||||
cv::line(m, center, center + static_cast<cv::Point>(ctr*axisln), cv::Scalar(0, 0, 255), 2);
|
||||
// center to top
|
||||
cv::line(m, center, center + static_cast<cv::Point>(ctt*axisln), cv::Scalar(0, 255, 0), 2);
|
||||
// center to forward
|
||||
cv::line(m, center, center + static_cast<cv::Point>(ctf*axisln), cv::Scalar(255, 0, 255), 2);
|
||||
}
|
||||
void vvec(cv::Mat &m, const cv::Mat &v, const cv::Rect &face_rc,
|
||||
const cv::Rect &left_rc, const cv::Rect &right_rc) {
|
||||
const auto scale = 0.002 * face_rc.width;
|
||||
|
||||
cv::Point3f gazeVector;
|
||||
const auto *gazePtr = v.ptr<float>();
|
||||
gazeVector.x = gazePtr[0];
|
||||
gazeVector.y = gazePtr[1];
|
||||
gazeVector.z = gazePtr[2];
|
||||
gazeVector = gazeVector / cv::norm(gazeVector);
|
||||
|
||||
const double arrowLength = 0.4 * face_rc.width;
|
||||
const auto left_mid = midp(left_rc);
|
||||
const auto right_mid = midp(right_rc);
|
||||
|
||||
cv::Point2f gazeArrow;
|
||||
gazeArrow.x = gazeVector.x;
|
||||
gazeArrow.y = -gazeVector.y;
|
||||
gazeArrow *= arrowLength;
|
||||
|
||||
cv::arrowedLine(m, left_mid, left_mid + gazeArrow, cv::Scalar(255, 0, 0), 2);
|
||||
cv::arrowedLine(m, right_mid, right_mid + gazeArrow, cv::Scalar(255, 0, 0), 2);
|
||||
|
||||
cv::Point2f gazeAngles;
|
||||
custom::gazeVectorToGazeAngles(gazeVector, gazeAngles);
|
||||
|
||||
cv::putText(m,
|
||||
cv::format("gaze angles: (h=%0.0f, v=%0.0f)",
|
||||
static_cast<double>(std::round(gazeAngles.x)),
|
||||
static_cast<double>(std::round(gazeAngles.y))),
|
||||
cv::Point(static_cast<int>(face_rc.tl().x),
|
||||
static_cast<int>(face_rc.br().y + 12. * face_rc.width / 100.)),
|
||||
cv::FONT_HERSHEY_PLAIN, scale * 2, cv::Scalar::all(255), 1);
|
||||
};
|
||||
} // anonymous namespace
|
||||
} // namespace vis
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
cmd.about(about);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
cv::GMat in;
|
||||
cv::GMat faces = cv::gapi::infer<custom::Faces>(in);
|
||||
cv::GOpaque<cv::Size> sz = cv::gapi::streaming::size(in);
|
||||
cv::GArray<cv::Rect> faces_rc = cv::gapi::parseSSD(faces, sz, 0.5f, true, true);
|
||||
cv::GArray<cv::GMat> angles_y, angles_p, angles_r;
|
||||
std::tie(angles_y, angles_p, angles_r) = cv::gapi::infer<custom::HeadPose>(faces_rc, in);
|
||||
cv::GArray<cv::GMat> heads_pos = custom::ProcessPoses::on(angles_y, angles_p, angles_r);
|
||||
cv::GArray<cv::GMat> landmarks = cv::gapi::infer<custom::Landmarks>(faces_rc, in);
|
||||
cv::GArray<cv::Rect> left_eyes, right_eyes;
|
||||
std::tie(left_eyes, right_eyes) = custom::ParseEyes::on(landmarks, faces_rc, sz);
|
||||
cv::GArray<cv::GMat> gaze_vectors = cv::gapi::infer2<custom::Gaze>( in
|
||||
, left_eyes
|
||||
, right_eyes
|
||||
, heads_pos);
|
||||
cv::GComputation graph(cv::GIn(in),
|
||||
cv::GOut( cv::gapi::copy(in)
|
||||
, faces_rc
|
||||
, left_eyes
|
||||
, right_eyes
|
||||
, heads_pos
|
||||
, gaze_vectors));
|
||||
|
||||
const auto input_file_name = cmd.get<std::string>("input");
|
||||
const auto face_model_path = cmd.get<std::string>("facem");
|
||||
const auto head_model_path = cmd.get<std::string>("headm");
|
||||
const auto lmrk_model_path = cmd.get<std::string>("landm");
|
||||
const auto gaze_model_path = cmd.get<std::string>("gazem");
|
||||
|
||||
auto face_net = cv::gapi::ie::Params<custom::Faces> {
|
||||
face_model_path, // path to topology IR
|
||||
weights_path(face_model_path), // path to weights
|
||||
cmd.get<std::string>("faced"), /// device specifier
|
||||
};
|
||||
auto head_net = cv::gapi::ie::Params<custom::HeadPose> {
|
||||
head_model_path, // path to topology IR
|
||||
weights_path(head_model_path), // path to weights
|
||||
cmd.get<std::string>("headd"), // device specifier
|
||||
}.cfgOutputLayers({"angle_y_fc", "angle_p_fc", "angle_r_fc"});
|
||||
auto landmarks_net = cv::gapi::ie::Params<custom::Landmarks> {
|
||||
lmrk_model_path, // path to topology IR
|
||||
weights_path(lmrk_model_path), // path to weights
|
||||
cmd.get<std::string>("landd"), // device specifier
|
||||
};
|
||||
auto gaze_net = cv::gapi::ie::Params<custom::Gaze> {
|
||||
gaze_model_path, // path to topology IR
|
||||
weights_path(gaze_model_path), // path to weights
|
||||
cmd.get<std::string>("gazed"), // device specifier
|
||||
}.cfgInputLayers({"left_eye_image", "right_eye_image", "head_pose_angles"});
|
||||
|
||||
auto kernels = cv::gapi::kernels< custom::OCVSize
|
||||
, custom::OCVParseEyes
|
||||
, custom::OCVProcessPoses>();
|
||||
auto networks = cv::gapi::networks(face_net, head_net, landmarks_net, gaze_net);
|
||||
auto pipeline = graph.compileStreaming(cv::compile_args(networks, kernels));
|
||||
|
||||
cv::TickMeter tm;
|
||||
cv::Mat image;
|
||||
std::vector<cv::Rect> out_faces, out_right_eyes, out_left_eyes;
|
||||
std::vector<cv::Mat> out_poses;
|
||||
std::vector<cv::Mat> out_gazes;
|
||||
std::size_t frames = 0u;
|
||||
std::cout << "Reading " << input_file_name << std::endl;
|
||||
|
||||
pipeline.setSource(cv::gapi::wip::make_src<cv::gapi::wip::GCaptureSource>(input_file_name));
|
||||
pipeline.start();
|
||||
tm.start();
|
||||
while (pipeline.pull(cv::gout( image
|
||||
, out_faces
|
||||
, out_left_eyes
|
||||
, out_right_eyes
|
||||
, out_poses
|
||||
, out_gazes))) {
|
||||
frames++;
|
||||
// Visualize results on the frame
|
||||
for (auto &&rc : out_faces) vis::bbox(image, rc);
|
||||
for (auto &&rc : out_left_eyes) vis::bbox(image, rc);
|
||||
for (auto &&rc : out_right_eyes) vis::bbox(image, rc);
|
||||
for (std::size_t i = 0u; i < out_faces.size(); i++) {
|
||||
vis::pose(image, out_poses[i], out_faces[i]);
|
||||
vis::vvec(image, out_gazes[i], out_faces[i], out_left_eyes[i], out_right_eyes[i]);
|
||||
}
|
||||
tm.stop();
|
||||
const auto fps_str = std::to_string(frames / tm.getTimeSec()) + " FPS";
|
||||
cv::putText(image, fps_str, {0,32}, cv::FONT_HERSHEY_SIMPLEX, 1.0, {0,255,0}, 2);
|
||||
cv::imshow("Out", image);
|
||||
cv::waitKey(1);
|
||||
tm.start();
|
||||
}
|
||||
tm.stop();
|
||||
std::cout << "Processed " << frames << " frames"
|
||||
<< " (" << frames / tm.getTimeSec() << " FPS)" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
#include <chrono>
|
||||
#include <iomanip>
|
||||
|
||||
#include "opencv2/imgproc.hpp"
|
||||
#include "opencv2/highgui.hpp"
|
||||
|
||||
#include "opencv2/gapi.hpp"
|
||||
#include "opencv2/gapi/core.hpp"
|
||||
#include "opencv2/gapi/imgproc.hpp"
|
||||
#include "opencv2/gapi/infer.hpp"
|
||||
#include "opencv2/gapi/infer/ie.hpp"
|
||||
#include "opencv2/gapi/infer/onnx.hpp"
|
||||
#include "opencv2/gapi/cpu/gcpukernel.hpp"
|
||||
#include "opencv2/gapi/streaming/cap.hpp"
|
||||
|
||||
namespace {
|
||||
const std::string keys =
|
||||
"{ h help | | print this help message }"
|
||||
"{ input | | Path to an input video file }"
|
||||
"{ fdm | | IE face detection model IR }"
|
||||
"{ fdw | | IE face detection model weights }"
|
||||
"{ fdd | | IE face detection device }"
|
||||
"{ emom | | ONNX emotions recognition model }"
|
||||
"{ output | | (Optional) Path to an output video file }"
|
||||
;
|
||||
} // namespace
|
||||
|
||||
namespace custom {
|
||||
G_API_NET(Faces, <cv::GMat(cv::GMat)>, "face-detector");
|
||||
G_API_NET(Emotions, <cv::GMat(cv::GMat)>, "emotions-recognition");
|
||||
|
||||
G_API_OP(PostProc, <cv::GArray<cv::Rect>(cv::GMat, cv::GMat)>, "custom.fd_postproc") {
|
||||
static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GMatDesc &) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL(OCVPostProc, PostProc) {
|
||||
static void run(const cv::Mat &in_ssd_result,
|
||||
const cv::Mat &in_frame,
|
||||
std::vector<cv::Rect> &out_faces) {
|
||||
const int MAX_PROPOSALS = 200;
|
||||
const int OBJECT_SIZE = 7;
|
||||
const cv::Size upscale = in_frame.size();
|
||||
const cv::Rect surface({0,0}, upscale);
|
||||
|
||||
out_faces.clear();
|
||||
|
||||
const float *data = in_ssd_result.ptr<float>();
|
||||
for (int i = 0; i < MAX_PROPOSALS; i++) {
|
||||
const float image_id = data[i * OBJECT_SIZE + 0]; // batch id
|
||||
const float confidence = data[i * OBJECT_SIZE + 2];
|
||||
const float rc_left = data[i * OBJECT_SIZE + 3];
|
||||
const float rc_top = data[i * OBJECT_SIZE + 4];
|
||||
const float rc_right = data[i * OBJECT_SIZE + 5];
|
||||
const float rc_bottom = data[i * OBJECT_SIZE + 6];
|
||||
|
||||
if (image_id < 0.f) { // indicates end of detections
|
||||
break;
|
||||
}
|
||||
if (confidence < 0.5f) {
|
||||
continue;
|
||||
}
|
||||
|
||||
cv::Rect rc;
|
||||
rc.x = static_cast<int>(rc_left * upscale.width);
|
||||
rc.y = static_cast<int>(rc_top * upscale.height);
|
||||
rc.width = static_cast<int>(rc_right * upscale.width) - rc.x;
|
||||
rc.height = static_cast<int>(rc_bottom * upscale.height) - rc.y;
|
||||
out_faces.push_back(rc & surface);
|
||||
}
|
||||
}
|
||||
};
|
||||
//! [Postproc]
|
||||
|
||||
} // namespace custom
|
||||
|
||||
namespace labels {
|
||||
// Labels as defined in
|
||||
// https://github.com/onnx/models/tree/master/vision/body_analysis/emotion_ferplus
|
||||
//
|
||||
const std::string emotions[] = {
|
||||
"neutral", "happiness", "surprise", "sadness", "anger", "disgust", "fear", "contempt"
|
||||
};
|
||||
namespace {
|
||||
template<typename Iter>
|
||||
std::vector<float> softmax(Iter begin, Iter end) {
|
||||
std::vector<float> prob(end - begin, 0.f);
|
||||
std::transform(begin, end, prob.begin(), [](float x) { return std::exp(x); });
|
||||
float sum = std::accumulate(prob.begin(), prob.end(), 0.0f);
|
||||
for (int i = 0; i < static_cast<int>(prob.size()); i++)
|
||||
prob[i] /= sum;
|
||||
return prob;
|
||||
}
|
||||
|
||||
void DrawResults(cv::Mat &frame,
|
||||
const std::vector<cv::Rect> &faces,
|
||||
const std::vector<cv::Mat> &out_emotions) {
|
||||
CV_Assert(faces.size() == out_emotions.size());
|
||||
|
||||
for (auto it = faces.begin(); it != faces.end(); ++it) {
|
||||
const auto idx = std::distance(faces.begin(), it);
|
||||
const auto &rc = *it;
|
||||
|
||||
const float *emotions_data = out_emotions[idx].ptr<float>();
|
||||
auto sm = softmax(emotions_data, emotions_data + 8);
|
||||
const auto emo_id = std::max_element(sm.begin(), sm.end()) - sm.begin();
|
||||
|
||||
const int ATTRIB_OFFSET = 15;
|
||||
cv::rectangle(frame, rc, {0, 255, 0}, 4);
|
||||
cv::putText(frame, emotions[emo_id],
|
||||
cv::Point(rc.x, rc.y - ATTRIB_OFFSET),
|
||||
cv::FONT_HERSHEY_COMPLEX_SMALL,
|
||||
1,
|
||||
cv::Scalar(0, 0, 255));
|
||||
|
||||
std::cout << emotions[emo_id] << " at " << rc << std::endl;
|
||||
}
|
||||
}
|
||||
} // anonymous namespace
|
||||
} // namespace labels
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
const std::string input = cmd.get<std::string>("input");
|
||||
const std::string output = cmd.get<std::string>("output");
|
||||
|
||||
// OpenVINO FD parameters here
|
||||
auto det_net = cv::gapi::ie::Params<custom::Faces> {
|
||||
cmd.get<std::string>("fdm"), // read cmd args: path to topology IR
|
||||
cmd.get<std::string>("fdw"), // read cmd args: path to weights
|
||||
cmd.get<std::string>("fdd"), // read cmd args: device specifier
|
||||
};
|
||||
|
||||
// ONNX Emotions parameters here
|
||||
auto emo_net = cv::gapi::onnx::Params<custom::Emotions> {
|
||||
cmd.get<std::string>("emom"), // read cmd args: path to the ONNX model
|
||||
}.cfgNormalize({false}); // model accepts 0..255 range in FP32
|
||||
|
||||
auto kernels = cv::gapi::kernels<custom::OCVPostProc>();
|
||||
auto networks = cv::gapi::networks(det_net, emo_net);
|
||||
|
||||
cv::GMat in;
|
||||
cv::GMat bgr = cv::gapi::copy(in);
|
||||
cv::GMat frame = cv::gapi::streaming::desync(bgr);
|
||||
cv::GMat detections = cv::gapi::infer<custom::Faces>(frame);
|
||||
cv::GArray<cv::Rect> faces = custom::PostProc::on(detections, frame);
|
||||
cv::GArray<cv::GMat> emotions = cv::gapi::infer<custom::Emotions>(faces, frame);
|
||||
auto pipeline = cv::GComputation(cv::GIn(in), cv::GOut(bgr, faces, emotions))
|
||||
.compileStreaming(cv::compile_args(kernels, networks));
|
||||
|
||||
auto in_src = cv::gapi::wip::make_src<cv::gapi::wip::GCaptureSource>(input);
|
||||
pipeline.setSource(cv::gin(in_src));
|
||||
|
||||
cv::util::optional<cv::Mat> out_frame;
|
||||
cv::util::optional<std::vector<cv::Rect>> out_faces;
|
||||
cv::util::optional<std::vector<cv::Mat>> out_emotions;
|
||||
|
||||
cv::Mat last_mat;
|
||||
std::vector<cv::Rect> last_faces;
|
||||
std::vector<cv::Mat> last_emotions;
|
||||
|
||||
cv::VideoWriter writer;
|
||||
cv::TickMeter tm;
|
||||
std::size_t frames = 0u;
|
||||
|
||||
tm.start();
|
||||
pipeline.start();
|
||||
while (pipeline.pull(cv::gout(out_frame, out_faces, out_emotions))) {
|
||||
++frames;
|
||||
if (out_faces && out_emotions) {
|
||||
last_faces = *out_faces;
|
||||
last_emotions = *out_emotions;
|
||||
}
|
||||
if (out_frame) {
|
||||
last_mat = *out_frame;
|
||||
labels::DrawResults(last_mat, last_faces, last_emotions);
|
||||
|
||||
if (!output.empty()) {
|
||||
if (!writer.isOpened()) {
|
||||
const auto sz = cv::Size{last_mat.cols, last_mat.rows};
|
||||
writer.open(output, cv::VideoWriter::fourcc('M','J','P','G'), 25.0, sz);
|
||||
CV_Assert(writer.isOpened());
|
||||
}
|
||||
writer << last_mat;
|
||||
}
|
||||
}
|
||||
if (!last_mat.empty()) {
|
||||
cv::imshow("Out", last_mat);
|
||||
cv::waitKey(1);
|
||||
}
|
||||
}
|
||||
tm.stop();
|
||||
std::cout << "Processed " << frames << " frames" << " (" << frames / tm.getTimeSec() << " FPS)" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <opencv2/imgcodecs.hpp>
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/imgproc.hpp>
|
||||
#include <opencv2/gapi/infer.hpp>
|
||||
#include <opencv2/gapi/render.hpp>
|
||||
#include <opencv2/gapi/infer/ie.hpp>
|
||||
#include <opencv2/gapi/cpu/gcpukernel.hpp>
|
||||
#include <opencv2/gapi/streaming/cap.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
#include <opencv2/gapi/infer/parsers.hpp>
|
||||
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ input | | Path to the input video file }"
|
||||
"{ facem | face-detection-adas-0001.xml | Path to OpenVINO IE face detection model (.xml) }"
|
||||
"{ faced | CPU | Target device for face detection model (e.g. CPU, GPU, VPU, ...) }"
|
||||
"{ r roi | -1,-1,-1,-1 | Region of interest (ROI) to use for inference. Identified automatically when not set }";
|
||||
|
||||
namespace {
|
||||
|
||||
std::string weights_path(const std::string &model_path) {
|
||||
const auto EXT_LEN = 4u;
|
||||
const auto sz = model_path.size();
|
||||
CV_Assert(sz > EXT_LEN);
|
||||
|
||||
auto ext = model_path.substr(sz - EXT_LEN);
|
||||
std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c){
|
||||
return static_cast<unsigned char>(std::tolower(c));
|
||||
});
|
||||
CV_Assert(ext == ".xml");
|
||||
return model_path.substr(0u, sz - EXT_LEN) + ".bin";
|
||||
}
|
||||
|
||||
cv::util::optional<cv::Rect> parse_roi(const std::string &rc) {
|
||||
cv::Rect rv;
|
||||
char delim[3];
|
||||
|
||||
std::stringstream is(rc);
|
||||
is >> rv.x >> delim[0] >> rv.y >> delim[1] >> rv.width >> delim[2] >> rv.height;
|
||||
if (is.bad()) {
|
||||
return cv::util::optional<cv::Rect>(); // empty value
|
||||
}
|
||||
const auto is_delim = [](char c) {
|
||||
return c == ',';
|
||||
};
|
||||
if (!std::all_of(std::begin(delim), std::end(delim), is_delim)) {
|
||||
return cv::util::optional<cv::Rect>(); // empty value
|
||||
|
||||
}
|
||||
if (rv.x < 0 || rv.y < 0 || rv.width <= 0 || rv.height <= 0) {
|
||||
return cv::util::optional<cv::Rect>(); // empty value
|
||||
}
|
||||
return cv::util::make_optional(std::move(rv));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
namespace custom {
|
||||
|
||||
G_API_NET(FaceDetector, <cv::GMat(cv::GMat)>, "face-detector");
|
||||
|
||||
using GDetections = cv::GArray<cv::Rect>;
|
||||
using GRect = cv::GOpaque<cv::Rect>;
|
||||
using GSize = cv::GOpaque<cv::Size>;
|
||||
using GPrims = cv::GArray<cv::gapi::wip::draw::Prim>;
|
||||
|
||||
G_API_OP(LocateROI, <GRect(cv::GMat)>, "sample.custom.locate-roi") {
|
||||
static cv::GOpaqueDesc outMeta(const cv::GMatDesc &) {
|
||||
return cv::empty_gopaque_desc();
|
||||
}
|
||||
};
|
||||
|
||||
G_API_OP(BBoxes, <GPrims(GDetections, GRect)>, "sample.custom.b-boxes") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc &, const cv::GOpaqueDesc &) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL(OCVLocateROI, LocateROI) {
|
||||
// This is the place where we can run extra analytics
|
||||
// on the input image frame and select the ROI (region
|
||||
// of interest) where we want to detect our objects (or
|
||||
// run any other inference).
|
||||
//
|
||||
// Currently it doesn't do anything intelligent,
|
||||
// but only crops the input image to square (this is
|
||||
// the most convenient aspect ratio for detectors to use)
|
||||
|
||||
static void run(const cv::Mat &in_mat, cv::Rect &out_rect) {
|
||||
|
||||
// Identify the central point & square size (- some padding)
|
||||
const auto center = cv::Point{in_mat.cols/2, in_mat.rows/2};
|
||||
auto sqside = std::min(in_mat.cols, in_mat.rows);
|
||||
|
||||
// Now build the central square ROI
|
||||
out_rect = cv::Rect{ center.x - sqside/2
|
||||
, center.y - sqside/2
|
||||
, sqside
|
||||
, sqside
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) {
|
||||
// This kernel converts the rectangles into G-API's
|
||||
// rendering primitives
|
||||
static void run(const std::vector<cv::Rect> &in_face_rcs,
|
||||
const cv::Rect &in_roi,
|
||||
std::vector<cv::gapi::wip::draw::Prim> &out_prims) {
|
||||
out_prims.clear();
|
||||
const auto cvt = [](const cv::Rect &rc, const cv::Scalar &clr) {
|
||||
return cv::gapi::wip::draw::Rect(rc, clr, 2);
|
||||
};
|
||||
out_prims.emplace_back(cvt(in_roi, CV_RGB(0,255,255))); // cyan
|
||||
for (auto &&rc : in_face_rcs) {
|
||||
out_prims.emplace_back(cvt(rc, CV_RGB(0,255,0))); // green
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace custom
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Prepare parameters first
|
||||
const std::string input = cmd.get<std::string>("input");
|
||||
const auto opt_roi = parse_roi(cmd.get<std::string>("roi"));
|
||||
|
||||
const auto face_model_path = cmd.get<std::string>("facem");
|
||||
auto face_net = cv::gapi::ie::Params<custom::FaceDetector> {
|
||||
face_model_path, // path to topology IR
|
||||
weights_path(face_model_path), // path to weights
|
||||
cmd.get<std::string>("faced"), // device specifier
|
||||
};
|
||||
auto kernels = cv::gapi::kernels
|
||||
<custom::OCVLocateROI
|
||||
, custom::OCVBBoxes>();
|
||||
auto networks = cv::gapi::networks(face_net);
|
||||
|
||||
// Now build the graph. The graph structure may vary
|
||||
// passed on the input parameters
|
||||
cv::GStreamingCompiled pipeline;
|
||||
auto inputs = cv::gin(cv::gapi::wip::make_src<cv::gapi::wip::GCaptureSource>(input));
|
||||
|
||||
cv::GMat in;
|
||||
cv::GOpaque<cv::Size> sz = cv::gapi::streaming::size(in);
|
||||
if (opt_roi.has_value()) {
|
||||
// Use the value provided by user
|
||||
std::cout << "Will run inference for static region "
|
||||
<< opt_roi.value()
|
||||
<< " only"
|
||||
<< std::endl;
|
||||
cv::GOpaque<cv::Rect> in_roi;
|
||||
auto blob = cv::gapi::infer<custom::FaceDetector>(in_roi, in);
|
||||
cv::GArray<cv::Rect> rcs = cv::gapi::parseSSD(blob, sz, 0.5f, true, true);
|
||||
auto out = cv::gapi::wip::draw::render3ch(in, custom::BBoxes::on(rcs, in_roi));
|
||||
pipeline = cv::GComputation(cv::GIn(in, in_roi), cv::GOut(out))
|
||||
.compileStreaming(cv::compile_args(kernels, networks));
|
||||
|
||||
// Since the ROI to detect is manual, make it part of the input vector
|
||||
inputs.push_back(cv::gin(opt_roi.value())[0]);
|
||||
} else {
|
||||
// Automatically detect ROI to infer. Make it output parameter
|
||||
std::cout << "ROI is not set or invalid. Locating it automatically"
|
||||
<< std::endl;
|
||||
cv::GOpaque<cv::Rect> roi = custom::LocateROI::on(in);
|
||||
auto blob = cv::gapi::infer<custom::FaceDetector>(roi, in);
|
||||
cv::GArray<cv::Rect> rcs = cv::gapi::parseSSD(blob, sz, 0.5f, true, true);
|
||||
auto out = cv::gapi::wip::draw::render3ch(in, custom::BBoxes::on(rcs, roi));
|
||||
pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out))
|
||||
.compileStreaming(cv::compile_args(kernels, networks));
|
||||
}
|
||||
|
||||
// The execution part
|
||||
pipeline.setSource(std::move(inputs));
|
||||
pipeline.start();
|
||||
|
||||
cv::Mat out;
|
||||
size_t frames = 0u;
|
||||
cv::TickMeter tm;
|
||||
tm.start();
|
||||
while (pipeline.pull(cv::gout(out))) {
|
||||
cv::imshow("Out", out);
|
||||
cv::waitKey(1);
|
||||
++frames;
|
||||
}
|
||||
tm.stop();
|
||||
std::cout << "Processed " << frames << " frames" << " (" << frames / tm.getTimeSec() << " FPS)" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <opencv2/imgcodecs.hpp>
|
||||
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/imgproc.hpp>
|
||||
#include <opencv2/gapi/infer.hpp>
|
||||
#include <opencv2/gapi/render.hpp>
|
||||
#include <opencv2/gapi/infer/onnx.hpp>
|
||||
#include <opencv2/gapi/cpu/gcpukernel.hpp>
|
||||
#include <opencv2/gapi/streaming/cap.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
#include <opencv2/gapi/infer/parsers.hpp>
|
||||
|
||||
namespace custom {
|
||||
|
||||
G_API_NET(ObjDetector, <cv::GMat(cv::GMat)>, "object-detector");
|
||||
|
||||
using GDetections = cv::GArray<cv::Rect>;
|
||||
using GSize = cv::GOpaque<cv::Size>;
|
||||
using GPrims = cv::GArray<cv::gapi::wip::draw::Prim>;
|
||||
|
||||
G_API_OP(BBoxes, <GPrims(GDetections)>, "sample.custom.b-boxes") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc &) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) {
|
||||
// This kernel converts the rectangles into G-API's
|
||||
// rendering primitives
|
||||
static void run(const std::vector<cv::Rect> &in_obj_rcs,
|
||||
std::vector<cv::gapi::wip::draw::Prim> &out_prims) {
|
||||
out_prims.clear();
|
||||
const auto cvt = [](const cv::Rect &rc, const cv::Scalar &clr) {
|
||||
return cv::gapi::wip::draw::Rect(rc, clr, 2);
|
||||
};
|
||||
for (auto &&rc : in_obj_rcs) {
|
||||
out_prims.emplace_back(cvt(rc, CV_RGB(0,255,0))); // green
|
||||
}
|
||||
|
||||
std::cout << "Detections:";
|
||||
for (auto &&rc : in_obj_rcs) std::cout << ' ' << rc;
|
||||
std::cout << std::endl;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace custom
|
||||
|
||||
namespace {
|
||||
void remap_ssd_ports(const std::unordered_map<std::string, cv::Mat> &onnx,
|
||||
std::unordered_map<std::string, cv::Mat> &gapi) {
|
||||
// Assemble ONNX-processed outputs back to a single 1x1x200x7 blob
|
||||
// to preserve compatibility with OpenVINO-based SSD pipeline
|
||||
const cv::Mat &num_detections = onnx.at("num_detections:0");
|
||||
const cv::Mat &detection_boxes = onnx.at("detection_boxes:0");
|
||||
const cv::Mat &detection_scores = onnx.at("detection_scores:0");
|
||||
const cv::Mat &detection_classes = onnx.at("detection_classes:0");
|
||||
|
||||
GAPI_Assert(num_detections.depth() == CV_32F);
|
||||
GAPI_Assert(detection_boxes.depth() == CV_32F);
|
||||
GAPI_Assert(detection_scores.depth() == CV_32F);
|
||||
GAPI_Assert(detection_classes.depth() == CV_32F);
|
||||
|
||||
cv::Mat &ssd_output = gapi.at("detection_output");
|
||||
|
||||
const int num_objects = static_cast<int>(num_detections.ptr<float>()[0]);
|
||||
const float *in_boxes = detection_boxes.ptr<float>();
|
||||
const float *in_scores = detection_scores.ptr<float>();
|
||||
const float *in_classes = detection_classes.ptr<float>();
|
||||
float *ptr = ssd_output.ptr<float>();
|
||||
|
||||
for (int i = 0; i < num_objects; i++) {
|
||||
ptr[0] = 0.f; // "image_id"
|
||||
ptr[1] = in_classes[i]; // "label"
|
||||
ptr[2] = in_scores[i]; // "confidence"
|
||||
ptr[3] = in_boxes[4*i + 1]; // left
|
||||
ptr[4] = in_boxes[4*i + 0]; // top
|
||||
ptr[5] = in_boxes[4*i + 3]; // right
|
||||
ptr[6] = in_boxes[4*i + 2]; // bottom
|
||||
|
||||
ptr += 7;
|
||||
in_boxes += 4;
|
||||
}
|
||||
if (num_objects < ssd_output.size[2]-1) {
|
||||
// put a -1 mark at the end of output blob if there is space left
|
||||
ptr[0] = -1.f;
|
||||
}
|
||||
}
|
||||
} // anonymous namespace
|
||||
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ input | | Path to the input video file }"
|
||||
"{ output | | (Optional) path to output video file }"
|
||||
"{ detm | | Path to an ONNX SSD object detection model (.onnx) }"
|
||||
;
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Prepare parameters first
|
||||
const std::string input = cmd.get<std::string>("input");
|
||||
const std::string output = cmd.get<std::string>("output");
|
||||
const auto obj_model_path = cmd.get<std::string>("detm");
|
||||
|
||||
auto obj_net = cv::gapi::onnx::Params<custom::ObjDetector>{obj_model_path}
|
||||
.cfgOutputLayers({"detection_output"})
|
||||
.cfgPostProc({cv::GMatDesc{CV_32F, {1,1,200,7}}}, remap_ssd_ports);
|
||||
auto kernels = cv::gapi::kernels<custom::OCVBBoxes>();
|
||||
auto networks = cv::gapi::networks(obj_net);
|
||||
|
||||
// Now build the graph
|
||||
cv::GMat in;
|
||||
auto blob = cv::gapi::infer<custom::ObjDetector>(in);
|
||||
cv::GArray<cv::Rect> rcs =
|
||||
cv::gapi::parseSSD(blob, cv::gapi::streaming::size(in), 0.5f, true, true);
|
||||
auto out = cv::gapi::wip::draw::render3ch(in, custom::BBoxes::on(rcs));
|
||||
cv::GStreamingCompiled pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out))
|
||||
.compileStreaming(cv::compile_args(kernels, networks));
|
||||
|
||||
auto inputs = cv::gin(cv::gapi::wip::make_src<cv::gapi::wip::GCaptureSource>(input));
|
||||
|
||||
// The execution part
|
||||
pipeline.setSource(std::move(inputs));
|
||||
|
||||
cv::TickMeter tm;
|
||||
cv::VideoWriter writer;
|
||||
size_t frames = 0u;
|
||||
cv::Mat outMat;
|
||||
|
||||
tm.start();
|
||||
pipeline.start();
|
||||
while (pipeline.pull(cv::gout(outMat))) {
|
||||
++frames;
|
||||
cv::imshow("Out", outMat);
|
||||
cv::waitKey(1);
|
||||
if (!output.empty()) {
|
||||
if (!writer.isOpened()) {
|
||||
const auto sz = cv::Size{outMat.cols, outMat.rows};
|
||||
writer.open(output, cv::VideoWriter::fourcc('M','J','P','G'), 25.0, sz);
|
||||
CV_Assert(writer.isOpened());
|
||||
}
|
||||
writer << outMat;
|
||||
}
|
||||
}
|
||||
tm.stop();
|
||||
std::cout << "Processed " << frames << " frames" << " (" << frames / tm.getTimeSec() << " FPS)" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <opencv2/imgcodecs.hpp>
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/imgproc.hpp>
|
||||
#include <opencv2/gapi/infer.hpp>
|
||||
#include <opencv2/gapi/infer/parsers.hpp>
|
||||
#include <opencv2/gapi/render.hpp>
|
||||
#include <opencv2/gapi/cpu/gcpukernel.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
|
||||
#include <opencv2/gapi/oak/oak.hpp>
|
||||
#include <opencv2/gapi/oak/infer.hpp>
|
||||
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ detector | | Path to compiled .blob face detector model }"
|
||||
"{ duration | 100 | Number of frames to pull from camera and run inference on }";
|
||||
|
||||
namespace custom {
|
||||
|
||||
G_API_NET(FaceDetector, <cv::GMat(cv::GFrame)>, "sample.custom.face-detector");
|
||||
|
||||
using GDetections = cv::GArray<cv::Rect>;
|
||||
using GSize = cv::GOpaque<cv::Size>;
|
||||
using GPrims = cv::GArray<cv::gapi::wip::draw::Prim>;
|
||||
|
||||
G_API_OP(BBoxes, <GPrims(GDetections)>, "sample.custom.b-boxes") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc &) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) {
|
||||
// This kernel converts the rectangles into G-API's
|
||||
// rendering primitives
|
||||
static void run(const std::vector<cv::Rect> &in_face_rcs,
|
||||
std::vector<cv::gapi::wip::draw::Prim> &out_prims) {
|
||||
out_prims.clear();
|
||||
const auto cvt = [](const cv::Rect &rc, const cv::Scalar &clr) {
|
||||
return cv::gapi::wip::draw::Rect(rc, clr, 2);
|
||||
};
|
||||
for (auto &&rc : in_face_rcs) {
|
||||
out_prims.emplace_back(cvt(rc, CV_RGB(0,255,0))); // green
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace custom
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
const auto det_name = cmd.get<std::string>("detector");
|
||||
const auto duration = cmd.get<int>("duration");
|
||||
|
||||
if (det_name.empty()) {
|
||||
std::cerr << "FATAL: path to detection model is not provided for the sample."
|
||||
<< "Please specify it with --detector options."
|
||||
<< std::endl;
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Prepare G-API kernels and networks packages:
|
||||
auto detector = cv::gapi::oak::Params<custom::FaceDetector>(det_name);
|
||||
auto networks = cv::gapi::networks(detector);
|
||||
|
||||
auto kernels = cv::gapi::combine(
|
||||
cv::gapi::kernels<custom::OCVBBoxes>(),
|
||||
cv::gapi::oak::kernels());
|
||||
|
||||
auto args = cv::compile_args(kernels, networks);
|
||||
|
||||
// Initialize graph structure
|
||||
cv::GFrame in;
|
||||
cv::GFrame copy = cv::gapi::oak::copy(in); // NV12 transfered to host + passthrough copy for infer
|
||||
cv::GOpaque<cv::Size> sz = cv::gapi::streaming::size(copy);
|
||||
|
||||
// infer is not affected by the actual copy here
|
||||
cv::GMat blob = cv::gapi::infer<custom::FaceDetector>(copy);
|
||||
// FIXME: OAK infer detects faces slightly out of frame bounds
|
||||
cv::GArray<cv::Rect> rcs = cv::gapi::parseSSD(blob, sz, 0.5f, true, false);
|
||||
auto rendered = cv::gapi::wip::draw::renderFrame(copy, custom::BBoxes::on(rcs));
|
||||
// on-the-fly conversion NV12->BGR
|
||||
cv::GMat out = cv::gapi::streaming::BGR(rendered);
|
||||
|
||||
auto pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out, rcs))
|
||||
.compileStreaming(std::move(args));
|
||||
|
||||
// Graph execution
|
||||
pipeline.setSource(cv::gapi::wip::make_src<cv::gapi::oak::ColorCamera>());
|
||||
pipeline.start();
|
||||
|
||||
cv::Mat out_mat;
|
||||
std::vector<cv::Rect> out_dets;
|
||||
int frames = 0;
|
||||
while (pipeline.pull(cv::gout(out_mat, out_dets))) {
|
||||
std::string name = "oak_infer_frame_" + std::to_string(frames) + ".png";
|
||||
|
||||
cv::imwrite(name, out_mat);
|
||||
|
||||
if (!out_dets.empty()) {
|
||||
std::cout << "Got " << out_dets.size() << " detections on frame #" << frames << std::endl;
|
||||
}
|
||||
|
||||
++frames;
|
||||
if (frames == duration) {
|
||||
pipeline.stop();
|
||||
break;
|
||||
}
|
||||
}
|
||||
std::cout << "Pipeline finished. Processed " << frames << " frames" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/cpu/core.hpp>
|
||||
#include <opencv2/gapi/gframe.hpp>
|
||||
#include <opencv2/gapi/media.hpp>
|
||||
|
||||
#include <opencv2/gapi/oak/oak.hpp>
|
||||
#include <opencv2/gapi/streaming/format.hpp> // BGR accessor
|
||||
|
||||
#include <opencv2/highgui.hpp> // CommandLineParser
|
||||
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ output | output.png | Path to the output file }";
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
const std::string output_name = cmd.get<std::string>("output");
|
||||
|
||||
cv::GFrame in;
|
||||
// Actually transfers data to host
|
||||
cv::GFrame copy = cv::gapi::oak::copy(in);
|
||||
// Default camera works only with nv12 format
|
||||
cv::GMat out = cv::gapi::streaming::Y(copy);
|
||||
|
||||
auto args = cv::compile_args(cv::gapi::oak::ColorCameraParams{},
|
||||
cv::gapi::oak::kernels());
|
||||
|
||||
auto pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)).compileStreaming(std::move(args));
|
||||
|
||||
// Graph execution /////////////////////////////////////////////////////////
|
||||
cv::Mat out_mat(1920, 1080, CV_8UC1);
|
||||
|
||||
pipeline.setSource(cv::gapi::wip::make_src<cv::gapi::oak::ColorCamera>());
|
||||
pipeline.start();
|
||||
|
||||
// pull 1 frame
|
||||
pipeline.pull(cv::gout(out_mat));
|
||||
|
||||
cv::imwrite(output_name, out_mat);
|
||||
|
||||
std::cout << "Pipeline finished: " << output_name << " file has been written." << std::endl;
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
#include <fstream>
|
||||
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/gframe.hpp>
|
||||
|
||||
#include <opencv2/gapi/oak/oak.hpp>
|
||||
#include <opencv2/gapi/streaming/format.hpp> // BGR accessor
|
||||
|
||||
#include <opencv2/highgui.hpp> // CommandLineParser
|
||||
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ output | output.h265 | Path to the output .h265 video file }";
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
const std::string output_name = cmd.get<std::string>("output");
|
||||
|
||||
cv::gapi::oak::EncoderConfig cfg;
|
||||
cfg.profile = cv::gapi::oak::EncoderConfig::Profile::H265_MAIN;
|
||||
|
||||
cv::GFrame in;
|
||||
cv::GArray<uint8_t> encoded = cv::gapi::oak::encode(in, cfg);
|
||||
|
||||
auto args = cv::compile_args(cv::gapi::oak::ColorCameraParams{}, cv::gapi::oak::kernels());
|
||||
|
||||
auto pipeline = cv::GComputation(cv::GIn(in), cv::GOut(encoded)).compileStreaming(std::move(args));
|
||||
|
||||
// Graph execution /////////////////////////////////////////////////////////
|
||||
pipeline.setSource(cv::gapi::wip::make_src<cv::gapi::oak::ColorCamera>());
|
||||
pipeline.start();
|
||||
|
||||
std::vector<uint8_t> out_h265_data;
|
||||
|
||||
std::ofstream out_h265_file;
|
||||
out_h265_file.open(output_name, std::ofstream::out | std::ofstream::binary | std::ofstream::trunc);
|
||||
|
||||
// Pull 300 frames from the camera
|
||||
uint32_t frames = 300;
|
||||
uint32_t pulled = 0;
|
||||
|
||||
while (pipeline.pull(cv::gout(out_h265_data))) {
|
||||
if (out_h265_file.is_open()) {
|
||||
out_h265_file.write(reinterpret_cast<const char*>(out_h265_data.data()),
|
||||
out_h265_data.size());
|
||||
}
|
||||
if (pulled++ == frames) {
|
||||
pipeline.stop();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << "Pipeline finished: " << output_name << " file has been written." << std::endl;
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/cpu/core.hpp>
|
||||
#include <opencv2/gapi/gframe.hpp>
|
||||
#include <opencv2/gapi/media.hpp>
|
||||
|
||||
#include <opencv2/gapi/oak/oak.hpp>
|
||||
#include <opencv2/gapi/streaming/format.hpp> // BGR accessor
|
||||
|
||||
#include <opencv2/highgui.hpp> // CommandLineParser
|
||||
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ output | output.png | Path to the output file }";
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
const std::string output_name = cmd.get<std::string>("output");
|
||||
|
||||
std::vector<int> h = {1, 0, -1,
|
||||
2, 0, -2,
|
||||
1, 0, -1};
|
||||
std::vector<int> v = { 1, 2, 1,
|
||||
0, 0, 0,
|
||||
-1, -2, -1};
|
||||
cv::Mat hk(3, 3, CV_32SC1, h.data());
|
||||
cv::Mat vk(3, 3, CV_32SC1, v.data());
|
||||
|
||||
// Heterogeneous pipeline:
|
||||
// OAK camera -> Sobel -> streaming accessor (CPU)
|
||||
cv::GFrame in;
|
||||
cv::GFrame sobel = cv::gapi::oak::sobelXY(in, hk, vk);
|
||||
// Default camera and then sobel work only with nv12 format
|
||||
cv::GMat out = cv::gapi::streaming::Y(sobel);
|
||||
|
||||
auto args = cv::compile_args(cv::gapi::oak::ColorCameraParams{},
|
||||
cv::gapi::oak::kernels());
|
||||
|
||||
auto pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)).compileStreaming(std::move(args));
|
||||
|
||||
// Graph execution /////////////////////////////////////////////////////////
|
||||
cv::Mat out_mat(1920, 1080, CV_8UC1);
|
||||
|
||||
pipeline.setSource(cv::gapi::wip::make_src<cv::gapi::oak::ColorCamera>());
|
||||
pipeline.start();
|
||||
|
||||
// pull 1 frame
|
||||
pipeline.pull(cv::gout(out_mat));
|
||||
|
||||
cv::imwrite(output_name, out_mat);
|
||||
|
||||
std::cout << "Pipeline finished: " << output_name << " file has been written." << std::endl;
|
||||
}
|
||||
@@ -0,0 +1,704 @@
|
||||
#include <algorithm>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <cctype>
|
||||
#include <tuple>
|
||||
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/cpu/gcpukernel.hpp>
|
||||
#include <opencv2/gapi/infer/ie.hpp>
|
||||
#include <opencv2/gapi/render.hpp>
|
||||
#include <opencv2/gapi/streaming/onevpl/source.hpp>
|
||||
#include <opencv2/highgui.hpp> // CommandLineParser
|
||||
#include <opencv2/gapi/infer/parsers.hpp>
|
||||
|
||||
#ifdef HAVE_INF_ENGINE
|
||||
#include <inference_engine.hpp> // ParamMap
|
||||
#endif // HAVE_INF_ENGINE
|
||||
|
||||
#ifdef HAVE_DIRECTX
|
||||
#ifdef HAVE_D3D11
|
||||
|
||||
// get rid of generate macro max/min/etc from DX side
|
||||
#define D3D11_NO_HELPERS
|
||||
#define NOMINMAX
|
||||
#include <d3d11.h>
|
||||
#undef NOMINMAX
|
||||
#undef D3D11_NO_HELPERS
|
||||
#endif // HAVE_D3D11
|
||||
#endif // HAVE_DIRECTX
|
||||
|
||||
#ifdef __linux__
|
||||
#if defined(HAVE_VA) || defined(HAVE_VA_INTEL)
|
||||
#include "va/va.h"
|
||||
#include "va/va_drm.h"
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#endif // defined(HAVE_VA) || defined(HAVE_VA_INTEL)
|
||||
#endif // __linux__
|
||||
|
||||
|
||||
const std::string about =
|
||||
"This is an OpenCV-based version of oneVPLSource decoder example";
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ input | | Path to the input demultiplexed video file }"
|
||||
"{ output | | Path to the output RAW video file. Use .avi extension }"
|
||||
"{ facem | face-detection-adas-0001.xml | Path to OpenVINO IE face detection model (.xml) }"
|
||||
"{ faced | GPU | Target device for face detection model (e.g. AUTO, GPU, VPU, ...) }"
|
||||
"{ cfg_params | | Semicolon separated list of oneVPL mfxVariants which is used for configuring source (see `MFXSetConfigFilterProperty` by https://spec.oneapi.io/versions/latest/elements/oneVPL/source/index.html) }"
|
||||
"{ streaming_queue_capacity | 1 | Streaming executor queue capacity. Calculated automatically if 0 }"
|
||||
"{ frames_pool_size | 0 | OneVPL source applies this parameter as preallocated frames pool size}"
|
||||
"{ vpp_frames_pool_size | 0 | OneVPL source applies this parameter as preallocated frames pool size for VPP preprocessing results}"
|
||||
"{ roi | -1,-1,-1,-1 | Region of interest (ROI) to use for inference. Identified automatically when not set }"
|
||||
"{ source_device | CPU | choose device for decoding }"
|
||||
"{ preproc_device | | choose device for preprocessing }";
|
||||
|
||||
|
||||
namespace {
|
||||
bool is_gpu(const std::string &device_name) {
|
||||
return device_name.find("GPU") != std::string::npos;
|
||||
}
|
||||
|
||||
std::string get_weights_path(const std::string &model_path) {
|
||||
const auto EXT_LEN = 4u;
|
||||
const auto sz = model_path.size();
|
||||
GAPI_Assert(sz > EXT_LEN);
|
||||
|
||||
auto ext = model_path.substr(sz - EXT_LEN);
|
||||
std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c){
|
||||
return static_cast<unsigned char>(std::tolower(c));
|
||||
});
|
||||
GAPI_Assert(ext == ".xml");
|
||||
return model_path.substr(0u, sz - EXT_LEN) + ".bin";
|
||||
}
|
||||
|
||||
// TODO: It duplicates infer_single_roi sample
|
||||
cv::util::optional<cv::Rect> parse_roi(const std::string &rc) {
|
||||
cv::Rect rv;
|
||||
char delim[3];
|
||||
|
||||
std::stringstream is(rc);
|
||||
is >> rv.x >> delim[0] >> rv.y >> delim[1] >> rv.width >> delim[2] >> rv.height;
|
||||
if (is.bad()) {
|
||||
return cv::util::optional<cv::Rect>(); // empty value
|
||||
}
|
||||
const auto is_delim = [](char c) {
|
||||
return c == ',';
|
||||
};
|
||||
if (!std::all_of(std::begin(delim), std::end(delim), is_delim)) {
|
||||
return cv::util::optional<cv::Rect>(); // empty value
|
||||
}
|
||||
if (rv.x < 0 || rv.y < 0 || rv.width <= 0 || rv.height <= 0) {
|
||||
return cv::util::optional<cv::Rect>(); // empty value
|
||||
}
|
||||
return cv::util::make_optional(std::move(rv));
|
||||
}
|
||||
|
||||
#ifdef HAVE_DIRECTX
|
||||
#ifdef HAVE_D3D11
|
||||
|
||||
// Since ATL headers might not be available on specific MSVS Build Tools
|
||||
// we use simple `CComPtr` implementation like as `ComPtrGuard`
|
||||
// which is not supposed to be the full functional replacement of `CComPtr`
|
||||
// and it uses as RAII to make sure utilization is correct
|
||||
template <typename COMNonManageableType>
|
||||
void release(COMNonManageableType *ptr) {
|
||||
if (ptr) {
|
||||
ptr->Release();
|
||||
}
|
||||
}
|
||||
|
||||
template <typename COMNonManageableType>
|
||||
using ComPtrGuard = std::unique_ptr<COMNonManageableType, decltype(&release<COMNonManageableType>)>;
|
||||
|
||||
template <typename COMNonManageableType>
|
||||
ComPtrGuard<COMNonManageableType> createCOMPtrGuard(COMNonManageableType *ptr = nullptr) {
|
||||
return ComPtrGuard<COMNonManageableType> {ptr, &release<COMNonManageableType>};
|
||||
}
|
||||
|
||||
|
||||
using AccelParamsType = std::tuple<ComPtrGuard<ID3D11Device>, ComPtrGuard<ID3D11DeviceContext>>;
|
||||
|
||||
AccelParamsType create_device_with_ctx(IDXGIAdapter* adapter) {
|
||||
UINT flags = 0;
|
||||
D3D_FEATURE_LEVEL feature_levels[] = { D3D_FEATURE_LEVEL_11_1,
|
||||
D3D_FEATURE_LEVEL_11_0,
|
||||
};
|
||||
D3D_FEATURE_LEVEL featureLevel;
|
||||
ID3D11Device* ret_device_ptr = nullptr;
|
||||
ID3D11DeviceContext* ret_ctx_ptr = nullptr;
|
||||
HRESULT err = D3D11CreateDevice(adapter, D3D_DRIVER_TYPE_UNKNOWN,
|
||||
nullptr, flags,
|
||||
feature_levels,
|
||||
ARRAYSIZE(feature_levels),
|
||||
D3D11_SDK_VERSION, &ret_device_ptr,
|
||||
&featureLevel, &ret_ctx_ptr);
|
||||
if (FAILED(err)) {
|
||||
throw std::runtime_error("Cannot create D3D11CreateDevice, error: " +
|
||||
std::to_string(HRESULT_CODE(err)));
|
||||
}
|
||||
|
||||
return std::make_tuple(createCOMPtrGuard(ret_device_ptr),
|
||||
createCOMPtrGuard(ret_ctx_ptr));
|
||||
}
|
||||
#endif // HAVE_D3D11
|
||||
#endif // HAVE_DIRECTX
|
||||
} // anonymous namespace
|
||||
|
||||
namespace custom {
|
||||
G_API_NET(FaceDetector, <cv::GMat(cv::GMat)>, "face-detector");
|
||||
|
||||
using GDetections = cv::GArray<cv::Rect>;
|
||||
using GRect = cv::GOpaque<cv::Rect>;
|
||||
using GSize = cv::GOpaque<cv::Size>;
|
||||
using GPrims = cv::GArray<cv::gapi::wip::draw::Prim>;
|
||||
|
||||
G_API_OP(ParseSSD, <GDetections(cv::GMat, GRect, GSize)>, "sample.custom.parse-ssd") {
|
||||
static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GOpaqueDesc &, const cv::GOpaqueDesc &) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
// TODO: It duplicates infer_single_roi sample
|
||||
G_API_OP(LocateROI, <GRect(GSize)>, "sample.custom.locate-roi") {
|
||||
static cv::GOpaqueDesc outMeta(const cv::GOpaqueDesc &) {
|
||||
return cv::empty_gopaque_desc();
|
||||
}
|
||||
};
|
||||
|
||||
G_API_OP(BBoxes, <GPrims(GDetections, GRect)>, "sample.custom.b-boxes") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc &, const cv::GOpaqueDesc &) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL(OCVLocateROI, LocateROI) {
|
||||
// This is the place where we can run extra analytics
|
||||
// on the input image frame and select the ROI (region
|
||||
// of interest) where we want to detect our objects (or
|
||||
// run any other inference).
|
||||
//
|
||||
// Currently it doesn't do anything intelligent,
|
||||
// but only crops the input image to square (this is
|
||||
// the most convenient aspect ratio for detectors to use)
|
||||
|
||||
static void run(const cv::Size& in_size,
|
||||
cv::Rect &out_rect) {
|
||||
|
||||
// Identify the central point & square size (- some padding)
|
||||
const auto center = cv::Point{in_size.width/2, in_size.height/2};
|
||||
auto sqside = std::min(in_size.width, in_size.height);
|
||||
|
||||
// Now build the central square ROI
|
||||
out_rect = cv::Rect{ center.x - sqside/2
|
||||
, center.y - sqside/2
|
||||
, sqside
|
||||
, sqside
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) {
|
||||
// This kernel converts the rectangles into G-API's
|
||||
// rendering primitives
|
||||
static void run(const std::vector<cv::Rect> &in_face_rcs,
|
||||
const cv::Rect &in_roi,
|
||||
std::vector<cv::gapi::wip::draw::Prim> &out_prims) {
|
||||
out_prims.clear();
|
||||
const auto cvt = [](const cv::Rect &rc, const cv::Scalar &clr) {
|
||||
return cv::gapi::wip::draw::Rect(rc, clr, 2);
|
||||
};
|
||||
out_prims.emplace_back(cvt(in_roi, CV_RGB(0,255,255))); // cyan
|
||||
for (auto &&rc : in_face_rcs) {
|
||||
out_prims.emplace_back(cvt(rc, CV_RGB(0,255,0))); // green
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL(OCVParseSSD, ParseSSD) {
|
||||
static void run(const cv::Mat &in_ssd_result,
|
||||
const cv::Rect &in_roi,
|
||||
const cv::Size &in_parent_size,
|
||||
std::vector<cv::Rect> &out_objects) {
|
||||
const auto &in_ssd_dims = in_ssd_result.size;
|
||||
GAPI_Assert(in_ssd_dims.dims() == 4u);
|
||||
|
||||
const int MAX_PROPOSALS = in_ssd_dims[2];
|
||||
const int OBJECT_SIZE = in_ssd_dims[3];
|
||||
GAPI_Assert(OBJECT_SIZE == 7); // fixed SSD object size
|
||||
|
||||
const cv::Size up_roi = in_roi.size();
|
||||
const cv::Rect surface({0,0}, in_parent_size);
|
||||
|
||||
out_objects.clear();
|
||||
|
||||
const float *data = in_ssd_result.ptr<float>();
|
||||
for (int i = 0; i < MAX_PROPOSALS; i++) {
|
||||
const float image_id = data[i * OBJECT_SIZE + 0];
|
||||
const float label = data[i * OBJECT_SIZE + 1];
|
||||
const float confidence = data[i * OBJECT_SIZE + 2];
|
||||
const float rc_left = data[i * OBJECT_SIZE + 3];
|
||||
const float rc_top = data[i * OBJECT_SIZE + 4];
|
||||
const float rc_right = data[i * OBJECT_SIZE + 5];
|
||||
const float rc_bottom = data[i * OBJECT_SIZE + 6];
|
||||
(void) label; // unused
|
||||
|
||||
if (image_id < 0.f) {
|
||||
break; // marks end-of-detections
|
||||
}
|
||||
if (confidence < 0.5f) {
|
||||
continue; // skip objects with low confidence
|
||||
}
|
||||
|
||||
// map relative coordinates to the original image scale
|
||||
// taking the ROI into account
|
||||
cv::Rect rc;
|
||||
rc.x = static_cast<int>(rc_left * up_roi.width);
|
||||
rc.y = static_cast<int>(rc_top * up_roi.height);
|
||||
rc.width = static_cast<int>(rc_right * up_roi.width) - rc.x;
|
||||
rc.height = static_cast<int>(rc_bottom * up_roi.height) - rc.y;
|
||||
rc.x += in_roi.x;
|
||||
rc.y += in_roi.y;
|
||||
out_objects.emplace_back(rc & surface);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace custom
|
||||
|
||||
namespace cfg {
|
||||
typename cv::gapi::wip::onevpl::CfgParam create_from_string(const std::string &line);
|
||||
|
||||
struct flow {
|
||||
flow(bool preproc, bool rctx) :
|
||||
vpl_preproc_enable(preproc),
|
||||
ie_remote_ctx_enable(rctx) {
|
||||
}
|
||||
bool vpl_preproc_enable = false;
|
||||
bool ie_remote_ctx_enable = false;
|
||||
};
|
||||
|
||||
using support_matrix =
|
||||
std::map <std::string/*source_dev_id*/,
|
||||
std::map<std::string/*preproc_device_id*/,
|
||||
std::map <std::string/*rctx device_id*/, std::shared_ptr<flow>>>>;
|
||||
support_matrix resolved_conf{{
|
||||
{"GPU", {{
|
||||
{"", {{ "CPU", std::make_shared<flow>(false, false)},
|
||||
{ "GPU", {/* unsupported:
|
||||
* ie GPU preproc isn't available */}}
|
||||
}},
|
||||
|
||||
{"CPU", {{ "CPU", {/* unsupported: preproc mix */}},
|
||||
{ "GPU", {/* unsupported: preproc mix */}}
|
||||
}},
|
||||
#if defined(HAVE_DIRECTX) && defined(HAVE_D3D11)
|
||||
{"GPU", {{ "CPU", std::make_shared<flow>(true, false)},
|
||||
{ "GPU", std::make_shared<flow>(true, true)}}}
|
||||
#else // TODO VAAPI under linux doesn't support GPU IE remote context
|
||||
{"GPU", {{ "CPU", std::make_shared<flow>(true, false)},
|
||||
{ "GPU", std::make_shared<flow>(true, false)}}}
|
||||
#endif
|
||||
}}
|
||||
},
|
||||
{"CPU", {{
|
||||
{"", {{ "CPU", std::make_shared<flow>(false, false)},
|
||||
{ "GPU", std::make_shared<flow>(false, false)}
|
||||
}},
|
||||
|
||||
{"CPU", {{ "CPU", std::make_shared<flow>(true, false)},
|
||||
{ "GPU", std::make_shared<flow>(true, false)}
|
||||
}},
|
||||
|
||||
{"GPU", {{ "CPU", {/* unsupported: preproc mix */}},
|
||||
{ "GPU", {/* unsupported: preproc mix */}}}}
|
||||
}}
|
||||
}
|
||||
}};
|
||||
|
||||
static void print_available_cfg(std::ostream &out,
|
||||
const std::string &source_device,
|
||||
const std::string &preproc_device,
|
||||
const std::string &ie_device_id) {
|
||||
const std::string source_device_cfg_name("--source_device=");
|
||||
const std::string preproc_device_cfg_name("--preproc_device=");
|
||||
const std::string ie_cfg_name("--faced=");
|
||||
out << "unsupported acceleration param combinations:\n"
|
||||
<< source_device_cfg_name << source_device << " "
|
||||
<< preproc_device_cfg_name << preproc_device << " "
|
||||
<< ie_cfg_name << ie_device_id <<
|
||||
"\n\nSupported matrix:\n\n" << std::endl;
|
||||
for (const auto &s_d : cfg::resolved_conf) {
|
||||
std::string prefix = source_device_cfg_name + s_d.first;
|
||||
for (const auto &p_d : s_d.second) {
|
||||
std::string mid_prefix = prefix + +"\t" + preproc_device_cfg_name +
|
||||
(p_d.first.empty() ? "" : p_d.first);
|
||||
for (const auto &i_d : p_d.second) {
|
||||
if (i_d.second) {
|
||||
std::cerr << mid_prefix << "\t" << ie_cfg_name <<i_d.first << std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
cmd.about(about);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
// get file name
|
||||
const auto file_path = cmd.get<std::string>("input");
|
||||
const auto output = cmd.get<std::string>("output");
|
||||
const auto opt_roi = parse_roi(cmd.get<std::string>("roi"));
|
||||
const auto face_model_path = cmd.get<std::string>("facem");
|
||||
const auto streaming_queue_capacity = cmd.get<uint32_t>("streaming_queue_capacity");
|
||||
const auto source_decode_queue_capacity = cmd.get<uint32_t>("frames_pool_size");
|
||||
const auto source_vpp_queue_capacity = cmd.get<uint32_t>("vpp_frames_pool_size");
|
||||
const auto device_id = cmd.get<std::string>("faced");
|
||||
const auto source_device = cmd.get<std::string>("source_device");
|
||||
const auto preproc_device = cmd.get<std::string>("preproc_device");
|
||||
|
||||
// validate support matrix
|
||||
std::shared_ptr<cfg::flow> flow_settings = cfg::resolved_conf[source_device][preproc_device][device_id];
|
||||
if (!flow_settings) {
|
||||
cfg::print_available_cfg(std::cerr, source_device, preproc_device, device_id);
|
||||
return -1;
|
||||
}
|
||||
|
||||
// check output file extension
|
||||
if (!output.empty()) {
|
||||
auto ext = output.find_last_of(".");
|
||||
if (ext == std::string::npos || (output.substr(ext + 1) != "avi")) {
|
||||
std::cerr << "Output file should have *.avi extension for output video" << std::endl;
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
// get oneVPL cfg params from cmd
|
||||
std::stringstream params_list(cmd.get<std::string>("cfg_params"));
|
||||
std::vector<cv::gapi::wip::onevpl::CfgParam> source_cfgs;
|
||||
try {
|
||||
std::string line;
|
||||
while (std::getline(params_list, line, ';')) {
|
||||
source_cfgs.push_back(cfg::create_from_string(line));
|
||||
}
|
||||
} catch (const std::exception& ex) {
|
||||
std::cerr << "Invalid cfg parameter: " << ex.what() << std::endl;
|
||||
return -1;
|
||||
}
|
||||
|
||||
// apply VPL source optimization params
|
||||
if (source_decode_queue_capacity != 0) {
|
||||
source_cfgs.push_back(cv::gapi::wip::onevpl::CfgParam::create_frames_pool_size(source_decode_queue_capacity));
|
||||
}
|
||||
if (source_vpp_queue_capacity != 0) {
|
||||
source_cfgs.push_back(cv::gapi::wip::onevpl::CfgParam::create_vpp_frames_pool_size(source_vpp_queue_capacity));
|
||||
}
|
||||
|
||||
auto face_net = cv::gapi::ie::Params<custom::FaceDetector> {
|
||||
face_model_path, // path to topology IR
|
||||
get_weights_path(face_model_path), // path to weights
|
||||
device_id
|
||||
};
|
||||
|
||||
// It is allowed (and highly recommended) to reuse predefined device_ptr & context_ptr objects
|
||||
// received from user application. Current sample demonstrate how to deal with this situation.
|
||||
//
|
||||
// But if you do not need this fine-grained acceleration devices configuration then
|
||||
// just use default constructors for onevpl::GSource, IE and preprocessing module.
|
||||
// But please pay attention that default pipeline construction in this case will be
|
||||
// very inefficient and carries out multiple CPU-GPU memory copies
|
||||
//
|
||||
// If you want to reach max performance and seize copy-free approach for specific
|
||||
// device & context selection then follow the steps below.
|
||||
// The situation is complicated a little bit in comparison with default configuration, thus
|
||||
// let's focusing this:
|
||||
//
|
||||
// - all component-participants (Source, Preprocessing, Inference)
|
||||
// must share the same device & context instances
|
||||
//
|
||||
// - you must wrapping your available device & context instancs into thin
|
||||
// `cv::gapi::wip::Device` & `cv::gapi::wip::Context`.
|
||||
// !!! Please pay attention that both objects are weak wrapper so you must ensure
|
||||
// that device & context would be alived before full pipeline created !!!
|
||||
//
|
||||
// - you should pass such wrappers as constructor arguments for each component in pipeline:
|
||||
// a) use extended constructor for `onevpl::GSource` for activating predefined device & context
|
||||
// b) use `cfgContextParams` method of `cv::gapi::ie::Params` to enable `PreprocessingEngine`
|
||||
// for predefined device & context
|
||||
// c) use `InferenceEngine::ParamMap` to activate remote ctx in Inference Engine for given
|
||||
// device & context
|
||||
//
|
||||
//
|
||||
//// P.S. the current sample supports heterogenous pipeline construction also.
|
||||
//// It is possible to make up mixed device approach.
|
||||
//// Please feel free to explore different configurations!
|
||||
|
||||
cv::util::optional<cv::gapi::wip::onevpl::Device> gpu_accel_device;
|
||||
cv::util::optional<cv::gapi::wip::onevpl::Context> gpu_accel_ctx;
|
||||
cv::gapi::wip::onevpl::Device cpu_accel_device = cv::gapi::wip::onevpl::create_host_device();
|
||||
cv::gapi::wip::onevpl::Context cpu_accel_ctx = cv::gapi::wip::onevpl::create_host_context();
|
||||
// create GPU device if requested
|
||||
if (is_gpu(device_id)
|
||||
|| is_gpu(source_device)
|
||||
|| is_gpu(preproc_device)) {
|
||||
#ifdef HAVE_DIRECTX
|
||||
#ifdef HAVE_D3D11
|
||||
// create DX11 device & context owning handles.
|
||||
// wip::Device & wip::Context provide non-owning semantic of resources and act
|
||||
// as weak references API wrappers in order to carry type-erased resources type
|
||||
// into appropriate modules: onevpl::GSource, PreprocEngine and InferenceEngine
|
||||
// Until modules are not created owner handles must stay alive
|
||||
auto dx11_dev = createCOMPtrGuard<ID3D11Device>();
|
||||
auto dx11_ctx = createCOMPtrGuard<ID3D11DeviceContext>();
|
||||
|
||||
auto adapter_factory = createCOMPtrGuard<IDXGIFactory>();
|
||||
{
|
||||
IDXGIFactory* out_factory = nullptr;
|
||||
HRESULT err = CreateDXGIFactory(__uuidof(IDXGIFactory),
|
||||
reinterpret_cast<void**>(&out_factory));
|
||||
if (FAILED(err)) {
|
||||
std::cerr << "Cannot create CreateDXGIFactory, error: " << HRESULT_CODE(err) << std::endl;
|
||||
return -1;
|
||||
}
|
||||
adapter_factory = createCOMPtrGuard(out_factory);
|
||||
}
|
||||
|
||||
auto intel_adapter = createCOMPtrGuard<IDXGIAdapter>();
|
||||
UINT adapter_index = 0;
|
||||
const unsigned int refIntelVendorID = 0x8086;
|
||||
IDXGIAdapter* out_adapter = nullptr;
|
||||
|
||||
while (adapter_factory->EnumAdapters(adapter_index, &out_adapter) != DXGI_ERROR_NOT_FOUND) {
|
||||
DXGI_ADAPTER_DESC desc{};
|
||||
out_adapter->GetDesc(&desc);
|
||||
if (desc.VendorId == refIntelVendorID) {
|
||||
intel_adapter = createCOMPtrGuard(out_adapter);
|
||||
break;
|
||||
}
|
||||
++adapter_index;
|
||||
}
|
||||
|
||||
if (!intel_adapter) {
|
||||
std::cerr << "No Intel GPU adapter on aboard. Exit" << std::endl;
|
||||
return -1;
|
||||
}
|
||||
|
||||
std::tie(dx11_dev, dx11_ctx) = create_device_with_ctx(intel_adapter.get());
|
||||
gpu_accel_device = cv::util::make_optional(
|
||||
cv::gapi::wip::onevpl::create_dx11_device(
|
||||
reinterpret_cast<void*>(dx11_dev.release()),
|
||||
"GPU"));
|
||||
gpu_accel_ctx = cv::util::make_optional(
|
||||
cv::gapi::wip::onevpl::create_dx11_context(
|
||||
reinterpret_cast<void*>(dx11_ctx.release())));
|
||||
#endif // HAVE_D3D11
|
||||
#endif // HAVE_DIRECTX
|
||||
#ifdef __linux__
|
||||
#if defined(HAVE_VA) || defined(HAVE_VA_INTEL)
|
||||
static const char *predefined_vaapi_devices_list[] {"/dev/dri/renderD128",
|
||||
"/dev/dri/renderD129",
|
||||
"/dev/dri/card0",
|
||||
"/dev/dri/card1",
|
||||
nullptr};
|
||||
std::stringstream ss;
|
||||
int device_fd = -1;
|
||||
VADisplay va_handle = nullptr;
|
||||
for (const char **device_path = predefined_vaapi_devices_list;
|
||||
*device_path != nullptr; device_path++) {
|
||||
device_fd = open(*device_path, O_RDWR);
|
||||
if (device_fd < 0) {
|
||||
std::string info("Cannot open GPU file: \"");
|
||||
info = info + *device_path + "\", error: " + strerror(errno);
|
||||
ss << info << std::endl;
|
||||
continue;
|
||||
}
|
||||
va_handle = vaGetDisplayDRM(device_fd);
|
||||
if (!va_handle) {
|
||||
close(device_fd);
|
||||
std::string info("VAAPI device vaGetDisplayDRM failed, error: ");
|
||||
info += strerror(errno);
|
||||
ss << info << std::endl;
|
||||
continue;
|
||||
}
|
||||
int major_version = 0, minor_version = 0;
|
||||
VAStatus status {};
|
||||
status = vaInitialize(va_handle, &major_version, &minor_version);
|
||||
if (VA_STATUS_SUCCESS != status) {
|
||||
close(device_fd);
|
||||
va_handle = nullptr;
|
||||
|
||||
std::string info("Cannot initialize VAAPI device, error: ");
|
||||
info += vaErrorStr(status);
|
||||
ss << info << std::endl;
|
||||
continue;
|
||||
}
|
||||
std::cout << "VAAPI created for device: " << *device_path << ", version: "
|
||||
<< major_version << "." << minor_version << std::endl;
|
||||
break;
|
||||
}
|
||||
|
||||
// check device creation
|
||||
if (!va_handle) {
|
||||
std::cerr << "Cannot create VAAPI device. Log:\n" << ss.str() << std::endl;
|
||||
return -1;
|
||||
}
|
||||
gpu_accel_device = cv::util::make_optional(
|
||||
cv::gapi::wip::onevpl::create_vaapi_device(reinterpret_cast<void*>(va_handle),
|
||||
"GPU"));
|
||||
gpu_accel_ctx = cv::util::make_optional(
|
||||
cv::gapi::wip::onevpl::create_vaapi_context(nullptr));
|
||||
#endif // defined(HAVE_VA) || defined(HAVE_VA_INTEL)
|
||||
#endif // #ifdef __linux__
|
||||
}
|
||||
|
||||
#ifdef HAVE_INF_ENGINE
|
||||
// activate remote ctx in Inference Engine for GPU device
|
||||
// when other pipeline component use the GPU device too
|
||||
if (flow_settings->ie_remote_ctx_enable) {
|
||||
InferenceEngine::ParamMap ctx_config({{"CONTEXT_TYPE", "VA_SHARED"},
|
||||
{"VA_DEVICE", gpu_accel_device.value().get_ptr()} });
|
||||
face_net.cfgContextParams(ctx_config);
|
||||
std::cout << "enforce InferenceEngine remote context on device: " << device_id << std::endl;
|
||||
|
||||
// NB: consider NV12 surface because it's one of native GPU image format
|
||||
face_net.pluginConfig({{"GPU_NV12_TWO_INPUTS", "YES" }});
|
||||
std::cout << "enforce InferenceEngine NV12 blob" << std::endl;
|
||||
}
|
||||
#endif // HAVE_INF_ENGINE
|
||||
|
||||
// turn on VPP PreprocessingEngine if available & requested
|
||||
if (flow_settings->vpl_preproc_enable) {
|
||||
if (is_gpu(preproc_device)) {
|
||||
// activate VPP PreprocessingEngine on GPU
|
||||
face_net.cfgPreprocessingParams(gpu_accel_device.value(),
|
||||
gpu_accel_ctx.value());
|
||||
} else {
|
||||
// activate VPP PreprocessingEngine on CPU
|
||||
face_net.cfgPreprocessingParams(cpu_accel_device,
|
||||
cpu_accel_ctx);
|
||||
}
|
||||
std::cout << "enforce VPP preprocessing on device: " << preproc_device << std::endl;
|
||||
} else {
|
||||
std::cout << "use InferenceEngine default preprocessing" << std::endl;
|
||||
}
|
||||
|
||||
auto kernels = cv::gapi::kernels
|
||||
< custom::OCVLocateROI
|
||||
, custom::OCVParseSSD
|
||||
, custom::OCVBBoxes>();
|
||||
auto networks = cv::gapi::networks(face_net);
|
||||
auto face_detection_args = cv::compile_args(networks, kernels);
|
||||
if (streaming_queue_capacity != 0) {
|
||||
face_detection_args += cv::compile_args(cv::gapi::streaming::queue_capacity{ streaming_queue_capacity });
|
||||
}
|
||||
|
||||
// Create source
|
||||
cv::gapi::wip::IStreamSource::Ptr cap;
|
||||
try {
|
||||
if (is_gpu(source_device)) {
|
||||
std::cout << "enforce VPL Source deconding on device: " << source_device << std::endl;
|
||||
// use special 'Device' constructor for `onevpl::GSource`
|
||||
cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs,
|
||||
gpu_accel_device.value(),
|
||||
gpu_accel_ctx.value());
|
||||
} else {
|
||||
cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs);
|
||||
}
|
||||
std::cout << "oneVPL source description: " << cap->descr_of() << std::endl;
|
||||
} catch (const std::exception& ex) {
|
||||
std::cerr << "Cannot create source: " << ex.what() << std::endl;
|
||||
return -1;
|
||||
}
|
||||
|
||||
cv::GMetaArg descr = cap->descr_of();
|
||||
auto frame_descr = cv::util::get<cv::GFrameDesc>(descr);
|
||||
cv::GOpaque<cv::Rect> in_roi;
|
||||
auto inputs = cv::gin(cap);
|
||||
|
||||
// Now build the graph
|
||||
cv::GFrame in;
|
||||
auto size = cv::gapi::streaming::size(in);
|
||||
auto graph_inputs = cv::GIn(in);
|
||||
if (!opt_roi.has_value()) {
|
||||
// Automatically detect ROI to infer. Make it output parameter
|
||||
std::cout << "ROI is not set or invalid. Locating it automatically"
|
||||
<< std::endl;
|
||||
in_roi = custom::LocateROI::on(size);
|
||||
} else {
|
||||
// Use the value provided by user
|
||||
std::cout << "Will run inference for static region "
|
||||
<< opt_roi.value()
|
||||
<< " only"
|
||||
<< std::endl;
|
||||
graph_inputs += cv::GIn(in_roi);
|
||||
inputs += cv::gin(opt_roi.value());
|
||||
}
|
||||
auto blob = cv::gapi::infer<custom::FaceDetector>(in_roi, in);
|
||||
cv::GArray<cv::Rect> rcs = custom::ParseSSD::on(blob, in_roi, size);
|
||||
auto out_frame = cv::gapi::wip::draw::renderFrame(in, custom::BBoxes::on(rcs, in_roi));
|
||||
auto out = cv::gapi::streaming::BGR(out_frame);
|
||||
cv::GStreamingCompiled pipeline = cv::GComputation(std::move(graph_inputs), cv::GOut(out)) // and move here
|
||||
.compileStreaming(std::move(face_detection_args));
|
||||
// The execution part
|
||||
pipeline.setSource(std::move(inputs));
|
||||
pipeline.start();
|
||||
|
||||
size_t frames = 0u;
|
||||
cv::TickMeter tm;
|
||||
cv::VideoWriter writer;
|
||||
if (!output.empty() && !writer.isOpened()) {
|
||||
const auto sz = cv::Size{frame_descr.size.width, frame_descr.size.height};
|
||||
writer.open(output, cv::VideoWriter::fourcc('M','J','P','G'), 25.0, sz);
|
||||
GAPI_Assert(writer.isOpened());
|
||||
}
|
||||
|
||||
cv::Mat outMat;
|
||||
tm.start();
|
||||
while (pipeline.pull(cv::gout(outMat))) {
|
||||
cv::imshow("Out", outMat);
|
||||
cv::waitKey(1);
|
||||
if (!output.empty()) {
|
||||
writer << outMat;
|
||||
}
|
||||
++frames;
|
||||
}
|
||||
tm.stop();
|
||||
std::cout << "Processed " << frames << " frames" << " (" << frames / tm.getTimeSec() << " FPS)" << std::endl;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
namespace cfg {
|
||||
typename cv::gapi::wip::onevpl::CfgParam create_from_string(const std::string &line) {
|
||||
using namespace cv::gapi::wip;
|
||||
|
||||
if (line.empty()) {
|
||||
throw std::runtime_error("Cannot parse CfgParam from emply line");
|
||||
}
|
||||
|
||||
std::string::size_type name_endline_pos = line.find(':');
|
||||
if (name_endline_pos == std::string::npos) {
|
||||
throw std::runtime_error("Cannot parse CfgParam from: " + line +
|
||||
"\nExpected separator \":\"");
|
||||
}
|
||||
|
||||
std::string name = line.substr(0, name_endline_pos);
|
||||
std::string value = line.substr(name_endline_pos + 1);
|
||||
|
||||
return cv::gapi::wip::onevpl::CfgParam::create(name, value,
|
||||
/* vpp params strongly optional */
|
||||
name.find("vpp.") == std::string::npos);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
#include <algorithm>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <cctype>
|
||||
#include <tuple>
|
||||
#include <memory>
|
||||
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/gpu/ggpukernel.hpp>
|
||||
#include <opencv2/gapi/streaming/onevpl/source.hpp>
|
||||
#include <opencv2/gapi/streaming/onevpl/data_provider_interface.hpp>
|
||||
#include <opencv2/gapi/streaming/onevpl/default.hpp>
|
||||
#include <opencv2/highgui.hpp> // CommandLineParser
|
||||
#include <opencv2/gapi/ocl/core.hpp>
|
||||
|
||||
const std::string about =
|
||||
"This is an example presents decoding on GPU using VPL Source and passing it to OpenCL backend";
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ input | | Path to the input video file. Use .avi extension }"
|
||||
"{ accel_mode | mfxImplDescription.AccelerationMode:MFX_ACCEL_MODE_VIA_D3D11 | Acceleration mode for VPL }";
|
||||
|
||||
namespace {
|
||||
namespace cfg {
|
||||
// FIXME: Move OneVPL arguments parser to a single place
|
||||
typename cv::gapi::wip::onevpl::CfgParam create_from_string(const std::string &line);
|
||||
} // namespace cfg
|
||||
} // anonymous namespace
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
cmd.about(about);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Get file name
|
||||
const auto input = cmd.get<std::string>("input");
|
||||
const auto accel_mode = cmd.get<std::string>("accel_mode");
|
||||
|
||||
// Create VPL config
|
||||
std::vector<cv::gapi::wip::onevpl::CfgParam> source_cfgs;
|
||||
source_cfgs.push_back(cfg::create_from_string(accel_mode));
|
||||
|
||||
// Create VPL-based source
|
||||
std::shared_ptr<cv::gapi::wip::onevpl::IDeviceSelector> default_device_selector =
|
||||
cv::gapi::wip::onevpl::getDefaultDeviceSelector(source_cfgs);
|
||||
|
||||
cv::gapi::wip::IStreamSource::Ptr source = cv::gapi::wip::make_onevpl_src(input, source_cfgs,
|
||||
default_device_selector);
|
||||
|
||||
// Build the graph
|
||||
cv::GFrame in; // input frame from VPL source
|
||||
auto bgr_gmat = cv::gapi::streaming::BGR(in); // conversion from VPL source frame to BGR UMat
|
||||
auto out = cv::gapi::blur(bgr_gmat, cv::Size(4,4)); // ocl kernel of blur operation
|
||||
|
||||
cv::GStreamingCompiled pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out))
|
||||
.compileStreaming(cv::compile_args(cv::gapi::core::ocl::kernels()));
|
||||
pipeline.setSource(std::move(source));
|
||||
|
||||
// The execution part
|
||||
size_t frames = 0u;
|
||||
cv::TickMeter tm;
|
||||
cv::Mat outMat;
|
||||
|
||||
pipeline.start();
|
||||
tm.start();
|
||||
|
||||
while (pipeline.pull(cv::gout(outMat))) {
|
||||
cv::imshow("OutVideo", outMat);
|
||||
cv::waitKey(1);
|
||||
++frames;
|
||||
}
|
||||
tm.stop();
|
||||
std::cout << "Processed " << frames << " frames" << " (" << frames / tm.getTimeSec() << " FPS)" << std::endl;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
namespace {
|
||||
namespace cfg {
|
||||
typename cv::gapi::wip::onevpl::CfgParam create_from_string(const std::string &line) {
|
||||
using namespace cv::gapi::wip;
|
||||
|
||||
if (line.empty()) {
|
||||
throw std::runtime_error("Cannot parse CfgParam from emply line");
|
||||
}
|
||||
|
||||
std::string::size_type name_endline_pos = line.find(':');
|
||||
if (name_endline_pos == std::string::npos) {
|
||||
throw std::runtime_error("Cannot parse CfgParam from: " + line +
|
||||
"\nExpected separator \":\"");
|
||||
}
|
||||
|
||||
std::string name = line.substr(0, name_endline_pos);
|
||||
std::string value = line.substr(name_endline_pos + 1);
|
||||
|
||||
return cv::gapi::wip::onevpl::CfgParam::create(name, value,
|
||||
/* vpp params strongly optional */
|
||||
name.find("vpp.") == std::string::npos);
|
||||
}
|
||||
} // namespace cfg
|
||||
} // anonymous namespace
|
||||
@@ -0,0 +1,541 @@
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <thread>
|
||||
#include <exception>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/highgui.hpp> // cv::CommandLineParser
|
||||
#include <opencv2/core/utils/filesystem.hpp>
|
||||
|
||||
#if defined(_WIN32)
|
||||
#define NOMINMAX
|
||||
#include <windows.h>
|
||||
#undef NOMINMAX
|
||||
#endif
|
||||
|
||||
#include "pipeline_modeling_tool/dummy_source.hpp"
|
||||
#include "pipeline_modeling_tool/utils.hpp"
|
||||
#include "pipeline_modeling_tool/pipeline_builder.hpp"
|
||||
|
||||
enum class AppMode {
|
||||
REALTIME,
|
||||
BENCHMARK
|
||||
};
|
||||
|
||||
static AppMode strToAppMode(const std::string& mode_str) {
|
||||
if (mode_str == "realtime") {
|
||||
return AppMode::REALTIME;
|
||||
} else if (mode_str == "benchmark") {
|
||||
return AppMode::BENCHMARK;
|
||||
} else {
|
||||
throw std::logic_error("Unsupported AppMode: " + mode_str +
|
||||
"\nPlease chose between: realtime and benchmark");
|
||||
}
|
||||
}
|
||||
|
||||
enum class WaitMode {
|
||||
BUSY,
|
||||
SLEEP
|
||||
};
|
||||
|
||||
static WaitMode strToWaitMode(const std::string& mode_str) {
|
||||
if (mode_str == "sleep") {
|
||||
return WaitMode::SLEEP;
|
||||
} else if (mode_str == "busy") {
|
||||
return WaitMode::BUSY;
|
||||
} else {
|
||||
throw std::logic_error("Unsupported wait mode: " + mode_str +
|
||||
"\nPlease chose between: busy (default) and sleep");
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
T read(const cv::FileNode& node) {
|
||||
return static_cast<T>(node);
|
||||
}
|
||||
|
||||
static cv::FileNode check_and_get_fn(const cv::FileNode& fn,
|
||||
const std::string& field,
|
||||
const std::string& uplvl) {
|
||||
const bool is_map = fn.isMap();
|
||||
if (!is_map || fn[field].empty()) {
|
||||
throw std::logic_error(uplvl + " must contain field: " + field);
|
||||
}
|
||||
return fn[field];
|
||||
}
|
||||
|
||||
static cv::FileNode check_and_get_fn(const cv::FileStorage& fs,
|
||||
const std::string& field,
|
||||
const std::string& uplvl) {
|
||||
auto fn = fs[field];
|
||||
if (fn.empty()) {
|
||||
throw std::logic_error(uplvl + " must contain field: " + field);
|
||||
}
|
||||
return fn;
|
||||
}
|
||||
|
||||
template <typename T, typename FileT>
|
||||
T check_and_read(const FileT& f,
|
||||
const std::string& field,
|
||||
const std::string& uplvl) {
|
||||
auto fn = check_and_get_fn(f, field, uplvl);
|
||||
return read<T>(fn);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
cv::optional<T> readOpt(const cv::FileNode& fn) {
|
||||
return fn.empty() ? cv::optional<T>() : cv::optional<T>(read<T>(fn));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
std::vector<T> readList(const cv::FileNode& fn,
|
||||
const std::string& field,
|
||||
const std::string& uplvl) {
|
||||
auto fn_field = check_and_get_fn(fn, field, uplvl);
|
||||
if (!fn_field.isSeq()) {
|
||||
throw std::logic_error(field + " in " + uplvl + " must be a sequence");
|
||||
}
|
||||
|
||||
std::vector<T> vec;
|
||||
for (auto iter : fn_field) {
|
||||
vec.push_back(read<T>(iter));
|
||||
}
|
||||
return vec;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
std::vector<T> readVec(const cv::FileNode& fn,
|
||||
const std::string& field,
|
||||
const std::string& uplvl) {
|
||||
auto fn_field = check_and_get_fn(fn, field, uplvl);
|
||||
|
||||
std::vector<T> vec;
|
||||
fn_field >> vec;
|
||||
return vec;
|
||||
}
|
||||
|
||||
static int strToPrecision(const std::string& precision) {
|
||||
static std::unordered_map<std::string, int> str_to_precision = {
|
||||
{"U8", CV_8U}, {"FP32", CV_32F}, {"FP16", CV_16F}
|
||||
};
|
||||
auto it = str_to_precision.find(precision);
|
||||
if (it == str_to_precision.end()) {
|
||||
throw std::logic_error("Unsupported precision: " + precision);
|
||||
}
|
||||
return it->second;
|
||||
}
|
||||
|
||||
template <>
|
||||
OutputDescr read<OutputDescr>(const cv::FileNode& fn) {
|
||||
auto dims = readVec<int>(fn, "dims", "output");
|
||||
auto str_prec = check_and_read<std::string>(fn, "precision", "output");
|
||||
return OutputDescr{dims, strToPrecision(str_prec)};
|
||||
}
|
||||
|
||||
template <>
|
||||
Edge read<Edge>(const cv::FileNode& fn) {
|
||||
auto from = check_and_read<std::string>(fn, "from", "edge");
|
||||
auto to = check_and_read<std::string>(fn, "to", "edge");
|
||||
|
||||
auto splitNameAndPort = [](const std::string& str) {
|
||||
auto pos = str.find(':');
|
||||
auto name =
|
||||
pos == std::string::npos ? str : std::string(str.c_str(), pos);
|
||||
size_t port =
|
||||
pos == std::string::npos ? 0 : std::atoi(str.c_str() + pos + 1);
|
||||
return std::make_pair(name, port);
|
||||
};
|
||||
|
||||
auto p1 = splitNameAndPort(from);
|
||||
auto p2 = splitNameAndPort(to);
|
||||
return Edge{Edge::P{p1.first, p1.second}, Edge::P{p2.first, p2.second}};
|
||||
}
|
||||
|
||||
static std::string getModelsPath() {
|
||||
static char* models_path_c = std::getenv("PIPELINE_MODELS_PATH");
|
||||
static std::string models_path = models_path_c ? models_path_c : ".";
|
||||
return models_path;
|
||||
}
|
||||
|
||||
template <>
|
||||
ModelPath read<ModelPath>(const cv::FileNode& fn) {
|
||||
using cv::utils::fs::join;
|
||||
if (!fn["xml"].empty() && !fn["bin"].empty()) {
|
||||
return ModelPath{LoadPath{join(getModelsPath(), fn["xml"].string()),
|
||||
join(getModelsPath(), fn["bin"].string())}};
|
||||
} else if (!fn["blob"].empty()){
|
||||
return ModelPath{ImportPath{join(getModelsPath(), fn["blob"].string())}};
|
||||
} else {
|
||||
const std::string emsg = R""""(
|
||||
Path to OpenVINO model must be specified in either of two formats:
|
||||
1.
|
||||
xml: path to *.xml
|
||||
bin: path to *.bin
|
||||
2.
|
||||
blob: path to *.blob
|
||||
)"""";
|
||||
throw std::logic_error(emsg);
|
||||
}
|
||||
}
|
||||
|
||||
static PLMode strToPLMode(const std::string& mode_str) {
|
||||
if (mode_str == "streaming") {
|
||||
return PLMode::STREAMING;
|
||||
} else if (mode_str == "regular") {
|
||||
return PLMode::REGULAR;
|
||||
} else {
|
||||
throw std::logic_error("Unsupported PLMode: " + mode_str +
|
||||
"\nPlease chose between: streaming and regular");
|
||||
}
|
||||
}
|
||||
|
||||
static cv::gapi::ie::InferMode strToInferMode(const std::string& infer_mode) {
|
||||
if (infer_mode == "async") {
|
||||
return cv::gapi::ie::InferMode::Async;
|
||||
} else if (infer_mode == "sync") {
|
||||
return cv::gapi::ie::InferMode::Sync;
|
||||
} else {
|
||||
throw std::logic_error("Unsupported Infer mode: " + infer_mode +
|
||||
"\nPlease chose between: async and sync");
|
||||
}
|
||||
}
|
||||
|
||||
template <>
|
||||
CallParams read<CallParams>(const cv::FileNode& fn) {
|
||||
auto name =
|
||||
check_and_read<std::string>(fn, "name", "node");
|
||||
// FIXME: Impossible to read size_t due OpenCV limitations.
|
||||
auto call_every_nth_opt = readOpt<int>(fn["call_every_nth"]);
|
||||
auto call_every_nth = call_every_nth_opt.value_or(1);
|
||||
if (call_every_nth <= 0) {
|
||||
throw std::logic_error(
|
||||
name + " call_every_nth must be greater than zero\n"
|
||||
"Current call_every_nth: " + std::to_string(call_every_nth));
|
||||
}
|
||||
return CallParams{std::move(name), static_cast<size_t>(call_every_nth)};
|
||||
}
|
||||
|
||||
template <typename V>
|
||||
std::map<std::string, V> readMap(const cv::FileNode& fn) {
|
||||
std::map<std::string, V> map;
|
||||
for (auto item : fn) {
|
||||
map.emplace(item.name(), read<V>(item));
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
template <>
|
||||
InferParams read<InferParams>(const cv::FileNode& fn) {
|
||||
auto name =
|
||||
check_and_read<std::string>(fn, "name", "node");
|
||||
|
||||
InferParams params;
|
||||
params.path = read<ModelPath>(fn);
|
||||
params.device = check_and_read<std::string>(fn, "device", name);
|
||||
params.input_layers = readList<std::string>(fn, "input_layers", name);
|
||||
params.output_layers = readList<std::string>(fn, "output_layers", name);
|
||||
params.config = readMap<std::string>(fn["config"]);
|
||||
|
||||
auto out_prec_str = readOpt<std::string>(fn["output_precision"]);
|
||||
if (out_prec_str.has_value()) {
|
||||
params.out_precision =
|
||||
cv::optional<int>(strToPrecision(out_prec_str.value()));
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
template <>
|
||||
DummyParams read<DummyParams>(const cv::FileNode& fn) {
|
||||
auto name =
|
||||
check_and_read<std::string>(fn, "name", "node");
|
||||
|
||||
DummyParams params;
|
||||
params.time = check_and_read<double>(fn, "time", name);
|
||||
if (params.time < 0) {
|
||||
throw std::logic_error(name + " time must be positive");
|
||||
}
|
||||
params.output = check_and_read<OutputDescr>(fn, "output", name);
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
static std::vector<std::string> parseExecList(const std::string& exec_list) {
|
||||
std::vector<std::string> pl_types;
|
||||
std::stringstream ss(exec_list);
|
||||
std::string pl_type;
|
||||
while (getline(ss, pl_type, ',')) {
|
||||
pl_types.push_back(pl_type);
|
||||
}
|
||||
return pl_types;
|
||||
}
|
||||
|
||||
static void loadConfig(const std::string& filename,
|
||||
std::map<std::string, std::string>& config) {
|
||||
cv::FileStorage fs(filename, cv::FileStorage::READ);
|
||||
if (!fs.isOpened()) {
|
||||
throw std::runtime_error("Failed to load config: " + filename);
|
||||
}
|
||||
|
||||
cv::FileNode root = fs.root();
|
||||
for (auto it = root.begin(); it != root.end(); ++it) {
|
||||
auto device = *it;
|
||||
if (!device.isMap()) {
|
||||
throw std::runtime_error("Failed to parse config: " + filename);
|
||||
}
|
||||
for (auto item : device) {
|
||||
config.emplace(item.name(), item.string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
#if defined(_WIN32)
|
||||
timeBeginPeriod(1);
|
||||
#endif
|
||||
try {
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message. }"
|
||||
"{ cfg | | Path to the config which is either"
|
||||
" YAML file or string. }"
|
||||
"{ load_config | | Optional. Path to XML/YAML/JSON file"
|
||||
" to load custom IE parameters. }"
|
||||
"{ cache_dir | | Optional. Enables caching of loaded models"
|
||||
" to specified directory. }"
|
||||
"{ log_file | | Optional. If file is specified, app will"
|
||||
" dump expanded execution information. }"
|
||||
"{ pl_mode | streaming | Optional. Pipeline mode: streaming/regular"
|
||||
" if it's specified will be applied for"
|
||||
" every pipeline. }"
|
||||
"{ qc | 1 | Optional. Calculated automatically by G-API"
|
||||
" if set to 0. If it's specified will be"
|
||||
" applied for every pipeline. }"
|
||||
"{ app_mode | realtime | Application mode (realtime/benchmark). }"
|
||||
"{ drop_frames | false | Drop frames if they come earlier than pipeline is completed. }"
|
||||
"{ exec_list | | A comma-separated list of pipelines that"
|
||||
" will be executed. Spaces around commas"
|
||||
" are prohibited. }"
|
||||
"{ infer_mode | async | OpenVINO inference mode (async/sync). }";
|
||||
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
const auto cfg = cmd.get<std::string>("cfg");
|
||||
const auto load_config = cmd.get<std::string>("load_config");
|
||||
const auto cached_dir = cmd.get<std::string>("cache_dir");
|
||||
const auto log_file = cmd.get<std::string>("log_file");
|
||||
const auto cmd_pl_mode = strToPLMode(cmd.get<std::string>("pl_mode"));
|
||||
const auto qc = cmd.get<int>("qc");
|
||||
const auto app_mode = strToAppMode(cmd.get<std::string>("app_mode"));
|
||||
const auto exec_str = cmd.get<std::string>("exec_list");
|
||||
const auto infer_mode = strToInferMode(cmd.get<std::string>("infer_mode"));
|
||||
const auto drop_frames = cmd.get<bool>("drop_frames");
|
||||
|
||||
cv::FileStorage fs;
|
||||
if (cfg.empty()) {
|
||||
throw std::logic_error("Config must be specified via --cfg option");
|
||||
}
|
||||
// NB: *.yml
|
||||
if (cfg.size() < 5) {
|
||||
throw std::logic_error("--cfg string must contain at least 5 symbols"
|
||||
" to determine if it's a file (*.yml) a or string");
|
||||
}
|
||||
if (cfg.substr(cfg.size() - 4, cfg.size()) == ".yml") {
|
||||
if (!fs.open(cfg, cv::FileStorage::READ)) {
|
||||
throw std::logic_error("Failed to open config file: " + cfg);
|
||||
}
|
||||
} else {
|
||||
fs = cv::FileStorage(cfg, cv::FileStorage::FORMAT_YAML |
|
||||
cv::FileStorage::MEMORY);
|
||||
}
|
||||
|
||||
std::map<std::string, std::string> gconfig;
|
||||
if (!load_config.empty()) {
|
||||
loadConfig(load_config, gconfig);
|
||||
}
|
||||
// NB: Takes priority over config from file
|
||||
if (!cached_dir.empty()) {
|
||||
gconfig =
|
||||
std::map<std::string, std::string>{{"CACHE_DIR", cached_dir}};
|
||||
}
|
||||
|
||||
auto opt_work_time_ms = readOpt<double>(fs["work_time"]);
|
||||
cv::optional<int64_t> opt_work_time_mcs;
|
||||
if (opt_work_time_ms) {
|
||||
const double work_time_ms = opt_work_time_ms.value();
|
||||
if (work_time_ms < 0) {
|
||||
throw std::logic_error("work_time must be positive");
|
||||
}
|
||||
opt_work_time_mcs = cv::optional<int64_t>(utils::ms_to_mcs(work_time_ms));
|
||||
}
|
||||
|
||||
auto pipelines_fn = check_and_get_fn(fs, "Pipelines", "Config");
|
||||
if (!pipelines_fn.isMap()) {
|
||||
throw std::logic_error("Pipelines field must be a map");
|
||||
}
|
||||
|
||||
auto exec_list = !exec_str.empty() ? parseExecList(exec_str)
|
||||
: pipelines_fn.keys();
|
||||
|
||||
|
||||
std::vector<Pipeline::Ptr> pipelines;
|
||||
pipelines.reserve(exec_list.size());
|
||||
// NB: Build pipelines based on config information
|
||||
PipelineBuilder builder;
|
||||
for (const auto& name : exec_list) {
|
||||
const auto& pl_fn = check_and_get_fn(pipelines_fn, name, "Pipelines");
|
||||
builder.setName(name);
|
||||
StopCriterion::Ptr stop_criterion;
|
||||
auto opt_num_iters = readOpt<int>(pl_fn["num_iters"]);
|
||||
// NB: num_iters for specific pipeline takes priority over global work_time.
|
||||
if (opt_num_iters) {
|
||||
stop_criterion.reset(new NumItersCriterion(opt_num_iters.value()));
|
||||
} else if (opt_work_time_mcs) {
|
||||
stop_criterion.reset(new ElapsedTimeCriterion(opt_work_time_mcs.value()));
|
||||
} else {
|
||||
throw std::logic_error(
|
||||
"Failed: Pipeline " + name + " doesn't have stop criterion!\n"
|
||||
"Please specify either work_time: <value> in the config root"
|
||||
" or num_iters: <value> for specific pipeline.");
|
||||
}
|
||||
builder.setStopCriterion(std::move(stop_criterion));
|
||||
|
||||
// NB: Set source
|
||||
{
|
||||
const auto& src_fn = check_and_get_fn(pl_fn, "source", name);
|
||||
auto src_name =
|
||||
check_and_read<std::string>(src_fn, "name", "source");
|
||||
auto latency =
|
||||
check_and_read<double>(src_fn, "latency", "source");
|
||||
auto output =
|
||||
check_and_read<OutputDescr>(src_fn, "output", "source");
|
||||
// NB: In case BENCHMARK mode sources work with zero latency.
|
||||
if (app_mode == AppMode::BENCHMARK) {
|
||||
latency = 0.0;
|
||||
}
|
||||
|
||||
const auto wait_mode =
|
||||
strToWaitMode(readOpt<std::string>(src_fn["wait_mode"]).value_or("busy"));
|
||||
auto wait_strategy = (wait_mode == WaitMode::SLEEP) ? utils::sleep : utils::busyWait;
|
||||
auto src = std::make_shared<DummySource>(
|
||||
utils::double_ms_t{latency}, output, drop_frames, std::move(wait_strategy));
|
||||
builder.setSource(src_name, src);
|
||||
}
|
||||
|
||||
const auto& nodes_fn = check_and_get_fn(pl_fn, "nodes", name);
|
||||
if (!nodes_fn.isSeq()) {
|
||||
throw std::logic_error("nodes in " + name + " must be a sequence");
|
||||
}
|
||||
|
||||
for (auto node_fn : nodes_fn) {
|
||||
auto call_params = read<CallParams>(node_fn);
|
||||
auto node_type =
|
||||
check_and_read<std::string>(node_fn, "type", "node");
|
||||
if (node_type == "Dummy") {
|
||||
builder.addDummy(call_params, read<DummyParams>(node_fn));
|
||||
} else if (node_type == "Infer") {
|
||||
auto infer_params = read<InferParams>(node_fn);
|
||||
try {
|
||||
utils::mergeMapWith(infer_params.config, gconfig);
|
||||
} catch (std::exception& e) {
|
||||
std::stringstream ss;
|
||||
ss << "Failed to merge global and local config for Infer node: "
|
||||
<< call_params.name << std::endl << e.what();
|
||||
throw std::logic_error(ss.str());
|
||||
}
|
||||
infer_params.mode = infer_mode;
|
||||
builder.addInfer(call_params, infer_params);
|
||||
} else {
|
||||
throw std::logic_error("Unsupported node type: " + node_type);
|
||||
}
|
||||
}
|
||||
|
||||
const auto edges_fn = check_and_get_fn(pl_fn, "edges", name);
|
||||
if (!edges_fn.isSeq()) {
|
||||
throw std::logic_error("edges in " + name + " must be a sequence");
|
||||
}
|
||||
for (auto edge_fn : edges_fn) {
|
||||
auto edge = read<Edge>(edge_fn);
|
||||
builder.addEdge(edge);
|
||||
}
|
||||
|
||||
auto cfg_pl_mode = readOpt<std::string>(pl_fn["mode"]);
|
||||
// NB: Pipeline mode from config takes priority over cmd.
|
||||
auto pl_mode = cfg_pl_mode.has_value()
|
||||
? strToPLMode(cfg_pl_mode.value()) : cmd_pl_mode;
|
||||
// NB: Using drop_frames with streaming pipelines will lead to
|
||||
// incorrect performance results.
|
||||
if (drop_frames && pl_mode == PLMode::STREAMING) {
|
||||
throw std::logic_error(
|
||||
"--drop_frames option is supported only for pipelines in \"regular\" mode");
|
||||
}
|
||||
|
||||
builder.setMode(pl_mode);
|
||||
|
||||
// NB: Queue capacity from config takes priority over cmd.
|
||||
auto config_qc = readOpt<int>(pl_fn["queue_capacity"]);
|
||||
auto queue_capacity = config_qc.has_value() ? config_qc.value() : qc;
|
||||
// NB: 0 is special constant that means
|
||||
// queue capacity should be calculated automatically.
|
||||
if (queue_capacity != 0) {
|
||||
builder.setQueueCapacity(queue_capacity);
|
||||
}
|
||||
|
||||
auto dump = readOpt<std::string>(pl_fn["dump"]);
|
||||
if (dump) {
|
||||
builder.setDumpFilePath(dump.value());
|
||||
}
|
||||
|
||||
pipelines.emplace_back(builder.build());
|
||||
}
|
||||
|
||||
// NB: Compille pipelines
|
||||
for (size_t i = 0; i < pipelines.size(); ++i) {
|
||||
pipelines[i]->compile();
|
||||
}
|
||||
|
||||
// NB: Execute pipelines
|
||||
std::vector<std::exception_ptr> eptrs(pipelines.size(), nullptr);
|
||||
std::vector<std::thread> threads(pipelines.size());
|
||||
for (size_t i = 0; i < pipelines.size(); ++i) {
|
||||
threads[i] = std::thread([&, i]() {
|
||||
try {
|
||||
pipelines[i]->run();
|
||||
} catch (...) {
|
||||
eptrs[i] = std::current_exception();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
std::ofstream file;
|
||||
if (!log_file.empty()) {
|
||||
file.open(log_file);
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < threads.size(); ++i) {
|
||||
threads[i].join();
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < threads.size(); ++i) {
|
||||
if (eptrs[i] != nullptr) {
|
||||
try {
|
||||
std::rethrow_exception(eptrs[i]);
|
||||
} catch (std::exception& e) {
|
||||
throw std::logic_error(pipelines[i]->name() + " failed: " + e.what());
|
||||
}
|
||||
}
|
||||
if (file.is_open()) {
|
||||
file << pipelines[i]->report().toStr(true) << std::endl;
|
||||
}
|
||||
std::cout << pipelines[i]->report().toStr() << std::endl;
|
||||
}
|
||||
} catch (const std::exception& e) {
|
||||
std::cout << e.what() << std::endl;
|
||||
throw;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
#ifndef OPENCV_GAPI_PIPELINE_MODELING_TOOL_DUMMY_SOURCE_HPP
|
||||
#define OPENCV_GAPI_PIPELINE_MODELING_TOOL_DUMMY_SOURCE_HPP
|
||||
|
||||
#include <thread>
|
||||
#include <memory>
|
||||
#include <chrono>
|
||||
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/streaming/cap.hpp> // cv::gapi::wip::IStreamSource
|
||||
|
||||
#include "utils.hpp"
|
||||
|
||||
class DummySource final: public cv::gapi::wip::IStreamSource {
|
||||
public:
|
||||
using WaitStrategy = std::function<void(std::chrono::microseconds)>;
|
||||
using Ptr = std::shared_ptr<DummySource>;
|
||||
using ts_t = std::chrono::microseconds;
|
||||
|
||||
template <typename DurationT>
|
||||
DummySource(const DurationT latency,
|
||||
const OutputDescr& output,
|
||||
const bool drop_frames,
|
||||
WaitStrategy&& wait);
|
||||
|
||||
bool pull(cv::gapi::wip::Data& data) override;
|
||||
cv::GMetaArg descr_of() const override;
|
||||
|
||||
private:
|
||||
int64_t m_latency;
|
||||
cv::Mat m_mat;
|
||||
bool m_drop_frames;
|
||||
int64_t m_next_tick_ts = -1;
|
||||
int64_t m_curr_seq_id = 0;
|
||||
WaitStrategy m_wait;
|
||||
};
|
||||
|
||||
template <typename DurationT>
|
||||
DummySource::DummySource(const DurationT latency,
|
||||
const OutputDescr& output,
|
||||
const bool drop_frames,
|
||||
WaitStrategy&& wait)
|
||||
: m_latency(std::chrono::duration_cast<ts_t>(latency).count()),
|
||||
m_drop_frames(drop_frames),
|
||||
m_wait(std::move(wait)) {
|
||||
utils::createNDMat(m_mat, output.dims, output.precision);
|
||||
utils::generateRandom(m_mat);
|
||||
}
|
||||
|
||||
bool DummySource::pull(cv::gapi::wip::Data& data) {
|
||||
using namespace std::chrono;
|
||||
using namespace cv::gapi::streaming;
|
||||
|
||||
// NB: Wait m_latency before return the first frame.
|
||||
if (m_next_tick_ts == -1) {
|
||||
m_next_tick_ts = utils::timestamp<ts_t>() + m_latency;
|
||||
}
|
||||
|
||||
int64_t curr_ts = utils::timestamp<ts_t>();
|
||||
if (curr_ts < m_next_tick_ts) {
|
||||
/*
|
||||
* curr_ts
|
||||
* |
|
||||
* ------|----*-----|------->
|
||||
* ^
|
||||
* m_next_tick_ts
|
||||
*
|
||||
*
|
||||
* NB: New frame will be produced at the m_next_tick_ts point.
|
||||
*/
|
||||
m_wait(ts_t{m_next_tick_ts - curr_ts});
|
||||
} else if (m_latency != 0) {
|
||||
/*
|
||||
* curr_ts
|
||||
* +1 +2 |
|
||||
* |----------|----------|----------|----*-----|------->
|
||||
* ^ ^
|
||||
* m_next_tick_ts ------------->
|
||||
*
|
||||
*/
|
||||
|
||||
// NB: Count how many frames have been produced since last pull (m_next_tick_ts).
|
||||
int64_t num_frames =
|
||||
static_cast<int64_t>((curr_ts - m_next_tick_ts) / m_latency);
|
||||
// NB: Shift m_next_tick_ts to the nearest tick before curr_ts.
|
||||
m_next_tick_ts += num_frames * m_latency;
|
||||
// NB: if drop_frames is enabled, update current seq_id and wait for the next tick, otherwise
|
||||
// return last written frame (+2 at the picture above) immediately.
|
||||
if (m_drop_frames) {
|
||||
// NB: Shift tick to the next frame.
|
||||
m_next_tick_ts += m_latency;
|
||||
// NB: Wait for the next frame.
|
||||
m_wait(ts_t{m_next_tick_ts - curr_ts});
|
||||
// NB: Drop already produced frames + update seq_id for the current.
|
||||
m_curr_seq_id += num_frames + 1;
|
||||
}
|
||||
}
|
||||
// NB: Just increase reference counter not to release mat memory
|
||||
// after assigning it to the data.
|
||||
cv::Mat mat = m_mat;
|
||||
data.meta[meta_tag::timestamp] = utils::timestamp<ts_t>();
|
||||
data.meta[meta_tag::seq_id] = m_curr_seq_id++;
|
||||
data = mat;
|
||||
m_next_tick_ts += m_latency;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
cv::GMetaArg DummySource::descr_of() const {
|
||||
return cv::GMetaArg{cv::descr_of(m_mat)};
|
||||
}
|
||||
|
||||
#endif // OPENCV_GAPI_PIPELINE_MODELING_TOOL_DUMMY_SOURCE_HPP
|
||||
@@ -0,0 +1,250 @@
|
||||
#ifndef OPENCV_GAPI_PIPELINE_MODELING_TOOL_PIPELINE_HPP
|
||||
#define OPENCV_GAPI_PIPELINE_MODELING_TOOL_PIPELINE_HPP
|
||||
|
||||
#include <iomanip>
|
||||
|
||||
struct PerfReport {
|
||||
std::string name;
|
||||
double avg_latency = 0.0;
|
||||
double min_latency = 0.0;
|
||||
double max_latency = 0.0;
|
||||
double first_latency = 0.0;
|
||||
double throughput = 0.0;
|
||||
double elapsed = 0.0;
|
||||
double warmup_time = 0.0;
|
||||
int64_t num_late_frames = 0;
|
||||
std::vector<double> latencies;
|
||||
std::vector<int64_t> seq_ids;
|
||||
|
||||
std::string toStr(bool expanded = false) const;
|
||||
};
|
||||
|
||||
std::string PerfReport::toStr(bool expand) const {
|
||||
const auto to_double_str = [](double val) {
|
||||
std::stringstream ss;
|
||||
ss << std::fixed << std::setprecision(3) << val;
|
||||
return ss.str();
|
||||
};
|
||||
|
||||
std::stringstream ss;
|
||||
ss << name << ": warm-up: " << to_double_str(warmup_time)
|
||||
<< " ms, execution time: " << to_double_str(elapsed)
|
||||
<< " ms, throughput: " << to_double_str(throughput)
|
||||
<< " FPS, latency: first: " << to_double_str(first_latency)
|
||||
<< " ms, min: " << to_double_str(min_latency)
|
||||
<< " ms, avg: " << to_double_str(avg_latency)
|
||||
<< " ms, max: " << to_double_str(max_latency)
|
||||
<< " ms, frames: " << num_late_frames << "/" << seq_ids.back()+1 << " (dropped/all)";
|
||||
if (expand) {
|
||||
for (size_t i = 0; i < latencies.size(); ++i) {
|
||||
ss << "\nFrame:" << i << "\nLatency: "
|
||||
<< to_double_str(latencies[i]) << " ms";
|
||||
}
|
||||
}
|
||||
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
class StopCriterion {
|
||||
public:
|
||||
using Ptr = std::unique_ptr<StopCriterion>;
|
||||
|
||||
virtual void start() = 0;
|
||||
virtual void iter() = 0;
|
||||
virtual bool done() = 0;
|
||||
virtual ~StopCriterion() = default;
|
||||
};
|
||||
|
||||
class Pipeline {
|
||||
public:
|
||||
using Ptr = std::shared_ptr<Pipeline>;
|
||||
|
||||
Pipeline(std::string&& name,
|
||||
cv::GComputation&& comp,
|
||||
std::shared_ptr<DummySource>&& src,
|
||||
StopCriterion::Ptr stop_criterion,
|
||||
cv::GCompileArgs&& args,
|
||||
const size_t num_outputs);
|
||||
|
||||
void compile();
|
||||
void run();
|
||||
|
||||
const PerfReport& report() const;
|
||||
const std::string& name() const { return m_name;}
|
||||
|
||||
virtual ~Pipeline() = default;
|
||||
|
||||
protected:
|
||||
virtual void _compile() = 0;
|
||||
virtual void run_iter() = 0;
|
||||
virtual void init() {};
|
||||
virtual void deinit() {};
|
||||
|
||||
void prepareOutputs();
|
||||
|
||||
std::string m_name;
|
||||
cv::GComputation m_comp;
|
||||
std::shared_ptr<DummySource> m_src;
|
||||
StopCriterion::Ptr m_stop_criterion;
|
||||
cv::GCompileArgs m_args;
|
||||
size_t m_num_outputs;
|
||||
PerfReport m_perf;
|
||||
|
||||
cv::GRunArgsP m_pipeline_outputs;
|
||||
std::vector<cv::Mat> m_out_mats;
|
||||
int64_t m_start_ts;
|
||||
int64_t m_seq_id;
|
||||
};
|
||||
|
||||
Pipeline::Pipeline(std::string&& name,
|
||||
cv::GComputation&& comp,
|
||||
std::shared_ptr<DummySource>&& src,
|
||||
StopCriterion::Ptr stop_criterion,
|
||||
cv::GCompileArgs&& args,
|
||||
const size_t num_outputs)
|
||||
: m_name(std::move(name)),
|
||||
m_comp(std::move(comp)),
|
||||
m_src(std::move(src)),
|
||||
m_stop_criterion(std::move(stop_criterion)),
|
||||
m_args(std::move(args)),
|
||||
m_num_outputs(num_outputs) {
|
||||
m_perf.name = m_name;
|
||||
}
|
||||
|
||||
void Pipeline::compile() {
|
||||
m_perf.warmup_time =
|
||||
utils::measure<utils::double_ms_t>([this]() {
|
||||
_compile();
|
||||
});
|
||||
}
|
||||
|
||||
void Pipeline::prepareOutputs() {
|
||||
// NB: N-2 buffers + timestamp + seq_id.
|
||||
m_out_mats.resize(m_num_outputs - 2);
|
||||
for (auto& m : m_out_mats) {
|
||||
m_pipeline_outputs += cv::gout(m);
|
||||
}
|
||||
m_pipeline_outputs += cv::gout(m_start_ts);
|
||||
m_pipeline_outputs += cv::gout(m_seq_id);
|
||||
}
|
||||
|
||||
void Pipeline::run() {
|
||||
using namespace std::chrono;
|
||||
|
||||
// NB: Allocate outputs for execution
|
||||
prepareOutputs();
|
||||
|
||||
// NB: Warm-up iteration invalidates source state
|
||||
// so need to copy it
|
||||
auto orig_src = m_src;
|
||||
auto copy_src = std::make_shared<DummySource>(*m_src);
|
||||
|
||||
// NB: Use copy for warm-up iteration
|
||||
m_src = copy_src;
|
||||
|
||||
// NB: Warm-up iteration
|
||||
init();
|
||||
run_iter();
|
||||
deinit();
|
||||
|
||||
// NB: Calculate first latency
|
||||
m_perf.first_latency = utils::double_ms_t{
|
||||
microseconds{utils::timestamp<microseconds>() - m_start_ts}}.count();
|
||||
|
||||
// NB: Now use original source
|
||||
m_src = orig_src;
|
||||
|
||||
// NB: Start measuring execution
|
||||
init();
|
||||
auto start = high_resolution_clock::now();
|
||||
m_stop_criterion->start();
|
||||
|
||||
while (true) {
|
||||
run_iter();
|
||||
const auto latency = utils::double_ms_t{
|
||||
microseconds{utils::timestamp<microseconds>() - m_start_ts}}.count();
|
||||
|
||||
m_perf.latencies.push_back(latency);
|
||||
m_perf.seq_ids.push_back(m_seq_id);
|
||||
|
||||
m_stop_criterion->iter();
|
||||
|
||||
if (m_stop_criterion->done()) {
|
||||
m_perf.elapsed = duration_cast<utils::double_ms_t>(
|
||||
high_resolution_clock::now() - start).count();
|
||||
deinit();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
m_perf.avg_latency = utils::avg(m_perf.latencies);
|
||||
m_perf.min_latency = utils::min(m_perf.latencies);
|
||||
m_perf.max_latency = utils::max(m_perf.latencies);
|
||||
|
||||
// NB: Count the number of dropped frames
|
||||
int64_t prev_seq_id = m_perf.seq_ids[0];
|
||||
for (size_t i = 1; i < m_perf.seq_ids.size(); ++i) {
|
||||
m_perf.num_late_frames += m_perf.seq_ids[i] - prev_seq_id - 1;
|
||||
prev_seq_id = m_perf.seq_ids[i];
|
||||
}
|
||||
|
||||
m_perf.throughput = (m_perf.latencies.size() / m_perf.elapsed) * 1000;
|
||||
}
|
||||
|
||||
const PerfReport& Pipeline::report() const {
|
||||
return m_perf;
|
||||
}
|
||||
|
||||
class StreamingPipeline : public Pipeline {
|
||||
public:
|
||||
using Pipeline::Pipeline;
|
||||
|
||||
private:
|
||||
void _compile() override {
|
||||
m_compiled =
|
||||
m_comp.compileStreaming({m_src->descr_of()},
|
||||
cv::GCompileArgs(m_args));
|
||||
}
|
||||
|
||||
virtual void init() override {
|
||||
m_compiled.setSource(m_src);
|
||||
m_compiled.start();
|
||||
}
|
||||
|
||||
virtual void deinit() override {
|
||||
m_compiled.stop();
|
||||
}
|
||||
|
||||
virtual void run_iter() override {
|
||||
m_compiled.pull(cv::GRunArgsP{m_pipeline_outputs});
|
||||
}
|
||||
|
||||
cv::GStreamingCompiled m_compiled;
|
||||
};
|
||||
|
||||
class RegularPipeline : public Pipeline {
|
||||
public:
|
||||
using Pipeline::Pipeline;
|
||||
|
||||
private:
|
||||
void _compile() override {
|
||||
m_compiled =
|
||||
m_comp.compile({m_src->descr_of()},
|
||||
cv::GCompileArgs(m_args));
|
||||
}
|
||||
|
||||
virtual void run_iter() override {
|
||||
cv::gapi::wip::Data data;
|
||||
m_src->pull(data);
|
||||
m_compiled({data}, cv::GRunArgsP{m_pipeline_outputs});
|
||||
}
|
||||
|
||||
cv::GCompiled m_compiled;
|
||||
};
|
||||
|
||||
enum class PLMode {
|
||||
REGULAR,
|
||||
STREAMING
|
||||
};
|
||||
|
||||
#endif // OPENCV_GAPI_PIPELINE_MODELING_TOOL_PIPELINE_HPP
|
||||
@@ -0,0 +1,692 @@
|
||||
#ifndef OPENCV_GAPI_PIPELINE_MODELING_TOOL_PIPELINE_BUILDER_HPP
|
||||
#define OPENCV_GAPI_PIPELINE_MODELING_TOOL_PIPELINE_BUILDER_HPP
|
||||
|
||||
#include <map>
|
||||
|
||||
#include <opencv2/gapi/infer.hpp> // cv::gapi::GNetPackage
|
||||
#include <opencv2/gapi/streaming/cap.hpp> // cv::gapi::wip::IStreamSource
|
||||
#include <opencv2/gapi/infer/ie.hpp> // cv::gapi::ie::Params
|
||||
#include <opencv2/gapi/gcommon.hpp> // cv::gapi::GCompileArgs
|
||||
#include <opencv2/gapi/cpu/gcpukernel.hpp> // GAPI_OCV_KERNEL
|
||||
#include <opencv2/gapi/gkernel.hpp> // G_API_OP
|
||||
|
||||
#include "pipeline.hpp"
|
||||
#include "utils.hpp"
|
||||
|
||||
struct Edge {
|
||||
struct P {
|
||||
std::string name;
|
||||
size_t port;
|
||||
};
|
||||
|
||||
P src;
|
||||
P dst;
|
||||
};
|
||||
|
||||
struct CallParams {
|
||||
std::string name;
|
||||
size_t call_every_nth;
|
||||
};
|
||||
|
||||
struct CallNode {
|
||||
using F = std::function<void(const cv::GProtoArgs&, cv::GProtoArgs&)>;
|
||||
|
||||
CallParams params;
|
||||
F run;
|
||||
};
|
||||
|
||||
struct DataNode {
|
||||
cv::optional<cv::GProtoArg> arg;
|
||||
};
|
||||
|
||||
struct Node {
|
||||
using Ptr = std::shared_ptr<Node>;
|
||||
using WPtr = std::weak_ptr<Node>;
|
||||
using Kind = cv::util::variant<CallNode, DataNode>;
|
||||
|
||||
std::vector<Node::WPtr> in_nodes;
|
||||
std::vector<Node::Ptr> out_nodes;
|
||||
Kind kind;
|
||||
};
|
||||
|
||||
struct SubGraphCall {
|
||||
G_API_OP(GSubGraph,
|
||||
<cv::GMat(cv::GMat, cv::GComputation, cv::GCompileArgs, size_t)>,
|
||||
"custom.subgraph") {
|
||||
static cv::GMatDesc outMeta(const cv::GMatDesc& in,
|
||||
cv::GComputation comp,
|
||||
cv::GCompileArgs compile_args,
|
||||
const size_t call_every_nth) {
|
||||
GAPI_Assert(call_every_nth > 0);
|
||||
auto out_metas =
|
||||
comp.compile(in, std::move(compile_args)).outMetas();
|
||||
GAPI_Assert(out_metas.size() == 1u);
|
||||
GAPI_Assert(cv::util::holds_alternative<cv::GMatDesc>(out_metas[0]));
|
||||
return cv::util::get<cv::GMatDesc>(out_metas[0]);
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
struct SubGraphState {
|
||||
cv::Mat last_result;
|
||||
cv::GCompiled cc;
|
||||
int call_counter = 0;
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL_ST(SubGraphImpl, GSubGraph, SubGraphState) {
|
||||
static void setup(const cv::GMatDesc& in,
|
||||
cv::GComputation comp,
|
||||
cv::GCompileArgs compile_args,
|
||||
const size_t /*call_every_nth*/,
|
||||
std::shared_ptr<SubGraphState>& state,
|
||||
const cv::GCompileArgs& /*args*/) {
|
||||
state.reset(new SubGraphState{});
|
||||
state->cc = comp.compile(in, std::move(compile_args));
|
||||
auto out_desc =
|
||||
cv::util::get<cv::GMatDesc>(state->cc.outMetas()[0]);
|
||||
utils::createNDMat(state->last_result,
|
||||
out_desc.dims,
|
||||
out_desc.depth);
|
||||
}
|
||||
|
||||
static void run(const cv::Mat& in,
|
||||
cv::GComputation /*comp*/,
|
||||
cv::GCompileArgs /*compile_args*/,
|
||||
const size_t call_every_nth,
|
||||
cv::Mat& out,
|
||||
SubGraphState& state) {
|
||||
// NB: Make a call on the first iteration and skip the furthers.
|
||||
if (state.call_counter == 0) {
|
||||
state.cc(in, state.last_result);
|
||||
}
|
||||
state.last_result.copyTo(out);
|
||||
state.call_counter = (state.call_counter + 1) % call_every_nth;
|
||||
}
|
||||
};
|
||||
|
||||
void operator()(const cv::GProtoArgs& inputs, cv::GProtoArgs& outputs);
|
||||
|
||||
size_t numInputs() const { return 1; }
|
||||
size_t numOutputs() const { return 1; }
|
||||
|
||||
cv::GComputation comp;
|
||||
cv::GCompileArgs compile_args;
|
||||
size_t call_every_nth;
|
||||
};
|
||||
|
||||
void SubGraphCall::operator()(const cv::GProtoArgs& inputs,
|
||||
cv::GProtoArgs& outputs) {
|
||||
GAPI_Assert(inputs.size() == 1u);
|
||||
GAPI_Assert(cv::util::holds_alternative<cv::GMat>(inputs[0]));
|
||||
GAPI_Assert(outputs.empty());
|
||||
auto in = cv::util::get<cv::GMat>(inputs[0]);
|
||||
outputs.emplace_back(GSubGraph::on(in, comp, compile_args, call_every_nth));
|
||||
}
|
||||
|
||||
struct DummyCall {
|
||||
G_API_OP(GDummy,
|
||||
<cv::GMat(cv::GMat, double, OutputDescr)>,
|
||||
"custom.dummy") {
|
||||
static cv::GMatDesc outMeta(const cv::GMatDesc& /* in */,
|
||||
double /* time */,
|
||||
const OutputDescr& output) {
|
||||
if (output.dims.size() == 2) {
|
||||
return cv::GMatDesc(output.precision,
|
||||
1,
|
||||
// NB: Dims[H, W] -> Size(W, H)
|
||||
cv::Size(output.dims[1], output.dims[0]));
|
||||
}
|
||||
return cv::GMatDesc(output.precision, output.dims);
|
||||
}
|
||||
};
|
||||
|
||||
struct DummyState {
|
||||
cv::Mat mat;
|
||||
};
|
||||
|
||||
// NB: Generate random mat once and then
|
||||
// copy to dst buffer on every iteration.
|
||||
GAPI_OCV_KERNEL_ST(GCPUDummy, GDummy, DummyState) {
|
||||
static void setup(const cv::GMatDesc& /*in*/,
|
||||
double /*time*/,
|
||||
const OutputDescr& output,
|
||||
std::shared_ptr<DummyState>& state,
|
||||
const cv::GCompileArgs& /*args*/) {
|
||||
state.reset(new DummyState{});
|
||||
utils::createNDMat(state->mat, output.dims, output.precision);
|
||||
utils::generateRandom(state->mat);
|
||||
}
|
||||
|
||||
static void run(const cv::Mat& /*in_mat*/,
|
||||
double time,
|
||||
const OutputDescr& /*output*/,
|
||||
cv::Mat& out_mat,
|
||||
DummyState& state) {
|
||||
using namespace std::chrono;
|
||||
auto start_ts = utils::timestamp<utils::double_ms_t>();
|
||||
state.mat.copyTo(out_mat);
|
||||
auto elapsed = utils::timestamp<utils::double_ms_t>() - start_ts;
|
||||
utils::busyWait(duration_cast<microseconds>(utils::double_ms_t{time-elapsed}));
|
||||
}
|
||||
};
|
||||
|
||||
void operator()(const cv::GProtoArgs& inputs, cv::GProtoArgs& outputs);
|
||||
|
||||
size_t numInputs() const { return 1; }
|
||||
size_t numOutputs() const { return 1; }
|
||||
|
||||
double time;
|
||||
OutputDescr output;
|
||||
};
|
||||
|
||||
void DummyCall::operator()(const cv::GProtoArgs& inputs,
|
||||
cv::GProtoArgs& outputs) {
|
||||
GAPI_Assert(inputs.size() == 1u);
|
||||
GAPI_Assert(cv::util::holds_alternative<cv::GMat>(inputs[0]));
|
||||
GAPI_Assert(outputs.empty());
|
||||
auto in = cv::util::get<cv::GMat>(inputs[0]);
|
||||
outputs.emplace_back(GDummy::on(in, time, output));
|
||||
}
|
||||
|
||||
struct InferCall {
|
||||
void operator()(const cv::GProtoArgs& inputs, cv::GProtoArgs& outputs);
|
||||
size_t numInputs() const { return input_layers.size(); }
|
||||
size_t numOutputs() const { return output_layers.size(); }
|
||||
|
||||
std::string tag;
|
||||
std::vector<std::string> input_layers;
|
||||
std::vector<std::string> output_layers;
|
||||
};
|
||||
|
||||
void InferCall::operator()(const cv::GProtoArgs& inputs,
|
||||
cv::GProtoArgs& outputs) {
|
||||
GAPI_Assert(inputs.size() == input_layers.size());
|
||||
GAPI_Assert(outputs.empty());
|
||||
|
||||
cv::GInferInputs g_inputs;
|
||||
// TODO: Add an opportunity not specify input/output layers in case
|
||||
// there is only single layer.
|
||||
for (size_t i = 0; i < inputs.size(); ++i) {
|
||||
// TODO: Support GFrame as well.
|
||||
GAPI_Assert(cv::util::holds_alternative<cv::GMat>(inputs[i]));
|
||||
auto in = cv::util::get<cv::GMat>(inputs[i]);
|
||||
g_inputs[input_layers[i]] = in;
|
||||
}
|
||||
auto g_outputs = cv::gapi::infer<cv::gapi::Generic>(tag, g_inputs);
|
||||
for (size_t i = 0; i < output_layers.size(); ++i) {
|
||||
outputs.emplace_back(g_outputs.at(output_layers[i]));
|
||||
}
|
||||
}
|
||||
|
||||
struct SourceCall {
|
||||
void operator()(const cv::GProtoArgs& inputs, cv::GProtoArgs& outputs);
|
||||
size_t numInputs() const { return 0; }
|
||||
size_t numOutputs() const { return 1; }
|
||||
};
|
||||
|
||||
void SourceCall::operator()(const cv::GProtoArgs& inputs,
|
||||
cv::GProtoArgs& outputs) {
|
||||
GAPI_Assert(inputs.empty());
|
||||
GAPI_Assert(outputs.empty());
|
||||
// NB: Since NV12 isn't exposed source always produce GMat.
|
||||
outputs.emplace_back(cv::GMat());
|
||||
}
|
||||
|
||||
struct LoadPath {
|
||||
std::string xml;
|
||||
std::string bin;
|
||||
};
|
||||
|
||||
struct ImportPath {
|
||||
std::string blob;
|
||||
};
|
||||
|
||||
using ModelPath = cv::util::variant<ImportPath, LoadPath>;
|
||||
|
||||
struct DummyParams {
|
||||
double time;
|
||||
OutputDescr output;
|
||||
};
|
||||
|
||||
struct InferParams {
|
||||
std::string name;
|
||||
ModelPath path;
|
||||
std::string device;
|
||||
std::vector<std::string> input_layers;
|
||||
std::vector<std::string> output_layers;
|
||||
std::map<std::string, std::string> config;
|
||||
cv::gapi::ie::InferMode mode;
|
||||
cv::util::optional<int> out_precision;
|
||||
};
|
||||
|
||||
class ElapsedTimeCriterion : public StopCriterion {
|
||||
public:
|
||||
ElapsedTimeCriterion(int64_t work_time_mcs);
|
||||
|
||||
void start() override;
|
||||
void iter() override;
|
||||
bool done() override;
|
||||
|
||||
private:
|
||||
int64_t m_work_time_mcs;
|
||||
int64_t m_start_ts = -1;
|
||||
int64_t m_curr_ts = -1;
|
||||
};
|
||||
|
||||
ElapsedTimeCriterion::ElapsedTimeCriterion(int64_t work_time_mcs)
|
||||
: m_work_time_mcs(work_time_mcs) {
|
||||
};
|
||||
|
||||
void ElapsedTimeCriterion::start() {
|
||||
m_start_ts = m_curr_ts = utils::timestamp<std::chrono::microseconds>();
|
||||
}
|
||||
|
||||
void ElapsedTimeCriterion::iter() {
|
||||
m_curr_ts = utils::timestamp<std::chrono::microseconds>();
|
||||
}
|
||||
|
||||
bool ElapsedTimeCriterion::done() {
|
||||
return (m_curr_ts - m_start_ts) >= m_work_time_mcs;
|
||||
}
|
||||
|
||||
class NumItersCriterion : public StopCriterion {
|
||||
public:
|
||||
NumItersCriterion(int64_t num_iters);
|
||||
|
||||
void start() override;
|
||||
void iter() override;
|
||||
bool done() override;
|
||||
|
||||
private:
|
||||
int64_t m_num_iters;
|
||||
int64_t m_curr_iters = 0;
|
||||
};
|
||||
|
||||
NumItersCriterion::NumItersCriterion(int64_t num_iters)
|
||||
: m_num_iters(num_iters) {
|
||||
}
|
||||
|
||||
void NumItersCriterion::start() {
|
||||
m_curr_iters = 0;
|
||||
}
|
||||
|
||||
void NumItersCriterion::iter() {
|
||||
++m_curr_iters;
|
||||
}
|
||||
|
||||
bool NumItersCriterion::done() {
|
||||
return m_curr_iters == m_num_iters;
|
||||
}
|
||||
|
||||
class PipelineBuilder {
|
||||
public:
|
||||
PipelineBuilder();
|
||||
void addDummy(const CallParams& call_params,
|
||||
const DummyParams& dummy_params);
|
||||
|
||||
void addInfer(const CallParams& call_params,
|
||||
const InferParams& infer_params);
|
||||
|
||||
void setSource(const std::string& name,
|
||||
std::shared_ptr<DummySource> src);
|
||||
|
||||
void addEdge(const Edge& edge);
|
||||
void setMode(PLMode mode);
|
||||
void setDumpFilePath(const std::string& dump);
|
||||
void setQueueCapacity(const size_t qc);
|
||||
void setName(const std::string& name);
|
||||
void setStopCriterion(StopCriterion::Ptr stop_criterion);
|
||||
|
||||
Pipeline::Ptr build();
|
||||
|
||||
private:
|
||||
template <typename CallT>
|
||||
void addCall(const CallParams& call_params,
|
||||
CallT&& call);
|
||||
|
||||
Pipeline::Ptr construct();
|
||||
|
||||
template <typename K, typename V>
|
||||
using M = std::unordered_map<K, V>;
|
||||
struct State {
|
||||
struct NodeEdges {
|
||||
std::vector<Edge> input_edges;
|
||||
std::vector<Edge> output_edges;
|
||||
};
|
||||
|
||||
M<std::string, Node::Ptr> calls_map;
|
||||
std::vector<Node::Ptr> all_calls;
|
||||
|
||||
cv::gapi::GNetPackage networks;
|
||||
cv::gapi::GKernelPackage kernels;
|
||||
cv::GCompileArgs compile_args;
|
||||
std::shared_ptr<DummySource> src;
|
||||
PLMode mode = PLMode::STREAMING;
|
||||
std::string name;
|
||||
StopCriterion::Ptr stop_criterion;
|
||||
};
|
||||
|
||||
std::unique_ptr<State> m_state;
|
||||
};
|
||||
|
||||
PipelineBuilder::PipelineBuilder() : m_state(new State{}) { };
|
||||
|
||||
void PipelineBuilder::addDummy(const CallParams& call_params,
|
||||
const DummyParams& dummy_params) {
|
||||
m_state->kernels.include<DummyCall::GCPUDummy>();
|
||||
addCall(call_params,
|
||||
DummyCall{dummy_params.time, dummy_params.output});
|
||||
}
|
||||
|
||||
template <typename CallT>
|
||||
void PipelineBuilder::addCall(const CallParams& call_params,
|
||||
CallT&& call) {
|
||||
|
||||
size_t num_inputs = call.numInputs();
|
||||
size_t num_outputs = call.numOutputs();
|
||||
Node::Ptr call_node(new Node{{},{},Node::Kind{CallNode{call_params,
|
||||
std::move(call)}}});
|
||||
// NB: Create placeholders for inputs.
|
||||
call_node->in_nodes.resize(num_inputs);
|
||||
// NB: Create outputs with empty data.
|
||||
for (size_t i = 0; i < num_outputs; ++i) {
|
||||
call_node->out_nodes.emplace_back(new Node{{call_node},
|
||||
{},
|
||||
Node::Kind{DataNode{}}});
|
||||
}
|
||||
|
||||
auto it = m_state->calls_map.find(call_params.name);
|
||||
if (it != m_state->calls_map.end()) {
|
||||
throw std::logic_error("Node: " + call_params.name + " already exists!");
|
||||
}
|
||||
m_state->calls_map.emplace(call_params.name, call_node);
|
||||
m_state->all_calls.emplace_back(call_node);
|
||||
}
|
||||
|
||||
void PipelineBuilder::addInfer(const CallParams& call_params,
|
||||
const InferParams& infer_params) {
|
||||
// NB: No default ctor for Params.
|
||||
std::unique_ptr<cv::gapi::ie::Params<cv::gapi::Generic>> pp;
|
||||
if (cv::util::holds_alternative<LoadPath>(infer_params.path)) {
|
||||
auto load_path = cv::util::get<LoadPath>(infer_params.path);
|
||||
pp.reset(new cv::gapi::ie::Params<cv::gapi::Generic>(call_params.name,
|
||||
load_path.xml,
|
||||
load_path.bin,
|
||||
infer_params.device));
|
||||
} else {
|
||||
GAPI_Assert(cv::util::holds_alternative<ImportPath>(infer_params.path));
|
||||
auto import_path = cv::util::get<ImportPath>(infer_params.path);
|
||||
pp.reset(new cv::gapi::ie::Params<cv::gapi::Generic>(call_params.name,
|
||||
import_path.blob,
|
||||
infer_params.device));
|
||||
}
|
||||
|
||||
pp->pluginConfig(infer_params.config);
|
||||
pp->cfgInferMode(infer_params.mode);
|
||||
if (infer_params.out_precision) {
|
||||
pp->cfgOutputPrecision(infer_params.out_precision.value());
|
||||
}
|
||||
m_state->networks += cv::gapi::networks(*pp);
|
||||
|
||||
addCall(call_params,
|
||||
InferCall{call_params.name,
|
||||
infer_params.input_layers,
|
||||
infer_params.output_layers});
|
||||
}
|
||||
|
||||
void PipelineBuilder::addEdge(const Edge& edge) {
|
||||
const auto& src_it = m_state->calls_map.find(edge.src.name);
|
||||
if (src_it == m_state->calls_map.end()) {
|
||||
throw std::logic_error("Failed to find node: " + edge.src.name);
|
||||
}
|
||||
auto src_node = src_it->second;
|
||||
if (src_node->out_nodes.size() <= edge.src.port) {
|
||||
throw std::logic_error("Failed to access node: " + edge.src.name +
|
||||
" by out port: " + std::to_string(edge.src.port));
|
||||
}
|
||||
|
||||
auto dst_it = m_state->calls_map.find(edge.dst.name);
|
||||
if (dst_it == m_state->calls_map.end()) {
|
||||
throw std::logic_error("Failed to find node: " + edge.dst.name);
|
||||
}
|
||||
auto dst_node = dst_it->second;
|
||||
if (dst_node->in_nodes.size() <= edge.dst.port) {
|
||||
throw std::logic_error("Failed to access node: " + edge.dst.name +
|
||||
" by in port: " + std::to_string(edge.dst.port));
|
||||
}
|
||||
|
||||
auto out_data = src_node->out_nodes[edge.src.port];
|
||||
auto& in_data = dst_node->in_nodes[edge.dst.port];
|
||||
// NB: in_data != nullptr.
|
||||
if (!in_data.expired()) {
|
||||
throw std::logic_error("Node: " + edge.dst.name +
|
||||
" already connected by in port: " +
|
||||
std::to_string(edge.dst.port));
|
||||
}
|
||||
dst_node->in_nodes[edge.dst.port] = out_data;
|
||||
out_data->out_nodes.push_back(dst_node);
|
||||
}
|
||||
|
||||
void PipelineBuilder::setSource(const std::string& name,
|
||||
std::shared_ptr<DummySource> src) {
|
||||
GAPI_Assert(!m_state->src && "Only single source pipelines are supported!");
|
||||
m_state->src = src;
|
||||
addCall(CallParams{name, 1u/*call_every_nth*/}, SourceCall{});
|
||||
}
|
||||
|
||||
void PipelineBuilder::setMode(PLMode mode) {
|
||||
m_state->mode = mode;
|
||||
}
|
||||
|
||||
void PipelineBuilder::setDumpFilePath(const std::string& dump) {
|
||||
m_state->compile_args.emplace_back(cv::graph_dump_path{dump});
|
||||
}
|
||||
|
||||
void PipelineBuilder::setQueueCapacity(const size_t qc) {
|
||||
m_state->compile_args.emplace_back(cv::gapi::streaming::queue_capacity{qc});
|
||||
}
|
||||
|
||||
void PipelineBuilder::setName(const std::string& name) {
|
||||
m_state->name = name;
|
||||
}
|
||||
|
||||
void PipelineBuilder::setStopCriterion(StopCriterion::Ptr stop_criterion) {
|
||||
m_state->stop_criterion = std::move(stop_criterion);
|
||||
}
|
||||
|
||||
static bool visit(Node::Ptr node,
|
||||
std::vector<Node::Ptr>& sorted,
|
||||
std::unordered_map<Node::Ptr, int>& visited) {
|
||||
if (!node) {
|
||||
throw std::logic_error("Found null node");
|
||||
}
|
||||
|
||||
visited[node] = 1;
|
||||
for (auto in : node->in_nodes) {
|
||||
auto in_node = in.lock();
|
||||
if (visited[in_node] == 0) {
|
||||
if (visit(in_node, sorted, visited)) {
|
||||
return true;
|
||||
}
|
||||
} else if (visited[in_node] == 1) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
visited[node] = 2;
|
||||
sorted.push_back(node);
|
||||
return false;
|
||||
}
|
||||
|
||||
static cv::optional<std::vector<Node::Ptr>>
|
||||
toposort(const std::vector<Node::Ptr> nodes) {
|
||||
std::vector<Node::Ptr> sorted;
|
||||
std::unordered_map<Node::Ptr, int> visited;
|
||||
for (auto n : nodes) {
|
||||
if (visit(n, sorted, visited)) {
|
||||
return cv::optional<std::vector<Node::Ptr>>{};
|
||||
}
|
||||
}
|
||||
return cv::util::make_optional(sorted);
|
||||
}
|
||||
|
||||
Pipeline::Ptr PipelineBuilder::construct() {
|
||||
// NB: Unlike G-API, pipeline_builder_tool graph always starts with CALL node
|
||||
// (not data) that produce datas, so the call node which doesn't have
|
||||
// inputs is considered as "producer" node.
|
||||
//
|
||||
// Graph always starts with CALL node and ends with DATA node.
|
||||
// Graph example: [source] -> (source:0) -> [PP] -> (PP:0)
|
||||
//
|
||||
// The algorithm is quite simple:
|
||||
// 0. Verify that every call input node exists (connected).
|
||||
// 1. Sort all nodes by visiting only call nodes,
|
||||
// since there is no data nodes that's not connected with any call node,
|
||||
// it's guarantee that every node will be visited.
|
||||
// 2. Fillter call nodes.
|
||||
// 3. Go through every call node.
|
||||
// FIXME: Add toposort in case user passed nodes
|
||||
// in arbitrary order which is unlikely happened.
|
||||
// 4. Extract proto input from every input node
|
||||
// 5. Run call and get outputs
|
||||
// 6. If call node doesn't have inputs it means that it's "producer" node,
|
||||
// so collect all outputs to graph_inputs vector.
|
||||
// 7. Assign proto outputs to output data nodes,
|
||||
// so the next calls can use them as inputs.
|
||||
cv::GProtoArgs graph_inputs;
|
||||
cv::GProtoArgs graph_outputs;
|
||||
// 0. Verify that every call input node exists (connected).
|
||||
for (auto call_node : m_state->all_calls) {
|
||||
for (size_t i = 0; i < call_node->in_nodes.size(); ++i) {
|
||||
const auto& in_data_node = call_node->in_nodes[i];
|
||||
// NB: in_data_node == nullptr.
|
||||
if (in_data_node.expired()) {
|
||||
const auto& call = cv::util::get<CallNode>(call_node->kind);
|
||||
throw std::logic_error(
|
||||
"Node: " + call.params.name + " in Pipeline: " + m_state->name +
|
||||
" has dangling input by in port: " + std::to_string(i));
|
||||
}
|
||||
}
|
||||
}
|
||||
// (0) Sort all nodes;
|
||||
auto has_sorted = toposort(m_state->all_calls);
|
||||
if (!has_sorted) {
|
||||
throw std::logic_error(
|
||||
"Pipeline: " + m_state->name + " has cyclic dependencies") ;
|
||||
}
|
||||
auto& sorted = has_sorted.value();
|
||||
// (1). Fillter call nodes.
|
||||
std::vector<Node::Ptr> sorted_calls;
|
||||
for (auto n : sorted) {
|
||||
if (cv::util::holds_alternative<CallNode>(n->kind)) {
|
||||
sorted_calls.push_back(n);
|
||||
}
|
||||
}
|
||||
|
||||
m_state->kernels.include<SubGraphCall::SubGraphImpl>();
|
||||
m_state->compile_args.emplace_back(m_state->networks);
|
||||
m_state->compile_args.emplace_back(m_state->kernels);
|
||||
|
||||
// (2). Go through every call node.
|
||||
for (auto call_node : sorted_calls) {
|
||||
auto& call = cv::util::get<CallNode>(call_node->kind);
|
||||
cv::GProtoArgs outputs;
|
||||
cv::GProtoArgs inputs;
|
||||
for (size_t i = 0; i < call_node->in_nodes.size(); ++i) {
|
||||
auto in_node = call_node->in_nodes.at(i);
|
||||
auto in_data = cv::util::get<DataNode>(in_node.lock()->kind);
|
||||
if (!in_data.arg.has_value()) {
|
||||
throw std::logic_error("data hasn't been provided");
|
||||
}
|
||||
// (3). Extract proto input from every input node.
|
||||
inputs.push_back(in_data.arg.value());
|
||||
}
|
||||
// NB: If node shouldn't be called on each iterations,
|
||||
// it should be wrapped into subgraph which is able to skip calling.
|
||||
if (call.params.call_every_nth != 1u) {
|
||||
// FIXME: Limitation of the subgraph operation (<GMat(GMat)>).
|
||||
// G-API doesn't support dynamic number of inputs/outputs.
|
||||
if (inputs.size() > 1u) {
|
||||
throw std::logic_error(
|
||||
"skip_frame_nth is supported only for single input subgraphs\n"
|
||||
"Current subgraph has " + std::to_string(inputs.size()) + " inputs");
|
||||
}
|
||||
|
||||
if (outputs.size() > 1u) {
|
||||
throw std::logic_error(
|
||||
"skip_frame_nth is supported only for single output subgraphs\n"
|
||||
"Current subgraph has " + std::to_string(inputs.size()) + " outputs");
|
||||
}
|
||||
// FIXME: Should be generalized.
|
||||
// Now every subgraph contains only single node
|
||||
// which has single input/output.
|
||||
GAPI_Assert(cv::util::holds_alternative<cv::GMat>(inputs[0]));
|
||||
cv::GProtoArgs subgr_inputs{cv::GProtoArg{cv::GMat()}};
|
||||
cv::GProtoArgs subgr_outputs;
|
||||
call.run(subgr_inputs, subgr_outputs);
|
||||
auto comp = cv::GComputation(cv::GProtoInputArgs{subgr_inputs},
|
||||
cv::GProtoOutputArgs{subgr_outputs});
|
||||
call = CallNode{CallParams{call.params.name, 1u/*call_every_nth*/},
|
||||
SubGraphCall{std::move(comp),
|
||||
m_state->compile_args,
|
||||
call.params.call_every_nth}};
|
||||
}
|
||||
// (4). Run call and get outputs.
|
||||
call.run(inputs, outputs);
|
||||
// (5) If call node doesn't have inputs
|
||||
// it means that it's input producer node (Source).
|
||||
if (call_node->in_nodes.empty()) {
|
||||
for (auto out : outputs) {
|
||||
graph_inputs.push_back(out);
|
||||
}
|
||||
}
|
||||
// (6). Assign proto outputs to output data nodes,
|
||||
// so the next calls can use them as inputs.
|
||||
GAPI_Assert(outputs.size() == call_node->out_nodes.size());
|
||||
for (size_t i = 0; i < outputs.size(); ++i) {
|
||||
auto out_node = call_node->out_nodes[i];
|
||||
auto& out_data = cv::util::get<DataNode>(out_node->kind);
|
||||
out_data.arg = cv::util::make_optional(outputs[i]);
|
||||
if (out_node->out_nodes.empty()) {
|
||||
graph_outputs.push_back(out_data.arg.value());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
GAPI_Assert(m_state->stop_criterion);
|
||||
GAPI_Assert(graph_inputs.size() == 1);
|
||||
GAPI_Assert(cv::util::holds_alternative<cv::GMat>(graph_inputs[0]));
|
||||
// FIXME: Handle GFrame when NV12 comes.
|
||||
const auto& graph_input = cv::util::get<cv::GMat>(graph_inputs[0]);
|
||||
graph_outputs.emplace_back(
|
||||
cv::gapi::streaming::timestamp(graph_input).strip());
|
||||
graph_outputs.emplace_back(
|
||||
cv::gapi::streaming::seq_id(graph_input).strip());
|
||||
|
||||
if (m_state->mode == PLMode::STREAMING) {
|
||||
return std::make_shared<StreamingPipeline>(std::move(m_state->name),
|
||||
cv::GComputation(
|
||||
cv::GProtoInputArgs{graph_inputs},
|
||||
cv::GProtoOutputArgs{graph_outputs}),
|
||||
std::move(m_state->src),
|
||||
std::move(m_state->stop_criterion),
|
||||
std::move(m_state->compile_args),
|
||||
graph_outputs.size());
|
||||
}
|
||||
GAPI_Assert(m_state->mode == PLMode::REGULAR);
|
||||
return std::make_shared<RegularPipeline>(std::move(m_state->name),
|
||||
cv::GComputation(
|
||||
cv::GProtoInputArgs{graph_inputs},
|
||||
cv::GProtoOutputArgs{graph_outputs}),
|
||||
std::move(m_state->src),
|
||||
std::move(m_state->stop_criterion),
|
||||
std::move(m_state->compile_args),
|
||||
graph_outputs.size());
|
||||
}
|
||||
|
||||
Pipeline::Ptr PipelineBuilder::build() {
|
||||
auto pipeline = construct();
|
||||
m_state.reset(new State{});
|
||||
return pipeline;
|
||||
}
|
||||
|
||||
#endif // OPENCV_GAPI_PIPELINE_MODELING_TOOL_PIPELINE_BUILDER_HPP
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,144 @@
|
||||
#ifndef OPENCV_GAPI_PIPELINE_MODELING_TOOL_UTILS_HPP
|
||||
#define OPENCV_GAPI_PIPELINE_MODELING_TOOL_UTILS_HPP
|
||||
|
||||
#include <map>
|
||||
|
||||
#include <opencv2/core.hpp>
|
||||
|
||||
#if defined(_WIN32)
|
||||
#include <windows.h>
|
||||
#endif
|
||||
|
||||
// FIXME: It's better to place it somewhere in common.hpp
|
||||
struct OutputDescr {
|
||||
std::vector<int> dims;
|
||||
int precision;
|
||||
};
|
||||
|
||||
namespace utils {
|
||||
|
||||
using double_ms_t = std::chrono::duration<double, std::milli>;
|
||||
|
||||
inline void createNDMat(cv::Mat& mat, const std::vector<int>& dims, int depth) {
|
||||
GAPI_Assert(!dims.empty());
|
||||
mat.create(dims, depth);
|
||||
if (dims.size() == 1) {
|
||||
//FIXME: Well-known 1D mat WA
|
||||
mat.dims = 1;
|
||||
}
|
||||
}
|
||||
|
||||
inline void generateRandom(cv::Mat& out) {
|
||||
switch (out.depth()) {
|
||||
case CV_8U:
|
||||
cv::randu(out, 0, 255);
|
||||
break;
|
||||
case CV_32F:
|
||||
cv::randu(out, 0.f, 1.f);
|
||||
break;
|
||||
case CV_16F: {
|
||||
std::vector<int> dims;
|
||||
for (int i = 0; i < out.size.dims(); ++i) {
|
||||
dims.push_back(out.size[i]);
|
||||
}
|
||||
cv::Mat fp32_mat;
|
||||
createNDMat(fp32_mat, dims, CV_32F);
|
||||
cv::randu(fp32_mat, 0.f, 1.f);
|
||||
fp32_mat.convertTo(out, out.type());
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw std::logic_error("Unsupported preprocessing depth");
|
||||
}
|
||||
}
|
||||
|
||||
inline void sleep(std::chrono::microseconds delay) {
|
||||
#if defined(_WIN32)
|
||||
// FIXME: Wrap it to RAII and instance only once.
|
||||
HANDLE timer = CreateWaitableTimer(NULL, true, NULL);
|
||||
if (!timer) {
|
||||
throw std::logic_error("Failed to create timer");
|
||||
}
|
||||
|
||||
LARGE_INTEGER li;
|
||||
using ns_t = std::chrono::nanoseconds;
|
||||
using ns_100_t = std::chrono::duration<ns_t::rep,
|
||||
std::ratio_multiply<std::ratio<100>, ns_t::period>>;
|
||||
// NB: QuadPart takes portions of 100 nanoseconds.
|
||||
li.QuadPart = -std::chrono::duration_cast<ns_100_t>(delay).count();
|
||||
|
||||
if(!SetWaitableTimer(timer, &li, 0, NULL, NULL, false)){
|
||||
CloseHandle(timer);
|
||||
throw std::logic_error("Failed to set timer");
|
||||
}
|
||||
if (WaitForSingleObject(timer, INFINITE) != WAIT_OBJECT_0) {
|
||||
CloseHandle(timer);
|
||||
throw std::logic_error("Failed to wait timer");
|
||||
}
|
||||
CloseHandle(timer);
|
||||
#else
|
||||
std::this_thread::sleep_for(delay);
|
||||
#endif
|
||||
}
|
||||
|
||||
template <typename duration_t>
|
||||
typename duration_t::rep measure(std::function<void()> f) {
|
||||
using namespace std::chrono;
|
||||
auto start = high_resolution_clock::now();
|
||||
f();
|
||||
return duration_cast<duration_t>(
|
||||
high_resolution_clock::now() - start).count();
|
||||
}
|
||||
|
||||
template <typename duration_t>
|
||||
typename duration_t::rep timestamp() {
|
||||
using namespace std::chrono;
|
||||
auto now = high_resolution_clock::now();
|
||||
return duration_cast<duration_t>(now.time_since_epoch()).count();
|
||||
}
|
||||
|
||||
inline void busyWait(std::chrono::microseconds delay) {
|
||||
auto start_ts = timestamp<std::chrono::microseconds>();
|
||||
auto end_ts = start_ts;
|
||||
auto time_to_wait = delay.count();
|
||||
|
||||
while (end_ts - start_ts < time_to_wait) {
|
||||
end_ts = timestamp<std::chrono::microseconds>();
|
||||
}
|
||||
}
|
||||
|
||||
template <typename K, typename V>
|
||||
void mergeMapWith(std::map<K, V>& target, const std::map<K, V>& second) {
|
||||
for (auto&& item : second) {
|
||||
auto it = target.find(item.first);
|
||||
if (it != target.end()) {
|
||||
throw std::logic_error("Error: key: " + it->first + " is already in target map");
|
||||
}
|
||||
target.insert(item);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
double avg(const std::vector<T>& vec) {
|
||||
return std::accumulate(vec.begin(), vec.end(), 0.0) / vec.size();
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
T max(const std::vector<T>& vec) {
|
||||
return *std::max_element(vec.begin(), vec.end());
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
T min(const std::vector<T>& vec) {
|
||||
return *std::min_element(vec.begin(), vec.end());
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
int64_t ms_to_mcs(T ms) {
|
||||
using namespace std::chrono;
|
||||
return duration_cast<microseconds>(duration<T, std::milli>(ms)).count();
|
||||
}
|
||||
|
||||
} // namespace utils
|
||||
|
||||
#endif // OPENCV_GAPI_PIPELINE_MODELING_TOOL_UTILS_HPP
|
||||
@@ -0,0 +1,167 @@
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
#include <cctype>
|
||||
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <opencv2/imgcodecs.hpp>
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/imgproc.hpp>
|
||||
#include <opencv2/gapi/infer.hpp>
|
||||
#include <opencv2/gapi/render.hpp>
|
||||
#include <opencv2/gapi/infer/ie.hpp>
|
||||
#include <opencv2/gapi/cpu/gcpukernel.hpp>
|
||||
#include <opencv2/gapi/streaming/cap.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
#include <opencv2/gapi/infer/parsers.hpp>
|
||||
|
||||
const std::string about =
|
||||
"This is an OpenCV-based version of Privacy Masking Camera example";
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ input | | Path to the input video file }"
|
||||
"{ platm | vehicle-license-plate-detection-barrier-0106.xml | Path to OpenVINO IE vehicle/plate detection model (.xml) }"
|
||||
"{ platd | CPU | Target device for vehicle/plate detection model (e.g. CPU, GPU, VPU, ...) }"
|
||||
"{ facem | face-detection-retail-0005.xml | Path to OpenVINO IE face detection model (.xml) }"
|
||||
"{ faced | CPU | Target device for face detection model (e.g. CPU, GPU, VPU, ...) }"
|
||||
"{ trad | false | Run processing in a traditional (non-pipelined) way }"
|
||||
"{ noshow | false | Don't display UI (improves performance) }";
|
||||
|
||||
namespace {
|
||||
|
||||
std::string weights_path(const std::string &model_path) {
|
||||
const auto EXT_LEN = 4u;
|
||||
const auto sz = model_path.size();
|
||||
CV_Assert(sz > EXT_LEN);
|
||||
|
||||
auto ext = model_path.substr(sz - EXT_LEN);
|
||||
|
||||
std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c){ return static_cast<unsigned char>(std::tolower(c)); });
|
||||
CV_Assert(ext == ".xml");
|
||||
|
||||
return model_path.substr(0u, sz - EXT_LEN) + ".bin";
|
||||
}
|
||||
} // namespace
|
||||
|
||||
namespace custom {
|
||||
|
||||
G_API_NET(VehLicDetector, <cv::GMat(cv::GMat)>, "vehicle-license-plate-detector");
|
||||
G_API_NET(FaceDetector, <cv::GMat(cv::GMat)>, "face-detector");
|
||||
|
||||
using GDetections = cv::GArray<cv::Rect>;
|
||||
|
||||
using GPrims = cv::GArray<cv::gapi::wip::draw::Prim>;
|
||||
|
||||
G_API_OP(ToMosaic, <GPrims(GDetections, GDetections)>, "custom.privacy_masking.to_mosaic") {
|
||||
static cv::GArrayDesc outMeta(const cv::GArrayDesc &, const cv::GArrayDesc &) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL(OCVToMosaic, ToMosaic) {
|
||||
static void run(const std::vector<cv::Rect> &in_plate_rcs,
|
||||
const std::vector<cv::Rect> &in_face_rcs,
|
||||
std::vector<cv::gapi::wip::draw::Prim> &out_prims) {
|
||||
out_prims.clear();
|
||||
const auto cvt = [](cv::Rect rc) {
|
||||
// Align the mosaic region to mosaic block size
|
||||
const int BLOCK_SIZE = 24;
|
||||
const int dw = BLOCK_SIZE - (rc.width % BLOCK_SIZE);
|
||||
const int dh = BLOCK_SIZE - (rc.height % BLOCK_SIZE);
|
||||
rc.width += dw;
|
||||
rc.height += dh;
|
||||
rc.x -= dw / 2;
|
||||
rc.y -= dh / 2;
|
||||
return cv::gapi::wip::draw::Mosaic{rc, BLOCK_SIZE, 0};
|
||||
};
|
||||
for (auto &&rc : in_plate_rcs) { out_prims.emplace_back(cvt(rc)); }
|
||||
for (auto &&rc : in_face_rcs) { out_prims.emplace_back(cvt(rc)); }
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace custom
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
cmd.about(about);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
const std::string input = cmd.get<std::string>("input");
|
||||
const bool no_show = cmd.get<bool>("noshow");
|
||||
const bool run_trad = cmd.get<bool>("trad");
|
||||
|
||||
cv::GMat in;
|
||||
cv::GMat blob_plates = cv::gapi::infer<custom::VehLicDetector>(in);
|
||||
cv::GMat blob_faces = cv::gapi::infer<custom::FaceDetector>(in);
|
||||
// VehLicDetector from Open Model Zoo marks vehicles with label "1" and
|
||||
// license plates with label "2", filter out license plates only.
|
||||
cv::GOpaque<cv::Size> sz = cv::gapi::streaming::size(in);
|
||||
cv::GArray<cv::Rect> rc_plates, rc_faces;
|
||||
cv::GArray<int> labels;
|
||||
std::tie(rc_plates, labels) = cv::gapi::parseSSD(blob_plates, sz, 0.5f, 2);
|
||||
// Face detector produces faces only so there's no need to filter by label,
|
||||
// pass "-1".
|
||||
std::tie(rc_faces, labels) = cv::gapi::parseSSD(blob_faces, sz, 0.5f, -1);
|
||||
cv::GMat out = cv::gapi::wip::draw::render3ch(in, custom::ToMosaic::on(rc_plates, rc_faces));
|
||||
cv::GComputation graph(in, out);
|
||||
|
||||
const auto plate_model_path = cmd.get<std::string>("platm");
|
||||
auto plate_net = cv::gapi::ie::Params<custom::VehLicDetector> {
|
||||
plate_model_path, // path to topology IR
|
||||
weights_path(plate_model_path), // path to weights
|
||||
cmd.get<std::string>("platd"), // device specifier
|
||||
};
|
||||
const auto face_model_path = cmd.get<std::string>("facem");
|
||||
auto face_net = cv::gapi::ie::Params<custom::FaceDetector> {
|
||||
face_model_path, // path to topology IR
|
||||
weights_path(face_model_path), // path to weights
|
||||
cmd.get<std::string>("faced"), // device specifier
|
||||
};
|
||||
auto kernels = cv::gapi::kernels<custom::OCVToMosaic>();
|
||||
auto networks = cv::gapi::networks(plate_net, face_net);
|
||||
|
||||
cv::TickMeter tm;
|
||||
cv::Mat out_frame;
|
||||
std::size_t frames = 0u;
|
||||
std::cout << "Reading " << input << std::endl;
|
||||
|
||||
if (run_trad) {
|
||||
cv::Mat in_frame;
|
||||
cv::VideoCapture cap(input);
|
||||
cap >> in_frame;
|
||||
|
||||
auto exec = graph.compile(cv::descr_of(in_frame), cv::compile_args(kernels, networks));
|
||||
tm.start();
|
||||
do {
|
||||
exec(in_frame, out_frame);
|
||||
if (!no_show) {
|
||||
cv::imshow("Out", out_frame);
|
||||
cv::waitKey(1);
|
||||
}
|
||||
frames++;
|
||||
} while (cap.read(in_frame));
|
||||
tm.stop();
|
||||
} else {
|
||||
auto pipeline = graph.compileStreaming(cv::compile_args(kernels, networks));
|
||||
pipeline.setSource(cv::gapi::wip::make_src<cv::gapi::wip::GCaptureSource>(input));
|
||||
pipeline.start();
|
||||
tm.start();
|
||||
|
||||
while (pipeline.pull(cv::gout(out_frame))) {
|
||||
frames++;
|
||||
if (!no_show) {
|
||||
cv::imshow("Out", out_frame);
|
||||
cv::waitKey(1);
|
||||
}
|
||||
}
|
||||
|
||||
tm.stop();
|
||||
}
|
||||
|
||||
std::cout << "Processed " << frames << " frames"
|
||||
<< " (" << frames / tm.getTimeSec() << " FPS)" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <opencv2/gapi/infer/ie.hpp>
|
||||
#include <opencv2/gapi/cpu/gcpukernel.hpp>
|
||||
#include <opencv2/gapi/streaming/cap.hpp>
|
||||
#include <opencv2/gapi/operators.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
|
||||
#include <opencv2/gapi/streaming/desync.hpp>
|
||||
#include <opencv2/gapi/streaming/format.hpp>
|
||||
|
||||
#include <iomanip>
|
||||
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ desync | false | Desynchronize inference }"
|
||||
"{ input | | Path to the input video file }"
|
||||
"{ output | | Path to the output video file }"
|
||||
"{ ssm | semantic-segmentation-adas-0001.xml | Path to OpenVINO IE semantic segmentation model (.xml) }";
|
||||
|
||||
// 20 colors for 20 classes of semantic-segmentation-adas-0001
|
||||
static std::vector<cv::Vec3b> colors = {
|
||||
{ 0, 0, 0 },
|
||||
{ 0, 0, 128 },
|
||||
{ 0, 128, 0 },
|
||||
{ 0, 128, 128 },
|
||||
{ 128, 0, 0 },
|
||||
{ 128, 0, 128 },
|
||||
{ 128, 128, 0 },
|
||||
{ 128, 128, 128 },
|
||||
{ 0, 0, 64 },
|
||||
{ 0, 0, 192 },
|
||||
{ 0, 128, 64 },
|
||||
{ 0, 128, 192 },
|
||||
{ 128, 0, 64 },
|
||||
{ 128, 0, 192 },
|
||||
{ 128, 128, 64 },
|
||||
{ 128, 128, 192 },
|
||||
{ 0, 64, 0 },
|
||||
{ 0, 64, 128 },
|
||||
{ 0, 192, 0 },
|
||||
{ 0, 192, 128 },
|
||||
{ 128, 64, 0 }
|
||||
};
|
||||
|
||||
namespace {
|
||||
std::string get_weights_path(const std::string &model_path) {
|
||||
const auto EXT_LEN = 4u;
|
||||
const auto sz = model_path.size();
|
||||
CV_Assert(sz > EXT_LEN);
|
||||
|
||||
auto ext = model_path.substr(sz - EXT_LEN);
|
||||
std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c){
|
||||
return static_cast<unsigned char>(std::tolower(c));
|
||||
});
|
||||
CV_Assert(ext == ".xml");
|
||||
return model_path.substr(0u, sz - EXT_LEN) + ".bin";
|
||||
}
|
||||
|
||||
bool isNumber(const std::string &str) {
|
||||
return !str.empty() && std::all_of(str.begin(), str.end(),
|
||||
[](unsigned char ch) { return std::isdigit(ch); });
|
||||
}
|
||||
|
||||
std::string toStr(double value) {
|
||||
std::stringstream ss;
|
||||
ss << std::fixed << std::setprecision(1) << value;
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
void classesToColors(const cv::Mat &out_blob,
|
||||
cv::Mat &mask_img) {
|
||||
const int H = out_blob.size[0];
|
||||
const int W = out_blob.size[1];
|
||||
|
||||
mask_img.create(H, W, CV_8UC3);
|
||||
GAPI_Assert(out_blob.type() == CV_8UC1);
|
||||
const uint8_t* const classes = out_blob.ptr<uint8_t>();
|
||||
|
||||
for (int rowId = 0; rowId < H; ++rowId) {
|
||||
for (int colId = 0; colId < W; ++colId) {
|
||||
uint8_t class_id = classes[rowId * W + colId];
|
||||
mask_img.at<cv::Vec3b>(rowId, colId) =
|
||||
class_id < colors.size()
|
||||
? colors[class_id]
|
||||
: cv::Vec3b{0, 0, 0}; // NB: sample supports 20 classes
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void probsToClasses(const cv::Mat& probs, cv::Mat& classes) {
|
||||
const int C = probs.size[1];
|
||||
const int H = probs.size[2];
|
||||
const int W = probs.size[3];
|
||||
|
||||
classes.create(H, W, CV_8UC1);
|
||||
GAPI_Assert(probs.depth() == CV_32F);
|
||||
float* out_p = reinterpret_cast<float*>(probs.data);
|
||||
uint8_t* classes_p = reinterpret_cast<uint8_t*>(classes.data);
|
||||
|
||||
for (int h = 0; h < H; ++h) {
|
||||
for (int w = 0; w < W; ++w) {
|
||||
double max = 0;
|
||||
int class_id = 0;
|
||||
for (int c = 0; c < C; ++c) {
|
||||
int idx = c * H * W + h * W + w;
|
||||
if (out_p[idx] > max) {
|
||||
max = out_p[idx];
|
||||
class_id = c;
|
||||
}
|
||||
}
|
||||
classes_p[h * W + w] = static_cast<uint8_t>(class_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
namespace vis {
|
||||
|
||||
static void putText(cv::Mat& mat, const cv::Point &position, const std::string &message) {
|
||||
auto fontFace = cv::FONT_HERSHEY_COMPLEX;
|
||||
int thickness = 2;
|
||||
cv::Scalar color = {200, 10, 10};
|
||||
double fontScale = 0.65;
|
||||
|
||||
cv::putText(mat, message, position, fontFace,
|
||||
fontScale, cv::Scalar(255, 255, 255), thickness + 1);
|
||||
cv::putText(mat, message, position, fontFace, fontScale, color, thickness);
|
||||
}
|
||||
|
||||
static void drawResults(cv::Mat &img, const cv::Mat &color_mask) {
|
||||
img = img / 2 + color_mask / 2;
|
||||
}
|
||||
|
||||
} // namespace vis
|
||||
|
||||
namespace custom {
|
||||
G_API_OP(PostProcessing, <cv::GMat(cv::GMat, cv::GMat)>, "sample.custom.post_processing") {
|
||||
static cv::GMatDesc outMeta(const cv::GMatDesc &in, const cv::GMatDesc &) {
|
||||
return in;
|
||||
}
|
||||
};
|
||||
|
||||
GAPI_OCV_KERNEL(OCVPostProcessing, PostProcessing) {
|
||||
static void run(const cv::Mat &in, const cv::Mat &out_blob, cv::Mat &out) {
|
||||
int C = -1, H = -1, W = -1;
|
||||
if (out_blob.size.dims() == 4u) {
|
||||
C = 1; H = 2, W = 3;
|
||||
} else if (out_blob.size.dims() == 3u) {
|
||||
C = 0; H = 1, W = 2;
|
||||
} else {
|
||||
throw std::logic_error(
|
||||
"Number of dimmensions for model output must be 3 or 4!");
|
||||
}
|
||||
cv::Mat classes;
|
||||
// NB: If output has more than single plane, it contains probabilities
|
||||
// otherwise class id.
|
||||
if (out_blob.size[C] > 1) {
|
||||
probsToClasses(out_blob, classes);
|
||||
} else {
|
||||
if (out_blob.depth() != CV_32S) {
|
||||
throw std::logic_error(
|
||||
"Single channel output must have integer precision!");
|
||||
}
|
||||
cv::Mat view(out_blob.size[H], // cols
|
||||
out_blob.size[W], // rows
|
||||
CV_32SC1,
|
||||
out_blob.data);
|
||||
view.convertTo(classes, CV_8UC1);
|
||||
}
|
||||
cv::Mat mask_img;
|
||||
classesToColors(classes, mask_img);
|
||||
cv::resize(mask_img, out, in.size(), 0, 0, cv::INTER_NEAREST);
|
||||
}
|
||||
};
|
||||
} // namespace custom
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Prepare parameters first
|
||||
const std::string input = cmd.get<std::string>("input");
|
||||
const std::string output = cmd.get<std::string>("output");
|
||||
const auto model_path = cmd.get<std::string>("ssm");
|
||||
const bool desync = cmd.get<bool>("desync");
|
||||
const auto weights_path = get_weights_path(model_path);
|
||||
const auto device = "CPU";
|
||||
G_API_NET(SemSegmNet, <cv::GMat(cv::GMat)>, "semantic-segmentation");
|
||||
const auto net = cv::gapi::ie::Params<SemSegmNet> {
|
||||
model_path, weights_path, device
|
||||
};
|
||||
const auto kernels = cv::gapi::kernels<custom::OCVPostProcessing>();
|
||||
const auto networks = cv::gapi::networks(net);
|
||||
|
||||
// Now build the graph
|
||||
cv::GMat in;
|
||||
cv::GMat bgr = cv::gapi::copy(in);
|
||||
cv::GMat frame = desync ? cv::gapi::streaming::desync(bgr) : bgr;
|
||||
cv::GMat out_blob = cv::gapi::infer<SemSegmNet>(frame);
|
||||
cv::GMat out = custom::PostProcessing::on(frame, out_blob);
|
||||
|
||||
cv::GStreamingCompiled pipeline = cv::GComputation(cv::GIn(in), cv::GOut(bgr, out))
|
||||
.compileStreaming(cv::compile_args(kernels, networks,
|
||||
cv::gapi::streaming::queue_capacity{1}));
|
||||
|
||||
std::shared_ptr<cv::gapi::wip::GCaptureSource> source;
|
||||
if (isNumber(input)) {
|
||||
source = std::make_shared<cv::gapi::wip::GCaptureSource>(
|
||||
std::stoi(input),
|
||||
std::map<int, double> {
|
||||
{cv::CAP_PROP_FRAME_WIDTH, 1280},
|
||||
{cv::CAP_PROP_FRAME_HEIGHT, 720},
|
||||
{cv::CAP_PROP_BUFFERSIZE, 1},
|
||||
{cv::CAP_PROP_AUTOFOCUS, true}
|
||||
}
|
||||
);
|
||||
} else {
|
||||
source = std::make_shared<cv::gapi::wip::GCaptureSource>(input);
|
||||
}
|
||||
auto inputs = cv::gin(
|
||||
static_cast<cv::gapi::wip::IStreamSource::Ptr>(source));
|
||||
|
||||
// The execution part
|
||||
pipeline.setSource(std::move(inputs));
|
||||
|
||||
cv::TickMeter tm;
|
||||
cv::VideoWriter writer;
|
||||
|
||||
cv::util::optional<cv::Mat> color_mask;
|
||||
cv::util::optional<cv::Mat> image;
|
||||
cv::Mat last_image;
|
||||
cv::Mat last_color_mask;
|
||||
|
||||
pipeline.start();
|
||||
tm.start();
|
||||
|
||||
std::size_t frames = 0u;
|
||||
std::size_t masks = 0u;
|
||||
while (pipeline.pull(cv::gout(image, color_mask))) {
|
||||
if (image.has_value()) {
|
||||
++frames;
|
||||
last_image = std::move(*image);
|
||||
}
|
||||
|
||||
if (color_mask.has_value()) {
|
||||
++masks;
|
||||
last_color_mask = std::move(*color_mask);
|
||||
}
|
||||
|
||||
if (!last_image.empty() && !last_color_mask.empty()) {
|
||||
tm.stop();
|
||||
|
||||
std::string stream_fps = "Stream FPS: " + toStr(frames / tm.getTimeSec());
|
||||
std::string inference_fps = "Inference FPS: " + toStr(masks / tm.getTimeSec());
|
||||
|
||||
cv::Mat tmp = last_image.clone();
|
||||
|
||||
vis::drawResults(tmp, last_color_mask);
|
||||
vis::putText(tmp, {10, 22}, stream_fps);
|
||||
vis::putText(tmp, {10, 22 + 30}, inference_fps);
|
||||
|
||||
cv::imshow("Out", tmp);
|
||||
cv::waitKey(1);
|
||||
if (!output.empty()) {
|
||||
if (!writer.isOpened()) {
|
||||
const auto sz = cv::Size{tmp.cols, tmp.rows};
|
||||
writer.open(output, cv::VideoWriter::fourcc('M','J','P','G'), 25.0, sz);
|
||||
CV_Assert(writer.isOpened());
|
||||
}
|
||||
writer << tmp;
|
||||
}
|
||||
|
||||
tm.start();
|
||||
}
|
||||
}
|
||||
tm.stop();
|
||||
std::cout << "Processed " << frames << " frames" << " ("
|
||||
<< frames / tm.getTimeSec()<< " FPS)" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
#include <opencv2/gapi.hpp> // G-API framework header
|
||||
#include <opencv2/gapi/imgproc.hpp> // cv::gapi::blur()
|
||||
#include <opencv2/highgui.hpp> // cv::imread/imwrite
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
if (argc < 3) return 1;
|
||||
|
||||
cv::GMat in; // Express the graph:
|
||||
cv::GMat out = cv::gapi::blur(in, cv::Size(3,3)); // `out` is a result of `blur` of `in`
|
||||
|
||||
cv::Mat in_mat = cv::imread(argv[1]); // Get the real data
|
||||
cv::Mat out_mat; // Output buffer (may be empty)
|
||||
|
||||
cv::GComputation(cv::GIn(in), cv::GOut(out)) // Declare a graph from `in` to `out`
|
||||
.apply(cv::gin(in_mat), cv::gout(out_mat)); // ...and run it immediately
|
||||
|
||||
cv::imwrite(argv[2], out_mat); // Save the result
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
#include <opencv2/videoio.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/imgproc.hpp>
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
|
||||
using namespace cv;
|
||||
Mat in_mat = imread("lena.png");
|
||||
Mat gx, gy;
|
||||
|
||||
Sobel(in_mat, gx, CV_32F, 1, 0);
|
||||
Sobel(in_mat, gy, CV_32F, 0, 1);
|
||||
|
||||
Mat mag;
|
||||
sqrt(gx.mul(gx) + gy.mul(gy), mag);
|
||||
|
||||
Mat out_mat;
|
||||
mag.convertTo(out_mat, CV_8U);
|
||||
|
||||
imwrite("lena-out.png", out_mat);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
#include <opencv2/videoio.hpp>
|
||||
#include <opencv2/highgui.hpp>
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/imgproc.hpp>
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
|
||||
using namespace cv;
|
||||
Mat in_mat = imread("lena.png");
|
||||
Mat out_mat;
|
||||
|
||||
GMat in;
|
||||
GMat gx = gapi::Sobel(in, CV_32F, 1, 0);
|
||||
GMat gy = gapi::Sobel(in, CV_32F, 0, 1);
|
||||
GMat mag = gapi::sqrt( gapi::mul(gx, gx)
|
||||
+ gapi::mul(gy, gy));
|
||||
GMat out = gapi::convertTo(mag, CV_8U);
|
||||
|
||||
GComputation sobel(GIn(in), GOut(out));
|
||||
sobel.apply(in_mat, out_mat);
|
||||
|
||||
imwrite("lena-out.png", out_mat);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,698 @@
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <cmath>
|
||||
#include <iostream>
|
||||
#include <limits>
|
||||
#include <numeric>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include <opencv2/gapi.hpp>
|
||||
#include <opencv2/gapi/core.hpp>
|
||||
#include <opencv2/gapi/cpu/gcpukernel.hpp>
|
||||
#include <opencv2/gapi/infer.hpp>
|
||||
#include <opencv2/gapi/infer/ie.hpp>
|
||||
#include <opencv2/gapi/streaming/cap.hpp>
|
||||
|
||||
#include <opencv2/highgui.hpp>
|
||||
#include <opencv2/core/utility.hpp>
|
||||
|
||||
const std::string about =
|
||||
"This is an OpenCV-based version of OMZ Text Detection example";
|
||||
const std::string keys =
|
||||
"{ h help | | Print this help message }"
|
||||
"{ input | | Path to the input video file }"
|
||||
"{ tdm | text-detection-0004.xml | Path to OpenVINO text detection model (.xml), versions 0003 and 0004 work }"
|
||||
"{ tdd | CPU | Target device for the text detector (e.g. CPU, GPU, VPU, ...) }"
|
||||
"{ trm | text-recognition-0012.xml | Path to OpenVINO text recognition model (.xml) }"
|
||||
"{ trd | CPU | Target device for the text recognition (e.g. CPU, GPU, VPU, ...) }"
|
||||
"{ bw | 0 | CTC beam search decoder bandwidth, if 0, a CTC greedy decoder is used}"
|
||||
"{ sset | 0123456789abcdefghijklmnopqrstuvwxyz | Symbol set to use with text recognition decoder. Shouldn't contain symbol #. }"
|
||||
"{ thr | 0.2 | Text recognition confidence threshold}"
|
||||
;
|
||||
|
||||
namespace {
|
||||
std::string weights_path(const std::string &model_path) {
|
||||
const auto EXT_LEN = 4u;
|
||||
const auto sz = model_path.size();
|
||||
CV_Assert(sz > EXT_LEN);
|
||||
|
||||
const auto ext = model_path.substr(sz - EXT_LEN);
|
||||
CV_Assert(cv::toLowerCase(ext) == ".xml");
|
||||
return model_path.substr(0u, sz - EXT_LEN) + ".bin";
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// Taken from OMZ samples as-is
|
||||
template<typename Iter>
|
||||
void softmax_and_choose(Iter begin, Iter end, int *argmax, float *prob) {
|
||||
auto max_element = std::max_element(begin, end);
|
||||
*argmax = static_cast<int>(std::distance(begin, max_element));
|
||||
float max_val = *max_element;
|
||||
double sum = 0;
|
||||
for (auto i = begin; i != end; i++) {
|
||||
sum += std::exp((*i) - max_val);
|
||||
}
|
||||
if (std::fabs(sum) < std::numeric_limits<double>::epsilon()) {
|
||||
throw std::logic_error("sum can't be equal to zero");
|
||||
}
|
||||
*prob = 1.0f / static_cast<float>(sum);
|
||||
}
|
||||
|
||||
template<typename Iter>
|
||||
std::vector<float> softmax(Iter begin, Iter end) {
|
||||
std::vector<float> prob(end - begin, 0.f);
|
||||
std::transform(begin, end, prob.begin(), [](float x) { return std::exp(x); });
|
||||
float sum = std::accumulate(prob.begin(), prob.end(), 0.0f);
|
||||
for (int i = 0; i < static_cast<int>(prob.size()); i++)
|
||||
prob[i] /= sum;
|
||||
return prob;
|
||||
}
|
||||
|
||||
struct BeamElement {
|
||||
std::vector<int> sentence; //!< The sequence of chars that will be a result of the beam element
|
||||
|
||||
float prob_blank; //!< The probability that the last char in CTC sequence
|
||||
//!< for the beam element is the special blank char
|
||||
|
||||
float prob_not_blank; //!< The probability that the last char in CTC sequence
|
||||
//!< for the beam element is NOT the special blank char
|
||||
|
||||
float prob() const { //!< The probability of the beam element.
|
||||
return prob_blank + prob_not_blank;
|
||||
}
|
||||
};
|
||||
|
||||
std::string CTCGreedyDecoder(const float *data,
|
||||
const std::size_t sz,
|
||||
const std::string &alphabet,
|
||||
const char pad_symbol,
|
||||
double *conf) {
|
||||
std::string res = "";
|
||||
bool prev_pad = false;
|
||||
*conf = 1;
|
||||
|
||||
const auto num_classes = alphabet.length();
|
||||
for (auto it = data; it != (data+sz); it += num_classes) {
|
||||
int argmax = 0;
|
||||
float prob = 0.f;
|
||||
|
||||
softmax_and_choose(it, it + num_classes, &argmax, &prob);
|
||||
(*conf) *= prob;
|
||||
|
||||
auto symbol = alphabet[argmax];
|
||||
if (symbol != pad_symbol) {
|
||||
if (res.empty() || prev_pad || (!res.empty() && symbol != res.back())) {
|
||||
prev_pad = false;
|
||||
res += symbol;
|
||||
}
|
||||
} else {
|
||||
prev_pad = true;
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
std::string CTCBeamSearchDecoder(const float *data,
|
||||
const std::size_t sz,
|
||||
const std::string &alphabet,
|
||||
double *conf,
|
||||
int bandwidth) {
|
||||
const auto num_classes = alphabet.length();
|
||||
|
||||
std::vector<BeamElement> curr;
|
||||
std::vector<BeamElement> last;
|
||||
|
||||
last.push_back(BeamElement{std::vector<int>(), 1.f, 0.f});
|
||||
|
||||
for (auto it = data; it != (data+sz); it += num_classes) {
|
||||
curr.clear();
|
||||
|
||||
std::vector<float> prob = softmax(it, it + num_classes);
|
||||
|
||||
for(const auto& candidate: last) {
|
||||
float prob_not_blank = 0.f;
|
||||
const std::vector<int>& candidate_sentence = candidate.sentence;
|
||||
if (!candidate_sentence.empty()) {
|
||||
int n = candidate_sentence.back();
|
||||
prob_not_blank = candidate.prob_not_blank * prob[n];
|
||||
}
|
||||
float prob_blank = candidate.prob() * prob[num_classes - 1];
|
||||
|
||||
auto check_res = std::find_if(curr.begin(),
|
||||
curr.end(),
|
||||
[&candidate_sentence](const BeamElement& n) {
|
||||
return n.sentence == candidate_sentence;
|
||||
});
|
||||
if (check_res == std::end(curr)) {
|
||||
curr.push_back(BeamElement{candidate.sentence, prob_blank, prob_not_blank});
|
||||
} else {
|
||||
check_res->prob_not_blank += prob_not_blank;
|
||||
if (check_res->prob_blank != 0.f) {
|
||||
throw std::logic_error("Probability that the last char in CTC-sequence "
|
||||
"is the special blank char must be zero here");
|
||||
}
|
||||
check_res->prob_blank = prob_blank;
|
||||
}
|
||||
|
||||
for (int i = 0; i < static_cast<int>(num_classes) - 1; i++) {
|
||||
auto extend = candidate_sentence;
|
||||
extend.push_back(i);
|
||||
|
||||
if (candidate_sentence.size() > 0 && candidate.sentence.back() == i) {
|
||||
prob_not_blank = prob[i] * candidate.prob_blank;
|
||||
} else {
|
||||
prob_not_blank = prob[i] * candidate.prob();
|
||||
}
|
||||
|
||||
auto check_res2 = std::find_if(curr.begin(),
|
||||
curr.end(),
|
||||
[&extend](const BeamElement &n) {
|
||||
return n.sentence == extend;
|
||||
});
|
||||
if (check_res2 == std::end(curr)) {
|
||||
curr.push_back(BeamElement{extend, 0.f, prob_not_blank});
|
||||
} else {
|
||||
check_res2->prob_not_blank += prob_not_blank;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sort(curr.begin(), curr.end(), [](const BeamElement &a, const BeamElement &b) -> bool {
|
||||
return a.prob() > b.prob();
|
||||
});
|
||||
|
||||
last.clear();
|
||||
int num_to_copy = std::min(bandwidth, static_cast<int>(curr.size()));
|
||||
for (int b = 0; b < num_to_copy; b++) {
|
||||
last.push_back(curr[b]);
|
||||
}
|
||||
}
|
||||
|
||||
*conf = last[0].prob();
|
||||
std::string res="";
|
||||
for (const auto& idx: last[0].sentence) {
|
||||
res += alphabet[idx];
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
} // anonymous namespace
|
||||
|
||||
namespace custom {
|
||||
namespace {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// Define networks for this sample
|
||||
using GMat2 = std::tuple<cv::GMat, cv::GMat>;
|
||||
G_API_NET(TextDetection,
|
||||
<GMat2(cv::GMat)>,
|
||||
"sample.custom.text_detect");
|
||||
|
||||
G_API_NET(TextRecognition,
|
||||
<cv::GMat(cv::GMat)>,
|
||||
"sample.custom.text_recogn");
|
||||
|
||||
// Define custom operations
|
||||
using GSize = cv::GOpaque<cv::Size>;
|
||||
using GRRects = cv::GArray<cv::RotatedRect>;
|
||||
G_API_OP(PostProcess,
|
||||
<GRRects(cv::GMat,cv::GMat,GSize,float,float)>,
|
||||
"sample.custom.text.post_proc") {
|
||||
static cv::GArrayDesc outMeta(const cv::GMatDesc &,
|
||||
const cv::GMatDesc &,
|
||||
const cv::GOpaqueDesc &,
|
||||
float,
|
||||
float) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
using GMats = cv::GArray<cv::GMat>;
|
||||
G_API_OP(CropLabels,
|
||||
<GMats(cv::GMat,GRRects,GSize)>,
|
||||
"sample.custom.text.crop") {
|
||||
static cv::GArrayDesc outMeta(const cv::GMatDesc &,
|
||||
const cv::GArrayDesc &,
|
||||
const cv::GOpaqueDesc &) {
|
||||
return cv::empty_array_desc();
|
||||
}
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// Implement custom operations
|
||||
GAPI_OCV_KERNEL(OCVPostProcess, PostProcess) {
|
||||
static void run(const cv::Mat &link,
|
||||
const cv::Mat &segm,
|
||||
const cv::Size &img_size,
|
||||
const float link_threshold,
|
||||
const float segm_threshold,
|
||||
std::vector<cv::RotatedRect> &out) {
|
||||
// NOTE: Taken from the OMZ text detection sample almost as-is
|
||||
const int kMinArea = 300;
|
||||
const int kMinHeight = 10;
|
||||
|
||||
const float *link_data_pointer = link.ptr<float>();
|
||||
std::vector<float> link_data(link_data_pointer, link_data_pointer + link.total());
|
||||
link_data = transpose4d(link_data, dimsToShape(link.size), {0, 2, 3, 1});
|
||||
softmax(link_data);
|
||||
link_data = sliceAndGetSecondChannel(link_data);
|
||||
std::vector<int> new_link_data_shape = {
|
||||
link.size[0],
|
||||
link.size[2],
|
||||
link.size[3],
|
||||
link.size[1]/2,
|
||||
};
|
||||
|
||||
const float *cls_data_pointer = segm.ptr<float>();
|
||||
std::vector<float> cls_data(cls_data_pointer, cls_data_pointer + segm.total());
|
||||
cls_data = transpose4d(cls_data, dimsToShape(segm.size), {0, 2, 3, 1});
|
||||
softmax(cls_data);
|
||||
cls_data = sliceAndGetSecondChannel(cls_data);
|
||||
std::vector<int> new_cls_data_shape = {
|
||||
segm.size[0],
|
||||
segm.size[2],
|
||||
segm.size[3],
|
||||
segm.size[1]/2,
|
||||
};
|
||||
|
||||
out = maskToBoxes(decodeImageByJoin(cls_data, new_cls_data_shape,
|
||||
link_data, new_link_data_shape,
|
||||
segm_threshold, link_threshold),
|
||||
static_cast<float>(kMinArea),
|
||||
static_cast<float>(kMinHeight),
|
||||
img_size);
|
||||
}
|
||||
|
||||
static std::vector<std::size_t> dimsToShape(const cv::MatSize &sz) {
|
||||
const int n_dims = sz.dims();
|
||||
std::vector<std::size_t> result;
|
||||
result.reserve(n_dims);
|
||||
|
||||
// cv::MatSize is not iterable...
|
||||
for (int i = 0; i < n_dims; i++) {
|
||||
result.emplace_back(static_cast<std::size_t>(sz[i]));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static void softmax(std::vector<float> &rdata) {
|
||||
// NOTE: Taken from the OMZ text detection sample almost as-is
|
||||
const size_t last_dim = 2;
|
||||
for (size_t i = 0 ; i < rdata.size(); i+=last_dim) {
|
||||
float m = std::max(rdata[i], rdata[i+1]);
|
||||
rdata[i] = std::exp(rdata[i] - m);
|
||||
rdata[i + 1] = std::exp(rdata[i + 1] - m);
|
||||
float s = rdata[i] + rdata[i + 1];
|
||||
rdata[i] /= s;
|
||||
rdata[i + 1] /= s;
|
||||
}
|
||||
}
|
||||
|
||||
static std::vector<float> transpose4d(const std::vector<float> &data,
|
||||
const std::vector<size_t> &shape,
|
||||
const std::vector<size_t> &axes) {
|
||||
// NOTE: Taken from the OMZ text detection sample almost as-is
|
||||
if (shape.size() != axes.size())
|
||||
throw std::runtime_error("Shape and axes must have the same dimension.");
|
||||
|
||||
for (size_t a : axes) {
|
||||
if (a >= shape.size())
|
||||
throw std::runtime_error("Axis must be less than dimension of shape.");
|
||||
}
|
||||
size_t total_size = shape[0]*shape[1]*shape[2]*shape[3];
|
||||
std::vector<size_t> steps {
|
||||
shape[axes[1]]*shape[axes[2]]*shape[axes[3]],
|
||||
shape[axes[2]]*shape[axes[3]],
|
||||
shape[axes[3]],
|
||||
1
|
||||
};
|
||||
|
||||
size_t source_data_idx = 0;
|
||||
std::vector<float> new_data(total_size, 0);
|
||||
std::vector<size_t> ids(shape.size());
|
||||
for (ids[0] = 0; ids[0] < shape[0]; ids[0]++) {
|
||||
for (ids[1] = 0; ids[1] < shape[1]; ids[1]++) {
|
||||
for (ids[2] = 0; ids[2] < shape[2]; ids[2]++) {
|
||||
for (ids[3]= 0; ids[3] < shape[3]; ids[3]++) {
|
||||
size_t new_data_idx = ids[axes[0]]*steps[0] + ids[axes[1]]*steps[1] +
|
||||
ids[axes[2]]*steps[2] + ids[axes[3]]*steps[3];
|
||||
new_data[new_data_idx] = data[source_data_idx++];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return new_data;
|
||||
}
|
||||
|
||||
static std::vector<float> sliceAndGetSecondChannel(const std::vector<float> &data) {
|
||||
// NOTE: Taken from the OMZ text detection sample almost as-is
|
||||
std::vector<float> new_data(data.size() / 2, 0);
|
||||
for (size_t i = 0; i < data.size() / 2; i++) {
|
||||
new_data[i] = data[2 * i + 1];
|
||||
}
|
||||
return new_data;
|
||||
}
|
||||
|
||||
static void join(const int p1,
|
||||
const int p2,
|
||||
std::unordered_map<int, int> &group_mask) {
|
||||
// NOTE: Taken from the OMZ text detection sample almost as-is
|
||||
const int root1 = findRoot(p1, group_mask);
|
||||
const int root2 = findRoot(p2, group_mask);
|
||||
if (root1 != root2) {
|
||||
group_mask[root1] = root2;
|
||||
}
|
||||
}
|
||||
|
||||
static cv::Mat decodeImageByJoin(const std::vector<float> &cls_data,
|
||||
const std::vector<int> &cls_data_shape,
|
||||
const std::vector<float> &link_data,
|
||||
const std::vector<int> &link_data_shape,
|
||||
float cls_conf_threshold,
|
||||
float link_conf_threshold) {
|
||||
// NOTE: Taken from the OMZ text detection sample almost as-is
|
||||
const int h = cls_data_shape[1];
|
||||
const int w = cls_data_shape[2];
|
||||
|
||||
std::vector<uchar> pixel_mask(h * w, 0);
|
||||
std::unordered_map<int, int> group_mask;
|
||||
std::vector<cv::Point> points;
|
||||
for (int i = 0; i < static_cast<int>(pixel_mask.size()); i++) {
|
||||
pixel_mask[i] = cls_data[i] >= cls_conf_threshold;
|
||||
if (pixel_mask[i]) {
|
||||
points.emplace_back(i % w, i / w);
|
||||
group_mask[i] = -1;
|
||||
}
|
||||
}
|
||||
std::vector<uchar> link_mask(link_data.size(), 0);
|
||||
for (size_t i = 0; i < link_mask.size(); i++) {
|
||||
link_mask[i] = link_data[i] >= link_conf_threshold;
|
||||
}
|
||||
size_t neighbours = size_t(link_data_shape[3]);
|
||||
for (const auto &point : points) {
|
||||
size_t neighbour = 0;
|
||||
for (int ny = point.y - 1; ny <= point.y + 1; ny++) {
|
||||
for (int nx = point.x - 1; nx <= point.x + 1; nx++) {
|
||||
if (nx == point.x && ny == point.y)
|
||||
continue;
|
||||
if (nx >= 0 && nx < w && ny >= 0 && ny < h) {
|
||||
uchar pixel_value = pixel_mask[size_t(ny) * size_t(w) + size_t(nx)];
|
||||
uchar link_value = link_mask[(size_t(point.y) * size_t(w) + size_t(point.x))
|
||||
*neighbours + neighbour];
|
||||
if (pixel_value && link_value) {
|
||||
join(point.x + point.y * w, nx + ny * w, group_mask);
|
||||
}
|
||||
}
|
||||
neighbour++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return get_all(points, w, h, group_mask);
|
||||
}
|
||||
|
||||
static cv::Mat get_all(const std::vector<cv::Point> &points,
|
||||
const int w,
|
||||
const int h,
|
||||
std::unordered_map<int, int> &group_mask) {
|
||||
// NOTE: Taken from the OMZ text detection sample almost as-is
|
||||
std::unordered_map<int, int> root_map;
|
||||
cv::Mat mask(h, w, CV_32S, cv::Scalar(0));
|
||||
for (const auto &point : points) {
|
||||
int point_root = findRoot(point.x + point.y * w, group_mask);
|
||||
if (root_map.find(point_root) == root_map.end()) {
|
||||
root_map.emplace(point_root, static_cast<int>(root_map.size() + 1));
|
||||
}
|
||||
mask.at<int>(point.x + point.y * w) = root_map[point_root];
|
||||
}
|
||||
return mask;
|
||||
}
|
||||
|
||||
static int findRoot(const int point,
|
||||
std::unordered_map<int, int> &group_mask) {
|
||||
// NOTE: Taken from the OMZ text detection sample almost as-is
|
||||
int root = point;
|
||||
bool update_parent = false;
|
||||
while (group_mask.at(root) != -1) {
|
||||
root = group_mask.at(root);
|
||||
update_parent = true;
|
||||
}
|
||||
if (update_parent) {
|
||||
group_mask[point] = root;
|
||||
}
|
||||
return root;
|
||||
}
|
||||
|
||||
static std::vector<cv::RotatedRect> maskToBoxes(const cv::Mat &mask,
|
||||
const float min_area,
|
||||
const float min_height,
|
||||
const cv::Size &image_size) {
|
||||
// NOTE: Taken from the OMZ text detection sample almost as-is
|
||||
std::vector<cv::RotatedRect> bboxes;
|
||||
double min_val = 0.;
|
||||
double max_val = 0.;
|
||||
cv::minMaxLoc(mask, &min_val, &max_val);
|
||||
int max_bbox_idx = static_cast<int>(max_val);
|
||||
cv::Mat resized_mask;
|
||||
cv::resize(mask, resized_mask, image_size, 0, 0, cv::INTER_NEAREST);
|
||||
|
||||
for (int i = 1; i <= max_bbox_idx; i++) {
|
||||
cv::Mat bbox_mask = resized_mask == i;
|
||||
std::vector<std::vector<cv::Point>> contours;
|
||||
|
||||
cv::findContours(bbox_mask, contours, cv::RETR_CCOMP, cv::CHAIN_APPROX_SIMPLE);
|
||||
if (contours.empty())
|
||||
continue;
|
||||
cv::RotatedRect r = cv::minAreaRect(contours[0]);
|
||||
if (std::min(r.size.width, r.size.height) < min_height)
|
||||
continue;
|
||||
if (r.size.area() < min_area)
|
||||
continue;
|
||||
bboxes.emplace_back(r);
|
||||
}
|
||||
return bboxes;
|
||||
}
|
||||
}; // GAPI_OCV_KERNEL(PostProcess)
|
||||
|
||||
GAPI_OCV_KERNEL(OCVCropLabels, CropLabels) {
|
||||
static void run(const cv::Mat &image,
|
||||
const std::vector<cv::RotatedRect> &detections,
|
||||
const cv::Size &outSize,
|
||||
std::vector<cv::Mat> &out) {
|
||||
out.clear();
|
||||
out.reserve(detections.size());
|
||||
cv::Mat crop(outSize, CV_8UC3, cv::Scalar(0));
|
||||
cv::Mat gray(outSize, CV_8UC1, cv::Scalar(0));
|
||||
std::vector<int> blob_shape = {1,1,outSize.height,outSize.width};
|
||||
|
||||
for (auto &&rr : detections) {
|
||||
std::vector<cv::Point2f> points(4);
|
||||
rr.points(points.data());
|
||||
|
||||
const auto top_left_point_idx = topLeftPointIdx(points);
|
||||
cv::Point2f point0 = points[static_cast<size_t>(top_left_point_idx)];
|
||||
cv::Point2f point1 = points[(top_left_point_idx + 1) % 4];
|
||||
cv::Point2f point2 = points[(top_left_point_idx + 2) % 4];
|
||||
|
||||
std::vector<cv::Point2f> from{point0, point1, point2};
|
||||
std::vector<cv::Point2f> to{
|
||||
cv::Point2f(0.0f, 0.0f),
|
||||
cv::Point2f(static_cast<float>(outSize.width-1), 0.0f),
|
||||
cv::Point2f(static_cast<float>(outSize.width-1),
|
||||
static_cast<float>(outSize.height-1))
|
||||
};
|
||||
cv::Mat M = cv::getAffineTransform(from, to);
|
||||
cv::warpAffine(image, crop, M, outSize);
|
||||
cv::cvtColor(crop, gray, cv::COLOR_BGR2GRAY);
|
||||
|
||||
cv::Mat blob;
|
||||
gray.convertTo(blob, CV_32F);
|
||||
out.push_back(blob.reshape(1, blob_shape)); // pass as 1,1,H,W instead of H,W
|
||||
}
|
||||
}
|
||||
|
||||
static int topLeftPointIdx(const std::vector<cv::Point2f> &points) {
|
||||
// NOTE: Taken from the OMZ text detection sample almost as-is
|
||||
cv::Point2f most_left(std::numeric_limits<float>::max(),
|
||||
std::numeric_limits<float>::max());
|
||||
cv::Point2f almost_most_left(std::numeric_limits<float>::max(),
|
||||
std::numeric_limits<float>::max());
|
||||
int most_left_idx = -1;
|
||||
int almost_most_left_idx = -1;
|
||||
|
||||
for (size_t i = 0; i < points.size() ; i++) {
|
||||
if (most_left.x > points[i].x) {
|
||||
if (most_left.x < std::numeric_limits<float>::max()) {
|
||||
almost_most_left = most_left;
|
||||
almost_most_left_idx = most_left_idx;
|
||||
}
|
||||
most_left = points[i];
|
||||
most_left_idx = static_cast<int>(i);
|
||||
}
|
||||
if (almost_most_left.x > points[i].x && points[i] != most_left) {
|
||||
almost_most_left = points[i];
|
||||
almost_most_left_idx = static_cast<int>(i);
|
||||
}
|
||||
}
|
||||
|
||||
if (almost_most_left.y < most_left.y) {
|
||||
most_left = almost_most_left;
|
||||
most_left_idx = almost_most_left_idx;
|
||||
}
|
||||
return most_left_idx;
|
||||
}
|
||||
|
||||
}; // GAPI_OCV_KERNEL(CropLabels)
|
||||
|
||||
} // anonymous namespace
|
||||
} // namespace custom
|
||||
|
||||
namespace vis {
|
||||
namespace {
|
||||
|
||||
void drawRotatedRect(cv::Mat &m, const cv::RotatedRect &rc) {
|
||||
std::vector<cv::Point2f> tmp_points(5);
|
||||
rc.points(tmp_points.data());
|
||||
tmp_points[4] = tmp_points[0];
|
||||
auto prev = tmp_points.begin(), it = prev+1;
|
||||
for (; it != tmp_points.end(); ++it) {
|
||||
cv::line(m, *prev, *it, cv::Scalar(50, 205, 50), 2);
|
||||
prev = it;
|
||||
}
|
||||
}
|
||||
|
||||
void drawText(cv::Mat &m, const cv::RotatedRect &rc, const std::string &str) {
|
||||
const int fface = cv::FONT_HERSHEY_SIMPLEX;
|
||||
const double scale = 0.7;
|
||||
const int thick = 1;
|
||||
int base = 0;
|
||||
const auto text_size = cv::getTextSize(str, fface, scale, thick, &base);
|
||||
|
||||
std::vector<cv::Point2f> tmp_points(4);
|
||||
rc.points(tmp_points.data());
|
||||
const auto tl_point_idx = custom::OCVCropLabels::topLeftPointIdx(tmp_points);
|
||||
cv::Point text_pos = tmp_points[tl_point_idx];
|
||||
text_pos.x = std::max(0, text_pos.x);
|
||||
text_pos.y = std::max(text_size.height, text_pos.y);
|
||||
|
||||
cv::rectangle(m,
|
||||
text_pos + cv::Point{0, base},
|
||||
text_pos + cv::Point{text_size.width, -text_size.height},
|
||||
CV_RGB(50, 205, 50),
|
||||
cv::FILLED);
|
||||
const auto white = CV_RGB(255, 255, 255);
|
||||
cv::putText(m, str, text_pos, fface, scale, white, thick, 8);
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
} // namespace vis
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
cv::CommandLineParser cmd(argc, argv, keys);
|
||||
cmd.about(about);
|
||||
if (cmd.has("help")) {
|
||||
cmd.printMessage();
|
||||
return 0;
|
||||
}
|
||||
const auto input_file_name = cmd.get<std::string>("input");
|
||||
const auto tdet_model_path = cmd.get<std::string>("tdm");
|
||||
const auto trec_model_path = cmd.get<std::string>("trm");
|
||||
const auto tdet_target_dev = cmd.get<std::string>("tdd");
|
||||
const auto trec_target_dev = cmd.get<std::string>("trd");
|
||||
const auto ctc_beam_dec_bw = cmd.get<int>("bw");
|
||||
const auto dec_conf_thresh = cmd.get<double>("thr");
|
||||
|
||||
const auto pad_symbol = '#';
|
||||
const auto symbol_set = cmd.get<std::string>("sset") + pad_symbol;
|
||||
|
||||
cv::GMat in;
|
||||
cv::GOpaque<cv::Size> in_rec_sz;
|
||||
cv::GMat link, segm;
|
||||
std::tie(link, segm) = cv::gapi::infer<custom::TextDetection>(in);
|
||||
cv::GOpaque<cv::Size> size = cv::gapi::streaming::size(in);
|
||||
cv::GArray<cv::RotatedRect> rrs = custom::PostProcess::on(link, segm, size, 0.8f, 0.8f);
|
||||
cv::GArray<cv::GMat> labels = custom::CropLabels::on(in, rrs, in_rec_sz);
|
||||
cv::GArray<cv::GMat> text = cv::gapi::infer2<custom::TextRecognition>(in, labels);
|
||||
|
||||
cv::GComputation graph(cv::GIn(in, in_rec_sz),
|
||||
cv::GOut(cv::gapi::copy(in), rrs, text));
|
||||
|
||||
// Text detection network
|
||||
auto tdet_net = cv::gapi::ie::Params<custom::TextDetection> {
|
||||
tdet_model_path, // path to topology IR
|
||||
weights_path(tdet_model_path), // path to weights
|
||||
tdet_target_dev, // device specifier
|
||||
}.cfgOutputLayers({"model/link_logits_/add", "model/segm_logits/add"});
|
||||
|
||||
auto trec_net = cv::gapi::ie::Params<custom::TextRecognition> {
|
||||
trec_model_path, // path to topology IR
|
||||
weights_path(trec_model_path), // path to weights
|
||||
trec_target_dev, // device specifier
|
||||
};
|
||||
auto networks = cv::gapi::networks(tdet_net, trec_net);
|
||||
|
||||
auto kernels = cv::gapi::kernels< custom::OCVPostProcess
|
||||
, custom::OCVCropLabels
|
||||
>();
|
||||
auto pipeline = graph.compileStreaming(cv::compile_args(kernels, networks));
|
||||
|
||||
std::cout << "Reading " << input_file_name << std::endl;
|
||||
|
||||
// Input stream
|
||||
auto in_src = cv::gapi::wip::make_src<cv::gapi::wip::GCaptureSource>(input_file_name);
|
||||
|
||||
// Text recognition input size (also an input parameter to the graph)
|
||||
auto in_rsz = cv::Size{ 120, 32 };
|
||||
|
||||
// Set the pipeline source & start the pipeline
|
||||
pipeline.setSource(cv::gin(in_src, in_rsz));
|
||||
pipeline.start();
|
||||
|
||||
// Declare the output data & run the processing loop
|
||||
cv::TickMeter tm;
|
||||
cv::Mat image;
|
||||
std::vector<cv::RotatedRect> out_rcs;
|
||||
std::vector<cv::Mat> out_text;
|
||||
|
||||
tm.start();
|
||||
int frames = 0;
|
||||
while (pipeline.pull(cv::gout(image, out_rcs, out_text))) {
|
||||
frames++;
|
||||
|
||||
CV_Assert(out_rcs.size() == out_text.size());
|
||||
const auto num_labels = out_rcs.size();
|
||||
|
||||
std::vector<cv::Point2f> tmp_points(4);
|
||||
for (std::size_t l = 0; l < num_labels; l++) {
|
||||
// Decode the recognized text in the rectangle
|
||||
const auto &blob = out_text[l];
|
||||
const float *data = blob.ptr<float>();
|
||||
const auto sz = blob.total();
|
||||
double conf = 1.0;
|
||||
const std::string res = ctc_beam_dec_bw == 0
|
||||
? CTCGreedyDecoder(data, sz, symbol_set, pad_symbol, &conf)
|
||||
: CTCBeamSearchDecoder(data, sz, symbol_set, &conf, ctc_beam_dec_bw);
|
||||
|
||||
// Draw a bounding box for this rotated rectangle
|
||||
const auto &rc = out_rcs[l];
|
||||
vis::drawRotatedRect(image, rc);
|
||||
|
||||
// Draw text, if decoded
|
||||
if (conf >= dec_conf_thresh) {
|
||||
vis::drawText(image, rc, res);
|
||||
}
|
||||
}
|
||||
tm.stop();
|
||||
cv::imshow("Out", image);
|
||||
cv::waitKey(1);
|
||||
tm.start();
|
||||
}
|
||||
tm.stop();
|
||||
std::cout << "Processed " << frames << " frames"
|
||||
<< " (" << frames / tm.getTimeSec() << " FPS)" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user