Files
2026-07-13 13:33:03 +08:00

422 lines
20 KiB
C++
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//
// RasterBufExecution.cpp
// MNN
//
// Created by MNN on 2020/05/12.
// Copyright © 2018, Alibaba Group Holding Limited
//
#ifndef MNN_OPENCL_BUFFER_CLOSED
#include "backend/opencl/execution/buffer/RasterBufExecution.hpp"
#include "core/Macro.h"
#include "core/TensorUtils.hpp"
#include "core/OpCommonUtils.hpp"
#include "backend/opencl/core/OpenCLBackend.hpp"
namespace MNN {
namespace OpenCL {
RasterBufExecution::RasterBufExecution(const std::vector<Tensor *> &inputs, const MNN::Op *op, Backend *backend)
: CommonExecution(backend, op) {
mOpenCLBackend = (OpenCLBackend *)backend;
//nothing to do
}
ErrorCode RasterBufExecution::onEncode(const std::vector<Tensor *> &____inputs, const std::vector<Tensor *> &outputs) {
#ifdef LOG_VERBOSE
MNN_PRINT("start RasterBufExecution onResize !\n");
#endif
mTempInput.clear();
mCombineInfo.clear();
mTempOutput = nullptr;
MNN_ASSERT(outputs.size() == 1);
auto output = outputs[0];
if (!____inputs.empty()) {
OpCommonUtils::rasterInputReset(____inputs, outputs[0]);
}
auto des = TensorUtils::getDescribe(output);
auto outputDes = TensorUtils::getDescribe(output);
auto regionNum = des->regions.size();
auto mOpenCLBackend = static_cast<OpenCLBackend*>(backend());
auto runtime = mOpenCLBackend->getOpenCLRuntime();
int kernel_idx = 0;
auto outputShape = tensorShapeFormat(output);
mFast = false;
if (outputDes->dimensionFormat == MNN_DATA_FORMAT_NC4HW4) {
mFast = true;
for (int i=0; i< des->regions.size(); ++i) {
auto& slice = des->regions[i];
if (TensorUtils::getDescribe(slice.origin)->dimensionFormat != MNN_DATA_FORMAT_NC4HW4) {
mFast = false;
break;
}
if (!OpCommonUtils::canBlitFast(slice, output, 4, true)) {
mFast = false;
break;
}
}
}
mNeedZero = !TensorUtils::regionIsFull(output);
mNeedZero = mNeedZero || ((outputShape[3] % 4) != 0 && MNN_DATA_FORMAT_NC4HW4 == outputDes->dimensionFormat && !mFast);
if(mFast == false){
CanCombine(outputs);
regionNum = mCombineInfo.size();
}
mUnits.resize(regionNum);
if(mNeedZero)
{
mUnits.resize(regionNum + 1);
int region[] = {outputShape[0], outputShape[3], outputShape[1], outputShape[2]};//nchw
if(MNN_DATA_FORMAT_NC4HW4 == outputDes->dimensionFormat){
region[1] = ROUND_UP(outputShape[3], 4);
}
Unit &unit = mUnits[kernel_idx++];
unit.kernel = runtime->buildKernel("raster_buf", "buffer_set_zero", {}, mOpenCLBackend->getPrecision(), output, output);
unit.localWorkSize = {8, 8};
unit.globalWorkSize = {(uint32_t)UP_DIV((region[2] * region[3]), 8)*8,
(uint32_t)UP_DIV((region[0] * region[1]), 8)*8};
int global_dim0 = region[2] * region[3];
int global_dim1 = region[0] * region[1];
uint32_t idx = 0;
cl_int ret = CL_SUCCESS;
ret |= unit.kernel->get().setArg(idx++, global_dim0);
ret |= unit.kernel->get().setArg(idx++, global_dim1);
ret |= unit.kernel->get().setArg(idx++, openCLBuffer(output));
if(ret != CL_SUCCESS)
{
MNN_PRINT("setArg err %d\n", (int)ret);
}
mOpenCLBackend->recordKernel2d(unit.kernel, {(uint32_t)UP_DIV((region[2] * region[3]), 8)*8,
(uint32_t)UP_DIV((region[0] * region[1]), 8)*8}, {8, 8});
}
if(mFast)
{
// nc4hw4 buffer raster
for (auto& slice : des->regions)
{
auto origin = slice.origin;
auto inputShape = tensorShapeFormat(origin);
Tensor::InsideDescribe::Region C4Region;
OpCommonUtils::turnToPackRegion(slice, C4Region, output, 4, true);
Unit &unit = mUnits[kernel_idx++];
unit.kernel = runtime->buildKernel("raster_buf", "raster_nc4hw4_buffer", {}, mOpenCLBackend->getPrecision(), origin, output);
const std::vector<uint32_t> gws = {(uint32_t)C4Region.size[2],
(uint32_t)C4Region.size[1],
(uint32_t)C4Region.size[0]};
uint32_t mMaxWorkGroupSize = static_cast<uint32_t>(runtime->getMaxWorkGroupSize(unit.kernel));
auto outputShape = tensorShapeFormat(output);
auto sliceShape = tensorShapeFormat(slice.origin);
uint32_t idx = 0;
cl_int ret = CL_SUCCESS;
ret |= unit.kernel->get().setArg(idx++, gws[0]);
ret |= unit.kernel->get().setArg(idx++, gws[1]);
ret |= unit.kernel->get().setArg(idx++, gws[2]);
ret |= unit.kernel->get().setArg(idx++, openCLBuffer(slice.origin));
ret |= unit.kernel->get().setArg(idx++, C4Region.src.offset);
ret |= unit.kernel->get().setArg(idx++, C4Region.src.stride[0]);
ret |= unit.kernel->get().setArg(idx++, C4Region.src.stride[1]);
ret |= unit.kernel->get().setArg(idx++, C4Region.src.stride[2]);
ret |= unit.kernel->get().setArg(idx++, sliceShape[1]);
ret |= unit.kernel->get().setArg(idx++, sliceShape[2]);
ret |= unit.kernel->get().setArg(idx++, sliceShape[3]);
ret |= unit.kernel->get().setArg(idx++, openCLBuffer(output));
ret |= unit.kernel->get().setArg(idx++, C4Region.dst.offset);
ret |= unit.kernel->get().setArg(idx++, C4Region.dst.stride[0]);
ret |= unit.kernel->get().setArg(idx++, C4Region.dst.stride[1]);
ret |= unit.kernel->get().setArg(idx++, C4Region.dst.stride[2]);
ret |= unit.kernel->get().setArg(idx++, outputShape[1]);
ret |= unit.kernel->get().setArg(idx++, outputShape[2]);
ret |= unit.kernel->get().setArg(idx++, outputShape[3]);
if(ret != CL_SUCCESS)
{
MNN_PRINT("setArg err %d\n", (int)ret);
}
std::string name = "raster_nc4hw4_buffer";
const std::vector<uint32_t> lws = localWS3DDefault(gws, mMaxWorkGroupSize, mOpenCLBackend->getOpenCLRuntime(), name, unit.kernel, mOpenCLBackend->getCLTuneLevel(), "raster_buf").first;
unit.localWorkSize = {lws[0], lws[1], lws[2]};
unit.globalWorkSize = {ROUND_UP(gws[0], std::max((uint32_t)1, lws[0])),
ROUND_UP(gws[1], std::max((uint32_t)1, lws[1])),
ROUND_UP(gws[2], std::max((uint32_t)1, lws[2]))};
mOpenCLBackend->recordKernel3d(unit.kernel, gws, lws);
}
return NO_ERROR;
}
for(auto& info : mCombineInfo){
auto slice = info.mRegion;
int nums = info.mCanCombineNum;
int src_offset = info.mSrc_offset;
int dst_offset = info.mDst_offset;
std::set<std::string> buildOptions;
auto origin = slice.origin;
auto inputShape = tensorShapeFormat(origin);
buildOptions.emplace("-DINPUT_FORMAT=" + std::to_string(TensorUtils::getDescribe(origin)->dimensionFormat));
buildOptions.emplace("-DOUTPUT_FORMAT=" + std::to_string(outputDes->dimensionFormat));
// Detect L2 cache-set thrashing in NC4HW4 tensors:
// When NC4HW4 tensor has N (batch) as power-of-2 and H*W=1,
// channel groups are spaced N*4 elements apart. Consecutive work-items
// access consecutive channels → different channel groups → same cache set.
// Fix: reshape 1D traversal into 2D (batch × channel) so consecutive
// work-items walk the batch dimension (contiguous in NC4HW4 memory).
bool inputIsNC4HW4 = TensorUtils::getDescribe(origin)->dimensionFormat == MNN_DATA_FORMAT_NC4HW4;
bool outputIsNC4HW4 = outputDes->dimensionFormat == MNN_DATA_FORMAT_NC4HW4;
auto isPow2 = [](int v) { return v > 0 && (v & (v - 1)) == 0; };
// Check if we have a 1D raster with NC4HW4 tensor whose batch dim is power-of-2
int nc4_N = 0, nc4_C = 0;
bool needTranspose = false;
if (slice.size[0] == 1 && slice.size[1] == 1 && slice.src.stride[2] == 1 && slice.dst.stride[2] == 1) {
if (inputIsNC4HW4 && inputShape[1] * inputShape[2] == 1) {
// Input is NC4HW4 with H*W=1, N=inputShape[0], C=inputShape[3]
nc4_N = inputShape[0];
nc4_C = inputShape[3];
} else if (outputIsNC4HW4 && outputShape[1] * outputShape[2] == 1) {
// Output is NC4HW4 with H*W=1, N=outputShape[0], C=outputShape[3]
nc4_N = outputShape[0];
nc4_C = outputShape[3];
}
if (nc4_N >= 256 && isPow2(nc4_N) && nc4_C > 4 && nc4_N * nc4_C == slice.size[2]) {
needTranspose = true;
}
}
Unit &unit = mUnits[kernel_idx++];
unit.kernel = runtime->buildKernel("raster_buf", "raster_direct_buffer", buildOptions, mOpenCLBackend->getPrecision(), origin, output);
if (needTranspose) {
// 2D traversal: x=batch(N), y=channel(C)
// inputIndex = x * C + y (instead of original x where in_c = x%C, in_b = x/C)
// This makes consecutive work-items access same channel group, different batches
const std::vector<uint32_t> gws = {(uint32_t)nc4_N * nums, (uint32_t)nc4_C, 1u};
uint32_t mMaxWorkGroupSize = static_cast<uint32_t>(runtime->getMaxWorkGroupSize(unit.kernel));
// Transposed strides: x walks batch (stride=C), y walks channel (stride=1)
int srcStride0_t = slice.src.stride[0];
int srcStride1_t = 1;
int srcStride2_t = nc4_C;
int dstStride0_t = slice.dst.stride[0];
int dstStride1_t = 1;
int dstStride2_t = nc4_C;
uint32_t idx = 0;
cl_int ret = CL_SUCCESS;
ret |= unit.kernel->get().setArg(idx++, gws[0]);
ret |= unit.kernel->get().setArg(idx++, gws[1]);
ret |= unit.kernel->get().setArg(idx++, gws[2]);
ret |= unit.kernel->get().setArg(idx++, (int)nc4_N); // size_x = N (batch per combine group)
ret |= unit.kernel->get().setArg(idx++, openCLBuffer(origin));
ret |= unit.kernel->get().setArg(idx++, slice.src.offset);
ret |= unit.kernel->get().setArg(idx++, src_offset);
ret |= unit.kernel->get().setArg(idx++, srcStride0_t);
ret |= unit.kernel->get().setArg(idx++, srcStride1_t);
ret |= unit.kernel->get().setArg(idx++, srcStride2_t);
ret |= unit.kernel->get().setArg(idx++, inputShape[2]);
ret |= unit.kernel->get().setArg(idx++, inputShape[1]);
ret |= unit.kernel->get().setArg(idx++, inputShape[3]);
ret |= unit.kernel->get().setArg(idx++, inputShape[0]);
ret |= unit.kernel->get().setArg(idx++, openCLBuffer(output));
ret |= unit.kernel->get().setArg(idx++, slice.dst.offset);
ret |= unit.kernel->get().setArg(idx++, dst_offset);
ret |= unit.kernel->get().setArg(idx++, dstStride0_t);
ret |= unit.kernel->get().setArg(idx++, dstStride1_t);
ret |= unit.kernel->get().setArg(idx++, dstStride2_t);
ret |= unit.kernel->get().setArg(idx++, outputShape[2]);
ret |= unit.kernel->get().setArg(idx++, outputShape[1]);
ret |= unit.kernel->get().setArg(idx++, outputShape[3]);
ret |= unit.kernel->get().setArg(idx++, outputShape[0]);
if (ret != CL_SUCCESS) {
MNN_PRINT("setArg err %d\n", (int)ret);
}
std::string name = "raster_buffer_transpose";
const std::vector<uint32_t> lws =
localWS3DDefault(gws, mMaxWorkGroupSize, mOpenCLBackend->getOpenCLRuntime(), name, unit.kernel,
mOpenCLBackend->getCLTuneLevel(), "raster_buf")
.first;
unit.localWorkSize = {lws[0], lws[1], lws[2]};
unit.globalWorkSize = {ROUND_UP(gws[0], std::max((uint32_t)1, lws[0])),
ROUND_UP(gws[1], std::max((uint32_t)1, lws[1])),
ROUND_UP(gws[2], std::max((uint32_t)1, lws[2]))};
mOpenCLBackend->recordKernel3d(unit.kernel, gws, lws);
} else {
// Original path
const std::vector<uint32_t> gws = {(uint32_t)slice.size[2] * nums, (uint32_t)slice.size[1],
(uint32_t)slice.size[0]};
uint32_t mMaxWorkGroupSize = static_cast<uint32_t>(runtime->getMaxWorkGroupSize(unit.kernel));
uint32_t idx = 0;
cl_int ret = CL_SUCCESS;
ret |= unit.kernel->get().setArg(idx++, gws[0]);
ret |= unit.kernel->get().setArg(idx++, gws[1]);
ret |= unit.kernel->get().setArg(idx++, gws[2]);
ret |= unit.kernel->get().setArg(idx++, slice.size[2]);
ret |= unit.kernel->get().setArg(idx++, openCLBuffer(origin));
ret |= unit.kernel->get().setArg(idx++, slice.src.offset);
ret |= unit.kernel->get().setArg(idx++, src_offset);
ret |= unit.kernel->get().setArg(idx++, slice.src.stride[0]);
ret |= unit.kernel->get().setArg(idx++, slice.src.stride[1]);
ret |= unit.kernel->get().setArg(idx++, slice.src.stride[2]);
ret |= unit.kernel->get().setArg(idx++, inputShape[2]);
ret |= unit.kernel->get().setArg(idx++, inputShape[1]);
ret |= unit.kernel->get().setArg(idx++, inputShape[3]);
ret |= unit.kernel->get().setArg(idx++, inputShape[0]);
ret |= unit.kernel->get().setArg(idx++, openCLBuffer(output));
ret |= unit.kernel->get().setArg(idx++, slice.dst.offset);
ret |= unit.kernel->get().setArg(idx++, dst_offset);
ret |= unit.kernel->get().setArg(idx++, slice.dst.stride[0]);
ret |= unit.kernel->get().setArg(idx++, slice.dst.stride[1]);
ret |= unit.kernel->get().setArg(idx++, slice.dst.stride[2]);
ret |= unit.kernel->get().setArg(idx++, outputShape[2]);
ret |= unit.kernel->get().setArg(idx++, outputShape[1]);
ret |= unit.kernel->get().setArg(idx++, outputShape[3]);
ret |= unit.kernel->get().setArg(idx++, outputShape[0]);
if (ret != CL_SUCCESS) {
MNN_PRINT("setArg err %d\n", (int)ret);
}
std::string name = "raster_buffer";
const std::vector<uint32_t> lws =
localWS3DDefault(gws, mMaxWorkGroupSize, mOpenCLBackend->getOpenCLRuntime(), name, unit.kernel,
mOpenCLBackend->getCLTuneLevel(), "raster_buf")
.first;
unit.localWorkSize = {lws[0], lws[1], lws[2]};
unit.globalWorkSize = {gws[0], gws[1], gws[2]};
mOpenCLBackend->recordKernel3d(unit.kernel, gws, lws);
}
}
#ifdef LOG_VERBOSE
MNN_PRINT("end RasterBufExecution onResize !\n");
#endif
return NO_ERROR;
}
class RasterBufCreator : public OpenCLBackend::Creator {
public:
virtual ~RasterBufCreator() = default;
virtual Execution *onCreate(const std::vector<Tensor *> &inputs, const std::vector<Tensor *> &outputs, const MNN::Op *op,
Backend *backend) const override {
for (int i = 0; i < inputs.size(); ++i) {
TensorUtils::setTensorSupportPack(inputs[i], false);
}
for (int i = 0; i < outputs.size(); ++i) {
TensorUtils::setTensorSupportPack(outputs[i], false);
}
OPENCL_CREATOR_CHECK(new RasterBufExecution(inputs, op, backend));
}
};
void RasterBufExecution::CanCombine(const std::vector<Tensor *> &outputs){
auto des = TensorUtils::getDescribe(outputs[0]);
auto regions = des->regions;
Tensor* origin;
int size0, size1, size2, src_offset, dst_offset, last_src_offset, last_dst_offset, src_sride0, src_sride1, src_sride2, dst_sride0, dst_sride1, dst_sride2;
int canCombineNum = 0;
for(auto& slice : des->regions){
bool res = true;
if(canCombineNum == 0){
origin = slice.origin;
size0 = slice.size[0];
size1 = slice.size[1];
size2 = slice.size[2];
src_sride0 = slice.src.stride[0];
src_sride1 = slice.src.stride[1];
src_sride2 = slice.src.stride[2];
dst_sride0 = slice.dst.stride[0];
dst_sride1 = slice.dst.stride[1];
dst_sride2 = slice.dst.stride[2];
canCombineNum++;
// push back
mCombineInfo.push_back(CanCombineInfo(slice, 0, 0, 1));
} else if(canCombineNum == 1){
res &= slice.origin == origin;
res &= slice.size[0] == size0;
res &= slice.size[1] == size1;
res &= slice.size[2] == size2;
res &= slice.src.stride[0] == src_sride0;
res &= slice.src.stride[1] == src_sride1;
res &= slice.src.stride[2] == src_sride2;
res &= slice.dst.stride[0] == dst_sride0;
res &= slice.dst.stride[1] == dst_sride1;
res &= slice.dst.stride[2] == dst_sride2;
if(res){
src_offset = slice.src.offset - last_src_offset;
dst_offset = slice.dst.offset - last_dst_offset;
canCombineNum++;
// change canCombineNum
mCombineInfo.back().mSrc_offset = src_offset;
mCombineInfo.back().mDst_offset = dst_offset;
mCombineInfo.back().mCanCombineNum = canCombineNum;
} else{
origin = slice.origin;
size0 = slice.size[0];
size1 = slice.size[1];
size2 = slice.size[2];
src_sride0 = slice.src.stride[0];
src_sride1 = slice.src.stride[1];
src_sride2 = slice.src.stride[2];
dst_sride0 = slice.dst.stride[0];
dst_sride1 = slice.dst.stride[1];
dst_sride2 = slice.dst.stride[2];
// recover
canCombineNum = 1;
// push back
mCombineInfo.push_back(CanCombineInfo(slice, 0, 0, 1));
}
} else{
res &= slice.origin == origin;
res &= slice.size[0] == size0;
res &= slice.size[1] == size1;
res &= slice.size[2] == size2;
res &= slice.src.stride[0] == src_sride0;
res &= slice.src.stride[1] == src_sride1;
res &= slice.src.stride[2] == src_sride2;
res &= slice.dst.stride[0] == dst_sride0;
res &= slice.dst.stride[1] == dst_sride1;
res &= slice.dst.stride[2] == dst_sride2;
res &= slice.src.offset - last_src_offset == src_offset;
res &= slice.dst.offset - last_dst_offset == dst_offset;
if(res){
canCombineNum++;
// change canCombineNum
mCombineInfo.back().mSrc_offset = src_offset;
mCombineInfo.back().mDst_offset = dst_offset;
mCombineInfo.back().mCanCombineNum = canCombineNum;
} else{
origin = slice.origin;
size0 = slice.size[0];
size1 = slice.size[1];
size2 = slice.size[2];
src_sride0 = slice.src.stride[0];
src_sride1 = slice.src.stride[1];
src_sride2 = slice.src.stride[2];
dst_sride0 = slice.dst.stride[0];
dst_sride1 = slice.dst.stride[1];
dst_sride2 = slice.dst.stride[2];
// recover
canCombineNum = 1;
// push back
mCombineInfo.push_back(CanCombineInfo(slice, 0, 0, 1));
}
}
last_src_offset = slice.src.offset;
last_dst_offset = slice.dst.offset;
}
}
REGISTER_OPENCL_OP_CREATOR(RasterBufCreator, OpType_Raster, BUFFER);
} // namespace OpenCL
} // namespace MNN
#endif /* MNN_OPENCL_BUFFER_CLOSED */