// // CommonOptFunction.h // MNN // // Created by MNN on 2018/07/16. // Copyright © 2018, Alibaba Group Holding Limited // #ifndef CommonOptFunction_h #define CommonOptFunction_h #include #include #include #include #include #include "core/Macro.h" #include "backend/cpu/compute/Int8FunctionsOpt.h" #ifdef MNN_SUPPORT_TRANSFORMER_FUSE #define MNN_FLASH_ATTENTION_BLOCK_SIZE 64 #endif extern "C" { #ifdef __aarch64__ #ifdef MNN_LOW_MEMORY void MNNGeneralIm2col_Fp32Arm82(float* destOrigin, float const** sourceGroup, const int32_t* info, const int32_t* el, int32_t LP, int32_t pack); void MNNGeneralIm2col_Fp32Arm86(float* destOrigin, float const** sourceGroup, const int32_t* info, const int32_t* el, int32_t LP, int32_t pack); void MNNGeneralIm2col_Fp32Sme2(float* destOrigin, float const** sourceGroup, const int32_t* info, const int32_t* el, int32_t LP, int32_t pack); void MNNLocalMinMaxFP32_Pack4(float* dstMin, float* dstMax, const float* source, size_t blockNum, size_t blockLU, size_t EP, size_t LP, size_t loadDstBuffer); void MNNLocalMinMaxFP32_Pack8(float* dstMin, float* dstMax, const float* source, size_t blockNum, size_t blockLU, size_t EP, size_t LP, size_t loadDstBuffer); void MNNDynamicQuantFP32_Pack4(const float* src, int8_t* dst, const float* scale, size_t src_depth_quad, size_t realSize, const float* bias, size_t pack); void MNNDynamicQuantFP32_Pack8(const float* src, int8_t* dst, const float* scale, size_t src_depth_quad, size_t realSize, const float* bias, size_t pack); void MNNAbsMaxFP32_Pack4(const float* source, float* absmax, size_t src_depth_quad, size_t realSize, int pack); void MNNAbsMaxFP32_Pack8(const float* source, float* absmax, size_t src_depth_quad, size_t realSize, int pack); void MNNQuantScaleFP32(float* absmax, float* quant_scale, float* dequant_scale, size_t thread, size_t batch); void MNNDynamicUpdateConvBiasScale(float* newbias, float* oldbias, float* weightKernelSum, float* inputZero, size_t ocQuad); #endif // MNN_LOW_MEMORY #ifdef MNN_SME2 void MNNPackedMatMulRemainFP32_SME2(float* C, const float* A, const float* B, size_t eSize, const size_t* parameter, const float* postParameters, const float* bias, const float* k, const float* b); #endif #endif // __aarch64__ void MNNQuantAttentionKey(int8_t* dst, const float* source, float* sumKey, float* maxKey, int32_t* params); void MNNQuantAttentionValue(int8_t* dst, const float* source, float* valueQuantInfo, int32_t* params); void MNNReluWithSlope(float* dst, const float* src, size_t sizeQuad, float slope); void MNNReluInt8(int8_t* dst, const int8_t* src, size_t size, ssize_t zeroPoint); void MNNReluWithSlopeChannel(float* dst, const float* src, const float* slope, size_t sizeQuad, size_t depthQuad); void MNNHardSwish(float* dst, const float* src, size_t size); void MNNGelu(float* dst, const float* src, size_t size, float* parameters); void MNNPackC4(float* dst, const float* src, size_t area, size_t depth, int* areaOffset); void MNNPackC4Origin(float* dst, const float* src, size_t area, size_t depth, int areaOffset); void MNNPackC2(double* dst, const double* src, size_t area, size_t depth, int* areaOffset); void MNNPackC2Origin(double* dst, const double* src, size_t area, size_t depth, int areaOffset); void MNNPackInt8C2(float* dst, const float* src, size_t area, size_t depth, int* areaOffset); void MNNPackInt8C2Origin(float* dst, const float* src, size_t area, size_t depth, int areaOffset); void MNNPackC4Int16(int16_t* dst, const int16_t* src, size_t area, size_t depth, int* areaOffset); void MNNPackC4Uint8(uint8_t* dst, const uint8_t* src, size_t area, size_t depth, int* areaOffset); void MNNUnpackC4(float* dst, const float* src, size_t area, size_t depth, int* areaOffset); void MNNUnpackC4Origin(float* dst, const float* src, size_t area, size_t depth, int areaOffset); void MNNUnpackC2(double* dst, const double* src, size_t area, size_t depth, int* areaOffset); void MNNUnpackC2Origin(double* dst, const double* src, size_t area, size_t depth, int areaOffset); void MNNUnpackC2Float(float* dst, const float* src, size_t area, size_t depth, int* areaOffset, int pack = 1); void MNNUnpackInt8C2(float* dst, const float* src, size_t area, size_t depth, int* areaOffset); void MNNUnpackInt8C2Origin(float* dst, const float* src, size_t area, size_t depth, int areaOffset); void MNNUnpackC4Int16(int16_t* dst, const int16_t* src, size_t area, size_t depth, int* areaOffset); void MNNUnpackC4Uint8(uint8_t* dst, const uint8_t* src, size_t area, size_t depth, int* areaOffset); void MNNScaleAndAddBias(float* dst, const float* src, const float* bias, const float* alpha, size_t planeNumber, size_t biasNumber); void MNNScaleAndAddBiasScalar(float* dst, const float* src, float bias, float alpha, size_t number); // TODO: Swap the name for MNNUnpackTranspose and MNNPackTranspose void MNNUnpackTranspose(float* dst, const float* src, size_t area, size_t depth, int* areaOffset); void MNNUnpackTransposeInt16(int16_t* dst, const int16_t* src, size_t area, size_t depth, int* areaOffset); void MNNUnpackTransposeUint8(uint8_t* dst, const uint8_t* src, size_t area, size_t depth, int* areaOffset); void MNNPackTranspose(float* dst, const float* src, size_t area, size_t depth, int* areaOffset); void MNNPackTransposeInt16(int16_t* dst, const int16_t* src, size_t area, size_t depth, int* areaOffset); void MNNPackTransposeUint8(uint8_t* dst, const uint8_t* src, size_t area, size_t depth, int* areaOffset); void MNNCopyC4WithStride(const float* source, float* dest, size_t srcStride, size_t dstStride, size_t count); void MNNAddC4WithStride(const float* source, float* dest, size_t srcStride, size_t dstStride, size_t count); void MNNUInt8ToInt16WithOffsetC4Common(int16_t* dst, const uint8_t* src, size_t zeroPoint, size_t sizeQuad, size_t dstStride, size_t srcStride); void MNNUInt8ToInt16WithOffsetC4Fast(int16_t* dst, const uint8_t* src, size_t zeroPoint, size_t sizeQuad, size_t depthQuad, size_t dstZStep, size_t srcZStep); void MNNMaxFloat(float* input, float* maxBuffer, int32_t inputCountUnit); void MNNMinFloat(float* input, float* maxBuffer, int32_t inputCountUnit); void MNNPowC8(float* dest, const float* source, const float* powfParam, size_t betaInt, size_t countC8); void MNNExpC8(float* dest, const float* source, float* offset, const float* parameters, size_t countC8); // Offset: o0, o1, o2, o3: dst = exp(src*o0+o2)+o1, o3 = o3+sum(dst) void MNNExp(float* dst, const float* src, float* offset, size_t dataSize); void MNNSin(float* dst, const float* src, size_t dataSize); void MNNTanh(float* dst, const float* src, size_t dataSize); void MNNSigmoid(float* dst, const float* src, size_t dataSize); void MNNSigmoidLowp(float* dst, const float* src, size_t dataSize); void MNNSiLu(float* dst, const float* src, size_t dataSize); void MNNSiLuLowp(float* dst, const float* src, size_t dataSize); void MNNReluWithSlopeCommon(float* dst, const float* src, size_t size, float slope); void MNNHardSwishCommon(float* dst, const float* src, size_t size); void MNNGeluCommon(float* dst, const float* src, size_t size); void MNNGeluStandardCommon(float* dst, const float* src, size_t size); void MNNNorm(float* dest, const float* source, const float* gamma, const float* beta, float epsilon, size_t size, bool RMSNorm = false); void MNNSoftmax(float* softmaxDst, const float* input, float* runningMax, float* runningSum, float* updateScale, int outside, int reduceSize, int kvSeqOffset, int validOffset, int pack = 1, bool mask = false); // Get Pack for MatMul's e , l , h , the pack number must be 1 or 4 * n void MNNGetMatMulPackMode(int* eP, int* lP, int* hP); void MNNGetSparseMatMulPackMode(int* eP, int* lP, int* hP); /** int number = info[0]; int eSrcStride = info[1]; int eDstStride = info[2]; int xStride = info[3]; el: number * 4 0: e 1: l 2: e-offset 3: l-offset */ void MNNPackC4ForMatMul_A(float* destOrigin, float const** sourceGroup, const int32_t* info, const int32_t* el); void MNNPackForMatMul_B(float* dest, const float* source, size_t h, size_t kernelsize, size_t ic, bool transpose); // parameters: e, l, h, CStride, AStride, BStride void MNNPackedMatMul(float* C, const float* A, const float* B, const size_t* parameter, const float* postParameters, const float* bias, const float* k, const float* b); void MNNFunctionInit(); void MNNPackedMatMulRemain(float* C, const float* A, const float* B, size_t eSize, const size_t* parameter, const float* postParameters, const float* bias, const float* k, const float* b); void MNNPackForSparseMatMul_B(float* dest, unsigned int* NNZMap, int* dataOffsetMap, int sparseBlockOC, const float* source, size_t h, size_t l, const int eP, bool transpose); struct SparseMatMulParas { float* C; const float* A; const float* B; unsigned int* NNZMap; int* dataOffsetMap; }; void MNNPackedSparseMatMulEpx1(float* C, const float* A, const float* B, size_t eSize, const size_t* parameter, const float* postParameters, const float* bias, unsigned int* NNZMap, int* dataOffsetMap); void MNNPackedSparseMatMulEpx4(float* C, const float* A, const float* B, size_t eSize, const size_t* parameter, const float* postParameters, const float* bias, unsigned int* NNZMap, int* dataOffsetMap); int MNNGetC4DivNumber(int hP); void MNNAxByClampBroadcastUnit(float* C, const float* A, const float* B, size_t width, size_t cStride, size_t aStride, size_t height, const float* parameters); // dim: 4-element, sizeDW, sizeDH, strideSW, strideDH void MNNTranspose32Bit(int32_t* dstO, const int32_t* srcO, int32_t* dim); // not C4 void MNNTranspose16Bit(int16_t* dstO, const int16_t* srcO, int32_t* dim); // not C4 void MNNVectorTop1Float(float* input, float* maxValue, int32_t* maxIndex, size_t inputCountUnit); void MNNVectorTop1Int32(int32_t* input, int32_t* maxValue, int32_t* maxIndex, size_t inputCountUnit); struct MatMulParam { int32_t e; int32_t l; int32_t h; int32_t numberThread; bool ATranspose; bool BTranspose; }; void MNNComputeMatMulForE_1(const float* A, const float* B, float* C, const float* biasPtr, const MatMulParam* param, size_t tId); void MNNCopyC4Int16WithStride(const float* sourceF, float* destF, size_t srcStride, size_t dstStride, size_t count); void MNNInt8ToInt16(int16_t* dest, const int8_t* source, size_t count); void MNNPackForMatMul_A(float* dst, const float* src, size_t E, size_t L, size_t eP, size_t lP, size_t bytes); struct SumByAxisParams { ssize_t kernelCountUnitDouble; ssize_t unitColBufferSize; ssize_t DST_XUNIT; ssize_t SRC_UNIT; ssize_t blockNum; ssize_t oneScale; ssize_t valid; ssize_t kernelxy; ssize_t LU; ssize_t inputBlock; }; #ifdef __aarch64__ void MNNPermuteSumWeightInt4Arm86(uint8_t* dest, uint8_t* source, size_t outside, size_t inside, float* kernelsum); void MNNPermuteSumWeightInt4Arm82(uint8_t* dest, uint8_t* source, size_t outside, size_t inside, float* kernelsum); void MNNSumWeightInt8Arm86(float* kernelsum, int8_t* source, size_t outside, size_t reduceAxis, size_t hP, size_t lP); void MNNSumWeightInt8Arm82(float* kernelsum, int8_t* source, size_t outside, size_t reduceAxis, size_t hP, size_t lP); #ifdef MNN_SME2 void MNNSumWeightInt8Sme2_Hp32(float* kernelsum, int8_t* source, size_t outside, size_t reduceAxis, size_t hP, size_t lP); void MNNSumWeightInt8Sme2_Hp128(float* kernelsum, int8_t* source, size_t outside, size_t reduceAxis, size_t hP, size_t lP); void MNNPermuteSumWeightInt4Sme2_Hp32(uint8_t* dest, uint8_t* source, size_t outside, size_t inside, float* kernelsum, int32_t* table); void MNNPermuteSumWeightInt4Sme2_Hp128(uint8_t* dest, uint8_t* source, size_t outside, size_t inside, float* kernelsum, int32_t* table); #endif #endif } typedef void (*MNNBinaryExecute)(void* outputRaw, const void* inputRaw0, const void* inputRaw1, int elementSize, int broadcastIndex); typedef void (*MNNUnaryExecute)(void* outputRaw, const void* inputRaw, int elementSize); typedef void (*MNNUnaryExecuteInt8)(void* outputRaw, const void* inputRaw, int elementSize, QuanPrePostParameters* params); typedef void (*MNNCopyWithStride)(uint8_t* dstO, const uint8_t* srcO, int size, int stride, int ds); typedef void (*MNNBinaryExecInt8)(int8_t* outputRaw, const int8_t* inputRaw0, const int8_t* inputRaw1, ssize_t* inputScalesInt32, float* inputScalesFp32, const QuanPrePostParameters* params, size_t elementSize, size_t needBroadcast); constexpr int InputTileMax = 14; // same value from DynamicGemm.h, cannot include from different backend code. namespace MNN { struct MatmulRelatedFunctions { // from coreFunctions void (*MNNSumWeightInt8)(float* kernelsum, int8_t* source, size_t outside, size_t reduceAxis, size_t hP, size_t lP) = nullptr; void (*MNNSumWeightInt8SmeHp128)(float* kernelsum, int8_t* source, size_t outside, size_t reduceAxis, size_t hP, size_t lP) = nullptr; void (*MNNReorderWeightInt4)(uint8_t* dest, const uint8_t* source, int32_t* shape, size_t size, float* kernelsum) = nullptr; void (*MNNGeneralIm2Col)(float* destOrigin, float const** sourceGroup, const int32_t* info, const int32_t* el, int32_t LP, int32_t pack) = nullptr; // from int8CoreFunctions void (*Int8GemmKernel)(int8_t* dst, const int8_t* src, const int8_t* weight, size_t src_depth_quad, size_t dst_step, size_t dst_depth_quad, const QuanPostTreatParameters* post, size_t realCount) = nullptr; void (*Int8GemmKernelFast)(int8_t* dst, const int8_t* src, const int8_t* weight, size_t src_depth_quad, size_t dst_step, size_t dst_depth_quad, const QuanPostTreatParameters* post, size_t realCount) = nullptr; void (*MNNGetGemmUnit)(int* UNIT, int* SRC_UNIT, int* DST_XUNIT) = nullptr; void (*MNNPackC4Int8ForMatMul_A)(int8_t* destOrigin, int8_t const** sourceGroup, const int32_t* info, const int32_t* el) = nullptr; void (*MNNGemmInt8AddBiasScale_Unit_FP16)(int8_t* dst, const int8_t* src, const int8_t* weight, size_t src_depth_quad, size_t dst_step, size_t dst_depth_quad, const QuanPostTreatParameters* post, size_t realDstCount) = nullptr; void (*MNNGemmInt8AddBiasScale_w4_Unit_FP16)(int8_t* dst, const int8_t* src, const int8_t* weight, size_t src_depth_quad, size_t dst_step, size_t dst_depth_quad, const QuanPostTreatParameters* post, size_t realDstCount) = nullptr; void (*MNNGemmInt8AddBiasScale_w2_Unit_FP16)(int8_t* dst, const int8_t* src, const int8_t* weight, size_t src_depth_quad, size_t dst_step, size_t dst_depth_quad, const QuanPostTreatParameters* post, size_t realDstCount) = nullptr; void (*MNNGemmInt8AddBiasScale_w3_Unit_FP16)(int8_t* dst, const int8_t* src, const int8_t* weight, size_t src_depth_quad, size_t dst_step, size_t dst_depth_quad, const QuanPostTreatParameters* post, size_t realDstCount) = nullptr; void (*MNNGemmInt8AddBiasScale_Unit_FP16_DecodeMax)(int8_t* dst, const int8_t* src, const int8_t* weight, size_t src_depth_quad, size_t dst_step, size_t dst_depth_quad, const QuanPostTreatParameters* post, size_t realDstCount) = nullptr; void (*MNNGemmInt8AddBiasScale_Unit_FP32_DecodeMax)(int8_t* dst, const int8_t* src, const int8_t* weight, size_t src_depth_quad, size_t dst_step, size_t dst_depth_quad, const QuanPostTreatParameters* post, size_t realDstCount) = nullptr; void (*MNNGemmInt8AddBiasScale_w4_Unit_FP16_DecodeMax)(int8_t* dst, const int8_t* src, const int8_t* weight, size_t src_depth_quad, size_t dst_step, size_t dst_depth_quad, const QuanPostTreatParameters* post, size_t realDstCount) = nullptr; void (*MNNGemmInt8AddBiasScale_w4_Unit_FP32_DecodeMax)(int8_t* dst, const int8_t* src, const int8_t* weight, size_t src_depth_quad, size_t dst_step, size_t dst_depth_quad, const QuanPostTreatParameters* post, size_t realDstCount) = nullptr; void (*Int8GemmKernel_W4)(int8_t* dst, const int8_t* src, const int8_t* weight, size_t src_depth_quad, size_t dst_step, size_t dst_depth_quad, const QuanPostTreatParameters* post, size_t realDstCount) = nullptr; void (*Int8GemmKernel_W2)(int8_t* dst, const int8_t* src, const int8_t* weight, size_t src_depth_quad, size_t dst_step, size_t dst_depth_quad, const QuanPostTreatParameters* post, size_t realDstCount) = nullptr; void (*Int8GemmKernel_W3)(int8_t* dst, const int8_t* src, const int8_t* weight, size_t src_depth_quad, size_t dst_step, size_t dst_depth_quad, const QuanPostTreatParameters* post, size_t realDstCount) = nullptr; void (*MNNSumByAxisLForMatmul_A)(float* dest, int8_t* source, const float* dequantScale, ssize_t realDstCount, SumByAxisParams sumParams) = nullptr; int eP; }; struct CoreFunctions { // cpu feature bool supportFp16arith = false; bool supportSDot = false; bool supportI8mm = false; bool supportSME2 = false; bool supportRVV = false; int smeCoreNumber = 0; /**MatMul Pack and Functions*/ void (*MNNGetMatMulPackMode)(int* eP, int* lP, int* hP); void (*MNNGetSparseMatMulPackMode)(int* eP, int* lP, int* hP); void (*MNNPackC4ForMatMul_A)(float* destOrigin, float const** sourceGroup, const int32_t* info, const int32_t* el); void (*MNNPackForMatMul_B)(float* dest, const float* source, size_t h, size_t kernelsize, size_t ic, bool transpose); void (*MNNGeneralIm2Col)(float* destOrigin, float const** sourceGroup, const int32_t* info, const int32_t* el, int32_t LP, int32_t pack); // parameters: e, l, h, CStride, AStride, BStride void (*MNNPackedMatMul)(float* C, const float* A, const float* B, const size_t* parameter, const float* postParameters, const float* bias, const float* k, const float* b); void (*MNNPackedMatMulRemain)(float* C, const float* A, const float* B, size_t eSize, const size_t* parameter, const float* postParameters, const float* bias, const float* k, const float* b); // int8 matmul related void (*MNNSumByAxisLForMatmul_A)(float* dest, int8_t* source, const float* dequantScale, ssize_t realDstCount, SumByAxisParams sumParams); void (*MNNReorderWeightInt4)(uint8_t* dest, const uint8_t* source, int32_t* shape, size_t size, float* kernelsum); void (*MNNSumWeightInt8)(float* kernlesum, int8_t* source, size_t outside, size_t reduceAxis, size_t hP, size_t lP); void (*MNNSumWeightInt8SmeHp128)(float* kernelsum, int8_t* source, size_t outside, size_t reduceAxis, size_t hP, size_t lP); // cpu dynamic quant void (*MNNAbsMax)(const float* source, float* absmax, size_t src_depth_quad, size_t realSize, int pack) = nullptr; void (*MNNQuantScale)(float* absmax, float* quant_scale, float* dequant_scale, size_t thread, size_t batch) = nullptr; void (*MNNDynamicQuant)(const float* src, int8_t* dst, const float* scale, size_t src_depth_quad, size_t realSize, int pack, const float* bias) = nullptr; void (*MNNComputeMatMulForH_1)(const float* A, const float* B, float* C, const float* biasPtr, const MatMulParam* param, size_t tId); void (*MNNComputeMatMulForE_1)(const float* A, const float* B, float* C, const float* biasPtr, const MatMulParam* param, size_t tId); // Rank-1 update: S[dk, dv] += k[dk] * delta[dv] (outer product add) void (*MNNRankOneUpdate)(float* S, const float* k, const float* delta, size_t dk, size_t dv); // Read-only dual MatVec: out_k = S^T @ k, out_q = S^T @ q (does NOT modify S) void (*MNNDualMatVec)(const float* S, const float* k, const float* q, float* out_k, float* out_q, size_t dk, size_t dv); // Fused decay + rank-1 update: S[i,j] = decay * S[i,j] + k[i] * delta[j] void (*MNNDecayRankOneUpdate)(float* S, const float* k, const float* delta, float decay, size_t dk, size_t dv); // Fused gated-delta-rule kernel. Computes (all in the backend's native // precision — fp32 in default backend, fp16 in arm82; pointer type is // float* by convention): // out_k = S^T @ k [d_v] // out_q = S^T @ q [d_v] // delta = beta * (v - decay * out_k) [d_v] // out = decay * out_q + kq * delta [d_v] (written) // S = decay * S + k ⊗ delta [d_k, d_v] (in-place) // 'kq' must be precomputed as dot(k,q) by the caller. void (*MNNFusedGatedDelta)(float* S, const float* k, const float* q, const float* v, float* out, float decay, float beta, float kq, size_t dk, size_t dv); void (*MNNCountMaxMinValue)(const float* source, float* minVal, float* maxVal, size_t size); void (*MNNNormPacked)(float* dest, const float* source, const float* gamma, const float* beta, float epsilon, size_t batch, size_t channels, bool RMSNorm); void (*MNNDynamicUpdateConvBiasScale)(float* newbias, float* oldbias, float* weightKernelSum, float* inputZero, size_t ocQuad); void (*MNNAsyQuantInfo)(float* scale, float* bias, float* qscale, float* qbias, float* dstMin, float* dstMax, const float* src, const size_t* info); void (*MNNAsyQuantFunc)(int8_t* dst, const float* src, float* qscale, float* qbias, const size_t* info); typedef void (*MNNPackedMatMulKernel)(float* C, const float* A, const float* B, const size_t* parameter, const float* postParameters, const float* bias); MNNPackedMatMulKernel MNNPackedMatMulOC16Functions[InputTileMax] = {0}; MNNPackedMatMulKernel MNNPackedMatMulOC32Functions[InputTileMax] = {0}; MNNPackedMatMulKernel MNNPackedMatMulOC48Functions[InputTileMax] = {0}; // For Atomic Op MNNBinaryExecute (*MNNSelectBinaryFunctionForFloat)(int opType); MNNUnaryExecute (*MNNSelectUnaryFunctionForFloat)(int opType, int precisionMode); #ifdef MNN_SUPPORT_QUANT_EXTEND MNNUnaryExecuteInt8 (*MNNSelectUnaryFunctionForInt8)(int opType) = nullptr; #endif // B matrix is sparsed typedef void (*MNNPackedSparseMatMul)(float* C, const float* A, const float* B, size_t eSize, const size_t* parameter, const float* postParameters, const float* bias, unsigned int* NNZMap, int* dataOffsetMap); void (*MNNAdjustOptimalSparseKernel)(int& sparseBlockOC, MNNPackedSparseMatMul& packedSparseMatMul); /**Lowp Backend Setting*/ void (*MNNFp32ToLowp)(const float* src, int16_t* dst, size_t size); void (*MNNLowpToFp32)(const int16_t* src, float* dst, size_t size); int bytes; // Byte for float int matmulBytes = 0; // Special bytes for dense matmul, C = A*B, A, B is matmulBytes, C is bytes. If 0, means the same as bytes /**NC4HW4's Functions*/ int pack; // For pack * bytes > 16 MNNCopyWithStride (*MNNSelectBlitFunction)(int blitBytes) = nullptr; void (*MNNPackCUnitInt16)(int16_t* dst, const int16_t* src, size_t area, size_t depth, int* areaOffset); void (*MNNUnpackCUnitInt16)(int16_t* dst, const int16_t* src, size_t area, size_t depth, int* areaOffset); void (*MNNPackCUnitTransposeInt16)(int16_t* dst, const int16_t* src, size_t area, size_t depth, int* areaOffset); void (*MNNUnpackCUnitTransposeInt16)(int16_t* dst, const int16_t* src, size_t area, size_t depth, int* areaOffset); void (*MNNPackCUnitInt8)(int8_t* dst, const int8_t* src, size_t area, size_t depth, int* areaOffset); void (*MNNUnpackCUnitInt8)(int8_t* dst, const int8_t* src, size_t area, size_t depth, int* areaOffset); void (*MNNPackCUnitTransposeInt8)(int8_t* dst, const int8_t* src, size_t area, size_t depth, int* areaOffset); void (*MNNUnpackCUnitTransposeInt8)(int8_t* dst, const int8_t* src, size_t area, size_t depth, int* areaOffset); void (*MNNPackCUnit)(float* dst, const float* src, size_t area, size_t depth, int* areaOffset); void (*MNNUnpackCUnit)(float* dst, const float* src, size_t area, size_t depth, int* areaOffset); void (*MNNPackCUnitTranspose)(float* dst, const float* src, size_t area, size_t depth, int* areaOffset); void (*MNNUnpackCUnitTranspose)(float* dst, const float* src, size_t area, size_t depth, int* areaOffset); // NC4HW4's compute function void (*MNNConvRunForLineDepthwise)(float* dst, const float* src, const float* weight, size_t width, size_t src_w_setup, size_t fw, size_t fh, size_t dilateX_step, size_t dilateY_step, size_t height, size_t srcHStep, size_t dstHStep, const float* bias, const float* parameters); void (*MNNAxByClampBroadcastUnit)(float* C, const float* A, const float* B, size_t width, size_t cStride, size_t aStride, size_t height, const float* parameters); void (*MNNMatrixAdd)(float* C, const float* A, const float* B, size_t widthC4, size_t cStride, size_t aStride, size_t bStride, size_t height); void (*MNNMatrixSub)(float* C, const float* A, const float* B, size_t widthC4, size_t cStride, size_t aStride, size_t bStride, size_t height); void (*MNNStrassenMergeCFunction)(float* c11, float* c12, float* c21, float* c22, float* xAddr, size_t cStride, size_t eSub, size_t hSub); void (*MNNScaleAndAddBias)(float* dst, const float* src, const float* bias, const float* alpha, size_t planeNumber, size_t biasNumber); void (*MNNGridSampleComputeCord)(float* dst, const float* src, size_t inH, size_t inW, size_t outH, size_t outW, bool alignCorners); void (*MNNGridSampleInterp)(float* outputPtr, const float* inputPtr, const float* cordPtr, size_t inH, size_t inW, size_t outW, size_t channelCUnit, size_t inOffset, size_t outOffset, bool sampleMode, bool padMode); void (*MNNGridSampleInterpGrad)(float* outputPtr, float* inputPtr, const float* cordPtr, size_t inH, size_t inW, size_t outW, size_t channelCUnit, size_t inOffset, size_t outOffset, bool sampleMode, bool padMode); void (*MNNGridSampleComputeCord3D)(float* dst, const float* src, size_t inD, size_t inH, size_t inW, size_t outD, size_t outH, size_t outW, bool alignCorners); void (*MNNGridSampleInterp3D)(float* outputPtr, const float* inputPtr, const float* cordPtr, size_t inD, size_t inH, size_t inW, size_t outW, size_t channelCUnit, size_t inOffset, size_t outOffset, bool sampleMode, bool padMode) = nullptr; void (*MNNRoiPoolingMax)(float* dst, const float* src, int hLen, int wLen, int iw); void (*MNNRoiAlignMax)(float* dst, const float* src, const std::vector>& vecPos, const std::vector>& vecArea, int samplingRatioArea, int pooledHeight, int pooledWidth); void (*MNNRoiAlignAvg)(float* dst, const float* src, const std::vector>& vecPos, const std::vector>& vecArea, int samplingRatioArea, int pooledHeight, int pooledWidth); float penalty; void (*MNNCopyC4WithStride)(const float* source, float* dest, size_t srcStride, size_t dstStride, size_t count); void (*MNNAddC4WithStride)(const float* source, float* dest, size_t srcStride, size_t dstStride, size_t count); typedef void (*WinoTransPackFunc)(float* srcBlock, float* dstStart, size_t dstStep); WinoTransPackFunc (*chooseWinoSourceTransformPack)(int k, int w, int ePack, int lPack, int packCUnit); typedef void (*WinoUnrollTransFunc)(const float* srcBlock, float* dstStart, size_t srcRowStep, size_t dstRowStep, size_t srcStep, size_t dstStep); typedef void (*WinoUnrollDestTransFunc)(const float* srcBlock, float* dstStart, const float* bias, const float* postParameters, size_t srcRowStep, size_t dstRowStep, size_t srcStep, size_t dstStep); WinoUnrollTransFunc (*chooseWinoSourceUnrollTransform)(int k, int w); void (*chooseWinoDestUnrollTransform)(WinoUnrollDestTransFunc* destFunctions, size_t maxUnit, int k, int h); void (*MNNDeconvRunForUnitDepthWise)(const float* dst, float* src, const float* weight, size_t fw, size_t fh, size_t weight_y_step, size_t dilateX_step, size_t dilateY_step); void (*MNNDeconvRunForLineDepthwise)(const float* dst, float* src, const float* weight, size_t width, size_t src_w_setup, size_t fw, size_t fh, size_t dilateX_step, size_t dilateY_step); void (*MNNDepthwiseConvFastKernel)(float* dst, const float* src, const float* weight, size_t width, size_t src_w_setup, size_t fw, size_t fh, size_t dilateX_step, size_t dilateY_step, size_t height, size_t srcHStep, size_t dstHStep, const float* bias, const float* parameters) = nullptr; void (*MNNReluWithSlopeChannel)(float* dst, const float* src, const float* slope, size_t sizeQuad, size_t depthQuad); void (*MNNPoolingAvg)(const void* channelInput, int inputWidth, int inputHeight, void* channelOutput, int outputWidth, int outputHeight, int kernelWidth, int kernelHeight, int strideWidth, int strideHeight, int padWidth, int padHeight, int padType, int countType); void (*MNNPoolingMax)(const void* channelInput, int inputWidth, int inputHeight, void* channelOutput, int outputWidth, int outputHeight, int kernelWidth, int kernelHeight, int strideWidth, int strideHeight, int padWidth, int padHeight, int padType, int countType); void (*MNNPoolingMaxWithRedice)(const void* channelInput, int inputWidth, int inputHeight, void* channelOutput, int outputWidth, int outputHeight, int kernelWidth, int kernelHeight, int strideWidth, int strideHeight, int padWidth, int padHeight, int padType, int countType, int* RediceOutput); // ImageProcess Funtions void (*MNNRGBAToBGRA)(const unsigned char* source, unsigned char* dest, size_t count); void (*MNNNV21ToRGBA)(const unsigned char* source, unsigned char* dest, size_t count); void (*MNNNV21ToRGB)(const unsigned char* source, unsigned char* dest, size_t count); void (*MNNNV21ToBGRA)(const unsigned char* source, unsigned char* dest, size_t count); void (*MNNNV21ToBGR)(const unsigned char* source, unsigned char* dest, size_t count); void (*MNNC1ToFloatC1)(const unsigned char* source, float* dest, const float* mean, const float* normal, size_t count); void (*MNNC3ToFloatC3)(const unsigned char* source, float* dest, const float* mean, const float* normal, size_t count); void (*MNNC3ToFloatRGBA)(const unsigned char* source, float* dest, const float* mean, const float* normal, size_t count); void (*MNNsampleBilinearCommon)(const unsigned char* source, unsigned char* dest, MNN::CV::Point* points, size_t count, size_t iw, size_t ih, size_t yStride, size_t bpp); void (*MNNSamplerC4Nearest)(const unsigned char* source, unsigned char* dest, MNN::CV::Point* points, size_t sta, size_t count, size_t capacity, size_t iw, size_t ih, size_t yStride); void (*MNNSamplerC4Bilinear)(const unsigned char* source, unsigned char* dest, MNN::CV::Point* points, size_t sta, size_t count, size_t capacity, size_t iw, size_t ih, size_t yStride); void (*MNNSampleC4Bilinear)(const unsigned char* source, unsigned char* dest, MNN::CV::Point* points, size_t sta, size_t count, size_t capacity, size_t iw, size_t ih, size_t yStride); void (*MNNSampleBilinear)(const unsigned char* source, unsigned char* dest, MNN::CV::Point* points, size_t count, size_t iw, size_t ih, size_t yStride, size_t bpp); void (*MNN4BitcopyWithStride)(uint8_t* dstO, const uint8_t* srcO, int size, int stride, int ds); void (*MNN2BitcopyWithStride)(uint8_t* dstO, const uint8_t* srcO, int size, int stride, int ds); void (*MNN1BitcopyWithStride)(uint8_t* dstO, const uint8_t* srcO, int size, int stride, int ds); void (*MNN4BitcopyFast)(uint8_t* dstO, const uint8_t* srcO, int size, int stride, int ds); void (*MNN2BitcopyFast)(uint8_t* dstO, const uint8_t* srcO, int size, int stride, int ds); void (*MNN1BitcopyFast)(uint8_t* dstO, const uint8_t* srcO, int size, int stride, int ds); void (*MNNAccumulateSequenceNumber)(float* dst, const float* src, int size); // Attention void (*MNNAttenPackAndScaleSingleHead)(float* dst, const float* srcHeadBase, size_t srcRowStride, const float* scale, const int32_t* units, size_t seqLen, size_t headDim); void (*MNNFlashAttentionUpdateBlockOutput)(float* dst, float* src, float* scale, float* normalizeScale, int depthQuad, int plane, int pack, int idx, int kvBlocks, int size, int bytes, int seqStart); void (*MNNSoftmax)(float* softmaxDst, const float* input, float* runningMax, float* runningSum, float* updateScale, int outside, int reduceSize, int kvSeqOffset, int validOffset, int pack, bool mask); void (*MNNQuantAttentionKey)(int8_t* dst, const float* source, float* sumKey, float* maxKey, int32_t* params); void (*MNNQuantAttentionValue)(int8_t* dst, const float* source, float* valueQuantInfo, int32_t* params); void (*MNNRoPECompute)(void* dst, const void* src, const void* cosEven, const void* cosOdd, const void* sinEven, const void* sinOdd, int numHead, int headDim, int ropeCutHeadDim); MatmulRelatedFunctions int8MatmulRelatedFunctions; MatmulRelatedFunctions arm82MatmulRelatedFunctions; }; void MNNCoreFunctionInit(); CoreFunctions* MNNGetCoreFunctions(); }; // namespace MNN #endif /* CommonOptFunction_h */