#include #include #include "BF16Unary.hpp" #include "VecHalf.hpp" #include "math/Vec.hpp" #include "backend/cpu/UnaryUtils.hpp" #include "BF16Backend.hpp" extern "C" { void NEON_MNNGelu_BF16(int16_t* dst, const int16_t* src, size_t size, float* parameters); } namespace MNN { using Vec4Half = MNN::Math::VecHalf<4>; using Vec4 = MNN::Math::Vec; struct Vec4Square { Vec4Half operator()(Vec4Half &x) const { return x * x; } }; struct Vec4Neg { Vec4Half operator()(Vec4Half &x) const { return -x; } }; struct Vec4Abs { Vec4Half operator()(Vec4Half &x) const { float v[4]; v[0] = fabs(x[0]); v[1] = fabs(x[1]); v[2] = fabs(x[2]); v[3] = fabs(x[3]); auto c = Vec4::load(v); Vec4Half value; value.value = std::move(c.value); return value; } }; template void BF16VecUnary(void *dstRaw, const void *src0Raw, int elementSize) { Compute Func; auto dst = (int16_t*)dstRaw; auto src0 = (int16_t*)src0Raw; const int sizeDivUnit = elementSize / 4; const int remainCount = elementSize - sizeDivUnit * 4; if (sizeDivUnit > 0) { for (int i = 0; i < sizeDivUnit; ++i) { Vec4Half a = Vec4Half::load(src0); Vec4Half::save(dst, Func(a)); src0 += 4; dst += 4; } } if (remainCount > 0) { int16_t tempSrc0[4]; int16_t tempDst[4]; ::memcpy(tempSrc0, src0, remainCount * sizeof(int16_t)); Vec4Half a = Vec4Half::load(tempSrc0); Vec4Half::save(tempDst, Func(a)); ::memcpy(dst, tempDst, remainCount * sizeof(int16_t)); } } #define BLOCK_SIZE 16 template static void _Wrap(void* outRaw, const void* inpRaw, int realSize) { Compute execute; float out[BLOCK_SIZE]; float inp[BLOCK_SIZE]; int b = realSize / BLOCK_SIZE; int remain = realSize % BLOCK_SIZE; auto bf16F = BF16Functions::get(); auto outR = (int16_t*)outRaw; auto inpR = (const int16_t*)inpRaw; for (int i=0; iMNNLowpToFp32(inpR, inp, BLOCK_SIZE); execute(out, inp, BLOCK_SIZE); bf16F->MNNFp32ToLowp(out, outR, BLOCK_SIZE); outR += BLOCK_SIZE; inpR += BLOCK_SIZE; } if (remain > 0) { bf16F->MNNLowpToFp32(inpR, inp, remain); execute(out, inp, remain); bf16F->MNNFp32ToLowp(out, outR, remain); } } struct _Exp { void operator()(void* outRaw, const void* inpRaw, int realSize) const { auto out = (float*)outRaw; auto inp = (const float*)inpRaw; float offset[4] = { 1.0f, 0.0f, 0.0f, 0.0f }; MNNExp(out, inp, offset, realSize); } }; struct _ExpM1 { void operator()(void* outRaw, const void* inpRaw, int realSize) const { auto out = (float*)outRaw; auto inp = (const float*)inpRaw; float offset[4] = { 1.0f, -1.0f, 0.0f, 0.0f }; MNNExp(out, inp, offset, realSize); } }; struct _Tanh { void operator()(void* outRaw, const void* inpRaw, int realSize) const { auto out = (float*)outRaw; auto inp = (const float*)inpRaw; MNNTanh(out, inp, realSize); } }; struct _Sigmoid { void operator()(void* outRaw, const void* inpRaw, int realSize) const { auto out = (float*)outRaw; auto inp = (const float*)inpRaw; MNNSigmoidLowp(out, inp, realSize); } }; struct _HardSwish { void operator()(void* outRaw, const void* inpRaw, int realSize) const { auto out = (float*)outRaw; auto inp = (const float*)inpRaw; MNNHardSwishCommon(out, inp, realSize); } }; struct _Gelu { void operator()(void* outRaw, const void* inpRaw, int realSize) const { auto out = (float*)outRaw; auto inp = (const float*)inpRaw; MNNGeluCommon(out, inp, realSize); } }; void BF16GELU (void* OutRaw, const void* inpRaw, int realSize) { int16_t* out = (int16_t*)OutRaw; const int16_t* inp = (const int16_t*)inpRaw; int sizeQuad = realSize / 8; int start = 0; float parameters[8] = {0.044715f, 0.79788458f, 378.f, 17325.f, 135135.f, 28.f, 3150.f, 62370.f}; if (sizeQuad > 0) { #ifdef MNN_USE_NEON NEON_MNNGelu_BF16(out, inp, sizeQuad, parameters); #endif start = sizeQuad * 8; } int16_t tempInp[8]; for (int i = start; i < realSize; i++) { tempInp[i-start] = inp[i]; } #ifdef MNN_USE_NEON NEON_MNNGelu_BF16(tempInp, tempInp, 1, parameters); #endif for (int i = start; i < realSize; i++) { out[i] = tempInp[i-start]; } } template struct _Unary { void operator()(void* outputPtr, const void* inputPtr, int elementSize) const { Func f; const T *inputData = (T*)inputPtr; T *outputData = (T *)outputPtr; for (int i=0; i; case UnaryOpOperation_SQUARE: return BF16VecUnary; case UnaryOpOperation_NEG: return BF16VecUnary; case UnaryOpOperation_RSQRT: return _Wrap<_Unary, float>>; case UnaryOpOperation_EXP: return _Wrap<_Exp>; case UnaryOpOperation_COS: return _Wrap<_Unary, float>>; case UnaryOpOperation_SIN: return _Wrap<_Unary, float>>; case UnaryOpOperation_SIGMOID: return _Wrap<_Sigmoid>; case UnaryOpOperation_TANH: return _Wrap<_Tanh>; case UnaryOpOperation_TAN: return _Wrap<_Unary, float>>; case UnaryOpOperation_ATAN: return _Wrap<_Unary, float>>; case UnaryOpOperation_SQRT: return _Wrap<_Unary, float>>; case UnaryOpOperation_CEIL: return _Wrap<_Unary, float>>; case UnaryOpOperation_RECIPROCAL: return _Wrap<_Unary, float>>; case UnaryOpOperation_LOG1P: return _Wrap<_Unary, float>>; case UnaryOpOperation_LOG: return _Wrap<_Unary, float>>; case UnaryOpOperation_FLOOR: return _Wrap<_Unary, float>>; case UnaryOpOperation_BNLL: return _Wrap<_Unary, float>>; case UnaryOpOperation_ACOSH: return _Wrap<_Unary, float>>; case UnaryOpOperation_SINH: return _Wrap<_Unary, float>>; case UnaryOpOperation_ASINH: return _Wrap<_Unary, float>>; case UnaryOpOperation_ATANH: return _Wrap<_Unary, float>>; case UnaryOpOperation_SIGN: return _Wrap<_Unary, float>>; case UnaryOpOperation_ROUND: return _Wrap<_Unary, float>>; case UnaryOpOperation_COSH: return _Wrap<_Unary, float>>; case UnaryOpOperation_ERF: return _Wrap<_Unary, float>>; case UnaryOpOperation_ERFC: return _Wrap<_Unary, float>>; case UnaryOpOperation_ERFINV: return _Wrap<_Unary, float>>; case UnaryOpOperation_EXPM1: return _Wrap<_ExpM1>; case UnaryOpOperation_ASIN: return _Wrap<_Unary, float>>; case UnaryOpOperation_ACOS: return _Wrap<_Unary, float>>; case UnaryOpOperation_HARDSWISH: return _Wrap<_HardSwish>; case UnaryOpOperation_GELU: #ifdef MNN_USE_NEON return BF16GELU; #else return _Wrap<_Gelu>; #endif default: MNN_ASSERT(false); break; } return nullptr; } };