wehub-resource-sync
1417 lines
44 KiB
ArmAsm
1417 lines
44 KiB
ArmAsm
//
|
|
// MNNGemmInt8AddBiasScale_ARMV82_Unit.S
|
|
// MNN
|
|
//
|
|
// Created by MNN on 2019/12/17.
|
|
// Copyright © 2018, Alibaba Group Holding Limited
|
|
//
|
|
|
|
#if defined(__aarch64__)
|
|
#include "MNNAsmGlobal.h"
|
|
|
|
.text
|
|
.align 5
|
|
|
|
.macro ADD_BIAS_FLOAT d0, d1, d2, d3, z0
|
|
fadd \d0\().4s, \d0\().4s, \z0\().4s
|
|
fadd \d1\().4s, \d1\().4s, \z0\().4s
|
|
fadd \d2\().4s, \d2\().4s, \z0\().4s
|
|
fadd \d3\().4s, \d3\().4s, \z0\().4s
|
|
.endm
|
|
|
|
.macro ADD_FLOAT d0, d1, d2, d3, s0, s1, s2, s3
|
|
fadd \d0\().4s, \d0\().4s, \s0\().4s
|
|
fadd \d1\().4s, \d1\().4s, \s1\().4s
|
|
fadd \d2\().4s, \d2\().4s, \s2\().4s
|
|
fadd \d3\().4s, \d3\().4s, \s3\().4s
|
|
.endm
|
|
|
|
.macro SET_BIAS d0, d1, d2, d3
|
|
movi \d0\().16b, #0
|
|
movi \d1\().16b, #0
|
|
movi \d2\().16b, #0
|
|
movi \d3\().16b, #0
|
|
.endm
|
|
.macro Int32ToFloat z0, z1, z2, z3
|
|
scvtf \z0\().4s, \z0\().4s
|
|
scvtf \z1\().4s, \z1\().4s
|
|
scvtf \z2\().4s, \z2\().4s
|
|
scvtf \z3\().4s, \z3\().4s
|
|
.endm
|
|
.macro MUL_SCALE s, d0, d1, d2, d3
|
|
fmul \d0\().4s, \d0\().4s, \s\().4s
|
|
fmul \d1\().4s, \d1\().4s, \s\().4s
|
|
fmul \d2\().4s, \d2\().4s, \s\().4s
|
|
fmul \d3\().4s, \d3\().4s, \s\().4s
|
|
.endm
|
|
.macro MUL_EXTRA_SCALE s, d0, d1, d2, d3
|
|
fmul \d0\().4s, \d0\().4s, \s\().s[0]
|
|
fmul \d1\().4s, \d1\().4s, \s\().s[1]
|
|
fmul \d2\().4s, \d2\().4s, \s\().s[2]
|
|
fmul \d3\().4s, \d3\().4s, \s\().s[3]
|
|
.endm
|
|
.macro FloatToInt32 z0, z1, z2, z3
|
|
fcvtas \z0\().4s, \z0\().4s
|
|
fcvtas \z1\().4s, \z1\().4s
|
|
fcvtas \z2\().4s, \z2\().4s
|
|
fcvtas \z3\().4s, \z3\().4s
|
|
.endm
|
|
.macro Int32ToInt16 s0, s1, s2, s3, d0, d1
|
|
sqxtn \d0\().4h, \s0\().4s
|
|
sqxtn2 \d0\().8h, \s1\().4s
|
|
sqxtn \d1\().4h, \s2\().4s
|
|
sqxtn2 \d1\().8h, \s3\().4s
|
|
.endm
|
|
.macro Int16ToInt8_ONE s0, s1, d0
|
|
sqxtn \d0\().8b, \s0\().8h
|
|
sqxtn2 \d0\().16b, \s1\().8h
|
|
.endm
|
|
.macro Int16ToInt8 s0, s1, s2, s3, d0, d1
|
|
Int16ToInt8_ONE \s0, \s1, \d0
|
|
Int16ToInt8_ONE \s2, \s3, \d1
|
|
.endm
|
|
.macro MLA_WEIGHTZERO d0, s0, s1, idx // idx for xKernelSum
|
|
fmla \d0\().4s, \s1\().4s, \s0\().s[\idx]
|
|
.endm
|
|
.macro ReLU_FP32 s0, s1, s2, s3, z0, z1 // z0:min z1:max
|
|
fmin \s0\().4s, \s0\().4s, \z1\().4s
|
|
fmin \s1\().4s, \s1\().4s, \z1\().4s
|
|
fmin \s2\().4s, \s2\().4s, \z1\().4s
|
|
fmin \s3\().4s, \s3\().4s, \z1\().4s
|
|
fmax \s0\().4s, \s0\().4s, \z0\().4s
|
|
fmax \s1\().4s, \s1\().4s, \z0\().4s
|
|
fmax \s2\().4s, \s2\().4s, \z0\().4s
|
|
fmax \s3\().4s, \s3\().4s, \z0\().4s
|
|
.endm
|
|
|
|
.macro REVERT_INPUT_DEQUANT_BIAS rg0, rg1, rg2, rg3
|
|
mul \rg1, \rg2, \rg3
|
|
sub \rg0, \rg0, \rg1
|
|
.endm
|
|
|
|
.macro REVERT_WEIGHT_KERNEL_SUM rg0, rg1, rg2, rg3
|
|
// y=UP_DIV(ocDiv4,(hp/pack))
|
|
add \rg1, \rg3, #1
|
|
lsr \rg1, \rg1, #1
|
|
// blockNum * y * (hp * sizeof(float))
|
|
mul \rg1, \rg2, \rg1
|
|
sub \rg0, \rg0, \rg1, LSL #5 // revert weight kernel sum
|
|
.endm
|
|
asm_function MNNGemmInt8AddBiasScale_ARMV82_Unit
|
|
/*
|
|
struct QuanPostTreatParameters {
|
|
const float* scale;
|
|
const float* biasFloat;
|
|
int32_t maxValue;
|
|
int32_t minValue;
|
|
int32_t useInt8 = 1; // Save result as int8_t dataType; otherwise float32.
|
|
float roundValuePos = 0.5f;
|
|
float roundValueNeg = -0.5f;
|
|
float* srcKernelSum;
|
|
float* weightKernelSum;
|
|
float* fp32minmax;
|
|
ssize_t blockNum = 1;
|
|
const int32_t* bias = nullptr;
|
|
const float* inputScale = nullptr;
|
|
const float* inputBias = nullptr;
|
|
float* accumBuffer = nullptr;
|
|
};
|
|
*/
|
|
|
|
//void MNNGemmInt8AddBiasScale_ARMV82_Unit(int8_t* dst, const int8_t* src,
|
|
// const int8_t* weight, size_t src_depth_quad, size_t dst_step, size_t dst_depth_quad,
|
|
// const QuanPostTreatParameters* parameters, size_t realDstCount);
|
|
|
|
//Auto: x0:dst, x1:src, x2:weight, x3:src_depth_quad, x4:dst_step
|
|
//x5:dst_depth_quad, x6: parameters, x7: realDstCount
|
|
|
|
//Load from x6: x9: bias, x8: xKernelSum, x23: fp32minmax
|
|
ldr x9, [x6, #8]
|
|
|
|
stp d14, d15, [sp, #(-16 * 10)]!
|
|
stp d12, d13, [sp, #(16 * 1)]
|
|
stp d10, d11, [sp, #(16 * 2)]
|
|
stp d8, d9, [sp, #(16 * 3)]
|
|
stp x21, x22, [sp, #(16 * 4)]
|
|
stp x19, x20, [sp, #(16 * 5)]
|
|
stp x23, x24, [sp, #(16 * 6)]
|
|
stp x25, x26, [sp, #(16 * 7)]
|
|
stp x27, x28, [sp, #(16 * 8)]
|
|
|
|
ldr x8, [x6, #40] // srcKernelSum
|
|
ldr x28, [x6, #48] // weightKernelSum
|
|
ldr x24, [x6, #80] // inputScale
|
|
ldr x27, [x6, #88] // inputBias
|
|
ldr x10, [x6, #96] // accumBuffer
|
|
ldr x26, [x6, #64] // blockNum
|
|
lsl x22, x7, #2 // eDest * SRC_UNIT
|
|
|
|
mov x25, #-32
|
|
add x23, x6, #16 // int8 max ptr
|
|
cbz x28, TILE_12
|
|
ldr x23, [x6, #56] // fp32minmax
|
|
cbz x27, TILE_12
|
|
sub x25, x22, #32
|
|
|
|
TILE_12:
|
|
cmp x7, #12
|
|
blt TILE_8
|
|
sub x4, x4, #128
|
|
mov x12, x2
|
|
mov x6, x0
|
|
mov x14, x5
|
|
mov x20, x9
|
|
mov x15, x8 // input kernel sum
|
|
mov x21, x24 // input dequant scale
|
|
cbnz x28, L8LoopDz_TILE_12
|
|
add x4, x4, #128 // int8 do not change
|
|
L8LoopDz_TILE_12:
|
|
cmp x14, #2
|
|
blt L4LoopDz_TILE_12
|
|
mov x11, x1
|
|
mov x19, #0
|
|
TILE12_BLOCKNUM:
|
|
mov x13, x3
|
|
|
|
SET_BIAS v8, v9, v10, v11
|
|
SET_BIAS v12, v13, v14, v15
|
|
SET_BIAS v16, v17, v18, v19
|
|
SET_BIAS v20, v21, v22, v23
|
|
SET_BIAS v24, v25, v26, v27
|
|
SET_BIAS v28, v29, v30, v31
|
|
|
|
L8LoopSz_TILE_12:
|
|
ld1 {v3.16b, v4.16b}, [x12], #32 // weight
|
|
ld1 {v0.16b, v1.16b, v2.16b}, [x11], x22 // src
|
|
.inst 0x4f80e068 // sdot v8.4s, v3.16b, v0.4b[0]
|
|
.inst 0x4fa0e069 // sdot v9.4s, v3.16b, v0.4b[1]
|
|
.inst 0x4f80e86a // sdot v10.4s, v3.16b, v0.4b[2]
|
|
.inst 0x4fa0e86b // sdot v11.4s, v3.16b, v0.4b[3]
|
|
|
|
.inst 0x4f81e06c // sdot v12.4s, v3.16b, v1.4b[0]
|
|
.inst 0x4fa1e06d // sdot v13.4s, v3.16b, v1.4b[1]
|
|
.inst 0x4f81e86e // sdot v14.4s, v3.16b, v1.4b[2]
|
|
.inst 0x4fa1e86f // sdot v15.4s, v3.16b, v1.4b[3]
|
|
|
|
.inst 0x4f82e070 // sdot v16.4s, v3.16b, v2.4b[0]
|
|
.inst 0x4fa2e071 // sdot v17.4s, v3.16b, v2.4b[1]
|
|
.inst 0x4f82e872 // sdot v18.4s, v3.16b, v2.4b[2]
|
|
.inst 0x4fa2e873 // sdot v19.4s, v3.16b, v2.4b[3]
|
|
.inst 0x4f80e094 // sdot v20.4s, v4.16b, v0.4b[0]
|
|
.inst 0x4fa0e095 // sdot v21.4s, v4.16b, v0.4b[1]
|
|
.inst 0x4f80e896 // sdot v22.4s, v4.16b, v0.4b[2]
|
|
.inst 0x4fa0e897 // sdot v23.4s, v4.16b, v0.4b[3]
|
|
|
|
.inst 0x4f81e098 // sdot v24.4s, v4.16b, v1.4b[0]
|
|
.inst 0x4fa1e099 // sdot v25.4s, v4.16b, v1.4b[1]
|
|
.inst 0x4f81e89a // sdot v26.4s, v4.16b, v1.4b[2]
|
|
.inst 0x4fa1e89b // sdot v27.4s, v4.16b, v1.4b[3]
|
|
subs x13, x13, #1
|
|
.inst 0x4f82e09c // sdot v28.4s, v4.16b, v2.4b[0]
|
|
.inst 0x4fa2e09d // sdot v29.4s, v4.16b, v2.4b[1]
|
|
.inst 0x4f82e89e // sdot v30.4s, v4.16b, v2.4b[2]
|
|
.inst 0x4fa2e89f // sdot v31.4s, v4.16b, v2.4b[3]
|
|
bne L8LoopSz_TILE_12
|
|
|
|
L8LoopSzEnd_TILE_12:
|
|
|
|
L8Tile12Quan:
|
|
ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [x12], #64 // weight scale&bias
|
|
ld1 {v4.4s, v5.4s, v6.4s}, [x8], x22 // input kernel sum
|
|
Int32ToFloat v8, v9, v10, v11
|
|
Int32ToFloat v12, v13, v14, v15
|
|
Int32ToFloat v16, v17, v18, v19
|
|
Int32ToFloat v20, v21, v22, v23
|
|
Int32ToFloat v24, v25, v26, v27
|
|
Int32ToFloat v28, v29, v30, v31
|
|
|
|
MUL_SCALE v0, v8, v9, v10, v11
|
|
MUL_SCALE v0, v12, v13, v14, v15
|
|
MUL_SCALE v0, v16, v17, v18, v19
|
|
MUL_SCALE v1, v20, v21, v22, v23
|
|
MUL_SCALE v1, v24, v25, v26, v27
|
|
MUL_SCALE v1, v28, v29, v30, v31
|
|
|
|
cbz x21, TILE12_L8_MLA_TERM
|
|
ld1 {v0.4s, v1.4s}, [x24], #32
|
|
ld1 {v7.4s}, [x24], x25
|
|
MUL_EXTRA_SCALE v0, v8, v9, v10, v11
|
|
MUL_EXTRA_SCALE v1, v12, v13, v14, v15
|
|
MUL_EXTRA_SCALE v7, v16, v17, v18, v19
|
|
MUL_EXTRA_SCALE v0, v20, v21, v22, v23
|
|
MUL_EXTRA_SCALE v1, v24, v25, v26, v27
|
|
MUL_EXTRA_SCALE v7, v28, v29, v30, v31
|
|
|
|
TILE12_L8_MLA_TERM:
|
|
MLA_WEIGHTZERO v8, v4, v2, 0 // tile:0, oc:0-3
|
|
MLA_WEIGHTZERO v9, v4, v2, 1 // tile:1, oc:0-3
|
|
MLA_WEIGHTZERO v10, v4, v2, 2 // tile:2, oc:0-3
|
|
MLA_WEIGHTZERO v11, v4, v2, 3 // tile:3, oc:0-3
|
|
MLA_WEIGHTZERO v12, v5, v2, 0 // tile:4, oc:0-3
|
|
MLA_WEIGHTZERO v13, v5, v2, 1 // tile:5, oc:0-3
|
|
MLA_WEIGHTZERO v14, v5, v2, 2 // tile:6, oc:0-3
|
|
MLA_WEIGHTZERO v15, v5, v2, 3 // tile:7, oc:0-3
|
|
MLA_WEIGHTZERO v16, v6, v2, 0 // tile:8, oc:0-3
|
|
MLA_WEIGHTZERO v17, v6, v2, 1 // tile:9, oc:0-3
|
|
MLA_WEIGHTZERO v18, v6, v2, 2 // tile:10, oc:0-3
|
|
MLA_WEIGHTZERO v19, v6, v2, 3 // tile:11, oc:0-3
|
|
|
|
MLA_WEIGHTZERO v20, v4, v3, 0 // tile:0, oc:4-7
|
|
MLA_WEIGHTZERO v21, v4, v3, 1 // tile:1, oc:4-7
|
|
MLA_WEIGHTZERO v22, v4, v3, 2 // tile:2, oc:4-7
|
|
MLA_WEIGHTZERO v23, v4, v3, 3 // tile:3, oc:4-7
|
|
MLA_WEIGHTZERO v24, v5, v3, 0 // tile:4, oc:4-7
|
|
MLA_WEIGHTZERO v25, v5, v3, 1 // tile:5, oc:4-7
|
|
MLA_WEIGHTZERO v26, v5, v3, 2 // tile:6, oc:4-7
|
|
MLA_WEIGHTZERO v27, v5, v3, 3 // tile:7, oc:4-7
|
|
MLA_WEIGHTZERO v28, v6, v3, 0 // tile:8, oc:4-7
|
|
MLA_WEIGHTZERO v29, v6, v3, 1 // tile:9, oc:4-7
|
|
MLA_WEIGHTZERO v30, v6, v3, 2 // tile:10, oc:4-7
|
|
MLA_WEIGHTZERO v31, v6, v3, 3 // tile:11, oc:4-7
|
|
|
|
|
|
cbz x27, TILE12_ADD_DSTV
|
|
ld1 {v0.4s, v1.4s, v2.4s}, [x27], x22 // input dequant bias
|
|
ld1 {v3.4s, v4.4s}, [x28], #32 // weight kernel sum
|
|
MLA_WEIGHTZERO v8, v0, v3, 0
|
|
MLA_WEIGHTZERO v9, v0, v3, 1
|
|
MLA_WEIGHTZERO v10, v0, v3, 2
|
|
MLA_WEIGHTZERO v11, v0, v3, 3
|
|
MLA_WEIGHTZERO v12, v1, v3, 0
|
|
MLA_WEIGHTZERO v13, v1, v3, 1
|
|
MLA_WEIGHTZERO v14, v1, v3, 2
|
|
MLA_WEIGHTZERO v15, v1, v3, 3
|
|
MLA_WEIGHTZERO v16, v2, v3, 0
|
|
MLA_WEIGHTZERO v17, v2, v3, 1
|
|
MLA_WEIGHTZERO v18, v2, v3, 2
|
|
MLA_WEIGHTZERO v19, v2, v3, 3
|
|
|
|
MLA_WEIGHTZERO v20, v0, v4, 0 // tile:0, oc:4-7
|
|
MLA_WEIGHTZERO v21, v0, v4, 1 // tile:1, oc:4-7
|
|
MLA_WEIGHTZERO v22, v0, v4, 2 // tile:2, oc:4-7
|
|
MLA_WEIGHTZERO v23, v0, v4, 3 // tile:3, oc:4-7
|
|
MLA_WEIGHTZERO v24, v1, v4, 0 // tile:4, oc:4-7
|
|
MLA_WEIGHTZERO v25, v1, v4, 1 // tile:5, oc:4-7
|
|
MLA_WEIGHTZERO v26, v1, v4, 2 // tile:6, oc:4-7
|
|
MLA_WEIGHTZERO v27, v1, v4, 3 // tile:7, oc:4-7
|
|
MLA_WEIGHTZERO v28, v2, v4, 0 // tile:8, oc:4-7
|
|
MLA_WEIGHTZERO v29, v2, v4, 1 // tile:9, oc:4-7
|
|
MLA_WEIGHTZERO v30, v2, v4, 2 // tile:10, oc:4-7
|
|
MLA_WEIGHTZERO v31, v2, v4, 3 // tile:11, oc:4-7
|
|
|
|
TILE12_ADD_DSTV:
|
|
cbz x19, TILE12_L8_ACCUM_BUFFER // x19=0: first block, do not add previous block result
|
|
ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [x10], #64
|
|
ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [x10], #64
|
|
ADD_FLOAT v8, v9, v10, v11, v0, v1, v2, v3
|
|
ADD_FLOAT v12, v13, v14, v15, v4, v5, v6, v7
|
|
ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [x10], #64
|
|
ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [x10], #64
|
|
ADD_FLOAT v16, v17, v18, v19, v0, v1, v2, v3
|
|
ADD_FLOAT v20, v21, v22, v23, v4, v5, v6, v7
|
|
ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [x10], #64
|
|
ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [x10]
|
|
ADD_FLOAT v24, v25, v26, v27, v0, v1, v2, v3
|
|
ADD_FLOAT v28, v29, v30, v31, v4, v5, v6, v7
|
|
sub x10, x10, #320
|
|
|
|
TILE12_L8_ACCUM_BUFFER:
|
|
add x19, x19, #1
|
|
cmp x19, x26
|
|
beq TILE12_POST
|
|
st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [x10], #64
|
|
st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [x10], #64
|
|
st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [x10], #64
|
|
st1 {v20.4s, v21.4s, v22.4s, v23.4s}, [x10], #64
|
|
st1 {v24.4s, v25.4s, v26.4s, v27.4s}, [x10], #64
|
|
st1 {v28.4s, v29.4s, v30.4s, v31.4s}, [x10]
|
|
sub x10, x10, #320
|
|
b TILE12_BLOCKNUM
|
|
|
|
TILE12_POST:
|
|
cbz x28, L8Tile12QuanUseInt8
|
|
sub x14, x14, #2
|
|
cbz x9, TILE12_RELU
|
|
ld1 {v0.4s, v1.4s}, [x20], #32
|
|
ADD_BIAS_FLOAT v8, v9, v10, v11, v0
|
|
ADD_BIAS_FLOAT v12, v13, v14, v15, v0
|
|
ADD_BIAS_FLOAT v16, v17, v18, v19, v0
|
|
ADD_BIAS_FLOAT v20, v21, v22, v23, v1
|
|
ADD_BIAS_FLOAT v24, v25, v26, v27, v1
|
|
ADD_BIAS_FLOAT v28, v29, v30, v31, v1
|
|
|
|
TILE12_RELU:
|
|
cbz x23, TILE12_STORE
|
|
ld1r {v0.4s}, [x23], #4 // f32 min
|
|
ld1r {v1.4s}, [x23] // f32 max
|
|
ReLU_FP32 v8, v9, v10, v11, v0, v1
|
|
ReLU_FP32 v12, v13, v14, v15, v0, v1
|
|
ReLU_FP32 v16, v17, v18, v19, v0, v1
|
|
ReLU_FP32 v20, v21, v22, v23, v0, v1
|
|
ReLU_FP32 v24, v25, v26, v27, v0, v1
|
|
ReLU_FP32 v28, v29, v30, v31, v0, v1
|
|
sub x23, x23, #4
|
|
|
|
TILE12_STORE:
|
|
st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [x6], #64
|
|
st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [x6], #64
|
|
st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [x6], x4
|
|
st1 {v20.4s, v21.4s, v22.4s, v23.4s}, [x6], #64
|
|
st1 {v24.4s, v25.4s, v26.4s, v27.4s}, [x6], #64
|
|
st1 {v28.4s, v29.4s, v30.4s, v31.4s}, [x6], x4
|
|
b L8Tile12LoopCheck
|
|
|
|
L8Tile12QuanUseInt8:
|
|
sub x14, x14, #2
|
|
ld1r {v7.4s}, [x23], #4 // int8 max
|
|
ld1r {v6.4s}, [x23] // int8 min
|
|
ld1 {v0.4s, v1.4s}, [x9], #32
|
|
dup v7.16b, v7.b[0]
|
|
dup v6.16b, v6.b[0]
|
|
ADD_BIAS_FLOAT v8, v9, v10, v11, v0
|
|
ADD_BIAS_FLOAT v12, v13, v14, v15, v0
|
|
ADD_BIAS_FLOAT v16, v17, v18, v19, v0
|
|
ADD_BIAS_FLOAT v20, v21, v22, v23, v1
|
|
ADD_BIAS_FLOAT v24, v25, v26, v27, v1
|
|
ADD_BIAS_FLOAT v28, v29, v30, v31, v1
|
|
|
|
sub x23, x23, #4
|
|
FloatToInt32 v8, v9, v10, v11
|
|
FloatToInt32 v12, v13, v14, v15
|
|
FloatToInt32 v16, v17, v18, v19
|
|
FloatToInt32 v20, v21, v22, v23
|
|
FloatToInt32 v24, v25, v26, v27
|
|
FloatToInt32 v28, v29, v30, v31
|
|
Int32ToInt16 v8, v9, v10, v11, v0, v1
|
|
Int32ToInt16 v12, v13, v14, v15, v2, v3
|
|
Int32ToInt16 v16, v17, v18, v19, v4, v5
|
|
Int32ToInt16 v20, v21, v22, v23, v8, v9
|
|
Int32ToInt16 v24, v25, v26, v27, v10, v11
|
|
Int32ToInt16 v28, v29, v30, v31, v12, v13
|
|
Int16ToInt8 v0, v1, v2, v3, v16, v17
|
|
Int16ToInt8 v4, v5, v8, v9, v18, v19
|
|
Int16ToInt8 v10, v11, v12, v13, v20, v21
|
|
smax v16.16b, v6.16b, v16.16b
|
|
smax v17.16b, v6.16b, v17.16b
|
|
smax v18.16b, v6.16b, v18.16b
|
|
smax v19.16b, v6.16b, v19.16b
|
|
smax v20.16b, v6.16b, v20.16b
|
|
smax v21.16b, v6.16b, v21.16b
|
|
smin v16.16b, v7.16b, v16.16b
|
|
smin v17.16b, v7.16b, v17.16b
|
|
smin v18.16b, v7.16b, v18.16b
|
|
smin v19.16b, v7.16b, v19.16b
|
|
smin v20.16b, v7.16b, v20.16b
|
|
smin v21.16b, v7.16b, v21.16b
|
|
st1 {v16.16b, v17.16b, v18.16b}, [x6], x4
|
|
st1 {v19.16b, v20.16b, v21.16b}, [x6], x4
|
|
|
|
L8Tile12LoopCheck:
|
|
cbz x14, Tile12End
|
|
mov x8, x15 // revert input kernel sum
|
|
mov x24, x21 // revert input dequant scale
|
|
cbz x27, L8LoopDz_TILE_12
|
|
REVERT_INPUT_DEQUANT_BIAS x27, x19, x26, x22
|
|
b L8LoopDz_TILE_12
|
|
|
|
L4LoopDz_TILE_12:
|
|
mov x11, x1
|
|
mov x19, #0
|
|
L4_TILE12_BLOCKNUM:
|
|
mov x13, x3
|
|
SET_BIAS v8, v9, v10, v11
|
|
SET_BIAS v12, v13, v14, v15
|
|
SET_BIAS v16, v17, v18, v19
|
|
|
|
L4_LoopSz_TILE_12:
|
|
ld1 {v3.16b}, [x12] // weight
|
|
ld1 {v0.16b, v1.16b, v2.16b}, [x11], x22 // src
|
|
.inst 0x4f80e068 // sdot v8.4s, v3.16b, v0.4b[0]
|
|
.inst 0x4fa0e069 // sdot v9.4s, v3.16b, v0.4b[1]
|
|
.inst 0x4f80e86a // sdot v10.4s, v3.16b, v0.4b[2]
|
|
.inst 0x4fa0e86b // sdot v11.4s, v3.16b, v0.4b[3]
|
|
.inst 0x4f81e06c // sdot v12.4s, v3.16b, v1.4b[0]
|
|
.inst 0x4fa1e06d // sdot v13.4s, v3.16b, v1.4b[1]
|
|
.inst 0x4f81e86e // sdot v14.4s, v3.16b, v1.4b[2]
|
|
.inst 0x4fa1e86f // sdot v15.4s, v3.16b, v1.4b[3]
|
|
add x12, x12, #32 // weight offset=lp*hp=32
|
|
subs x13, x13, #1
|
|
.inst 0x4f82e070 // sdot v16.4s, v3.16b, v2.4b[0]
|
|
.inst 0x4fa2e071 // sdot v17.4s, v3.16b, v2.4b[1]
|
|
.inst 0x4f82e872 // sdot v18.4s, v3.16b, v2.4b[2]
|
|
.inst 0x4fa2e873 // sdot v19.4s, v3.16b, v2.4b[3]
|
|
bne L4_LoopSz_TILE_12
|
|
|
|
|
|
L4_Tile12Quan:
|
|
ld1 {v0.4s}, [x12] // scale
|
|
add x12, x12, #32
|
|
ld1 {v2.4s, v3.4s, v4.4s}, [x8], x22 // x kernel sum
|
|
ld1 {v5.4s}, [x12] // weight quan zeropoint
|
|
add x12, x12, #32
|
|
Int32ToFloat v8, v9, v10, v11
|
|
Int32ToFloat v12, v13, v14, v15
|
|
Int32ToFloat v16, v17, v18, v19
|
|
MUL_SCALE v0, v8, v9, v10, v11
|
|
MUL_SCALE v0, v12, v13, v14, v15
|
|
MUL_SCALE v0, v16, v17, v18, v19
|
|
|
|
cbz x21, TILE12_L4_MLA
|
|
ld1 {v0.4s, v1.4s}, [x24], #32
|
|
ld1 {v7.4s}, [x24], x25
|
|
MUL_EXTRA_SCALE v0, v8, v9, v10, v11
|
|
MUL_EXTRA_SCALE v1, v12, v13, v14, v15
|
|
MUL_EXTRA_SCALE v7, v16, v17, v18, v19
|
|
|
|
TILE12_L4_MLA:
|
|
MLA_WEIGHTZERO v8, v2, v5, 0 // tile:0, oc:0-3
|
|
MLA_WEIGHTZERO v9, v2, v5, 1 // tile:1, oc:0-3
|
|
MLA_WEIGHTZERO v10, v2, v5, 2 // tile:2, oc:0-3
|
|
MLA_WEIGHTZERO v11, v2, v5, 3 // tile:3, oc:0-3
|
|
MLA_WEIGHTZERO v12, v3, v5, 0 // tile:4, oc:0-3
|
|
MLA_WEIGHTZERO v13, v3, v5, 1 // tile:5, oc:0-3
|
|
MLA_WEIGHTZERO v14, v3, v5, 2 // tile:6, oc:0-3
|
|
MLA_WEIGHTZERO v15, v3, v5, 3 // tile:7, oc:0-3
|
|
MLA_WEIGHTZERO v16, v4, v5, 0 // tile:8, oc:0-3
|
|
MLA_WEIGHTZERO v17, v4, v5, 1 // tile:9, oc:0-3
|
|
MLA_WEIGHTZERO v18, v4, v5, 2 // tile:10, oc:0-3
|
|
MLA_WEIGHTZERO v19, v4, v5, 3 // tile:11, oc:0-3
|
|
|
|
cbz x27, L4_TILE12_ADD_DSTV
|
|
ld1 {v0.4s, v1.4s, v2.4s}, [x27], x22 // input dequant bias
|
|
ld1 {v3.4s}, [x28] // weight kernel sum
|
|
MLA_WEIGHTZERO v8, v0, v3, 0
|
|
MLA_WEIGHTZERO v9, v0, v3, 1
|
|
MLA_WEIGHTZERO v10, v0, v3, 2
|
|
MLA_WEIGHTZERO v11, v0, v3, 3
|
|
MLA_WEIGHTZERO v12, v1, v3, 0
|
|
MLA_WEIGHTZERO v13, v1, v3, 1
|
|
MLA_WEIGHTZERO v14, v1, v3, 2
|
|
MLA_WEIGHTZERO v15, v1, v3, 3
|
|
MLA_WEIGHTZERO v16, v2, v3, 0
|
|
MLA_WEIGHTZERO v17, v2, v3, 1
|
|
MLA_WEIGHTZERO v18, v2, v3, 2
|
|
MLA_WEIGHTZERO v19, v2, v3, 3
|
|
add x28, x28, #32
|
|
|
|
L4_TILE12_ADD_DSTV:
|
|
cbz x19, L4_TILE12_ACCUM_BUFFER // x19=0: first block, do not add previous block result
|
|
ld1 {v20.4s, v21.4s, v22.4s, v23.4s}, [x10], #64
|
|
ld1 {v24.4s, v25.4s, v26.4s, v27.4s}, [x10], #64
|
|
ld1 {v28.4s, v29.4s, v30.4s, v31.4s}, [x10]
|
|
ADD_FLOAT v8, v9, v10, v11, v20, v21, v22, v23
|
|
ADD_FLOAT v12, v13, v14, v15, v24, v25, v26, v27
|
|
ADD_FLOAT v16, v17, v18, v19, v28, v29, v30, v31
|
|
sub x10, x10, #128
|
|
|
|
L4_TILE12_ACCUM_BUFFER:
|
|
add x19, x19, #1
|
|
cmp x19, x26
|
|
beq L4_TILE12_POST
|
|
st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [x10], #64
|
|
st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [x10], #64
|
|
st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [x10]
|
|
sub x10, x10, #128
|
|
b L4_TILE12_BLOCKNUM
|
|
|
|
L4_TILE12_POST:
|
|
cbz x28, L4Tile12QuanUseInt8
|
|
cbz x9, L4_TILE12_RELU
|
|
ld1 {v0.4s}, [x20], #16
|
|
ADD_BIAS_FLOAT v8, v9, v10, v11, v0
|
|
ADD_BIAS_FLOAT v12, v13, v14, v15, v0
|
|
ADD_BIAS_FLOAT v16, v17, v18, v19, v0
|
|
L4_TILE12_RELU:
|
|
cbz x23, L4_TILE12_STORE
|
|
ld1r {v0.4s}, [x23], #4 // f32 min
|
|
ld1r {v1.4s}, [x23] // f32 max
|
|
ReLU_FP32 v8, v9, v10, v11, v0, v1
|
|
ReLU_FP32 v12, v13, v14, v15, v0, v1
|
|
ReLU_FP32 v16, v17, v18, v19, v0, v1
|
|
sub x23, x23, #4
|
|
|
|
L4_TILE12_STORE:
|
|
st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [x6], #64
|
|
st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [x6], #64
|
|
st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [x6], x4
|
|
b Tile12End
|
|
|
|
L4Tile12QuanUseInt8:
|
|
ld1r {v7.4s}, [x23], #4 // int8 max
|
|
ld1r {v6.4s}, [x23] // int8 min
|
|
ld1 {v0.4s}, [x9] // bias
|
|
dup v7.16b, v7.b[0]
|
|
dup v6.16b, v6.b[0]
|
|
ADD_BIAS_FLOAT v8, v9, v10, v11, v0
|
|
ADD_BIAS_FLOAT v12, v13, v14, v15, v0
|
|
ADD_BIAS_FLOAT v16, v17, v18, v19, v0
|
|
sub x23, x23, #4
|
|
FloatToInt32 v8, v9, v10, v11
|
|
FloatToInt32 v12, v13, v14, v15
|
|
FloatToInt32 v16, v17, v18, v19
|
|
Int32ToInt16 v8, v9, v10, v11, v0, v1
|
|
Int32ToInt16 v12, v13, v14, v15, v2, v3
|
|
Int32ToInt16 v16, v17, v18, v19, v4, v5
|
|
Int16ToInt8 v0, v1, v2, v3, v16, v17
|
|
Int16ToInt8_ONE v4, v5, v18
|
|
smax v16.16b, v6.16b, v16.16b
|
|
smax v17.16b, v6.16b, v17.16b
|
|
smax v18.16b, v6.16b, v18.16b
|
|
smin v16.16b, v7.16b, v16.16b
|
|
smin v17.16b, v7.16b, v17.16b
|
|
smin v18.16b, v7.16b, v18.16b
|
|
st1 {v16.16b, v17.16b, v18.16b}, [x6], x4
|
|
b Tile12UpdateAddr
|
|
|
|
Tile12End:
|
|
add x4, x4, #128 // revert x4, int8 do not need
|
|
Tile12UpdateAddr:
|
|
add x0, x0, #192
|
|
sub x7, x7, #12
|
|
cbz x7, End
|
|
add x1, x1, #48
|
|
add x8, x15, #48
|
|
add x24, x21, #48
|
|
|
|
|
|
cbz x27, TILE_8
|
|
REVERT_INPUT_DEQUANT_BIAS x27, x19, x26, x22
|
|
REVERT_WEIGHT_KERNEL_SUM x28, x14, x26, x5
|
|
add x27, x27, #48
|
|
|
|
TILE_8:
|
|
mov x25, #0
|
|
cbz x28, TILE_Remain
|
|
cbz x27, TILE_Remain
|
|
mov x25, x22
|
|
TILE_Remain:
|
|
|
|
cmp x7, #8
|
|
blt TILE_4
|
|
mov x6, x0
|
|
mov x12, x2
|
|
mov x14, x5
|
|
mov x20, x9 // bias
|
|
mov x15, x8 // input kernel sum
|
|
mov x21, x24 // input dequant scale
|
|
cbz x28, L8LoopDz_TILE_8
|
|
sub x4, x4, #64 // float output dst_Z_step-64 for Tile8
|
|
|
|
L8LoopDz_TILE_8:
|
|
cmp x14, #2
|
|
blt L4LoopDz_TILE_8
|
|
mov x11, x1
|
|
mov x19, #0
|
|
TILE8_BLOCKNUM:
|
|
mov x13, x3
|
|
|
|
SET_BIAS v8, v9, v10, v11
|
|
SET_BIAS v12, v13, v14, v15
|
|
SET_BIAS v16, v17, v18, v19
|
|
SET_BIAS v20, v21, v22, v23
|
|
|
|
L8LoopSz_TILE_8:
|
|
ld1 {v3.16b, v4.16b}, [x12], #32 // weight
|
|
ld1 {v0.16b, v1.16b}, [x11], x22 // src
|
|
.inst 0x4f80e068 // sdot v8.4s, v3.16b, v0.4b[0]
|
|
.inst 0x4fa0e069 // sdot v9.4s, v3.16b, v0.4b[1]
|
|
.inst 0x4f80e86a // sdot v10.4s, v3.16b, v0.4b[2]
|
|
.inst 0x4fa0e86b // sdot v11.4s, v3.16b, v0.4b[3]
|
|
|
|
.inst 0x4f81e06c // sdot v12.4s, v3.16b, v1.4b[0]
|
|
.inst 0x4fa1e06d // sdot v13.4s, v3.16b, v1.4b[1]
|
|
.inst 0x4f81e86e // sdot v14.4s, v3.16b, v1.4b[2]
|
|
.inst 0x4fa1e86f // sdot v15.4s, v3.16b, v1.4b[3]
|
|
|
|
.inst 0x4f80e090 // sdot v16.4s, v4.16b, v0.4b[0]
|
|
.inst 0x4fa0e091 // sdot v17.4s, v4.16b, v0.4b[1]
|
|
.inst 0x4f80e892 // sdot v18.4s, v4.16b, v0.4b[2]
|
|
.inst 0x4fa0e893 // sdot v19.4s, v4.16b, v0.4b[3]
|
|
subs x13, x13, #1
|
|
.inst 0x4f81e094 // sdot v20.4s, v4.16b, v1.4b[0]
|
|
.inst 0x4fa1e095 // sdot v21.4s, v4.16b, v1.4b[1]
|
|
.inst 0x4f81e896 // sdot v22.4s, v4.16b, v1.4b[2]
|
|
.inst 0x4fa1e897 // sdot v23.4s, v4.16b, v1.4b[3]
|
|
bne L8LoopSz_TILE_8
|
|
|
|
L8Tile8Quan:
|
|
ld1 {v0.4s, v1.4s}, [x12], #32 // scale
|
|
ld1 {v2.4s, v3.4s}, [x8], x22 // x kernel sum
|
|
ld1 {v24.4s, v25.4s}, [x12], #32 // weight quan zeropoint
|
|
Int32ToFloat v8, v9, v10, v11
|
|
Int32ToFloat v12, v13, v14, v15
|
|
Int32ToFloat v16, v17, v18, v19
|
|
Int32ToFloat v20, v21, v22, v23
|
|
MUL_SCALE v0, v8, v9, v10, v11
|
|
MUL_SCALE v0, v12, v13, v14, v15
|
|
MUL_SCALE v1, v16, v17, v18, v19
|
|
MUL_SCALE v1, v20, v21, v22, v23
|
|
|
|
cbz x21, TILE8_L8_MLA
|
|
ld1 {v4.4s, v5.4s}, [x24], x25
|
|
MUL_EXTRA_SCALE v4, v8, v9, v10, v11
|
|
MUL_EXTRA_SCALE v5, v12, v13, v14, v15
|
|
MUL_EXTRA_SCALE v4, v16, v17, v18, v19
|
|
MUL_EXTRA_SCALE v5, v20, v21, v22, v23
|
|
|
|
TILE8_L8_MLA:
|
|
MLA_WEIGHTZERO v8, v2, v24, 0 // tile:0, oc:0-3
|
|
MLA_WEIGHTZERO v9, v2, v24, 1 // tile:1, oc:0-3
|
|
MLA_WEIGHTZERO v10, v2, v24, 2 // tile:2, oc:0-3
|
|
MLA_WEIGHTZERO v11, v2, v24, 3 // tile:3, oc:0-3
|
|
MLA_WEIGHTZERO v12, v3, v24, 0 // tile:4, oc:0-3
|
|
MLA_WEIGHTZERO v13, v3, v24, 1 // tile:5, oc:0-3
|
|
MLA_WEIGHTZERO v14, v3, v24, 2 // tile:6, oc:0-3
|
|
MLA_WEIGHTZERO v15, v3, v24, 3 // tile:7, oc:0-3
|
|
MLA_WEIGHTZERO v16, v2, v25, 0 // tile:0, oc:4-7
|
|
MLA_WEIGHTZERO v17, v2, v25, 1 // tile:1, oc:4-7
|
|
MLA_WEIGHTZERO v18, v2, v25, 2 // tile:2, oc:4-7
|
|
MLA_WEIGHTZERO v19, v2, v25, 3 // tile:3, oc:4-7
|
|
MLA_WEIGHTZERO v20, v3, v25, 0 // tile:4, oc:4-7
|
|
MLA_WEIGHTZERO v21, v3, v25, 1 // tile:5, oc:4-7
|
|
MLA_WEIGHTZERO v22, v3, v25, 2 // tile:6, oc:4-7
|
|
MLA_WEIGHTZERO v23, v3, v25, 3 // tile:7, oc:4-7
|
|
|
|
cbz x27, TILE8_ADD_DSTV
|
|
ld1 {v2.4s, v3.4s}, [x27], x25
|
|
ld1 {v24.4s, v25.4s}, [x28], #32
|
|
MLA_WEIGHTZERO v8, v2, v24, 0 // tile:0, oc:0-3
|
|
MLA_WEIGHTZERO v9, v2, v24, 1 // tile:1, oc:0-3
|
|
MLA_WEIGHTZERO v10, v2, v24, 2 // tile:2, oc:0-3
|
|
MLA_WEIGHTZERO v11, v2, v24, 3 // tile:3, oc:0-3
|
|
MLA_WEIGHTZERO v12, v3, v24, 0 // tile:4, oc:0-3
|
|
MLA_WEIGHTZERO v13, v3, v24, 1 // tile:5, oc:0-3
|
|
MLA_WEIGHTZERO v14, v3, v24, 2 // tile:6, oc:0-3
|
|
MLA_WEIGHTZERO v15, v3, v24, 3 // tile:7, oc:0-3
|
|
MLA_WEIGHTZERO v16, v2, v25, 0 // tile:0, oc:4-7
|
|
MLA_WEIGHTZERO v17, v2, v25, 1 // tile:1, oc:4-7
|
|
MLA_WEIGHTZERO v18, v2, v25, 2 // tile:2, oc:4-7
|
|
MLA_WEIGHTZERO v19, v2, v25, 3 // tile:3, oc:4-7
|
|
MLA_WEIGHTZERO v20, v3, v25, 0 // tile:4, oc:4-7
|
|
MLA_WEIGHTZERO v21, v3, v25, 1 // tile:5, oc:4-7
|
|
MLA_WEIGHTZERO v22, v3, v25, 2 // tile:6, oc:4-7
|
|
MLA_WEIGHTZERO v23, v3, v25, 3 // tile:7, oc:4-7
|
|
|
|
TILE8_ADD_DSTV:
|
|
cbz x19, TILE8_L8_ACCUM_BUFFER
|
|
ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [x10], #64
|
|
ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [x10], #64
|
|
ld1 {v24.4s, v25.4s, v26.4s, v27.4s}, [x10], #64
|
|
ld1 {v28.4s, v29.4s, v30.4s, v31.4s}, [x10]
|
|
ADD_FLOAT v8, v9, v10, v11, v0, v1, v2, v3
|
|
ADD_FLOAT v12, v13, v14, v15, v4, v5, v6, v7
|
|
ADD_FLOAT v16, v17, v18, v19, v24, v25, v26, v27
|
|
ADD_FLOAT v20, v21, v22, v23, v28, v29, v30, v31
|
|
sub x10, x10, #192
|
|
|
|
TILE8_L8_ACCUM_BUFFER:
|
|
add x19, x19, #1
|
|
cmp x19, x26
|
|
beq TILE8_POST
|
|
st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [x10], #64
|
|
st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [x10], #64
|
|
st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [x10], #64
|
|
st1 {v20.4s, v21.4s, v22.4s, v23.4s}, [x10]
|
|
sub x10, x10, #192
|
|
b TILE8_BLOCKNUM
|
|
|
|
TILE8_POST:
|
|
cbz x28, L8Tile8QuanUseInt8
|
|
sub x14, x14, #2 // oc-2
|
|
cbz x9, TILE8_RELU
|
|
ld1 {v0.4s, v1.4s}, [x20], #32
|
|
ADD_BIAS_FLOAT v8, v9, v10, v11, v0
|
|
ADD_BIAS_FLOAT v12, v13, v14, v15, v0
|
|
ADD_BIAS_FLOAT v16, v17, v18, v19, v1
|
|
ADD_BIAS_FLOAT v20, v21, v22, v23, v1
|
|
|
|
TILE8_RELU:
|
|
cbz x23, TILE8_STORE
|
|
ld1r {v0.4s}, [x23], #4 // f32 min
|
|
ld1r {v1.4s}, [x23] // f32 max
|
|
ReLU_FP32 v8, v9, v10, v11, v0, v1
|
|
ReLU_FP32 v12, v13, v14, v15, v0, v1
|
|
ReLU_FP32 v16, v17, v18, v19, v0, v1
|
|
ReLU_FP32 v20, v21, v22, v23, v0, v1
|
|
sub x23, x23, #4
|
|
|
|
TILE8_STORE:
|
|
st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [x6], #64
|
|
st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [x6], x4
|
|
st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [x6], #64
|
|
st1 {v20.4s, v21.4s, v22.4s, v23.4s}, [x6], x4
|
|
cbz x14, TILE8_FLOAT_END
|
|
b L8Tile8LoopCheck
|
|
|
|
L8Tile8QuanUseInt8:
|
|
sub x14, x14, #2
|
|
ld1r {v7.4s}, [x23], #4 // int8 max
|
|
ld1r {v6.4s}, [x23] // int8 min
|
|
ld1 {v0.4s, v1.4s}, [x20], #32
|
|
dup v7.16b, v7.b[0]
|
|
dup v6.16b, v6.b[0]
|
|
ADD_BIAS_FLOAT v8, v9, v10, v11, v0
|
|
ADD_BIAS_FLOAT v12, v13, v14, v15, v0
|
|
ADD_BIAS_FLOAT v16, v17, v18, v19, v1
|
|
ADD_BIAS_FLOAT v20, v21, v22, v23, v1
|
|
sub x23, x23, #4
|
|
FloatToInt32 v8, v9, v10, v11
|
|
FloatToInt32 v12, v13, v14, v15
|
|
FloatToInt32 v16, v17, v18, v19
|
|
FloatToInt32 v20, v21, v22, v23
|
|
Int32ToInt16 v8, v9, v10, v11, v0, v1
|
|
Int32ToInt16 v12, v13, v14, v15, v2, v3
|
|
Int32ToInt16 v16, v17, v18, v19, v4, v5
|
|
Int32ToInt16 v20, v21, v22, v23, v8, v9
|
|
Int16ToInt8 v0, v1, v2, v3, v16, v17
|
|
Int16ToInt8 v4, v5, v8, v9, v18, v19
|
|
smax v16.16b, v6.16b, v16.16b
|
|
smax v17.16b, v6.16b, v17.16b
|
|
smax v18.16b, v6.16b, v18.16b
|
|
smax v19.16b, v6.16b, v19.16b
|
|
smin v16.16b, v7.16b, v16.16b
|
|
smin v17.16b, v7.16b, v17.16b
|
|
smin v18.16b, v7.16b, v18.16b
|
|
smin v19.16b, v7.16b, v19.16b
|
|
st1 {v16.16b, v17.16b}, [x6], x4
|
|
st1 {v18.16b, v19.16b}, [x6], x4
|
|
cbz x14, TILE8_INT8_END
|
|
|
|
L8Tile8LoopCheck:
|
|
mov x8, x15 // revert input kernel sum
|
|
mov x24, x21 // revert input dequant scale
|
|
cbz x27, L8LoopDz_TILE_8
|
|
REVERT_INPUT_DEQUANT_BIAS x27, x19, x26, x22
|
|
b L8LoopDz_TILE_8
|
|
|
|
L4LoopDz_TILE_8:
|
|
mov x11, x1
|
|
mov x19, #0
|
|
L4_TILE8_BLOCKNUM:
|
|
mov x13, x3
|
|
|
|
SET_BIAS v8, v9, v10, v11
|
|
SET_BIAS v12, v13, v14, v15
|
|
|
|
L4LoopSz_TILE_8:
|
|
ld1 {v3.16b}, [x12] // weight
|
|
ld1 {v0.16b, v1.16b}, [x11], x22 // src
|
|
.inst 0x4f80e068 // sdot v8.4s, v3.16b, v0.4b[0]
|
|
.inst 0x4fa0e069 // sdot v9.4s, v3.16b, v0.4b[1]
|
|
.inst 0x4f80e86a // sdot v10.4s, v3.16b, v0.4b[2]
|
|
.inst 0x4fa0e86b // sdot v11.4s, v3.16b, v0.4b[3]
|
|
add x12, x12, #32 // weight offset=lp*hp
|
|
subs x13, x13, #1
|
|
.inst 0x4f81e06c // sdot v12.4s, v3.16b, v1.4b[0]
|
|
.inst 0x4fa1e06d // sdot v13.4s, v3.16b, v1.4b[1]
|
|
.inst 0x4f81e86e // sdot v14.4s, v3.16b, v1.4b[2]
|
|
.inst 0x4fa1e86f // sdot v15.4s, v3.16b, v1.4b[3]
|
|
bne L4LoopSz_TILE_8
|
|
|
|
L4Tile8Quan:
|
|
ld1 {v0.4s}, [x12] // scale
|
|
add x12, x12, #32
|
|
ld1 {v2.4s, v3.4s}, [x8], x22 // x kernel sum
|
|
ld1 {v24.4s}, [x12] // weight quan zeropoint
|
|
add x12, x12, #32
|
|
Int32ToFloat v8, v9, v10, v11
|
|
Int32ToFloat v12, v13, v14, v15
|
|
MUL_SCALE v0, v8, v9, v10, v11
|
|
MUL_SCALE v0, v12, v13, v14, v15
|
|
|
|
cbz x21, TILE8_L4_MLA
|
|
ld1 {v4.4s, v5.4s}, [x24], x25
|
|
MUL_EXTRA_SCALE v4, v8, v9, v10, v11
|
|
MUL_EXTRA_SCALE v5, v12, v13, v14, v15
|
|
|
|
TILE8_L4_MLA:
|
|
MLA_WEIGHTZERO v8, v2, v24, 0 // tile:0, oc:0-3
|
|
MLA_WEIGHTZERO v9, v2, v24, 1 // tile:1, oc:0-3
|
|
MLA_WEIGHTZERO v10, v2, v24, 2 // tile:2, oc:0-3
|
|
MLA_WEIGHTZERO v11, v2, v24, 3 // tile:3, oc:0-3
|
|
MLA_WEIGHTZERO v12, v3, v24, 0 // tile:4, oc:0-3
|
|
MLA_WEIGHTZERO v13, v3, v24, 1 // tile:5, oc:0-3
|
|
MLA_WEIGHTZERO v14, v3, v24, 2 // tile:6, oc:0-3
|
|
MLA_WEIGHTZERO v15, v3, v24, 3 // tile:7, oc:0-3
|
|
|
|
cbz x27, L4_TILE8_ADD_DSTV
|
|
ld1 {v2.4s, v3.4s}, [x27], x25
|
|
ld1 {v24.4s}, [x28]
|
|
MLA_WEIGHTZERO v8, v2, v24, 0 // tile:0, oc:0-3
|
|
MLA_WEIGHTZERO v9, v2, v24, 1 // tile:1, oc:0-3
|
|
MLA_WEIGHTZERO v10, v2, v24, 2 // tile:2, oc:0-3
|
|
MLA_WEIGHTZERO v11, v2, v24, 3 // tile:3, oc:0-3
|
|
MLA_WEIGHTZERO v12, v3, v24, 0 // tile:4, oc:0-3
|
|
MLA_WEIGHTZERO v13, v3, v24, 1 // tile:5, oc:0-3
|
|
MLA_WEIGHTZERO v14, v3, v24, 2 // tile:6, oc:0-3
|
|
MLA_WEIGHTZERO v15, v3, v24, 3 // tile:7, oc:0-3
|
|
add x28, x28, #32
|
|
|
|
L4_TILE8_ADD_DSTV:
|
|
cbz x19, TILE8_L4_ACCUM_BUFFER
|
|
ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [x10], #64
|
|
ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [x10]
|
|
ADD_FLOAT v8, v9, v10, v11, v0, v1, v2, v3
|
|
ADD_FLOAT v12, v13, v14, v15, v4, v5, v6, v7
|
|
sub x10, x10, #64
|
|
|
|
TILE8_L4_ACCUM_BUFFER:
|
|
add x19, x19, #1
|
|
cmp x19, x26
|
|
beq L4_TILE8_POST
|
|
st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [x10], #64
|
|
st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [x10]
|
|
sub x10, x10, #64
|
|
b L4_TILE8_BLOCKNUM
|
|
|
|
L4_TILE8_POST:
|
|
cbz x28, L4Tile8QuanUseInt8
|
|
cbz x9, L4_TILE8_RELU
|
|
ld1 {v0.4s}, [x20], #16
|
|
ADD_BIAS_FLOAT v8, v9, v10, v11, v0
|
|
ADD_BIAS_FLOAT v12, v13, v14, v15, v0
|
|
|
|
L4_TILE8_RELU:
|
|
cbz x23, L4_TILE8_STORE
|
|
ld1r {v0.4s}, [x23], #4 // f32 min
|
|
ld1r {v1.4s}, [x23] // f32 max
|
|
ReLU_FP32 v8, v9, v10, v11, v0, v1
|
|
ReLU_FP32 v12, v13, v14, v15, v0, v1
|
|
sub x23, x23, #4
|
|
|
|
L4_TILE8_STORE:
|
|
st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [x6], #64
|
|
st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [x6], x4
|
|
b TILE8_FLOAT_END
|
|
|
|
L4Tile8QuanUseInt8:
|
|
ld1r {v7.4s}, [x23], #4 // int8 max
|
|
ld1r {v6.4s}, [x23] // int8 min
|
|
ld1 {v4.4s}, [x20], #16
|
|
dup v7.16b, v7.b[0]
|
|
dup v6.16b, v6.b[0]
|
|
ADD_BIAS_FLOAT v8, v9, v10, v11, v4
|
|
ADD_BIAS_FLOAT v12, v13, v14, v15, v4
|
|
sub x23, x23, #4
|
|
FloatToInt32 v8, v9, v10, v11
|
|
FloatToInt32 v12, v13, v14, v15
|
|
Int32ToInt16 v8, v9, v10, v11, v0, v1
|
|
Int32ToInt16 v12, v13, v14, v15, v2, v3
|
|
Int16ToInt8 v0, v1, v2, v3, v16, v17
|
|
smax v16.16b, v6.16b, v16.16b
|
|
smax v17.16b, v6.16b, v17.16b
|
|
smin v16.16b, v7.16b, v16.16b
|
|
smin v17.16b, v7.16b, v17.16b
|
|
st1 {v16.16b, v17.16b}, [x6], x4
|
|
b TILE8_INT8_END
|
|
|
|
TILE8_FLOAT_END:
|
|
add x0, x0, #128
|
|
sub x7, x7, #8
|
|
cbz x7, End
|
|
add x1, x1, #32
|
|
add x8, x15, #32
|
|
add x24, x21, #32
|
|
add x4, x4, #64 // revert dst step
|
|
cbz x27, TILE_4
|
|
REVERT_INPUT_DEQUANT_BIAS x27, x19, x26, x22
|
|
REVERT_WEIGHT_KERNEL_SUM x28, x14, x26, x5
|
|
add x27, x27, #32
|
|
b TILE_4
|
|
TILE8_INT8_END:
|
|
add x0, x0, #32
|
|
sub x7, x7, #8
|
|
cbz x7, End
|
|
add x1, x1, #32
|
|
add x8, x15, #32
|
|
cbz x21, TILE_4
|
|
add x24, x21, #32
|
|
|
|
TILE_4:
|
|
cmp x7, #4
|
|
blt TILE_1_Init
|
|
mov x6, x0
|
|
mov x12, x2
|
|
mov x14, x5
|
|
mov x20, x9
|
|
mov x15, x8 // input kernel sum
|
|
mov x21, x24 // input dequant scale
|
|
|
|
L8LoopDz_TILE_4:
|
|
cmp x14, #2
|
|
blt L4LoopDz_TILE_4
|
|
mov x11, x1
|
|
mov x19, #0
|
|
TILE4_BLOCKNUM:
|
|
mov x13, x3
|
|
SET_BIAS v8, v9, v10, v11
|
|
SET_BIAS v12, v13, v14, v15
|
|
|
|
L8LoopSz_TILE_4:
|
|
ld1 {v3.16b, v4.16b}, [x12], #32 // weight
|
|
ld1 {v0.16b}, [x11], x22 // src
|
|
.inst 0x4f80e068 // sdot v8.4s, v3.16b, v0.4b[0]
|
|
.inst 0x4fa0e069 // sdot v9.4s, v3.16b, v0.4b[1]
|
|
.inst 0x4f80e86a // sdot v10.4s, v3.16b, v0.4b[2]
|
|
.inst 0x4fa0e86b // sdot v11.4s, v3.16b, v0.4b[3]
|
|
|
|
subs x13, x13, #1
|
|
.inst 0x4f80e08c // sdot v12.4s, v4.16b, v0.4b[0]
|
|
.inst 0x4fa0e08d // sdot v13.4s, v4.16b, v0.4b[1]
|
|
.inst 0x4f80e88e // sdot v14.4s, v4.16b, v0.4b[2]
|
|
.inst 0x4fa0e88f // sdot v15.4s, v4.16b, v0.4b[3]
|
|
bne L8LoopSz_TILE_4
|
|
|
|
L8Tile4Quan:
|
|
ld1 {v0.4s, v1.4s}, [x12], #32 // scale
|
|
ld1 {v2.4s}, [x8], x22 // x kernel sum
|
|
ld1 {v24.4s, v25.4s}, [x12], #32 // weight quan zeropoint
|
|
Int32ToFloat v8, v9, v10, v11
|
|
Int32ToFloat v12, v13, v14, v15
|
|
MUL_SCALE v0, v8, v9, v10, v11
|
|
MUL_SCALE v1, v12, v13, v14, v15
|
|
|
|
cbz x21, TILE4_L8_MLA
|
|
ld1 {v4.4s}, [x24], x25
|
|
MUL_EXTRA_SCALE v4, v8, v9, v10, v11
|
|
MUL_EXTRA_SCALE v4, v12, v13, v14, v15
|
|
|
|
TILE4_L8_MLA:
|
|
MLA_WEIGHTZERO v8, v2, v24, 0 // tile:0, oc:0-3
|
|
MLA_WEIGHTZERO v9, v2, v24, 1 // tile:1, oc:0-3
|
|
MLA_WEIGHTZERO v10, v2, v24, 2 // tile:2, oc:0-3
|
|
MLA_WEIGHTZERO v11, v2, v24, 3 // tile:3, oc:0-3
|
|
MLA_WEIGHTZERO v12, v2, v25, 0 // tile:0, oc:4-7
|
|
MLA_WEIGHTZERO v13, v2, v25, 1 // tile:1, oc:4-7
|
|
MLA_WEIGHTZERO v14, v2, v25, 2 // tile:2, oc:4-7
|
|
MLA_WEIGHTZERO v15, v2, v25, 3 // tile:3, oc:4-7
|
|
|
|
|
|
cbz x27, TILE4_ADD_DSTV
|
|
ld1 {v2.4s}, [x27], x25
|
|
ld1 {v24.4s, v25.4s}, [x28], #32
|
|
MLA_WEIGHTZERO v8, v2, v24, 0 // tile:0, oc:0-3
|
|
MLA_WEIGHTZERO v9, v2, v24, 1 // tile:1, oc:0-3
|
|
MLA_WEIGHTZERO v10, v2, v24, 2 // tile:2, oc:0-3
|
|
MLA_WEIGHTZERO v11, v2, v24, 3 // tile:3, oc:0-3
|
|
MLA_WEIGHTZERO v12, v2, v25, 0 // tile:0, oc:4-7
|
|
MLA_WEIGHTZERO v13, v2, v25, 1 // tile:1, oc:4-7
|
|
MLA_WEIGHTZERO v14, v2, v25, 2 // tile:2, oc:4-7
|
|
MLA_WEIGHTZERO v15, v2, v25, 3 // tile:3, oc:4-7
|
|
|
|
TILE4_ADD_DSTV:
|
|
cbz x19, TILE4_L8_ACCUM_BUFFER
|
|
ld1 {v16.4s, v17.4s, v18.4s, v19.4s}, [x10], #64
|
|
ld1 {v20.4s, v21.4s, v22.4s, v23.4s}, [x10]
|
|
ADD_FLOAT v8, v9, v10, v11, v16, v17, v18, v19
|
|
ADD_FLOAT v12, v13, v14, v15, v20, v21, v22, v23
|
|
sub x10, x10, #64
|
|
|
|
TILE4_L8_ACCUM_BUFFER:
|
|
add x19, x19, #1
|
|
cmp x19, x26
|
|
beq TILE4_POST
|
|
st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [x10], #64
|
|
st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [x10]
|
|
sub x10, x10, #64
|
|
b TILE4_BLOCKNUM
|
|
|
|
TILE4_POST:
|
|
cbz x28, L8Tile4QuanUseInt8
|
|
sub x14, x14, #2
|
|
cbz x9, TILE4_RELU
|
|
ld1 {v4.4s, v5.4s}, [x20], #32
|
|
ADD_BIAS_FLOAT v8, v9, v10, v11, v4
|
|
ADD_BIAS_FLOAT v12, v13, v14, v15, v5
|
|
TILE4_RELU:
|
|
cbz x23, TILE4_STORE
|
|
ld1r {v26.4s}, [x23], #4 // f32 min
|
|
ld1r {v27.4s}, [x23] // f32 max
|
|
ReLU_FP32 v8, v9, v10, v11, v26, v27
|
|
ReLU_FP32 v12, v13, v14, v15, v26, v27
|
|
sub x23, x23, #4
|
|
|
|
TILE4_STORE:
|
|
st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [x6], x4
|
|
st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [x6], x4
|
|
cbz x14, TILE4_FLOAT_END
|
|
b L8Tile4LoopCheck
|
|
|
|
L8Tile4QuanUseInt8:
|
|
sub x14, x14, #2
|
|
ld1r {v7.4s}, [x23], #4 // int8 max
|
|
ld1r {v6.4s}, [x23] // int8 min
|
|
ld1 {v4.4s, v5.4s}, [x20], #32
|
|
dup v7.16b, v7.b[0]
|
|
dup v6.16b, v6.b[0]
|
|
ADD_BIAS_FLOAT v8, v9, v10, v11, v4
|
|
ADD_BIAS_FLOAT v12, v13, v14, v15, v5
|
|
sub x23, x23, #4
|
|
FloatToInt32 v8, v9, v10, v11
|
|
FloatToInt32 v12, v13, v14, v15
|
|
Int32ToInt16 v8, v9, v10, v11, v0, v1
|
|
Int32ToInt16 v12, v13, v14, v15, v2, v3
|
|
Int16ToInt8 v0, v1, v2, v3, v16, v17
|
|
smax v16.16b, v6.16b, v16.16b
|
|
smax v17.16b, v6.16b, v17.16b
|
|
smin v16.16b, v7.16b, v16.16b
|
|
smin v17.16b, v7.16b, v17.16b
|
|
st1 {v16.16b}, [x6], x4
|
|
st1 {v17.16b}, [x6], x4
|
|
cbz x14, TILE4_INT8_END
|
|
|
|
L8Tile4LoopCheck:
|
|
mov x8, x15
|
|
mov x24, x21 // revert input dequant scale
|
|
cbz x27, L8LoopDz_TILE_4
|
|
REVERT_INPUT_DEQUANT_BIAS x27, x19, x26, x22
|
|
b L8LoopDz_TILE_4
|
|
|
|
L4LoopDz_TILE_4:
|
|
mov x11, x1
|
|
mov x19, #0
|
|
L4_TILE4_BLOCKNUM:
|
|
mov x13, x3
|
|
SET_BIAS v8, v9, v10, v11
|
|
|
|
L4LoopSz_TILE_4:
|
|
ld1 {v3.16b}, [x12] // weight
|
|
ld1 {v0.16b}, [x11], x22 // src
|
|
subs x13, x13, #1
|
|
add x12, x12, #32 // weight offset = lp*hp
|
|
.inst 0x4f80e068 // sdot v8.4s, v3.16b, v0.4b[0]
|
|
.inst 0x4fa0e069 // sdot v9.4s, v3.16b, v0.4b[1]
|
|
.inst 0x4f80e86a // sdot v10.4s, v3.16b, v0.4b[2]
|
|
.inst 0x4fa0e86b // sdot v11.4s, v3.16b, v0.4b[3]
|
|
bne L4LoopSz_TILE_4
|
|
|
|
L4Tile4Quan:
|
|
ld1 {v0.4s}, [x12] // scale
|
|
add x12, x12, #32
|
|
ld1 {v2.4s}, [x8], x22 // x kernel sum
|
|
ld1 {v24.4s}, [x12] // weight quan zeropoint
|
|
add x12, x12, #32
|
|
Int32ToFloat v8, v9, v10, v11
|
|
MUL_SCALE v0, v8, v9, v10, v11
|
|
|
|
cbz x21, TILE4_L4_MLA
|
|
ld1 {v4.4s}, [x24], x25
|
|
MUL_EXTRA_SCALE v4, v8, v9, v10, v11
|
|
|
|
TILE4_L4_MLA:
|
|
MLA_WEIGHTZERO v8, v2, v24, 0 // tile:0, oc:0-3
|
|
MLA_WEIGHTZERO v9, v2, v24, 1 // tile:1, oc:0-3
|
|
MLA_WEIGHTZERO v10, v2, v24, 2 // tile:2, oc:0-3
|
|
MLA_WEIGHTZERO v11, v2, v24, 3 // tile:3, oc:0-3
|
|
|
|
cbz x27, L4_TILE4_ADD_DSTV
|
|
ld1 {v2.4s}, [x27], x25
|
|
ld1 {v24.4s}, [x28]
|
|
MLA_WEIGHTZERO v8, v2, v24, 0 // tile:0, oc:0-3
|
|
MLA_WEIGHTZERO v9, v2, v24, 1 // tile:1, oc:0-3
|
|
MLA_WEIGHTZERO v10, v2, v24, 2 // tile:2, oc:0-3
|
|
MLA_WEIGHTZERO v11, v2, v24, 3 // tile:3, oc:0-3
|
|
add x28, x28, #32
|
|
|
|
L4_TILE4_ADD_DSTV:
|
|
cbz x19, TILE4_L4_ACCUM_BUFFER
|
|
ld1 {v16.4s, v17.4s, v18.4s, v19.4s}, [x10]
|
|
ADD_FLOAT v8, v9, v10, v11, v16, v17, v18, v19
|
|
|
|
TILE4_L4_ACCUM_BUFFER:
|
|
add x19, x19, #1
|
|
cmp x19, x26
|
|
beq L4_TILE4_POST
|
|
st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [x10]
|
|
b L4_TILE4_BLOCKNUM
|
|
|
|
L4_TILE4_POST:
|
|
cbz x28, L4Tile4QuanUseInt8
|
|
cbz x9, L4_TILE4_RELU
|
|
ld1 {v4.4s}, [x20], #16
|
|
ADD_BIAS_FLOAT v8, v9, v10, v11, v4
|
|
L4_TILE4_RELU:
|
|
cbz x23, L4_TILE4_STORE
|
|
ld1r {v26.4s}, [x23], #4 // f32 min
|
|
ld1r {v27.4s}, [x23] // f32 max
|
|
ReLU_FP32 v8, v9, v10, v11, v26, v27
|
|
sub x23, x23, #4
|
|
|
|
L4_TILE4_STORE:
|
|
st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [x6], x4
|
|
b TILE4_FLOAT_END
|
|
|
|
L4Tile4QuanUseInt8:
|
|
ld1r {v7.4s}, [x23], #4 // int8 max
|
|
ld1r {v6.4s}, [x23] // int8 min
|
|
ld1 {v3.4s}, [x20], #16
|
|
dup v7.16b, v7.b[0]
|
|
dup v6.16b, v6.b[0]
|
|
ADD_BIAS_FLOAT v8, v9, v10, v11, v3
|
|
sub x23, x23, #4
|
|
FloatToInt32 v8, v9, v10, v11
|
|
Int32ToInt16 v8, v9, v10, v11, v0, v1
|
|
Int16ToInt8_ONE v0, v1, v16
|
|
smax v16.16b, v6.16b, v16.16b
|
|
smin v16.16b, v7.16b, v16.16b
|
|
st1 {v16.16b}, [x6], x4
|
|
b TILE4_INT8_END
|
|
|
|
TILE4_FLOAT_END:
|
|
add x0, x0, #64
|
|
sub x7, x7, #4
|
|
cbz x7, End
|
|
add x1, x1, #16
|
|
add x8, x15, #16
|
|
add x24, x21, #16
|
|
cbz x27, TILE_1_Init_FLOAT
|
|
REVERT_INPUT_DEQUANT_BIAS x27, x19, x26, x22
|
|
REVERT_WEIGHT_KERNEL_SUM x28, x14, x26, x5
|
|
add x27, x27, #16
|
|
b TILE_1_Init_FLOAT
|
|
|
|
TILE4_INT8_END:
|
|
add x0, x0, #16
|
|
sub x7, x7, #4
|
|
cbz x7, End
|
|
add x1, x1, #16
|
|
add x8, x15, #16
|
|
cbz x21, TILE_1_Init_INT8
|
|
add x24, x21, #16
|
|
|
|
TILE_1_Init:
|
|
cbnz x28, TILE_1_Init_FLOAT
|
|
TILE_1_Init_INT8:
|
|
cbz x23, TILE_1
|
|
ld1r {v7.4s}, [x23], #4 // int8 max
|
|
ld1r {v6.4s}, [x23] // int8 min
|
|
dup v7.16b, v7.b[0]
|
|
dup v6.16b, v6.b[0]
|
|
b TILE_1
|
|
|
|
TILE_1_Init_FLOAT:
|
|
cbz x23, TILE_1
|
|
ld1r {v26.4s}, [x23], #4 // f32 min
|
|
ld1r {v27.4s}, [x23] // f32 max
|
|
|
|
TILE_1:
|
|
mov x6, x0
|
|
mov x12, x2
|
|
mov x14, x5
|
|
mov x20, x9
|
|
mov x15, x8 // input kernel sum
|
|
mov x21, x24 // input dequant scale
|
|
L8LoopDz_TILE_1:
|
|
cmp x14, #2
|
|
blt L4LoopDz_TILE_1
|
|
mov x11, x1
|
|
mov x19, #0
|
|
TILE1_BLOCKNUM:
|
|
mov x13, x3
|
|
|
|
movi v8.16b, #0
|
|
movi v9.16b, #0
|
|
L8LoopSz_TILE_1:
|
|
ld1 {v3.16b, v4.16b}, [x12], #32 // weight
|
|
ld1 {v0.s}[0], [x11], x22 // src
|
|
.inst 0x4f80e068 // sdot v8.4s, v3.16b, v0.4b[0]
|
|
subs x13, x13, #1
|
|
.inst 0x4f80e089 // sdot v9.4s, v4.16b, v0.4b[0]
|
|
bne L8LoopSz_TILE_1
|
|
|
|
L8Tile1Quan:
|
|
ld1 {v0.4s, v1.4s}, [x12], #32 // scale
|
|
ld1 {v2.s}[0], [x8], x22 // x kernel sum
|
|
ld1 {v24.4s, v25.4s}, [x12], #32 // weight quan zeropoint
|
|
scvtf v8.4s, v8.4s
|
|
scvtf v9.4s, v9.4s
|
|
fmul v8.4s, v8.4s, v0.4s
|
|
fmul v9.4s, v9.4s, v1.4s
|
|
|
|
cbz x21, TILE1_L8_MLA
|
|
ld1 {v4.s}[0], [x24], x25
|
|
fmul v8.4s, v8.4s, v4.s[0]
|
|
fmul v9.4s, v9.4s, v4.s[0]
|
|
|
|
TILE1_L8_MLA:
|
|
MLA_WEIGHTZERO v8, v2, v24, 0 // tile:0, oc:0-3
|
|
MLA_WEIGHTZERO v9, v2, v25, 0 // tile:0, oc:4-7
|
|
|
|
|
|
|
|
cbz x27, TILE1_ADD_DSTV
|
|
ld1 {v2.s}[0], [x27], x25
|
|
ld1 {v24.4s, v25.4s}, [x28], #32
|
|
MLA_WEIGHTZERO v8, v2, v24, 0 // tile:0, oc:0-3
|
|
MLA_WEIGHTZERO v9, v2, v25, 0 // tile:0, oc:4-7
|
|
|
|
TILE1_ADD_DSTV:
|
|
cbz x19, TILE1_L8_ACCUM_BUFFER
|
|
ld1 {v10.4s, v11.4s}, [x10]
|
|
fadd v8.4s, v8.4s, v10.4s
|
|
fadd v9.4s, v9.4s, v11.4s
|
|
|
|
TILE1_L8_ACCUM_BUFFER:
|
|
add x19, x19, #1
|
|
cmp x19, x26
|
|
beq TILE1_POST
|
|
st1 {v8.4s, v9.4s}, [x10]
|
|
b TILE1_BLOCKNUM
|
|
|
|
TILE1_POST:
|
|
cbz x28, L8Tile1QuanUseInt8
|
|
sub x14, x14, #2
|
|
cbz x9, TILE1_RELU
|
|
ld1 {v10.4s, v11.4s}, [x20], #32
|
|
fadd v8.4s, v8.4s, v10.4s
|
|
fadd v9.4s, v9.4s, v11.4s
|
|
TILE1_RELU:
|
|
cbz x23, TILE1_STORE
|
|
fmin v8.4s, v8.4s, v27.4s
|
|
fmin v9.4s, v9.4s, v27.4s
|
|
fmax v8.4s, v8.4s, v26.4s
|
|
fmax v9.4s, v9.4s, v26.4s
|
|
|
|
TILE1_STORE:
|
|
st1 {v8.4s}, [x6], x4
|
|
st1 {v9.4s}, [x6], x4
|
|
cbz x14, TILE1_FLOAT_END
|
|
b L8Tile1LoopCheck
|
|
|
|
L8Tile1QuanUseInt8:
|
|
sub x14, x14, #2
|
|
ld1 {v10.4s, v11.4s}, [x20], #32
|
|
fadd v8.4s, v8.4s, v10.4s
|
|
fadd v9.4s, v9.4s, v11.4s
|
|
fcvtas v8.4s, v8.4s
|
|
fcvtas v9.4s, v9.4s
|
|
sqxtn v0.4h, v8.4s
|
|
sqxtn2 v0.8h, v9.4s
|
|
sqxtn v16.8b, v0.8h
|
|
smax v16.16b, v6.16b, v16.16b
|
|
smin v16.16b, v7.16b, v16.16b
|
|
st1 {v16.s}[0], [x6], x4
|
|
st1 {v16.s}[1], [x6], x4
|
|
cbz x14, TILE1_INT8_END
|
|
|
|
L8Tile1LoopCheck:
|
|
mov x8, x15
|
|
mov x24, x21 // revert input dequant scale
|
|
cbz x27, L8LoopDz_TILE_1
|
|
REVERT_INPUT_DEQUANT_BIAS x27, x19, x26, x22
|
|
b L8LoopDz_TILE_1
|
|
|
|
L4LoopDz_TILE_1:
|
|
mov x11, x1
|
|
mov x19, #0
|
|
L4_TILE1_BLOCKNUM:
|
|
mov x13, x3
|
|
movi v8.16b, #0
|
|
L4LoopSz_TILE_1:
|
|
ld1 {v3.16b}, [x12] // weight
|
|
ld1 {v0.s}[0], [x11], x22 // src
|
|
subs x13, x13, #1
|
|
add x12, x12, #32 // weight offset = lp*hp
|
|
.inst 0x4f80e068 // sdot v8.4s, v3.16b, v0.4b[0]
|
|
bne L4LoopSz_TILE_1
|
|
|
|
L4Tile1Quan:
|
|
ld1 {v0.4s}, [x12] // scale
|
|
add x12, x12, #32
|
|
ld1 {v2.s}[0], [x8], x22 // x kernel sum
|
|
ld1 {v24.4s}, [x12] // weight quan zeropoint
|
|
add x12, x12, #32
|
|
scvtf v8.4s, v8.4s
|
|
fmul v8.4s, v8.4s, v0.4s
|
|
|
|
cbz x21, TILE1_L4_MLA
|
|
ld1 {v4.s}[0], [x24], x25
|
|
fmul v8.4s, v8.4s, v4.s[0]
|
|
|
|
TILE1_L4_MLA:
|
|
MLA_WEIGHTZERO v8, v2, v24, 0 // tile:0, oc:0-3
|
|
|
|
cbz x27, L4_TILE1_ADD_DSTV
|
|
ld1 {v2.s}[0], [x27], x25
|
|
ld1 {v24.4s}, [x28]
|
|
MLA_WEIGHTZERO v8, v2, v24, 0 // tile:0, oc:0-3
|
|
add x28, x28, #32
|
|
|
|
L4_TILE1_ADD_DSTV:
|
|
cbz x19, L4_TILE1_L8_ACCUM_BUFFER
|
|
ld1 {v10.4s}, [x10]
|
|
fadd v8.4s, v8.4s, v10.4s
|
|
|
|
L4_TILE1_L8_ACCUM_BUFFER:
|
|
add x19, x19, #1
|
|
cmp x19, x26
|
|
beq L4_TILE1_POST
|
|
st1 {v8.4s}, [x10]
|
|
b L4_TILE1_BLOCKNUM
|
|
|
|
|
|
|
|
L4_TILE1_POST:
|
|
cbz x28, L4Tile1QuanUseInt8
|
|
cbz x9, L4_TILE1_RELU
|
|
ld1 {v10.4s}, [x20], #16
|
|
fadd v8.4s, v8.4s, v10.4s
|
|
L4_TILE1_RELU:
|
|
cbz x23, L4_TILE1_STORE
|
|
fmin v8.4s, v8.4s, v27.4s
|
|
fmax v8.4s, v8.4s, v26.4s
|
|
|
|
L4_TILE1_STORE:
|
|
st1 {v8.4s}, [x6], x4
|
|
b TILE1_FLOAT_END
|
|
|
|
L4Tile1QuanUseInt8:
|
|
ld1 {v4.4s}, [x20], #16
|
|
fadd v8.4s, v8.4s, v4.4s
|
|
fcvtas v8.4s, v8.4s
|
|
sqxtn v0.4h, v8.4s
|
|
sqxtn v16.8b, v0.8h
|
|
smax v16.8b, v6.8b, v16.8b
|
|
smin v16.8b, v7.8b, v16.8b
|
|
st1 {v16.s}[0], [x6], x4
|
|
b TILE1_INT8_END
|
|
|
|
TILE1_FLOAT_END:
|
|
subs x7, x7, #1
|
|
beq End
|
|
|
|
add x0, x0, #16
|
|
add x24, x21, #4
|
|
add x1, x1, #4
|
|
add x8, x15, #4
|
|
cbz x27, TILE_1
|
|
REVERT_INPUT_DEQUANT_BIAS x27, x19, x26, x22
|
|
REVERT_WEIGHT_KERNEL_SUM x28, x14, x26, x5
|
|
add x27, x27, #4
|
|
b TILE_1
|
|
|
|
TILE1_INT8_END:
|
|
subs x7, x7, #1
|
|
beq End
|
|
|
|
add x0, x0, #4
|
|
add x1, x1, #4
|
|
add x8, x15, #4
|
|
cbz x21, TILE_1
|
|
add x24, x21, #4
|
|
b TILE_1
|
|
|
|
End:
|
|
ldp x27, x28, [sp, #(16 * 8)]
|
|
ldp x25, x26, [sp, #(16 * 7)]
|
|
ldp x23, x24, [sp, #(16 * 6)]
|
|
ldp x19, x20, [sp, #(16 * 5)]
|
|
ldp x21, x22, [sp, #(16 * 4)]
|
|
ldp d8, d9, [sp, #(16 * 3)]
|
|
ldp d10, d11, [sp, #(16 * 2)]
|
|
ldp d12, d13, [sp, #(16 * 1)]
|
|
ldp d14, d15, [sp], #(16 * 10)
|
|
ret
|
|
|
|
#endif // __aarch64__
|