wehub-resource-sync
287 lines
10 KiB
Plaintext
287 lines
10 KiB
Plaintext
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
#include "paddle/phi/kernels/gpu/fused_token_prune_kernel.h"
|
|
#include <limits>
|
|
|
|
#include "paddle/phi/backends/gpu/gpu_context.h"
|
|
#include "paddle/phi/backends/gpu/gpu_launch_config.h"
|
|
#include "paddle/phi/common/data_type.h"
|
|
#include "paddle/phi/core/dense_tensor.h"
|
|
#include "paddle/phi/core/kernel_registry.h"
|
|
#include "paddle/phi/kernels/funcs/cub.h"
|
|
#include "paddle/phi/kernels/funcs/elementwise/elementwise_op_broadcast.cu.h"
|
|
#include "paddle/phi/kernels/funcs/fused_token_prune_utils.h"
|
|
|
|
namespace phi {
|
|
|
|
using SegmentOffsetIter = funcs::SegmentOffsetIter;
|
|
|
|
template <typename T>
|
|
struct AttnMaskFunctor {
|
|
inline HOSTDEVICE T operator()(const T a, const T b) const {
|
|
return b >= 0 ? a : 0;
|
|
}
|
|
};
|
|
|
|
__global__ void FillIndex(int64_t* indices, int num_raws, int num_cols) {
|
|
int64_t num_threads = static_cast<int64_t>(num_raws) * num_cols;
|
|
int64_t tid =
|
|
static_cast<int64_t>(threadIdx.x) +
|
|
static_cast<int64_t>(blockIdx.x) * static_cast<int64_t>(blockDim.x);
|
|
int stride = blockDim.x * gridDim.x;
|
|
|
|
for (; tid < num_threads; tid += stride) {
|
|
int col = tid % num_cols;
|
|
indices[tid] = static_cast<int64_t>(col);
|
|
}
|
|
}
|
|
|
|
template <typename T>
|
|
__global__ void TakeAlongAxis(const T* src,
|
|
T* dst,
|
|
int64_t* indices,
|
|
int num_raws,
|
|
int src_num_cols,
|
|
int dst_num_cols,
|
|
int num_elements) {
|
|
int64_t num_threads = static_cast<int64_t>(num_raws) * dst_num_cols;
|
|
int64_t tid =
|
|
static_cast<int64_t>(threadIdx.x) +
|
|
static_cast<int64_t>(blockIdx.x) * static_cast<int64_t>(blockDim.x);
|
|
int stride = blockDim.x * gridDim.x;
|
|
|
|
for (; tid < num_threads; tid += stride) {
|
|
int raw = tid / dst_num_cols;
|
|
int col = tid % dst_num_cols;
|
|
for (int i = 0; i < num_elements; ++i) {
|
|
dst[tid * num_elements + i] =
|
|
*(src + (raw * src_num_cols + indices[tid]) * num_elements + i);
|
|
}
|
|
}
|
|
}
|
|
|
|
template <typename T>
|
|
__global__ void MaximumFirst(T* mat, int num_raws, int num_cols, T max_value) {
|
|
int num_threads = num_raws;
|
|
int64_t tid =
|
|
static_cast<int64_t>(blockIdx.x) * static_cast<int64_t>(blockDim.x) +
|
|
static_cast<int64_t>(threadIdx.x);
|
|
int stride = blockDim.x * gridDim.x;
|
|
|
|
for (; tid < num_threads; tid += stride) {
|
|
mat[tid * num_cols] = max_value;
|
|
}
|
|
}
|
|
|
|
template <typename T, typename Context>
|
|
void FusedTokenPruneOpCUDAKernel(const Context& dev_ctx,
|
|
const DenseTensor& attn,
|
|
const DenseTensor& x,
|
|
const DenseTensor& mask,
|
|
const DenseTensor& new_mask,
|
|
bool keep_first_token,
|
|
bool keep_order,
|
|
DenseTensor* slimmed_x,
|
|
DenseTensor* cls_inds) {
|
|
// Input dims
|
|
auto attn_dims = attn.dims();
|
|
auto x_dims = x.dims();
|
|
auto new_mask_dims = new_mask.dims();
|
|
|
|
auto bsz = attn_dims[0];
|
|
auto num_heads = attn_dims[1];
|
|
auto max_seq_len = attn_dims[2];
|
|
auto c = x_dims[2];
|
|
PADDLE_ENFORCE_LE_INT_MAX(new_mask_dims[2], "slimmed_x_len");
|
|
int slimmed_x_len = static_cast<int>(new_mask_dims[2]);
|
|
|
|
// Outputs
|
|
DenseTensor* out_slimmed_x = slimmed_x;
|
|
DenseTensor* slimmed_indices = cls_inds;
|
|
auto* out_slimmed_x_data = dev_ctx.template Alloc<T>(out_slimmed_x);
|
|
auto* slimmed_indices_data = dev_ctx.template Alloc<int64_t>(slimmed_indices);
|
|
|
|
// Intermediate variable
|
|
DenseTensor attn_tmp;
|
|
attn_tmp.Resize(attn_dims);
|
|
auto* attn_tmp_data = dev_ctx.template Alloc<T>(&attn_tmp);
|
|
DenseTensor attn_accu;
|
|
attn_accu.Resize({bsz, max_seq_len});
|
|
auto* attn_accu_data = dev_ctx.template Alloc<T>(&attn_accu);
|
|
DenseTensor attn_accu_indices;
|
|
attn_accu_indices.Resize({bsz, max_seq_len});
|
|
auto* attn_accu_indices_data =
|
|
dev_ctx.template Alloc<int64_t>(&attn_accu_indices);
|
|
DenseTensor sort_attn_accu;
|
|
sort_attn_accu.Resize({bsz, max_seq_len});
|
|
auto* sort_attn_accu_data = dev_ctx.template Alloc<T>(&sort_attn_accu);
|
|
DenseTensor sort_attn_accu_indices;
|
|
sort_attn_accu_indices.Resize({bsz, max_seq_len});
|
|
auto* sort_attn_accu_indices_data =
|
|
dev_ctx.template Alloc<int64_t>(&sort_attn_accu_indices);
|
|
|
|
DenseTensor temp_storage;
|
|
|
|
// 1. Filter attn by mask
|
|
std::vector<const DenseTensor*> ins;
|
|
std::vector<DenseTensor*> outs;
|
|
ins.emplace_back(&attn);
|
|
ins.emplace_back(&mask);
|
|
outs.emplace_back(&attn_tmp);
|
|
funcs::LaunchElementwiseCudaKernel<T>(
|
|
dev_ctx, ins, &outs, AttnMaskFunctor<T>());
|
|
|
|
// 2. Reduce sum
|
|
const std::vector<int64_t> reduce_dims{1, 2};
|
|
Reduce<T, kps::SumOps>(
|
|
dev_ctx, attn_tmp, false, reduce_dims, attn_accu.dtype(), &attn_accu);
|
|
|
|
// 3. Prepare token indices
|
|
backends::gpu::GpuLaunchConfig config =
|
|
backends::gpu::GetGpuLaunchConfig1D(dev_ctx, bsz * max_seq_len);
|
|
FillIndex<<<config.block_per_grid,
|
|
config.thread_per_block,
|
|
0,
|
|
dev_ctx.stream()>>>(attn_accu_indices_data, bsz, max_seq_len);
|
|
|
|
// 4. Sort token indices by attn
|
|
if (keep_first_token) {
|
|
T max = std::numeric_limits<T>::max();
|
|
config = backends::gpu::GetGpuLaunchConfig1D(dev_ctx, bsz);
|
|
MaximumFirst<T>
|
|
<<<config.block_per_grid,
|
|
config.thread_per_block,
|
|
0,
|
|
dev_ctx.stream()>>>(attn_accu_data, bsz, max_seq_len, max);
|
|
}
|
|
size_t temp_storage_bytes = -1;
|
|
PADDLE_ENFORCE_LE_INT_MAX(bsz * max_seq_len, "bsz * max_seq_len");
|
|
int num_items = static_cast<int>(bsz * max_seq_len);
|
|
PADDLE_ENFORCE_LE_INT_MAX(bsz, "bsz");
|
|
int num_segments = static_cast<int>(bsz);
|
|
|
|
cub::CountingInputIterator<int64_t> counting_iter(0);
|
|
cub::TransformInputIterator<int64_t,
|
|
SegmentOffsetIter,
|
|
cub::CountingInputIterator<int64_t>>
|
|
segment_offsets_t(counting_iter, SegmentOffsetIter(max_seq_len));
|
|
// Determine temporary device storage requirements
|
|
PADDLE_ENFORCE_GPU_SUCCESS(cub::DeviceSegmentedRadixSort::SortPairsDescending(
|
|
nullptr,
|
|
temp_storage_bytes,
|
|
attn_accu_data,
|
|
sort_attn_accu_data,
|
|
attn_accu_indices_data,
|
|
sort_attn_accu_indices_data,
|
|
num_items,
|
|
num_segments,
|
|
segment_offsets_t,
|
|
segment_offsets_t + 1,
|
|
0,
|
|
sizeof(T) * 8,
|
|
dev_ctx.stream()));
|
|
// Allocate temporary storage
|
|
int64_t temp_size = temp_storage_bytes;
|
|
temp_storage.Resize({temp_size});
|
|
auto* temp_storage_data = dev_ctx.template Alloc<uint8_t>(&temp_storage);
|
|
|
|
// Run sorting operation
|
|
PADDLE_ENFORCE_GPU_SUCCESS(cub::DeviceSegmentedRadixSort::SortPairsDescending(
|
|
temp_storage_data,
|
|
temp_storage_bytes,
|
|
attn_accu_data,
|
|
sort_attn_accu_data,
|
|
attn_accu_indices_data,
|
|
sort_attn_accu_indices_data,
|
|
num_items,
|
|
num_segments,
|
|
segment_offsets_t,
|
|
segment_offsets_t + 1,
|
|
0,
|
|
sizeof(T) * 8,
|
|
dev_ctx.stream()));
|
|
// 5. Slice
|
|
auto slimmed_indices_tmp = funcs::Slice<int64_t>(dev_ctx,
|
|
sort_attn_accu_indices,
|
|
{1} /*axes*/,
|
|
{0} /*starts*/,
|
|
{slimmed_x_len} /*ends*/);
|
|
if (keep_order) {
|
|
// 6. reorder
|
|
PADDLE_ENFORCE_LE_INT_MAX(bsz * slimmed_x_len, "bsz * slimmed_x_len");
|
|
num_items = static_cast<int>(bsz * slimmed_x_len);
|
|
temp_storage_bytes = -1;
|
|
cub::TransformInputIterator<int64_t,
|
|
SegmentOffsetIter,
|
|
cub::CountingInputIterator<int64_t>>
|
|
segment_offsets_t2(counting_iter, SegmentOffsetIter(slimmed_x_len));
|
|
PADDLE_ENFORCE_GPU_SUCCESS(cub::DeviceSegmentedRadixSort::SortKeys(
|
|
nullptr,
|
|
temp_storage_bytes,
|
|
static_cast<int64_t*>(slimmed_indices_tmp.data()),
|
|
static_cast<int64_t*>(slimmed_indices->data()),
|
|
num_items,
|
|
num_segments,
|
|
segment_offsets_t2,
|
|
segment_offsets_t2 + 1,
|
|
0,
|
|
sizeof(int64_t) * 8,
|
|
dev_ctx.stream()));
|
|
temp_size = temp_storage_bytes;
|
|
temp_storage.Resize({temp_size});
|
|
temp_storage_data = dev_ctx.template Alloc<uint8_t>(&temp_storage);
|
|
PADDLE_ENFORCE_GPU_SUCCESS(cub::DeviceSegmentedRadixSort::SortKeys(
|
|
temp_storage_data,
|
|
temp_storage_bytes,
|
|
static_cast<int64_t*>(slimmed_indices_tmp.data()),
|
|
static_cast<int64_t*>(slimmed_indices->data()),
|
|
num_items,
|
|
num_segments,
|
|
segment_offsets_t2,
|
|
segment_offsets_t2 + 1,
|
|
0,
|
|
sizeof(int64_t) * 8,
|
|
dev_ctx.stream()));
|
|
} else {
|
|
Copy(dev_ctx,
|
|
slimmed_indices_tmp,
|
|
dev_ctx.GetPlace(),
|
|
false,
|
|
slimmed_indices);
|
|
}
|
|
// 7. Get slimmed X by indices
|
|
config = backends::gpu::GetGpuLaunchConfig1D(dev_ctx, bsz * slimmed_x_len);
|
|
TakeAlongAxis<T>
|
|
<<<config.block_per_grid, config.thread_per_block, 0, dev_ctx.stream()>>>(
|
|
x.data<T>(),
|
|
out_slimmed_x_data,
|
|
slimmed_indices->data<int64_t>(),
|
|
bsz,
|
|
max_seq_len,
|
|
slimmed_x_len,
|
|
c);
|
|
}
|
|
|
|
} // namespace phi
|
|
|
|
PD_REGISTER_KERNEL(fused_token_prune,
|
|
GPU,
|
|
ALL_LAYOUT,
|
|
phi::FusedTokenPruneOpCUDAKernel,
|
|
float,
|
|
double) {
|
|
kernel->OutputAt(1).SetDataType(phi::DataType::INT64);
|
|
}
|