wehub-resource-sync
306 lines
10 KiB
Plaintext
306 lines
10 KiB
Plaintext
// Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
#include "paddle/phi/kernels/min_max_with_index_kernel.h"
|
|
|
|
#include "paddle/phi/backends/gpu/gpu_context.h"
|
|
#include "paddle/phi/core/kernel_registry.h"
|
|
|
|
#if defined(__NVCC__) || defined(__HIPCC__)
|
|
#include <limits>
|
|
|
|
#include "paddle/common/ddim.h"
|
|
#include "paddle/phi/core/utils/data_type.h"
|
|
#include "paddle/phi/kernels/funcs/cub.h"
|
|
#include "paddle/phi/kernels/funcs/math_function.h"
|
|
namespace phi {
|
|
|
|
namespace { // NOLINT
|
|
template <typename K, typename V>
|
|
using KeyValuePair = cub::KeyValuePair<K, V>;
|
|
|
|
} // namespace
|
|
|
|
#define FIXED_BLOCK_DIM_CASE_BASE(log2_block_dim, ...) \
|
|
case (1 << (log2_block_dim)): { \
|
|
constexpr auto kBlockDim = (1 << (log2_block_dim)); \
|
|
__VA_ARGS__; \
|
|
} break
|
|
|
|
#define FIXED_BLOCK_DIM_CASE(...) \
|
|
FIXED_BLOCK_DIM_CASE_BASE(10, ##__VA_ARGS__); \
|
|
FIXED_BLOCK_DIM_CASE_BASE(9, ##__VA_ARGS__); \
|
|
FIXED_BLOCK_DIM_CASE_BASE(8, ##__VA_ARGS__); \
|
|
FIXED_BLOCK_DIM_CASE_BASE(7, ##__VA_ARGS__); \
|
|
FIXED_BLOCK_DIM_CASE_BASE(6, ##__VA_ARGS__); \
|
|
FIXED_BLOCK_DIM_CASE_BASE(5, ##__VA_ARGS__); \
|
|
FIXED_BLOCK_DIM_CASE_BASE(4, ##__VA_ARGS__); \
|
|
FIXED_BLOCK_DIM_CASE_BASE(3, ##__VA_ARGS__);
|
|
|
|
template <typename T,
|
|
typename IndType,
|
|
class Reducer,
|
|
size_t BlockDim,
|
|
typename IndexType>
|
|
__global__ void MinMaxWithIndexKernel(const int64_t height, // n * h
|
|
const int64_t width, // c
|
|
const int64_t post_size, // h
|
|
const Reducer reducer,
|
|
const T init,
|
|
const T* in,
|
|
T* val_out,
|
|
IndType* key_out) {
|
|
typedef cub::BlockReduce<KeyValuePair<IndexType, T>, BlockDim> BlockReduce;
|
|
__shared__ typename BlockReduce::TempStorage temp_storage;
|
|
|
|
for (IndexType idx = blockIdx.x; idx < height; idx += gridDim.x) {
|
|
KeyValuePair<IndexType, T> kv_pair = {-1, init};
|
|
IndexType h = idx / post_size;
|
|
IndexType w = idx % post_size;
|
|
for (IndexType k = threadIdx.x; k < width; k += blockDim.x) {
|
|
kv_pair =
|
|
reducer({k, in[h * width * post_size + k * post_size + w]}, kv_pair);
|
|
}
|
|
kv_pair = BlockReduce(temp_storage).Reduce(kv_pair, reducer);
|
|
if (threadIdx.x == 0) {
|
|
val_out[idx] = static_cast<T>(kv_pair.value);
|
|
key_out[idx] = static_cast<IndType>(kv_pair.key);
|
|
}
|
|
__syncthreads();
|
|
}
|
|
}
|
|
|
|
template <typename T, typename IndType, class Reducer, typename IndexType>
|
|
void ComputeMinMaxWithIndex(const GPUContext& dev_ctx,
|
|
const DenseTensor& input,
|
|
DenseTensor* values,
|
|
DenseTensor* indices,
|
|
const int64_t pre,
|
|
const int64_t post,
|
|
const int64_t n) {
|
|
auto cu_stream = dev_ctx.stream();
|
|
auto ComputeBlockSize = [](int64_t col) {
|
|
auto block_size = 8;
|
|
if (col > 512)
|
|
block_size = 1024;
|
|
else if (col > 256)
|
|
block_size = 512;
|
|
else if (col > 128)
|
|
block_size = 256;
|
|
else if (col > 64)
|
|
block_size = 128;
|
|
else if (col > 32)
|
|
block_size = 64;
|
|
else if (col > 16)
|
|
block_size = 32;
|
|
else if (col > 8)
|
|
block_size = 16;
|
|
return block_size;
|
|
};
|
|
|
|
int64_t max_grid_dimx = dev_ctx.GetCUDAMaxGridDimSize()[0];
|
|
int64_t height = pre * post;
|
|
int64_t width = n;
|
|
int64_t grid_size = height < max_grid_dimx ? height : max_grid_dimx;
|
|
|
|
const T* in_data = input.data<T>();
|
|
|
|
T* val_data = dev_ctx.template Alloc<T>(values);
|
|
IndType* ind_data = dev_ctx.template Alloc<IndType>(indices);
|
|
|
|
if (typeid(Reducer) == typeid(cub::ArgMax)) {
|
|
switch (ComputeBlockSize(width)) {
|
|
FIXED_BLOCK_DIM_CASE(
|
|
MinMaxWithIndexKernel<T, IndType, Reducer, kBlockDim, IndexType>
|
|
<<<grid_size, kBlockDim, 0, cu_stream>>>(
|
|
height,
|
|
width,
|
|
post,
|
|
Reducer(),
|
|
std::numeric_limits<T>::lowest(),
|
|
in_data,
|
|
val_data,
|
|
ind_data));
|
|
}
|
|
} else {
|
|
switch (ComputeBlockSize(width)) {
|
|
FIXED_BLOCK_DIM_CASE(
|
|
MinMaxWithIndexKernel<T, IndType, Reducer, kBlockDim, IndexType>
|
|
<<<grid_size, kBlockDim, 0, cu_stream>>>(
|
|
height,
|
|
width,
|
|
post,
|
|
Reducer(),
|
|
std::numeric_limits<T>::max(),
|
|
in_data,
|
|
val_data,
|
|
ind_data));
|
|
}
|
|
}
|
|
}
|
|
|
|
template <typename Context, typename T, class Reducer>
|
|
struct VisitDataCudaMinMaxWithIndexFunctor {
|
|
const Context& dev_ctx;
|
|
const DenseTensor& x;
|
|
int64_t axis;
|
|
bool keepdims;
|
|
bool flatten;
|
|
DenseTensor* val_out;
|
|
DenseTensor* ind_out;
|
|
|
|
explicit VisitDataCudaMinMaxWithIndexFunctor(const Context& dev_ctx,
|
|
const DenseTensor& x,
|
|
int64_t axis,
|
|
bool keepdims,
|
|
bool flatten,
|
|
DenseTensor* val_out,
|
|
DenseTensor* ind_out)
|
|
: dev_ctx(dev_ctx),
|
|
x(x),
|
|
axis(axis),
|
|
keepdims(keepdims),
|
|
flatten(flatten),
|
|
val_out(val_out),
|
|
ind_out(ind_out) {}
|
|
|
|
template <typename IndType>
|
|
void apply() const {
|
|
DDim x_dims;
|
|
int new_axis = axis;
|
|
if (flatten) {
|
|
x_dims = make_ddim({x.numel()});
|
|
// if flatten, the axis just as 0
|
|
new_axis = 0;
|
|
} else {
|
|
x_dims = x.dims();
|
|
if (axis < 0) new_axis = axis + x.dims().size();
|
|
}
|
|
if (x.numel() == 0) {
|
|
dev_ctx.template Alloc<T>(val_out);
|
|
dev_ctx.template Alloc<IndType>(ind_out);
|
|
return;
|
|
}
|
|
// For 0D Tensor
|
|
if (x.dims().size() == 0) {
|
|
dev_ctx.template Alloc<T>(val_out);
|
|
dev_ctx.template Alloc<IndType>(ind_out);
|
|
funcs::set_constant(dev_ctx, ind_out, static_cast<IndType>(0));
|
|
Copy<Context>(dev_ctx, x, dev_ctx.GetPlace(), false, val_out);
|
|
return;
|
|
}
|
|
|
|
int64_t numel = x.numel();
|
|
int64_t groups = numel / x_dims[new_axis];
|
|
int64_t pre = 1;
|
|
int64_t post = 1;
|
|
int64_t n = x_dims[new_axis];
|
|
|
|
for (int i = 0; i < new_axis; i++) {
|
|
pre *= x_dims[i];
|
|
}
|
|
|
|
for (int i = new_axis + 1; i < x_dims.size(); i++) {
|
|
post *= x_dims[i];
|
|
}
|
|
|
|
if (numel > std::numeric_limits<int32_t>::max()) {
|
|
ComputeMinMaxWithIndex<T, IndType, Reducer, int64_t>(
|
|
dev_ctx, x, val_out, ind_out, pre, post, n);
|
|
} else {
|
|
ComputeMinMaxWithIndex<T, IndType, Reducer, int32_t>(
|
|
dev_ctx, x, val_out, ind_out, pre, post, n);
|
|
}
|
|
}
|
|
};
|
|
|
|
template <typename Context, typename T, class Reducer>
|
|
void MinMaxWithIndexOpCUDAKernel(const Context& dev_ctx,
|
|
const DenseTensor& x,
|
|
const Scalar& axis,
|
|
bool keepdims,
|
|
bool flatten,
|
|
DenseTensor* val_out,
|
|
DenseTensor* ind_out) {
|
|
PADDLE_ENFORCE_GE(
|
|
x.numel(),
|
|
0,
|
|
common::errors::InvalidArgument(
|
|
"(min/max)_with_index input numel must > 0, bug got %d", x.numel()));
|
|
phi::VisitDataTypeTiny(
|
|
DataType::INT64,
|
|
VisitDataCudaMinMaxWithIndexFunctor<Context, T, Reducer>(
|
|
dev_ctx, x, axis.to<int64_t>(), keepdims, flatten, val_out, ind_out));
|
|
}
|
|
|
|
template <typename T, typename Context>
|
|
void MinWithIndexKernel(const Context& dev_ctx,
|
|
const DenseTensor& x,
|
|
const Scalar& dim,
|
|
bool keepdim,
|
|
bool flatten,
|
|
DenseTensor* val_out,
|
|
DenseTensor* ind_out) {
|
|
MinMaxWithIndexOpCUDAKernel<Context, T, cub::ArgMin>(
|
|
dev_ctx, x, dim, keepdim, flatten, val_out, ind_out);
|
|
}
|
|
|
|
template <typename T, typename Context>
|
|
void MaxWithIndexKernel(const Context& dev_ctx,
|
|
const DenseTensor& x,
|
|
const Scalar& dim,
|
|
bool keepdim,
|
|
bool flatten,
|
|
DenseTensor* val_out,
|
|
DenseTensor* ind_out) {
|
|
MinMaxWithIndexOpCUDAKernel<Context, T, cub::ArgMax>(
|
|
dev_ctx, x, dim, keepdim, flatten, val_out, ind_out);
|
|
}
|
|
|
|
#endif
|
|
|
|
} // namespace phi
|
|
|
|
PD_REGISTER_KERNEL(min_with_index,
|
|
GPU,
|
|
ALL_LAYOUT,
|
|
phi::MinWithIndexKernel,
|
|
phi::float16,
|
|
phi::bfloat16,
|
|
float,
|
|
double,
|
|
int32_t,
|
|
int64_t,
|
|
int16_t,
|
|
uint8_t) {
|
|
kernel->OutputAt(0).SetDataType(kernel->InputAt(0).dtype);
|
|
kernel->OutputAt(1).SetDataType(phi::DataType::INT64);
|
|
}
|
|
|
|
PD_REGISTER_KERNEL(max_with_index,
|
|
GPU,
|
|
ALL_LAYOUT,
|
|
phi::MaxWithIndexKernel,
|
|
phi::float16,
|
|
phi::bfloat16,
|
|
float,
|
|
double,
|
|
int32_t,
|
|
int64_t,
|
|
int16_t,
|
|
uint8_t) {
|
|
kernel->OutputAt(0).SetDataType(kernel->InputAt(0).dtype);
|
|
kernel->OutputAt(1).SetDataType(phi::DataType::INT64);
|
|
}
|