chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,296 @@
|
||||
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include "paddle/phi/kernels/sparse/matmul_kernel.h"
|
||||
#include "paddle/common/ddim.h"
|
||||
#include "paddle/phi/backends/gpu/gpu_context.h"
|
||||
#include "paddle/phi/core/enforce.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/core/meta_tensor.h"
|
||||
#include "paddle/phi/core/sparse_coo_tensor.h"
|
||||
#include "paddle/phi/core/sparse_csr_tensor.h"
|
||||
#include "paddle/phi/core/tensor_utils.h"
|
||||
#include "paddle/phi/kernels/empty_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function_impl.h"
|
||||
#include "paddle/phi/kernels/funcs/sparse/sparse_blas.h"
|
||||
#include "paddle/phi/kernels/sparse/empty_kernel.h"
|
||||
#include "paddle/phi/kernels/sparse/impl/unary_kernel_impl.h"
|
||||
#include "paddle/phi/kernels/sparse/sparse_utils_kernel.h"
|
||||
|
||||
namespace phi {
|
||||
namespace sparse {
|
||||
|
||||
template <typename T, typename Context, typename TensorType>
|
||||
void MatmulKernelImpl(const Context& dev_ctx,
|
||||
const TensorType& x,
|
||||
const DenseTensor& y,
|
||||
DenseTensor* out) {
|
||||
#if defined(PADDLE_WITH_CUDA) || HIP_VERSION >= 402
|
||||
std::vector<int64_t> xdim_vec = vectorize(x.dims());
|
||||
std::vector<int64_t> ydim_vec = vectorize(y.dims());
|
||||
auto x_ndims = xdim_vec.size();
|
||||
auto y_ndims = ydim_vec.size();
|
||||
PADDLE_ENFORCE_EQ(x_ndims,
|
||||
y_ndims,
|
||||
common::errors::PreconditionNotMet(
|
||||
"The dims size of Input(x) and Input(y) "
|
||||
"should be equal, But received X's "
|
||||
"dimensions=%d, Y's dimensions=%d.",
|
||||
x_ndims,
|
||||
y_ndims));
|
||||
PADDLE_ENFORCE_GE(
|
||||
x_ndims,
|
||||
2,
|
||||
common::errors::InvalidArgument("the dims size of Input(x) and "
|
||||
"Input(y) must be greater than "
|
||||
"or equal to 2."));
|
||||
|
||||
for (size_t i = 0; i < x_ndims - 2; ++i) {
|
||||
PADDLE_ENFORCE_EQ(xdim_vec[i],
|
||||
ydim_vec[i],
|
||||
common::errors::InvalidArgument(
|
||||
"x.dim[%d] and x.dim[%d] must be equal.", i, i));
|
||||
}
|
||||
|
||||
PADDLE_ENFORCE_GE(
|
||||
xdim_vec[x_ndims - 1],
|
||||
ydim_vec[y_ndims - 2],
|
||||
common::errors::PreconditionNotMet(
|
||||
"The shape of Input(x) and Input(y) is not suitable for matmul "
|
||||
"operation, x_dim[-1] must be equal to y_dim[-2]."));
|
||||
|
||||
// InferMeta of DenseTensor 'out'
|
||||
std::vector<int64_t> out_dim_vec(ydim_vec);
|
||||
out_dim_vec[y_ndims - 2] = xdim_vec[x_ndims - 2];
|
||||
out_dim_vec[y_ndims - 1] = ydim_vec[y_ndims - 1];
|
||||
MetaTensor meta_out(out);
|
||||
meta_out.set_dims(make_ddim(out_dim_vec));
|
||||
meta_out.set_dtype(y.dtype());
|
||||
// Ensure the output DenseTensor has a proper dense layout, not sparse layout
|
||||
meta_out.set_layout(DataLayout::NCHW);
|
||||
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
|
||||
#ifdef PADDLE_WITH_HIP
|
||||
funcs::SetConstant<Context, T> set_zero;
|
||||
set_zero(dev_ctx, out, static_cast<T>(0.0f));
|
||||
#endif
|
||||
|
||||
auto sparse_blas = funcs::sparse::GetSparseBlas<Context, T>(dev_ctx);
|
||||
sparse_blas.SPMM(
|
||||
false, false, static_cast<T>(1), x, y, static_cast<T>(0), out);
|
||||
#endif
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void MatmulCooDenseKernel(const Context& dev_ctx,
|
||||
const SparseCooTensor& x,
|
||||
const DenseTensor& y,
|
||||
DenseTensor* out) {
|
||||
MatmulKernelImpl<T>(dev_ctx, x, y, out);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void MatmulCsrDenseKernel(const Context& dev_ctx,
|
||||
const SparseCsrTensor& x,
|
||||
const DenseTensor& y,
|
||||
DenseTensor* out) {
|
||||
MatmulKernelImpl<T>(dev_ctx, x, y, out);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void MatmulCsrCsrKernel(const Context& dev_ctx,
|
||||
const SparseCsrTensor& x,
|
||||
const SparseCsrTensor& y,
|
||||
SparseCsrTensor* out) {
|
||||
#if defined(PADDLE_WITH_CUDA)
|
||||
std::vector<int64_t> xdim_vec = vectorize(x.dims());
|
||||
std::vector<int64_t> ydim_vec = vectorize(y.dims());
|
||||
auto x_ndims = xdim_vec.size();
|
||||
auto y_ndims = ydim_vec.size();
|
||||
PADDLE_ENFORCE_EQ(x_ndims,
|
||||
y_ndims,
|
||||
common::errors::PreconditionNotMet(
|
||||
"The dims size of Input(x) and Input(y) "
|
||||
"should be equal, But received X's "
|
||||
"dimensions=%d, Y's dimensions=%d.",
|
||||
x_ndims,
|
||||
y_ndims));
|
||||
PADDLE_ENFORCE_GE(
|
||||
x_ndims,
|
||||
2,
|
||||
common::errors::InvalidArgument("the dims size of Input(x) and "
|
||||
"Input(y) must be greater than "
|
||||
"or equal to 2."));
|
||||
|
||||
for (size_t i = 0; i < x_ndims - 2; ++i) {
|
||||
PADDLE_ENFORCE_EQ(xdim_vec[i],
|
||||
ydim_vec[i],
|
||||
common::errors::InvalidArgument(
|
||||
"x.dim[%d] and x.dim[%d] must be equal.", i, i));
|
||||
}
|
||||
|
||||
PADDLE_ENFORCE_GE(
|
||||
xdim_vec[x_ndims - 1],
|
||||
ydim_vec[y_ndims - 2],
|
||||
common::errors::PreconditionNotMet(
|
||||
"The shape of Input(x) and Input(y) is not suitable for matmul "
|
||||
"operation, x_dim[-1] must be equal to y_dim[-2]."));
|
||||
|
||||
auto sparse_blas = funcs::sparse::GetSparseBlas<Context, T>(dev_ctx);
|
||||
sparse_blas.SPGEMM(
|
||||
false, false, static_cast<T>(1), x, y, static_cast<T>(0), out);
|
||||
#endif
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void MatmulCooCooKernel(const Context& dev_ctx,
|
||||
const SparseCooTensor& x,
|
||||
const SparseCooTensor& y,
|
||||
SparseCooTensor* out) {
|
||||
// 'cusparseSpGEMM' only support CSR now, so use COO->CSR->COO.
|
||||
SparseCsrTensor x_csr = CooToCsr<T, Context>(dev_ctx, x);
|
||||
SparseCsrTensor y_csr = CooToCsr<T, Context>(dev_ctx, y);
|
||||
SparseCsrTensor out_csr;
|
||||
out_csr.set_dims(out->dims());
|
||||
MatmulCsrCsrKernel<T>(dev_ctx, x_csr, y_csr, &out_csr);
|
||||
CsrToCooKernel<T>(dev_ctx, out_csr, out);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void MaskedMatmulCsrKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& y,
|
||||
const SparseCsrTensor& mask,
|
||||
SparseCsrTensor* out) {
|
||||
#if defined(PADDLE_WITH_CUDA)
|
||||
std::vector<int64_t> xdim_vec = vectorize(x.dims());
|
||||
std::vector<int64_t> ydim_vec = vectorize(y.dims());
|
||||
std::vector<int64_t> maskdim_vec = vectorize(mask.dims());
|
||||
|
||||
auto x_ndims = xdim_vec.size();
|
||||
auto y_ndims = ydim_vec.size();
|
||||
auto mask_ndims = maskdim_vec.size();
|
||||
|
||||
PADDLE_ENFORCE_EQ(x_ndims,
|
||||
y_ndims,
|
||||
common::errors::PreconditionNotMet(
|
||||
"The dims size of Input(x) and Input(y) "
|
||||
"should be equal, But received X's "
|
||||
"dimensions=%d, Y's dimensions=%d.",
|
||||
x_ndims,
|
||||
y_ndims));
|
||||
PADDLE_ENFORCE_EQ(x_ndims,
|
||||
mask_ndims,
|
||||
common::errors::PreconditionNotMet(
|
||||
"The dims size of Input(x) and Input(mask) "
|
||||
"should be equal, But received X's "
|
||||
"dimensions=%d, mask's dimensions=%d.",
|
||||
x_ndims,
|
||||
mask_ndims));
|
||||
PADDLE_ENFORCE_GE(
|
||||
x_ndims,
|
||||
2,
|
||||
common::errors::InvalidArgument("the dims size of Input(x) and "
|
||||
"Input(y) must be greater than "
|
||||
"or equal to 2."));
|
||||
|
||||
for (size_t i = 0; i < x_ndims - 2; ++i) {
|
||||
PADDLE_ENFORCE_EQ(xdim_vec[i],
|
||||
ydim_vec[i],
|
||||
common::errors::InvalidArgument(
|
||||
"x.dim[%d] and x.dim[%d] must match.", i, i));
|
||||
PADDLE_ENFORCE_EQ(xdim_vec[i],
|
||||
maskdim_vec[i],
|
||||
common::errors::InvalidArgument(
|
||||
"x.dim[%d] and mask.dim[%d] must match.", i, i));
|
||||
}
|
||||
|
||||
PADDLE_ENFORCE_GE(
|
||||
xdim_vec[x_ndims - 1],
|
||||
ydim_vec[y_ndims - 2],
|
||||
common::errors::PreconditionNotMet(
|
||||
"The shape of Input(x) and Input(y) is not suitable for matmul "
|
||||
"operation, x_dim[-1] must be equal to y_dim[-2]."));
|
||||
|
||||
PADDLE_ENFORCE_EQ(
|
||||
maskdim_vec[mask_ndims - 2],
|
||||
xdim_vec[x_ndims - 2],
|
||||
common::errors::PreconditionNotMet(
|
||||
"The shape of Input(x) and Input(y) is not suitable for matmul "
|
||||
"operation, mask_dim[-2] must be equal to x_dim[-2]."));
|
||||
|
||||
PADDLE_ENFORCE_EQ(
|
||||
maskdim_vec[mask_ndims - 1],
|
||||
ydim_vec[y_ndims - 1],
|
||||
common::errors::PreconditionNotMet(
|
||||
"The shape of Input(x) and Input(y) is not suitable for matmul "
|
||||
"operation, mask_dim[-1] must be equal to y_dim[-1]."));
|
||||
|
||||
// InferMeta of SparseCsrTensor 'out', CreateLikeInferMeta
|
||||
EmptyLikeCsrKernel<T, Context>(dev_ctx, mask, out);
|
||||
|
||||
auto sparse_blas = funcs::sparse::GetSparseBlas<Context, T>(dev_ctx);
|
||||
sparse_blas.SDDMM(
|
||||
false, false, static_cast<T>(1), x, y, static_cast<T>(0), out);
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace sparse
|
||||
} // namespace phi
|
||||
|
||||
PD_REGISTER_KERNEL(matmul_csr_dense,
|
||||
GPU,
|
||||
ALL_LAYOUT,
|
||||
phi::sparse::MatmulCsrDenseKernel,
|
||||
float,
|
||||
double) {
|
||||
kernel->InputAt(0).SetDataLayout(phi::DataLayout::SPARSE_CSR);
|
||||
}
|
||||
|
||||
PD_REGISTER_KERNEL(matmul_coo_dense,
|
||||
GPU,
|
||||
ALL_LAYOUT,
|
||||
phi::sparse::MatmulCooDenseKernel,
|
||||
float,
|
||||
double) {
|
||||
kernel->InputAt(0).SetDataLayout(phi::DataLayout::SPARSE_COO);
|
||||
}
|
||||
|
||||
PD_REGISTER_KERNEL(matmul_coo_coo,
|
||||
GPU,
|
||||
ALL_LAYOUT,
|
||||
phi::sparse::MatmulCooCooKernel,
|
||||
float,
|
||||
double) {
|
||||
kernel->InputAt(0).SetDataLayout(phi::DataLayout::SPARSE_COO);
|
||||
kernel->InputAt(1).SetDataLayout(phi::DataLayout::SPARSE_COO);
|
||||
}
|
||||
|
||||
PD_REGISTER_KERNEL(matmul_csr_csr,
|
||||
GPU,
|
||||
ALL_LAYOUT,
|
||||
phi::sparse::MatmulCsrCsrKernel,
|
||||
float,
|
||||
double) {
|
||||
kernel->InputAt(0).SetDataLayout(phi::DataLayout::SPARSE_CSR);
|
||||
kernel->InputAt(1).SetDataLayout(phi::DataLayout::SPARSE_CSR);
|
||||
}
|
||||
|
||||
PD_REGISTER_KERNEL(masked_matmul_csr,
|
||||
GPU,
|
||||
ALL_LAYOUT,
|
||||
phi::sparse::MaskedMatmulCsrKernel,
|
||||
float,
|
||||
double) {}
|
||||
Reference in New Issue
Block a user