chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,441 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef PADDLE_WITH_HIP
|
||||
// HIP not support cusolver
|
||||
|
||||
#include "paddle/phi/kernels/svd_kernel.h"
|
||||
|
||||
#include "paddle/phi/backends/dynload/cusolver.h"
|
||||
#include "paddle/phi/common/memory_utils.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/kernels/complex_kernel.h"
|
||||
#include "paddle/phi/kernels/empty_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/complex_functors.h"
|
||||
#include "paddle/phi/kernels/transpose_kernel.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <class T>
|
||||
static void GesvdjBatched(const GPUContext& dev_ctx,
|
||||
int batchSize,
|
||||
int m,
|
||||
int n,
|
||||
int k,
|
||||
T* A,
|
||||
T* U,
|
||||
T* V,
|
||||
dtype::Real<T>* S,
|
||||
int* info,
|
||||
int thin_UV = 1);
|
||||
|
||||
template <>
|
||||
void GesvdjBatched<float>(const GPUContext& dev_ctx,
|
||||
int batchSize,
|
||||
int m,
|
||||
int n,
|
||||
int k,
|
||||
float* A,
|
||||
float* U,
|
||||
float* V,
|
||||
float* S,
|
||||
int* info,
|
||||
int thin_UV) {
|
||||
/* compute singular vectors */
|
||||
const cusolverEigMode_t jobz =
|
||||
CUSOLVER_EIG_MODE_VECTOR; /* compute singular vectors */
|
||||
gesvdjInfo_t gesvdj_params = NULL;
|
||||
int lda = m;
|
||||
int ldu = m;
|
||||
int ldt = n;
|
||||
int lwork = 0;
|
||||
auto handle = dev_ctx.cusolver_dn_handle();
|
||||
PADDLE_ENFORCE_GPU_SUCCESS(
|
||||
dynload::cusolverDnCreateGesvdjInfo(&gesvdj_params));
|
||||
PADDLE_ENFORCE_GPU_SUCCESS(
|
||||
dynload::cusolverDnSgesvdj_bufferSize(handle,
|
||||
jobz,
|
||||
thin_UV,
|
||||
m,
|
||||
n,
|
||||
A,
|
||||
lda,
|
||||
S,
|
||||
U,
|
||||
ldu,
|
||||
V,
|
||||
ldt,
|
||||
&lwork,
|
||||
gesvdj_params));
|
||||
auto workspace =
|
||||
memory_utils::Alloc(dev_ctx.GetPlace(),
|
||||
lwork * sizeof(float),
|
||||
Stream(reinterpret_cast<StreamId>(dev_ctx.stream())));
|
||||
float* workspace_ptr = reinterpret_cast<float*>(workspace->ptr());
|
||||
int64_t stride_A = static_cast<int64_t>(lda) * n;
|
||||
int64_t stride_U = static_cast<int64_t>(ldu) * (thin_UV ? k : m);
|
||||
int64_t stride_V = static_cast<int64_t>(ldt) * (thin_UV ? k : n);
|
||||
for (int i = 0; i < batchSize; ++i) {
|
||||
PADDLE_ENFORCE_GPU_SUCCESS(dynload::cusolverDnSgesvdj(handle,
|
||||
jobz,
|
||||
thin_UV,
|
||||
m,
|
||||
n,
|
||||
A + stride_A * i,
|
||||
lda,
|
||||
S + k * i,
|
||||
U + stride_U * i,
|
||||
ldu,
|
||||
V + stride_V * i,
|
||||
ldt,
|
||||
workspace_ptr,
|
||||
lwork,
|
||||
info,
|
||||
gesvdj_params));
|
||||
// check the error info
|
||||
int error_info;
|
||||
memory_utils::Copy(CPUPlace(),
|
||||
&error_info,
|
||||
dev_ctx.GetPlace(),
|
||||
info,
|
||||
sizeof(int),
|
||||
dev_ctx.stream());
|
||||
PADDLE_ENFORCE_EQ(
|
||||
error_info,
|
||||
0,
|
||||
common::errors::PreconditionNotMet(
|
||||
"For batch [%d]: CUSolver SVD is not zero. [%d]", i, error_info));
|
||||
}
|
||||
PADDLE_ENFORCE_GPU_SUCCESS(
|
||||
dynload::cusolverDnDestroyGesvdjInfo(gesvdj_params));
|
||||
}
|
||||
|
||||
template <>
|
||||
void GesvdjBatched<double>(const GPUContext& dev_ctx,
|
||||
int batchSize,
|
||||
int m,
|
||||
int n,
|
||||
int k,
|
||||
double* A,
|
||||
double* U,
|
||||
double* V,
|
||||
double* S,
|
||||
int* info,
|
||||
int thin_UV) {
|
||||
/* compute singular vectors */
|
||||
const cusolverEigMode_t jobz =
|
||||
CUSOLVER_EIG_MODE_VECTOR; /* compute singular vectors */
|
||||
gesvdjInfo_t gesvdj_params = NULL;
|
||||
int lda = m;
|
||||
int ldu = m;
|
||||
int ldt = n;
|
||||
int lwork = 0;
|
||||
auto handle = dev_ctx.cusolver_dn_handle();
|
||||
PADDLE_ENFORCE_GPU_SUCCESS(
|
||||
dynload::cusolverDnCreateGesvdjInfo(&gesvdj_params));
|
||||
PADDLE_ENFORCE_GPU_SUCCESS(
|
||||
dynload::cusolverDnDgesvdj_bufferSize(handle,
|
||||
jobz,
|
||||
thin_UV,
|
||||
m,
|
||||
n,
|
||||
A,
|
||||
lda,
|
||||
S,
|
||||
U,
|
||||
ldu,
|
||||
V,
|
||||
ldt,
|
||||
&lwork,
|
||||
gesvdj_params));
|
||||
auto workspace =
|
||||
memory_utils::Alloc(dev_ctx.GetPlace(),
|
||||
lwork * sizeof(double),
|
||||
Stream(reinterpret_cast<StreamId>(dev_ctx.stream())));
|
||||
double* workspace_ptr = reinterpret_cast<double*>(workspace->ptr());
|
||||
int64_t stride_A = static_cast<int64_t>(lda) * n;
|
||||
int64_t stride_U = static_cast<int64_t>(ldu) * (thin_UV ? k : m);
|
||||
int64_t stride_V = static_cast<int64_t>(ldt) * (thin_UV ? k : n);
|
||||
for (int i = 0; i < batchSize; ++i) {
|
||||
PADDLE_ENFORCE_GPU_SUCCESS(dynload::cusolverDnDgesvdj(handle,
|
||||
jobz,
|
||||
thin_UV,
|
||||
m,
|
||||
n,
|
||||
A + stride_A * i,
|
||||
lda,
|
||||
S + k * i,
|
||||
U + stride_U * i,
|
||||
ldu,
|
||||
V + stride_V * i,
|
||||
ldt,
|
||||
workspace_ptr,
|
||||
lwork,
|
||||
info,
|
||||
gesvdj_params));
|
||||
// check the error info
|
||||
int error_info;
|
||||
memory_utils::Copy(CPUPlace(),
|
||||
&error_info,
|
||||
dev_ctx.GetPlace(),
|
||||
info,
|
||||
sizeof(int),
|
||||
dev_ctx.stream());
|
||||
PADDLE_ENFORCE_EQ(
|
||||
error_info,
|
||||
0,
|
||||
common::errors::PreconditionNotMet(
|
||||
"For batch [%d]: CUSolver SVD is not zero. [%d]", i, error_info));
|
||||
}
|
||||
PADDLE_ENFORCE_GPU_SUCCESS(
|
||||
dynload::cusolverDnDestroyGesvdjInfo(gesvdj_params));
|
||||
}
|
||||
|
||||
template <>
|
||||
void GesvdjBatched<complex64>(const GPUContext& dev_ctx,
|
||||
int batchSize,
|
||||
int m,
|
||||
int n,
|
||||
int k,
|
||||
complex64* A,
|
||||
complex64* U,
|
||||
complex64* V,
|
||||
float* S,
|
||||
int* info,
|
||||
int thin_UV) {
|
||||
/* compute singular vectors */
|
||||
const cusolverEigMode_t jobz =
|
||||
CUSOLVER_EIG_MODE_VECTOR; /* compute singular vectors */
|
||||
gesvdjInfo_t gesvdj_params = NULL;
|
||||
int lda = m;
|
||||
int ldu = m;
|
||||
int ldt = n;
|
||||
int lwork = 0;
|
||||
auto handle = dev_ctx.cusolver_dn_handle();
|
||||
PADDLE_ENFORCE_GPU_SUCCESS(
|
||||
dynload::cusolverDnCreateGesvdjInfo(&gesvdj_params));
|
||||
PADDLE_ENFORCE_GPU_SUCCESS(
|
||||
dynload::cusolverDnCgesvdj_bufferSize(handle,
|
||||
jobz,
|
||||
thin_UV,
|
||||
m,
|
||||
n,
|
||||
reinterpret_cast<cuComplex*>(A),
|
||||
lda,
|
||||
S,
|
||||
reinterpret_cast<cuComplex*>(U),
|
||||
ldu,
|
||||
reinterpret_cast<cuComplex*>(V),
|
||||
ldt,
|
||||
&lwork,
|
||||
gesvdj_params));
|
||||
auto workspace =
|
||||
memory_utils::Alloc(dev_ctx.GetPlace(),
|
||||
lwork * sizeof(complex64),
|
||||
Stream(reinterpret_cast<StreamId>(dev_ctx.stream())));
|
||||
complex64* workspace_ptr = reinterpret_cast<complex64*>(workspace->ptr());
|
||||
int64_t stride_A = static_cast<int64_t>(lda) * n;
|
||||
int64_t stride_U = static_cast<int64_t>(ldu) * (thin_UV ? k : m);
|
||||
int64_t stride_V = static_cast<int64_t>(ldt) * (thin_UV ? k : n);
|
||||
for (int i = 0; i < batchSize; ++i) {
|
||||
PADDLE_ENFORCE_GPU_SUCCESS(dynload::cusolverDnCgesvdj(
|
||||
handle,
|
||||
jobz,
|
||||
thin_UV,
|
||||
m,
|
||||
n,
|
||||
reinterpret_cast<cuComplex*>(A + stride_A * i),
|
||||
lda,
|
||||
reinterpret_cast<float*>(S + k * i),
|
||||
reinterpret_cast<cuComplex*>(U + stride_U * i),
|
||||
ldu,
|
||||
reinterpret_cast<cuComplex*>(V + stride_V * i),
|
||||
ldt,
|
||||
reinterpret_cast<cuComplex*>(workspace_ptr),
|
||||
lwork,
|
||||
info,
|
||||
gesvdj_params));
|
||||
// check the error info
|
||||
int error_info;
|
||||
memory_utils::Copy(CPUPlace(),
|
||||
&error_info,
|
||||
dev_ctx.GetPlace(),
|
||||
info,
|
||||
sizeof(int),
|
||||
dev_ctx.stream());
|
||||
PADDLE_ENFORCE_EQ(
|
||||
error_info,
|
||||
0,
|
||||
common::errors::PreconditionNotMet(
|
||||
"For batch [%d]: CUSolver SVD is not zero. [%d]", i, error_info));
|
||||
}
|
||||
PADDLE_ENFORCE_GPU_SUCCESS(
|
||||
dynload::cusolverDnDestroyGesvdjInfo(gesvdj_params));
|
||||
}
|
||||
|
||||
template <>
|
||||
void GesvdjBatched<complex128>(const GPUContext& dev_ctx,
|
||||
int batchSize,
|
||||
int m,
|
||||
int n,
|
||||
int k,
|
||||
complex128* A,
|
||||
complex128* U,
|
||||
complex128* V,
|
||||
double* S,
|
||||
int* info,
|
||||
int thin_UV) {
|
||||
/* compute singular vectors */
|
||||
const cusolverEigMode_t jobz =
|
||||
CUSOLVER_EIG_MODE_VECTOR; /* compute singular vectors */
|
||||
gesvdjInfo_t gesvdj_params = NULL;
|
||||
int lda = m;
|
||||
int ldu = m;
|
||||
int ldt = n;
|
||||
int lwork = 0;
|
||||
auto handle = dev_ctx.cusolver_dn_handle();
|
||||
PADDLE_ENFORCE_GPU_SUCCESS(
|
||||
dynload::cusolverDnCreateGesvdjInfo(&gesvdj_params));
|
||||
PADDLE_ENFORCE_GPU_SUCCESS(dynload::cusolverDnZgesvdj_bufferSize(
|
||||
handle,
|
||||
jobz,
|
||||
thin_UV,
|
||||
m,
|
||||
n,
|
||||
reinterpret_cast<cuDoubleComplex*>(A),
|
||||
lda,
|
||||
S,
|
||||
reinterpret_cast<cuDoubleComplex*>(U),
|
||||
ldu,
|
||||
reinterpret_cast<cuDoubleComplex*>(V),
|
||||
ldt,
|
||||
&lwork,
|
||||
gesvdj_params));
|
||||
auto workspace =
|
||||
memory_utils::Alloc(dev_ctx.GetPlace(),
|
||||
lwork * sizeof(complex128),
|
||||
Stream(reinterpret_cast<StreamId>(dev_ctx.stream())));
|
||||
complex128* workspace_ptr = reinterpret_cast<complex128*>(workspace->ptr());
|
||||
int64_t stride_A = static_cast<int64_t>(lda) * n;
|
||||
int64_t stride_U = static_cast<int64_t>(ldu) * (thin_UV ? k : m);
|
||||
int64_t stride_V = static_cast<int64_t>(ldt) * (thin_UV ? k : n);
|
||||
for (int i = 0; i < batchSize; ++i) {
|
||||
PADDLE_ENFORCE_GPU_SUCCESS(dynload::cusolverDnZgesvdj(
|
||||
handle,
|
||||
jobz,
|
||||
thin_UV,
|
||||
m,
|
||||
n,
|
||||
reinterpret_cast<cuDoubleComplex*>(A + stride_A * i),
|
||||
lda,
|
||||
reinterpret_cast<double*>(S + k * i),
|
||||
reinterpret_cast<cuDoubleComplex*>(U + stride_U * i),
|
||||
ldu,
|
||||
reinterpret_cast<cuDoubleComplex*>(V + stride_V * i),
|
||||
ldt,
|
||||
reinterpret_cast<cuDoubleComplex*>(workspace_ptr),
|
||||
lwork,
|
||||
info,
|
||||
gesvdj_params));
|
||||
// check the error info
|
||||
int error_info;
|
||||
memory_utils::Copy(CPUPlace(),
|
||||
&error_info,
|
||||
dev_ctx.GetPlace(),
|
||||
info,
|
||||
sizeof(int),
|
||||
dev_ctx.stream());
|
||||
PADDLE_ENFORCE_EQ(
|
||||
error_info,
|
||||
0,
|
||||
common::errors::PreconditionNotMet(
|
||||
"For batch [%d]: CUSolver SVD is not zero. [%d]", i, error_info));
|
||||
}
|
||||
PADDLE_ENFORCE_GPU_SUCCESS(
|
||||
dynload::cusolverDnDestroyGesvdjInfo(gesvdj_params));
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void SvdKernel(const Context& dev_ctx,
|
||||
const DenseTensor& X,
|
||||
bool full_matrices,
|
||||
DenseTensor* U,
|
||||
DenseTensor* S,
|
||||
DenseTensor* VH) {
|
||||
if (X.numel() == 0) {
|
||||
dev_ctx.template Alloc<T>(U);
|
||||
dev_ctx.template Alloc<dtype::Real<T>>(S);
|
||||
dev_ctx.template Alloc<T>(VH);
|
||||
return;
|
||||
}
|
||||
auto& dims = X.dims();
|
||||
int64_t batch_count64 = 1;
|
||||
for (int i = 0; i < dims.size() - 2; i++) {
|
||||
batch_count64 *= dims[i];
|
||||
}
|
||||
// TODO(large-tensor): cusolver batch_count not support int64
|
||||
PADDLE_ENFORCE_LE_INT_MAX(batch_count64, "batch_count");
|
||||
int batch_count = static_cast<int>(batch_count64);
|
||||
|
||||
int rank = dims.size();
|
||||
int64_t m = dims[rank - 2];
|
||||
int64_t n = dims[rank - 1];
|
||||
// TODO(large-tensor): cusolver m/n not support int64
|
||||
PADDLE_ENFORCE_LE_INT_MAX(m, "m");
|
||||
PADDLE_ENFORCE_LE_INT_MAX(n, "n");
|
||||
int m_int = static_cast<int>(m);
|
||||
int n_int = static_cast<int>(n);
|
||||
|
||||
auto* u_data = dev_ctx.template Alloc<T>(U);
|
||||
auto* vh_data = dev_ctx.template Alloc<T>(VH);
|
||||
auto* s_data = dev_ctx.template Alloc<dtype::Real<T>>(S);
|
||||
// NOTE:(@xiongkun03)
|
||||
// matrices are assumed to be stored in column-major order in cusolver
|
||||
// then view A as n x m and do A^T SVD, we can avoid transpose
|
||||
// Must Copy X once, because the gesvdj will change the origin input matrix
|
||||
DenseTensor x_tmp;
|
||||
Copy(dev_ctx, X, dev_ctx.GetPlace(), false, &x_tmp);
|
||||
auto info = Empty<int, Context>(dev_ctx, {batch_count});
|
||||
int* info_ptr = reinterpret_cast<int*>(info.data());
|
||||
|
||||
GesvdjBatched<T>(dev_ctx,
|
||||
batch_count,
|
||||
n_int,
|
||||
m_int,
|
||||
std::min(m_int, n_int),
|
||||
dev_ctx.template Alloc<T>(&x_tmp),
|
||||
vh_data,
|
||||
u_data,
|
||||
s_data,
|
||||
info_ptr,
|
||||
!full_matrices);
|
||||
|
||||
auto UT_dim = U->dims();
|
||||
std::swap(UT_dim[rank - 1], UT_dim[rank - 2]); // Get the dim of UT_dim
|
||||
U->Resize(UT_dim); // U is entirely UT
|
||||
auto tmp_U = TransposeLast2Dim<T>(dev_ctx, Conj<T, Context>(dev_ctx, *U));
|
||||
U->ShareDataWith(tmp_U); // U becomse UT, aka VT;
|
||||
}
|
||||
} // namespace phi
|
||||
|
||||
PD_REGISTER_KERNEL(svd, // cuda_only
|
||||
GPU,
|
||||
ALL_LAYOUT,
|
||||
phi::SvdKernel,
|
||||
float,
|
||||
double,
|
||||
phi::complex64,
|
||||
phi::complex128) {}
|
||||
|
||||
#endif // not PADDLE_WITH_HIP
|
||||
Reference in New Issue
Block a user