// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#ifndef PADDLE_WITH_HIP
// HIP not support cusolver

#include "paddle/phi/kernels/svd_kernel.h"

#include "paddle/phi/backends/dynload/cusolver.h"
#include "paddle/phi/common/memory_utils.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/complex_kernel.h"
#include "paddle/phi/kernels/empty_kernel.h"
#include "paddle/phi/kernels/funcs/complex_functors.h"
#include "paddle/phi/kernels/transpose_kernel.h"

namespace phi {

template <class T>
static void GesvdjBatched(const GPUContext& dev_ctx,
                          int batchSize,
                          int m,
                          int n,
                          int k,
                          T* A,
                          T* U,
                          T* V,
                          dtype::Real<T>* S,
                          int* info,
                          int thin_UV = 1);

template <>
void GesvdjBatched<float>(const GPUContext& dev_ctx,
                          int batchSize,
                          int m,
                          int n,
                          int k,
                          float* A,
                          float* U,
                          float* V,
                          float* S,
                          int* info,
                          int thin_UV) {
  /* compute singular vectors */
  const cusolverEigMode_t jobz =
      CUSOLVER_EIG_MODE_VECTOR; /* compute singular vectors */
  gesvdjInfo_t gesvdj_params = NULL;
  int lda = m;
  int ldu = m;
  int ldt = n;
  int lwork = 0;
  auto handle = dev_ctx.cusolver_dn_handle();
  PADDLE_ENFORCE_GPU_SUCCESS(
      dynload::cusolverDnCreateGesvdjInfo(&gesvdj_params));
  PADDLE_ENFORCE_GPU_SUCCESS(
      dynload::cusolverDnSgesvdj_bufferSize(handle,
                                            jobz,
                                            thin_UV,
                                            m,
                                            n,
                                            A,
                                            lda,
                                            S,
                                            U,
                                            ldu,
                                            V,
                                            ldt,
                                            &lwork,
                                            gesvdj_params));
  auto workspace =
      memory_utils::Alloc(dev_ctx.GetPlace(),
                          lwork * sizeof(float),
                          Stream(reinterpret_cast<StreamId>(dev_ctx.stream())));
  float* workspace_ptr = reinterpret_cast<float*>(workspace->ptr());
  int64_t stride_A = static_cast<int64_t>(lda) * n;
  int64_t stride_U = static_cast<int64_t>(ldu) * (thin_UV ? k : m);
  int64_t stride_V = static_cast<int64_t>(ldt) * (thin_UV ? k : n);
  for (int i = 0; i < batchSize; ++i) {
    PADDLE_ENFORCE_GPU_SUCCESS(dynload::cusolverDnSgesvdj(handle,
                                                          jobz,
                                                          thin_UV,
                                                          m,
                                                          n,
                                                          A + stride_A * i,
                                                          lda,
                                                          S + k * i,
                                                          U + stride_U * i,
                                                          ldu,
                                                          V + stride_V * i,
                                                          ldt,
                                                          workspace_ptr,
                                                          lwork,
                                                          info,
                                                          gesvdj_params));
    // check the error info
    int error_info;
    memory_utils::Copy(CPUPlace(),
                       &error_info,
                       dev_ctx.GetPlace(),
                       info,
                       sizeof(int),
                       dev_ctx.stream());
    PADDLE_ENFORCE_EQ(
        error_info,
        0,
        common::errors::PreconditionNotMet(
            "For batch [%d]: CUSolver SVD is not zero. [%d]", i, error_info));
  }
  PADDLE_ENFORCE_GPU_SUCCESS(
      dynload::cusolverDnDestroyGesvdjInfo(gesvdj_params));
}

template <>
void GesvdjBatched<double>(const GPUContext& dev_ctx,
                           int batchSize,
                           int m,
                           int n,
                           int k,
                           double* A,
                           double* U,
                           double* V,
                           double* S,
                           int* info,
                           int thin_UV) {
  /* compute singular vectors */
  const cusolverEigMode_t jobz =
      CUSOLVER_EIG_MODE_VECTOR; /* compute singular vectors */
  gesvdjInfo_t gesvdj_params = NULL;
  int lda = m;
  int ldu = m;
  int ldt = n;
  int lwork = 0;
  auto handle = dev_ctx.cusolver_dn_handle();
  PADDLE_ENFORCE_GPU_SUCCESS(
      dynload::cusolverDnCreateGesvdjInfo(&gesvdj_params));
  PADDLE_ENFORCE_GPU_SUCCESS(
      dynload::cusolverDnDgesvdj_bufferSize(handle,
                                            jobz,
                                            thin_UV,
                                            m,
                                            n,
                                            A,
                                            lda,
                                            S,
                                            U,
                                            ldu,
                                            V,
                                            ldt,
                                            &lwork,
                                            gesvdj_params));
  auto workspace =
      memory_utils::Alloc(dev_ctx.GetPlace(),
                          lwork * sizeof(double),
                          Stream(reinterpret_cast<StreamId>(dev_ctx.stream())));
  double* workspace_ptr = reinterpret_cast<double*>(workspace->ptr());
  int64_t stride_A = static_cast<int64_t>(lda) * n;
  int64_t stride_U = static_cast<int64_t>(ldu) * (thin_UV ? k : m);
  int64_t stride_V = static_cast<int64_t>(ldt) * (thin_UV ? k : n);
  for (int i = 0; i < batchSize; ++i) {
    PADDLE_ENFORCE_GPU_SUCCESS(dynload::cusolverDnDgesvdj(handle,
                                                          jobz,
                                                          thin_UV,
                                                          m,
                                                          n,
                                                          A + stride_A * i,
                                                          lda,
                                                          S + k * i,
                                                          U + stride_U * i,
                                                          ldu,
                                                          V + stride_V * i,
                                                          ldt,
                                                          workspace_ptr,
                                                          lwork,
                                                          info,
                                                          gesvdj_params));
    // check the error info
    int error_info;
    memory_utils::Copy(CPUPlace(),
                       &error_info,
                       dev_ctx.GetPlace(),
                       info,
                       sizeof(int),
                       dev_ctx.stream());
    PADDLE_ENFORCE_EQ(
        error_info,
        0,
        common::errors::PreconditionNotMet(
            "For batch [%d]: CUSolver SVD is not zero. [%d]", i, error_info));
  }
  PADDLE_ENFORCE_GPU_SUCCESS(
      dynload::cusolverDnDestroyGesvdjInfo(gesvdj_params));
}

template <>
void GesvdjBatched<complex64>(const GPUContext& dev_ctx,
                              int batchSize,
                              int m,
                              int n,
                              int k,
                              complex64* A,
                              complex64* U,
                              complex64* V,
                              float* S,
                              int* info,
                              int thin_UV) {
  /* compute singular vectors */
  const cusolverEigMode_t jobz =
      CUSOLVER_EIG_MODE_VECTOR; /* compute singular vectors */
  gesvdjInfo_t gesvdj_params = NULL;
  int lda = m;
  int ldu = m;
  int ldt = n;
  int lwork = 0;
  auto handle = dev_ctx.cusolver_dn_handle();
  PADDLE_ENFORCE_GPU_SUCCESS(
      dynload::cusolverDnCreateGesvdjInfo(&gesvdj_params));
  PADDLE_ENFORCE_GPU_SUCCESS(
      dynload::cusolverDnCgesvdj_bufferSize(handle,
                                            jobz,
                                            thin_UV,
                                            m,
                                            n,
                                            reinterpret_cast<cuComplex*>(A),
                                            lda,
                                            S,
                                            reinterpret_cast<cuComplex*>(U),
                                            ldu,
                                            reinterpret_cast<cuComplex*>(V),
                                            ldt,
                                            &lwork,
                                            gesvdj_params));
  auto workspace =
      memory_utils::Alloc(dev_ctx.GetPlace(),
                          lwork * sizeof(complex64),
                          Stream(reinterpret_cast<StreamId>(dev_ctx.stream())));
  complex64* workspace_ptr = reinterpret_cast<complex64*>(workspace->ptr());
  int64_t stride_A = static_cast<int64_t>(lda) * n;
  int64_t stride_U = static_cast<int64_t>(ldu) * (thin_UV ? k : m);
  int64_t stride_V = static_cast<int64_t>(ldt) * (thin_UV ? k : n);
  for (int i = 0; i < batchSize; ++i) {
    PADDLE_ENFORCE_GPU_SUCCESS(dynload::cusolverDnCgesvdj(
        handle,
        jobz,
        thin_UV,
        m,
        n,
        reinterpret_cast<cuComplex*>(A + stride_A * i),
        lda,
        reinterpret_cast<float*>(S + k * i),
        reinterpret_cast<cuComplex*>(U + stride_U * i),
        ldu,
        reinterpret_cast<cuComplex*>(V + stride_V * i),
        ldt,
        reinterpret_cast<cuComplex*>(workspace_ptr),
        lwork,
        info,
        gesvdj_params));
    // check the error info
    int error_info;
    memory_utils::Copy(CPUPlace(),
                       &error_info,
                       dev_ctx.GetPlace(),
                       info,
                       sizeof(int),
                       dev_ctx.stream());
    PADDLE_ENFORCE_EQ(
        error_info,
        0,
        common::errors::PreconditionNotMet(
            "For batch [%d]: CUSolver SVD is not zero. [%d]", i, error_info));
  }
  PADDLE_ENFORCE_GPU_SUCCESS(
      dynload::cusolverDnDestroyGesvdjInfo(gesvdj_params));
}

template <>
void GesvdjBatched<complex128>(const GPUContext& dev_ctx,
                               int batchSize,
                               int m,
                               int n,
                               int k,
                               complex128* A,
                               complex128* U,
                               complex128* V,
                               double* S,
                               int* info,
                               int thin_UV) {
  /* compute singular vectors */
  const cusolverEigMode_t jobz =
      CUSOLVER_EIG_MODE_VECTOR; /* compute singular vectors */
  gesvdjInfo_t gesvdj_params = NULL;
  int lda = m;
  int ldu = m;
  int ldt = n;
  int lwork = 0;
  auto handle = dev_ctx.cusolver_dn_handle();
  PADDLE_ENFORCE_GPU_SUCCESS(
      dynload::cusolverDnCreateGesvdjInfo(&gesvdj_params));
  PADDLE_ENFORCE_GPU_SUCCESS(dynload::cusolverDnZgesvdj_bufferSize(
      handle,
      jobz,
      thin_UV,
      m,
      n,
      reinterpret_cast<cuDoubleComplex*>(A),
      lda,
      S,
      reinterpret_cast<cuDoubleComplex*>(U),
      ldu,
      reinterpret_cast<cuDoubleComplex*>(V),
      ldt,
      &lwork,
      gesvdj_params));
  auto workspace =
      memory_utils::Alloc(dev_ctx.GetPlace(),
                          lwork * sizeof(complex128),
                          Stream(reinterpret_cast<StreamId>(dev_ctx.stream())));
  complex128* workspace_ptr = reinterpret_cast<complex128*>(workspace->ptr());
  int64_t stride_A = static_cast<int64_t>(lda) * n;
  int64_t stride_U = static_cast<int64_t>(ldu) * (thin_UV ? k : m);
  int64_t stride_V = static_cast<int64_t>(ldt) * (thin_UV ? k : n);
  for (int i = 0; i < batchSize; ++i) {
    PADDLE_ENFORCE_GPU_SUCCESS(dynload::cusolverDnZgesvdj(
        handle,
        jobz,
        thin_UV,
        m,
        n,
        reinterpret_cast<cuDoubleComplex*>(A + stride_A * i),
        lda,
        reinterpret_cast<double*>(S + k * i),
        reinterpret_cast<cuDoubleComplex*>(U + stride_U * i),
        ldu,
        reinterpret_cast<cuDoubleComplex*>(V + stride_V * i),
        ldt,
        reinterpret_cast<cuDoubleComplex*>(workspace_ptr),
        lwork,
        info,
        gesvdj_params));
    // check the error info
    int error_info;
    memory_utils::Copy(CPUPlace(),
                       &error_info,
                       dev_ctx.GetPlace(),
                       info,
                       sizeof(int),
                       dev_ctx.stream());
    PADDLE_ENFORCE_EQ(
        error_info,
        0,
        common::errors::PreconditionNotMet(
            "For batch [%d]: CUSolver SVD is not zero. [%d]", i, error_info));
  }
  PADDLE_ENFORCE_GPU_SUCCESS(
      dynload::cusolverDnDestroyGesvdjInfo(gesvdj_params));
}

template <typename T, typename Context>
void SvdKernel(const Context& dev_ctx,
               const DenseTensor& X,
               bool full_matrices,
               DenseTensor* U,
               DenseTensor* S,
               DenseTensor* VH) {
  if (X.numel() == 0) {
    dev_ctx.template Alloc<T>(U);
    dev_ctx.template Alloc<dtype::Real<T>>(S);
    dev_ctx.template Alloc<T>(VH);
    return;
  }
  auto& dims = X.dims();
  int64_t batch_count64 = 1;
  for (int i = 0; i < dims.size() - 2; i++) {
    batch_count64 *= dims[i];
  }
  // TODO(large-tensor): cusolver batch_count not support int64
  PADDLE_ENFORCE_LE_INT_MAX(batch_count64, "batch_count");
  int batch_count = static_cast<int>(batch_count64);

  int rank = dims.size();
  int64_t m = dims[rank - 2];
  int64_t n = dims[rank - 1];
  // TODO(large-tensor): cusolver m/n not support int64
  PADDLE_ENFORCE_LE_INT_MAX(m, "m");
  PADDLE_ENFORCE_LE_INT_MAX(n, "n");
  int m_int = static_cast<int>(m);
  int n_int = static_cast<int>(n);

  auto* u_data = dev_ctx.template Alloc<T>(U);
  auto* vh_data = dev_ctx.template Alloc<T>(VH);
  auto* s_data = dev_ctx.template Alloc<dtype::Real<T>>(S);
  // NOTE:(@xiongkun03)
  // matrices are assumed to be stored in column-major order in cusolver
  // then view A as n x m and do A^T SVD, we can avoid transpose
  // Must Copy X once, because the gesvdj will change the origin input matrix
  DenseTensor x_tmp;
  Copy(dev_ctx, X, dev_ctx.GetPlace(), false, &x_tmp);
  auto info = Empty<int, Context>(dev_ctx, {batch_count});
  int* info_ptr = reinterpret_cast<int*>(info.data());

  GesvdjBatched<T>(dev_ctx,
                   batch_count,
                   n_int,
                   m_int,
                   std::min(m_int, n_int),
                   dev_ctx.template Alloc<T>(&x_tmp),
                   vh_data,
                   u_data,
                   s_data,
                   info_ptr,
                   !full_matrices);

  auto UT_dim = U->dims();
  std::swap(UT_dim[rank - 1], UT_dim[rank - 2]);  // Get the dim of UT_dim
  U->Resize(UT_dim);                              // U is entirely UT
  auto tmp_U = TransposeLast2Dim<T>(dev_ctx, Conj<T, Context>(dev_ctx, *U));
  U->ShareDataWith(tmp_U);  // U becomse UT, aka VT;
}
}  // namespace phi

PD_REGISTER_KERNEL(svd,  // cuda_only
                   GPU,
                   ALL_LAYOUT,
                   phi::SvdKernel,
                   float,
                   double,
                   phi::complex64,
                   phi::complex128) {}

#endif  // not PADDLE_WITH_HIP