paddlepaddle--paddle/paddle/phi/kernels/gpu/diag_grad_kernel.cu

// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "paddle/phi/backends/gpu/gpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/diag_kernel.h"

#include "paddle/common/enforce.h"
#include "paddle/phi/kernels/funcs/diag_functor.h"
#include "paddle/phi/kernels/funcs/math_function.h"

namespace phi {

// Extract the diagonal of a matrix 'dout' to a matrix 'dx'
template <typename T>
__global__ void ExtractDiagonalKernel(const T* dout,
                                      T* dx,
                                      int64_t start,
                                      int64_t dx_length,
                                      const int64_t sumStride,
                                      const int64_t xStride) {
  for (int64_t idx =
           static_cast<int64_t>(blockIdx.x) * static_cast<int64_t>(blockDim.x) +
           static_cast<int64_t>(threadIdx.x);
       idx < dx_length;
       idx += gridDim.x * blockDim.x) {
    const int64_t outOffset = start + sumStride * idx;
    dx[xStride * idx] = dout[outOffset];
  }
}

// Paste a vector 'dout' to the diagonal of a matrix 'dx'
template <typename T>
__global__ void PasteDiagonalKernel(const T* dout,
                                    T* dx,
                                    int64_t start,
                                    int64_t size,
                                    const int64_t sumStride,
                                    const int64_t outStride) {
  for (int64_t idx =
           static_cast<int64_t>(blockIdx.x) * static_cast<int64_t>(blockDim.x) +
           static_cast<int64_t>(threadIdx.x);
       idx < size;
       idx += gridDim.x * blockDim.x) {
    int64_t xOffset = start + sumStride * idx;
    dx[xOffset] = dout[outStride * idx];
  }
}

template <typename T, typename Context>
void DiagGradKernel(const Context& dev_ctx,
                    const DenseTensor& x,
                    const DenseTensor& out_grad,
                    int offset,
                    DenseTensor* x_grad) {
  T* dx_data = dev_ctx.template Alloc<T>(x_grad);
  if (x_grad && x_grad->numel() == 0) return;
  auto* dout_data = out_grad.data<T>();
  auto dx_dims = x_grad->dims();
  auto dout_dims = out_grad.dims();

  auto GetBlockGridSize = [&dev_ctx](int64_t size) {
    const int64_t block_size =
        std::min(size, static_cast<int64_t>(dev_ctx.GetMaxThreadsPerBlock()));
    int64_t max_threads = dev_ctx.GetMaxPhysicalThreadCount();
    const int64_t max_blocks =
        std::max(((max_threads - 1) / block_size + 1), static_cast<int64_t>(1));
    const int64_t grid_size =
        std::min(max_blocks, (size + block_size - 1) / block_size);
    return std::tuple<int64_t, int64_t>{block_size, grid_size};
  };

  if (dx_dims.size() <= 1) {
    int64_t dx_length = (dx_dims.size() == 1 ? dx_dims[0] : 1);
    int64_t size = (offset > 0) ? dx_length + offset : dx_length - offset;
    int64_t dx_stride = 1;
    if (size > 0) {
      int64_t dout_stride_0 = funcs::ComputeStride(0, dout_dims);
      int64_t dout_stride_1 = funcs::ComputeStride(1, dout_dims);
      int64_t start =
          (offset >= 0 ? offset * dout_stride_1 : -offset * dout_stride_0);

      std::tuple<int64_t, int64_t> block_grid_size = GetBlockGridSize(size);
      const int64_t grid_64 = std::get<1>(block_grid_size);
      const int64_t block_64 = std::get<0>(block_grid_size);
      PADDLE_ENFORCE_LE_UINT32_MAX(grid_64, "grid");
      PADDLE_ENFORCE_LE_UINT32_MAX(block_64, "block");
      uint32_t grid = static_cast<uint32_t>(grid_64);
      uint32_t block = static_cast<uint32_t>(block_64);
      ExtractDiagonalKernel<T>
          <<<grid, block, 0, dev_ctx.stream()>>>(dout_data,
                                                 dx_data,
                                                 start,
                                                 dx_length,
                                                 dout_stride_0 + dout_stride_1,
                                                 dx_stride);
    }
  } else {
    funcs::SetConstant<Context, T> set_padding_value;
    set_padding_value(dev_ctx, x_grad, static_cast<T>(0));

    int64_t dx_stride_0 = funcs::ComputeStride(0, dx_dims);
    int64_t dx_stride_1 = funcs::ComputeStride(1, dx_dims);
    int64_t size;
    if (offset > 0) {
      size = std::min(dx_dims[0], dx_dims[1] - offset);
    } else {
      size = std::min(dx_dims[0] + offset, dx_dims[1]);
    }

    if (size > 0) {
      int64_t start =
          (offset >= 0 ? offset * dx_stride_1 : -offset * dx_stride_0);
      int64_t dout_stride_0 = funcs::ComputeStride(0, dout_dims);
      std::tuple<int64_t, int64_t> block_grid_size = GetBlockGridSize(size);
      const int64_t grid_64 = std::get<1>(block_grid_size);
      const int64_t block_64 = std::get<0>(block_grid_size);
      PADDLE_ENFORCE_LE_UINT32_MAX(grid_64, "grid");
      PADDLE_ENFORCE_LE_UINT32_MAX(block_64, "block");
      uint32_t grid = static_cast<uint32_t>(grid_64);
      uint32_t block = static_cast<uint32_t>(block_64);
      PasteDiagonalKernel<T>
          <<<grid, block, 0, dev_ctx.stream()>>>(dout_data,
                                                 dx_data,
                                                 start,
                                                 size,
                                                 dx_stride_0 + dx_stride_1,
                                                 dout_stride_0);
    }
  }
}

}  // namespace phi

PD_REGISTER_KERNEL(diag_grad,
                   GPU,
                   ALL_LAYOUT,
                   phi::DiagGradKernel,
                   phi::float16,
                   phi::bfloat16,
                   int,
                   int64_t,
                   float,
                   double,
                   phi::complex64,
                   phi::complex128) {}