// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "paddle/phi/kernels/conv_kernel.h"
#include "glog/logging.h"
#include "paddle/phi/kernels/full_kernel.h"
#include "paddle/phi/kernels/gpudnn/conv_gpudnn.h"

#include "paddle/phi/backends/context_pool.h"
#include "paddle/phi/backends/gpu/gpu_context.h"
#include "paddle/phi/core/dense_tensor.h"
#include "paddle/phi/core/kernel_registry.h"

#ifdef PADDLE_WITH_HIP
#include "paddle/phi/kernels/gpudnn/conv_miopen_helper.h"
#else
#include "paddle/phi/kernels/gpudnn/conv_cudnn_v7.h"
#endif

#include "paddle/phi/backends/gpu/cuda/cudnn_workspace_helper.h"
#include "paddle/phi/kernels/cpu/conv_util.h"
#include "paddle/phi/kernels/funcs/batch_norm_utils.h"
#include "paddle/phi/kernels/funcs/padding.h"
#include "paddle/phi/kernels/impl/conv_cudnn_impl.h"

#ifdef PADDLE_WITH_CUDNN_FRONTEND
// clang-format off
#include "paddle/phi/backends/dynload/cudnn_frontend.h"
#include "paddle/phi/kernels/autotune/cache.h"
#include "paddle/phi/kernels/gpudnn/conv_cudnn_frontend.h"
// clang-format on
#endif

#include "paddle/common/flags.h"

COMMON_DECLARE_bool(manually_trans_conv_filter);
COMMON_DECLARE_bool(use_accuracy_compatible_kernel);

namespace phi {
template <typename T, typename Context>
void ConvCudnnKernelImplV7(const DenseTensor* transformed_input,
                           const DenseTensor* transformed_filter_channel,
                           const Context& dev_ctx,
                           const std::vector<int>& strides,
                           const std::vector<int>& padding_common,
                           const std::vector<int>& dilations,
                           DataLayout compute_format,
                           DataLayout layout,
                           bool exhaustive_search,
                           bool deterministic,
                           int groups,
                           DenseTensor* transformed_output) {
  const T* input_data = transformed_input->data<T>();
  const T* filter_data = transformed_filter_channel->data<T>();
  T* output_data = transformed_output->data<T>();

  auto handle = dev_ctx.cudnn_handle();
  auto workspace_handle = dev_ctx.cudnn_workspace_handle();

  auto layout_format = phi::backends::gpu::GetCudnnTensorFormat(layout);
  auto dtype = phi::backends::gpu::CudnnDataType<T>::type;

  // ------------------- cudnn descriptors ---------------------
  ConvArgs args{handle,
                transformed_input,
                transformed_filter_channel,
                transformed_output,
                strides,
                padding_common,
                dilations,
                dtype,
                groups,
                compute_format};

#ifdef PADDLE_WITH_HIP
  // MIOPEN need to set groups in cdesc in miopen_desc.h
  args.cdesc.set(
      dtype, padding_common, strides, dilations, phi::AllowTF32Cudnn(), groups);
#else
  args.cdesc.set(
      dtype, padding_common, strides, dilations, phi::AllowTF32Cudnn());
#endif

#if defined(PADDLE_WITH_CUDA)
  // cudnn 7 can support groups, no need to do it manually
  // FIXME(typhoonzero): find a better way to disable groups
  // rather than setting it to 1.
  PADDLE_ENFORCE_GPU_SUCCESS(
      phi::dynload::cudnnSetConvolutionGroupCount(args.cdesc.desc(), groups));
  groups = 1;
#endif
#ifdef PADDLE_WITH_HIP
  // MIOPEN do not set groups in wdesc after set groups in cdesc
  groups = 1;
#endif
  args.idesc.set(*transformed_input, layout_format);
  args.wdesc.set(*transformed_filter_channel, layout_format, groups);
  args.odesc.set(*transformed_output, layout_format);
  int i_n, i_c, i_d, i_h, i_w;
  int o_n, o_c, o_d, o_h, o_w;

  if (compute_format == DataLayout::NHWC) {
    GetNCDHW(transformed_input->dims(),
             DataLayout::NHWC,
             &i_n,
             &i_c,
             &i_d,
             &i_h,
             &i_w);
    GetNCDHW(transformed_output->dims(),
             DataLayout::NHWC,
             &o_n,
             &o_c,
             &o_d,
             &o_h,
             &o_w);
  } else {
    GetNCDHW(transformed_input->dims(),
             DataLayout::NCHW,
             &i_n,
             &i_c,
             &i_d,
             &i_h,
             &i_w);
    GetNCDHW(transformed_output->dims(),
             DataLayout::NCHW,
             &o_n,
             &o_c,
             &o_d,
             &o_h,
             &o_w);
  }

  int64_t group_offset_in =
      static_cast<int64_t>(i_c) / groups * i_h * i_w * i_d;
  int64_t group_offset_out =
      static_cast<int64_t>(o_c) / groups * o_h * o_w * o_d;
  int64_t group_offset_filter = transformed_filter_channel->numel() / groups;
  // ------------------- cudnn conv workspace ---------------------
  size_t workspace_size = 0;  // final workspace to allocate.
// ------------------- cudnn conv algorithm ---------------------
#ifdef PADDLE_WITH_HIP
  SearchResult<miopenConvFwdAlgorithm_t> fwd_result;
  using search = SearchAlgorithm<miopenConvFwdAlgorithm_t>;
  workspace_size = search::GetWorkspaceSize(args);
  fwd_result.algo = search::Find<T>(
      args, exhaustive_search, deterministic, workspace_size, dev_ctx);
#else
  SearchResult<cudnnConvolutionFwdAlgo_t> fwd_result;
  using search = SearchAlgorithm<ConvKind::kForward>;
  fwd_result = search::Find<T>(dev_ctx, args, exhaustive_search, deterministic);
  workspace_size = fwd_result.workspace_size;
#endif

#if defined(PADDLE_WITH_CUDA)
  // when groups > 1, SearchAlgorithm find algo is CUDNN_CONVOLUTION_\
    // FWD_ALGO_WINOGRAD_NONFUSED, but this kind of algorithm is unstable
  // in forward computation, so change the algorithm to CUDNN_CONVOLUTION_\
    // FWD_ALGO_IMPLICIT_GEMM manually.
  if (groups > 1) {
    fwd_result.algo = static_cast<cudnnConvolutionFwdAlgo_t>(0);
  }
#endif

  // ------------------- cudnn conv forward ---------------------
  ScalingParamType<T> alpha = 1.0f;
  ScalingParamType<T> beta = 0.0f;

  // NOTE(zhiqiu): inplace addto is not supported in double grad yet.
  // ScalingParamType<T> beta = dev_ctx.Attr<bool>("use_addto") ? 1.0f : 0.0f;
  // VLOG(4) << "Conv: use_addto = " << dev_ctx.Attr<bool>("use_addto");

#ifdef PADDLE_WITH_HIP
  workspace_handle.RunFunc(
      [&](void* workspace_ptr) {
        PADDLE_ENFORCE_GPU_SUCCESS(
            phi::dynload::miopenConvolutionForward(handle,
                                                   &alpha,
                                                   args.idesc.desc(),
                                                   input_data,
                                                   args.wdesc.desc(),
                                                   filter_data,
                                                   args.cdesc.desc(),
                                                   fwd_result.algo,
                                                   &beta,
                                                   args.odesc.desc(),
                                                   output_data,
                                                   workspace_ptr,
                                                   workspace_size));
      },
      workspace_size);
#else
  ConvRunner<T, ConvKind::kForward>::Apply(dev_ctx,
                                           args,
                                           fwd_result,
                                           input_data,
                                           filter_data,
                                           output_data,
                                           groups,
                                           group_offset_in,
                                           group_offset_filter,
                                           group_offset_out,
                                           workspace_size,
                                           &workspace_handle,
                                           false);
#endif
}

#ifdef PADDLE_WITH_CUDNN_FRONTEND
template <typename T, typename Context>
void ConvCudnnKernelImplV8(const DenseTensor* input_tensor,
                           const DenseTensor* filter_channel_tensor,
                           const Context& dev_ctx,
                           const std::vector<int>& strides,
                           const std::vector<int>& padding_common,
                           const std::vector<int>& dilations,
                           DataLayout layout,
                           bool exhaustive_search,
                           bool deterministic,
                           int groups,
                           DenseTensor* output_tensor) {
  auto& plan_cache = phi::autotune::AutoTuneCache::Instance().GetConvV8(
      phi::autotune::AlgorithmType::kConvForwardV8);

  T* input_data = const_cast<T*>(input_tensor->data<T>());
  T* filter_data = const_cast<T*>(filter_channel_tensor->data<T>());
  T* output_data = output_tensor->data<T>();
  cudnnHandle_t handle = const_cast<cudnnHandle_t>(dev_ctx.cudnn_handle());
  auto workspace_handle = dev_ctx.cudnn_workspace_handle();

  auto layout_format = phi::backends::gpu::GetCudnnTensorFormat(layout);
  auto dtype = phi::backends::gpu::CudnnDataType<T>::type;

  float alpha = 1.0f;
  float beta = 0.0f;

  using helper = CudnnFrontendConvHelper;
  auto op_graph = helper::BuildConvOperationGraph<
      CUDNN_BACKEND_OPERATION_CONVOLUTION_FORWARD_DESCRIPTOR>(
      input_tensor,
      output_tensor,
      filter_channel_tensor,
      layout_format,
      strides,
      padding_common,
      dilations,
      dtype,
      handle,
      alpha,
      beta);

  if (plan_cache.FindPlan(op_graph, handle)) {
    const cudnn_frontend::ExecutionPlan* cached_plan = nullptr;
    int64_t workspace_size = 0;
    plan_cache.GetPlanAndWorkspaceSize(
        op_graph, &cached_plan, &workspace_size, handle);
    helper::ExecutePlan(handle,
                        &workspace_handle,
                        input_data,
                        output_data,
                        filter_data,
                        cached_plan->get_raw_desc(),
                        workspace_size);
    return;
  }

  auto plans = helper::FindExecutionPlans(&op_graph,
                                          exhaustive_search,
                                          deterministic,
                                          input_data,
                                          output_data,
                                          filter_data,
                                          handle,
                                          &workspace_handle,
                                          input_tensor->dtype());

  helper::ExecutePlansAndCache(handle,
                               &workspace_handle,
                               input_data,
                               output_data,
                               filter_data,
                               &plans,
                               exhaustive_search,
                               op_graph,
                               &plan_cache);
}
#endif

template <typename T, typename Context>
void ConvCudnnKernel(const Context& dev_ctx,
                     const DenseTensor& input,
                     const DenseTensor& filter,
                     const std::vector<int>& strides,
                     const std::vector<int>& paddings_t,
                     const std::string& padding_algorithm,
                     const std::vector<int>& dilations_t,
                     int groups,
                     const std::string& data_format,
                     DenseTensor* output) {
  if (input.numel() == 0 || filter.numel() == 0) {
    Full<T, Context>(dev_ctx, output->dims(), 0, output);
    return;
  }
  dev_ctx.template Alloc<T>(output);
  std::vector<int> paddings = paddings_t;
  std::vector<int> dilations = dilations_t;

  bool has_exhaustive_search = dev_ctx.HasDnnAttr("exhaustive_search");
  VLOG(4) << "GPUContext contains `exhaustive_search`: "
          << has_exhaustive_search;
  bool exhaustive_search_attr =
      has_exhaustive_search
          ? PADDLE_GET_CONST(bool, dev_ctx.GetDnnAttr("exhaustive_search"))
          : false;
  bool exhaustive_search =
      FLAGS_cudnn_exhaustive_search || exhaustive_search_attr;
  bool deterministic = FLAGS_cudnn_deterministic;
  PADDLE_ENFORCE_EQ(exhaustive_search && deterministic,
                    false,
                    common::errors::InvalidArgument(
                        "Can't set exhaustive_search True and "
                        "FLAGS_cudnn_deterministic True at same time."));

  const bool channel_last = (data_format == "NHWC" || data_format == "NDHWC");
  auto dtype = phi::backends::gpu::CudnnDataType<T>::type;

#ifdef PADDLE_WITH_HIP
  // HIP MIOPEN ONLY SUPPORT NCHW format
  auto compute_format = DataLayout::NCHW;
#else
#if CUDNN_VERSION_MIN(8, 1, 0)
  const bool compute_in_nhwc =
      FLAGS_use_accuracy_compatible_kernel
          ? channel_last
          : ((dtype == CUDNN_DATA_HALF || dtype == CUDNN_DATA_BFLOAT16) &&
             IsVoltaOrLater(dev_ctx));
#else
  // Tensor Core introduced from Volta GPUs supports more faster conv op
  // with FP16 in NHWC data format. (BF16 require cudnn >= 8.1.0)
  const bool compute_in_nhwc =
      dtype == CUDNN_DATA_HALF && IsVoltaOrLater(dev_ctx);
#endif
  // We will only do data format conversion from NHWC to NCHW.
  // cudnn will convert NCHW to NHWC automatically on Tensor Core.
  auto compute_format =
      compute_in_nhwc && channel_last ? DataLayout::NHWC : DataLayout::NCHW;
#endif
  VLOG(3) << "Compute ConvOp with cuDNN:"
          << " data_format=" << data_format << " compute_format="
          << (compute_format == DataLayout::NHWC ? "NHWC" : "NCHW");

  // ------------ transformed tensor -----------
  DenseTensor transformed_input_channel(input.type());
  DenseTensor transformed_output(output->type());
  DenseTensor transformed_filter_channel(filter.type());

  if (channel_last && compute_format == DataLayout::NCHW) {
    VLOG(3) << "Transform input tensor from NHWC to NCHW.";
    ResizeToChannelFirst<Context, T>(
        dev_ctx, &input, &transformed_input_channel);
    TransToChannelFirst<Context, T>(
        dev_ctx, &input, &transformed_input_channel);

    ResizeToChannelFirst<Context, T>(dev_ctx, output, &transformed_output);

  } else {
    transformed_input_channel.ShareDataWith(input);
    transformed_output.ShareDataWith(*output);
  }
  if (compute_format == DataLayout::NHWC && !FLAGS_manually_trans_conv_filter) {
    VLOG(3) << "Transform filter tensor from NCHW to NHWC.";
    ResizeToChannelLast<Context, T>(
        dev_ctx, &filter, &transformed_filter_channel);
    TransToChannelLast<Context, T>(
        dev_ctx, &filter, &transformed_filter_channel);
  } else {
    transformed_filter_channel.ShareDataWith(filter);
  }

  // update padding and dilation
  auto in_dims = transformed_input_channel.dims();
  auto filter_dims = transformed_filter_channel.dims();
  DDim in_data_dims;
  DDim filter_data_dims;

  if (compute_format == DataLayout::NCHW) {
    in_data_dims = slice_ddim(in_dims, 2, in_dims.size());
    filter_data_dims = slice_ddim(filter_dims, 2, filter_dims.size());
  } else {
    in_data_dims = slice_ddim(in_dims, 1, in_dims.size() - 1);
    filter_data_dims = slice_ddim(filter_dims, 1, filter_dims.size() - 1);
  }

  std::vector<int> ksize = vectorize<int>(filter_data_dims);
  UpdatePaddingAndDilation(
      &paddings, &dilations, padding_algorithm, in_data_dims, strides, ksize);

  int data_dim = strides.size();  // 2d or 3d
  bool is_sys_pad = funcs::IsSymmetricPadding(paddings, data_dim);

  DenseTensor transformed_input;
  std::vector<int> padding_common(data_dim, 0);
  if (!is_sys_pad) {
    std::vector<int> padding_diff(data_dim);
    std::vector<int> new_input_shape_vec(data_dim + 2);
    new_input_shape_vec[0] = transformed_input_channel.dims()[0];

    if (compute_format == DataLayout::NCHW) {
      new_input_shape_vec[1] = transformed_input_channel.dims()[1];
    } else {
      new_input_shape_vec[data_dim + 1] =
          transformed_input_channel.dims()[data_dim + 1];
    }

    std::vector<int> input_pad(transformed_input_channel.dims().size() * 2, 0);
    for (size_t i = 0; i < data_dim; ++i) {
      padding_diff[i] = std::abs(paddings[2 * i] - paddings[2 * i + 1]);
      padding_common[i] = std::min(paddings[2 * i], paddings[2 * i + 1]);
      if (compute_format == DataLayout::NCHW) {
        new_input_shape_vec[i + 2] =
            transformed_input_channel.dims()[i + 2] + padding_diff[i];
      } else {
        new_input_shape_vec[i + 1] =
            transformed_input_channel.dims()[i + 1] + padding_diff[i];
      }
      if (compute_format == DataLayout::NCHW) {
        input_pad[2 * i + 4] = paddings[2 * i] - padding_common[i];
        input_pad[2 * i + 4 + 1] = paddings[2 * i + 1] - padding_common[i];
      } else {
        input_pad[2 * i + 2] = paddings[2 * i] - padding_common[i];
        input_pad[2 * i + 2 + 1] = paddings[2 * i + 1] - padding_common[i];
      }
    }
    DDim new_input_shape(make_ddim(new_input_shape_vec));
    transformed_input.Resize(new_input_shape);
    dev_ctx.template Alloc<T>(&transformed_input);

    const int rank = transformed_input_channel.dims().size();
    T pad_value(0.0);
    switch (rank) {
      case 4: {
        funcs::PadFunction<Context, T, 4>(dev_ctx,
                                          input_pad,
                                          transformed_input_channel,
                                          pad_value,
                                          &transformed_input);
      } break;
      case 5: {
        funcs::PadFunction<Context, T, 5>(dev_ctx,
                                          input_pad,
                                          transformed_input_channel,
                                          pad_value,
                                          &transformed_input);
      } break;
      default:
        PADDLE_THROW(common::errors::InvalidArgument(
            "ConvOp only support tensors with 4 or 5 dimensions."));
    }

  } else {
    transformed_input.ShareDataWith(transformed_input_channel);
    if (paddings.size() == data_dim) {
      for (size_t i = 0; i < data_dim; ++i) {
        padding_common[i] = paddings[i];
      }
    } else {
      for (size_t i = 0; i < data_dim; ++i) {
        padding_common[i] = paddings[2 * i];
      }
    }
  }

  DataLayout layout =
      compute_format == DataLayout::NHWC ? DataLayout::NHWC : DataLayout::NCHW;
  if (transformed_input.dims().size() == 5) {
    layout = compute_format == DataLayout::NHWC ? DataLayout::NDHWC
                                                : DataLayout::NCDHW;
  }

  CUDNN_ENFORCE_TENSOR_SIZE_SUPPORTED(transformed_input);
  CUDNN_ENFORCE_TENSOR_SIZE_SUPPORTED(transformed_filter_channel);
  CUDNN_ENFORCE_TENSOR_SIZE_SUPPORTED(transformed_output);
#ifdef PADDLE_WITH_CUDNN_FRONTEND
  if (dynload ::IsCudnnFrontendEnabled() &&
      (FLAGS_use_accuracy_compatible_kernel || groups == 1))
    ConvCudnnKernelImplV8<T>(&transformed_input,
                             &transformed_filter_channel,
                             dev_ctx,
                             strides,
                             padding_common,
                             dilations,
                             layout,
                             exhaustive_search,
                             deterministic,
                             groups,
                             &transformed_output);
  else
    ConvCudnnKernelImplV7<T>(&transformed_input,
                             &transformed_filter_channel,
                             dev_ctx,
                             strides,
                             padding_common,
                             dilations,
                             compute_format,
                             layout,
                             exhaustive_search,
                             deterministic,
                             groups,
                             &transformed_output);
#else
  ConvCudnnKernelImplV7<T>(&transformed_input,
                           &transformed_filter_channel,
                           dev_ctx,
                           strides,
                           padding_common,
                           dilations,
                           compute_format,
                           layout,
                           exhaustive_search,
                           deterministic,
                           groups,
                           &transformed_output);
#endif

  if (channel_last && compute_format == DataLayout::NCHW) {
    TransToChannelLast<Context, T>(dev_ctx, &transformed_output, output);
  }
}

template <typename T, typename Context>
void Conv3DCudnnKernel(const Context& dev_ctx,
                       const DenseTensor& input,
                       const DenseTensor& filter,
                       const std::vector<int>& strides,
                       const std::vector<int>& paddings,
                       const std::string& padding_algorithm,
                       int groups,
                       const std::vector<int>& dilations,
                       const std::string& data_format,
                       DenseTensor* out) {
  ConvCudnnKernel<T>(dev_ctx,
                     input,
                     filter,
                     strides,
                     paddings,
                     padding_algorithm,
                     dilations,
                     groups,
                     data_format,
                     out);
}
}  // namespace phi

#ifdef PADDLE_WITH_HIP
PD_REGISTER_KERNEL(conv2d,
                   GPUDNN,
                   ALL_LAYOUT,
                   phi::ConvCudnnKernel,
                   float,
                   phi::float16,
                   phi::bfloat16) {}

PD_REGISTER_KERNEL(conv3d,
                   GPUDNN,
                   ALL_LAYOUT,
                   phi::Conv3DCudnnKernel,
                   float,
                   phi::float16,
                   phi::bfloat16) {}

PD_REGISTER_KERNEL(depthwise_conv2d,
                   GPUDNN,
                   ALL_LAYOUT,
                   phi::DepthwiseConvCudnnKernel,
                   float,
                   phi::float16,
                   phi::bfloat16) {}

#else
#if CUDNN_VERSION_MIN(8, 1, 0)
PD_REGISTER_KERNEL(conv2d,
                   GPUDNN,
                   ALL_LAYOUT,
                   phi::ConvCudnnKernel,
                   float,
                   double,
                   phi::float16,
                   phi::bfloat16) {}

PD_REGISTER_KERNEL(conv3d,
                   GPUDNN,
                   ALL_LAYOUT,
                   phi::Conv3DCudnnKernel,
                   float,
                   double,
                   phi::float16,
                   phi::bfloat16) {}
#elif CUDNN_VERSION_MIN(8, 6, 0) && CUDA_VERSION >= 11800 && \
    defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 890
PD_REGISTER_KERNEL(conv2d,
                   GPUDNN,
                   ALL_LAYOUT,
                   phi::ConvCudnnKernel,
                   float,
                   double,
                   phi::float8_e4m3fn,
                   phi::float16,
                   phi::bfloat16) {}
#else
PD_REGISTER_KERNEL(conv2d,
                   GPUDNN,
                   ALL_LAYOUT,
                   phi::ConvCudnnKernel,
                   float,
                   double,
                   phi::float16) {}

PD_REGISTER_KERNEL(conv3d,
                   GPUDNN,
                   ALL_LAYOUT,
                   phi::Conv3DCudnnKernel,
                   float,
                   double,
                   phi::float16) {}
#endif

#endif