chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,953 @@
|
||||
// Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <numeric>
|
||||
#include <type_traits>
|
||||
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "paddle/common/flags.h"
|
||||
#include "paddle/phi/backends/gpu/gpu_context.h"
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/concat_kernel.h"
|
||||
#include "paddle/phi/kernels/contiguous_kernel.h"
|
||||
#include "paddle/phi/kernels/conv_kernel.h"
|
||||
#include "paddle/phi/kernels/cpu/conv_util.h"
|
||||
#include "paddle/phi/kernels/elementwise_add_kernel.h"
|
||||
#include "paddle/phi/kernels/fill_kernel.h"
|
||||
#include "paddle/phi/kernels/full_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/batch_norm_utils.h"
|
||||
#include "paddle/phi/kernels/funcs/blas/blas.h"
|
||||
#include "paddle/phi/kernels/funcs/im2col_slow.cuh"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
#include "paddle/phi/kernels/funcs/vol2col_slow.cuh"
|
||||
#include "paddle/phi/kernels/reduce_sum_kernel.h"
|
||||
#include "paddle/phi/kernels/slice_kernel.h"
|
||||
|
||||
COMMON_DECLARE_bool(use_accuracy_compatible_kernel);
|
||||
|
||||
namespace phi {
|
||||
template <typename T>
|
||||
static inline T div_rtn(T x, T y) {
|
||||
int q = x / y;
|
||||
int r = x % y;
|
||||
if ((r != 0) && ((r < 0) != (y < 0))) --q;
|
||||
return q;
|
||||
}
|
||||
|
||||
template <typename C,
|
||||
std::enable_if_t<std::is_integral_v<typename C::value_type>, int> = 0>
|
||||
inline int64_t multiply_integers(const C& container) {
|
||||
return std::accumulate(container.begin(),
|
||||
container.end(),
|
||||
static_cast<int64_t>(1),
|
||||
std::multiplies<>());
|
||||
}
|
||||
|
||||
template <typename Iter,
|
||||
std::enable_if_t<std::is_integral_v<
|
||||
typename std::iterator_traits<Iter>::value_type>,
|
||||
int> = 0>
|
||||
inline int64_t multiply_integers(Iter begin, Iter end) {
|
||||
return std::accumulate(
|
||||
begin, end, static_cast<int64_t>(1), std::multiplies<>());
|
||||
}
|
||||
|
||||
template <int64_t dim>
|
||||
std::vector<int64_t> GetOutputSpatialSize(
|
||||
const DenseTensor& input,
|
||||
const std::vector<int64_t>& kernel_size,
|
||||
const std::vector<int64_t>& stride_size,
|
||||
const std::vector<int64_t>& pad_size,
|
||||
const std::vector<int64_t>& dilation_size) {
|
||||
std::vector<int64_t> sizes;
|
||||
auto input_dim = input.dims().size();
|
||||
|
||||
for (int64_t index = 0; index < dim; ++index) {
|
||||
int64_t input_size = input.dims()[index + input_dim - dim];
|
||||
int64_t kernel = kernel_size[index];
|
||||
int64_t stride = stride_size[index];
|
||||
int64_t pad = pad_size[index];
|
||||
int64_t dilation = dilation_size[index];
|
||||
|
||||
int64_t numerator = input_size + 2 * pad - (dilation * (kernel - 1) + 1);
|
||||
int64_t size = div_rtn<int64_t>(numerator, stride) + 1;
|
||||
|
||||
sizes.push_back(size);
|
||||
}
|
||||
|
||||
return sizes;
|
||||
}
|
||||
|
||||
template <int64_t dim>
|
||||
std::vector<int64_t> GetOutputSize(const DenseTensor& input,
|
||||
const DenseTensor& weight,
|
||||
const std::vector<int64_t>& kernel_size,
|
||||
const std::vector<int64_t>& stride_size,
|
||||
const std::vector<int64_t>& pad_size,
|
||||
const std::vector<int64_t>& dilation_size) {
|
||||
auto output_size = GetOutputSpatialSize<dim>(
|
||||
input, kernel_size, stride_size, pad_size, dilation_size);
|
||||
|
||||
output_size.insert(output_size.begin(), weight.dims()[0]);
|
||||
|
||||
if (input.dims().size() == dim + 2) {
|
||||
output_size.insert(output_size.begin(), input.dims()[0]);
|
||||
}
|
||||
|
||||
return output_size;
|
||||
}
|
||||
|
||||
template <typename T, typename Context, int64_t dim>
|
||||
void hvol2col(const Context& dev_ctx,
|
||||
const T* data_hvol,
|
||||
int channels,
|
||||
const std::vector<int64_t>& input_size,
|
||||
const std::vector<int64_t>& output_size,
|
||||
const std::vector<int64_t>& kernel_size,
|
||||
const std::vector<int64_t>& stride_size,
|
||||
const std::vector<int64_t>& pad_size,
|
||||
const std::vector<int64_t>& dilation_size,
|
||||
T* data_col) {
|
||||
if (dim == 3) {
|
||||
funcs::vol2col_slow<T, Context>(dev_ctx,
|
||||
data_hvol,
|
||||
channels,
|
||||
input_size[0],
|
||||
input_size[1],
|
||||
input_size[2],
|
||||
output_size[0],
|
||||
output_size[1],
|
||||
output_size[2],
|
||||
kernel_size[0],
|
||||
kernel_size[1],
|
||||
kernel_size[2],
|
||||
pad_size[0],
|
||||
pad_size[1],
|
||||
pad_size[2],
|
||||
stride_size[0],
|
||||
stride_size[1],
|
||||
stride_size[2],
|
||||
dilation_size[0],
|
||||
dilation_size[1],
|
||||
dilation_size[2],
|
||||
data_col);
|
||||
} else if (dim == 2) {
|
||||
funcs::im2col_slow<T, Context>(dev_ctx,
|
||||
data_hvol,
|
||||
channels,
|
||||
input_size[0],
|
||||
input_size[1],
|
||||
output_size[0],
|
||||
output_size[1],
|
||||
kernel_size[0],
|
||||
kernel_size[1],
|
||||
pad_size[0],
|
||||
pad_size[1],
|
||||
stride_size[0],
|
||||
stride_size[1],
|
||||
dilation_size[0],
|
||||
dilation_size[1],
|
||||
data_col);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename Context, int64_t dim>
|
||||
void col2hvol(const Context& dev_ctx,
|
||||
const T* data_col,
|
||||
const int channels,
|
||||
const std::vector<int64_t>& input_size,
|
||||
const std::vector<int64_t>& output_size,
|
||||
const std::vector<int64_t>& kernel_size,
|
||||
const std::vector<int64_t>& stride_size,
|
||||
const std::vector<int64_t>& pad_size,
|
||||
const std::vector<int64_t>& dilation_size,
|
||||
T* data_hvol) {
|
||||
if (dim == 3) {
|
||||
funcs::col2vol_slow<T, T, Context>(dev_ctx,
|
||||
data_col,
|
||||
channels,
|
||||
input_size[0],
|
||||
input_size[1],
|
||||
input_size[2],
|
||||
output_size[0],
|
||||
output_size[1],
|
||||
output_size[2],
|
||||
kernel_size[0],
|
||||
kernel_size[1],
|
||||
kernel_size[2],
|
||||
pad_size[0],
|
||||
pad_size[1],
|
||||
pad_size[2],
|
||||
stride_size[0],
|
||||
stride_size[1],
|
||||
stride_size[2],
|
||||
dilation_size[0],
|
||||
dilation_size[1],
|
||||
dilation_size[2],
|
||||
data_hvol);
|
||||
}
|
||||
if (dim == 2) {
|
||||
funcs::col2im_slow<T, T, Context>(dev_ctx,
|
||||
data_col,
|
||||
channels,
|
||||
input_size[0],
|
||||
input_size[1],
|
||||
output_size[0],
|
||||
output_size[1],
|
||||
kernel_size[0],
|
||||
kernel_size[1],
|
||||
pad_size[0],
|
||||
pad_size[1],
|
||||
stride_size[0],
|
||||
stride_size[1],
|
||||
dilation_size[0],
|
||||
dilation_size[1],
|
||||
data_hvol);
|
||||
}
|
||||
}
|
||||
|
||||
// Select View function
|
||||
template <typename T>
|
||||
DenseTensor Select(const DenseTensor& src, int64_t index) {
|
||||
DenseTensor out;
|
||||
out.ShareDataWith(src);
|
||||
auto dims = src.dims();
|
||||
std::vector<int64_t> new_dims;
|
||||
for (int i = 1; i < dims.size(); ++i) {
|
||||
new_dims.push_back(dims[i]);
|
||||
}
|
||||
out.Resize(new_dims);
|
||||
int64_t stride_0 = src.numel() / dims[0];
|
||||
size_t offset_bytes = index * stride_0 * sizeof(T);
|
||||
out.set_offset(src.offset() + offset_bytes);
|
||||
return out;
|
||||
}
|
||||
|
||||
template <typename T, typename Context, int Dims>
|
||||
void SlowConvDilatedAllCUDAImpl(const Context& dev_ctx,
|
||||
DenseTensor* output,
|
||||
const DenseTensor* input,
|
||||
const DenseTensor* weight,
|
||||
const DenseTensor* bias,
|
||||
const DenseTensor* grad_output,
|
||||
DenseTensor* grad_input,
|
||||
DenseTensor* grad_weight,
|
||||
DenseTensor* grad_bias,
|
||||
const std::vector<int64_t>& kernel_size,
|
||||
const std::vector<int64_t>& strides,
|
||||
const std::vector<int64_t>& paddings,
|
||||
const std::vector<int64_t>& dilations) {
|
||||
const int64_t batch_size = input->dims()[0];
|
||||
const int64_t input_channels = weight->dims()[1];
|
||||
const int64_t output_channels = weight->dims()[0];
|
||||
|
||||
std::vector<int64_t> input_spatial_size;
|
||||
for (int i = 2; i < input->dims().size(); ++i) {
|
||||
input_spatial_size.push_back(input->dims()[i]);
|
||||
}
|
||||
|
||||
std::vector<int64_t> output_spatial_size = GetOutputSpatialSize<Dims>(
|
||||
*input, kernel_size, strides, paddings, dilations);
|
||||
|
||||
int64_t kernel_volume = multiply_integers(kernel_size);
|
||||
int64_t output_volume = multiply_integers(output_spatial_size);
|
||||
|
||||
// Buffer
|
||||
int64_t col_dim0 = input_channels * kernel_volume;
|
||||
int64_t col_dim1 = output_volume;
|
||||
|
||||
DenseTensor columns;
|
||||
if (output || grad_weight || grad_input) {
|
||||
columns.Resize({col_dim0, col_dim1});
|
||||
dev_ctx.template Alloc<T>(&columns);
|
||||
}
|
||||
|
||||
// Initialize
|
||||
funcs::SetConstant<Context, T> set_zero;
|
||||
if (grad_weight) set_zero(dev_ctx, grad_weight, static_cast<T>(0));
|
||||
if (grad_bias) set_zero(dev_ctx, grad_bias, static_cast<T>(0));
|
||||
if (output && !bias) set_zero(dev_ctx, output, static_cast<T>(0));
|
||||
|
||||
// Bias CPU Mirror
|
||||
DenseTensor bias_cpu;
|
||||
const T* bias_cpu_data = nullptr;
|
||||
if (output && bias) {
|
||||
Copy(dev_ctx, *bias, CPUPlace(), true, &bias_cpu);
|
||||
bias_cpu_data = bias_cpu.data<T>();
|
||||
}
|
||||
|
||||
DenseTensor grad_output_n;
|
||||
std::vector<int64_t> sum_axes;
|
||||
for (int i = 0; i < Dims; ++i) sum_axes.push_back(i + 1);
|
||||
|
||||
auto blas = funcs::GetBlas<Context, T>(dev_ctx);
|
||||
|
||||
for (int elt = 0; elt < batch_size; ++elt) {
|
||||
T* columns_ptr = columns.data<T>();
|
||||
|
||||
// Prepare Input Slice View
|
||||
DenseTensor input_n = Select<T>(*input, elt);
|
||||
const T* input_ptr_raw = input_n.data<T>();
|
||||
|
||||
// Forward
|
||||
if (output) {
|
||||
DenseTensor output_n = Select<T>(*output, elt);
|
||||
T* output_ptr_raw = output_n.data<T>();
|
||||
|
||||
if (bias) {
|
||||
for (int n = 0; n < output_channels; ++n) {
|
||||
DenseTensor out_slice = Select<T>(output_n, n);
|
||||
|
||||
FillKernel<T, Context>(
|
||||
dev_ctx, out_slice, Scalar(bias_cpu_data[n]), &out_slice);
|
||||
}
|
||||
}
|
||||
|
||||
hvol2col<T, Context, Dims>(dev_ctx,
|
||||
input_ptr_raw,
|
||||
input_channels,
|
||||
input_spatial_size,
|
||||
output_spatial_size,
|
||||
kernel_size,
|
||||
strides,
|
||||
paddings,
|
||||
dilations,
|
||||
columns_ptr);
|
||||
blas.GEMM(false, // TransA
|
||||
false, // TransB
|
||||
static_cast<int>(output_channels), // M
|
||||
static_cast<int>(col_dim1), // N
|
||||
static_cast<int>(col_dim0), // K
|
||||
static_cast<T>(1), // alpha
|
||||
weight->data<T>(), // A
|
||||
static_cast<int>(col_dim0), // lda
|
||||
columns_ptr, // B
|
||||
static_cast<int>(col_dim1), // ldb
|
||||
static_cast<T>(1), // beta = 1 (Accumulate)
|
||||
output_ptr_raw, // C
|
||||
static_cast<int>(col_dim1) // ldc
|
||||
);
|
||||
} else {
|
||||
grad_output_n = Select<T>(*grad_output, elt);
|
||||
}
|
||||
|
||||
// Backward Grad Input
|
||||
if (grad_input) {
|
||||
DenseTensor grad_input_n = Select<T>(*grad_input, elt);
|
||||
T* grad_input_ptr_raw = grad_input_n.data<T>();
|
||||
const T* grad_output_ptr_raw = grad_output_n.data<T>();
|
||||
|
||||
blas.GEMM(true, // TransA
|
||||
false, // TransB
|
||||
static_cast<int>(col_dim0), // M
|
||||
static_cast<int>(col_dim1), // N
|
||||
static_cast<int>(output_channels), // K
|
||||
static_cast<T>(1), // alpha
|
||||
weight->data<T>(), // A
|
||||
static_cast<int>(col_dim0), // lda
|
||||
grad_output_ptr_raw, // B
|
||||
static_cast<int>(col_dim1), // ldb
|
||||
static_cast<T>(0), // beta
|
||||
columns_ptr, // C
|
||||
static_cast<int>(col_dim1) // ldc
|
||||
);
|
||||
|
||||
col2hvol<T, Context, Dims>(dev_ctx,
|
||||
columns_ptr,
|
||||
input_channels,
|
||||
input_spatial_size,
|
||||
output_spatial_size,
|
||||
kernel_size,
|
||||
strides,
|
||||
paddings,
|
||||
dilations,
|
||||
grad_input_ptr_raw);
|
||||
}
|
||||
|
||||
// Backward Grad Weight
|
||||
if (grad_weight) {
|
||||
const T* grad_output_ptr_raw = grad_output_n.data<T>();
|
||||
|
||||
hvol2col<T, Context, Dims>(dev_ctx,
|
||||
input_ptr_raw,
|
||||
input_channels,
|
||||
input_spatial_size,
|
||||
output_spatial_size,
|
||||
kernel_size,
|
||||
strides,
|
||||
paddings,
|
||||
dilations,
|
||||
columns_ptr);
|
||||
|
||||
blas.GEMM(false, // TransA
|
||||
true, // TransB
|
||||
static_cast<int>(output_channels), // M
|
||||
static_cast<int>(col_dim0), // N
|
||||
static_cast<int>(col_dim1), // K
|
||||
static_cast<T>(1), // alpha
|
||||
grad_output_ptr_raw, // A
|
||||
static_cast<int>(col_dim1), // lda
|
||||
columns_ptr, // B
|
||||
static_cast<int>(col_dim1), // ldb
|
||||
static_cast<T>(1), // beta
|
||||
grad_weight->data<T>(), // C
|
||||
static_cast<int>(col_dim0) // ldc
|
||||
);
|
||||
}
|
||||
|
||||
// Backward Grad Bias
|
||||
if (grad_bias) {
|
||||
DenseTensor sum_result = Sum<T, Context>(dev_ctx,
|
||||
grad_output_n,
|
||||
IntArray(sum_axes),
|
||||
CppTypeToDataType<T>::Type(),
|
||||
false);
|
||||
Add<T, Context>(dev_ctx, *grad_bias, sum_result, grad_bias);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename Context, int64_t dim>
|
||||
void SlowConvBackwardNoGroup(const Context& dev_ctx,
|
||||
const DenseTensor& grad_output,
|
||||
const DenseTensor& input,
|
||||
const DenseTensor& weight,
|
||||
const std::vector<int64_t>& kernel_size,
|
||||
const std::vector<int64_t>& strides,
|
||||
const std::vector<int64_t>& paddings,
|
||||
const std::vector<int64_t>& dilations,
|
||||
DenseTensor* grad_input,
|
||||
DenseTensor* grad_weight,
|
||||
DenseTensor* grad_bias) {
|
||||
int64_t rank = input.dims().size();
|
||||
bool is_batch = (rank == (dim + 2));
|
||||
|
||||
// tensor.unsqueeze(0)
|
||||
auto make_batch_view = [&](const DenseTensor& src, DenseTensor& dst) {
|
||||
if (!is_batch) {
|
||||
dst.ShareDataWith(src);
|
||||
std::vector<int64_t> new_shape = {1};
|
||||
for (int i = 0; i < src.dims().size(); ++i)
|
||||
new_shape.push_back(src.dims()[i]);
|
||||
dst.Resize(new_shape);
|
||||
} else {
|
||||
dst.ShareDataWith(src);
|
||||
}
|
||||
};
|
||||
|
||||
DenseTensor grad_output_;
|
||||
make_batch_view(grad_output, grad_output_);
|
||||
|
||||
DenseTensor input_;
|
||||
make_batch_view(input, input_);
|
||||
|
||||
const DenseTensor& weight_ = weight;
|
||||
|
||||
DenseTensor grad_input_view;
|
||||
DenseTensor* grad_input_ptr = nullptr;
|
||||
|
||||
if (grad_input) {
|
||||
dev_ctx.template Alloc<T>(grad_input);
|
||||
|
||||
if (!is_batch) {
|
||||
grad_input_view.ShareDataWith(*grad_input);
|
||||
std::vector<int64_t> new_shape = {1};
|
||||
for (int i = 0; i < grad_input->dims().size(); ++i)
|
||||
new_shape.push_back(grad_input->dims()[i]);
|
||||
grad_input_view.Resize(new_shape);
|
||||
grad_input_ptr = &grad_input_view;
|
||||
} else {
|
||||
grad_input_ptr = grad_input;
|
||||
}
|
||||
}
|
||||
|
||||
DenseTensor* grad_weight_ptr = nullptr;
|
||||
if (grad_weight) {
|
||||
dev_ctx.template Alloc<T>(grad_weight);
|
||||
grad_weight_ptr = grad_weight;
|
||||
}
|
||||
|
||||
DenseTensor* grad_bias_ptr = nullptr;
|
||||
if (grad_bias) {
|
||||
dev_ctx.template Alloc<T>(grad_bias);
|
||||
grad_bias_ptr = grad_bias;
|
||||
}
|
||||
|
||||
SlowConvDilatedAllCUDAImpl<T, Context, dim>(
|
||||
dev_ctx,
|
||||
nullptr, // [Output]
|
||||
&input_, // [Input]
|
||||
&weight_, // [Weight]
|
||||
nullptr, // [Bias]
|
||||
&grad_output_, // [GradOutput]
|
||||
grad_input_ptr, // [GradInput]
|
||||
grad_weight_ptr, // [GradWeight]
|
||||
grad_bias_ptr, // [GradBias] (New)
|
||||
kernel_size,
|
||||
strides,
|
||||
paddings,
|
||||
dilations);
|
||||
}
|
||||
|
||||
template <typename T, typename Context, int64_t dim>
|
||||
void SlowConvNoGroup(const Context& dev_ctx,
|
||||
const DenseTensor& input,
|
||||
const DenseTensor& weight,
|
||||
const DenseTensor* bias,
|
||||
const std::vector<int64_t>& kernel_size,
|
||||
const std::vector<int64_t>& strides,
|
||||
const std::vector<int64_t>& paddings,
|
||||
const std::vector<int64_t>& dilations,
|
||||
DenseTensor* output) {
|
||||
int64_t rank = input.dims().size();
|
||||
bool is_batch = (rank == (dim + 2));
|
||||
|
||||
// (is_batch ? input.contiguous() : input.contiguous().unsqueeze(0));
|
||||
DenseTensor input_;
|
||||
if (!is_batch) {
|
||||
input_.ShareDataWith(input);
|
||||
std::vector<int64_t> new_shape = {1};
|
||||
for (int i = 0; i < rank; ++i) new_shape.push_back(input.dims()[i]);
|
||||
input_.Resize(new_shape);
|
||||
} else {
|
||||
input_.ShareDataWith(input);
|
||||
}
|
||||
|
||||
const DenseTensor& weight_ = weight;
|
||||
|
||||
// (is_batch ? output : output.unsqueeze(0));
|
||||
if (output) dev_ctx.template Alloc<T>(output);
|
||||
|
||||
DenseTensor output_;
|
||||
if (!is_batch) {
|
||||
output_.ShareDataWith(*output);
|
||||
|
||||
std::vector<int64_t> out_shape = {1};
|
||||
for (int i = 0; i < output->dims().size(); ++i) {
|
||||
out_shape.push_back(output->dims()[i]);
|
||||
}
|
||||
output_.Resize(out_shape);
|
||||
} else {
|
||||
output_.ShareDataWith(*output);
|
||||
}
|
||||
|
||||
SlowConvDilatedAllCUDAImpl<T, Context, dim>(dev_ctx,
|
||||
&output_, // [Output]
|
||||
&input_, // [Input]
|
||||
&weight_, // [Weight]
|
||||
bias, // [Bias]
|
||||
nullptr, // [GradOutput]
|
||||
nullptr, // [GradInput]
|
||||
nullptr, // [GradWeight]
|
||||
nullptr, // [GradBias]
|
||||
kernel_size,
|
||||
strides,
|
||||
paddings,
|
||||
dilations);
|
||||
}
|
||||
|
||||
template <typename T, typename Context, int64_t dim>
|
||||
void SlowConvForward(const Context& dev_ctx,
|
||||
const DenseTensor& input,
|
||||
const DenseTensor& filter_t,
|
||||
const paddle::optional<DenseTensor>& bias,
|
||||
const std::vector<int>& strides,
|
||||
const std::vector<int>& paddings_t,
|
||||
const std::string& padding_algorithm,
|
||||
int groups,
|
||||
const std::vector<int>& dilations_t,
|
||||
const std::string& data_format,
|
||||
DenseTensor* output) {
|
||||
std::vector<int> paddings = paddings_t;
|
||||
std::vector<int> dilations = dilations_t;
|
||||
DenseTensor filter = filter_t;
|
||||
|
||||
if (input.numel() == 0 || filter.numel() == 0) {
|
||||
Full<T, Context>(dev_ctx, output->dims(), 0, output);
|
||||
return;
|
||||
}
|
||||
|
||||
dev_ctx.template Alloc<T>(output);
|
||||
|
||||
const bool channel_last = (data_format == "NHWC" || data_format == "NDHWC");
|
||||
|
||||
DenseTensor transformed_input(input.type());
|
||||
DenseTensor transformed_output(output->type());
|
||||
|
||||
if (channel_last) {
|
||||
ResizeToChannelFirst<Context, T>(dev_ctx, &input, &transformed_input);
|
||||
TransToChannelFirst<Context, T>(dev_ctx, &input, &transformed_input);
|
||||
|
||||
ResizeToChannelFirst<Context, T>(dev_ctx, output, &transformed_output);
|
||||
|
||||
} else {
|
||||
transformed_input = input;
|
||||
transformed_output = *output;
|
||||
}
|
||||
|
||||
// update padding and dilation
|
||||
auto trans_in_dims = transformed_input.dims();
|
||||
auto filter_dims = filter.dims();
|
||||
|
||||
DDim in_data_dims = slice_ddim(trans_in_dims, 2, trans_in_dims.size());
|
||||
DDim filter_data_dims = slice_ddim(filter_dims, 2, filter_dims.size());
|
||||
|
||||
std::vector<int> ksize = vectorize<int>(filter_data_dims);
|
||||
UpdatePaddingAndDilation(
|
||||
&paddings, &dilations, padding_algorithm, in_data_dims, strides, ksize);
|
||||
|
||||
// =================================================================
|
||||
// Contiguous & Grouping
|
||||
// =================================================================
|
||||
DenseTensor input_contiguous;
|
||||
ContiguousKernel<T, Context>(dev_ctx, transformed_input, &input_contiguous);
|
||||
|
||||
DenseTensor weight_contiguous;
|
||||
ContiguousKernel<T, Context>(dev_ctx, filter_t, &weight_contiguous);
|
||||
|
||||
auto to_int64_vec = [](const std::vector<int>& in) {
|
||||
return std::vector<int64_t>(in.begin(), in.end());
|
||||
};
|
||||
|
||||
const DenseTensor* bias_ptr = bias.get_ptr();
|
||||
DenseTensor bias_contiguous;
|
||||
|
||||
if (bias_ptr) {
|
||||
ContiguousKernel<T, Context>(dev_ctx, *bias_ptr, &bias_contiguous);
|
||||
bias_ptr = &bias_contiguous;
|
||||
}
|
||||
|
||||
if (groups == 1) {
|
||||
SlowConvNoGroup<T, Context, dim>(dev_ctx,
|
||||
input_contiguous,
|
||||
weight_contiguous,
|
||||
bias_ptr,
|
||||
to_int64_vec(ksize),
|
||||
to_int64_vec(strides),
|
||||
to_int64_vec(paddings),
|
||||
to_int64_vec(dilations),
|
||||
&transformed_output);
|
||||
|
||||
} else {
|
||||
int64_t in_rank = input_contiguous.dims().size();
|
||||
bool has_batch = (in_rank == dim + 2);
|
||||
int channel_dim = has_batch ? 1 : 0;
|
||||
|
||||
int64_t in_channels = input_contiguous.dims()[channel_dim];
|
||||
int64_t out_channels = weight_contiguous.dims()[0];
|
||||
|
||||
int64_t in_g_sz = in_channels / groups;
|
||||
int64_t out_g_sz = out_channels / groups;
|
||||
|
||||
std::vector<DenseTensor> outputs(groups);
|
||||
|
||||
for (int g = 0; g < groups; ++g) {
|
||||
// Slice Input (Channel)
|
||||
DenseTensor input_g;
|
||||
SliceKernel<T, Context>(dev_ctx,
|
||||
input_contiguous,
|
||||
{channel_dim},
|
||||
{g * in_g_sz},
|
||||
{(g + 1) * in_g_sz},
|
||||
{1},
|
||||
{},
|
||||
&input_g);
|
||||
|
||||
// Slice Weight (OutChannel dim 0)
|
||||
DenseTensor weight_g;
|
||||
SliceKernel<T, Context>(dev_ctx,
|
||||
weight_contiguous,
|
||||
{0},
|
||||
{g * out_g_sz},
|
||||
{(g + 1) * out_g_sz},
|
||||
{1},
|
||||
{},
|
||||
&weight_g);
|
||||
|
||||
// Slice Bias (OutChannel dim 0)
|
||||
DenseTensor bias_g;
|
||||
const DenseTensor* bias_g_ptr = nullptr;
|
||||
if (bias_ptr) {
|
||||
SliceKernel<T, Context>(dev_ctx,
|
||||
*bias_ptr,
|
||||
{0},
|
||||
{g * out_g_sz},
|
||||
{(g + 1) * out_g_sz},
|
||||
{1},
|
||||
{},
|
||||
&bias_g);
|
||||
bias_g_ptr = &bias_g;
|
||||
}
|
||||
|
||||
DenseTensor output_g;
|
||||
auto out_shape = transformed_output.dims();
|
||||
out_shape[channel_dim] = out_g_sz;
|
||||
output_g.Resize(out_shape);
|
||||
dev_ctx.template Alloc<T>(&output_g);
|
||||
|
||||
SlowConvNoGroup<T, Context, dim>(dev_ctx,
|
||||
input_g,
|
||||
weight_g,
|
||||
bias_g_ptr,
|
||||
to_int64_vec(ksize),
|
||||
to_int64_vec(strides),
|
||||
to_int64_vec(paddings),
|
||||
to_int64_vec(dilations),
|
||||
&output_g);
|
||||
|
||||
outputs[g] = output_g;
|
||||
}
|
||||
|
||||
// Concat
|
||||
std::vector<const DenseTensor*> outputs_ptr;
|
||||
for (auto& t : outputs) outputs_ptr.push_back(&t);
|
||||
|
||||
ConcatKernel<T, Context>(
|
||||
dev_ctx, outputs_ptr, channel_dim, &transformed_output);
|
||||
}
|
||||
|
||||
if (channel_last) {
|
||||
TransToChannelLast<Context, T>(dev_ctx, &transformed_output, output);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename Context, int64_t dim>
|
||||
void SlowConvBackward(const Context& dev_ctx,
|
||||
const DenseTensor& input,
|
||||
const DenseTensor& filter_t,
|
||||
const DenseTensor& output_grad,
|
||||
const std::vector<int>& strides,
|
||||
const std::vector<int>& paddings_t,
|
||||
const std::string& padding_algorithm,
|
||||
const std::vector<int>& dilations_t,
|
||||
int groups,
|
||||
const std::string& data_format,
|
||||
DenseTensor* input_grad,
|
||||
DenseTensor* filter_grad,
|
||||
DenseTensor* bias_grad) {
|
||||
if (!input_grad && !filter_grad && !bias_grad) return;
|
||||
std::vector<int> paddings = paddings_t;
|
||||
std::vector<int> dilations = dilations_t;
|
||||
|
||||
DenseTensor filter = filter_t;
|
||||
// 0-size
|
||||
if (input.numel() == 0 || filter_t.numel() == 0) {
|
||||
if (input_grad) dev_ctx.template Alloc<T>(input_grad);
|
||||
if (filter_grad) {
|
||||
Full<T, Context>(dev_ctx, filter_grad->dims(), 0, filter_grad);
|
||||
}
|
||||
if (bias_grad) {
|
||||
dev_ctx.template Alloc<T>(bias_grad);
|
||||
Full<T, Context>(dev_ctx, bias_grad->dims(), 0, bias_grad);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (input_grad) dev_ctx.template Alloc<T>(input_grad);
|
||||
if (filter_grad) dev_ctx.template Alloc<T>(filter_grad);
|
||||
if (bias_grad) dev_ctx.template Alloc<T>(bias_grad);
|
||||
|
||||
const bool channel_last = (data_format == "NHWC" || data_format == "NDHWC");
|
||||
|
||||
DenseTensor transformed_input(input.type());
|
||||
DenseTensor transformed_output_grad(output_grad.type());
|
||||
|
||||
if (channel_last) {
|
||||
ResizeToChannelFirst<Context, T>(dev_ctx, &input, &transformed_input);
|
||||
TransToChannelFirst<Context, T>(dev_ctx, &input, &transformed_input);
|
||||
|
||||
ResizeToChannelFirst<Context, T>(
|
||||
dev_ctx, &output_grad, &transformed_output_grad);
|
||||
TransToChannelFirst<Context, T>(
|
||||
dev_ctx, &output_grad, &transformed_output_grad);
|
||||
} else {
|
||||
transformed_input = input;
|
||||
transformed_output_grad = output_grad;
|
||||
}
|
||||
|
||||
// update padding and dilation
|
||||
auto in_dims = transformed_input.dims();
|
||||
auto filter_dims = filter.dims();
|
||||
DDim in_data_dims = slice_ddim(in_dims, 2, in_dims.size());
|
||||
DDim filter_data_dims = slice_ddim(filter_dims, 2, filter_dims.size());
|
||||
std::vector<int> ksize = vectorize<int>(filter_data_dims);
|
||||
UpdatePaddingAndDilation<int>(
|
||||
&paddings, &dilations, padding_algorithm, in_data_dims, strides, ksize);
|
||||
|
||||
// =================================================================
|
||||
// Contiguous & Grouping
|
||||
// =================================================================
|
||||
DenseTensor tmp_input_grad;
|
||||
DenseTensor* t_input_grad_ptr = nullptr;
|
||||
DenseTensor* t_filter_grad_ptr = filter_grad;
|
||||
DenseTensor* t_bias_grad_ptr = bias_grad;
|
||||
|
||||
if (input_grad) {
|
||||
if (channel_last) {
|
||||
tmp_input_grad.Resize(transformed_input.dims());
|
||||
t_input_grad_ptr = &tmp_input_grad;
|
||||
} else {
|
||||
t_input_grad_ptr = input_grad;
|
||||
}
|
||||
}
|
||||
|
||||
// Contiguous
|
||||
DenseTensor grad_output_cont;
|
||||
ContiguousKernel<T, Context>(
|
||||
dev_ctx, transformed_output_grad, &grad_output_cont);
|
||||
|
||||
DenseTensor input_cont;
|
||||
ContiguousKernel<T, Context>(dev_ctx, transformed_input, &input_cont);
|
||||
|
||||
DenseTensor weight_cont;
|
||||
ContiguousKernel<T, Context>(dev_ctx, filter, &weight_cont);
|
||||
|
||||
auto to_int64_vec = [](const std::vector<int>& in) {
|
||||
return std::vector<int64_t>(in.begin(), in.end());
|
||||
};
|
||||
|
||||
// Group
|
||||
if (groups == 1) {
|
||||
SlowConvBackwardNoGroup<T, Context, dim>(dev_ctx,
|
||||
grad_output_cont,
|
||||
input_cont,
|
||||
weight_cont,
|
||||
to_int64_vec(ksize),
|
||||
to_int64_vec(strides),
|
||||
to_int64_vec(paddings),
|
||||
to_int64_vec(dilations),
|
||||
t_input_grad_ptr,
|
||||
t_filter_grad_ptr,
|
||||
t_bias_grad_ptr);
|
||||
} else {
|
||||
int64_t in_rank = input_cont.dims().size();
|
||||
bool has_batch = (in_rank == dim + 2);
|
||||
int channel_dim = has_batch ? 1 : 0;
|
||||
|
||||
int64_t in_channels = input_cont.dims()[channel_dim];
|
||||
int64_t out_channels = grad_output_cont.dims()[channel_dim];
|
||||
|
||||
int64_t in_g_sz = in_channels / groups;
|
||||
int64_t out_g_sz = out_channels / groups;
|
||||
|
||||
std::vector<DenseTensor> grad_inputs_g(groups);
|
||||
std::vector<DenseTensor> grad_weights_g(groups);
|
||||
std::vector<DenseTensor> grad_biases_g(groups);
|
||||
|
||||
for (int g = 0; g < groups; ++g) {
|
||||
// Slice GradOutput (Channel)
|
||||
DenseTensor grad_output_g;
|
||||
SliceKernel<T, Context>(dev_ctx,
|
||||
grad_output_cont,
|
||||
{channel_dim},
|
||||
{g * out_g_sz},
|
||||
{(g + 1) * out_g_sz},
|
||||
{1},
|
||||
{},
|
||||
&grad_output_g);
|
||||
|
||||
// Slice Input (Channel)
|
||||
DenseTensor input_g;
|
||||
SliceKernel<T, Context>(dev_ctx,
|
||||
input_cont,
|
||||
{channel_dim},
|
||||
{g * in_g_sz},
|
||||
{(g + 1) * in_g_sz},
|
||||
{1},
|
||||
{},
|
||||
&input_g);
|
||||
|
||||
// Slice Weight (Output Channel / dim 0)
|
||||
DenseTensor weight_g;
|
||||
SliceKernel<T, Context>(dev_ctx,
|
||||
weight_cont,
|
||||
{0},
|
||||
{g * out_g_sz},
|
||||
{(g + 1) * out_g_sz},
|
||||
{1},
|
||||
{},
|
||||
&weight_g);
|
||||
|
||||
DenseTensor grad_input_g_tensor;
|
||||
DenseTensor grad_weight_g_tensor;
|
||||
DenseTensor grad_bias_g_tensor;
|
||||
|
||||
if (t_input_grad_ptr) {
|
||||
auto g_shape = t_input_grad_ptr->dims();
|
||||
g_shape[channel_dim] = in_g_sz;
|
||||
grad_input_g_tensor.Resize(g_shape);
|
||||
}
|
||||
if (t_filter_grad_ptr) {
|
||||
auto w_shape = t_filter_grad_ptr->dims();
|
||||
w_shape[0] = out_g_sz;
|
||||
grad_weight_g_tensor.Resize(w_shape);
|
||||
}
|
||||
if (t_bias_grad_ptr) {
|
||||
auto b_shape = t_bias_grad_ptr->dims();
|
||||
b_shape[0] = out_g_sz;
|
||||
grad_bias_g_tensor.Resize(b_shape);
|
||||
}
|
||||
|
||||
SlowConvBackwardNoGroup<T, Context, dim>(
|
||||
dev_ctx,
|
||||
grad_output_g,
|
||||
input_g,
|
||||
weight_g,
|
||||
to_int64_vec(ksize),
|
||||
to_int64_vec(strides),
|
||||
to_int64_vec(paddings),
|
||||
to_int64_vec(dilations),
|
||||
(t_input_grad_ptr ? &grad_input_g_tensor : nullptr),
|
||||
(t_filter_grad_ptr ? &grad_weight_g_tensor : nullptr),
|
||||
(t_bias_grad_ptr ? &grad_bias_g_tensor : nullptr));
|
||||
|
||||
if (t_input_grad_ptr) grad_inputs_g[g] = grad_input_g_tensor;
|
||||
if (t_filter_grad_ptr) grad_weights_g[g] = grad_weight_g_tensor;
|
||||
if (t_bias_grad_ptr) grad_biases_g[g] = grad_bias_g_tensor;
|
||||
}
|
||||
|
||||
// Concat Input Grad
|
||||
if (t_input_grad_ptr) {
|
||||
std::vector<const DenseTensor*> ptrs;
|
||||
for (auto& t : grad_inputs_g) ptrs.push_back(&t);
|
||||
ConcatKernel<T, Context>(dev_ctx, ptrs, channel_dim, t_input_grad_ptr);
|
||||
}
|
||||
|
||||
// Concat Weight Grad
|
||||
if (t_filter_grad_ptr) {
|
||||
std::vector<const DenseTensor*> ptrs;
|
||||
for (auto& t : grad_weights_g) ptrs.push_back(&t);
|
||||
ConcatKernel<T, Context>(dev_ctx, ptrs, 0, t_filter_grad_ptr);
|
||||
}
|
||||
|
||||
// Concat Bias Grad
|
||||
if (t_bias_grad_ptr) {
|
||||
std::vector<const DenseTensor*> ptrs;
|
||||
for (auto& t : grad_biases_g) ptrs.push_back(&t);
|
||||
ConcatKernel<T, Context>(dev_ctx, ptrs, 0, t_bias_grad_ptr);
|
||||
}
|
||||
}
|
||||
|
||||
if (channel_last && input_grad) {
|
||||
TransToChannelLast<Context, T>(dev_ctx, t_input_grad_ptr, input_grad);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
Reference in New Issue
Block a user