chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,141 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/abs_grad_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/complex_functors.h"
|
||||
#include "paddle/phi/kernels/funcs/elementwise_base.h"
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
#if defined(__NVCC__)
|
||||
|
||||
template <typename T>
|
||||
struct AbsGradCUDAFunctor {
|
||||
HOSTDEVICE inline AbsGradCUDAFunctor() {}
|
||||
|
||||
HOSTDEVICE inline T operator()(const T x, const T dout) const {
|
||||
T output;
|
||||
if (x == T(0)) {
|
||||
output = T(0);
|
||||
} else {
|
||||
output = T(dout) * (x / T(std::abs(x)));
|
||||
}
|
||||
return output;
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct AbsGradCUDAFunctor<bfloat16> {
|
||||
HOSTDEVICE inline AbsGradCUDAFunctor() {}
|
||||
|
||||
HOSTDEVICE inline bfloat16 operator()(const bfloat16 x,
|
||||
const bfloat16 dout) const {
|
||||
bfloat16 output;
|
||||
if (x == bfloat16(0)) {
|
||||
output = static_cast<bfloat16>(0);
|
||||
} else {
|
||||
output = (dout) * (x / abs(x));
|
||||
}
|
||||
return output;
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct AbsGradCUDAFunctor<complex64> {
|
||||
HOSTDEVICE inline AbsGradCUDAFunctor() {}
|
||||
HOSTDEVICE inline complex64 operator()(const complex64 x,
|
||||
const float dout) const {
|
||||
complex64 output;
|
||||
if (x == complex64(0)) {
|
||||
output = complex64(0);
|
||||
} else {
|
||||
output = complex64(dout) * (x / complex64(abs(x)));
|
||||
}
|
||||
return output;
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct AbsGradCUDAFunctor<complex128> {
|
||||
HOSTDEVICE inline AbsGradCUDAFunctor() {}
|
||||
HOSTDEVICE inline complex128 operator()(const complex128 x,
|
||||
const double dout) const {
|
||||
complex128 output;
|
||||
if (x == complex128(0)) {
|
||||
output = complex128(0);
|
||||
} else {
|
||||
output = complex128(dout) * (x / complex128(abs(x)));
|
||||
}
|
||||
return output;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
void AbsGradKernelImpl(const GPUContext& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& dout,
|
||||
DenseTensor* dx) {
|
||||
std::vector<const DenseTensor*> ins = {&x, &dout};
|
||||
std::vector<DenseTensor*> outs = {dx};
|
||||
dev_ctx.Alloc<T>(dx);
|
||||
AbsGradCUDAFunctor<T> abs_grad_cuda_functor;
|
||||
funcs::ElementwiseKernel<T>(dev_ctx, ins, &outs, abs_grad_cuda_functor);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void AbsGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& dout,
|
||||
DenseTensor* dx) {
|
||||
AbsGradKernelImpl<T>(dev_ctx, x, dout, dx);
|
||||
}
|
||||
#else
|
||||
template <typename T, typename Context>
|
||||
void AbsGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& dout,
|
||||
DenseTensor* dx) {
|
||||
auto numel = dout.numel();
|
||||
auto* dout_data = dout.data<dtype::Real<T>>();
|
||||
auto* x_data = x.data<T>();
|
||||
|
||||
dev_ctx.template Alloc<T>(dx, static_cast<size_t>(numel * sizeof(T)));
|
||||
auto* dx_data = dx->data<T>();
|
||||
|
||||
funcs::ForRange<Context> for_range(dev_ctx, numel);
|
||||
funcs::AbsGradFunctor<T> functor(dout_data, x_data, dx_data, numel);
|
||||
for_range(functor);
|
||||
}
|
||||
|
||||
#endif
|
||||
template <typename T, typename Context>
|
||||
void AbsDoubleGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& ddx,
|
||||
DenseTensor* ddout) {
|
||||
auto numel = ddx.numel();
|
||||
auto* ddx_data = ddx.data<T>();
|
||||
auto* x_data = x.data<T>();
|
||||
dev_ctx.template Alloc<T>(ddout, static_cast<size_t>(numel * sizeof(T)));
|
||||
auto* ddout_data = ddout->data<T>();
|
||||
|
||||
funcs::ForRange<Context> for_range(dev_ctx, numel);
|
||||
funcs::AbsGradGradFunctor<T> functor(ddx_data, x_data, ddout_data, numel);
|
||||
for_range(functor);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,276 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
#include <cmath>
|
||||
#include <string>
|
||||
#include "glog/logging.h"
|
||||
|
||||
#include "paddle/phi/backends/cpu/cpu_context.h"
|
||||
#include "paddle/phi/backends/gpu/gpu_context.h"
|
||||
#include "paddle/phi/common/amp_type_traits.h"
|
||||
#include "paddle/phi/common/data_type.h"
|
||||
#include "paddle/phi/common/place.h"
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
|
||||
#include "paddle/phi/common/memory_utils.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename Context, typename T>
|
||||
struct AccuracyCheckFunctor {
|
||||
void operator()(const Context& dev_ctx,
|
||||
const DenseTensor& in,
|
||||
const DenseTensor& other,
|
||||
const std::string& fn_name,
|
||||
const float rtol,
|
||||
const float atol,
|
||||
bool equal_nan,
|
||||
DenseTensor* output);
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct AccuracyCheckFunctor<CPUContext, T> {
|
||||
void operator()(const CPUContext& dev_ctx,
|
||||
const DenseTensor& in,
|
||||
const DenseTensor& other,
|
||||
const std::string& fn_name,
|
||||
const double rtol,
|
||||
const double atol,
|
||||
bool equal_nan,
|
||||
DenseTensor* output) {
|
||||
auto* in_a = in.data<T>();
|
||||
auto* in_b = other.data<T>();
|
||||
auto* out_data = dev_ctx.template Alloc<bool>(output);
|
||||
auto num = in.numel();
|
||||
// *out_data = true;
|
||||
for (int i = 0; i < num; i++) {
|
||||
out_data[i] = true;
|
||||
}
|
||||
bool val;
|
||||
int res_index = -1;
|
||||
for (int i = 0; i < num; i++) {
|
||||
const double a = in_a[i], b = in_b[i];
|
||||
|
||||
if (std::isnan(a) || std::isnan(b)) {
|
||||
val = equal_nan && std::isnan(a) == std::isnan(b);
|
||||
} else {
|
||||
double left = (a > b ? a - b : b - a);
|
||||
double right = atol + (b > 0 ? rtol * b : (-rtol) * b);
|
||||
double diff = (left > right ? left - right : right - left);
|
||||
val = a == b || left <= right || diff <= 1e-10;
|
||||
}
|
||||
// *out_data &= val;
|
||||
out_data[i] = val;
|
||||
if (!val) {
|
||||
VLOG(2) << "Accuracy check failed between" << a << " and " << b
|
||||
<< " at index= " << i;
|
||||
res_index = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
PADDLE_ENFORCE_EQ(val,
|
||||
true,
|
||||
common::errors::PreconditionNotMet(
|
||||
"Accuracy check failed, kernel name %s, res index %d",
|
||||
fn_name,
|
||||
res_index));
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct AccuracyCheckFunctor<CPUContext, dtype::complex<T>> {
|
||||
void operator()(const CPUContext& dev_ctx,
|
||||
const DenseTensor& in,
|
||||
const DenseTensor& other,
|
||||
const std::string& fn_name,
|
||||
const double rtol,
|
||||
const double atol,
|
||||
bool equal_nan,
|
||||
DenseTensor* output) {
|
||||
auto* in_a = in.data<dtype::complex<T>>();
|
||||
auto* in_b = other.data<dtype::complex<T>>();
|
||||
auto* out_data = dev_ctx.template Alloc<bool>(output);
|
||||
auto num = in.numel();
|
||||
// *out_data = true;
|
||||
for (int i = 0; i < num; i++) {
|
||||
out_data[i] = true;
|
||||
}
|
||||
bool val = false;
|
||||
int res_index = -1;
|
||||
for (int i = 0; i < num; i++) {
|
||||
const dtype::complex<T> a = in_a[i], b = in_b[i];
|
||||
if (std::isnan(a) || std::isnan(b)) {
|
||||
val = equal_nan && std::isnan(a) == std::isnan(b);
|
||||
} else {
|
||||
T left = abs(a - b);
|
||||
T right = atol + rtol * abs(b);
|
||||
T diff = abs(left - right);
|
||||
val = a == b || left <= right || diff <= 1e-10;
|
||||
// *out_data &= val;
|
||||
out_data[i] = val;
|
||||
if (!val) {
|
||||
res_index = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
PADDLE_ENFORCE_EQ(val,
|
||||
true,
|
||||
common::errors::PreconditionNotMet(
|
||||
"Accuracy check failed, kernel name %s, res index %d",
|
||||
fn_name,
|
||||
res_index));
|
||||
}
|
||||
};
|
||||
|
||||
#if defined(__NVCC__) || defined(__HIPCC__)
|
||||
template <typename T>
|
||||
__global__ void AccuracyCheckCUDAKernel(const T* in_data,
|
||||
const T* other_data,
|
||||
const double rtol,
|
||||
const double atol,
|
||||
bool equal_nan,
|
||||
int64_t num,
|
||||
bool* out_data) {
|
||||
int64_t idx = static_cast<int64_t>(blockIdx.x) * blockDim.x + threadIdx.x;
|
||||
bool val;
|
||||
using MPType = typename MPTypeTrait<T>::Type;
|
||||
for (int64_t i = idx; i < num; i += blockDim.x * gridDim.x) {
|
||||
const double a = static_cast<MPType>(in_data[i]);
|
||||
const double b = static_cast<MPType>(other_data[i]);
|
||||
if (isnan(a) || isnan(b)) {
|
||||
val = equal_nan && isnan(a) == isnan(b);
|
||||
} else {
|
||||
double left = (a > b ? a - b : b - a);
|
||||
double right = atol + (b > 0 ? rtol * b : (-rtol) * b);
|
||||
double diff = (left > right ? left - right : right - left);
|
||||
val = a == b || left <= right || diff <= 1e-10;
|
||||
}
|
||||
out_data[i] = val;
|
||||
if (!val) {
|
||||
*out_data = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
template <>
|
||||
__global__ void AccuracyCheckCUDAKernel<complex64>(const complex64* in_data,
|
||||
const complex64* other_data,
|
||||
const double rtol,
|
||||
const double atol,
|
||||
bool equal_nan,
|
||||
int64_t num,
|
||||
bool* out_data) {
|
||||
int64_t idx = static_cast<int64_t>(blockIdx.x) * blockDim.x + threadIdx.x;
|
||||
bool val;
|
||||
for (int64_t i = idx; i < num; i += blockDim.x * gridDim.x) {
|
||||
const complex64 a = in_data[i];
|
||||
const complex64 b = other_data[i];
|
||||
if (isnan(a) || isnan(b)) {
|
||||
val = equal_nan && isnan(a) == isnan(b);
|
||||
} else {
|
||||
float left = abs(a - b);
|
||||
float right = atol + rtol * abs(b);
|
||||
float diff = abs(left - right);
|
||||
val = a == b || left <= right || diff <= 1e-10;
|
||||
}
|
||||
out_data[i] = val;
|
||||
if (!val) {
|
||||
*out_data = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <>
|
||||
__global__ void AccuracyCheckCUDAKernel<complex128>(
|
||||
const complex128* in_data,
|
||||
const complex128* other_data,
|
||||
const double rtol,
|
||||
const double atol,
|
||||
bool equal_nan,
|
||||
int64_t num,
|
||||
bool* out_data) {
|
||||
int64_t idx = static_cast<int64_t>(blockIdx.x) * blockDim.x + threadIdx.x;
|
||||
bool val;
|
||||
for (int64_t i = idx; i < num; i += blockDim.x * gridDim.x) {
|
||||
const complex128 a = in_data[i];
|
||||
const complex128 b = other_data[i];
|
||||
if (isnan(a) || isnan(b)) {
|
||||
val = equal_nan && isnan(a) == isnan(b);
|
||||
} else {
|
||||
double left = abs(a - b);
|
||||
double right = atol + rtol * abs(b);
|
||||
double diff = abs(left - right);
|
||||
val = a == b || left <= right || diff <= 1e-10;
|
||||
}
|
||||
out_data[i] = val;
|
||||
if (!val) {
|
||||
*out_data = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
struct AccuracyCheckFunctor<GPUContext, T> {
|
||||
void operator()(const GPUContext& dev_ctx,
|
||||
const DenseTensor& in,
|
||||
const DenseTensor& other,
|
||||
const std::string& fn_name,
|
||||
const double rtol,
|
||||
const double atol,
|
||||
bool equal_nan,
|
||||
DenseTensor* output) {
|
||||
int64_t num = in.numel();
|
||||
const T* in_data = in.data<T>();
|
||||
const T* other_data = other.data<T>();
|
||||
bool* out_data = dev_ctx.template Alloc<bool>(output);
|
||||
int block = 1024;
|
||||
int64_t grid = (block - 1 + num) / block;
|
||||
grid = (grid > block) ? block : grid;
|
||||
#ifdef PADDLE_WITH_HIP
|
||||
hipMemset(out_data, true, num * sizeof(bool));
|
||||
#else
|
||||
cudaMemset(out_data, true, num * sizeof(bool));
|
||||
#endif
|
||||
AccuracyCheckCUDAKernel<T><<<grid, block, 0, dev_ctx.stream()>>>(
|
||||
in_data, other_data, rtol, atol, equal_nan, num, out_data);
|
||||
|
||||
DenseTensor out_cpu;
|
||||
Copy(dev_ctx, *output, CPUPlace(), true, &out_cpu);
|
||||
auto data_ptr = out_cpu.data<bool>();
|
||||
|
||||
PADDLE_ENFORCE_EQ(*data_ptr,
|
||||
true,
|
||||
common::errors::PreconditionNotMet(
|
||||
"Accuracy check failed, kernel name %s", fn_name));
|
||||
}
|
||||
};
|
||||
#endif
|
||||
|
||||
template <typename T, typename Context>
|
||||
void AccuracyCheckKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& y,
|
||||
const std::string& fn_name,
|
||||
const double rtol,
|
||||
const double atol,
|
||||
bool equal_nan,
|
||||
DenseTensor* out) {
|
||||
AccuracyCheckFunctor<Context, T>()(
|
||||
dev_ctx, x, y, fn_name, rtol, atol, equal_nan, out);
|
||||
}
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,726 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "glog/logging.h"
|
||||
|
||||
#include "paddle/phi/backends/all_context.h"
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/activation_kernel.h"
|
||||
#include "paddle/phi/kernels/elementwise_add_kernel.h"
|
||||
#include "paddle/phi/kernels/elementwise_multiply_kernel.h"
|
||||
#include "paddle/phi/kernels/full_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/activation_functor.h"
|
||||
#include "paddle/phi/kernels/scale_kernel.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context, typename Functor>
|
||||
void ActivationGradImpl(const Context& dev_ctx,
|
||||
const DenseTensor* X,
|
||||
const DenseTensor* Out,
|
||||
const DenseTensor* dOut,
|
||||
DenseTensor* dX,
|
||||
const Functor& functor) {
|
||||
if (static_cast<int>(Functor::FwdDeps()) &
|
||||
static_cast<int>(funcs::ActBwdOpFwdDeps::kDepOut)) {
|
||||
PADDLE_ENFORCE_NOT_NULL(
|
||||
Out, errors::NotFound("The input DenseTensor Out can not be nullptr"));
|
||||
}
|
||||
PADDLE_ENFORCE_NOT_NULL(
|
||||
dOut, errors::NotFound("The input DenseTensor dOut can not be nullptr"));
|
||||
PADDLE_ENFORCE_NOT_NULL(
|
||||
dX, errors::NotFound("The output DenseTensor dX can not be nullptr"));
|
||||
if (!Out) {
|
||||
Out = dOut; // fake out
|
||||
}
|
||||
if (static_cast<int>(Functor::FwdDeps()) &
|
||||
static_cast<int>(funcs::ActBwdOpFwdDeps::kDepX)) {
|
||||
PADDLE_ENFORCE_NOT_NULL(
|
||||
X, errors::NotFound("The input DenseTensor X can not be nullptr"));
|
||||
} else {
|
||||
VLOG(10) << "Inplace activation of Op Functor: " << typeid(Functor).name();
|
||||
X = dX;
|
||||
}
|
||||
|
||||
dev_ctx.template Alloc<T>(dX);
|
||||
if (dX->numel() == 0) {
|
||||
return;
|
||||
}
|
||||
auto dout = EigenVector<T>::Flatten(
|
||||
GET_DATA_SAFELY(dOut, "Input", "Out@GRAD", "ActivationGrad"));
|
||||
auto out = EigenVector<T>::Flatten(
|
||||
GET_DATA_SAFELY(Out, "Input", "Out", "ActivationGrad"));
|
||||
auto dx = EigenVector<T>::Flatten(
|
||||
GET_DATA_SAFELY(dX, "Input", "X@GRAD", "ActivationGrad"));
|
||||
auto x = EigenVector<T>::Flatten(
|
||||
GET_DATA_SAFELY(X, "Input", "X", "ActivationGrad"));
|
||||
auto* place = dev_ctx.eigen_device();
|
||||
functor(*place, x, out, dout, dx);
|
||||
}
|
||||
|
||||
template <typename T, typename Context, typename Functor>
|
||||
void ActivationDoubleGradImpl(const Context& dev_ctx,
|
||||
const DenseTensor* X,
|
||||
const DenseTensor* Out,
|
||||
const DenseTensor* ddX,
|
||||
DenseTensor* dX,
|
||||
DenseTensor* dOut,
|
||||
DenseTensor* ddOut,
|
||||
const Functor& functor) {
|
||||
if (static_cast<int>(Functor::FwdDeps()) &
|
||||
static_cast<int>(funcs::ActBwdOpFwdDeps::kDepX)) {
|
||||
PADDLE_ENFORCE_NOT_NULL(
|
||||
X, errors::NotFound("The input DenseTensor X can not be nullptr"));
|
||||
} else {
|
||||
VLOG(10) << "Inplace activation of Op Functor: " << typeid(Functor).name();
|
||||
X = ddX;
|
||||
}
|
||||
if (static_cast<int>(Functor::FwdDeps()) &
|
||||
static_cast<int>(funcs::ActBwdOpFwdDeps::kDepOut)) {
|
||||
PADDLE_ENFORCE_NOT_NULL(
|
||||
Out, errors::NotFound("The input DenseTensor Out can not be nullptr"));
|
||||
} else {
|
||||
VLOG(10) << "Inplace activation of Op Functor: " << typeid(Functor).name();
|
||||
Out = ddX;
|
||||
}
|
||||
|
||||
if (ddOut) {
|
||||
dev_ctx.template Alloc<T>(ddOut);
|
||||
}
|
||||
if (dOut) {
|
||||
dev_ctx.template Alloc<T>(dOut);
|
||||
}
|
||||
if (dX) {
|
||||
dX->Resize(Out->dims());
|
||||
dev_ctx.template Alloc<T>(dX);
|
||||
}
|
||||
|
||||
functor(dev_ctx, X, Out, ddX, ddOut, dOut, dX);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ReluDoubleGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& out,
|
||||
const DenseTensor& ddx,
|
||||
DenseTensor* ddout) {
|
||||
funcs::ReluGradGradFunctor<T> relu_double_grad_functor;
|
||||
ActivationDoubleGradImpl<T, Context, funcs::ReluGradGradFunctor<T>>(
|
||||
dev_ctx,
|
||||
nullptr,
|
||||
&out,
|
||||
&ddx,
|
||||
nullptr,
|
||||
nullptr,
|
||||
ddout,
|
||||
relu_double_grad_functor);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void LeakyReluDoubleGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& ddx,
|
||||
double alpha,
|
||||
DenseTensor* ddout) {
|
||||
funcs::LeakyReluGradGradFunctor<T> leaky_relu_double_grad_functor;
|
||||
leaky_relu_double_grad_functor.alpha = alpha;
|
||||
ActivationDoubleGradImpl<T, Context, funcs::LeakyReluGradGradFunctor<T>>(
|
||||
dev_ctx,
|
||||
&x,
|
||||
nullptr,
|
||||
&ddx,
|
||||
nullptr,
|
||||
nullptr,
|
||||
ddout,
|
||||
leaky_relu_double_grad_functor);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void TanhDoubleGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& out,
|
||||
const DenseTensor& dout,
|
||||
const DenseTensor& ddx,
|
||||
DenseTensor* dout_new,
|
||||
DenseTensor* ddout) {
|
||||
if (dout_new) {
|
||||
dout_new->Resize(out.dims());
|
||||
dev_ctx.template Alloc<T>(dout_new);
|
||||
}
|
||||
if (ddout) {
|
||||
ddout->Resize(out.dims());
|
||||
dev_ctx.template Alloc<T>(ddout);
|
||||
}
|
||||
funcs::TanhGradGradFunctor<T> functor;
|
||||
functor(dev_ctx, &out, &ddx, &dout, dout_new, ddout);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void TanhTripleGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& out,
|
||||
const DenseTensor& dout,
|
||||
const DenseTensor& ddx,
|
||||
const optional<DenseTensor>& d_dout_new,
|
||||
const optional<DenseTensor>& d_ddout,
|
||||
DenseTensor* d_out_new,
|
||||
DenseTensor* d_dout,
|
||||
DenseTensor* d_ddx) {
|
||||
if (d_dout) {
|
||||
d_dout->Resize(out.dims());
|
||||
dev_ctx.template Alloc<T>(d_dout);
|
||||
}
|
||||
if (d_out_new) {
|
||||
d_out_new->Resize(out.dims());
|
||||
dev_ctx.template Alloc<T>(d_out_new);
|
||||
}
|
||||
if (d_ddx) {
|
||||
d_ddx->Resize(ddx.dims());
|
||||
dev_ctx.template Alloc<T>(d_ddx);
|
||||
}
|
||||
funcs::TanhTripleGradFunctor<T> functor;
|
||||
functor(dev_ctx,
|
||||
&out,
|
||||
&ddx,
|
||||
&dout,
|
||||
d_ddout.get_ptr(),
|
||||
d_dout_new.get_ptr(), // input
|
||||
d_dout,
|
||||
d_out_new,
|
||||
d_ddx); // output
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void EluDoubleGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& dout,
|
||||
const DenseTensor& ddx,
|
||||
float alpha,
|
||||
DenseTensor* dx,
|
||||
DenseTensor* ddout) {
|
||||
if (dx) {
|
||||
dx->Resize(x.dims());
|
||||
dev_ctx.template Alloc<T>(dx);
|
||||
}
|
||||
if (ddout) {
|
||||
dev_ctx.template Alloc<T>(ddout);
|
||||
}
|
||||
funcs::ELUGradGradFunctor<T> functor;
|
||||
functor.alpha = alpha;
|
||||
functor(dev_ctx, &x, &ddx, ddout, &dout, dx);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void LogitGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& out_grad,
|
||||
double eps,
|
||||
DenseTensor* x_grad) {
|
||||
dev_ctx.template Alloc<T>(x_grad);
|
||||
|
||||
auto eigen_x = EigenVector<T>::Flatten(x);
|
||||
auto eigen_dout = EigenVector<T>::Flatten(out_grad);
|
||||
auto eigen_dx = EigenVector<T>::Flatten(*x_grad);
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
auto eigen_p = EigenVector<T>::Flatten(x);
|
||||
|
||||
funcs::LogitGradFunctor<T> functor;
|
||||
functor(place, eigen_x, eigen_dout, eigen_dx, eigen_p, eps);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void SigmoidDoubleGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& out,
|
||||
const DenseTensor& dout,
|
||||
const DenseTensor& ddx,
|
||||
DenseTensor* dout_new,
|
||||
DenseTensor* ddout) {
|
||||
if (dout_new) {
|
||||
dout_new->Resize(out.dims());
|
||||
dev_ctx.template Alloc<T>(dout_new);
|
||||
}
|
||||
if (ddout) {
|
||||
ddout->Resize(out.dims());
|
||||
dev_ctx.template Alloc<T>(ddout);
|
||||
}
|
||||
funcs::SigmoidGradGradFunctor<T> functor;
|
||||
functor(dev_ctx, &out, &ddx, &dout, dout_new, ddout);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void SigmoidTripleGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& out,
|
||||
const DenseTensor& dout,
|
||||
const DenseTensor& ddx,
|
||||
const DenseTensor& d_dout_new,
|
||||
const optional<DenseTensor>& d_ddout,
|
||||
DenseTensor* d_out_new,
|
||||
DenseTensor* d_dout,
|
||||
DenseTensor* d_ddx) {
|
||||
if (d_dout) {
|
||||
d_dout->Resize(out.dims());
|
||||
dev_ctx.template Alloc<T>(d_dout);
|
||||
}
|
||||
if (d_out_new) {
|
||||
d_out_new->Resize(out.dims());
|
||||
dev_ctx.template Alloc<T>(d_out_new);
|
||||
}
|
||||
if (d_ddx) {
|
||||
d_ddx->Resize(ddx.dims());
|
||||
dev_ctx.template Alloc<T>(d_ddx);
|
||||
}
|
||||
funcs::SigmoidTripleGradFunctor<T> functor;
|
||||
functor(dev_ctx,
|
||||
&out,
|
||||
&ddx,
|
||||
&dout,
|
||||
d_ddout.get_ptr(),
|
||||
&d_dout_new,
|
||||
d_dout,
|
||||
d_out_new,
|
||||
d_ddx);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void LogDoubleGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& dout,
|
||||
const DenseTensor& ddx,
|
||||
DenseTensor* dx,
|
||||
DenseTensor* ddout) {
|
||||
if (dx) {
|
||||
dx->Resize(x.dims());
|
||||
dev_ctx.template Alloc<T>(dx);
|
||||
}
|
||||
if (ddout) {
|
||||
dev_ctx.template Alloc<T>(ddout);
|
||||
}
|
||||
funcs::LogGradGradFunctor<T> functor;
|
||||
functor(dev_ctx, &x, &ddx, ddout, &dout, dx);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void PowDoubleGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& dout,
|
||||
const DenseTensor& ddx,
|
||||
const Scalar& factor,
|
||||
DenseTensor* dx,
|
||||
DenseTensor* ddout) {
|
||||
PADDLE_ENFORCE_NOT_NULL(
|
||||
dx, errors::NotFound("The output DenseTensor DX can not be nullptr"));
|
||||
float exponent = factor.to<float>();
|
||||
if (dx) {
|
||||
if (exponent == 1) {
|
||||
*dx = FullLike<T, Context>(dev_ctx, x, static_cast<T>(0));
|
||||
} else {
|
||||
DenseTensor dx_tmp1 = Multiply<T, Context>(dev_ctx, dout, ddx);
|
||||
DenseTensor dx_tmp2 = Multiply<T, Context>(
|
||||
dev_ctx, dx_tmp1, Pow<T, Context>(dev_ctx, x, exponent - 2));
|
||||
*dx = Scale<T, Context>(
|
||||
dev_ctx, dx_tmp2, exponent * (exponent - 1), 0.0, true);
|
||||
}
|
||||
}
|
||||
if (ddout) {
|
||||
DenseTensor ddout_tmp = Multiply<T, Context>(
|
||||
dev_ctx, ddx, Pow<T, Context>(dev_ctx, x, exponent - 1));
|
||||
*ddout = Scale<T, Context>(dev_ctx, ddout_tmp, exponent, 0.0, true);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void PowTripleGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& dout,
|
||||
const DenseTensor& ddx,
|
||||
const DenseTensor& d_dx,
|
||||
const optional<DenseTensor>& d_ddout,
|
||||
const Scalar& factor,
|
||||
DenseTensor* out_d_x,
|
||||
DenseTensor* out_d_dout,
|
||||
DenseTensor* out_d_ddx) {
|
||||
PADDLE_ENFORCE_NOT_NULL(
|
||||
out_d_x,
|
||||
errors::NotFound("The output DenseTensor D_X can not be nullptr"));
|
||||
float exponent = factor.to<float>();
|
||||
if (exponent != 2 && exponent != 1) {
|
||||
// case1: b != 2 and b != 1
|
||||
// D_X = D_DX * DDX * DOut * b * (b-1) * (b-2) * X^(b-3)
|
||||
// + D_DDOut * DDX * b * (b-1) * X^(b-2)
|
||||
if (out_d_x) {
|
||||
DenseTensor out_d_x_tmp1 = Multiply<T, Context>(dev_ctx, d_dx, ddx);
|
||||
DenseTensor out_d_x_tmp2 =
|
||||
Scale<T, Context>(dev_ctx,
|
||||
Pow<T, Context>(dev_ctx, x, exponent - 3),
|
||||
exponent * (exponent - 1) * (exponent - 2),
|
||||
0.0,
|
||||
true);
|
||||
DenseTensor out_d_x_part1 = Multiply<T, Context>(
|
||||
dev_ctx,
|
||||
Multiply<T, Context>(dev_ctx, out_d_x_tmp1, dout),
|
||||
out_d_x_tmp2);
|
||||
|
||||
if (d_ddout.get_ptr()) {
|
||||
DenseTensor out_d_x_tmp3 =
|
||||
Multiply<T, Context>(dev_ctx, d_ddout.get(), ddx);
|
||||
DenseTensor out_d_x_tmp4 =
|
||||
Scale<T, Context>(dev_ctx,
|
||||
Pow<T, Context>(dev_ctx, x, exponent - 2),
|
||||
exponent * (exponent - 1),
|
||||
0.0,
|
||||
true);
|
||||
DenseTensor out_d_x_part2 =
|
||||
Multiply<T, Context>(dev_ctx, out_d_x_tmp3, out_d_x_tmp4);
|
||||
*out_d_x = Add<T, Context>(dev_ctx, out_d_x_part1, out_d_x_part2);
|
||||
} else {
|
||||
*out_d_x = out_d_x_part1;
|
||||
}
|
||||
}
|
||||
// D_DOut = D_DX * DDX * b * (b-1) * X^(b-2)
|
||||
if (out_d_dout) {
|
||||
DenseTensor out_d_x_tmp = Multiply<T, Context>(dev_ctx, d_dx, ddx);
|
||||
DenseTensor out_d_dout_tmp =
|
||||
Scale<T, Context>(dev_ctx,
|
||||
Pow<T, Context>(dev_ctx, x, exponent - 2),
|
||||
exponent * (exponent - 1),
|
||||
0.0,
|
||||
true);
|
||||
|
||||
*out_d_dout = Multiply<T, Context>(dev_ctx, out_d_x_tmp, out_d_dout_tmp);
|
||||
}
|
||||
// D_DDX = D_DX * DOut * b * (b-1) * X^(b-2) + D_DDOut * b * X^(b-1)
|
||||
if (out_d_ddx) {
|
||||
DenseTensor out_d_ddx_tmp1 = Multiply<T, Context>(dev_ctx, d_dx, dout);
|
||||
DenseTensor out_d_dout_tmp =
|
||||
Scale<T, Context>(dev_ctx,
|
||||
Pow<T, Context>(dev_ctx, x, exponent - 2),
|
||||
exponent * (exponent - 1),
|
||||
0.0,
|
||||
true);
|
||||
DenseTensor out_d_ddx_part1 =
|
||||
Multiply<T, Context>(dev_ctx, out_d_ddx_tmp1, out_d_dout_tmp);
|
||||
if (d_ddout.get_ptr()) {
|
||||
DenseTensor out_d_ddx_tmp2 =
|
||||
Scale<T, Context>(dev_ctx,
|
||||
Pow<T, Context>(dev_ctx, x, exponent - 1),
|
||||
exponent,
|
||||
0.0,
|
||||
true);
|
||||
DenseTensor out_d_ddx_part2 =
|
||||
Multiply<T, Context>(dev_ctx, d_ddout.get(), out_d_ddx_tmp2);
|
||||
*out_d_ddx = Add<T, Context>(dev_ctx, out_d_ddx_part1, out_d_ddx_part2);
|
||||
} else {
|
||||
*out_d_ddx = out_d_ddx_part1;
|
||||
}
|
||||
}
|
||||
} else if (exponent == 2) {
|
||||
// case2: b = 2
|
||||
// D_X = D_DDOut * DDX * b * (b-1) * X^(b-2)
|
||||
if (out_d_x) {
|
||||
if (d_ddout.get_ptr()) {
|
||||
DenseTensor out_d_x_tmp1 =
|
||||
Multiply<T, Context>(dev_ctx, d_ddout.get(), ddx);
|
||||
DenseTensor out_d_x_tmp2 =
|
||||
Scale<T, Context>(dev_ctx,
|
||||
Pow<T, Context>(dev_ctx, x, exponent - 2),
|
||||
exponent * (exponent - 1),
|
||||
0.0,
|
||||
true);
|
||||
*out_d_x = Multiply<T, Context>(dev_ctx, out_d_x_tmp1, out_d_x_tmp2);
|
||||
} else {
|
||||
*out_d_x = FullLike<T, Context>(dev_ctx, x, static_cast<T>(0));
|
||||
}
|
||||
}
|
||||
// D_DOut = D_DX * DDX * b * (b-1) * X^(b-2)
|
||||
if (out_d_dout) {
|
||||
DenseTensor out_d_dout_tmp1 = Multiply<T, Context>(dev_ctx, d_dx, ddx);
|
||||
DenseTensor out_d_dout_tmp2 =
|
||||
Scale<T, Context>(dev_ctx,
|
||||
Pow<T, Context>(dev_ctx, x, exponent - 2),
|
||||
exponent * (exponent - 1),
|
||||
0.0,
|
||||
true);
|
||||
|
||||
*out_d_dout =
|
||||
Multiply<T, Context>(dev_ctx, out_d_dout_tmp1, out_d_dout_tmp2);
|
||||
}
|
||||
// D_DDX = D_DX * DOut * b * (b-1) * X^(b-2) + D_DDOut * b * X^(b-1)
|
||||
if (out_d_ddx) {
|
||||
DenseTensor out_d_ddx_tmp1 = Multiply<T, Context>(dev_ctx, d_dx, dout);
|
||||
DenseTensor out_d_dout_tmp2 =
|
||||
Scale<T, Context>(dev_ctx,
|
||||
Pow<T, Context>(dev_ctx, x, exponent - 2),
|
||||
exponent * (exponent - 1),
|
||||
0.0,
|
||||
true);
|
||||
DenseTensor out_d_ddx_part1 =
|
||||
Multiply<T, Context>(dev_ctx, out_d_ddx_tmp1, out_d_dout_tmp2);
|
||||
|
||||
if (d_ddout.get_ptr()) {
|
||||
DenseTensor out_d_ddx_tmp2 =
|
||||
Scale<T, Context>(dev_ctx,
|
||||
Pow<T, Context>(dev_ctx, x, exponent - 1),
|
||||
exponent,
|
||||
0.0,
|
||||
true);
|
||||
DenseTensor out_d_ddx_part2 =
|
||||
Multiply<T, Context>(dev_ctx, d_ddout.get(), out_d_ddx_tmp2);
|
||||
*out_d_ddx = Add<T, Context>(dev_ctx, out_d_ddx_part1, out_d_ddx_part2);
|
||||
} else {
|
||||
*out_d_ddx = out_d_ddx_part1;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// case3: b = 1
|
||||
// D_X = D_DX * DDX * DOut * b * (b-1) * (b-2) * X^(b-3)
|
||||
if (out_d_x) {
|
||||
DenseTensor out_d_x_tmp1 = Multiply<T, Context>(dev_ctx, d_dx, ddx);
|
||||
DenseTensor out_d_x_tmp2 =
|
||||
Scale<T, Context>(dev_ctx,
|
||||
Pow<T, Context>(dev_ctx, x, exponent - 3),
|
||||
exponent * (exponent - 1) * (exponent - 2),
|
||||
0.0,
|
||||
true);
|
||||
|
||||
*out_d_x = Multiply<T, Context>(
|
||||
dev_ctx,
|
||||
Multiply<T, Context>(dev_ctx, out_d_x_tmp1, dout),
|
||||
out_d_x_tmp2);
|
||||
}
|
||||
// D_DOut = 0
|
||||
if (out_d_dout) {
|
||||
*out_d_dout = FullLike<T, Context>(dev_ctx, dout, static_cast<T>(0));
|
||||
}
|
||||
// D_DDX = D_DDOut * b * X^(b-1)
|
||||
if (out_d_ddx) {
|
||||
if (d_ddout.get_ptr()) {
|
||||
DenseTensor out_d_ddx_tmp =
|
||||
Scale<T, Context>(dev_ctx,
|
||||
Pow<T, Context>(dev_ctx, x, exponent - 1),
|
||||
exponent,
|
||||
0.0,
|
||||
true);
|
||||
|
||||
*out_d_ddx =
|
||||
Multiply<T, Context>(dev_ctx, d_ddout.get(), out_d_ddx_tmp);
|
||||
} else {
|
||||
*out_d_ddx = FullLike<T, Context>(dev_ctx, ddx, static_cast<T>(0));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void SqrtDoubleGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& out,
|
||||
const DenseTensor& dx,
|
||||
const DenseTensor& ddx,
|
||||
DenseTensor* dout,
|
||||
DenseTensor* ddout) {
|
||||
if (dout) {
|
||||
dout->Resize(out.dims());
|
||||
dev_ctx.template Alloc<T>(dout);
|
||||
}
|
||||
if (ddout) {
|
||||
ddout->Resize(out.dims());
|
||||
dev_ctx.template Alloc<T>(ddout);
|
||||
}
|
||||
|
||||
funcs::SqrtGradGradFunctor<T> functor;
|
||||
functor(dev_ctx, &out, &dx, &ddx, dout, ddout);
|
||||
}
|
||||
|
||||
// rsqrt Grad: dx = -0.5 * dy * y * y * y
|
||||
// rsqrt GradGrad: ddy = -0.5 * ddx * y * y * y, dy = (3 / y) * dx * ddx
|
||||
template <typename T, typename Context>
|
||||
void RsqrtDoubleGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& out,
|
||||
const DenseTensor& dx,
|
||||
const DenseTensor& ddx,
|
||||
DenseTensor* dout,
|
||||
DenseTensor* ddout) {
|
||||
if (dout) {
|
||||
dout->Resize(out.dims());
|
||||
dev_ctx.template Alloc<T>(dout);
|
||||
}
|
||||
if (ddout) {
|
||||
ddout->Resize(out.dims());
|
||||
dev_ctx.template Alloc<T>(ddout);
|
||||
}
|
||||
|
||||
funcs::RsqrtGradGradFunctor<T> functor;
|
||||
functor(dev_ctx, &out, &dx, &ddx, dout, ddout);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void CeluDoubleGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& dout,
|
||||
const DenseTensor& ddx,
|
||||
float alpha,
|
||||
DenseTensor* dx,
|
||||
DenseTensor* ddout) {
|
||||
if (dx) {
|
||||
dx->Resize(x.dims());
|
||||
dev_ctx.template Alloc<T>(dx);
|
||||
}
|
||||
if (ddout) {
|
||||
dev_ctx.template Alloc<T>(ddout);
|
||||
}
|
||||
|
||||
funcs::CELUGradGradFunctor<T> functor;
|
||||
auto attrs = functor.GetAttrs();
|
||||
*(attrs[0].second) = alpha;
|
||||
functor(dev_ctx, &x, &dout, &ddx, dx, ddout);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void SoftplusDoubleGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& dout,
|
||||
const DenseTensor& ddx,
|
||||
double beta,
|
||||
double threshold,
|
||||
DenseTensor* dx,
|
||||
DenseTensor* ddout) {
|
||||
if (dx) {
|
||||
dx->Resize(x.dims());
|
||||
dev_ctx.template Alloc<T>(dx);
|
||||
}
|
||||
if (ddout) {
|
||||
dev_ctx.template Alloc<T>(ddout);
|
||||
}
|
||||
|
||||
funcs::SoftplusDoubleGradFunctor<T> functor;
|
||||
auto attrs = functor.GetAttrs();
|
||||
*(attrs[0].second) = beta;
|
||||
*(attrs[1].second) = threshold;
|
||||
functor(dev_ctx, &x, &dout, &ddx, dx, ddout);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void SquareDoubleGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& dout,
|
||||
const DenseTensor& ddx,
|
||||
DenseTensor* dx,
|
||||
DenseTensor* ddout) {
|
||||
if (dx) {
|
||||
dx->Resize(x.dims());
|
||||
dev_ctx.template Alloc<T>(dx);
|
||||
}
|
||||
if (ddout) {
|
||||
dev_ctx.template Alloc<T>(ddout);
|
||||
}
|
||||
|
||||
funcs::SquareGradGradFunctor<T> functor;
|
||||
functor(dev_ctx, &x, &dout, &ddx, dx, ddout);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void SinDoubleGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& dout,
|
||||
const DenseTensor& ddx,
|
||||
DenseTensor* dx,
|
||||
DenseTensor* ddout) {
|
||||
if (dx) {
|
||||
dev_ctx.template Alloc<T>(dx);
|
||||
}
|
||||
if (ddout) {
|
||||
dev_ctx.template Alloc<T>(ddout);
|
||||
}
|
||||
funcs::SinDoubleGradFunctor<T> functor;
|
||||
functor(dev_ctx, &x, &dout, &ddx, dx, ddout);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void SinTripleGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const optional<DenseTensor>& dout,
|
||||
const optional<DenseTensor>& ddx,
|
||||
const DenseTensor& d_dx_new,
|
||||
const optional<DenseTensor>& d_ddout,
|
||||
DenseTensor* d_x_new,
|
||||
DenseTensor* d_dout,
|
||||
DenseTensor* d_ddx) {
|
||||
if (d_dout) {
|
||||
dev_ctx.template Alloc<T>(d_dout);
|
||||
}
|
||||
if (d_x_new) {
|
||||
dev_ctx.template Alloc<T>(d_x_new);
|
||||
}
|
||||
if (d_ddx) {
|
||||
dev_ctx.template Alloc<T>(d_ddx);
|
||||
}
|
||||
funcs::SinTripleGradFunctor<T> functor;
|
||||
functor(dev_ctx,
|
||||
&x,
|
||||
ddx.get_ptr(),
|
||||
dout.get_ptr(),
|
||||
d_ddout.get_ptr(),
|
||||
&d_dx_new, // input
|
||||
d_dout,
|
||||
d_x_new,
|
||||
d_ddx); // output
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void CosDoubleGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& dout,
|
||||
const DenseTensor& ddx,
|
||||
DenseTensor* dx,
|
||||
DenseTensor* ddout) {
|
||||
if (dx) {
|
||||
dev_ctx.template Alloc<T>(dx);
|
||||
}
|
||||
if (ddout) {
|
||||
dev_ctx.template Alloc<T>(ddout);
|
||||
}
|
||||
funcs::CosDoubleGradFunctor<T> functor;
|
||||
functor(dev_ctx, &x, &dout, &ddx, dx, ddout);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void CosTripleGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const optional<DenseTensor>& dout,
|
||||
const optional<DenseTensor>& ddx,
|
||||
const DenseTensor& d_dx_new,
|
||||
const optional<DenseTensor>& d_ddout,
|
||||
DenseTensor* d_x_new,
|
||||
DenseTensor* d_dout,
|
||||
DenseTensor* d_ddx) {
|
||||
if (d_dout) {
|
||||
dev_ctx.template Alloc<T>(d_dout);
|
||||
}
|
||||
if (d_x_new) {
|
||||
dev_ctx.template Alloc<T>(d_x_new);
|
||||
}
|
||||
if (d_ddx) {
|
||||
dev_ctx.template Alloc<T>(d_ddx);
|
||||
}
|
||||
funcs::CosTripleGradFunctor<T> functor;
|
||||
functor(dev_ctx,
|
||||
&x,
|
||||
ddx.get_ptr(),
|
||||
dout.get_ptr(),
|
||||
d_ddout.get_ptr(),
|
||||
&d_dx_new, // input
|
||||
d_dout,
|
||||
d_x_new,
|
||||
d_ddx); // output
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,151 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/backends/all_context.h"
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/funcs/activation_functor.h"
|
||||
#include "paddle/phi/kernels/funcs/sleef_vectorized_math.h"
|
||||
// #include "paddle/phi/kernels/funcs/blas/blas.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
#define ToString(x) #x
|
||||
|
||||
template <typename T, typename U, typename Context, typename Functor>
|
||||
void ActivationImpl(const Context& dev_ctx,
|
||||
const DenseTensor& X,
|
||||
DenseTensor* Out,
|
||||
const Functor& functor) {
|
||||
PADDLE_ENFORCE_NOT_NULL(Out,
|
||||
errors::NotFound("Output Out should not be nullptr"));
|
||||
dev_ctx.template Alloc<U>(Out);
|
||||
if (Out->numel() == 0) {
|
||||
return;
|
||||
}
|
||||
auto x =
|
||||
EigenVector<T>::Flatten(GET_DATA_SAFELY(&X, "Input", "X", "Activation"));
|
||||
auto out = EigenVector<U>::Flatten(
|
||||
GET_DATA_SAFELY(Out, "Output", "Out", "Activation"));
|
||||
auto* place = dev_ctx.eigen_device();
|
||||
functor(*place, x, out);
|
||||
}
|
||||
|
||||
// Vectorized Sin implementation for CPU - high precision
|
||||
// Only enabled for float/double on CPU to ensure bit-level alignment
|
||||
template <typename T, typename Context>
|
||||
void VectorizedSinImpl(const Context& dev_ctx,
|
||||
const DenseTensor& X,
|
||||
DenseTensor* Out) {
|
||||
PADDLE_ENFORCE_NOT_NULL(Out,
|
||||
errors::NotFound("Output Out should not be nullptr"));
|
||||
dev_ctx.template Alloc<T>(Out);
|
||||
if (Out->numel() == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const T* x_data = X.data<T>();
|
||||
T* out_data = Out->data<T>();
|
||||
int64_t numel = X.numel();
|
||||
|
||||
// Check if data is contiguous and use vectorized path
|
||||
if (funcs::sleef_vec::should_use_vectorized_path(x_data, out_data, numel)) {
|
||||
funcs::sleef_vec::vsin(out_data, x_data, numel);
|
||||
} else {
|
||||
// Fallback to Eigen-based implementation
|
||||
auto x = EigenVector<T>::Flatten(GET_DATA_SAFELY(&X, "Input", "X", "Sin"));
|
||||
auto out =
|
||||
EigenVector<T>::Flatten(GET_DATA_SAFELY(Out, "Output", "Out", "Sin"));
|
||||
auto* place = dev_ctx.eigen_device();
|
||||
out.device(*place) = x.unaryExpr(funcs::Sine<T>()).eval();
|
||||
}
|
||||
}
|
||||
|
||||
// Vectorized Cos implementation for CPU - high precision
|
||||
template <typename T, typename Context>
|
||||
void VectorizedCosImpl(const Context& dev_ctx,
|
||||
const DenseTensor& X,
|
||||
DenseTensor* Out) {
|
||||
PADDLE_ENFORCE_NOT_NULL(Out,
|
||||
errors::NotFound("Output Out should not be nullptr"));
|
||||
dev_ctx.template Alloc<T>(Out);
|
||||
if (Out->numel() == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const T* x_data = X.data<T>();
|
||||
T* out_data = Out->data<T>();
|
||||
int64_t numel = X.numel();
|
||||
|
||||
// Check if data is contiguous and use vectorized path
|
||||
if (funcs::sleef_vec::should_use_vectorized_path(x_data, out_data, numel)) {
|
||||
funcs::sleef_vec::vcos(out_data, x_data, numel);
|
||||
} else {
|
||||
// Fallback to Eigen-based implementation
|
||||
auto x = EigenVector<T>::Flatten(GET_DATA_SAFELY(&X, "Input", "X", "Cos"));
|
||||
auto out =
|
||||
EigenVector<T>::Flatten(GET_DATA_SAFELY(Out, "Output", "Out", "Cos"));
|
||||
auto* place = dev_ctx.eigen_device();
|
||||
out.device(*place) = x.unaryExpr(funcs::Cosine<T>()).eval();
|
||||
}
|
||||
}
|
||||
|
||||
// Vectorized Exp implementation for CPU - high precision
|
||||
template <typename T, typename Context>
|
||||
void VectorizedExpImpl(const Context& dev_ctx,
|
||||
const DenseTensor& X,
|
||||
DenseTensor* Out) {
|
||||
PADDLE_ENFORCE_NOT_NULL(Out,
|
||||
errors::NotFound("Output Out should not be nullptr"));
|
||||
dev_ctx.template Alloc<T>(Out);
|
||||
if (Out->numel() == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const T* x_data = X.data<T>();
|
||||
T* out_data = Out->data<T>();
|
||||
int64_t numel = X.numel();
|
||||
|
||||
// Check if data is contiguous and use vectorized path
|
||||
if (funcs::sleef_vec::should_use_vectorized_path_for_exp(
|
||||
x_data, out_data, numel)) {
|
||||
funcs::sleef_vec::vexp(out_data, x_data, numel);
|
||||
} else {
|
||||
// Fallback to Eigen-based implementation
|
||||
auto x = EigenVector<T>::Flatten(GET_DATA_SAFELY(&X, "Input", "X", "Exp"));
|
||||
auto out =
|
||||
EigenVector<T>::Flatten(GET_DATA_SAFELY(Out, "Output", "Out", "Exp"));
|
||||
auto* place = dev_ctx.eigen_device();
|
||||
out.device(*place) = x.exp();
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void LogitKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
double eps,
|
||||
DenseTensor* out) {
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
|
||||
auto eigen_out = EigenVector<T>::Flatten(*out);
|
||||
auto eigen_in = EigenVector<T>::Flatten(x);
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
auto eigen_p = EigenVector<T>::Flatten(*out);
|
||||
|
||||
funcs::LogitFunctor<T> functor;
|
||||
functor(place, eigen_in, eigen_out, eigen_p, eps);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,83 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
#include "paddle/phi/common/amp_type_traits.h"
|
||||
#include "paddle/phi/kernels/adadelta_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void AdadeltaKernel(const Context& dev_ctx,
|
||||
const DenseTensor& param,
|
||||
const DenseTensor& grad,
|
||||
const DenseTensor& avg_squared_grad,
|
||||
const DenseTensor& avg_squared_update,
|
||||
const DenseTensor& learning_rate,
|
||||
const optional<DenseTensor>& master_param,
|
||||
float rho,
|
||||
float epsilon,
|
||||
bool multi_precision,
|
||||
DenseTensor* param_out,
|
||||
DenseTensor* avg_squared_grad_out,
|
||||
DenseTensor* avg_squared_update_out,
|
||||
DenseTensor* master_param_outs) {
|
||||
using MT = typename dtype::template MPTypeTrait<T>::Type;
|
||||
dev_ctx.template Alloc<T>(param_out);
|
||||
dev_ctx.template Alloc<MT>(avg_squared_grad_out);
|
||||
dev_ctx.template Alloc<MT>(avg_squared_update_out);
|
||||
|
||||
MT rho_ = static_cast<MT>(rho);
|
||||
MT epsilon_ = static_cast<MT>(epsilon);
|
||||
|
||||
auto eigen_param = EigenVector<T>::Flatten(param);
|
||||
auto eigen_grad = EigenVector<T>::Flatten(grad);
|
||||
// Squared gradient accumulator
|
||||
auto eigen_avg_squared_grad = EigenVector<MT>::Flatten(avg_squared_grad);
|
||||
// Squared updates accumulator
|
||||
auto eigen_avg_squared_update = EigenVector<MT>::Flatten(avg_squared_update);
|
||||
auto eigen_param_out = EigenVector<T>::Flatten(*param_out);
|
||||
auto eigen_avg_squared_grad_out =
|
||||
EigenVector<MT>::Flatten(*avg_squared_grad_out);
|
||||
auto eigen_avg_squared_update_out =
|
||||
EigenVector<MT>::Flatten(*avg_squared_update_out);
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
auto eigen_grad_cast = eigen_grad.template cast<MT>();
|
||||
eigen_avg_squared_grad_out.device(place) =
|
||||
rho_ * eigen_avg_squared_grad + (1 - rho_) * eigen_grad_cast.square();
|
||||
auto update =
|
||||
-(((eigen_avg_squared_update + epsilon_).sqrt()) /
|
||||
((eigen_avg_squared_grad_out + epsilon_).sqrt()) * eigen_grad_cast);
|
||||
Eigen::DSizes<int, 1> m_dsize(avg_squared_update_out->numel());
|
||||
auto lr = EigenVector<MT>::Flatten(learning_rate);
|
||||
if (multi_precision) {
|
||||
auto eigen_master_param_out = EigenVector<MT>::Flatten(*master_param_outs);
|
||||
auto eigen_master_param = EigenVector<MT>::Flatten(*master_param);
|
||||
|
||||
eigen_master_param_out.device(place) =
|
||||
eigen_master_param + lr.broadcast(m_dsize) * update;
|
||||
eigen_param_out.device(place) =
|
||||
(eigen_param.template cast<MT>() + lr.broadcast(m_dsize) * update)
|
||||
.template cast<T>();
|
||||
} else {
|
||||
eigen_param_out.device(place) =
|
||||
eigen_param + (lr.broadcast(m_dsize) * update).template cast<T>();
|
||||
}
|
||||
eigen_avg_squared_update_out.device(place) =
|
||||
rho_ * eigen_avg_squared_update + (1 - rho_) * update.square();
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,129 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/adagrad_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename Context, typename T>
|
||||
struct SparseAdagradFunctor {
|
||||
void operator()(const Context& dev_ctx,
|
||||
const SelectedRows& grad,
|
||||
const DenseTensor& learning_rate,
|
||||
T epsilon,
|
||||
DenseTensor* moment,
|
||||
DenseTensor* param);
|
||||
};
|
||||
|
||||
template <typename Context, typename T>
|
||||
struct DenseAdagradFunctor {
|
||||
void operator()(const Context& dev_ctx,
|
||||
const DenseTensor& param_t,
|
||||
const DenseTensor& grad_t,
|
||||
const DenseTensor& moment_t,
|
||||
const DenseTensor& learning_rate,
|
||||
const optional<DenseTensor>& master_param,
|
||||
float epsilon_t,
|
||||
bool multi_precision,
|
||||
DenseTensor* param_out_tensor,
|
||||
DenseTensor* moment_out_tensor,
|
||||
DenseTensor* master_param_outs);
|
||||
};
|
||||
|
||||
template <typename Context, typename T>
|
||||
SelectedRows SquareSelectedRows(const Context& dev_ctx,
|
||||
const SelectedRows& input) {
|
||||
SelectedRows out;
|
||||
out.set_rows(input.rows());
|
||||
out.set_height(input.height());
|
||||
out.mutable_value()->Resize(input.value().dims());
|
||||
dev_ctx.template Alloc<T>(out.mutable_value());
|
||||
auto e_out = EigenVector<T>::Flatten(*(out.mutable_value()));
|
||||
auto e_in = EigenVector<T>::Flatten(input.value());
|
||||
e_out.device(*dev_ctx.eigen_device()) = e_in.square();
|
||||
return out;
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void AdagradDenseKernel(const Context& dev_ctx,
|
||||
const DenseTensor& param_t,
|
||||
const DenseTensor& grad_t,
|
||||
const DenseTensor& moment_t,
|
||||
const DenseTensor& learning_rate,
|
||||
const optional<DenseTensor>& master_param,
|
||||
float epsilon_t,
|
||||
bool multi_precision,
|
||||
DenseTensor* param_out_tensor,
|
||||
DenseTensor* moment_out_tensor,
|
||||
DenseTensor* master_param_outs) {
|
||||
DenseAdagradFunctor<Context, T> functor;
|
||||
functor(dev_ctx,
|
||||
param_t,
|
||||
grad_t,
|
||||
moment_t,
|
||||
learning_rate,
|
||||
master_param,
|
||||
epsilon_t,
|
||||
multi_precision,
|
||||
param_out_tensor,
|
||||
moment_out_tensor,
|
||||
master_param_outs);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void AdagradSparseKernel(const Context& dev_ctx,
|
||||
const DenseTensor& param_t,
|
||||
const SelectedRows& grad_t,
|
||||
const DenseTensor& moment_t,
|
||||
const DenseTensor& learning_rate,
|
||||
const optional<DenseTensor>& master_param UNUSED,
|
||||
float epsilon_t,
|
||||
bool multi_precision UNUSED,
|
||||
DenseTensor* param_out,
|
||||
DenseTensor* moment_out,
|
||||
DenseTensor* master_param_outs UNUSED) {
|
||||
auto* param_out_tensor = param_out;
|
||||
auto* moment_out_tensor = moment_out;
|
||||
|
||||
dev_ctx.template Alloc<T>(param_out_tensor);
|
||||
dev_ctx.template Alloc<T>(moment_out_tensor);
|
||||
|
||||
T epsilon = static_cast<T>(epsilon_t);
|
||||
|
||||
auto* param_tensor = ¶m_t;
|
||||
PADDLE_ENFORCE_EQ(param_tensor->IsSharedBufferWith(*param_out_tensor),
|
||||
true,
|
||||
common::errors::InvalidArgument(
|
||||
"the input tensor not equal with output tensor"));
|
||||
|
||||
auto* moment_tensor = &moment_t;
|
||||
PADDLE_ENFORCE_EQ(moment_tensor->IsSharedBufferWith(*moment_out_tensor),
|
||||
true,
|
||||
common::errors::InvalidArgument(
|
||||
"the input moment not equal with output moment"));
|
||||
|
||||
SparseAdagradFunctor<Context, T> functor;
|
||||
functor(dev_ctx,
|
||||
grad_t,
|
||||
learning_rate,
|
||||
epsilon,
|
||||
moment_out_tensor,
|
||||
param_out_tensor);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,72 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/adamax_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/eigen_function.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void AdamaxKernel(const Context& dev_ctx,
|
||||
const DenseTensor& param,
|
||||
const DenseTensor& grad,
|
||||
const DenseTensor& learning_rate,
|
||||
const DenseTensor& moment,
|
||||
const DenseTensor& inf_norm,
|
||||
const DenseTensor& beta1_pow,
|
||||
const optional<DenseTensor>& master_param UNUSED,
|
||||
float beta1,
|
||||
float beta2,
|
||||
float epsilon,
|
||||
bool multi_precision UNUSED,
|
||||
DenseTensor* param_out,
|
||||
DenseTensor* moment_out,
|
||||
DenseTensor* inf_norm_out,
|
||||
DenseTensor* master_param_outs UNUSED) {
|
||||
dev_ctx.template Alloc<T>(param_out);
|
||||
dev_ctx.template Alloc<T>(moment_out);
|
||||
dev_ctx.template Alloc<T>(inf_norm_out);
|
||||
|
||||
T beta1_ = static_cast<T>(beta1);
|
||||
T beta2_ = static_cast<T>(beta2);
|
||||
T epsilon_ = static_cast<T>(epsilon);
|
||||
|
||||
auto eigen_param = EigenVector<T>::Flatten(param);
|
||||
auto eigen_grad = EigenVector<T>::Flatten(grad);
|
||||
auto eigen_moment = EigenVector<T>::Flatten(moment);
|
||||
auto eigen_inf_norm = EigenVector<T>::Flatten(inf_norm);
|
||||
auto eigen_lr = EigenVector<T>::Flatten(learning_rate);
|
||||
auto eigen_beta1_pow = EigenVector<T>::Flatten(beta1_pow);
|
||||
|
||||
auto eigen_param_out = EigenVector<T>::Flatten(*param_out);
|
||||
auto eigen_moment_out = EigenVector<T>::Flatten(*moment_out);
|
||||
auto eigen_inf_norm_out = EigenVector<T>::Flatten(*inf_norm_out);
|
||||
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
|
||||
eigen_moment_out.device(place) =
|
||||
beta1_ * eigen_moment + (static_cast<T>(1) - beta1_) * eigen_grad;
|
||||
eigen_inf_norm_out.device(place) =
|
||||
eigen_grad.abs().cwiseMax((beta2_ * eigen_inf_norm) + epsilon_);
|
||||
auto lr_t = eigen_lr / (static_cast<T>(1) - eigen_beta1_pow);
|
||||
Eigen::DSizes<int, 1> m_dsize(moment_out->numel());
|
||||
eigen_param_out.device(place) =
|
||||
eigen_param -
|
||||
lr_t.broadcast(m_dsize) * (eigen_moment_out / eigen_inf_norm_out);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,80 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/add_n_kernel.h"
|
||||
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/core/tensor_utils.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
|
||||
#include "paddle/phi/kernels/funcs/selected_rows_functor.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void AddNArrayKernel(const Context& dev_ctx,
|
||||
const std::vector<const TensorArray*>& x,
|
||||
TensorArray* out) {
|
||||
for (auto& ele : *out) {
|
||||
dev_ctx.template Alloc<T>(&ele);
|
||||
}
|
||||
bool in_place = true;
|
||||
if (x.size() > 0 && x[0]->size() == out->size()) {
|
||||
for (size_t i = 0; i < out->size(); i++) {
|
||||
if (x[0]->at(i).IsInitialized() &&
|
||||
out->at(i).data() != x[0]->at(i).data()) {
|
||||
in_place = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
in_place = false;
|
||||
}
|
||||
for (size_t i = in_place ? 1 : 0; i < x.size(); ++i) {
|
||||
auto* in_array = x.at(i);
|
||||
|
||||
for (size_t j = 0; j < in_array->size(); ++j) {
|
||||
if (in_array->at(j).IsInitialized() && (in_array->at(j).numel() != 0)) {
|
||||
if (j >= out->size()) {
|
||||
out->resize(j + 1);
|
||||
}
|
||||
if (!out->at(j).IsInitialized() || (out->at(j).numel() == 0)) {
|
||||
Copy<Context>(dev_ctx,
|
||||
in_array->at(j),
|
||||
in_array->at(j).place(),
|
||||
false,
|
||||
&out->at(j));
|
||||
out->at(j).set_lod(in_array->at(j).lod());
|
||||
} else {
|
||||
PADDLE_ENFORCE_EQ(
|
||||
out->at(j).lod(),
|
||||
in_array->at(j).lod(),
|
||||
common::errors::InvalidArgument(
|
||||
"The lod message between inputs[%d] and"
|
||||
" outputs[%d] must be same, but now is not same.",
|
||||
j,
|
||||
j));
|
||||
auto in = EigenVector<T>::Flatten(in_array->at(j));
|
||||
auto result = EigenVector<T>::Flatten(out->at(j));
|
||||
result.device(*dev_ctx.eigen_device()) = result + in;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,221 @@
|
||||
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <type_traits>
|
||||
|
||||
#include "glog/logging.h"
|
||||
|
||||
#include "paddle/phi/common/amp_type_traits.h"
|
||||
#include "paddle/phi/kernels/addmm_grad_kernel.h"
|
||||
#include "paddle/phi/kernels/full_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/blas/blas.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/eigen_function.h"
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T>
|
||||
struct CopyOrScaleFunctor {
|
||||
CopyOrScaleFunctor(const float scale, const T* x, T* output, int64_t numel)
|
||||
: scale_(scale), x_(x), output_(output), numel_(numel) {}
|
||||
|
||||
HOSTDEVICE void operator()(int64_t idx) const {
|
||||
using MPType = typename MPTypeTrait<T>::Type;
|
||||
const MPType mp_scale = static_cast<MPType>(scale_);
|
||||
const MPType mp_x = static_cast<MPType>(x_[idx]);
|
||||
output_[idx] = static_cast<T>(mp_scale * mp_x);
|
||||
}
|
||||
|
||||
private:
|
||||
const float scale_;
|
||||
const T* x_;
|
||||
T* output_;
|
||||
int64_t numel_;
|
||||
};
|
||||
|
||||
using Array1 = Eigen::DSizes<int64_t, 1>;
|
||||
using Array2 = Eigen::DSizes<int64_t, 2>;
|
||||
|
||||
template <typename T, typename Context>
|
||||
void AddmmGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& input,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& y,
|
||||
const DenseTensor& out_grad,
|
||||
float alpha,
|
||||
float beta,
|
||||
DenseTensor* input_grad,
|
||||
DenseTensor* x_grad,
|
||||
DenseTensor* y_grad) {
|
||||
if (out_grad.numel() == 0) {
|
||||
if (input_grad) {
|
||||
Full<T, Context>(dev_ctx, input_grad->dims(), 0, input_grad);
|
||||
}
|
||||
if (x_grad) {
|
||||
Full<T, Context>(dev_ctx, x_grad->dims(), 0, x_grad);
|
||||
}
|
||||
if (y_grad) {
|
||||
Full<T, Context>(dev_ctx, y_grad->dims(), 0, y_grad);
|
||||
}
|
||||
return;
|
||||
}
|
||||
using MPType = typename MPTypeTrait<T>::Type;
|
||||
bool is_float16_or_bfloat16 = false;
|
||||
bool is_big_tensor = false;
|
||||
if (input.numel() * input.dims()[1] > std::numeric_limits<int>::max() ||
|
||||
x.numel() > std::numeric_limits<int>::max() ||
|
||||
y.numel() * y.dims()[1] > std::numeric_limits<int>::max()) {
|
||||
is_big_tensor = true;
|
||||
}
|
||||
if (std::is_same<T, float16>::value || std::is_same<T, bfloat16>::value) {
|
||||
is_float16_or_bfloat16 = true;
|
||||
}
|
||||
|
||||
auto in_dims = input.dims();
|
||||
if (input.dims().size() == 1) {
|
||||
in_dims = {1, input.dims()[0]};
|
||||
input_grad->Resize(in_dims);
|
||||
}
|
||||
int64_t total_elems = 0;
|
||||
|
||||
VLOG(3) << "alpha: " << alpha << " beta: " << beta;
|
||||
|
||||
if (input_grad != nullptr) {
|
||||
input_grad->set_lod(out_grad.lod());
|
||||
}
|
||||
if (x_grad != nullptr) {
|
||||
x_grad->set_lod(x.lod());
|
||||
}
|
||||
if (y_grad != nullptr) {
|
||||
y_grad->set_lod(y.lod());
|
||||
}
|
||||
|
||||
auto blas = funcs::GetBlas<Context, T>(dev_ctx);
|
||||
auto mt_blas = funcs::GetBlas<Context, MPType>(dev_ctx);
|
||||
if (input_grad) {
|
||||
dev_ctx.template Alloc<T>(input_grad);
|
||||
total_elems = in_dims[0] * in_dims[1];
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
auto eigen_dout = EigenTensor<T, 2>::From(out_grad);
|
||||
auto eigen_dinput = EigenTensor<T, 2>::From(*input_grad);
|
||||
|
||||
bool row_compress = in_dims[0] != out_grad.dims()[0];
|
||||
bool col_compress = in_dims[1] != out_grad.dims()[1];
|
||||
auto eigen_dinput_shape =
|
||||
Array2(input_grad->dims()[0], input_grad->dims()[1]);
|
||||
|
||||
if (row_compress && col_compress) {
|
||||
if (!is_float16_or_bfloat16 && !is_big_tensor) {
|
||||
eigen_dinput.device(place) =
|
||||
eigen_dout.sum().eval().reshape(eigen_dinput_shape);
|
||||
} else {
|
||||
eigen_dinput.device(place) = eigen_dout.template cast<MPType>()
|
||||
.sum()
|
||||
.eval()
|
||||
.reshape(eigen_dinput_shape)
|
||||
.template cast<T>();
|
||||
}
|
||||
} else if (row_compress) {
|
||||
if (!is_float16_or_bfloat16 && !is_big_tensor) {
|
||||
eigen_dinput.device(place) =
|
||||
eigen_dout.sum(Array1(0)).eval().reshape(eigen_dinput_shape);
|
||||
} else {
|
||||
eigen_dinput.device(place) = eigen_dout.template cast<MPType>()
|
||||
.sum(Array1(0))
|
||||
.eval()
|
||||
.reshape(eigen_dinput_shape)
|
||||
.template cast<T>();
|
||||
}
|
||||
} else if (col_compress) {
|
||||
if (!is_float16_or_bfloat16 && !is_big_tensor) {
|
||||
eigen_dinput.device(place) =
|
||||
eigen_dout.sum(Array1(1)).eval().reshape(eigen_dinput_shape);
|
||||
} else {
|
||||
eigen_dinput.device(place) = eigen_dout.template cast<MPType>()
|
||||
.sum(Array1(1))
|
||||
.eval()
|
||||
.reshape(eigen_dinput_shape)
|
||||
.template cast<T>();
|
||||
}
|
||||
} else {
|
||||
// The VCOPY does not support the float16, bfloat16
|
||||
if (!is_float16_or_bfloat16 && !is_big_tensor) {
|
||||
mt_blas.VCOPY(
|
||||
total_elems, out_grad.data<MPType>(), input_grad->data<MPType>());
|
||||
} else {
|
||||
funcs::ForRange<Context> for_range(dev_ctx, total_elems);
|
||||
CopyOrScaleFunctor<T> functor(
|
||||
1, out_grad.data<T>(), input_grad->data<T>(), total_elems);
|
||||
for_range(functor);
|
||||
}
|
||||
}
|
||||
|
||||
// The SCAL does not support the float16, bfloat16
|
||||
if (!is_float16_or_bfloat16 && !is_big_tensor) {
|
||||
mt_blas.SCAL(total_elems, beta, input_grad->data<MPType>());
|
||||
} else {
|
||||
funcs::ForRange<Context> for_range(dev_ctx, total_elems);
|
||||
CopyOrScaleFunctor<T> functor(
|
||||
beta, input_grad->data<T>(), input_grad->data<T>(), total_elems);
|
||||
for_range(functor);
|
||||
}
|
||||
|
||||
if (input.dims().size() == 1) {
|
||||
input_grad->Resize(input.dims());
|
||||
}
|
||||
}
|
||||
if (x_grad && x_grad->numel() == 0) {
|
||||
dev_ctx.template Alloc<T>(x_grad);
|
||||
Full<T, Context>(dev_ctx, y_grad->dims(), 0, y_grad);
|
||||
return;
|
||||
}
|
||||
if (y_grad && y_grad->numel() == 0) {
|
||||
dev_ctx.template Alloc<T>(y_grad);
|
||||
Full<T, Context>(dev_ctx, x_grad->dims(), 0, x_grad);
|
||||
return;
|
||||
}
|
||||
if (x_grad) {
|
||||
dev_ctx.template Alloc<T>(x_grad);
|
||||
total_elems = x.dims()[0] * x.dims()[1];
|
||||
// x_grad = out_grad * y'. x_grad: M x K, out_grad : M x N, y : K x N
|
||||
blas.MatMul(out_grad, false, y, true, x_grad);
|
||||
if (!is_float16_or_bfloat16 && !is_big_tensor) {
|
||||
mt_blas.SCAL(total_elems, alpha, x_grad->data<MPType>());
|
||||
} else {
|
||||
funcs::ForRange<Context> for_range(dev_ctx, total_elems);
|
||||
CopyOrScaleFunctor<T> functor(
|
||||
alpha, x_grad->data<T>(), x_grad->data<T>(), total_elems);
|
||||
for_range(functor);
|
||||
}
|
||||
}
|
||||
if (y_grad) {
|
||||
dev_ctx.template Alloc<T>(y_grad);
|
||||
total_elems = x.dims()[1] * y.dims()[1];
|
||||
// y_grad = x' * out_grad. y_grad K x N, out_grad : M x N, x : M x K
|
||||
blas.MatMul(x, true, out_grad, false, y_grad);
|
||||
if (!is_float16_or_bfloat16 && !is_big_tensor) {
|
||||
mt_blas.SCAL(total_elems, alpha, y_grad->data<MPType>());
|
||||
} else {
|
||||
funcs::ForRange<Context> for_range(dev_ctx, total_elems);
|
||||
CopyOrScaleFunctor<T> functor(
|
||||
alpha, y_grad->data<T>(), y_grad->data<T>(), total_elems);
|
||||
for_range(functor);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,152 @@
|
||||
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <type_traits>
|
||||
|
||||
#include "glog/logging.h"
|
||||
|
||||
#include "paddle/phi/common/amp_type_traits.h"
|
||||
#include "paddle/phi/kernels/addmm_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/blas/blas.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/eigen_function.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
using Array1 = Eigen::DSizes<int64_t, 1>;
|
||||
using Array2 = Eigen::DSizes<int64_t, 2>;
|
||||
|
||||
template <typename T, typename Context>
|
||||
void AddmmKernel(const Context& dev_ctx,
|
||||
const DenseTensor& input,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& y,
|
||||
float beta,
|
||||
float alpha,
|
||||
DenseTensor* out) {
|
||||
auto input_dims = input.dims();
|
||||
auto x_dims = x.dims();
|
||||
auto y_dims = y.dims();
|
||||
|
||||
DenseTensor input_2d(input);
|
||||
if (input.dims().size() == 1) {
|
||||
input_dims = {1, input.dims()[0]};
|
||||
input_2d.Resize(input_dims);
|
||||
}
|
||||
|
||||
// broadcast mode check
|
||||
if (x_dims[0] != input_dims[0]) {
|
||||
PADDLE_ENFORCE_EQ(input_dims[0],
|
||||
1,
|
||||
errors::InvalidArgument(
|
||||
"When x_dims[0] is not equal with input_dims[0], "
|
||||
"input_dims[0] must be 1 but got %s",
|
||||
input_dims[0]));
|
||||
PADDLE_ENFORCE_EQ(y_dims[1] == input_dims[1] || input_dims[1] == 1,
|
||||
true,
|
||||
errors::InvalidArgument(
|
||||
"The input tensor shape mismatch, input shape=[%s], "
|
||||
"x shape=[%s], y shape=[%s]",
|
||||
input_dims,
|
||||
x_dims,
|
||||
y_dims));
|
||||
}
|
||||
// broadcast mode check
|
||||
if (y_dims[1] != input_dims[1]) {
|
||||
PADDLE_ENFORCE_EQ(input_dims[1],
|
||||
1,
|
||||
errors::InvalidArgument(
|
||||
"When y_dims[1] is not equal with input_dims[0], "
|
||||
"input_dims[0] must be 1 but got %s",
|
||||
input_dims[1]));
|
||||
PADDLE_ENFORCE_EQ(x_dims[0] == input_dims[0] || input_dims[0] == 1,
|
||||
true,
|
||||
errors::InvalidArgument(
|
||||
"The input tensor shape mismatch, input shape=[%s], "
|
||||
"x shape=[%s], y shape=[%s]",
|
||||
input_dims,
|
||||
x_dims,
|
||||
y_dims));
|
||||
}
|
||||
// broadcast mode check
|
||||
PADDLE_ENFORCE_EQ(
|
||||
x_dims[1],
|
||||
y_dims[0],
|
||||
errors::InvalidArgument(
|
||||
"The input tensor X's width must be equal with matrix Y' height. "
|
||||
"But received X's shape = [%s], Y's shape = [%s].",
|
||||
x_dims[1],
|
||||
y_dims[0]));
|
||||
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
if (out->numel() == 0) return;
|
||||
|
||||
auto blas = funcs::GetBlas<Context, T>(dev_ctx);
|
||||
|
||||
// calc broadcast dim
|
||||
Array2 bcast_dims;
|
||||
bcast_dims[0] = x_dims[0] / input_dims[0];
|
||||
bcast_dims[1] = y_dims[1] / input_dims[1];
|
||||
VLOG(3) << "bcast_dims=[" << bcast_dims[0] << "," << bcast_dims[1] << "]";
|
||||
// broadcast using eigen
|
||||
const DenseTensor& const_ref_input = input_2d;
|
||||
auto eigen_input = EigenTensor<T, 2>::From(const_ref_input);
|
||||
auto eigen_out = EigenTensor<T, 2>::From(*out);
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
funcs::EigenBroadcast<std::decay_t<decltype(place)>, T, 2>::Eval(
|
||||
place, eigen_out, eigen_input, bcast_dims);
|
||||
|
||||
// Just return input X beta
|
||||
if (x.numel() == 0 || y.numel() == 0) {
|
||||
auto eigen_out2 = EigenVector<T>::Flatten(*out);
|
||||
eigen_out2.device(place) = eigen_out2 * static_cast<T>(beta);
|
||||
return;
|
||||
}
|
||||
|
||||
using MPType = typename MPTypeTrait<T>::Type;
|
||||
if constexpr (std::is_same_v<MPType, float>) {
|
||||
float t_alpha = alpha;
|
||||
float t_beta = beta;
|
||||
blas.GEMM(CblasNoTrans,
|
||||
CblasNoTrans,
|
||||
x_dims[0],
|
||||
y_dims[1],
|
||||
x_dims[1],
|
||||
t_alpha,
|
||||
x.data<T>(),
|
||||
y.data<T>(),
|
||||
t_beta,
|
||||
out->data<T>());
|
||||
} else {
|
||||
T t_alpha = static_cast<T>(alpha);
|
||||
T t_beta = static_cast<T>(beta);
|
||||
blas.GEMM(false,
|
||||
false,
|
||||
x_dims[0],
|
||||
y_dims[1],
|
||||
x_dims[1],
|
||||
t_alpha,
|
||||
x.data<T>(),
|
||||
x_dims[1],
|
||||
y.data<T>(),
|
||||
y_dims[1],
|
||||
t_beta,
|
||||
out->data<T>(),
|
||||
y_dims[1]);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,180 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/common/hostdevice.h"
|
||||
#include "paddle/phi/common/amp_type_traits.h"
|
||||
#include "paddle/phi/kernels/amp_kernel.h"
|
||||
#include "paddle/phi/kernels/full_kernel.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T>
|
||||
inline HOSTDEVICE bool CheckFinite(T value) {
|
||||
#if defined(PADDLE_WITH_CUDA) && defined(__NVCC__)
|
||||
return isfinite(value);
|
||||
#else
|
||||
return std::isfinite(value);
|
||||
#endif
|
||||
}
|
||||
|
||||
inline HOSTDEVICE bool IsFoundNanInf(const bool found_nan_inf_data) {
|
||||
return found_nan_inf_data;
|
||||
}
|
||||
|
||||
inline HOSTDEVICE bool IsFoundNanInf(const bool* found_nan_inf_data) {
|
||||
return *found_nan_inf_data;
|
||||
}
|
||||
|
||||
template <typename T, typename FoundInfFlagT>
|
||||
inline HOSTDEVICE void Update(const FoundInfFlagT found_inf_data,
|
||||
const T* pre_loss_scaling_data,
|
||||
const int* good_in_data,
|
||||
const int* bad_in_data,
|
||||
const int incr_every_n_steps,
|
||||
const int decr_every_n_nan_or_inf,
|
||||
const float incr_ratio,
|
||||
const float decr_ratio,
|
||||
T* updated_loss_scaling_data,
|
||||
int* good_out_data,
|
||||
int* bad_out_data) {
|
||||
if (IsFoundNanInf(found_inf_data)) {
|
||||
*good_out_data = 0;
|
||||
*bad_out_data = *bad_in_data + 1;
|
||||
if (*bad_out_data == decr_every_n_nan_or_inf) {
|
||||
T new_loss_scaling = *pre_loss_scaling_data * decr_ratio;
|
||||
*updated_loss_scaling_data = new_loss_scaling < static_cast<T>(1)
|
||||
? static_cast<T>(1)
|
||||
: new_loss_scaling;
|
||||
*bad_out_data = 0;
|
||||
}
|
||||
} else {
|
||||
*bad_out_data = 0;
|
||||
*good_out_data = *good_in_data + 1;
|
||||
if (*good_out_data == incr_every_n_steps) {
|
||||
T new_loss_scaling = *pre_loss_scaling_data * incr_ratio;
|
||||
*updated_loss_scaling_data = CheckFinite(new_loss_scaling)
|
||||
? new_loss_scaling
|
||||
: *pre_loss_scaling_data;
|
||||
*good_out_data = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Context, typename T>
|
||||
class LazyZeros {
|
||||
public:
|
||||
void operator()(const Context& dev_ctx UNUSED,
|
||||
const bool* found_inf_data UNUSED,
|
||||
const std::vector<const DenseTensor*>& xs UNUSED,
|
||||
const std::vector<DenseTensor*>& outs UNUSED) const {}
|
||||
};
|
||||
|
||||
template <typename Context, typename T, bool IsFoundInfOnCPU>
|
||||
class UpdateLossScalingFunctor {
|
||||
public:
|
||||
void operator()(const Context& dev_ctx,
|
||||
const bool* found_inf_data,
|
||||
const T* pre_loss_scaling_data,
|
||||
const int* good_in_data,
|
||||
const int* bad_in_data,
|
||||
const int incr_every_n_steps,
|
||||
const int decr_every_n_nan_or_inf,
|
||||
const float incr_ratio,
|
||||
const float decr_ratio,
|
||||
T* updated_loss_scaling_data,
|
||||
int* good_out_data,
|
||||
int* bad_out_data) const;
|
||||
};
|
||||
|
||||
template <typename T, typename Context>
|
||||
void UpdateLossScalingKernel(const Context& dev_ctx,
|
||||
const std::vector<const DenseTensor*>& xs,
|
||||
const DenseTensor& found_infinite,
|
||||
const DenseTensor& prev_loss_scaling,
|
||||
const DenseTensor& in_good_steps,
|
||||
const DenseTensor& in_bad_steps,
|
||||
int incr_every_n_steps,
|
||||
int decr_every_n_nan_or_inf,
|
||||
float incr_ratio,
|
||||
float decr_ratio,
|
||||
const Scalar& stop_update,
|
||||
std::vector<DenseTensor*> outs,
|
||||
DenseTensor* loss_scaling,
|
||||
DenseTensor* out_good_steps,
|
||||
DenseTensor* out_bad_steps) {
|
||||
using MT = typename MPTypeTrait<T>::Type;
|
||||
|
||||
PADDLE_ENFORCE_EQ(found_infinite.numel(),
|
||||
1,
|
||||
common::errors::InvalidArgument(
|
||||
"FoundInfinite must has only one element."));
|
||||
const bool* found_inf_data = found_infinite.data<bool>();
|
||||
bool is_found_inf_on_cpu =
|
||||
found_infinite.place().GetType() == AllocationType::CPU;
|
||||
|
||||
if (is_found_inf_on_cpu) {
|
||||
if (*found_inf_data) {
|
||||
for (auto* out : outs) {
|
||||
Full<T>(dev_ctx, vectorize(out->dims()), static_cast<T>(0), out);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
LazyZeros<Context, T>{}(dev_ctx, found_inf_data, xs, outs);
|
||||
}
|
||||
|
||||
auto stop_update_val = stop_update.to<bool>();
|
||||
if (stop_update_val) {
|
||||
return;
|
||||
}
|
||||
|
||||
const MT* pre_loss_scaling_data = prev_loss_scaling.data<MT>();
|
||||
const int* good_in_data = in_good_steps.data<int>();
|
||||
const int* bad_in_data = in_bad_steps.data<int>();
|
||||
|
||||
MT* updated_loss_scaling_data = dev_ctx.template Alloc<MT>(loss_scaling);
|
||||
int* good_out_data = dev_ctx.template Alloc<int>(out_good_steps);
|
||||
int* bad_out_data = dev_ctx.template Alloc<int>(out_bad_steps);
|
||||
|
||||
if (is_found_inf_on_cpu) {
|
||||
UpdateLossScalingFunctor<Context, MT, true>{}(dev_ctx,
|
||||
found_inf_data,
|
||||
pre_loss_scaling_data,
|
||||
good_in_data,
|
||||
bad_in_data,
|
||||
incr_every_n_steps,
|
||||
decr_every_n_nan_or_inf,
|
||||
incr_ratio,
|
||||
decr_ratio,
|
||||
updated_loss_scaling_data,
|
||||
good_out_data,
|
||||
bad_out_data);
|
||||
} else {
|
||||
UpdateLossScalingFunctor<Context, MT, false>{}(dev_ctx,
|
||||
found_inf_data,
|
||||
pre_loss_scaling_data,
|
||||
good_in_data,
|
||||
bad_in_data,
|
||||
incr_every_n_steps,
|
||||
decr_every_n_nan_or_inf,
|
||||
incr_ratio,
|
||||
decr_ratio,
|
||||
updated_loss_scaling_data,
|
||||
good_out_data,
|
||||
bad_out_data);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,125 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
#include <algorithm>
|
||||
#include <vector>
|
||||
|
||||
#include "paddle/phi/common/transform.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
#ifdef PADDLE_WITH_CUDA
|
||||
#ifndef _WIN32
|
||||
template <typename T>
|
||||
extern __global__ void GenAnchors(T* out,
|
||||
const T* aspect_ratios,
|
||||
const int ar_num,
|
||||
const T* anchor_sizes,
|
||||
const int as_num,
|
||||
const T* stride,
|
||||
const int sd_num,
|
||||
const int height,
|
||||
const int width,
|
||||
const T offset);
|
||||
|
||||
template <typename T>
|
||||
extern __global__ void SetVariance(T* out,
|
||||
const T* var,
|
||||
const int vnum,
|
||||
const int num);
|
||||
#endif
|
||||
#endif
|
||||
|
||||
template <typename T, typename Context>
|
||||
void AnchorGeneratorOpKernel(const Context& dev_ctx,
|
||||
const DenseTensor& input_in,
|
||||
const std::vector<float>& anchor_sizes,
|
||||
const std::vector<float>& aspect_ratios,
|
||||
const std::vector<float>& variances,
|
||||
const std::vector<float>& stride,
|
||||
float offset_in,
|
||||
DenseTensor* anchors_out,
|
||||
DenseTensor* variances_out) {
|
||||
auto* input = &input_in;
|
||||
auto* anchors = anchors_out;
|
||||
auto* vars = variances_out;
|
||||
|
||||
T offset = static_cast<T>(offset_in);
|
||||
|
||||
auto feature_width = input->dims()[3];
|
||||
auto feature_height = input->dims()[2];
|
||||
|
||||
T stride_width, stride_height;
|
||||
stride_width = stride[0];
|
||||
stride_height = stride[1];
|
||||
|
||||
int num_anchors = aspect_ratios.size() * anchor_sizes.size();
|
||||
|
||||
dev_ctx.template Alloc<T>(anchors);
|
||||
dev_ctx.template Alloc<T>(vars);
|
||||
|
||||
auto e_anchors = EigenTensor<T, 4>::From(*anchors);
|
||||
for (int h_idx = 0; h_idx < feature_height; ++h_idx) {
|
||||
for (int w_idx = 0; w_idx < feature_width; ++w_idx) {
|
||||
T x_ctr = (w_idx * stride_width) + offset * (stride_width - 1);
|
||||
T y_ctr = (h_idx * stride_height) + offset * (stride_height - 1);
|
||||
T area, area_ratios;
|
||||
T base_w, base_h;
|
||||
T scale_w, scale_h;
|
||||
T anchor_width, anchor_height;
|
||||
int idx = 0;
|
||||
for (size_t r = 0; r < aspect_ratios.size(); ++r) {
|
||||
auto ar = aspect_ratios[r];
|
||||
for (size_t s = 0; s < anchor_sizes.size(); ++s) {
|
||||
auto anchor_size = anchor_sizes[s];
|
||||
area = stride_width * stride_height;
|
||||
area_ratios = area / ar;
|
||||
base_w = round(sqrt(area_ratios));
|
||||
base_h = round(base_w * ar);
|
||||
scale_w = anchor_size / stride_width;
|
||||
scale_h = anchor_size / stride_height;
|
||||
anchor_width = scale_w * base_w;
|
||||
anchor_height = scale_h * base_h;
|
||||
e_anchors(h_idx, w_idx, idx, 0) = (x_ctr - 0.5 * (anchor_width - 1));
|
||||
e_anchors(h_idx, w_idx, idx, 1) = (y_ctr - 0.5 * (anchor_height - 1));
|
||||
e_anchors(h_idx, w_idx, idx, 2) = (x_ctr + 0.5 * (anchor_width - 1));
|
||||
e_anchors(h_idx, w_idx, idx, 3) = (y_ctr + 0.5 * (anchor_height - 1));
|
||||
idx++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
DenseTensor var_t;
|
||||
var_t.Resize({1, static_cast<int>(variances.size())});
|
||||
dev_ctx.template Alloc<T>(&var_t);
|
||||
auto var_et = EigenTensor<T, 2>::From(var_t);
|
||||
for (size_t i = 0; i < variances.size(); ++i) {
|
||||
var_et(0, i) = variances[i];
|
||||
}
|
||||
|
||||
int anchor_num = feature_height * feature_width * num_anchors;
|
||||
auto var_dim = vars->dims();
|
||||
vars->Resize({anchor_num, static_cast<int>(variances.size())});
|
||||
|
||||
auto e_vars = EigenMatrix<T, Eigen::RowMajor>::From(*vars);
|
||||
e_vars = var_et.broadcast(Eigen::DSizes<int, 2>(anchor_num, 1));
|
||||
|
||||
vars->Resize(var_dim);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,43 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/funcs/complex_functors.h"
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void AngleGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& out_grad,
|
||||
DenseTensor* x_grad) {
|
||||
auto numel = out_grad.numel();
|
||||
auto* dout_data = out_grad.data<dtype::Real<T>>();
|
||||
auto* x_data = x.data<T>();
|
||||
x_grad->Resize(out_grad.dims());
|
||||
if (x_grad->numel() == 0) {
|
||||
dev_ctx.template Alloc<T>(x_grad);
|
||||
return;
|
||||
}
|
||||
auto* dx_data = dev_ctx.template Alloc<T>(x_grad);
|
||||
|
||||
funcs::ForRange<Context> for_range(dev_ctx, numel);
|
||||
funcs::AngleGradFunctor<T> functor(dout_data, x_data, dx_data, numel);
|
||||
for_range(functor);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,41 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/funcs/complex_functors.h"
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void AngleKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
DenseTensor* out) {
|
||||
auto numel = x.numel();
|
||||
auto* x_data = x.data<T>();
|
||||
out->Resize(x.dims());
|
||||
if (out->numel() == 0) {
|
||||
dev_ctx.template Alloc<dtype::Real<T>>(out);
|
||||
return;
|
||||
}
|
||||
auto* out_data = dev_ctx.template Alloc<dtype::Real<T>>(out);
|
||||
|
||||
funcs::ForRange<Context> for_range(dev_ctx, numel);
|
||||
funcs::AngleFunctor<T> functor(x_data, out_data, numel);
|
||||
for_range(functor);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,46 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/as_complex_kernel.h"
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/core/tensor_utils.h"
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
/**
|
||||
* @brief This operator is used to return a complex tensor represented by an
|
||||
* old-fashioned real tensor. The size of the last dimension of the input tensor
|
||||
* should be 2, which corresponds to 'real' and 'complex', respectively.
|
||||
*
|
||||
* @param dev_ctx device context
|
||||
* @param x the input tensor of as_complex
|
||||
* @param out the output tensor of as_complex
|
||||
*/
|
||||
template <typename T, typename Context>
|
||||
void AsComplexKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
DenseTensor* out) {
|
||||
dev_ctx.template Alloc<dtype::complex<T>>(out);
|
||||
auto out_dims_original = out->dims();
|
||||
Copy(dev_ctx, x, dev_ctx.GetPlace(), false, out);
|
||||
out->Resize(out_dims_original); // restored the shape.
|
||||
out->set_type(CppTypeToDataType<dtype::complex<T>>::Type()); // restored the
|
||||
// dtype.
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,47 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/as_real_kernel.h"
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/core/tensor_utils.h"
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
/**
|
||||
* @brief This operator is used to return an old-fashioned real tensor from a
|
||||
* complex tensor. The size of the last dimension of the output tensor is 2,
|
||||
* which corresponds to 'real' and 'complex', respectively.
|
||||
*
|
||||
* @param dev_ctx device context
|
||||
* @param x the input tensor of as_real
|
||||
* @param out the output tensor of as_real
|
||||
*/
|
||||
template <typename T, typename Context>
|
||||
void AsRealKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
DenseTensor* out) {
|
||||
dev_ctx.template Alloc<typename T::value_type>(out);
|
||||
auto out_dims_original = out->dims();
|
||||
Copy(dev_ctx, x, dev_ctx.GetPlace(), false, out);
|
||||
out->Resize(out_dims_original); // restored the shape.
|
||||
out->set_type(
|
||||
CppTypeToDataType<typename T::value_type>::Type()); // restored the
|
||||
// dtype.
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,199 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/core/tensor_utils.h"
|
||||
#include "paddle/phi/kernels/atan2_grad_kernel.h"
|
||||
#include "paddle/phi/kernels/broadcast_tensors_kernel.h"
|
||||
#include "paddle/phi/kernels/full_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
#include "paddle/phi/kernels/reduce_sum_kernel.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
// dx1 = dout * x2 / ((x1)^2 + (x2)^2)
|
||||
// dx2 = - dout * x1 / ((x1)^2 + (x2)^2)
|
||||
template <typename T>
|
||||
struct Atan2GradFunctor {
|
||||
Atan2GradFunctor(
|
||||
const T* x1, const T* x2, const T* dout, T* dx1, T* dx2, int64_t numel)
|
||||
: x1_(x1), x2_(x2), dout_(dout), dx1_(dx1), dx2_(dx2), numel_(numel) {}
|
||||
|
||||
HOSTDEVICE void operator()(int64_t idx) const {
|
||||
float x1 = static_cast<float>(x1_[idx]);
|
||||
float x2 = static_cast<float>(x2_[idx]);
|
||||
float x = x1 * x1 + x2 * x2;
|
||||
if (dx1_) {
|
||||
dx1_[idx] = static_cast<T>(static_cast<float>(dout_[idx]) * x2 / x);
|
||||
}
|
||||
if (dx2_) {
|
||||
dx2_[idx] = static_cast<T>(-static_cast<float>(dout_[idx]) * x1 / x);
|
||||
}
|
||||
}
|
||||
|
||||
const T* x1_;
|
||||
const T* x2_;
|
||||
const T* dout_;
|
||||
T* dx1_;
|
||||
T* dx2_;
|
||||
int64_t numel_;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct Atan2GradFunctor<double> {
|
||||
Atan2GradFunctor(const double* x1,
|
||||
const double* x2,
|
||||
const double* dout,
|
||||
double* dx1,
|
||||
double* dx2,
|
||||
int64_t numel)
|
||||
: x1_(x1), x2_(x2), dout_(dout), dx1_(dx1), dx2_(dx2), numel_(numel) {}
|
||||
|
||||
HOSTDEVICE void operator()(int64_t idx) const {
|
||||
auto x = x1_[idx] * x1_[idx] + x2_[idx] * x2_[idx];
|
||||
if (dx1_) {
|
||||
dx1_[idx] = dout_[idx] * x2_[idx] / x;
|
||||
}
|
||||
if (dx2_) {
|
||||
dx2_[idx] = -dout_[idx] * x1_[idx] / x;
|
||||
}
|
||||
}
|
||||
|
||||
const double* x1_;
|
||||
const double* x2_;
|
||||
const double* dout_;
|
||||
double* dx1_;
|
||||
double* dx2_;
|
||||
int64_t numel_;
|
||||
};
|
||||
|
||||
template <typename T, typename Context>
|
||||
void Atan2GradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& y,
|
||||
const DenseTensor& out_grad,
|
||||
DenseTensor* x_grad,
|
||||
DenseTensor* y_grad) {
|
||||
if (out_grad.numel() == 0) {
|
||||
if (x_grad) {
|
||||
dev_ctx.template Alloc<T>(x_grad);
|
||||
if (x_grad->numel() != 0) {
|
||||
Full<T, Context>(dev_ctx, x_grad->dims(), 0, x_grad);
|
||||
}
|
||||
}
|
||||
if (y_grad) {
|
||||
dev_ctx.template Alloc<T>(y_grad);
|
||||
if (y_grad->numel() != 0) {
|
||||
Full<T, Context>(dev_ctx, y_grad->dims(), 0, y_grad);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (x.dims() == y.dims() && x.dims() == out_grad.dims()) {
|
||||
auto numel = x.numel();
|
||||
auto x_data = x.data<T>();
|
||||
auto y_data = y.data<T>();
|
||||
auto out_grad_data = out_grad.data<T>();
|
||||
|
||||
auto* x_grad_data = x_grad ? dev_ctx.template Alloc<T>(
|
||||
x_grad, size_t(x.numel() * sizeof(T)))
|
||||
: nullptr;
|
||||
auto* y_grad_data = y_grad ? dev_ctx.template Alloc<T>(
|
||||
y_grad, size_t(y.numel() * sizeof(T)))
|
||||
: nullptr;
|
||||
|
||||
funcs::ForRange<Context> for_range(dev_ctx, numel);
|
||||
Atan2GradFunctor<T> functor(
|
||||
x_data, y_data, out_grad_data, x_grad_data, y_grad_data, numel);
|
||||
for_range(functor);
|
||||
} else {
|
||||
DenseTensor b_x, b_y;
|
||||
b_x.Resize(out_grad.dims());
|
||||
b_y.Resize(out_grad.dims());
|
||||
|
||||
std::vector<const DenseTensor*> inputs = {&x, &y};
|
||||
std::vector<DenseTensor*> outputs = {&b_x, &b_y};
|
||||
BroadcastTensorsKernel<T, Context>(dev_ctx, inputs, outputs);
|
||||
|
||||
DenseTensor dx_b, dy_b;
|
||||
T* dx_b_data = nullptr;
|
||||
T* dy_b_data = nullptr;
|
||||
std::vector<int64_t> x_axes, y_axes;
|
||||
|
||||
if (x_grad) {
|
||||
int in_rank = x.dims().size();
|
||||
int out_rank = out_grad.dims().size();
|
||||
int diff = out_rank - in_rank;
|
||||
for (int i = 0; i < diff; ++i) x_axes.push_back(i);
|
||||
for (int i = 0; i < in_rank; ++i) {
|
||||
if (x.dims()[i] == 1 && out_grad.dims()[i + diff] > 1) {
|
||||
x_axes.push_back(i + diff);
|
||||
}
|
||||
}
|
||||
if (x_axes.empty()) {
|
||||
dev_ctx.template Alloc<T>(x_grad);
|
||||
dx_b_data = x_grad->data<T>();
|
||||
} else {
|
||||
dx_b.Resize(out_grad.dims());
|
||||
dx_b_data = dev_ctx.template Alloc<T>(&dx_b);
|
||||
}
|
||||
}
|
||||
|
||||
if (y_grad) {
|
||||
int in_rank = y.dims().size();
|
||||
int out_rank = out_grad.dims().size();
|
||||
int diff = out_rank - in_rank;
|
||||
for (int i = 0; i < diff; ++i) y_axes.push_back(i);
|
||||
for (int i = 0; i < in_rank; ++i) {
|
||||
if (y.dims()[i] == 1 && out_grad.dims()[i + diff] > 1) {
|
||||
y_axes.push_back(i + diff);
|
||||
}
|
||||
}
|
||||
if (y_axes.empty()) {
|
||||
dev_ctx.template Alloc<T>(y_grad);
|
||||
dy_b_data = y_grad->data<T>();
|
||||
} else {
|
||||
dy_b.Resize(out_grad.dims());
|
||||
dy_b_data = dev_ctx.template Alloc<T>(&dy_b);
|
||||
}
|
||||
}
|
||||
|
||||
auto numel = out_grad.numel();
|
||||
funcs::ForRange<Context> for_range(dev_ctx, numel);
|
||||
Atan2GradFunctor<T> functor(b_x.data<T>(),
|
||||
b_y.data<T>(),
|
||||
out_grad.data<T>(),
|
||||
dx_b_data,
|
||||
dy_b_data,
|
||||
numel);
|
||||
for_range(functor);
|
||||
|
||||
if (x_grad && !x_axes.empty()) {
|
||||
SumKernel<T, Context>(
|
||||
dev_ctx, dx_b, IntArray(x_axes), x_grad->dtype(), false, x_grad);
|
||||
x_grad->Resize(x.dims());
|
||||
}
|
||||
|
||||
if (y_grad && !y_axes.empty()) {
|
||||
SumKernel<T, Context>(
|
||||
dev_ctx, dy_b, IntArray(y_axes), y_grad->dtype(), false, y_grad);
|
||||
y_grad->Resize(y.dims());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,110 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/core/device_context.h"
|
||||
#include "paddle/phi/kernels/atan2_kernel.h"
|
||||
#include "paddle/phi/kernels/broadcast_tensors_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/common_shape.h"
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
|
||||
namespace phi {
|
||||
template <typename T>
|
||||
struct Atan2Out {
|
||||
using type = T;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct Atan2Out<int32_t> {
|
||||
using type = double;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct Atan2Out<int64_t> {
|
||||
using type = double;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct Atan2Functor {
|
||||
Atan2Functor(const T* x1,
|
||||
const T* x2,
|
||||
typename Atan2Out<T>::type* out,
|
||||
int64_t numel)
|
||||
: x1_(x1), x2_(x2), out_(out), numel_(numel) {}
|
||||
|
||||
HOSTDEVICE void operator()(int64_t idx) const {
|
||||
out_[idx] = static_cast<typename Atan2Out<T>::type>(
|
||||
::atan2f(static_cast<float>(x1_[idx]), static_cast<float>(x2_[idx])));
|
||||
}
|
||||
|
||||
const T* x1_;
|
||||
const T* x2_;
|
||||
typename Atan2Out<T>::type* out_;
|
||||
int64_t numel_;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct Atan2Functor<double> {
|
||||
Atan2Functor(const double* x1, const double* x2, double* out, int64_t numel)
|
||||
: x1_(x1), x2_(x2), out_(out), numel_(numel) {}
|
||||
|
||||
HOSTDEVICE void operator()(int64_t idx) const {
|
||||
out_[idx] = ::atan2(x1_[idx], x2_[idx]);
|
||||
}
|
||||
|
||||
const double* x1_;
|
||||
const double* x2_;
|
||||
double* out_;
|
||||
int64_t numel_;
|
||||
};
|
||||
|
||||
template <typename T, typename Context>
|
||||
void Atan2Kernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& y,
|
||||
DenseTensor* out) {
|
||||
dev_ctx.template Alloc<typename Atan2Out<T>::type>(out);
|
||||
if (out->numel() == 0) return;
|
||||
|
||||
if (x.dims() == y.dims()) {
|
||||
const auto numel = out->numel();
|
||||
const auto* x_data = x.data<T>();
|
||||
const auto* y_data = y.data<T>();
|
||||
|
||||
auto* out_data = out->data<typename Atan2Out<T>::type>();
|
||||
funcs::ForRange<Context> for_range(dev_ctx, numel);
|
||||
Atan2Functor<T> functor(x_data, y_data, out_data, numel);
|
||||
for_range(functor);
|
||||
} else {
|
||||
DenseTensor b_x, b_y;
|
||||
// Calculate broadcasted dims
|
||||
b_x.Resize(out->dims());
|
||||
b_y.Resize(out->dims());
|
||||
std::vector<const DenseTensor*> inputs = {&x, &y};
|
||||
std::vector<DenseTensor*> outputs = {&b_x, &b_y};
|
||||
BroadcastTensorsKernel<T, Context>(dev_ctx, inputs, outputs);
|
||||
|
||||
const auto numel = out->numel();
|
||||
const auto* x_data = b_x.data<T>();
|
||||
const auto* y_data = b_y.data<T>();
|
||||
auto* out_data = out->data<typename Atan2Out<T>::type>();
|
||||
funcs::ForRange<Context> for_range(dev_ctx, numel);
|
||||
Atan2Functor<T> functor(x_data, y_data, out_data, numel);
|
||||
for_range(functor);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,143 @@
|
||||
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
#include "paddle/phi/kernels/average_accumulates_kernel.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
#include "paddle/phi/common/memory_utils.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void AverageAccumulatesKernel(const Context& dev_ctx,
|
||||
const DenseTensor& param,
|
||||
const DenseTensor& in_sum_1,
|
||||
const DenseTensor& in_sum_2,
|
||||
const DenseTensor& in_sum_3,
|
||||
const DenseTensor& in_num_accumulates,
|
||||
const DenseTensor& in_old_num_accumulates,
|
||||
const DenseTensor& in_num_updates,
|
||||
float average_window,
|
||||
int64_t max_average_window,
|
||||
int64_t min_average_window,
|
||||
DenseTensor* out_sum_1,
|
||||
DenseTensor* out_sum_2,
|
||||
DenseTensor* out_sum_3,
|
||||
DenseTensor* out_num_accumulates,
|
||||
DenseTensor* out_old_num_accumulates,
|
||||
DenseTensor* out_num_updates) {
|
||||
// It is used to avoid loss of precision
|
||||
static const int64_t kMaxNumAccumulates = 16384;
|
||||
// Get accumulators from input
|
||||
// int64_t num_updates = 0;
|
||||
// int64_t num_accumulates = 0;
|
||||
// int64_t old_num_accumulates = 0;
|
||||
|
||||
auto num_updates_cpu = memory_utils::Alloc(CPUPlace(), sizeof(int64_t));
|
||||
int64_t* num_updates_cpu_ptr =
|
||||
reinterpret_cast<int64_t*>(num_updates_cpu->ptr());
|
||||
|
||||
auto num_accumulates_cpu = memory_utils::Alloc(CPUPlace(), sizeof(int64_t));
|
||||
int64_t* num_accumulates_cpu_ptr =
|
||||
reinterpret_cast<int64_t*>(num_accumulates_cpu->ptr());
|
||||
|
||||
auto old_num_accumulates_cpu =
|
||||
memory_utils::Alloc(CPUPlace(), sizeof(int64_t));
|
||||
int64_t* old_num_accumulates_cpu_ptr =
|
||||
reinterpret_cast<int64_t*>(old_num_accumulates_cpu->ptr());
|
||||
|
||||
GetAccumulators<Context>(dev_ctx,
|
||||
in_num_accumulates,
|
||||
in_old_num_accumulates,
|
||||
in_num_updates,
|
||||
num_updates_cpu_ptr,
|
||||
num_accumulates_cpu_ptr,
|
||||
old_num_accumulates_cpu_ptr);
|
||||
// Get attrs
|
||||
// float average_window = dev_ctx.Attr<float>("average_window");
|
||||
// int64_t max_average_window = dev_ctx.Attr<int64_t>("max_average_window");
|
||||
// int64_t min_average_window = dev_ctx.Attr<int64_t>("min_average_window");
|
||||
PADDLE_ENFORCE_LE(
|
||||
min_average_window,
|
||||
max_average_window,
|
||||
errors::InvalidArgument(
|
||||
"The min_average_window > "
|
||||
"max_average_window is not right, min_average_window is %ld, "
|
||||
"max_average_window is %ld.",
|
||||
min_average_window,
|
||||
max_average_window));
|
||||
|
||||
// Get inputs
|
||||
// auto* param = dev_ctx.Input<DenseTensor>("param");
|
||||
// auto* in_sum_1 = dev_ctx.Input<DenseTensor>("in_sum_1");
|
||||
// auto* in_sum_2 = dev_ctx.Input<DenseTensor>("in_sum_2");
|
||||
// auto* in_sum_3 = dev_ctx.Input<DenseTensor>("in_sum_3");
|
||||
auto param_tensor = EigenVector<T>::Flatten(param);
|
||||
auto in_sum_1_tensor = EigenVector<T>::Flatten(in_sum_1);
|
||||
auto in_sum_2_tensor = EigenVector<T>::Flatten(in_sum_2);
|
||||
auto in_sum_3_tensor = EigenVector<T>::Flatten(in_sum_3);
|
||||
|
||||
// Get outputs
|
||||
// auto* out_sum_1 = dev_ctx.Output<DenseTensor>("out_sum_1");
|
||||
// auto* out_sum_2 = dev_ctx.Output<DenseTensor>("out_sum_2");
|
||||
// auto* out_sum_3 = dev_ctx.Output<DenseTensor>("out_sum_3");
|
||||
dev_ctx.template Alloc<T>(out_sum_1);
|
||||
dev_ctx.template Alloc<T>(out_sum_2);
|
||||
dev_ctx.template Alloc<T>(out_sum_3);
|
||||
|
||||
auto out_sum_1_tensor = EigenVector<T>::Flatten(*out_sum_1);
|
||||
auto out_sum_2_tensor = EigenVector<T>::Flatten(*out_sum_2);
|
||||
auto out_sum_3_tensor = EigenVector<T>::Flatten(*out_sum_3);
|
||||
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
|
||||
funcs::SetConstant<Context, T> constant_functor;
|
||||
++(*num_updates_cpu_ptr);
|
||||
++(*num_accumulates_cpu_ptr);
|
||||
out_sum_1_tensor.device(place) = in_sum_1_tensor + param_tensor;
|
||||
out_sum_2_tensor.device(place) = in_sum_2_tensor;
|
||||
out_sum_3_tensor.device(place) = in_sum_3_tensor;
|
||||
if ((*num_updates_cpu_ptr) % kMaxNumAccumulates == 0) {
|
||||
// Move the sum to a different buffer to avoid loss of precision due to
|
||||
// too many sums.
|
||||
out_sum_2_tensor.device(place) = in_sum_2_tensor + in_sum_1_tensor;
|
||||
constant_functor(dev_ctx, out_sum_1, static_cast<T>(0));
|
||||
}
|
||||
if ((*num_accumulates_cpu_ptr) >= min_average_window &&
|
||||
(*num_accumulates_cpu_ptr) >=
|
||||
std::min<int64_t>(max_average_window,
|
||||
(*num_updates_cpu_ptr) * average_window)) {
|
||||
// Now the average window is too long, discard the old sum.
|
||||
out_sum_3_tensor.device(place) = in_sum_1_tensor + in_sum_2_tensor;
|
||||
constant_functor(dev_ctx, out_sum_1, static_cast<T>(0));
|
||||
constant_functor(dev_ctx, out_sum_2, static_cast<T>(0));
|
||||
(*old_num_accumulates_cpu_ptr) = (*num_accumulates_cpu_ptr);
|
||||
(*num_accumulates_cpu_ptr) = 0;
|
||||
}
|
||||
|
||||
// Set accumulators to output
|
||||
SetAccumulators<Context>(dev_ctx,
|
||||
*num_updates_cpu_ptr,
|
||||
*num_accumulates_cpu_ptr,
|
||||
*old_num_accumulates_cpu_ptr,
|
||||
out_num_accumulates,
|
||||
out_old_num_accumulates,
|
||||
out_num_updates);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,318 @@
|
||||
/* Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
#pragma once
|
||||
|
||||
#include <type_traits>
|
||||
|
||||
#include "glog/logging.h"
|
||||
|
||||
#include "paddle/common/flags.h"
|
||||
#include "paddle/phi/common/amp_type_traits.h"
|
||||
#include "paddle/phi/kernels/baddbmm_grad_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/blas/blas.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/eigen_function.h"
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
|
||||
COMMON_DECLARE_bool(use_accuracy_compatible_kernel);
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T>
|
||||
struct BCopyOrScaleFunctor {
|
||||
BCopyOrScaleFunctor(const float scale, const T* x, T* output, int64_t numel)
|
||||
: scale_(scale), x_(x), output_(output), numel_(numel) {}
|
||||
|
||||
HOSTDEVICE void operator()(int64_t idx) const {
|
||||
using MPType = typename MPTypeTrait<T>::Type;
|
||||
const MPType mp_scale = static_cast<MPType>(scale_);
|
||||
const MPType mp_x = static_cast<MPType>(x_[idx]);
|
||||
output_[idx] = static_cast<T>(mp_scale * mp_x);
|
||||
}
|
||||
|
||||
private:
|
||||
const float scale_;
|
||||
const T* x_;
|
||||
T* output_;
|
||||
int64_t numel_;
|
||||
};
|
||||
|
||||
using Array1 = Eigen::DSizes<int64_t, 1>;
|
||||
using Array2 = Eigen::DSizes<int64_t, 2>;
|
||||
using Array3 = Eigen::DSizes<int64_t, 3>;
|
||||
|
||||
template <typename T, typename Context>
|
||||
void BaddbmmGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& input,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& y,
|
||||
const DenseTensor& out_grad,
|
||||
float alpha,
|
||||
float beta,
|
||||
DenseTensor* input_grad,
|
||||
DenseTensor* x_grad,
|
||||
DenseTensor* y_grad) {
|
||||
using MPType = typename MPTypeTrait<T>::Type;
|
||||
bool is_float16_or_bfloat16 = false;
|
||||
if (std::is_same<T, float16>::value || std::is_same<T, bfloat16>::value) {
|
||||
is_float16_or_bfloat16 = true;
|
||||
}
|
||||
|
||||
auto in_dims = input.dims();
|
||||
if (input.dims().size() == 2) {
|
||||
in_dims = {input.dims()[0], 1, input.dims()[1]};
|
||||
input_grad->Resize(in_dims);
|
||||
}
|
||||
int64_t total_elems = 0;
|
||||
|
||||
VLOG(3) << "alpha: " << alpha << " beta: " << beta;
|
||||
|
||||
if (input_grad != nullptr) {
|
||||
input_grad->set_lod(out_grad.lod());
|
||||
}
|
||||
if (x_grad != nullptr) {
|
||||
x_grad->set_lod(x.lod());
|
||||
}
|
||||
if (y_grad != nullptr) {
|
||||
y_grad->set_lod(y.lod());
|
||||
}
|
||||
|
||||
auto blas = funcs::GetBlas<Context, T>(dev_ctx);
|
||||
auto mt_blas = funcs::GetBlas<Context, MPType>(dev_ctx);
|
||||
if (input_grad) {
|
||||
dev_ctx.template Alloc<T>(input_grad);
|
||||
total_elems = in_dims[0] * in_dims[1] * in_dims[2];
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
auto eigen_dout = EigenTensor<T, 3>::From(out_grad);
|
||||
auto eigen_dinput = EigenTensor<T, 3>::From(*input_grad);
|
||||
|
||||
bool batch_compress = in_dims[0] != out_grad.dims()[0];
|
||||
bool row_compress = in_dims[1] != out_grad.dims()[1];
|
||||
bool col_compress = in_dims[2] != out_grad.dims()[2];
|
||||
auto eigen_dinput_shape = Array3(
|
||||
input_grad->dims()[0], input_grad->dims()[1], input_grad->dims()[2]);
|
||||
|
||||
if (batch_compress && row_compress && col_compress) {
|
||||
if (!is_float16_or_bfloat16) {
|
||||
eigen_dinput.device(place) =
|
||||
eigen_dout.sum().eval().reshape(eigen_dinput_shape);
|
||||
} else {
|
||||
eigen_dinput.device(place) = eigen_dout.template cast<MPType>()
|
||||
.sum()
|
||||
.eval()
|
||||
.reshape(eigen_dinput_shape)
|
||||
.template cast<T>();
|
||||
}
|
||||
} else if (batch_compress && row_compress) {
|
||||
if (!is_float16_or_bfloat16) {
|
||||
eigen_dinput.device(place) =
|
||||
eigen_dout.sum(Array2(0, 1)).eval().reshape(eigen_dinput_shape);
|
||||
} else {
|
||||
eigen_dinput.device(place) = eigen_dout.template cast<MPType>()
|
||||
.sum(Array2(0, 1))
|
||||
.eval()
|
||||
.reshape(eigen_dinput_shape)
|
||||
.template cast<T>();
|
||||
}
|
||||
} else if (batch_compress && col_compress) {
|
||||
if (!is_float16_or_bfloat16) {
|
||||
eigen_dinput.device(place) =
|
||||
eigen_dout.sum(Array2(0, 2)).eval().reshape(eigen_dinput_shape);
|
||||
} else {
|
||||
eigen_dinput.device(place) = eigen_dout.template cast<MPType>()
|
||||
.sum(Array2(0, 2))
|
||||
.eval()
|
||||
.reshape(eigen_dinput_shape)
|
||||
.template cast<T>();
|
||||
}
|
||||
} else if (row_compress && col_compress) {
|
||||
if (!is_float16_or_bfloat16) {
|
||||
eigen_dinput.device(place) =
|
||||
eigen_dout.sum(Array2(1, 2)).eval().reshape(eigen_dinput_shape);
|
||||
} else {
|
||||
eigen_dinput.device(place) = eigen_dout.template cast<MPType>()
|
||||
.sum(Array2(1, 2))
|
||||
.eval()
|
||||
.reshape(eigen_dinput_shape)
|
||||
.template cast<T>();
|
||||
}
|
||||
} else if (batch_compress) {
|
||||
if (!is_float16_or_bfloat16) {
|
||||
eigen_dinput.device(place) =
|
||||
eigen_dout.sum(Array1(0)).eval().reshape(eigen_dinput_shape);
|
||||
} else {
|
||||
eigen_dinput.device(place) = eigen_dout.template cast<MPType>()
|
||||
.sum(Array1(0))
|
||||
.eval()
|
||||
.reshape(eigen_dinput_shape)
|
||||
.template cast<T>();
|
||||
}
|
||||
} else if (row_compress) {
|
||||
if (!is_float16_or_bfloat16) {
|
||||
eigen_dinput.device(place) =
|
||||
eigen_dout.sum(Array1(1)).eval().reshape(eigen_dinput_shape);
|
||||
} else {
|
||||
eigen_dinput.device(place) = eigen_dout.template cast<MPType>()
|
||||
.sum(Array1(1))
|
||||
.eval()
|
||||
.reshape(eigen_dinput_shape)
|
||||
.template cast<T>();
|
||||
}
|
||||
} else if (col_compress) {
|
||||
if (!is_float16_or_bfloat16) {
|
||||
eigen_dinput.device(place) =
|
||||
eigen_dout.sum(Array1(2)).eval().reshape(eigen_dinput_shape);
|
||||
} else {
|
||||
eigen_dinput.device(place) = eigen_dout.template cast<MPType>()
|
||||
.sum(Array1(2))
|
||||
.eval()
|
||||
.reshape(eigen_dinput_shape)
|
||||
.template cast<T>();
|
||||
}
|
||||
} else {
|
||||
// The VCOPY does not support the float16, bfloat16
|
||||
if (!is_float16_or_bfloat16) {
|
||||
mt_blas.VCOPY(
|
||||
total_elems, out_grad.data<MPType>(), input_grad->data<MPType>());
|
||||
} else {
|
||||
funcs::ForRange<Context> for_range(dev_ctx, total_elems);
|
||||
BCopyOrScaleFunctor<T> functor(
|
||||
1, out_grad.data<T>(), input_grad->data<T>(), total_elems);
|
||||
for_range(functor);
|
||||
}
|
||||
}
|
||||
|
||||
// The SCAL does not support the float16, bfloat16
|
||||
if (!is_float16_or_bfloat16) {
|
||||
mt_blas.SCAL(total_elems, beta, input_grad->data<MPType>());
|
||||
} else {
|
||||
funcs::ForRange<Context> for_range(dev_ctx, total_elems);
|
||||
BCopyOrScaleFunctor<T> functor(
|
||||
beta, input_grad->data<T>(), input_grad->data<T>(), total_elems);
|
||||
for_range(functor);
|
||||
}
|
||||
}
|
||||
if (x_grad) {
|
||||
dev_ctx.template Alloc<T>(x_grad);
|
||||
total_elems = x.dims()[0] * x.dims()[1] * x.dims()[2];
|
||||
// x_grad = alpha * out_grad @ y^T
|
||||
// out_grad: [B, M, N], y: [B, K, N], x_grad: [B, M, K]
|
||||
int64_t B_dim = x.dims()[0];
|
||||
int64_t M_dim = x.dims()[1];
|
||||
int64_t K_dim = x.dims()[2];
|
||||
int64_t N_dim = y.dims()[2];
|
||||
if constexpr (std::is_same_v<MPType, float>) {
|
||||
float gemm_alpha = FLAGS_use_accuracy_compatible_kernel ? 1.0f : alpha;
|
||||
float zero = 0.0f;
|
||||
blas.BatchedGEMM(CblasNoTrans,
|
||||
CblasTrans,
|
||||
M_dim,
|
||||
K_dim,
|
||||
N_dim,
|
||||
gemm_alpha,
|
||||
out_grad.data<T>(),
|
||||
y.data<T>(),
|
||||
zero,
|
||||
x_grad->data<T>(),
|
||||
B_dim,
|
||||
M_dim * N_dim,
|
||||
K_dim * N_dim);
|
||||
} else {
|
||||
T gemm_alpha = FLAGS_use_accuracy_compatible_kernel
|
||||
? static_cast<T>(1)
|
||||
: static_cast<T>(alpha);
|
||||
T zero = static_cast<T>(0);
|
||||
blas.BatchedGEMM(CblasNoTrans,
|
||||
CblasTrans,
|
||||
M_dim,
|
||||
K_dim,
|
||||
N_dim,
|
||||
gemm_alpha,
|
||||
out_grad.data<T>(),
|
||||
y.data<T>(),
|
||||
zero,
|
||||
x_grad->data<T>(),
|
||||
B_dim,
|
||||
M_dim * N_dim,
|
||||
K_dim * N_dim);
|
||||
}
|
||||
if (FLAGS_use_accuracy_compatible_kernel) {
|
||||
if (!is_float16_or_bfloat16) {
|
||||
mt_blas.SCAL(total_elems, alpha, x_grad->data<MPType>());
|
||||
} else {
|
||||
funcs::ForRange<Context> for_range(dev_ctx, total_elems);
|
||||
BCopyOrScaleFunctor<T> functor(
|
||||
alpha, x_grad->data<T>(), x_grad->data<T>(), total_elems);
|
||||
for_range(functor);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (y_grad) {
|
||||
dev_ctx.template Alloc<T>(y_grad);
|
||||
total_elems = y.dims()[0] * y.dims()[1] * y.dims()[2];
|
||||
// y_grad = alpha * x^T @ out_grad
|
||||
// x: [B, M, K], out_grad: [B, M, N], y_grad: [B, K, N]
|
||||
int64_t B_dim = x.dims()[0];
|
||||
int64_t M_dim = x.dims()[1];
|
||||
int64_t K_dim = x.dims()[2];
|
||||
int64_t N_dim = y.dims()[2];
|
||||
if constexpr (std::is_same_v<MPType, float>) {
|
||||
float gemm_alpha = FLAGS_use_accuracy_compatible_kernel ? 1.0f : alpha;
|
||||
float zero = 0.0f;
|
||||
blas.BatchedGEMM(CblasTrans,
|
||||
CblasNoTrans,
|
||||
K_dim,
|
||||
N_dim,
|
||||
M_dim,
|
||||
gemm_alpha,
|
||||
x.data<T>(),
|
||||
out_grad.data<T>(),
|
||||
zero,
|
||||
y_grad->data<T>(),
|
||||
B_dim,
|
||||
M_dim * K_dim,
|
||||
M_dim * N_dim);
|
||||
} else {
|
||||
T gemm_alpha = FLAGS_use_accuracy_compatible_kernel
|
||||
? static_cast<T>(1)
|
||||
: static_cast<T>(alpha);
|
||||
T zero = static_cast<T>(0);
|
||||
blas.BatchedGEMM(CblasTrans,
|
||||
CblasNoTrans,
|
||||
K_dim,
|
||||
N_dim,
|
||||
M_dim,
|
||||
gemm_alpha,
|
||||
x.data<T>(),
|
||||
out_grad.data<T>(),
|
||||
zero,
|
||||
y_grad->data<T>(),
|
||||
B_dim,
|
||||
M_dim * K_dim,
|
||||
M_dim * N_dim);
|
||||
}
|
||||
if (FLAGS_use_accuracy_compatible_kernel) {
|
||||
if (!is_float16_or_bfloat16) {
|
||||
mt_blas.SCAL(total_elems, alpha, y_grad->data<MPType>());
|
||||
} else {
|
||||
funcs::ForRange<Context> for_range(dev_ctx, total_elems);
|
||||
BCopyOrScaleFunctor<T> functor(
|
||||
alpha, y_grad->data<T>(), y_grad->data<T>(), total_elems);
|
||||
for_range(functor);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,222 @@
|
||||
/* Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <type_traits>
|
||||
|
||||
#include "glog/logging.h"
|
||||
|
||||
#include "paddle/phi/common/amp_type_traits.h"
|
||||
#include "paddle/phi/kernels/baddbmm_kernel.h"
|
||||
#include "paddle/phi/kernels/cast_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/blas/blas.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/eigen_function.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
using Array1 = Eigen::DSizes<int64_t, 1>;
|
||||
using Array2 = Eigen::DSizes<int64_t, 2>;
|
||||
using Array3 = Eigen::DSizes<int64_t, 3>;
|
||||
|
||||
template <typename T, typename Context>
|
||||
void BaddbmmKernel(const Context& dev_ctx,
|
||||
const DenseTensor& input,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& y,
|
||||
float beta,
|
||||
float alpha,
|
||||
DataType out_dtype,
|
||||
DenseTensor* out) {
|
||||
auto input_dims = input.dims();
|
||||
auto x_dims = x.dims();
|
||||
auto y_dims = y.dims();
|
||||
|
||||
DenseTensor input_3d(input);
|
||||
if (input.dims().size() == 2) {
|
||||
input_dims = {input.dims()[0], 1, input.dims()[1]};
|
||||
input_3d.Resize(input_dims);
|
||||
}
|
||||
|
||||
// broadcast mode check
|
||||
if (x_dims[0] != input_dims[0]) {
|
||||
PADDLE_ENFORCE_EQ(input_dims[0],
|
||||
1,
|
||||
errors::InvalidArgument(
|
||||
"When x_dims[0] is not equal with input_dims[0], "
|
||||
"input_dims[0] must be 1 but got %s",
|
||||
input_dims[0]));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
(x_dims[1] == input_dims[1] || input_dims[1] == 1) &&
|
||||
(y_dims[2] == input_dims[2] || input_dims[2] == 1),
|
||||
true,
|
||||
errors::InvalidArgument(
|
||||
"When x_dims[0] is not equal with input_dims[0], "
|
||||
"x_dims[1] and y_dims[2] must be equal with input_dims[1] and "
|
||||
"input_dims[2] respectively, or input_dims[1] and input_dims[2] "
|
||||
"must be 1. But got x_dims[1] = %s, input_dims[1] = %s, y_dims[2] "
|
||||
"= %s, input_dims[2] = %s",
|
||||
x_dims[1],
|
||||
input_dims[1],
|
||||
y_dims[2],
|
||||
input_dims[2]));
|
||||
}
|
||||
|
||||
if (x_dims[1] != input_dims[1]) {
|
||||
PADDLE_ENFORCE_EQ(input_dims[1],
|
||||
1,
|
||||
errors::InvalidArgument(
|
||||
"When x_dims[1] is not equal with input_dims[1], "
|
||||
"input_dims[1] must be 1 but got %s",
|
||||
input_dims[1]));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
(x_dims[0] == input_dims[0] || input_dims[0] == 1) &&
|
||||
(y_dims[2] == input_dims[2] || input_dims[2] == 1),
|
||||
true,
|
||||
errors::InvalidArgument(
|
||||
"When x_dims[1] is not equal with input_dims[1], "
|
||||
"x_dims[0] and y_dims[2] must be equal with input_dims[0] and "
|
||||
"input_dims[2] respectively, or input_dims[0] and input_dims[2] "
|
||||
"must be 1. But got x_dims[0] = %s, input_dims[0] = %s, y_dims[2] "
|
||||
"= %s, input_dims[2] = %s",
|
||||
x_dims[0],
|
||||
input_dims[0],
|
||||
y_dims[2],
|
||||
input_dims[2]));
|
||||
}
|
||||
|
||||
if (y_dims[2] != input_dims[2]) {
|
||||
PADDLE_ENFORCE_EQ(input_dims[2],
|
||||
1,
|
||||
errors::InvalidArgument(
|
||||
"When y_dims[2] is not equal with input_dims[2], "
|
||||
"input_dims[2] must be 1 but got %s",
|
||||
input_dims[2]));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
(x_dims[0] == input_dims[0] || input_dims[0] == 1) &&
|
||||
(x_dims[1] == input_dims[1] || input_dims[1] == 1),
|
||||
true,
|
||||
errors::InvalidArgument(
|
||||
"When y_dims[2] is not equal with input_dims[2], "
|
||||
"x_dims[0] and x_dims[1] must be equal with input_dims[0] and "
|
||||
"input_dims[1] respectively, or input_dims[0] and input_dims[1] "
|
||||
"must be 1. But got x_dims[0] = %s, input_dims[0] = %s, x_dims[1] "
|
||||
"= %s, input_dims[1] = %s",
|
||||
x_dims[0],
|
||||
input_dims[0],
|
||||
x_dims[1],
|
||||
input_dims[1]));
|
||||
}
|
||||
PADDLE_ENFORCE_EQ(
|
||||
x_dims[2],
|
||||
y_dims[1],
|
||||
errors::InvalidArgument(
|
||||
"The input tensor X's width must be equal with matrix Y' height. "
|
||||
"But received X's shape = [%s], Y's shape = [%s].",
|
||||
x_dims[2],
|
||||
y_dims[1]));
|
||||
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
auto blas = funcs::GetBlas<Context, T>(dev_ctx);
|
||||
|
||||
// calc broadcast dim
|
||||
Array3 bcast_dims;
|
||||
bcast_dims[0] = x_dims[0] / input_dims[0];
|
||||
bcast_dims[1] = x_dims[1] / input_dims[1];
|
||||
bcast_dims[2] = y_dims[2] / input_dims[2];
|
||||
VLOG(3) << "bcast_dims=[" << bcast_dims[0] << "," << bcast_dims[1] << ","
|
||||
<< bcast_dims[2] << "]";
|
||||
|
||||
// broadcast using eigen
|
||||
const DenseTensor& const_ref_input = input_3d;
|
||||
auto eigen_input = EigenTensor<T, 3>::From(const_ref_input);
|
||||
auto eigen_out = EigenTensor<T, 3>::From(*out);
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
funcs::EigenBroadcast<std::decay_t<decltype(place)>, T, 3>::Eval(
|
||||
place, eigen_out, eigen_input, bcast_dims);
|
||||
|
||||
using MPType = typename MPTypeTrait<T>::Type;
|
||||
|
||||
// special case for MPType
|
||||
if constexpr (std::is_same_v<MPType, float>) {
|
||||
VLOG(4) << "Function: baddbmm, Type of T: " << typeid(T).name();
|
||||
VLOG(4) << "Function: baddbmm, Type of MPType: " << typeid(MPType).name();
|
||||
float t_alpha = alpha;
|
||||
float t_beta = beta;
|
||||
if (x_dims[0] == 1) {
|
||||
blas.GEMM(CblasNoTrans,
|
||||
CblasNoTrans,
|
||||
x_dims[1],
|
||||
y_dims[2],
|
||||
x_dims[2],
|
||||
t_alpha,
|
||||
x.data<T>(),
|
||||
y.data<T>(),
|
||||
t_beta,
|
||||
out->data<T>());
|
||||
} else {
|
||||
blas.BatchedGEMM(CblasNoTrans,
|
||||
CblasNoTrans,
|
||||
x_dims[1],
|
||||
y_dims[2],
|
||||
x_dims[2],
|
||||
t_alpha,
|
||||
x.data<T>(),
|
||||
y.data<T>(),
|
||||
t_beta,
|
||||
out->data<T>(),
|
||||
x_dims[0],
|
||||
x_dims[1] * x_dims[2],
|
||||
x_dims[2] * y_dims[2]);
|
||||
}
|
||||
} else {
|
||||
T t_alpha = static_cast<T>(alpha);
|
||||
T t_beta = static_cast<T>(beta);
|
||||
if (x_dims[0] == 1) {
|
||||
blas.GEMM(CblasNoTrans,
|
||||
CblasNoTrans,
|
||||
x_dims[1],
|
||||
y_dims[2],
|
||||
x_dims[2],
|
||||
t_alpha,
|
||||
x.data<T>(),
|
||||
y.data<T>(),
|
||||
t_beta,
|
||||
out->data<T>());
|
||||
} else {
|
||||
blas.BatchedGEMM(CblasNoTrans,
|
||||
CblasNoTrans,
|
||||
x_dims[1],
|
||||
y_dims[2],
|
||||
x_dims[2],
|
||||
t_alpha,
|
||||
x.data<T>(),
|
||||
y.data<T>(),
|
||||
t_beta,
|
||||
out->data<T>(),
|
||||
x_dims[0],
|
||||
x_dims[1] * x_dims[2],
|
||||
x_dims[2] * y_dims[2]);
|
||||
// x_dims[2] == y_dims[1]
|
||||
}
|
||||
}
|
||||
|
||||
// Handle out_dtype conversion if specified
|
||||
if (out_dtype != DataType::UNDEFINED && out_dtype != out->dtype()) {
|
||||
CastKernel<T>(dev_ctx, *out, out_dtype, out);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,167 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/funcs/beam_search_decode.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
struct BeamSearchDecodeFunctor {
|
||||
BeamSearchDecodeFunctor(const TensorArray& step_ids,
|
||||
const TensorArray& step_scores,
|
||||
DenseTensor* id_tensor,
|
||||
DenseTensor* score_tensor,
|
||||
size_t beam_size,
|
||||
int end_id)
|
||||
: beam_size_(beam_size),
|
||||
end_id_(end_id),
|
||||
step_ids_origin_(step_ids),
|
||||
step_scores_origin_(step_scores),
|
||||
id_tensor_(id_tensor),
|
||||
score_tensor_(score_tensor) {
|
||||
tensor_on_gpu_ = false;
|
||||
// First make a copy of GPU data on CPU
|
||||
if (step_ids_origin_[0].place().GetType() == AllocationType::GPU ||
|
||||
step_ids_origin_[0].place().GetType() == AllocationType::CUSTOM) {
|
||||
if (step_ids_origin_[0].place().GetType() == AllocationType::GPU ||
|
||||
step_ids_origin_[0].place().GetType() == AllocationType::CUSTOM) {
|
||||
tensor_on_gpu_ = true;
|
||||
}
|
||||
DeviceContextPool& pool = DeviceContextPool::Instance();
|
||||
auto* dev_ctx = pool.Get(step_ids_origin_[0].place());
|
||||
// Copy all tensors in the input tensor array
|
||||
for (auto& step_id : step_ids_origin_) {
|
||||
DenseTensor out;
|
||||
if (step_id.numel() > 0) {
|
||||
if (tensor_on_gpu_) {
|
||||
dev_ctx->Wait();
|
||||
}
|
||||
Copy(*dev_ctx, step_id, CPUPlace(), false, &out);
|
||||
dev_ctx->Wait();
|
||||
}
|
||||
|
||||
out.set_lod(step_id.lod());
|
||||
step_ids_.push_back(out);
|
||||
}
|
||||
}
|
||||
if (step_scores_origin_[0].place().GetType() == AllocationType::GPU ||
|
||||
step_scores_origin_[0].place().GetType() == AllocationType::CUSTOM) {
|
||||
if (step_scores_origin_[0].place().GetType() == AllocationType::GPU ||
|
||||
step_scores_origin_[0].place().GetType() == AllocationType::CUSTOM) {
|
||||
tensor_on_gpu_ = true;
|
||||
}
|
||||
DeviceContextPool& pool = DeviceContextPool::Instance();
|
||||
auto* dev_ctx = pool.Get(step_scores_origin_[0].place());
|
||||
// Copy all tensors in the input tensor array
|
||||
for (auto& step_score : step_scores_origin_) {
|
||||
DenseTensor out;
|
||||
if (step_score.numel() > 0) {
|
||||
if (tensor_on_gpu_) {
|
||||
dev_ctx->Wait();
|
||||
}
|
||||
Copy(*dev_ctx, step_score, CPUPlace(), false, &out);
|
||||
dev_ctx->Wait();
|
||||
}
|
||||
|
||||
out.set_lod(step_score.lod());
|
||||
step_scores_.push_back(out);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void apply_mix() const {
|
||||
if (std::is_same<bool, T>::value) {
|
||||
PADDLE_THROW(common::errors::InvalidArgument(
|
||||
"beam search decode op does not support bool!"));
|
||||
|
||||
} else {
|
||||
funcs::BeamSearchDecoder<T> beam_search_decoder(beam_size_, end_id_);
|
||||
// Check if the tensor is on GPU. If so, use the CPU copy instead
|
||||
if (tensor_on_gpu_) {
|
||||
beam_search_decoder.Backtrace(
|
||||
step_ids_, step_scores_, id_tensor_, score_tensor_);
|
||||
} else {
|
||||
beam_search_decoder.Backtrace(
|
||||
step_ids_origin_, step_scores_origin_, id_tensor_, score_tensor_);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool tensor_on_gpu_;
|
||||
size_t beam_size_;
|
||||
int end_id_;
|
||||
// TODO(Superjomn) Here might result serious performance issue in the
|
||||
// concurrency
|
||||
// scenarios.
|
||||
const TensorArray& step_ids_origin_;
|
||||
const TensorArray& step_scores_origin_;
|
||||
TensorArray step_ids_ = TensorArray();
|
||||
TensorArray step_scores_ = TensorArray();
|
||||
DenseTensor* id_tensor_;
|
||||
DenseTensor* score_tensor_;
|
||||
};
|
||||
|
||||
template <typename T, typename Context>
|
||||
void BeamSearchDecodeOpKernel(const Context& dev_ctx,
|
||||
const TensorArray& ids_in,
|
||||
const TensorArray& scores_in,
|
||||
int beam_size,
|
||||
int end_id,
|
||||
DenseTensor* sentence_ids,
|
||||
DenseTensor* sentence_scores) {
|
||||
const TensorArray* ids = &ids_in;
|
||||
const TensorArray* scores = &scores_in;
|
||||
const size_t step_num = ids->size();
|
||||
PADDLE_ENFORCE_GT(
|
||||
step_num,
|
||||
0UL,
|
||||
common::errors::InvalidArgument(
|
||||
"beam search steps, which is the "
|
||||
"size of Input(Ids) TensorArray. beam search steps should "
|
||||
"be larger than 0, but received %d. ",
|
||||
step_num));
|
||||
const size_t source_num = ids->at(0).lod().at(0).size() - 1;
|
||||
PADDLE_ENFORCE_GT(
|
||||
source_num,
|
||||
0UL,
|
||||
common::errors::InvalidArgument(
|
||||
"source_num is the sequence number of the "
|
||||
"first decoding step, indicating by Input(Ids)[0].lod[0].size. "
|
||||
"The number of source_num should be larger than "
|
||||
"0, but received %d. ",
|
||||
source_num));
|
||||
|
||||
for (size_t i = 0; i < step_num; ++i) {
|
||||
size_t tmp = ids->at(i).lod().size();
|
||||
PADDLE_ENFORCE_EQ(
|
||||
tmp,
|
||||
2UL,
|
||||
common::errors::InvalidArgument(
|
||||
"For the i step in beam search steps,"
|
||||
"the size of Input(Ids)[i].lod() should larger than 2,"
|
||||
"but received %d. ",
|
||||
tmp));
|
||||
}
|
||||
|
||||
// prepare output
|
||||
DenseTensor* sentenceIds = sentence_ids;
|
||||
DenseTensor* sentenceScores = sentence_scores;
|
||||
BeamSearchDecodeFunctor bs(
|
||||
*ids, *scores, sentenceIds, sentenceScores, beam_size, end_id);
|
||||
bs.apply_mix<T>();
|
||||
}
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,74 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/funcs/math/beam_search.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void BeamSearchOpKernel(const Context &dev_ctx,
|
||||
const DenseTensor &pre_ids_in,
|
||||
const DenseTensor &pre_scores_in,
|
||||
const DenseTensor &ids_in,
|
||||
const DenseTensor &scores_in,
|
||||
int level,
|
||||
int beam_size,
|
||||
int end_id,
|
||||
bool is_accumulated,
|
||||
DenseTensor *selected_ids,
|
||||
DenseTensor *selected_scores,
|
||||
DenseTensor *parent_idx) {
|
||||
auto *ids = &ids_in;
|
||||
auto *scores = &scores_in;
|
||||
auto *pre_ids = &pre_ids_in;
|
||||
auto *pre_scores = &pre_scores_in;
|
||||
|
||||
PADDLE_ENFORCE_NOT_NULL(
|
||||
scores,
|
||||
common::errors::NotFound("Input(scores) of BeamSearchOp is not found."));
|
||||
PADDLE_ENFORCE_NOT_NULL(
|
||||
pre_ids,
|
||||
common::errors::NotFound("Input(pre_ids) of BeamSearchOp is not found."));
|
||||
PADDLE_ENFORCE_NOT_NULL(
|
||||
pre_scores,
|
||||
common::errors::NotFound(
|
||||
"Input(pre_scores) of BeamSearchOp is not found."));
|
||||
|
||||
PADDLE_ENFORCE_NOT_NULL(
|
||||
selected_ids,
|
||||
common::errors::NotFound(
|
||||
"Output(selected_ids) of BeamSearchOp is not found."));
|
||||
PADDLE_ENFORCE_NOT_NULL(
|
||||
selected_scores,
|
||||
common::errors::NotFound(
|
||||
"Output(selected_scores) of BeamSearchOp is not found."));
|
||||
|
||||
math::BeamSearchFunctor<Context, T> alg;
|
||||
alg(dev_ctx,
|
||||
pre_ids,
|
||||
pre_scores,
|
||||
ids,
|
||||
scores,
|
||||
selected_ids,
|
||||
selected_scores,
|
||||
parent_idx,
|
||||
level,
|
||||
beam_size,
|
||||
end_id,
|
||||
is_accumulated);
|
||||
}
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,166 @@
|
||||
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/backends/gpu/gpu_context.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/kernels/impl/bessel_kernel_cuda_impl.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T>
|
||||
struct CudaI0GradFunctor {
|
||||
__device__ __forceinline__ T operator()(const T _x, const T _out_grad) const {
|
||||
using MT = typename MPTypeTrait<T>::Type;
|
||||
const MT mp_x = static_cast<MT>(_x);
|
||||
const MT mp_out_grad = static_cast<MT>(_out_grad);
|
||||
// get output of i1
|
||||
MT x = std::abs(mp_x);
|
||||
if (x <= MT{8.0}) {
|
||||
auto coeff_pair_A = ChebyshevCoefficientsI1e_A<MT>();
|
||||
auto A = std::get<0>(coeff_pair_A);
|
||||
auto len = std::get<1>(coeff_pair_A);
|
||||
MT y = (x / MT{2.0}) - MT{2.0};
|
||||
|
||||
const MT i1_out = std::exp(x) * x * Chbevl<MT>(y, A, len);
|
||||
const MT i1_data = (mp_x < MT{0.0}) ? -i1_out : i1_out;
|
||||
// calculate i0 gradient
|
||||
return static_cast<T>(i1_data * mp_out_grad);
|
||||
}
|
||||
auto coeff_pair_B = ChebyshevCoefficientsI1e_B<MT>();
|
||||
auto B = std::get<0>(coeff_pair_B);
|
||||
auto len = std::get<1>(coeff_pair_B);
|
||||
MT y = (MT{32.0} / x) - MT{2.0};
|
||||
|
||||
const MT i1_out = (std::exp(x) * Chbevl<MT>(y, B, len)) / std::sqrt(x);
|
||||
const MT i1_data = (mp_x < MT{0.0}) ? -i1_out : i1_out;
|
||||
|
||||
return static_cast<T>(i1_data * mp_out_grad);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct CudaI0eGradFunctor {
|
||||
__device__ __forceinline__ T operator()(const T _x,
|
||||
const T _out,
|
||||
const T _out_grad) const {
|
||||
using MT = typename MPTypeTrait<T>::Type;
|
||||
const MT mp_x = static_cast<MT>(_x);
|
||||
const MT mp_out = static_cast<MT>(_out);
|
||||
const MT mp_out_grad = static_cast<MT>(_out_grad);
|
||||
// get output of i1e
|
||||
MT x = std::abs(mp_x);
|
||||
if (x <= MT{8.0}) {
|
||||
auto coeff_pair_A = ChebyshevCoefficientsI1e_A<MT>();
|
||||
auto A = std::get<0>(coeff_pair_A);
|
||||
auto len = std::get<1>(coeff_pair_A);
|
||||
MT y = (x / MT{2.0}) - MT{2.0};
|
||||
|
||||
const MT i1e_out = Chbevl<MT>(y, A, len) * x;
|
||||
const MT i1e_data = (mp_x < MT{0.0}) ? -i1e_out : i1e_out;
|
||||
// calculate i0e gradient
|
||||
return static_cast<T>((i1e_data - std::copysign(MT{1.0}, mp_x) * mp_out) *
|
||||
mp_out_grad);
|
||||
}
|
||||
auto coeff_pair_B = ChebyshevCoefficientsI1e_B<MT>();
|
||||
auto B = std::get<0>(coeff_pair_B);
|
||||
auto len = std::get<1>(coeff_pair_B);
|
||||
MT y = (MT{32.0} / x) - MT{2.0};
|
||||
|
||||
const MT i1e_out = Chbevl<MT>(y, B, len) / std::sqrt(x);
|
||||
const MT i1e_data = (mp_x < MT{0.0}) ? -i1e_out : i1e_out;
|
||||
|
||||
return static_cast<T>((i1e_data - std::copysign(MT{1.0}, mp_x) * mp_out) *
|
||||
mp_out_grad);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct CudaI1GradFunctor {
|
||||
__device__ __forceinline__ T operator()(const T _x,
|
||||
const T _out,
|
||||
const T _out_grad) const {
|
||||
using MT = typename MPTypeTrait<T>::Type;
|
||||
const MT mp_x = static_cast<MT>(_x);
|
||||
const MT mp_out = static_cast<MT>(_out);
|
||||
const MT mp_out_grad = static_cast<MT>(_out_grad);
|
||||
MT x = std::abs(mp_x);
|
||||
if (x <= MT{8.0}) {
|
||||
auto coeff_pair_A = ChebyshevCoefficientsI0e_A<MT>();
|
||||
auto A = std::get<0>(coeff_pair_A);
|
||||
auto len = std::get<1>(coeff_pair_A);
|
||||
MT y = (x / MT{2.0}) - MT{2.0};
|
||||
MT eps = static_cast<MT>(std::numeric_limits<T>::epsilon());
|
||||
|
||||
if (x <= eps) {
|
||||
MT out = (MT{0.5}) * mp_out_grad;
|
||||
return static_cast<T>(out);
|
||||
} else {
|
||||
return static_cast<T>(
|
||||
(std::exp(x) * Chbevl<MT>(y, A, len) - mp_out / mp_x) *
|
||||
mp_out_grad);
|
||||
}
|
||||
}
|
||||
auto coeff_pair_B = ChebyshevCoefficientsI0e_B<MT>();
|
||||
auto B = std::get<0>(coeff_pair_B);
|
||||
auto len = std::get<1>(coeff_pair_B);
|
||||
MT y = (MT{32.0} / x) - MT{2.0};
|
||||
|
||||
return static_cast<T>(
|
||||
(std::exp(x) * Chbevl<MT>(y, B, len) / std::sqrt(x) - mp_out / mp_x) *
|
||||
mp_out_grad);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct CudaI1eGradFunctor {
|
||||
__device__ __forceinline__ T operator()(const T _x,
|
||||
const T _out,
|
||||
const T _out_grad) const {
|
||||
using MT = typename MPTypeTrait<T>::Type;
|
||||
const MT mp_x = static_cast<MT>(_x);
|
||||
const MT mp_out = static_cast<MT>(_out);
|
||||
const MT mp_out_grad = static_cast<MT>(_out_grad);
|
||||
MT x = std::abs(mp_x);
|
||||
if (x <= MT{8.0}) {
|
||||
auto coeff_pair_A = ChebyshevCoefficientsI0e_A<MT>();
|
||||
auto A = std::get<0>(coeff_pair_A);
|
||||
auto len = std::get<1>(coeff_pair_A);
|
||||
MT y = (x / MT{2.0}) - MT{2.0};
|
||||
MT eps = static_cast<MT>(std::numeric_limits<T>::epsilon());
|
||||
|
||||
if (x <= eps) {
|
||||
MT out = (MT{0.5}) * mp_out_grad;
|
||||
return static_cast<T>(out);
|
||||
} else {
|
||||
MT out = (Chbevl<MT>(y, A, len) -
|
||||
mp_out * (std::copysign(MT{1.0}, mp_x) + (MT{1.0}) / mp_x)) *
|
||||
mp_out_grad;
|
||||
return static_cast<T>(out);
|
||||
}
|
||||
}
|
||||
auto coeff_pair_B = ChebyshevCoefficientsI0e_B<MT>();
|
||||
auto B = std::get<0>(coeff_pair_B);
|
||||
auto len = std::get<1>(coeff_pair_B);
|
||||
MT y = (MT{32.0} / x) - MT{2.0};
|
||||
|
||||
return static_cast<T>(
|
||||
(Chbevl<T>(y, B, len) / std::sqrt(x) -
|
||||
mp_out * (std::copysign(MT{1.0}, mp_x) + (MT{1.0}) / mp_x)) *
|
||||
mp_out_grad);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,211 @@
|
||||
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/backends/all_context.h"
|
||||
#include "paddle/phi/common/amp_type_traits.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
#include "paddle/phi/kernels/impl/bessel_kernel_impl.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T>
|
||||
struct I0GradFunctor {
|
||||
I0GradFunctor(const T* x, const T* out_grad, T* x_grad, int64_t numel)
|
||||
: inp_x_(x),
|
||||
inp_out_grad_(out_grad),
|
||||
output_x_grad_(x_grad),
|
||||
numel_(numel) {}
|
||||
|
||||
HOSTDEVICE void operator()(int64_t idx) const {
|
||||
using MT = typename MPTypeTrait<T>::Type;
|
||||
const MT mp_x = static_cast<MT>(inp_x_[idx]);
|
||||
const MT mp_out_grad = static_cast<MT>(inp_out_grad_[idx]);
|
||||
|
||||
MT x = std::abs(mp_x);
|
||||
if (x <= T{8.0}) {
|
||||
auto coeff_pair_A = ChebyshevCoefficientsI1e_A<MT>();
|
||||
auto A = std::get<0>(coeff_pair_A);
|
||||
auto len = std::get<1>(coeff_pair_A);
|
||||
MT y = (x / MT{2.0}) - MT{2.0};
|
||||
|
||||
const MT i1_out = std::exp(x) * x * Chbevl<MT>(y, A, len);
|
||||
const MT i1_data = (mp_x < T{0.0}) ? -i1_out : i1_out;
|
||||
output_x_grad_[idx] = static_cast<T>(i1_data * mp_out_grad);
|
||||
} else {
|
||||
auto coeff_pair_B = ChebyshevCoefficientsI1e_B<MT>();
|
||||
auto B = std::get<0>(coeff_pair_B);
|
||||
auto len = std::get<1>(coeff_pair_B);
|
||||
MT y = (MT{32.0} / x) - MT{2.0};
|
||||
|
||||
const MT i1_out = (std::exp(x) * Chbevl<MT>(y, B, len)) / std::sqrt(x);
|
||||
const MT i1_data = (mp_x < MT{0.0}) ? -i1_out : i1_out;
|
||||
output_x_grad_[idx] = static_cast<T>(i1_data * mp_out_grad);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
const T* inp_x_;
|
||||
const T* inp_out_grad_;
|
||||
T* output_x_grad_;
|
||||
int64_t numel_;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct I0eGradFunctor {
|
||||
I0eGradFunctor(
|
||||
const T* x, const T* out, const T* out_grad, T* x_grad, int64_t numel)
|
||||
: inp_x_(x),
|
||||
inp_out_(out),
|
||||
inp_out_grad_(out_grad),
|
||||
output_x_grad_(x_grad),
|
||||
numel_(numel) {}
|
||||
|
||||
HOSTDEVICE void operator()(int64_t idx) const {
|
||||
T x = std::abs(inp_x_[idx]);
|
||||
if (x <= T{8.0}) {
|
||||
auto coeff_pair_A = ChebyshevCoefficientsI1e_A<T>();
|
||||
auto A = std::get<0>(coeff_pair_A);
|
||||
auto len = std::get<1>(coeff_pair_A);
|
||||
T y = (x / T{2.0}) - T{2.0};
|
||||
|
||||
const T out = Chbevl<T>(y, A, len) * x;
|
||||
const T i1e_out = (inp_x_[idx] < T{0.0}) ? -out : out;
|
||||
output_x_grad_[idx] =
|
||||
(i1e_out - std::copysign(T{1.0}, inp_x_[idx]) * inp_out_[idx]) *
|
||||
inp_out_grad_[idx];
|
||||
} else {
|
||||
auto coeff_pair_B = ChebyshevCoefficientsI1e_B<T>();
|
||||
auto B = std::get<0>(coeff_pair_B);
|
||||
auto len = std::get<1>(coeff_pair_B);
|
||||
T y = (T{32.0} / x) - T{2.0};
|
||||
|
||||
const T out = Chbevl<T>(y, B, len) / std::sqrt(x);
|
||||
const T i1e_out = (inp_x_[idx] < T{0.0}) ? -out : out;
|
||||
output_x_grad_[idx] =
|
||||
(i1e_out - std::copysign(T{1.0}, inp_x_[idx]) * inp_out_[idx]) *
|
||||
inp_out_grad_[idx];
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
const T* inp_x_;
|
||||
const T* inp_out_;
|
||||
const T* inp_out_grad_;
|
||||
T* output_x_grad_;
|
||||
int64_t numel_;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct I1GradFunctor {
|
||||
I1GradFunctor(
|
||||
const T* x, const T* out, const T* out_grad, T* x_grad, int64_t numel)
|
||||
: input_x_(x),
|
||||
input_out_(out),
|
||||
input_out_grad_(out_grad),
|
||||
output_x_grad_(x_grad),
|
||||
numel_(numel) {}
|
||||
|
||||
HOSTDEVICE void operator()(int64_t idx) const {
|
||||
T x = std::abs(input_x_[idx]);
|
||||
T x_ = input_x_[idx];
|
||||
T out_ = input_out_[idx];
|
||||
T out_grad_ = input_out_grad_[idx];
|
||||
if (x <= T{8.0}) {
|
||||
auto coeff_pair_A = ChebyshevCoefficientsI0e_A<T>();
|
||||
auto A = std::get<0>(coeff_pair_A);
|
||||
auto len = std::get<1>(coeff_pair_A);
|
||||
T y = (x / T{2.0}) - T{2.0};
|
||||
T eps = std::numeric_limits<T>::epsilon();
|
||||
|
||||
if (x <= eps) {
|
||||
output_x_grad_[idx] = static_cast<T>(T{0.5} * out_grad_);
|
||||
} else {
|
||||
output_x_grad_[idx] = static_cast<T>(
|
||||
(std::exp(x) * Chbevl<T>(y, A, len) - out_ / x_) * out_grad_);
|
||||
}
|
||||
} else {
|
||||
auto coeff_pair_B = ChebyshevCoefficientsI0e_B<T>();
|
||||
auto B = std::get<0>(coeff_pair_B);
|
||||
auto len = std::get<1>(coeff_pair_B);
|
||||
T y = (T{32.0} / x) - T{2.0};
|
||||
|
||||
output_x_grad_[idx] = static_cast<T>(
|
||||
(std::exp(x) * Chbevl<T>(y, B, len) / std::sqrt(x) - out_ / x_) *
|
||||
out_grad_);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
const T* input_x_;
|
||||
const T* input_out_;
|
||||
const T* input_out_grad_;
|
||||
T* output_x_grad_;
|
||||
int64_t numel_;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct I1eGradFunctor {
|
||||
I1eGradFunctor(
|
||||
const T* x, const T* out, const T* out_grad, T* x_grad, int64_t numel)
|
||||
: input_x_(x),
|
||||
input_out_(out),
|
||||
input_out_grad_(out_grad),
|
||||
output_x_grad_(x_grad),
|
||||
numel_(numel) {}
|
||||
|
||||
HOSTDEVICE void operator()(int64_t idx) const {
|
||||
T x = std::abs(input_x_[idx]);
|
||||
T x_ = input_x_[idx];
|
||||
T out_ = input_out_[idx];
|
||||
T out_grad_ = input_out_grad_[idx];
|
||||
if (x <= T{8.0}) {
|
||||
auto coeff_pair_A = ChebyshevCoefficientsI0e_A<T>();
|
||||
auto A = std::get<0>(coeff_pair_A);
|
||||
auto len = std::get<1>(coeff_pair_A);
|
||||
T y = (x / T{2.0}) - T{2.0};
|
||||
T eps = std::numeric_limits<T>::epsilon();
|
||||
|
||||
if (x <= eps) {
|
||||
output_x_grad_[idx] = static_cast<T>(T{0.5} * out_grad_);
|
||||
} else {
|
||||
output_x_grad_[idx] =
|
||||
static_cast<T>((Chbevl<T>(y, A, len) -
|
||||
out_ * (std::copysign(T{1.0}, x_) + T{1.0} / x_)) *
|
||||
out_grad_);
|
||||
}
|
||||
} else {
|
||||
auto coeff_pair_B = ChebyshevCoefficientsI0e_B<T>();
|
||||
auto B = std::get<0>(coeff_pair_B);
|
||||
auto len = std::get<1>(coeff_pair_B);
|
||||
T y = (T{32.0} / x) - T{2.0};
|
||||
|
||||
output_x_grad_[idx] =
|
||||
static_cast<T>((Chbevl<T>(y, B, len) / std::sqrt(x) -
|
||||
out_ * (std::copysign(T{1.0}, x_) + T{1.0} / x_)) *
|
||||
out_grad_);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
const T* input_x_;
|
||||
const T* input_out_;
|
||||
const T* input_out_grad_;
|
||||
T* output_x_grad_;
|
||||
int64_t numel_;
|
||||
};
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,279 @@
|
||||
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/backends/gpu/gpu_context.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T>
|
||||
__host__ __device__ std::tuple<const T*, size_t> ChebyshevCoefficientsI0e_A() {
|
||||
/* Chebyshev coefficients for I0e(x) in the interval [0,8]. */
|
||||
static const T coeff[] = {
|
||||
-4.41534164647933937950E-18, 3.33079451882223809783E-17,
|
||||
-2.43127984654795469359E-16, 1.71539128555513303061E-15,
|
||||
-1.16853328779934516808E-14, 7.67618549860493561688E-14,
|
||||
-4.85644678311192946090E-13, 2.95505266312963983461E-12,
|
||||
-1.72682629144155570723E-11, 9.67580903537323691224E-11,
|
||||
-5.18979560163526290666E-10, 2.65982372468238665035E-9,
|
||||
-1.30002500998624804212E-8, 6.04699502254191894932E-8,
|
||||
-2.67079385394061173391E-7, 1.11738753912010371815E-6,
|
||||
-4.41673835845875056359E-6, 1.64484480707288970893E-5,
|
||||
-5.75419501008210370398E-5, 1.88502885095841655729E-4,
|
||||
-5.76375574538582365885E-4, 1.63947561694133579842E-3,
|
||||
-4.32430999505057594430E-3, 1.05464603945949983183E-2,
|
||||
-2.37374148058994688156E-2, 4.93052842396707084878E-2,
|
||||
-9.49010970480476444210E-2, 1.71620901522208775349E-1,
|
||||
-3.04682672343198398683E-1, 6.76795274409476084995E-1};
|
||||
return std::make_tuple(coeff, 30);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ __device__ std::tuple<const T*, size_t> ChebyshevCoefficientsI0e_B() {
|
||||
/* Chebyshev coefficients for I0e(x) in the inverted interval [8,infinity]. */
|
||||
static const T coeff[] = {
|
||||
-7.23318048787475395456E-18, -4.83050448594418207126E-18,
|
||||
4.46562142029675999901E-17, 3.46122286769746109310E-17,
|
||||
-2.82762398051658348494E-16, -3.42548561967721913462E-16,
|
||||
1.77256013305652638360E-15, 3.81168066935262242075E-15,
|
||||
-9.55484669882830764870E-15, -4.15056934728722208663E-14,
|
||||
1.54008621752140982691E-14, 3.85277838274214270114E-13,
|
||||
7.18012445138366623367E-13, -1.79417853150680611778E-12,
|
||||
-1.32158118404477131188E-11, -3.14991652796324136454E-11,
|
||||
1.18891471078464383424E-11, 4.94060238822496958910E-10,
|
||||
3.39623202570838634515E-9, 2.26666899049817806459E-8,
|
||||
2.04891858946906374183E-7, 2.89137052083475648297E-6,
|
||||
6.88975834691682398426E-5, 3.36911647825569408990E-3,
|
||||
8.04490411014108831608E-1};
|
||||
|
||||
return std::make_tuple(coeff, 25);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ __device__ T Chbevl(T x, const T array[], size_t len) {
|
||||
T b0, b1, b2;
|
||||
|
||||
b0 = array[0];
|
||||
b1 = static_cast<T>(0.0);
|
||||
|
||||
for (size_t i = 1; i < len; ++i) {
|
||||
b2 = b1;
|
||||
b1 = b0;
|
||||
b0 = x * b1 - b2 + array[i];
|
||||
}
|
||||
|
||||
return (static_cast<T>(0.5) * (b0 - b2));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
struct CudaI0Functor {
|
||||
__device__ __forceinline__ T operator()(const T _x) const {
|
||||
using MT = typename MPTypeTrait<T>::Type;
|
||||
const MT mp_x = static_cast<MT>(_x);
|
||||
MT x = std::abs(mp_x);
|
||||
if (x <= MT{8.0}) {
|
||||
auto coeff_pair_A = ChebyshevCoefficientsI0e_A<MT>();
|
||||
auto A = std::get<0>(coeff_pair_A);
|
||||
auto len = std::get<1>(coeff_pair_A);
|
||||
MT y = (x / MT{2.0}) - MT{2.0};
|
||||
|
||||
return static_cast<T>(std::exp(x) * Chbevl<MT>(y, A, len));
|
||||
}
|
||||
auto coeff_pair_B = ChebyshevCoefficientsI0e_B<MT>();
|
||||
auto B = std::get<0>(coeff_pair_B);
|
||||
auto len = std::get<1>(coeff_pair_B);
|
||||
MT y = (MT{32.0} / x) - MT{2.0};
|
||||
|
||||
return static_cast<T>(std::exp(x) * Chbevl<T>(y, B, len) / std::sqrt(x));
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct CudaI0eFunctor {
|
||||
__device__ __forceinline__ T operator()(const T _x) const {
|
||||
using MT = typename MPTypeTrait<T>::Type;
|
||||
const MT mp_x = static_cast<MT>(_x);
|
||||
MT x = std::abs(mp_x);
|
||||
if (x <= MT{8.0}) {
|
||||
auto coeff_pair_A = ChebyshevCoefficientsI0e_A<MT>();
|
||||
auto A = std::get<0>(coeff_pair_A);
|
||||
auto len = std::get<1>(coeff_pair_A);
|
||||
MT y = (x / MT{2.0}) - MT{2.0};
|
||||
|
||||
return static_cast<T>(Chbevl<MT>(y, A, len));
|
||||
}
|
||||
auto coeff_pair_B = ChebyshevCoefficientsI0e_B<MT>();
|
||||
auto B = std::get<0>(coeff_pair_B);
|
||||
auto len = std::get<1>(coeff_pair_B);
|
||||
MT y = (MT{32.0} / x) - MT{2.0};
|
||||
|
||||
return static_cast<T>(Chbevl<T>(y, B, len) / std::sqrt(x));
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
__host__ __device__ typename std::enable_if<std::is_same<double, T>::value,
|
||||
std::tuple<const T*, size_t>>::type
|
||||
ChebyshevCoefficientsI1e_A() {
|
||||
/* Chebyshev coefficients for exp(-x) I1(x)
|
||||
* in the interval [0,8].
|
||||
*
|
||||
* lim(x->0){ exp(-x) I1(x) / x } = 1/2.
|
||||
*/
|
||||
static const T coeff[] = {
|
||||
2.77791411276104639959E-18, -2.11142121435816608115E-17,
|
||||
1.55363195773620046921E-16, -1.10559694773538630805E-15,
|
||||
7.60068429473540693410E-15, -5.04218550472791168711E-14,
|
||||
3.22379336594557470981E-13, -1.98397439776494371520E-12,
|
||||
1.17361862988909016308E-11, -6.66348972350202774223E-11,
|
||||
3.62559028155211703701E-10, -1.88724975172282928790E-9,
|
||||
9.38153738649577178388E-9, -4.44505912879632808065E-8,
|
||||
2.00329475355213526229E-7, -8.56872026469545474066E-7,
|
||||
3.47025130813767847674E-6, -1.32731636560394358279E-5,
|
||||
4.78156510755005422638E-5, -1.61760815825896745588E-4,
|
||||
5.12285956168575772895E-4, -1.51357245063125314899E-3,
|
||||
4.15642294431288815669E-3, -1.05640848946261981558E-2,
|
||||
2.47264490306265168283E-2, -5.29459812080949914269E-2,
|
||||
1.02643658689847095384E-1, -1.76416518357834055153E-1,
|
||||
2.52587186443633654823E-1};
|
||||
return std::make_tuple(coeff, 29);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ __device__ typename std::enable_if<std::is_same<float, T>::value,
|
||||
std::tuple<const T*, size_t>>::type
|
||||
ChebyshevCoefficientsI1e_A() {
|
||||
/* Chebyshev coefficients for exp(-x) I1(x)
|
||||
* in the interval [0,8].
|
||||
*
|
||||
* lim(x->0){ exp(-x) I1(x) / x } = 1/2.
|
||||
*/
|
||||
static const T coeff[] = {9.38153738649577178388E-9f,
|
||||
-4.44505912879632808065E-8f,
|
||||
2.00329475355213526229E-7f,
|
||||
-8.56872026469545474066E-7f,
|
||||
3.47025130813767847674E-6f,
|
||||
-1.32731636560394358279E-5f,
|
||||
4.78156510755005422638E-5f,
|
||||
-1.61760815825896745588E-4f,
|
||||
5.12285956168575772895E-4f,
|
||||
-1.51357245063125314899E-3f,
|
||||
4.15642294431288815669E-3f,
|
||||
-1.05640848946261981558E-2f,
|
||||
2.47264490306265168283E-2f,
|
||||
-5.29459812080949914269E-2f,
|
||||
1.02643658689847095384E-1f,
|
||||
-1.76416518357834055153E-1f,
|
||||
2.52587186443633654823E-1f};
|
||||
return std::make_tuple(coeff, 17);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ __device__ typename std::enable_if<std::is_same<double, T>::value,
|
||||
std::tuple<const T*, size_t>>::type
|
||||
ChebyshevCoefficientsI1e_B() {
|
||||
/* Chebyshev coefficients for exp(-x) sqrt(x) I1(x)
|
||||
* in the inverted interval [8,infinity].
|
||||
*
|
||||
* lim(x->inf){ exp(-x) sqrt(x) I1(x) } = 1/sqrt(2pi).
|
||||
*/
|
||||
static const T coeff[] = {
|
||||
7.51729631084210481353E-18, 4.41434832307170791151E-18,
|
||||
-4.65030536848935832153E-17, -3.20952592199342395980E-17,
|
||||
2.96262899764595013876E-16, 3.30820231092092828324E-16,
|
||||
-1.88035477551078244854E-15, -3.81440307243700780478E-15,
|
||||
1.04202769841288027642E-14, 4.27244001671195135429E-14,
|
||||
-2.10154184277266431302E-14, -4.08355111109219731823E-13,
|
||||
-7.19855177624590851209E-13, 2.03562854414708950722E-12,
|
||||
1.41258074366137813316E-11, 3.25260358301548823856E-11,
|
||||
-1.89749581235054123450E-11, -5.58974346219658380687E-10,
|
||||
-3.83538038596423702205E-9, -2.63146884688951950684E-8,
|
||||
-2.51223623787020892529E-7, -3.88256480887769039346E-6,
|
||||
-1.10588938762623716291E-4, -9.76109749136146840777E-3,
|
||||
7.78576235018280120474E-1};
|
||||
|
||||
return std::make_tuple(coeff, 25);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ __device__ typename std::enable_if<std::is_same<float, T>::value,
|
||||
std::tuple<const T*, size_t>>::type
|
||||
ChebyshevCoefficientsI1e_B() {
|
||||
/* Chebyshev coefficients for exp(-x) sqrt(x) I1(x)
|
||||
* in the inverted interval [8,infinity].
|
||||
*
|
||||
* lim(x->inf){ exp(-x) sqrt(x) I1(x) } = 1/sqrt(2pi).
|
||||
*/
|
||||
static const T coeff[] = {-3.83538038596423702205E-9f,
|
||||
-2.63146884688951950684E-8f,
|
||||
-2.51223623787020892529E-7f,
|
||||
-3.88256480887769039346E-6f,
|
||||
-1.10588938762623716291E-4f,
|
||||
-9.76109749136146840777E-3f,
|
||||
7.78576235018280120474E-1f};
|
||||
|
||||
return std::make_tuple(coeff, 7);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
struct CudaI1Functor {
|
||||
__device__ __forceinline__ T operator()(const T _x) const {
|
||||
using MT = typename MPTypeTrait<T>::Type;
|
||||
const MT mp_x = static_cast<MT>(_x);
|
||||
MT x = std::abs(mp_x);
|
||||
if (x <= MT{8.0}) {
|
||||
auto coeff_pair_A = ChebyshevCoefficientsI1e_A<MT>();
|
||||
auto A = std::get<0>(coeff_pair_A);
|
||||
auto len = std::get<1>(coeff_pair_A);
|
||||
MT y = (x / MT{2.0}) - MT{2.0};
|
||||
const T out = std::exp(x) * x * Chbevl<MT>(y, A, len);
|
||||
return (mp_x < MT{0.0}) ? -out : out;
|
||||
}
|
||||
auto coeff_pair_B = ChebyshevCoefficientsI1e_B<MT>();
|
||||
auto B = std::get<0>(coeff_pair_B);
|
||||
auto len = std::get<1>(coeff_pair_B);
|
||||
MT y = (MT{32.0} / x) - MT{2.0};
|
||||
const T out = (std::exp(x) * Chbevl<MT>(y, B, len)) / std::sqrt(x);
|
||||
return (mp_x < MT{0.0}) ? -out : out;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct CudaI1eFunctor {
|
||||
__device__ __forceinline__ T operator()(const T _x) const {
|
||||
using MT = typename MPTypeTrait<T>::Type;
|
||||
const MT mp_x = static_cast<MT>(_x);
|
||||
MT x = std::abs(mp_x);
|
||||
if (x <= MT{8.0}) {
|
||||
auto coeff_pair_A = ChebyshevCoefficientsI1e_A<MT>();
|
||||
auto A = std::get<0>(coeff_pair_A);
|
||||
auto len = std::get<1>(coeff_pair_A);
|
||||
MT y = (x / MT{2.0}) - MT{2.0};
|
||||
|
||||
const T out = static_cast<T>(Chbevl<T>(y, A, len) * x);
|
||||
return (mp_x < MT{0.0}) ? -out : out;
|
||||
}
|
||||
auto coeff_pair_B = ChebyshevCoefficientsI1e_B<MT>();
|
||||
auto B = std::get<0>(coeff_pair_B);
|
||||
auto len = std::get<1>(coeff_pair_B);
|
||||
MT y = (MT{32.0} / x) - MT{2.0};
|
||||
|
||||
const T out = static_cast<T>(Chbevl<T>(y, B, len) / std::sqrt(x));
|
||||
return (mp_x < MT{0.0}) ? -out : out;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,315 @@
|
||||
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/backends/all_context.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T>
|
||||
static inline std::tuple<const T*, size_t> ChebyshevCoefficientsI0e_A() {
|
||||
/* Chebyshev coefficients for exp(-x) I0(x)
|
||||
* in the interval [0,8].
|
||||
*
|
||||
* lim(x->0) { exp(-x) I0(x) } = 1.
|
||||
*/
|
||||
static const T coeff[] = {
|
||||
-4.41534164647933937950E-18, 3.33079451882223809783E-17,
|
||||
-2.43127984654795469359E-16, 1.71539128555513303061E-15,
|
||||
-1.16853328779934516808E-14, 7.67618549860493561688E-14,
|
||||
-4.85644678311192946090E-13, 2.95505266312963983461E-12,
|
||||
-1.72682629144155570723E-11, 9.67580903537323691224E-11,
|
||||
-5.18979560163526290666E-10, 2.65982372468238665035E-9,
|
||||
-1.30002500998624804212E-8, 6.04699502254191894932E-8,
|
||||
-2.67079385394061173391E-7, 1.11738753912010371815E-6,
|
||||
-4.41673835845875056359E-6, 1.64484480707288970893E-5,
|
||||
-5.75419501008210370398E-5, 1.88502885095841655729E-4,
|
||||
-5.76375574538582365885E-4, 1.63947561694133579842E-3,
|
||||
-4.32430999505057594430E-3, 1.05464603945949983183E-2,
|
||||
-2.37374148058994688156E-2, 4.93052842396707084878E-2,
|
||||
-9.49010970480476444210E-2, 1.71620901522208775349E-1,
|
||||
-3.04682672343198398683E-1, 6.76795274409476084995E-1};
|
||||
return std::make_tuple(coeff, 30);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static inline std::tuple<const T*, size_t> ChebyshevCoefficientsI0e_B() {
|
||||
/* Chebyshev coefficients for exp(-x) sqrt(x) I0(x)
|
||||
* in the inverted interval [8,infinity].
|
||||
*
|
||||
* lim(x->inf){ exp(-x) sqrt(x) I0(x) } = 1/sqrt(2pi).
|
||||
*/
|
||||
static const T coeff[] = {
|
||||
-7.23318048787475395456E-18, -4.83050448594418207126E-18,
|
||||
4.46562142029675999901E-17, 3.46122286769746109310E-17,
|
||||
-2.82762398051658348494E-16, -3.42548561967721913462E-16,
|
||||
1.77256013305652638360E-15, 3.81168066935262242075E-15,
|
||||
-9.55484669882830764870E-15, -4.15056934728722208663E-14,
|
||||
1.54008621752140982691E-14, 3.85277838274214270114E-13,
|
||||
7.18012445138366623367E-13, -1.79417853150680611778E-12,
|
||||
-1.32158118404477131188E-11, -3.14991652796324136454E-11,
|
||||
1.18891471078464383424E-11, 4.94060238822496958910E-10,
|
||||
3.39623202570838634515E-9, 2.26666899049817806459E-8,
|
||||
2.04891858946906374183E-7, 2.89137052083475648297E-6,
|
||||
6.88975834691682398426E-5, 3.36911647825569408990E-3,
|
||||
8.04490411014108831608E-1};
|
||||
|
||||
return std::make_tuple(coeff, 25);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static inline T Chbevl(T x, const T array[], size_t len) {
|
||||
T b0, b1, b2;
|
||||
|
||||
b0 = array[0];
|
||||
b1 = static_cast<T>(0.0);
|
||||
|
||||
for (size_t i = 1; i < len; ++i) {
|
||||
b2 = b1;
|
||||
b1 = b0;
|
||||
b0 = x * b1 - b2 + array[i];
|
||||
}
|
||||
|
||||
return (static_cast<T>(0.5) * (b0 - b2));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
struct I0eFunctor {
|
||||
I0eFunctor(const T* input, T* output, int64_t numel)
|
||||
: input_(input), output_(output), numel_(numel) {}
|
||||
|
||||
HOSTDEVICE void operator()(int64_t idx) const {
|
||||
T x = std::abs(input_[idx]);
|
||||
if (x <= T{8.0}) {
|
||||
auto coeff_pair_A = ChebyshevCoefficientsI0e_A<T>();
|
||||
auto A = std::get<0>(coeff_pair_A);
|
||||
auto len = std::get<1>(coeff_pair_A);
|
||||
T y = (x / T{2.0}) - T{2.0};
|
||||
|
||||
output_[idx] = static_cast<T>(Chbevl<T>(y, A, len));
|
||||
} else {
|
||||
auto coeff_pair_B = ChebyshevCoefficientsI0e_B<T>();
|
||||
auto B = std::get<0>(coeff_pair_B);
|
||||
auto len = std::get<1>(coeff_pair_B);
|
||||
T y = (T{32.0} / x) - T{2.0};
|
||||
|
||||
output_[idx] = static_cast<T>(Chbevl<T>(y, B, len) / std::sqrt(x));
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
const T* input_;
|
||||
T* output_;
|
||||
int64_t numel_;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct I0Functor {
|
||||
I0Functor(const T* input, T* output, int64_t numel)
|
||||
: input_(input), output_(output), numel_(numel) {}
|
||||
|
||||
HOSTDEVICE void operator()(int64_t idx) const {
|
||||
T x = std::abs(input_[idx]);
|
||||
if (x <= T{8.0}) {
|
||||
auto coeff_pair_A = ChebyshevCoefficientsI0e_A<T>();
|
||||
auto A = std::get<0>(coeff_pair_A);
|
||||
auto len = std::get<1>(coeff_pair_A);
|
||||
T y = (x / T{2.0}) - T{2.0};
|
||||
|
||||
output_[idx] = static_cast<T>(std::exp(x) * Chbevl<T>(y, A, len));
|
||||
} else {
|
||||
auto coeff_pair_B = ChebyshevCoefficientsI0e_B<T>();
|
||||
auto B = std::get<0>(coeff_pair_B);
|
||||
auto len = std::get<1>(coeff_pair_B);
|
||||
T y = (T{32.0} / x) - T{2.0};
|
||||
|
||||
output_[idx] =
|
||||
static_cast<T>(std::exp(x) * Chbevl<T>(y, B, len) / std::sqrt(x));
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
const T* input_;
|
||||
T* output_;
|
||||
int64_t numel_;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
static inline typename std::enable_if<std::is_same<double, T>::value,
|
||||
std::tuple<const T*, size_t>>::type
|
||||
ChebyshevCoefficientsI1e_A() {
|
||||
/* Chebyshev coefficients for exp(-x) I1(x)
|
||||
* in the interval [0,8].
|
||||
*
|
||||
* lim(x->0){ exp(-x) I1(x) / x } = 1/2.
|
||||
*/
|
||||
static const T coeff[] = {
|
||||
2.77791411276104639959E-18, -2.11142121435816608115E-17,
|
||||
1.55363195773620046921E-16, -1.10559694773538630805E-15,
|
||||
7.60068429473540693410E-15, -5.04218550472791168711E-14,
|
||||
3.22379336594557470981E-13, -1.98397439776494371520E-12,
|
||||
1.17361862988909016308E-11, -6.66348972350202774223E-11,
|
||||
3.62559028155211703701E-10, -1.88724975172282928790E-9,
|
||||
9.38153738649577178388E-9, -4.44505912879632808065E-8,
|
||||
2.00329475355213526229E-7, -8.56872026469545474066E-7,
|
||||
3.47025130813767847674E-6, -1.32731636560394358279E-5,
|
||||
4.78156510755005422638E-5, -1.61760815825896745588E-4,
|
||||
5.12285956168575772895E-4, -1.51357245063125314899E-3,
|
||||
4.15642294431288815669E-3, -1.05640848946261981558E-2,
|
||||
2.47264490306265168283E-2, -5.29459812080949914269E-2,
|
||||
1.02643658689847095384E-1, -1.76416518357834055153E-1,
|
||||
2.52587186443633654823E-1};
|
||||
return std::make_tuple(coeff, 29);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static inline typename std::enable_if<std::is_same<float, T>::value,
|
||||
std::tuple<const T*, size_t>>::type
|
||||
ChebyshevCoefficientsI1e_A() {
|
||||
/* Chebyshev coefficients for exp(-x) I1(x)
|
||||
* in the interval [0,8].
|
||||
*
|
||||
* lim(x->0){ exp(-x) I1(x) / x } = 1/2.
|
||||
*/
|
||||
static const T coeff[] = {9.38153738649577178388E-9f,
|
||||
-4.44505912879632808065E-8f,
|
||||
2.00329475355213526229E-7f,
|
||||
-8.56872026469545474066E-7f,
|
||||
3.47025130813767847674E-6f,
|
||||
-1.32731636560394358279E-5f,
|
||||
4.78156510755005422638E-5f,
|
||||
-1.61760815825896745588E-4f,
|
||||
5.12285956168575772895E-4f,
|
||||
-1.51357245063125314899E-3f,
|
||||
4.15642294431288815669E-3f,
|
||||
-1.05640848946261981558E-2f,
|
||||
2.47264490306265168283E-2f,
|
||||
-5.29459812080949914269E-2f,
|
||||
1.02643658689847095384E-1f,
|
||||
-1.76416518357834055153E-1f,
|
||||
2.52587186443633654823E-1f};
|
||||
return std::make_tuple(coeff, 17);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static inline typename std::enable_if<std::is_same<double, T>::value,
|
||||
std::tuple<const T*, size_t>>::type
|
||||
ChebyshevCoefficientsI1e_B() {
|
||||
/* Chebyshev coefficients for exp(-x) sqrt(x) I1(x)
|
||||
* in the inverted interval [8,infinity].
|
||||
*
|
||||
* lim(x->inf){ exp(-x) sqrt(x) I1(x) } = 1/sqrt(2pi).
|
||||
*/
|
||||
static const T coeff[] = {
|
||||
7.51729631084210481353E-18, 4.41434832307170791151E-18,
|
||||
-4.65030536848935832153E-17, -3.20952592199342395980E-17,
|
||||
2.96262899764595013876E-16, 3.30820231092092828324E-16,
|
||||
-1.88035477551078244854E-15, -3.81440307243700780478E-15,
|
||||
1.04202769841288027642E-14, 4.27244001671195135429E-14,
|
||||
-2.10154184277266431302E-14, -4.08355111109219731823E-13,
|
||||
-7.19855177624590851209E-13, 2.03562854414708950722E-12,
|
||||
1.41258074366137813316E-11, 3.25260358301548823856E-11,
|
||||
-1.89749581235054123450E-11, -5.58974346219658380687E-10,
|
||||
-3.83538038596423702205E-9, -2.63146884688951950684E-8,
|
||||
-2.51223623787020892529E-7, -3.88256480887769039346E-6,
|
||||
-1.10588938762623716291E-4, -9.76109749136146840777E-3,
|
||||
7.78576235018280120474E-1};
|
||||
|
||||
return std::make_tuple(coeff, 25);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static inline typename std::enable_if<std::is_same<float, T>::value,
|
||||
std::tuple<const T*, size_t>>::type
|
||||
ChebyshevCoefficientsI1e_B() {
|
||||
/* Chebyshev coefficients for exp(-x) sqrt(x) I1(x)
|
||||
* in the inverted interval [8,infinity].
|
||||
*
|
||||
* lim(x->inf){ exp(-x) sqrt(x) I1(x) } = 1/sqrt(2pi).
|
||||
*/
|
||||
static const T coeff[] = {-3.83538038596423702205E-9f,
|
||||
-2.63146884688951950684E-8f,
|
||||
-2.51223623787020892529E-7f,
|
||||
-3.88256480887769039346E-6f,
|
||||
-1.10588938762623716291E-4f,
|
||||
-9.76109749136146840777E-3f,
|
||||
7.78576235018280120474E-1f};
|
||||
|
||||
return std::make_tuple(coeff, 7);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
struct I1Functor {
|
||||
I1Functor(const T* input, T* output, int64_t numel)
|
||||
: input_(input), output_(output), numel_(numel) {}
|
||||
|
||||
HOSTDEVICE void operator()(int64_t idx) const {
|
||||
T x = std::abs(input_[idx]);
|
||||
if (x <= T{8.0}) {
|
||||
auto coeff_pair_A = ChebyshevCoefficientsI1e_A<T>();
|
||||
auto A = std::get<0>(coeff_pair_A);
|
||||
auto len = std::get<1>(coeff_pair_A);
|
||||
T y = (x / T{2.0}) - T{2.0};
|
||||
const T out = std::exp(x) * x * Chbevl(y, A, len);
|
||||
output_[idx] = (input_[idx] < T{0.0}) ? -out : out;
|
||||
} else {
|
||||
auto coeff_pair_B = ChebyshevCoefficientsI1e_B<T>();
|
||||
auto B = std::get<0>(coeff_pair_B);
|
||||
auto len = std::get<1>(coeff_pair_B);
|
||||
T y = (T{32.0} / x) - T{2.0};
|
||||
const T out = (std::exp(x) * Chbevl(y, B, len)) / std::sqrt(x);
|
||||
output_[idx] = (input_[idx] < T{0.0}) ? -out : out;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
const T* input_;
|
||||
T* output_;
|
||||
int64_t numel_;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct I1eFunctor {
|
||||
I1eFunctor(const T* input, T* output, int64_t numel)
|
||||
: input_(input), output_(output), numel_(numel) {}
|
||||
|
||||
HOSTDEVICE void operator()(int64_t idx) const {
|
||||
T x = std::abs(input_[idx]);
|
||||
if (x <= T{8.0}) {
|
||||
auto coeff_pair_A = ChebyshevCoefficientsI1e_A<T>();
|
||||
auto A = std::get<0>(coeff_pair_A);
|
||||
auto len = std::get<1>(coeff_pair_A);
|
||||
T y = (x / T{2.0}) - T{2.0};
|
||||
const T out = Chbevl<T>(y, A, len) * x;
|
||||
output_[idx] = (input_[idx] < T{0.0}) ? -out : out;
|
||||
} else {
|
||||
auto coeff_pair_B = ChebyshevCoefficientsI1e_B<T>();
|
||||
auto B = std::get<0>(coeff_pair_B);
|
||||
auto len = std::get<1>(coeff_pair_B);
|
||||
T y = (T{32.0} / x) - T{2.0};
|
||||
|
||||
const T out = Chbevl<T>(y, B, len) / std::sqrt(x);
|
||||
output_[idx] = (input_[idx] < T{0.0}) ? -out : out;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
const T* input_;
|
||||
T* output_;
|
||||
int64_t numel_;
|
||||
};
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,143 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/funcs/blas/blas.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void BilinearGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& y,
|
||||
const DenseTensor& weight,
|
||||
const DenseTensor& dout,
|
||||
DenseTensor* dx,
|
||||
DenseTensor* dy,
|
||||
DenseTensor* dweight,
|
||||
DenseTensor* dbias) {
|
||||
auto batch_size = x.dims()[0];
|
||||
auto weight_dims = weight.dims();
|
||||
int out_dim = weight_dims[0];
|
||||
auto x_dim = weight_dims[1];
|
||||
auto y_dim = weight_dims[2];
|
||||
|
||||
auto x_mat = EigenMatrix<T>::From(x);
|
||||
auto y_mat = EigenMatrix<T>::From(y);
|
||||
auto dout_mat = EigenMatrix<T>::From(dout);
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
// Create the intermediate variable to calculate the Output(Y@GRAD).
|
||||
DenseTensor x_scale;
|
||||
x_scale.Resize({batch_size, x_dim});
|
||||
dev_ctx.template Alloc<T>(&x_scale);
|
||||
auto x_scale_mat = EigenMatrix<T>::From(x_scale);
|
||||
|
||||
// Create the intermediate variable to calculate the Output(X@GRAD).
|
||||
DenseTensor y_scale;
|
||||
y_scale.Resize({batch_size, y_dim});
|
||||
dev_ctx.template Alloc<T>(&y_scale);
|
||||
auto y_scale_mat = EigenMatrix<T>::From(y_scale);
|
||||
|
||||
funcs::SetConstant<Context, T> set_zero;
|
||||
|
||||
if (dx) {
|
||||
dev_ctx.template Alloc<T>(dx);
|
||||
set_zero(dev_ctx, dx, static_cast<T>(0));
|
||||
}
|
||||
|
||||
if (dy) {
|
||||
dev_ctx.template Alloc<T>(dy);
|
||||
set_zero(dev_ctx, dy, static_cast<T>(0));
|
||||
}
|
||||
|
||||
if (dweight) {
|
||||
dev_ctx.template Alloc<T>(dweight);
|
||||
}
|
||||
|
||||
auto blas = funcs::GetBlas<Context, T>(dev_ctx);
|
||||
|
||||
// Calculate the Output(X@GRAD) and Output(Y@GRAD).
|
||||
if (dx || dy || dweight) {
|
||||
Eigen::DSizes<int, 2> bcast_for_x(1, y_dim);
|
||||
Eigen::DSizes<int, 2> bcast_for_y(1, x_dim);
|
||||
Eigen::DSizes<int, 2> bcast_for_weight(1, x_dim);
|
||||
|
||||
for (int i = 0; i < out_dim; ++i) {
|
||||
DenseTensor weight_i = weight.Slice(i, i + 1).Resize({x_dim, y_dim});
|
||||
auto output_vec = dout_mat.chip(i, 1);
|
||||
|
||||
if (dx) {
|
||||
y_scale_mat.device(place) =
|
||||
output_vec.reshape(Eigen::DSizes<int, 2>(batch_size, 1))
|
||||
.broadcast(bcast_for_x) *
|
||||
y_mat;
|
||||
blas.GEMM(CblasNoTrans,
|
||||
CblasTrans,
|
||||
batch_size,
|
||||
x_dim,
|
||||
y_dim,
|
||||
1,
|
||||
y_scale.data<T>(),
|
||||
weight_i.data<T>(),
|
||||
1,
|
||||
dx->data<T>());
|
||||
}
|
||||
|
||||
if (dy || dweight) {
|
||||
auto output_vec_y =
|
||||
output_vec.reshape(Eigen::DSizes<int, 2>(batch_size, 1))
|
||||
.broadcast(bcast_for_y);
|
||||
x_scale_mat.device(place) = output_vec_y * x_mat;
|
||||
if (dy) {
|
||||
blas.GEMM(CblasNoTrans,
|
||||
CblasNoTrans,
|
||||
batch_size,
|
||||
y_dim,
|
||||
x_dim,
|
||||
1,
|
||||
x_scale.data<T>(),
|
||||
weight_i.data<T>(),
|
||||
1,
|
||||
dy->data<T>());
|
||||
}
|
||||
if (dweight) {
|
||||
DenseTensor dweight_i =
|
||||
dweight->Slice(i, i + 1).Resize({x_dim, y_dim});
|
||||
blas.GEMM(CblasTrans,
|
||||
CblasNoTrans,
|
||||
x_dim,
|
||||
y_dim,
|
||||
batch_size,
|
||||
1,
|
||||
x_scale.data<T>(),
|
||||
y.data<T>(),
|
||||
0,
|
||||
dweight_i.data<T>());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// calculate the gradient of Input(Bias).
|
||||
if (dbias) {
|
||||
dev_ctx.template Alloc<T>(dbias);
|
||||
auto dbias_mat = EigenVector<T>::Flatten(*dbias);
|
||||
dbias_mat.device(place) = dout_mat.sum(Eigen::DSizes<int, 1>(0));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,74 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/funcs/blas/blas.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/utils/optional.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void BilinearKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& y,
|
||||
const DenseTensor& weight,
|
||||
const optional<DenseTensor>& bias,
|
||||
DenseTensor* out) {
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
|
||||
auto y_mat = EigenMatrix<T>::From(y);
|
||||
auto output_mat = EigenMatrix<T>::From(*out);
|
||||
|
||||
auto batch_size = x.dims()[0];
|
||||
auto weight_dims = weight.dims();
|
||||
int64_t out_dim = weight_dims[0];
|
||||
auto x_dim = weight_dims[1];
|
||||
auto y_dim = weight_dims[2];
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
|
||||
// Create the intermediate variable to calculate the result of
|
||||
// Input(X) multiplied by Input(Weight_i), the formula is:
|
||||
// left_mul = X Weight_i.
|
||||
DenseTensor left_mul;
|
||||
left_mul.Resize({batch_size, y_dim});
|
||||
dev_ctx.template Alloc<T>(&left_mul);
|
||||
auto left_mul_mat = EigenMatrix<T>::From(left_mul);
|
||||
|
||||
for (int64_t i = 0; i < out_dim; ++i) {
|
||||
auto output_col_vec = output_mat.chip(i, 1);
|
||||
DenseTensor weight_mat = weight.Slice(i, i + 1).Resize({x_dim, y_dim});
|
||||
funcs::GetBlas<Context, T>(dev_ctx).GEMM(CblasNoTrans,
|
||||
CblasNoTrans,
|
||||
batch_size,
|
||||
y_dim,
|
||||
x_dim,
|
||||
1,
|
||||
x.data<T>(),
|
||||
weight_mat.data<T>(),
|
||||
0,
|
||||
left_mul.data<T>());
|
||||
output_col_vec.device(place) =
|
||||
(left_mul_mat * y_mat).sum(Eigen::DSizes<int, 1>(1));
|
||||
}
|
||||
if (bias.get_ptr()) {
|
||||
auto bias_vec = EigenMatrix<T>::From(*(bias.get_ptr()));
|
||||
Eigen::DSizes<int, 2> bcast(batch_size, 1);
|
||||
output_mat.device(place) = bias_vec.broadcast(bcast) + output_mat;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,106 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/bmm_grad_kernel.h"
|
||||
|
||||
#include "paddle/phi/kernels/funcs/blas/blas.h"
|
||||
#include "paddle/phi/kernels/impl/matmul_grad_kernel_impl.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void MatMul(const Context& dev_ctx,
|
||||
const DenseTensor& a,
|
||||
bool trans_a,
|
||||
const DenseTensor& b,
|
||||
bool trans_b,
|
||||
DenseTensor* out) {
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
auto blas = funcs::GetBlas<Context, T>(dev_ctx);
|
||||
auto mat_dim_a = funcs::CreateMatrixDescriptor(a.dims(), 0, trans_a);
|
||||
auto mat_dim_b = funcs::CreateMatrixDescriptor(b.dims(), 0, trans_b);
|
||||
|
||||
blas.MatMul(a, mat_dim_a, b, mat_dim_b, T(1), out, T(0));
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void CalcInputGrad(const Context& dev_ctx,
|
||||
const DenseTensor& a,
|
||||
bool trans_a,
|
||||
const DenseTensor& b,
|
||||
bool trans_b,
|
||||
DenseTensor* out) {
|
||||
if (out == nullptr) return;
|
||||
MatMul<T, Context>(dev_ctx, a, trans_a, b, trans_b, out);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void BmmGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& y,
|
||||
const DenseTensor& out_grad,
|
||||
DenseTensor* x_grad,
|
||||
DenseTensor* y_grad) {
|
||||
if (x_grad && x_grad->numel() == 0) {
|
||||
dev_ctx.template Alloc<T>(x_grad);
|
||||
Full<T, Context>(dev_ctx, y.dims(), 0, y_grad);
|
||||
return;
|
||||
}
|
||||
if (y_grad && y_grad->numel() == 0) {
|
||||
dev_ctx.template Alloc<T>(y_grad);
|
||||
Full<T, Context>(dev_ctx, x.dims(), 0, x_grad);
|
||||
return;
|
||||
}
|
||||
DenseTensor x_help = x;
|
||||
DenseTensor y_help = y;
|
||||
DenseTensor out_grad_help = out_grad;
|
||||
ReshapeXYOutIntoMatrixSequence(
|
||||
&x_help, &y_help, &out_grad_help, false, false);
|
||||
|
||||
DDim dx_dims;
|
||||
if (x_grad) {
|
||||
dx_dims = x_grad->dims();
|
||||
if (dx_dims != x_help.dims()) {
|
||||
x_grad->Resize(x_help.dims());
|
||||
}
|
||||
}
|
||||
|
||||
DDim dy_dims;
|
||||
if (y_grad) {
|
||||
dy_dims = y_grad->dims();
|
||||
if (dy_dims != y_help.dims()) {
|
||||
y_grad->Resize(y_help.dims());
|
||||
}
|
||||
}
|
||||
|
||||
CalcInputGrad<T, Context>(
|
||||
dev_ctx, out_grad_help, false, y_help, true, x_grad);
|
||||
CalcInputGrad<T, Context>(
|
||||
dev_ctx, x_help, true, out_grad_help, false, y_grad);
|
||||
|
||||
if (x_grad) {
|
||||
if (dx_dims != x_help.dims()) {
|
||||
x_grad->Resize(dx_dims);
|
||||
}
|
||||
}
|
||||
if (y_grad) {
|
||||
if (dy_dims != y_help.dims()) {
|
||||
y_grad->Resize(dy_dims);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,42 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/bmm_kernel.h"
|
||||
|
||||
#include "paddle/phi/kernels/funcs/blas/blas.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void BmmKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& y,
|
||||
DenseTensor* out) {
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
|
||||
if (x.numel() == 0 || y.numel() == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto blas = funcs::GetBlas<Context, T>(dev_ctx);
|
||||
|
||||
auto mat_dim_a = funcs::CreateMatrixDescriptor(x.dims(), 0, false);
|
||||
auto mat_dim_b = funcs::CreateMatrixDescriptor(y.dims(), 0, false);
|
||||
|
||||
blas.MatMul(x, mat_dim_a, y, mat_dim_b, T(1), out, T(0));
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,52 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
#include <string>
|
||||
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/kernels/funcs/detection/bbox_util.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void BoxClipKernel(const Context& dev_ctx,
|
||||
const DenseTensor& input,
|
||||
const DenseTensor& im_info,
|
||||
DenseTensor* output) {
|
||||
auto* input_box = &input;
|
||||
auto* im_info_p = &im_info;
|
||||
auto* output_box = output;
|
||||
dev_ctx.template Alloc<T>(output_box);
|
||||
|
||||
if (input_box->lod().size()) {
|
||||
PADDLE_ENFORCE_EQ(input_box->lod().size(),
|
||||
1UL,
|
||||
common::errors::InvalidArgument(
|
||||
"Input(Input) of BoxClip only supports 1 level "
|
||||
"of LoD. But received the "
|
||||
"level = %d",
|
||||
input_box->lod().size()));
|
||||
}
|
||||
auto box_lod = input_box->lod().back();
|
||||
int64_t n = static_cast<int64_t>(box_lod.size() - 1);
|
||||
for (int i = 0; i < n; ++i) {
|
||||
DenseTensor im_info_slice = im_info_p->Slice(i, i + 1);
|
||||
DenseTensor box_slice = input_box->Slice(box_lod[i], box_lod[i + 1]);
|
||||
DenseTensor output_slice = output_box->Slice(box_lod[i], box_lod[i + 1]);
|
||||
funcs::ClipTiledBoxes<T>(dev_ctx, im_info_slice, box_slice, &output_slice);
|
||||
}
|
||||
}
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,43 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "paddle/phi/core/enforce.h"
|
||||
|
||||
namespace phi {
|
||||
namespace funcs {
|
||||
|
||||
enum class BoxCodeType { kEncodeCenterSize = 0, kDecodeCenterSize = 1 };
|
||||
|
||||
inline BoxCodeType GetBoxCodeType(const std::string &type) {
|
||||
PADDLE_ENFORCE_EQ(
|
||||
(type == "encode_center_size") || (type == "decode_center_size"),
|
||||
true,
|
||||
common::errors::InvalidArgument(
|
||||
"The 'code_type' attribute in BoxCoder"
|
||||
" must be 'encode_center_size' or 'decode_center_size'. "
|
||||
"But received 'code_type' is %s",
|
||||
type));
|
||||
if (type == "encode_center_size") {
|
||||
return BoxCodeType::kEncodeCenterSize;
|
||||
} else {
|
||||
return BoxCodeType::kDecodeCenterSize;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace funcs
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,129 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/core/enforce.h"
|
||||
#include "paddle/phi/core/tensor_utils.h"
|
||||
#include "paddle/phi/kernels/broadcast_tensors_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/eigen_function.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
|
||||
#define SWITCH_OUT_RANK_CASE(n) \
|
||||
case n: { \
|
||||
ApplyBroadcast<T, Context, n>(dev_ctx, in_tensors[i], out_tensors[i]); \
|
||||
break; \
|
||||
}
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context, int OutRank>
|
||||
void ApplyBroadcast(const Context& dev_ctx,
|
||||
const DenseTensor* input_tensor,
|
||||
DenseTensor* output_tensor) {
|
||||
const auto& input_dims = input_tensor->dims();
|
||||
const auto& output_dims = output_tensor->dims();
|
||||
|
||||
int in_rank = input_dims.size();
|
||||
int out_rank = output_dims.size();
|
||||
|
||||
// 1. Collect bcast_dims, each element of which indicates how many
|
||||
// times we need to replicate along the corresponding dimension
|
||||
// 2. Collect new_input_dims_vec. Eigen::broadcast requires same rank for
|
||||
// both input and output tensors, so we need to initialize input X with
|
||||
// expanded dims: "new_input_dims_vec"
|
||||
Eigen::DSizes<int64_t, OutRank> bcast_dims;
|
||||
std::vector<int64_t> new_input_dims_vec(out_rank);
|
||||
for (int i = 0; i < out_rank; i++) {
|
||||
int in_axis = in_rank - i - 1;
|
||||
int out_axis = out_rank - i - 1;
|
||||
|
||||
bcast_dims[out_axis] = output_dims[out_axis];
|
||||
new_input_dims_vec[out_axis] = 1;
|
||||
if (in_axis >= 0 && input_dims[in_axis] == output_dims[out_axis]) {
|
||||
bcast_dims[out_axis] = 1;
|
||||
new_input_dims_vec[out_axis] = input_dims[in_axis];
|
||||
}
|
||||
}
|
||||
auto new_input_dims = make_ddim(new_input_dims_vec);
|
||||
|
||||
// Initialize input X with new_input_dims_vec, so it's rank-aligned with the
|
||||
// output
|
||||
auto x = EigenTensor<T, OutRank>::From(*input_tensor, new_input_dims);
|
||||
|
||||
dev_ctx.template Alloc<T>(output_tensor);
|
||||
auto y = EigenTensor<T, OutRank>::From(*output_tensor, output_dims);
|
||||
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
funcs::EigenBroadcast<std::decay_t<decltype(place)>, T, OutRank>::Eval(
|
||||
place, y, x, bcast_dims);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void BroadcastTensorsKernel(const Context& dev_ctx,
|
||||
const std::vector<const DenseTensor*>& x,
|
||||
std::vector<DenseTensor*> out) {
|
||||
const auto& in_tensors = x;
|
||||
auto out_tensors = out;
|
||||
size_t num_ins = in_tensors.size();
|
||||
|
||||
PADDLE_ENFORCE_GE(
|
||||
num_ins,
|
||||
1,
|
||||
errors::InvalidArgument(
|
||||
"Expected at least 1 input tensor, but only received %d.",
|
||||
in_tensors.size()));
|
||||
|
||||
PADDLE_ENFORCE_EQ(num_ins,
|
||||
out_tensors.size(),
|
||||
errors::InvalidArgument(
|
||||
"BroadcastTensorsOp expects equal number of inputs and "
|
||||
"outputs,but received: %d inputs v.s %d outputs",
|
||||
num_ins,
|
||||
out_tensors.size()));
|
||||
|
||||
// Eigen has no support for dynamic ranked tensor
|
||||
// Thus we perform static expansion for each possible ranks
|
||||
for (size_t i = 0; i < num_ins; i++) {
|
||||
int out_rank = out_tensors[i]->dims().size();
|
||||
switch (out_rank) {
|
||||
case 0: {
|
||||
const DenseTensor* src = in_tensors[i];
|
||||
DenseTensor* dst = out_tensors[i];
|
||||
Copy(dev_ctx, *src, src->place(), false, dst);
|
||||
break;
|
||||
}
|
||||
SWITCH_OUT_RANK_CASE(1)
|
||||
SWITCH_OUT_RANK_CASE(2)
|
||||
SWITCH_OUT_RANK_CASE(3)
|
||||
SWITCH_OUT_RANK_CASE(4)
|
||||
SWITCH_OUT_RANK_CASE(5)
|
||||
SWITCH_OUT_RANK_CASE(6)
|
||||
SWITCH_OUT_RANK_CASE(7)
|
||||
SWITCH_OUT_RANK_CASE(8)
|
||||
SWITCH_OUT_RANK_CASE(9)
|
||||
default: {
|
||||
PADDLE_THROW(common::errors::InvalidArgument(
|
||||
"Target tensor rank out of range. "
|
||||
"Maximum supported rank for broadcast is: 9"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,41 @@
|
||||
// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/c_identity_kernel.h"
|
||||
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void CIdentityKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
int ring_id,
|
||||
bool use_calc_stream,
|
||||
bool use_model_parallel,
|
||||
DenseTensor* out) {
|
||||
PADDLE_ENFORCE_GE(
|
||||
ring_id,
|
||||
0,
|
||||
errors::InvalidArgument(
|
||||
"The ring_id (%d) for c_identity op must be non-negative.", ring_id));
|
||||
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
|
||||
Copy(dev_ctx, x, dev_ctx.GetPlace(), false, out);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,61 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ChannelShuffleGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& out_grad,
|
||||
int groups,
|
||||
const std::string& data_format,
|
||||
DenseTensor* x_grad) {
|
||||
auto* dout = &out_grad;
|
||||
auto* dx = x_grad;
|
||||
dev_ctx.template Alloc<T>(dx);
|
||||
if (dx && dx->numel() == 0) {
|
||||
return;
|
||||
}
|
||||
bool channel_last = (data_format == "NHWC");
|
||||
const auto& do_dims = dout->dims();
|
||||
const auto& dx_dims = dx->dims();
|
||||
|
||||
DenseTensor t(*dout);
|
||||
if (!channel_last) {
|
||||
t.Resize({do_dims[0], do_dims[1] / groups, groups, do_dims[2], do_dims[3]});
|
||||
} else {
|
||||
t.Resize({do_dims[0], do_dims[1], do_dims[2], do_dims[3] / groups, groups});
|
||||
}
|
||||
auto axis = !channel_last ? std::vector<int>{0, 2, 1, 3, 4}
|
||||
: std::vector<int>{0, 1, 2, 4, 3};
|
||||
|
||||
DenseTensor o(*dx);
|
||||
if (!channel_last) {
|
||||
o.Resize({dx_dims[0], groups, dx_dims[1] / groups, dx_dims[2], dx_dims[3]});
|
||||
} else {
|
||||
o.Resize({dx_dims[0], dx_dims[1], dx_dims[2], groups, dx_dims[3] / groups});
|
||||
}
|
||||
funcs::Transpose<Context, T, 5> trans;
|
||||
trans(dev_ctx, t, &o, axis);
|
||||
dx->Resize(dx_dims);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,60 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ChannelShuffleKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
int groups,
|
||||
const std::string& data_format,
|
||||
DenseTensor* out) {
|
||||
auto* in = &x;
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
if (out && out->numel() == 0) {
|
||||
return;
|
||||
}
|
||||
bool channel_last = (data_format == "NHWC");
|
||||
const auto& in_dims = in->dims();
|
||||
const auto& o_dims = out->dims();
|
||||
|
||||
DenseTensor t(*in);
|
||||
if (!channel_last) {
|
||||
t.Resize({in_dims[0], groups, in_dims[1] / groups, in_dims[2], in_dims[3]});
|
||||
} else {
|
||||
t.Resize({in_dims[0], in_dims[1], in_dims[2], groups, in_dims[3] / groups});
|
||||
}
|
||||
auto axis = !channel_last ? std::vector<int>{0, 2, 1, 3, 4}
|
||||
: std::vector<int>{0, 1, 2, 4, 3};
|
||||
|
||||
DenseTensor o(*out);
|
||||
if (!channel_last) {
|
||||
o.Resize({in_dims[0], in_dims[1] / groups, groups, in_dims[2], in_dims[3]});
|
||||
} else {
|
||||
o.Resize({in_dims[0], in_dims[1], in_dims[2], in_dims[3] / groups, groups});
|
||||
}
|
||||
funcs::Transpose<Context, T, 5> trans;
|
||||
trans(dev_ctx, t, &o, axis);
|
||||
out->Resize(o_dims);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,341 @@
|
||||
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/cholesky_grad_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/blas/blas.h"
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename Context, typename T>
|
||||
inline void TransCompute(const int dim,
|
||||
const Context& dev_ctx,
|
||||
const DenseTensor& in,
|
||||
DenseTensor* out,
|
||||
const std::vector<int>& axis) {
|
||||
switch (dim) {
|
||||
case 1:
|
||||
funcs::Transpose<Context, T, 1> trans1;
|
||||
trans1(dev_ctx, in, out, axis);
|
||||
break;
|
||||
case 2:
|
||||
funcs::Transpose<Context, T, 2> trans2;
|
||||
trans2(dev_ctx, in, out, axis);
|
||||
break;
|
||||
case 3:
|
||||
funcs::Transpose<Context, T, 3> trans3;
|
||||
trans3(dev_ctx, in, out, axis);
|
||||
break;
|
||||
case 4:
|
||||
funcs::Transpose<Context, T, 4> trans4;
|
||||
trans4(dev_ctx, in, out, axis);
|
||||
break;
|
||||
case 5:
|
||||
funcs::Transpose<Context, T, 5> trans5;
|
||||
trans5(dev_ctx, in, out, axis);
|
||||
break;
|
||||
case 6:
|
||||
funcs::Transpose<Context, T, 6> trans6;
|
||||
trans6(dev_ctx, in, out, axis);
|
||||
break;
|
||||
default:
|
||||
// for dim >= 7 situation
|
||||
funcs::TransposeNormal<Context, T> trans_normal;
|
||||
trans_normal(dev_ctx, in, out, axis);
|
||||
}
|
||||
}
|
||||
|
||||
/*! Use these functors to implement tril, triu, diagonal and other operators */
|
||||
template <typename T>
|
||||
struct EyeFunctor {
|
||||
EyeFunctor(const int m, const int n, T* output)
|
||||
: m_(m), n_(n), output_(output) {}
|
||||
|
||||
HOSTDEVICE void operator()(size_t index) const {
|
||||
const int global_row = index / n_;
|
||||
const int col = index - global_row * n_;
|
||||
const int batch = global_row / m_;
|
||||
const int row = global_row - batch * m_;
|
||||
output_[index] = col == row ? static_cast<T>(1) : static_cast<T>(0);
|
||||
}
|
||||
|
||||
const int m_, n_;
|
||||
T* output_;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct MatrixSetDiagFunctor {
|
||||
/*! Overwrite specified diagonals of output by the values in diagonal.
|
||||
* diagonals can be a central band specified by num_diags and
|
||||
* upper_diag_index, where upper_diag_index=0 refers to the main diagonal,
|
||||
* positive value means superdiagonal and negative value means subdiagonal.
|
||||
* When it is a band, `diag` has a shape [i, j, ..., num_diags, max_diag_len]
|
||||
* and the num_diags diagonals has a up to down layout. Otherwise it has a
|
||||
* shape [i, j, ..., max_diag_len].
|
||||
*/
|
||||
MatrixSetDiagFunctor(const int m,
|
||||
const int n,
|
||||
const int num_diags,
|
||||
const int max_diag_len,
|
||||
const int upper_diag_index,
|
||||
const T* diag,
|
||||
T* output)
|
||||
: m_(m),
|
||||
n_(n),
|
||||
num_diags_(num_diags),
|
||||
max_diag_len_(max_diag_len),
|
||||
upper_diag_index_(upper_diag_index),
|
||||
diag_(diag),
|
||||
output_(output) {}
|
||||
|
||||
HOSTDEVICE void operator()(size_t index) const {
|
||||
const int batch_and_diag_index = index / max_diag_len_;
|
||||
const int index_in_the_diagonal =
|
||||
index - batch_and_diag_index * max_diag_len_;
|
||||
const int batch = batch_and_diag_index / num_diags_;
|
||||
const int diag_index_in_input = batch_and_diag_index - batch * num_diags_;
|
||||
// diag_index=0 refers to the main diagonal
|
||||
const int diag_index = upper_diag_index_ - diag_index_in_input;
|
||||
// shift down for subdiagonal if diag_index < 0
|
||||
const int y_index =
|
||||
index_in_the_diagonal + (0 > -diag_index ? 0 : -diag_index);
|
||||
// shift right for superdiagonal if diag_index > 0
|
||||
const int x_index =
|
||||
index_in_the_diagonal + (0 > diag_index ? 0 : diag_index);
|
||||
|
||||
// Upper-bound checks for diagonals shorter than max_diag_len.
|
||||
// y_index and x_index are nonnegative by construction.
|
||||
if (y_index < m_ && x_index < n_) {
|
||||
const int64_t out_index =
|
||||
static_cast<int64_t>(batch) * m_ * n_ + y_index * n_ + x_index;
|
||||
output_[out_index] = diag_[index];
|
||||
}
|
||||
}
|
||||
|
||||
const int m_, n_, num_diags_, max_diag_len_, upper_diag_index_;
|
||||
const T* diag_;
|
||||
T* output_;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct MatrixDiagPartFunctor {
|
||||
/*! Similar to MatrixSetDiagFunctor but return the diagonals. diag_index=0
|
||||
* refers to the main diagonal, positive value means superdiagonal and
|
||||
* negative value means subdiagonal */
|
||||
MatrixDiagPartFunctor(const int m,
|
||||
const int n,
|
||||
const int num_diags,
|
||||
const int max_diag_len,
|
||||
const int upper_diag_index,
|
||||
const T padding,
|
||||
const T* input,
|
||||
T* output)
|
||||
: m_(m),
|
||||
n_(n),
|
||||
num_diags_(num_diags),
|
||||
max_diag_len_(max_diag_len),
|
||||
upper_diag_index_(upper_diag_index),
|
||||
input_(input),
|
||||
output_(output) {}
|
||||
|
||||
HOSTDEVICE void operator()(size_t index) const {
|
||||
const int batch_and_mapped_diag_index = index / max_diag_len_;
|
||||
const int index_in_the_diagonal =
|
||||
index - batch_and_mapped_diag_index * max_diag_len_;
|
||||
const int batch = batch_and_mapped_diag_index / num_diags_;
|
||||
const int mapped_diag_index =
|
||||
batch_and_mapped_diag_index - batch * num_diags_;
|
||||
// diag_index=0 refers to the main diagonal
|
||||
const int diag_index = upper_diag_index_ - mapped_diag_index;
|
||||
// shift down for subdiagonal if diag_index < 0
|
||||
const int y_index =
|
||||
index_in_the_diagonal + (0 > -diag_index ? 0 : -diag_index);
|
||||
// shift right for superdiagonal if diag_index > 0
|
||||
const int x_index =
|
||||
index_in_the_diagonal + (0 > diag_index ? 0 : diag_index);
|
||||
if (y_index < m_ && x_index < n_) {
|
||||
output_[index] = input_[batch * m_ * n_ + y_index * m_ + x_index];
|
||||
} else {
|
||||
output_[index] = padding_;
|
||||
}
|
||||
}
|
||||
|
||||
const int m_, n_, num_diags_, max_diag_len_, upper_diag_index_;
|
||||
const T padding_;
|
||||
const T* input_;
|
||||
T* output_;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct MatrixBandPartScaleEndFunctor {
|
||||
/*! Compared with MatrixBandPartFunctor, it scale up values at the end of
|
||||
* band. It can be used to fuse the following operations, which actually
|
||||
* output triangular with diagonal scaled up:
|
||||
* 1. dig = matrix_diag_part(middle)
|
||||
* 2. middle = matrix_set_diag(middle, diag * scalar)
|
||||
* 3. middle = matrix_band_part(middle, -1, 0)
|
||||
*/
|
||||
MatrixBandPartScaleEndFunctor(const int m,
|
||||
const int n,
|
||||
const int num_lower_diags,
|
||||
const int num_upper_diags,
|
||||
const T scale,
|
||||
const T* input,
|
||||
T* output)
|
||||
: m_(m),
|
||||
n_(n),
|
||||
num_lower_diags_(num_lower_diags),
|
||||
num_upper_diags_(num_upper_diags),
|
||||
scale_(scale),
|
||||
input_(input),
|
||||
output_(output) {}
|
||||
|
||||
HOSTDEVICE void operator()(size_t index) const {
|
||||
const int col = index % n_;
|
||||
const int row = (index / n_) % m_;
|
||||
const int band_start = (num_lower_diags_ < 0 ? 0 : row - num_lower_diags_);
|
||||
const int band_end =
|
||||
(num_upper_diags_ < 0 ? n_ : row + num_upper_diags_ + 1);
|
||||
if (col < band_start || col >= band_end) {
|
||||
output_[index] = 0;
|
||||
} else if (col == band_end - 1) {
|
||||
output_[index] = scale_ * input_[index];
|
||||
} else {
|
||||
output_[index] = input_[index];
|
||||
}
|
||||
}
|
||||
|
||||
const int m_, n_, num_lower_diags_, num_upper_diags_;
|
||||
const T scale_;
|
||||
const T* input_;
|
||||
T* output_;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct AddtoScaleFunctor {
|
||||
AddtoScaleFunctor(const T scale, const T* input, T* output)
|
||||
: scale_(scale), input_(input), output_(output) {}
|
||||
HOSTDEVICE void operator()(size_t index) const {
|
||||
output_[index] += input_[index];
|
||||
output_[index] *= scale_;
|
||||
}
|
||||
const T scale_;
|
||||
const T* input_;
|
||||
T* output_;
|
||||
};
|
||||
|
||||
template <typename T, typename Context>
|
||||
void CholeskyGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& out,
|
||||
const DenseTensor& out_grad,
|
||||
bool upper,
|
||||
DenseTensor* x_grad) {
|
||||
if (x_grad->numel() == 0) {
|
||||
dev_ctx.template Alloc<T>(x_grad);
|
||||
return;
|
||||
}
|
||||
|
||||
auto* x_grad_data = dev_ctx.template Alloc<T>(x_grad);
|
||||
auto& dims = out.dims();
|
||||
int batch_count = 1;
|
||||
for (int i = 0; i < dims.size() - 2; i++) {
|
||||
batch_count *= dims[i];
|
||||
}
|
||||
auto m = dims[dims.size() - 1];
|
||||
int64_t tensor_size = static_cast<int64_t>(batch_count) * m * m;
|
||||
|
||||
std::vector<int> axis(dims.size() - 2);
|
||||
std::iota(axis.begin(), axis.end(), 0);
|
||||
axis.insert(axis.end(), {dims.size() - 1, dims.size() - 2});
|
||||
DenseTensor l, l_grad;
|
||||
if (upper) {
|
||||
l.Resize(dims);
|
||||
dev_ctx.template Alloc<T>(&l);
|
||||
l_grad.Resize(dims);
|
||||
dev_ctx.template Alloc<T>(&l_grad);
|
||||
TransCompute<Context, T>(dims.size(), dev_ctx, out, &l, axis);
|
||||
TransCompute<Context, T>(dims.size(), dev_ctx, out_grad, &l_grad, axis);
|
||||
} else {
|
||||
l = out;
|
||||
l_grad = out_grad;
|
||||
}
|
||||
auto* l_data = l.data<T>();
|
||||
|
||||
/*！ refer to Iain Murray (2016); arXiv 1602.07527 */
|
||||
/*! phi = matmul(L.transpose(-1, -2), grad) */
|
||||
DenseTensor middle;
|
||||
middle.Resize(dims);
|
||||
auto* middle_data = dev_ctx.template Alloc<T>(&middle);
|
||||
auto trans_desc = funcs::CreateMatrixDescriptor(dims, 0, true);
|
||||
auto no_trans_desc = funcs::CreateMatrixDescriptor(dims, 0, false);
|
||||
auto blas = funcs::GetBlas<Context, T>(dev_ctx);
|
||||
blas.MatMul(l, trans_desc, l_grad, no_trans_desc, T(1), &middle, T(0));
|
||||
|
||||
/*! phi.tril_().diagonal(0, -2, -1).mul_(0.5) */
|
||||
funcs::ForRange<Context> for_range(dev_ctx, tensor_size);
|
||||
MatrixBandPartScaleEndFunctor<T> matrix_band_part_scale_end_functor(
|
||||
m,
|
||||
m,
|
||||
/* num_lower_diags */ m,
|
||||
/* num_upper_diags */ 0,
|
||||
/* scale */ 0.5,
|
||||
middle_data,
|
||||
middle_data);
|
||||
for_range(matrix_band_part_scale_end_functor);
|
||||
|
||||
// Compute inverse by solving the triangular linear system AX = B, where B
|
||||
// is the identity matrix. The matrix X would be overwritten on B
|
||||
DenseTensor identity;
|
||||
identity.Resize(dims);
|
||||
auto* identity_data = dev_ctx.template Alloc<T>(&identity);
|
||||
EyeFunctor<T> eye_functor(m, m, identity_data);
|
||||
for_range(eye_functor);
|
||||
// TODO(guosheng): use trsmBatched for GPU
|
||||
for (int i = 0; i < batch_count; i++) {
|
||||
int64_t offset = static_cast<int64_t>(i) * m * m;
|
||||
blas.TRSM(/*side*/ CblasLeft,
|
||||
/*uplo*/ CblasLower,
|
||||
/*trans*/ CblasNoTrans,
|
||||
/*diag*/ CblasNonUnit,
|
||||
/*m*/ m,
|
||||
/*n*/ m,
|
||||
/*alpha*/ T(1),
|
||||
l_data + offset,
|
||||
/*lda*/ m,
|
||||
identity_data + offset,
|
||||
/*ldb*/ m);
|
||||
}
|
||||
DenseTensor& l_inverse = identity;
|
||||
|
||||
/*! x_grad = matmul(matmul(L_inverse.transpose(-1, -2), phi), L_inverse) */
|
||||
DenseTensor middle1;
|
||||
middle1.Resize(dims);
|
||||
dev_ctx.template Alloc<T>(&middle1);
|
||||
blas.MatMul(
|
||||
l_inverse, trans_desc, middle, no_trans_desc, T(1), &middle1, T(0));
|
||||
blas.MatMul(
|
||||
middle1, no_trans_desc, l_inverse, no_trans_desc, T(1), x_grad, T(0));
|
||||
|
||||
/*! x_grad.add(x_grad.transpose(-1, -2)).mul_(0.5) */
|
||||
DenseTensor x_grad_trans;
|
||||
x_grad_trans.Resize(dims);
|
||||
auto* x_grad_trans_data = dev_ctx.template Alloc<T>(&x_grad_trans);
|
||||
TransCompute<Context, T>(dims.size(), dev_ctx, *x_grad, &x_grad_trans, axis);
|
||||
AddtoScaleFunctor<T> addto_scale_functor(0.5, x_grad_trans_data, x_grad_data);
|
||||
for_range(addto_scale_functor);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,146 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/cholesky_solve_grad_kernel.h"
|
||||
#include "paddle/phi/kernels/cholesky_solve_kernel.h"
|
||||
#include "paddle/phi/kernels/complex_kernel.h"
|
||||
#include "paddle/phi/kernels/elementwise_add_kernel.h"
|
||||
#include "paddle/phi/kernels/empty_kernel.h"
|
||||
#include "paddle/phi/kernels/expand_kernel.h"
|
||||
#include "paddle/phi/kernels/full_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/blas/blas.h"
|
||||
#include "paddle/phi/kernels/funcs/common_shape.h"
|
||||
#include "paddle/phi/kernels/funcs/complex_functors.h"
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
#include "paddle/phi/kernels/funcs/matrix_reduce.h"
|
||||
#include "paddle/phi/kernels/funcs/tril_triu_compute.h"
|
||||
#include "paddle/phi/kernels/transpose_kernel.h"
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void CholeskySolveGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& y,
|
||||
const DenseTensor& out,
|
||||
const DenseTensor& dout,
|
||||
bool upper,
|
||||
DenseTensor* dx,
|
||||
DenseTensor* dy) {
|
||||
if (dout.numel() == 0) {
|
||||
if (dx) {
|
||||
dev_ctx.template Alloc<T>(dx);
|
||||
if (dx->numel() != 0) {
|
||||
Full<T, Context>(dev_ctx, dx->dims(), 0, dx);
|
||||
}
|
||||
}
|
||||
if (dy) {
|
||||
dev_ctx.template Alloc<T>(dy);
|
||||
if (dy->numel() != 0) {
|
||||
Full<T, Context>(dev_ctx, dy->dims(), 0, dy);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
// get broadcast dim
|
||||
std::vector<int64_t> x_bst_dims_vec;
|
||||
std::vector<int64_t> y_bst_dims_vec;
|
||||
std::tie(x_bst_dims_vec, y_bst_dims_vec) =
|
||||
funcs::MatrixGetBroadcastDims(x, y);
|
||||
IntArray x_bst_dims(x_bst_dims_vec);
|
||||
IntArray y_bst_dims(y_bst_dims_vec);
|
||||
|
||||
// Tensor broadcast to temp 'y_bst'
|
||||
DenseTensor y_bst = Empty<T, Context>(dev_ctx, y_bst_dims);
|
||||
ExpandKernel<T, Context>(dev_ctx, y, y_bst_dims, &y_bst);
|
||||
|
||||
// reuse forward to calculate dx_bst, which is broad_cast of dx
|
||||
DenseTensor dx_bst = Empty<T, Context>(dev_ctx, x_bst_dims);
|
||||
CholeskySolveKernel<T, Context>(dev_ctx, dout, y_bst, upper, &dx_bst);
|
||||
|
||||
// get 'dx' according to 'dx_bst'
|
||||
dx->Resize(x.dims());
|
||||
dev_ctx.template Alloc<T>(dx);
|
||||
if (dx_bst.dims() == x.dims()) {
|
||||
Copy<Context>(dev_ctx, dx_bst, dev_ctx.GetPlace(), false, dx);
|
||||
} else {
|
||||
funcs::MatrixReduceSumFunctor<T, Context> functor;
|
||||
functor(dev_ctx, dx_bst, dx);
|
||||
dx->Resize(x.dims());
|
||||
}
|
||||
|
||||
// calculate out's conjugate for complex
|
||||
DenseTensor out_conj = Conj<T, Context>(dev_ctx, out);
|
||||
out_conj = TransposeLast2Dim<T>(dev_ctx, out_conj);
|
||||
|
||||
DenseTensor commonterm = Empty<T, Context>(dev_ctx, y_bst_dims);
|
||||
auto blas = funcs::GetBlas<Context, T>(dev_ctx);
|
||||
blas.MatMul(dx_bst,
|
||||
funcs::CreateMatrixDescriptor(dx_bst.dims(), 0, false),
|
||||
out_conj,
|
||||
funcs::CreateMatrixDescriptor(out_conj.dims(), 0, false),
|
||||
static_cast<T>(1),
|
||||
&commonterm,
|
||||
static_cast<T>(0));
|
||||
|
||||
// calculate commonterm's conjugate for complex
|
||||
DenseTensor commonterm_conj = Conj<T, Context>(dev_ctx, commonterm);
|
||||
commonterm_conj = TransposeLast2Dim<T>(dev_ctx, commonterm_conj);
|
||||
|
||||
AddKernel<T>(dev_ctx, commonterm, commonterm_conj, &commonterm);
|
||||
|
||||
DenseTensor dy_bst = Empty<T, Context>(dev_ctx, y_bst_dims);
|
||||
if (upper) {
|
||||
blas.MatMul(y_bst,
|
||||
funcs::CreateMatrixDescriptor(y_bst.dims(), 0, false),
|
||||
commonterm,
|
||||
funcs::CreateMatrixDescriptor(commonterm.dims(), 0, false),
|
||||
static_cast<T>(-1),
|
||||
&dy_bst,
|
||||
static_cast<T>(0));
|
||||
} else {
|
||||
blas.MatMul(commonterm,
|
||||
funcs::CreateMatrixDescriptor(commonterm.dims(), 0, false),
|
||||
y_bst,
|
||||
funcs::CreateMatrixDescriptor(y_bst.dims(), 0, false),
|
||||
static_cast<T>(-1),
|
||||
&dy_bst,
|
||||
static_cast<T>(0));
|
||||
}
|
||||
|
||||
// get upper or lower of 'dy_bst'
|
||||
DenseTensor dy_bst_upper = Empty<T, Context>(dev_ctx, y_bst_dims);
|
||||
|
||||
int y_bst_ndim = y_bst_dims_vec.size();
|
||||
const auto H = y_bst_dims_vec[y_bst_ndim - 2];
|
||||
const auto W = y_bst_dims_vec[y_bst_ndim - 1];
|
||||
funcs::ForRange<Context> y_for_range(dev_ctx, dy_bst.numel());
|
||||
funcs::TrilTriuCompute<T> tril_triu_functor(
|
||||
dy_bst.data<T>(), 0, !upper, H, W, dy_bst_upper.data<T>());
|
||||
y_for_range(tril_triu_functor);
|
||||
|
||||
// get 'dy' according to 'dy_bst'
|
||||
dy->Resize(y.dims());
|
||||
dev_ctx.template Alloc<T>(dy);
|
||||
if (dy_bst_upper.dims() == y.dims()) {
|
||||
Copy<Context>(dev_ctx, dy_bst_upper, dev_ctx.GetPlace(), false, dy);
|
||||
} else {
|
||||
funcs::MatrixReduceSumFunctor<T, Context> functor;
|
||||
functor(dev_ctx, dy_bst_upper, dy);
|
||||
dy->Resize(y.dims());
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,105 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/cholesky_solve_kernel.h"
|
||||
#include "paddle/phi/kernels/complex_kernel.h"
|
||||
#include "paddle/phi/kernels/empty_kernel.h"
|
||||
#include "paddle/phi/kernels/expand_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/common_shape.h"
|
||||
#include "paddle/phi/kernels/transpose_kernel.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
class CholeskySolveFunctor {
|
||||
public:
|
||||
void operator()(const Context& dev_ctx,
|
||||
bool upper,
|
||||
int M,
|
||||
int N,
|
||||
T* Adata,
|
||||
int lda,
|
||||
T* Bdata,
|
||||
int* devInfo);
|
||||
};
|
||||
|
||||
template <typename T, typename Context>
|
||||
void CholeskySolveKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& y,
|
||||
bool upper,
|
||||
DenseTensor* out) {
|
||||
if (out && out->numel() == 0) {
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
return;
|
||||
}
|
||||
// get broadcast dim
|
||||
std::vector<int64_t> x_bst_dims_vec;
|
||||
std::vector<int64_t> y_bst_dims_vec;
|
||||
std::tie(x_bst_dims_vec, y_bst_dims_vec) =
|
||||
funcs::MatrixGetBroadcastDims(x, y);
|
||||
IntArray x_bst_dims(x_bst_dims_vec);
|
||||
IntArray y_bst_dims(y_bst_dims_vec);
|
||||
|
||||
DenseTensor y_bst = Empty<T, Context>(dev_ctx, y_bst_dims);
|
||||
ExpandKernel<T, Context>(dev_ctx, y, y_bst_dims, &y_bst);
|
||||
|
||||
// Tensor broadcast to temp 'x_bst' and 'y_bst'
|
||||
DenseTensor x_bst = Empty<T, Context>(dev_ctx, x_bst_dims);
|
||||
ExpandKernel<T, Context>(dev_ctx, x, x_bst_dims, &x_bst);
|
||||
|
||||
// calculate y_bst's conjugate for complex
|
||||
DenseTensor y_bst_conj = Conj<T, Context>(dev_ctx, y_bst);
|
||||
y_bst_conj = TransposeLast2Dim<T>(dev_ctx, y_bst_conj);
|
||||
T* y_bst_conj_data = y_bst_conj.data<T>();
|
||||
|
||||
// calculate x_bst's conjugate for complex
|
||||
DenseTensor x_bst_conj = Conj<T, Context>(dev_ctx, x_bst);
|
||||
x_bst_conj = TransposeLast2Dim<T>(dev_ctx, x_bst_conj);
|
||||
|
||||
// copy x_bst's conjugate to 'result'
|
||||
DenseTensor result;
|
||||
Copy<Context>(dev_ctx, x_bst_conj, dev_ctx.GetPlace(), false, &result);
|
||||
T* res_data = result.data<T>();
|
||||
|
||||
// CPU use lapack, GPU use cusolver
|
||||
int x_bst_ndim = x_bst_dims_vec.size();
|
||||
int M = static_cast<int>(x_bst_dims_vec[x_bst_ndim - 2]);
|
||||
int N = static_cast<int>(x_bst_dims_vec[x_bst_ndim - 1]);
|
||||
int batchsize = product(slice_ddim(x_bst.dims(), 0, x_bst_ndim - 2));
|
||||
|
||||
DenseTensor info = Empty<int, Context>(dev_ctx, IntArray({batchsize}));
|
||||
int* info_data = info.data<int>();
|
||||
|
||||
CholeskySolveFunctor<T, Context> functor;
|
||||
for (int i = 0; i < batchsize; ++i) {
|
||||
functor(dev_ctx,
|
||||
upper,
|
||||
M,
|
||||
N,
|
||||
y_bst_conj_data + i * M * M,
|
||||
std::max(1, M),
|
||||
res_data + i * M * N,
|
||||
info_data + i);
|
||||
}
|
||||
|
||||
// calculate out's conjugate for complex
|
||||
result = TransposeLast2Dim<T>(dev_ctx, result);
|
||||
out->Resize(x_bst_dims_vec);
|
||||
ConjKernel<T, Context>(dev_ctx, result, out);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,389 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/utils/optional.h"
|
||||
|
||||
namespace phi {
|
||||
struct Segment {
|
||||
int begin;
|
||||
int end;
|
||||
int type;
|
||||
bool operator==(const Segment& y) const {
|
||||
return begin == y.begin && end == y.end && type == y.type;
|
||||
}
|
||||
};
|
||||
|
||||
bool ChunkEnd(int prev_tag,
|
||||
int prev_type,
|
||||
int tag,
|
||||
int type,
|
||||
int other_chunk_type,
|
||||
int tag_begin,
|
||||
int tag_inside,
|
||||
int tag_end,
|
||||
int tag_single);
|
||||
|
||||
bool ChunkBegin(int prev_tag,
|
||||
int prev_type,
|
||||
int tag,
|
||||
int type,
|
||||
int other_chunk_type,
|
||||
int tag_begin,
|
||||
int tag_inside,
|
||||
int tag_end,
|
||||
int tag_single);
|
||||
|
||||
void EvalOneSeq(const int64_t* output,
|
||||
const int64_t* label,
|
||||
int length,
|
||||
std::vector<Segment>* output_segments,
|
||||
std::vector<Segment>* label_segments,
|
||||
int64_t* num_output_segments,
|
||||
int64_t* num_label_segments,
|
||||
int64_t* num_correct,
|
||||
int num_chunk_types,
|
||||
int num_tag_types,
|
||||
int other_chunk_type,
|
||||
int tag_begin,
|
||||
int tag_inside,
|
||||
int tag_end,
|
||||
int tag_single,
|
||||
const std::set<int>& excluded_chunk_types);
|
||||
|
||||
void GetSegments(const int64_t* label,
|
||||
int length,
|
||||
std::vector<Segment>* segments,
|
||||
int num_chunk_types,
|
||||
int num_tag_types,
|
||||
int other_chunk_type,
|
||||
int tag_begin,
|
||||
int tag_inside,
|
||||
int tag_end,
|
||||
int tag_single) {
|
||||
segments->clear();
|
||||
segments->reserve(length);
|
||||
int chunk_start = 0;
|
||||
bool in_chunk = false;
|
||||
int tag = -1;
|
||||
int type = other_chunk_type;
|
||||
for (int i = 0; i < length; ++i) {
|
||||
int prev_tag = tag;
|
||||
int prev_type = type;
|
||||
PADDLE_ENFORCE_LE(
|
||||
label[i],
|
||||
num_chunk_types * num_tag_types,
|
||||
common::errors::InvalidArgument(
|
||||
"The value of Input(Label) should be less than the number of "
|
||||
"chunk types times the number of tag types, but received %d "
|
||||
"(Label) vs %d (chunk types) * %d (tag types).",
|
||||
label[i],
|
||||
num_chunk_types,
|
||||
num_tag_types));
|
||||
tag = label[i] % num_tag_types;
|
||||
type = label[i] / num_tag_types;
|
||||
if (in_chunk && ChunkEnd(prev_tag,
|
||||
prev_type,
|
||||
tag,
|
||||
type,
|
||||
other_chunk_type,
|
||||
tag_begin,
|
||||
tag_inside,
|
||||
tag_end,
|
||||
tag_single)) {
|
||||
Segment segment{
|
||||
chunk_start, // begin
|
||||
i - 1, // end
|
||||
prev_type,
|
||||
};
|
||||
segments->push_back(segment);
|
||||
in_chunk = false;
|
||||
}
|
||||
if (ChunkBegin(prev_tag,
|
||||
prev_type,
|
||||
tag,
|
||||
type,
|
||||
other_chunk_type,
|
||||
tag_begin,
|
||||
tag_inside,
|
||||
tag_end,
|
||||
tag_single)) {
|
||||
chunk_start = i;
|
||||
in_chunk = true;
|
||||
}
|
||||
}
|
||||
if (in_chunk) {
|
||||
Segment segment{
|
||||
chunk_start, // begin
|
||||
length - 1, // end
|
||||
type,
|
||||
};
|
||||
segments->push_back(segment);
|
||||
}
|
||||
}
|
||||
|
||||
bool ChunkEnd(int prev_tag,
|
||||
int prev_type,
|
||||
int tag,
|
||||
int type,
|
||||
int other_chunk_type,
|
||||
int tag_begin,
|
||||
int tag_inside,
|
||||
int tag_end,
|
||||
int tag_single) {
|
||||
if (prev_type == other_chunk_type) return false;
|
||||
if (type == other_chunk_type) return true;
|
||||
if (type != prev_type) return true;
|
||||
if (prev_tag == tag_begin) return tag == tag_begin || tag == tag_single;
|
||||
if (prev_tag == tag_inside) return tag == tag_begin || tag == tag_single;
|
||||
if (prev_tag == tag_end) return true;
|
||||
if (prev_tag == tag_single) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ChunkBegin(int prev_tag,
|
||||
int prev_type,
|
||||
int tag,
|
||||
int type,
|
||||
int other_chunk_type,
|
||||
int tag_begin,
|
||||
int tag_inside,
|
||||
int tag_end,
|
||||
int tag_single) {
|
||||
if (prev_type == other_chunk_type) return type != other_chunk_type;
|
||||
if (type == other_chunk_type) return false;
|
||||
if (type != prev_type) return true;
|
||||
if (tag == tag_begin) return true;
|
||||
if (tag == tag_inside) return prev_tag == tag_end || prev_tag == tag_single;
|
||||
if (tag == tag_end) return prev_tag == tag_end || prev_tag == tag_single;
|
||||
if (tag == tag_single) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ChunkEvalKernel(const Context& dev_ctx,
|
||||
const DenseTensor& inference,
|
||||
const DenseTensor& label,
|
||||
const optional<DenseTensor>& seq_length,
|
||||
int num_chunk_types,
|
||||
const std::string& chunk_scheme,
|
||||
const std::vector<int>& excluded_chunk_types,
|
||||
DenseTensor* precision,
|
||||
DenseTensor* recall,
|
||||
DenseTensor* f1_score,
|
||||
DenseTensor* num_infer_chunks,
|
||||
DenseTensor* num_label_chunks,
|
||||
DenseTensor* num_correct_chunks) {
|
||||
// initialize to parse configurations
|
||||
int num_tag_types;
|
||||
int other_chunk_type;
|
||||
int tag_begin, tag_inside, tag_end, tag_single;
|
||||
std::vector<Segment> label_segments;
|
||||
std::vector<Segment> output_segments;
|
||||
std::set<int> excluded_chunk_types_new;
|
||||
|
||||
if (chunk_scheme == "IOB") {
|
||||
num_tag_types = 2;
|
||||
tag_begin = 0;
|
||||
tag_inside = 1;
|
||||
tag_end = -1;
|
||||
tag_single = -1;
|
||||
} else if (chunk_scheme == "IOE") {
|
||||
num_tag_types = 2;
|
||||
tag_begin = -1;
|
||||
tag_inside = 0;
|
||||
tag_end = 1;
|
||||
tag_single = -1;
|
||||
} else if (chunk_scheme == "IOBES") {
|
||||
num_tag_types = 4;
|
||||
tag_begin = 0;
|
||||
tag_inside = 1;
|
||||
tag_end = 2;
|
||||
tag_single = 3;
|
||||
} else if (chunk_scheme == "plain") {
|
||||
num_tag_types = 1;
|
||||
tag_begin = -1;
|
||||
tag_inside = -1;
|
||||
tag_end = -1;
|
||||
tag_single = -1;
|
||||
} else {
|
||||
PADDLE_THROW(common::errors::InvalidArgument("Unknown chunk scheme."));
|
||||
}
|
||||
other_chunk_type = num_chunk_types;
|
||||
excluded_chunk_types_new.insert(excluded_chunk_types.begin(),
|
||||
excluded_chunk_types.end());
|
||||
|
||||
const int64_t* inference_data = inference.data<int64_t>();
|
||||
const int64_t* label_data = label.data<int64_t>();
|
||||
T* precision_data = dev_ctx.template Alloc<T>(precision);
|
||||
T* recall_data = dev_ctx.template Alloc<T>(recall);
|
||||
T* f1_data = dev_ctx.template Alloc<T>(f1_score);
|
||||
int64_t* num_infer_chunks_data =
|
||||
dev_ctx.template Alloc<int64_t>(num_infer_chunks);
|
||||
int64_t* num_label_chunks_data =
|
||||
dev_ctx.template Alloc<int64_t>(num_label_chunks);
|
||||
int64_t* num_correct_chunks_data =
|
||||
dev_ctx.template Alloc<int64_t>(num_correct_chunks);
|
||||
*num_infer_chunks_data = 0;
|
||||
*num_label_chunks_data = 0;
|
||||
*num_correct_chunks_data = 0;
|
||||
|
||||
auto lod = label.lod();
|
||||
bool use_padding = lod.empty();
|
||||
int num_sequences = 0;
|
||||
|
||||
if (use_padding) {
|
||||
auto dim1 = inference.dims()[1];
|
||||
auto* seq_length_t = seq_length.get_ptr();
|
||||
auto* seq_length_data = seq_length_t->data<int64_t>();
|
||||
num_sequences = seq_length_t->dims()[0];
|
||||
|
||||
for (int i = 0; i < num_sequences; ++i) {
|
||||
int seq_length = seq_length_data[i];
|
||||
EvalOneSeq(inference_data + i * dim1,
|
||||
label_data + i * dim1,
|
||||
seq_length,
|
||||
&output_segments,
|
||||
&label_segments,
|
||||
num_infer_chunks_data,
|
||||
num_label_chunks_data,
|
||||
num_correct_chunks_data,
|
||||
num_chunk_types,
|
||||
num_tag_types,
|
||||
other_chunk_type,
|
||||
tag_begin,
|
||||
tag_inside,
|
||||
tag_end,
|
||||
tag_single,
|
||||
excluded_chunk_types_new);
|
||||
}
|
||||
} else {
|
||||
PADDLE_ENFORCE_EQ(
|
||||
lod.size(),
|
||||
1UL,
|
||||
common::errors::InvalidArgument(
|
||||
"Only support one level LoD sequence now, but received %d.",
|
||||
lod.size()));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
lod,
|
||||
inference.lod(),
|
||||
common::errors::InvalidArgument(
|
||||
"Input(Inference) and Input(Label) of Op(chunk_eval) should have "
|
||||
"same LoD information."));
|
||||
num_sequences = lod[0].size() - 1;
|
||||
|
||||
for (int i = 0; i < num_sequences; ++i) {
|
||||
int seq_length = lod[0][i + 1] - lod[0][i];
|
||||
EvalOneSeq(inference_data + lod[0][i],
|
||||
label_data + lod[0][i],
|
||||
seq_length,
|
||||
&output_segments,
|
||||
&label_segments,
|
||||
num_infer_chunks_data,
|
||||
num_label_chunks_data,
|
||||
num_correct_chunks_data,
|
||||
num_chunk_types,
|
||||
num_tag_types,
|
||||
other_chunk_type,
|
||||
tag_begin,
|
||||
tag_inside,
|
||||
tag_end,
|
||||
tag_single,
|
||||
excluded_chunk_types_new);
|
||||
}
|
||||
}
|
||||
|
||||
*precision_data =
|
||||
!(*num_infer_chunks_data)
|
||||
? 0
|
||||
: static_cast<T>(*num_correct_chunks_data) / (*num_infer_chunks_data);
|
||||
*recall_data =
|
||||
!(*num_label_chunks_data)
|
||||
? 0
|
||||
: static_cast<T>(*num_correct_chunks_data) / (*num_label_chunks_data);
|
||||
*f1_data = !(*num_correct_chunks_data)
|
||||
? 0
|
||||
: 2 * (*precision_data) * (*recall_data) /
|
||||
((*precision_data) + (*recall_data));
|
||||
}
|
||||
|
||||
void EvalOneSeq(const int64_t* output,
|
||||
const int64_t* label,
|
||||
int length,
|
||||
std::vector<Segment>* output_segments,
|
||||
std::vector<Segment>* label_segments,
|
||||
int64_t* num_output_segments,
|
||||
int64_t* num_label_segments,
|
||||
int64_t* num_correct,
|
||||
int num_chunk_types,
|
||||
int num_tag_types,
|
||||
int other_chunk_type,
|
||||
int tag_begin,
|
||||
int tag_inside,
|
||||
int tag_end,
|
||||
int tag_single,
|
||||
const std::set<int>& excluded_chunk_types) {
|
||||
GetSegments(output,
|
||||
length,
|
||||
output_segments,
|
||||
num_chunk_types,
|
||||
num_tag_types,
|
||||
other_chunk_type,
|
||||
tag_begin,
|
||||
tag_inside,
|
||||
tag_end,
|
||||
tag_single);
|
||||
GetSegments(label,
|
||||
length,
|
||||
label_segments,
|
||||
num_chunk_types,
|
||||
num_tag_types,
|
||||
other_chunk_type,
|
||||
tag_begin,
|
||||
tag_inside,
|
||||
tag_end,
|
||||
tag_single);
|
||||
size_t i = 0, j = 0;
|
||||
while (i < output_segments->size() && j < label_segments->size()) {
|
||||
if (output_segments->at(i) == label_segments->at(j) &&
|
||||
excluded_chunk_types.count(output_segments->at(i).type) != 1) {
|
||||
++(*num_correct);
|
||||
}
|
||||
if (output_segments->at(i).end < label_segments->at(j).end) {
|
||||
++i;
|
||||
} else if (output_segments->at(i).end > label_segments->at(j).end) {
|
||||
++j;
|
||||
} else {
|
||||
++i;
|
||||
++j;
|
||||
}
|
||||
}
|
||||
for (auto& segment : (*label_segments)) {
|
||||
if (excluded_chunk_types.count(segment.type) != 1) {
|
||||
++(*num_label_segments);
|
||||
}
|
||||
}
|
||||
for (auto& segment : (*output_segments)) {
|
||||
if (excluded_chunk_types.count(segment.type) != 1) {
|
||||
++(*num_output_segments);
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,55 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ClipByNormFunctor(const Context& dev_ctx,
|
||||
const DenseTensor& in,
|
||||
float max_norm,
|
||||
DenseTensor* output) {
|
||||
auto input = ∈
|
||||
dev_ctx.template Alloc<T>(output);
|
||||
|
||||
PADDLE_ENFORCE_NOT_NULL(input,
|
||||
common::errors::InvalidArgument(
|
||||
"Input(X) of ClipByNormOp should not be null. "
|
||||
"Please check if it is created correctly."));
|
||||
|
||||
auto x = EigenVector<T>::Flatten(*input);
|
||||
auto out = EigenVector<T>::Flatten(*output);
|
||||
auto x_norm = x.square().sum().sqrt();
|
||||
auto* place = dev_ctx.eigen_device();
|
||||
|
||||
auto temp = (x_norm <= max_norm).template cast<T>();
|
||||
auto epsilon = ((x_norm <= static_cast<T>(1e-30)).all().template cast<T>()) *
|
||||
static_cast<T>(1e-6);
|
||||
|
||||
auto scaling =
|
||||
temp + (static_cast<T>(1) - temp) * max_norm / (x_norm + epsilon);
|
||||
Eigen::array<int, 1> one_dim{{1}};
|
||||
Eigen::DSizes<int, 1> m_dsize(input->numel());
|
||||
if (dev_ctx.GetPlace() == CPUPlace()) {
|
||||
out.device(*place) = x * scaling.reshape(one_dim).eval().broadcast(m_dsize);
|
||||
} else {
|
||||
out.device(*place) = x * scaling.reshape(one_dim).broadcast(m_dsize);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,71 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/backends/all_context.h"
|
||||
#include "paddle/phi/common/transform.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/kernels/clip_kernel.h"
|
||||
#if defined(__NVCC__) || defined(__HIPCC__)
|
||||
#include "paddle/phi/kernels/funcs/broadcast_function.h"
|
||||
#endif
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T>
|
||||
class ClipGradFunctor {
|
||||
public:
|
||||
explicit ClipGradFunctor(const T min, const T max) : min_(min), max_(max) {}
|
||||
HOSTDEVICE T operator()(const T x, const T y) const {
|
||||
return (y >= min_ && y <= max_) ? x : static_cast<T>(0);
|
||||
}
|
||||
|
||||
private:
|
||||
T min_;
|
||||
T max_;
|
||||
};
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ClipGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& out_grad,
|
||||
const Scalar& min,
|
||||
const Scalar& max,
|
||||
DenseTensor* x_grad) {
|
||||
auto max_ = max.to<T>();
|
||||
auto min_ = min.to<T>();
|
||||
|
||||
#if defined(__NVCC__) || defined(__HIPCC__)
|
||||
std::vector<const DenseTensor*> ins = {&out_grad, &x};
|
||||
std::vector<DenseTensor*> outs = {x_grad};
|
||||
auto functor = ClipGradFunctor<T>(min_, max_);
|
||||
dev_ctx.template Alloc<T>(x_grad);
|
||||
funcs::ElementwiseKernel<T>(dev_ctx, ins, &outs, functor);
|
||||
#else
|
||||
int64_t numel = out_grad.numel();
|
||||
auto* d_x_data = dev_ctx.template Alloc<T>(x_grad);
|
||||
const T* d_out_data = out_grad.data<T>();
|
||||
const T* x_data = x.data<T>();
|
||||
Transform<Context> trans;
|
||||
trans(dev_ctx,
|
||||
d_out_data,
|
||||
d_out_data + numel,
|
||||
x_data,
|
||||
d_x_data,
|
||||
ClipGradFunctor<T>(min_, max_));
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,74 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/backends/all_context.h"
|
||||
#include "paddle/phi/common/transform.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/kernels/clip_kernel.h"
|
||||
#if defined(__NVCC__) || defined(__HIPCC__)
|
||||
#include "paddle/phi/kernels/funcs/broadcast_function.h"
|
||||
#endif
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T>
|
||||
class ClipFunctor {
|
||||
public:
|
||||
explicit ClipFunctor(const T min, const T max) : min_(min), max_(max) {}
|
||||
HOSTDEVICE T operator()(const T x) const {
|
||||
return x < min_ ? min_ : x > max_ ? max_ : x;
|
||||
}
|
||||
|
||||
private:
|
||||
T min_;
|
||||
T max_;
|
||||
};
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ClipKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const Scalar& min,
|
||||
const Scalar& max,
|
||||
DenseTensor* out) {
|
||||
auto max_ = max.to<T>();
|
||||
auto min_ = min.to<T>();
|
||||
|
||||
PADDLE_ENFORCE_LE(
|
||||
min_,
|
||||
max_,
|
||||
errors::InvalidArgument("max should be greater than or equal to min. "
|
||||
"But received min = %f, max = %f",
|
||||
static_cast<float>(min_),
|
||||
static_cast<float>(max_)));
|
||||
|
||||
T* out_data = dev_ctx.template Alloc<T>(out);
|
||||
const T* x_data = x.data<T>();
|
||||
int64_t numel = x.numel();
|
||||
if (dev_ctx.GetPlace().GetType() == AllocationType::GPU) {
|
||||
#if defined(__NVCC__) || defined(__HIPCC__)
|
||||
std::vector<const DenseTensor*> ins = {&x};
|
||||
std::vector<DenseTensor*> outs = {out};
|
||||
auto functor = ClipFunctor<T>(min_, max_);
|
||||
funcs::ElementwiseKernel<T>(dev_ctx, ins, &outs, functor);
|
||||
#endif
|
||||
} else {
|
||||
Transform<Context> trans;
|
||||
trans(
|
||||
dev_ctx, x_data, x_data + numel, out_data, ClipFunctor<T>(min_, max_));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,198 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <numeric>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
const int kBoxDim = 4;
|
||||
|
||||
template <typename T>
|
||||
struct ScoreWithID {
|
||||
T score;
|
||||
int batch_id;
|
||||
int index;
|
||||
int level;
|
||||
ScoreWithID() {
|
||||
batch_id = -1;
|
||||
index = -1;
|
||||
level = -1;
|
||||
}
|
||||
ScoreWithID(T score_, int batch_id_, int index_, int level_) {
|
||||
score = score_;
|
||||
batch_id = batch_id_;
|
||||
index = index_;
|
||||
level = level_;
|
||||
}
|
||||
};
|
||||
template <typename T>
|
||||
static inline bool CompareByScore(ScoreWithID<T> a, ScoreWithID<T> b) {
|
||||
return a.score >= b.score;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static inline bool CompareByBatchid(ScoreWithID<T> a, ScoreWithID<T> b) {
|
||||
return a.batch_id < b.batch_id;
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void CollectFpnProposalsOpKernel(
|
||||
const Context& dev_ctx,
|
||||
const std::vector<const DenseTensor*>& multi_level_rois,
|
||||
const std::vector<const DenseTensor*>& multi_level_scores,
|
||||
const optional<std::vector<const DenseTensor*>>& multi_level_rois_num,
|
||||
int post_nms_topn,
|
||||
DenseTensor* fpn_rois_out,
|
||||
DenseTensor* rois_num_out) {
|
||||
auto multi_layer_rois = multi_level_rois;
|
||||
|
||||
auto multi_layer_scores = multi_level_scores;
|
||||
auto multi_rois_num = multi_level_rois_num
|
||||
? multi_level_rois_num.get()
|
||||
: std::vector<const DenseTensor*>();
|
||||
int num_size = multi_rois_num.size();
|
||||
|
||||
auto* fpn_rois = fpn_rois_out;
|
||||
|
||||
PADDLE_ENFORCE_GE(post_nms_topn,
|
||||
0UL,
|
||||
common::errors::InvalidArgument(
|
||||
"The parameter post_nms_topn must be "
|
||||
"a positive integer. But received post_nms_topn = %d",
|
||||
post_nms_topn));
|
||||
|
||||
// assert that the length of Rois and scores are same
|
||||
PADDLE_ENFORCE_EQ(
|
||||
multi_layer_rois.size(),
|
||||
multi_layer_scores.size(),
|
||||
common::errors::InvalidArgument(
|
||||
"The number of RoIs and Scores should"
|
||||
" be the same. But received number of RoIs is %d, number of Scores "
|
||||
"is %d",
|
||||
multi_layer_rois.size(),
|
||||
multi_layer_scores.size()));
|
||||
// Check if the lod information of two DenseTensor is same
|
||||
const int num_fpn_level = multi_layer_rois.size();
|
||||
std::vector<int> integral_of_all_rois(num_fpn_level + 1, 0);
|
||||
for (int i = 0; i < num_fpn_level; ++i) {
|
||||
int all_rois = 0;
|
||||
if (num_size == 0) {
|
||||
auto cur_rois_lod = multi_layer_rois[i]->lod().back();
|
||||
all_rois = cur_rois_lod[cur_rois_lod.size() - 1];
|
||||
} else {
|
||||
const int* cur_rois_num = multi_rois_num[i]->data<int>();
|
||||
all_rois = std::accumulate(
|
||||
cur_rois_num, cur_rois_num + multi_rois_num[i]->numel(), 0);
|
||||
}
|
||||
integral_of_all_rois[i + 1] = integral_of_all_rois[i] + all_rois;
|
||||
}
|
||||
|
||||
const int batch_size = (num_size == 0)
|
||||
? multi_layer_rois[0]->lod().back().size() - 1
|
||||
: multi_rois_num[0]->numel();
|
||||
// concatenate all fpn rois scores into a list
|
||||
// create a vector to store all scores
|
||||
std::vector<ScoreWithID<T>> scores_of_all_rois(
|
||||
integral_of_all_rois[num_fpn_level], ScoreWithID<T>());
|
||||
for (int i = 0; i < num_fpn_level; ++i) {
|
||||
const T* cur_level_scores = multi_layer_scores[i]->data<T>();
|
||||
int cur_level_num = integral_of_all_rois[i + 1] - integral_of_all_rois[i];
|
||||
int cur_batch_id = 0;
|
||||
int pre_num = 0;
|
||||
for (int j = 0; j < cur_level_num; ++j) {
|
||||
if (num_size == 0) {
|
||||
auto cur_scores_lod = multi_layer_scores[i]->lod().back();
|
||||
if (static_cast<size_t>(j) >= cur_scores_lod[cur_batch_id + 1]) {
|
||||
cur_batch_id++;
|
||||
}
|
||||
} else {
|
||||
const int* rois_num_data = multi_rois_num[i]->data<int>();
|
||||
if (j >= pre_num + rois_num_data[cur_batch_id]) {
|
||||
pre_num += rois_num_data[cur_batch_id];
|
||||
cur_batch_id++;
|
||||
}
|
||||
}
|
||||
int cur_index = j + integral_of_all_rois[i];
|
||||
scores_of_all_rois[cur_index].score = cur_level_scores[j];
|
||||
scores_of_all_rois[cur_index].index = j;
|
||||
scores_of_all_rois[cur_index].level = i;
|
||||
scores_of_all_rois[cur_index].batch_id = cur_batch_id;
|
||||
}
|
||||
}
|
||||
// keep top post_nms_topn rois
|
||||
// sort the rois by the score
|
||||
if (post_nms_topn > integral_of_all_rois[num_fpn_level]) {
|
||||
post_nms_topn = integral_of_all_rois[num_fpn_level];
|
||||
}
|
||||
std::stable_sort(
|
||||
scores_of_all_rois.begin(), scores_of_all_rois.end(), CompareByScore<T>);
|
||||
scores_of_all_rois.resize(post_nms_topn);
|
||||
// sort by batch id
|
||||
std::stable_sort(scores_of_all_rois.begin(),
|
||||
scores_of_all_rois.end(),
|
||||
CompareByBatchid<T>);
|
||||
// create a pointer array
|
||||
std::vector<const T*> multi_fpn_rois_data(num_fpn_level);
|
||||
for (int i = 0; i < num_fpn_level; ++i) {
|
||||
multi_fpn_rois_data[i] = multi_layer_rois[i]->data<T>();
|
||||
}
|
||||
// initialize the outputs
|
||||
fpn_rois->Resize({post_nms_topn, kBoxDim});
|
||||
dev_ctx.template Alloc<T>(fpn_rois);
|
||||
T* fpn_rois_data = fpn_rois->data<T>();
|
||||
std::vector<size_t> lod0(1, 0);
|
||||
int cur_batch_id = 0;
|
||||
std::vector<int64_t> num_per_batch;
|
||||
int pre_idx = 0;
|
||||
int cur_num = 0;
|
||||
for (int i = 0; i < post_nms_topn; ++i) {
|
||||
int cur_fpn_level = scores_of_all_rois[i].level;
|
||||
int cur_level_index = scores_of_all_rois[i].index;
|
||||
memcpy(fpn_rois_data,
|
||||
multi_fpn_rois_data[cur_fpn_level] + cur_level_index * kBoxDim,
|
||||
kBoxDim * sizeof(T));
|
||||
fpn_rois_data += kBoxDim;
|
||||
if (scores_of_all_rois[i].batch_id != cur_batch_id) {
|
||||
cur_batch_id = scores_of_all_rois[i].batch_id;
|
||||
lod0.emplace_back(i);
|
||||
cur_num = i - pre_idx;
|
||||
pre_idx = i;
|
||||
num_per_batch.emplace_back(cur_num);
|
||||
}
|
||||
}
|
||||
num_per_batch.emplace_back(post_nms_topn - pre_idx);
|
||||
if (rois_num_out != nullptr) {
|
||||
auto* rois_num = rois_num_out;
|
||||
rois_num->Resize({batch_size});
|
||||
int* rois_num_data = dev_ctx.template Alloc<int>(rois_num);
|
||||
for (int i = 0; i < batch_size; i++) {
|
||||
rois_num_data[i] = num_per_batch[i];
|
||||
}
|
||||
}
|
||||
lod0.emplace_back(post_nms_topn);
|
||||
LegacyLoD lod;
|
||||
lod.emplace_back(lod0);
|
||||
fpn_rois->set_lod(lod);
|
||||
}
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,92 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/compare_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/compare_functors.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T,
|
||||
typename Context,
|
||||
typename Functor,
|
||||
typename InverseFunctor>
|
||||
inline void CompareKernelImpl(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& y,
|
||||
int axis,
|
||||
DenseTensor* out);
|
||||
|
||||
template <typename T,
|
||||
typename Context,
|
||||
typename Functor,
|
||||
typename InverseFunctor>
|
||||
inline void InplaceCompareKernelImpl(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& y,
|
||||
int axis,
|
||||
DenseTensor* out);
|
||||
|
||||
template <typename T, typename Context, typename Functor>
|
||||
inline void CompareAllKernelImpl(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& y,
|
||||
DenseTensor* out);
|
||||
|
||||
#define DEFINE_COMPARE_KERNEL(name, functor, inverse_functor) \
|
||||
template <typename T, typename Context> \
|
||||
void name##Kernel(const Context& dev_ctx, \
|
||||
const DenseTensor& x, \
|
||||
const DenseTensor& y, \
|
||||
DenseTensor* out) { \
|
||||
if (out->IsSharedWith(x)) { \
|
||||
InplaceCompareKernelImpl<T, Context, functor<T>, inverse_functor<T>>( \
|
||||
dev_ctx, x, y, -1, out); \
|
||||
} else { \
|
||||
CompareKernelImpl<T, Context, functor<T>, inverse_functor<T>>( \
|
||||
dev_ctx, x, y, -1, out); \
|
||||
} \
|
||||
}
|
||||
|
||||
DEFINE_COMPARE_KERNEL(LessThan,
|
||||
funcs::LessThanFunctor,
|
||||
funcs::GreaterThanFunctor)
|
||||
DEFINE_COMPARE_KERNEL(LessEqual,
|
||||
funcs::LessEqualFunctor,
|
||||
funcs::GreaterEqualFunctor)
|
||||
DEFINE_COMPARE_KERNEL(GreaterThan,
|
||||
funcs::GreaterThanFunctor,
|
||||
funcs::LessThanFunctor)
|
||||
DEFINE_COMPARE_KERNEL(GreaterEqual,
|
||||
funcs::GreaterEqualFunctor,
|
||||
funcs::LessEqualFunctor)
|
||||
DEFINE_COMPARE_KERNEL(Equal, funcs::EqualFunctor, funcs::EqualFunctor)
|
||||
DEFINE_COMPARE_KERNEL(NotEqual, funcs::NotEqualFunctor, funcs::NotEqualFunctor)
|
||||
#undef DEFINE_COMPARE_KERNEL
|
||||
|
||||
#define DEFINE_COMPARE_ALL_KERNEL(compare_all_kernel, functor) \
|
||||
template <typename T, typename Context> \
|
||||
void compare_all_kernel(const Context& dev_ctx, \
|
||||
const DenseTensor& x, \
|
||||
const DenseTensor& y, \
|
||||
DenseTensor* out) { \
|
||||
CompareAllKernelImpl<T, Context, functor<T>>(dev_ctx, x, y, out); \
|
||||
}
|
||||
|
||||
DEFINE_COMPARE_ALL_KERNEL(EqualAllKernel, funcs::EqualFunctor)
|
||||
#undef DEFINE_COMPARE_ALL_KERNEL
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,123 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/full_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/complex_functors.h"
|
||||
#include "paddle/phi/kernels/funcs/elementwise_grad_base.h"
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void RealGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& dout,
|
||||
DenseTensor* dx) {
|
||||
if (dx && dx->numel() == 0) {
|
||||
dev_ctx.template Alloc<T>(dx);
|
||||
return;
|
||||
}
|
||||
auto numel = dout.numel();
|
||||
auto* dout_data = dout.data<dtype::Real<T>>();
|
||||
auto* dx_data =
|
||||
dev_ctx.template Alloc<T>(dx, static_cast<size_t>(numel * sizeof(T)));
|
||||
|
||||
funcs::ForRange<Context> for_range(dev_ctx, numel);
|
||||
funcs::RealToComplexFunctor<T> functor(dout_data, dx_data, numel);
|
||||
for_range(functor);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ImagGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& dout,
|
||||
DenseTensor* dx) {
|
||||
if (dx && dx->numel() == 0) {
|
||||
dev_ctx.template Alloc<T>(dx);
|
||||
return;
|
||||
}
|
||||
auto numel = dout.numel();
|
||||
auto* dout_data = dout.data<dtype::Real<T>>();
|
||||
auto* dx_data =
|
||||
dev_ctx.template Alloc<T>(dx, static_cast<size_t>(numel * sizeof(T)));
|
||||
|
||||
funcs::ForRange<Context> for_range(dev_ctx, numel);
|
||||
funcs::ImagToComplexFunctor<T> functor(dout_data, dx_data, numel);
|
||||
for_range(functor);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
struct ComplexGradForRealFunctor {
|
||||
inline HOSTDEVICE T operator()(const T x UNUSED,
|
||||
const T y UNUSED,
|
||||
const dtype::complex<T> out UNUSED,
|
||||
const dtype::complex<T> dout) {
|
||||
return dout.real;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct ComplexGradForImagFunctor {
|
||||
inline HOSTDEVICE T operator()(const T x UNUSED,
|
||||
const T y UNUSED,
|
||||
const dtype::complex<T> out UNUSED,
|
||||
const dtype::complex<T> dout) {
|
||||
return dout.imag;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ComplexGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& y,
|
||||
const DenseTensor& dout,
|
||||
DenseTensor* dx,
|
||||
DenseTensor* dy) {
|
||||
using C = dtype::complex<T>;
|
||||
if (dout.numel() == 0) {
|
||||
if (dx) {
|
||||
if (dx->numel() == 0) {
|
||||
dev_ctx.template Alloc<T>(dx);
|
||||
} else {
|
||||
Full<T, Context>(dev_ctx, dx->dims(), 0, dx);
|
||||
}
|
||||
}
|
||||
if (dy) {
|
||||
if (dy->numel() == 0) {
|
||||
dev_ctx.template Alloc<T>(dy);
|
||||
} else {
|
||||
Full<T, Context>(dev_ctx, dy->dims(), 0, dy);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
// skip out in a hacky way
|
||||
auto out = dout;
|
||||
funcs::ElemwiseGradCompute<Context,
|
||||
T,
|
||||
ComplexGradForRealFunctor<T>,
|
||||
ComplexGradForImagFunctor<T>,
|
||||
C>(dev_ctx,
|
||||
x,
|
||||
y,
|
||||
out,
|
||||
dout,
|
||||
/*axis*/ -1,
|
||||
dx,
|
||||
dy,
|
||||
ComplexGradForRealFunctor<T>(),
|
||||
ComplexGradForImagFunctor<T>());
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,122 @@
|
||||
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/funcs/broadcast_function.h"
|
||||
#include "paddle/phi/kernels/funcs/complex_functors.h"
|
||||
#include "paddle/phi/kernels/funcs/elementwise_base.h"
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ConjKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
DenseTensor* out) {
|
||||
if (out->numel() == 0) {
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
return;
|
||||
}
|
||||
auto numel = x.numel();
|
||||
auto* x_data = x.data<T>();
|
||||
auto* out_data = dev_ctx.template Alloc<T>(out);
|
||||
|
||||
funcs::ForRange<Context> for_range(dev_ctx, numel);
|
||||
funcs::ConjFunctor<T> functor(x_data, numel, out_data);
|
||||
for_range(functor);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void RealKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
DenseTensor* out) {
|
||||
if (out->numel() == 0) {
|
||||
dev_ctx.template Alloc<dtype::Real<T>>(out);
|
||||
return;
|
||||
}
|
||||
auto numel = x.numel();
|
||||
auto* x_data = x.data<T>();
|
||||
auto* out_data = dev_ctx.template Alloc<dtype::Real<T>>(
|
||||
out, static_cast<size_t>(numel * sizeof(dtype::Real<T>)));
|
||||
|
||||
funcs::ForRange<Context> for_range(dev_ctx, numel);
|
||||
funcs::RealFunctor<T> functor(x_data, out_data, numel);
|
||||
for_range(functor);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ImagKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
DenseTensor* out) {
|
||||
if (out->numel() == 0) {
|
||||
dev_ctx.template Alloc<dtype::Real<T>>(out);
|
||||
return;
|
||||
}
|
||||
auto numel = x.numel();
|
||||
auto* x_data = x.data<T>();
|
||||
auto* out_data = dev_ctx.template Alloc<dtype::Real<T>>(
|
||||
out, static_cast<size_t>(numel * sizeof(dtype::Real<T>)));
|
||||
|
||||
funcs::ForRange<Context> for_range(dev_ctx, numel);
|
||||
funcs::ImagFunctor<T> functor(x_data, out_data, numel);
|
||||
for_range(functor);
|
||||
}
|
||||
|
||||
// functors to use with ElementwiseComputeEx
|
||||
template <typename T>
|
||||
struct RealAndImagToComplexFunctor {
|
||||
inline HOSTDEVICE dtype::complex<T> operator()(const T x, const T y) {
|
||||
return dtype::complex<T>(x, y);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct ImagAndRealToComplexFunctor {
|
||||
inline HOSTDEVICE dtype::complex<T> operator()(const T y, const T x) {
|
||||
return dtype::complex<T>(x, y);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ComplexKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& y,
|
||||
DenseTensor* out) {
|
||||
using C = dtype::complex<T>;
|
||||
if (out->numel() == 0) {
|
||||
dev_ctx.template Alloc<C>(out);
|
||||
return;
|
||||
}
|
||||
dev_ctx.template Alloc<C>(out);
|
||||
|
||||
// NOTE(chenfeiyu): be careful of the caveats of calling elementwise-related
|
||||
// facility functions
|
||||
#if defined(__NVCC__) || defined(__HIPCC__)
|
||||
funcs::ElementwiseCompute<RealAndImagToComplexFunctor<T>, T, C>(
|
||||
dev_ctx, x, y, RealAndImagToComplexFunctor<T>(), out);
|
||||
#else
|
||||
auto x_dims = x.dims();
|
||||
auto y_dims = y.dims();
|
||||
if (x_dims.size() >= y_dims.size()) {
|
||||
funcs::ElementwiseCompute<RealAndImagToComplexFunctor<T>, T, C>(
|
||||
dev_ctx, x, y, RealAndImagToComplexFunctor<T>(), out);
|
||||
} else {
|
||||
funcs::ElementwiseCompute<ImagAndRealToComplexFunctor<T>, T, C>(
|
||||
dev_ctx, x, y, ImagAndRealToComplexFunctor<T>(), out);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,74 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/concat_grad_kernel.h"
|
||||
#include "paddle/phi/kernels/full_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/concat_and_split_functor.h"
|
||||
#include "paddle/phi/kernels/funcs/concat_funcs.h"
|
||||
#include "paddle/phi/kernels/funcs/strided_memcpy.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ConcatGradKernel(const Context& dev_ctx,
|
||||
const std::vector<const DenseTensor*>& x,
|
||||
const DenseTensor& out_grad,
|
||||
const Scalar& axis_scalar,
|
||||
std::vector<DenseTensor*> x_grad) {
|
||||
auto outs = x_grad;
|
||||
{
|
||||
auto dx = x_grad;
|
||||
for (size_t i = 0; i < dx.size(); ++i) {
|
||||
if (dx[i] != nullptr) {
|
||||
dx[i]->set_lod(x[i]->lod());
|
||||
}
|
||||
}
|
||||
}
|
||||
PADDLE_ENFORCE_NOT_NULL(
|
||||
x[0],
|
||||
common::errors::NotFound("The first input tensor is not initialized."));
|
||||
|
||||
auto axis = axis_scalar.to<int>();
|
||||
axis = funcs::ComputeAxis(static_cast<int64_t>(axis),
|
||||
static_cast<int64_t>(x[0]->dims().size()));
|
||||
// get output tensor that the name is not kEmptyVarName
|
||||
std::vector<DenseTensor*> outputs;
|
||||
for (size_t j = 0; j < outs.size(); ++j) {
|
||||
if (outs[j]) {
|
||||
dev_ctx.template Alloc<T>(outs[j]);
|
||||
outputs.push_back(outs[j]);
|
||||
} else {
|
||||
outputs.push_back(nullptr);
|
||||
}
|
||||
}
|
||||
// if the out_grad.numel() == 0 ,the all x and x_grad must be zero size
|
||||
// tensor, so just return
|
||||
if (out_grad.numel() == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Sometimes direct copies will be faster, this maybe need deeply analysis.
|
||||
if (axis == 0 && outs.size() < 10) {
|
||||
std::vector<const DenseTensor*> ref_shape;
|
||||
ref_shape.insert(ref_shape.begin(), x.begin(), x.end());
|
||||
funcs::StridedMemcpyWithAxis0<T, Context>(
|
||||
dev_ctx, out_grad, ref_shape, &outputs);
|
||||
} else {
|
||||
funcs::SplitFunctor<Context, T> split_functor;
|
||||
split_functor(dev_ctx, out_grad, x, static_cast<int>(axis), &outputs);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,86 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/backends/gpu/gpu_context.h"
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#ifdef PADDLE_WITH_HIP
|
||||
#include "paddle/phi/kernels/gpudnn/conv_miopen_helper.h"
|
||||
#else
|
||||
#include "paddle/phi/kernels/gpudnn/conv_cudnn_v7.h"
|
||||
#endif
|
||||
|
||||
#include "paddle/phi/backends/dynload/cudnn.h"
|
||||
#include "paddle/phi/backends/gpu/cuda/cudnn_workspace_helper.h"
|
||||
#include "paddle/phi/kernels/cpu/conv_util.h"
|
||||
#include "paddle/phi/kernels/funcs/batch_norm_utils.h"
|
||||
#include "paddle/phi/kernels/funcs/padding.h"
|
||||
|
||||
COMMON_DECLARE_bool(cudnn_deterministic);
|
||||
PD_DECLARE_int64(conv_workspace_size_limit);
|
||||
COMMON_DECLARE_bool(cudnn_exhaustive_search);
|
||||
|
||||
namespace phi {
|
||||
|
||||
static inline bool IsVoltaOrLater(const GPUContext& dev_ctx) {
|
||||
return dev_ctx.GetComputeCapability() >= 70;
|
||||
}
|
||||
|
||||
// inline cudnnTensorFormat_t GetCudnnTensorFormat(
|
||||
// const DataLayout& order) { // Not use
|
||||
// switch (order) {
|
||||
// case DataLayout::NHWC:
|
||||
// return CUDNN_TENSOR_NHWC;
|
||||
// case DataLayout::NCHW:
|
||||
// return CUDNN_TENSOR_NCHW;
|
||||
// case DataLayout::NCDHW:
|
||||
// return CUDNN_TENSOR_NCHW; // NOTE: cudnn treat NdTensor as the same
|
||||
// case DataLayout::NDHWC:
|
||||
// return CUDNN_TENSOR_NHWC; // add, liyamei
|
||||
// default:
|
||||
// PADDLE_THROW(common::errors::Unimplemented(
|
||||
// "CUDNN has no equivalent dataLayout for input order."));
|
||||
// }
|
||||
// return CUDNN_TENSOR_NCHW;
|
||||
// }
|
||||
|
||||
/*
|
||||
static inline void GetNCDHW(const DDim& dims,
|
||||
const DataLayout& layout,
|
||||
int* N,
|
||||
int* C,
|
||||
int* D,
|
||||
int* H,
|
||||
int* W) {
|
||||
*N = dims[0];
|
||||
*C = layout == DataLayout::NCHW ? dims[1] : dims[dims.size() - 1];
|
||||
int i = layout == DataLayout::NCHW ? 0 : 1;
|
||||
if (dims.size() == 5) {
|
||||
*D = dims[2 - i];
|
||||
*H = dims[3 - i];
|
||||
*W = dims[4 - i];
|
||||
} else {
|
||||
*D = 1;
|
||||
*H = dims[2 - i];
|
||||
*W = dims[3 - i];
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
} // namespace phi
|
||||
|
||||
// PD_REGISTER_KERNEL(convdnn, GPU, ALL_LAYOUT, phi::ConvKernel, float, double
|
||||
// ) {}
|
||||
@@ -0,0 +1,552 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/cpu/conv_util.h"
|
||||
#include "paddle/phi/kernels/full_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/batch_norm_utils.h"
|
||||
#include "paddle/phi/kernels/funcs/blas/blas.h"
|
||||
#include "paddle/phi/kernels/funcs/im2col.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
#include "paddle/phi/kernels/funcs/vol2col.h"
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ConvGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& input,
|
||||
const DenseTensor& filter,
|
||||
const DenseTensor& output_grad,
|
||||
const std::vector<int>& strides,
|
||||
const std::vector<int>& paddings,
|
||||
const std::string& padding_algorithm,
|
||||
const std::vector<int>& dilations,
|
||||
int groups,
|
||||
const std::string& data_format,
|
||||
DenseTensor* input_grad,
|
||||
DenseTensor* filter_grad) {
|
||||
// The filter and filter_grad will be reshaped in the calculations,
|
||||
// so here use an assignment operation,
|
||||
// that avoids modifying the variable in the Scope.
|
||||
|
||||
if (!input_grad && !filter_grad) return;
|
||||
std::vector<int> paddings_ = paddings;
|
||||
std::vector<int> dilations_ = dilations;
|
||||
|
||||
DenseTensor filter_ = filter;
|
||||
// 0-size
|
||||
if (input.numel() == 0 || filter.numel() == 0) {
|
||||
if (input_grad) dev_ctx.template Alloc<T>(input_grad);
|
||||
if (filter_grad) {
|
||||
Full<T, Context>(dev_ctx, filter_grad->dims(), 0, filter_grad);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const bool channel_last = (data_format == "NHWC" || data_format == "NDHWC");
|
||||
|
||||
DenseTensor transformed_input(input.type());
|
||||
DenseTensor transformed_output_grad(output_grad.type());
|
||||
|
||||
if (channel_last) {
|
||||
ResizeToChannelFirst<Context, T>(dev_ctx, &input, &transformed_input);
|
||||
TransToChannelFirst<Context, T>(dev_ctx, &input, &transformed_input);
|
||||
|
||||
ResizeToChannelFirst<Context, T>(
|
||||
dev_ctx, &output_grad, &transformed_output_grad);
|
||||
TransToChannelFirst<Context, T>(
|
||||
dev_ctx, &output_grad, &transformed_output_grad);
|
||||
} else {
|
||||
transformed_input = input;
|
||||
transformed_output_grad = output_grad;
|
||||
}
|
||||
|
||||
// update padding and dilation
|
||||
auto in_dims = transformed_input.dims();
|
||||
auto filter_dims = filter.dims();
|
||||
DDim in_data_dims = slice_ddim(in_dims, 2, in_dims.size());
|
||||
DDim filter_data_dims = slice_ddim(filter_dims, 2, filter_dims.size());
|
||||
std::vector<int> ksize = vectorize<int>(filter_data_dims);
|
||||
UpdatePaddingAndDilation<int>(
|
||||
&paddings_, &dilations_, padding_algorithm, in_data_dims, strides, ksize);
|
||||
|
||||
const int64_t batch_size = transformed_input.dims()[0];
|
||||
|
||||
// filter_shape_vec: {k_o, k_i, k_h, k_w} or {k_o, k_i, k_d, k_h, k_w}
|
||||
std::vector<int64_t> filter_shape_vec(vectorize(filter.dims()));
|
||||
// output_shape_vec: {o_n, o_c, o_h, o_w} or {o_n, o_c, o_d, o_h, o_w}
|
||||
std::vector<int64_t> output_shape_vec(
|
||||
vectorize(transformed_output_grad.dims()));
|
||||
|
||||
// use col_shape in the im2col calculation
|
||||
// col_shape_vec: {i_c/g, k_h, k_w, o_h, o_w} or {i_c/g, k_d, k_h, k_w, o_d,
|
||||
// o_h, o_w}
|
||||
size_t data_dim = filter_shape_vec.size() - 2;
|
||||
std::vector<int64_t> col_shape_vec(1 + 2 * data_dim);
|
||||
col_shape_vec[0] = transformed_input.dims()[1] / groups;
|
||||
for (size_t j = 0; j < data_dim; ++j) {
|
||||
col_shape_vec[j + 1] = filter_shape_vec[j + 2];
|
||||
col_shape_vec[j + 1 + data_dim] = output_shape_vec[j + 2];
|
||||
}
|
||||
DDim col_shape(make_ddim(col_shape_vec));
|
||||
|
||||
// use col_matrix_shape in the gemm calculation
|
||||
// size: (i_c/g * k_h * k_w, o_h * o_w)
|
||||
// or
|
||||
// (i_c/g * k_d * k_h * k_w, o_d * o_h * o_w)
|
||||
DDim col_matrix_shape = flatten_to_2d(col_shape, data_dim + 1);
|
||||
|
||||
DDim input_shape =
|
||||
slice_ddim(transformed_input.dims(), 1, transformed_input.dims().size());
|
||||
|
||||
DDim filter_matrix_shape = {filter.dims()[0],
|
||||
filter.numel() / filter.dims()[0]};
|
||||
filter_.Resize(filter_matrix_shape);
|
||||
|
||||
DDim output_matrix_shape = {
|
||||
transformed_output_grad.dims()[1],
|
||||
transformed_output_grad.numel() / (transformed_output_grad.dims()[0] *
|
||||
transformed_output_grad.dims()[1])};
|
||||
|
||||
// convolution backward input operator: gemm + col2im(or col2vol)
|
||||
// convolution backward weight operator: im2col(or vol2col) + gemm
|
||||
int64_t in_step = transformed_input.dims()[1] / groups;
|
||||
int64_t out_step = transformed_output_grad.dims()[1] / groups;
|
||||
|
||||
bool is_expand = IsExpand(filter_shape_vec, strides, paddings_, dilations_);
|
||||
|
||||
DenseTensor col;
|
||||
// col_matrix shares the same piece of data with col,
|
||||
// but will be reshaped into a two-dimensional matrix shape
|
||||
// to call the matrix multiplication interface.
|
||||
DenseTensor col_matrix;
|
||||
if (is_expand) {
|
||||
col.Resize(col_shape);
|
||||
dev_ctx.template Alloc<T>(&col);
|
||||
col_matrix.ShareDataWith(col);
|
||||
col_matrix.Resize(col_matrix_shape);
|
||||
}
|
||||
|
||||
funcs::SetConstant<Context, T> set_zero;
|
||||
auto blas = funcs::GetBlas<Context, T>(dev_ctx);
|
||||
|
||||
if (input_grad) {
|
||||
dev_ctx.template Alloc<T>(input_grad);
|
||||
DenseTensor transformed_input_grad(input_grad->type());
|
||||
if (channel_last) {
|
||||
ResizeToChannelFirst<Context, T>(
|
||||
dev_ctx, input_grad, &transformed_input_grad);
|
||||
|
||||
} else {
|
||||
transformed_input_grad = *input_grad;
|
||||
}
|
||||
// if is_expand is false, the operation of set_zero is unnecessary,
|
||||
// because math::matmul will reset input_grad.
|
||||
if (is_expand) {
|
||||
set_zero(dev_ctx, &transformed_input_grad, static_cast<T>(0));
|
||||
}
|
||||
funcs::Col2ImFunctor<funcs::ColFormat::CFO, Context, T> col2im;
|
||||
funcs::Col2VolFunctor<Context, T> col2vol;
|
||||
|
||||
for (int64_t i = 0; i < batch_size; i++) {
|
||||
DenseTensor out_grad_batch =
|
||||
transformed_output_grad.Slice(i, i + 1).Resize(output_matrix_shape);
|
||||
DenseTensor in_grad_batch =
|
||||
transformed_input_grad.Slice(i, i + 1).Resize(input_shape);
|
||||
for (int g = 0; g < groups; g++) {
|
||||
// gemm
|
||||
DenseTensor out_grad_slice =
|
||||
out_grad_batch.Slice(g * out_step, (g + 1) * out_step);
|
||||
DenseTensor filter_slice =
|
||||
filter_.Slice(g * out_step, (g + 1) * out_step);
|
||||
|
||||
DenseTensor in_grad_slice =
|
||||
in_grad_batch.Slice(g * in_step, (g + 1) * in_step);
|
||||
|
||||
if (!is_expand) {
|
||||
col_matrix.ShareDataWith(in_grad_slice);
|
||||
col_matrix.Resize(col_matrix_shape);
|
||||
}
|
||||
blas.MatMul(filter_slice,
|
||||
true,
|
||||
out_grad_slice,
|
||||
false,
|
||||
T(1.0),
|
||||
&col_matrix,
|
||||
T(0.0));
|
||||
|
||||
if (is_expand && data_dim == 2U) {
|
||||
col2im(dev_ctx,
|
||||
col,
|
||||
dilations_,
|
||||
strides,
|
||||
std::vector<int>{
|
||||
paddings_[0], paddings_[2], paddings_[1], paddings_[3]},
|
||||
&in_grad_slice);
|
||||
} else if (is_expand && data_dim == 3U) {
|
||||
col2vol(dev_ctx, col, dilations_, strides, paddings_, &in_grad_slice);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (channel_last) {
|
||||
TransToChannelLast<Context, T>(
|
||||
dev_ctx, &transformed_input_grad, input_grad);
|
||||
}
|
||||
}
|
||||
|
||||
if (filter_grad) {
|
||||
dev_ctx.template Alloc<T>(filter_grad);
|
||||
DenseTensor filter_grad_ = *filter_grad;
|
||||
filter_grad_.Resize(filter_matrix_shape);
|
||||
set_zero(dev_ctx, filter_grad, static_cast<T>(0));
|
||||
funcs::Im2ColFunctor<funcs::ColFormat::CFO, Context, T> im2col;
|
||||
funcs::Vol2ColFunctor<Context, T> vol2col;
|
||||
for (int i = 0; i < batch_size; i++) {
|
||||
DenseTensor out_grad_batch =
|
||||
transformed_output_grad.Slice(i, i + 1).Resize(output_matrix_shape);
|
||||
DenseTensor in_batch =
|
||||
transformed_input.Slice(i, i + 1).Resize(input_shape);
|
||||
for (int g = 0; g < groups; g++) {
|
||||
// im2col
|
||||
DenseTensor out_grad_slice =
|
||||
out_grad_batch.Slice(g * out_step, (g + 1) * out_step);
|
||||
DenseTensor in_slice = in_batch.Slice(g * in_step, (g + 1) * in_step);
|
||||
|
||||
if (!is_expand) {
|
||||
col.ShareDataWith(in_slice);
|
||||
col_matrix.ShareDataWith(col);
|
||||
col_matrix.Resize(col_matrix_shape);
|
||||
} else if (data_dim == 2U) {
|
||||
im2col(dev_ctx,
|
||||
in_slice,
|
||||
dilations_,
|
||||
strides,
|
||||
std::vector<int>{
|
||||
paddings_[0], paddings_[2], paddings_[1], paddings_[3]},
|
||||
&col);
|
||||
|
||||
} else if (data_dim == 3U) {
|
||||
vol2col(dev_ctx, in_slice, dilations_, strides, paddings_, &col);
|
||||
}
|
||||
|
||||
// gemm
|
||||
DenseTensor filter_grad_slice =
|
||||
filter_grad_.Slice(g * out_step, (g + 1) * out_step);
|
||||
blas.MatMul(out_grad_slice,
|
||||
false,
|
||||
col_matrix,
|
||||
true,
|
||||
T(1.0),
|
||||
&filter_grad_slice,
|
||||
T(1.0));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ConvGradGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& input,
|
||||
const DenseTensor& filter,
|
||||
const DenseTensor& out_grad,
|
||||
const optional<DenseTensor>& input_grad_grad,
|
||||
const optional<DenseTensor>& filter_grad_grad,
|
||||
const std::vector<int>& strides,
|
||||
const std::vector<int>& paddings,
|
||||
const std::string& padding_algorithm,
|
||||
const std::vector<int>& dilations,
|
||||
int groups,
|
||||
const std::string& data_format,
|
||||
DenseTensor* input_grad,
|
||||
DenseTensor* filter_grad,
|
||||
DenseTensor* out_grad_grad) {
|
||||
const DenseTensor* X = &input;
|
||||
const DenseTensor* dY = &out_grad;
|
||||
const DenseTensor* ddX = input_grad_grad.get_ptr();
|
||||
const DenseTensor* ddW_in = filter_grad_grad.get_ptr();
|
||||
|
||||
DenseTensor* ddY = out_grad_grad;
|
||||
DenseTensor* dW = filter_grad;
|
||||
DenseTensor* dX = input_grad;
|
||||
DenseTensor W = filter;
|
||||
|
||||
if (!ddY && !dW && !dX) return;
|
||||
|
||||
std::vector<int> paddings_ = paddings;
|
||||
std::vector<int> dilations_ = dilations;
|
||||
|
||||
const bool channel_last = (data_format == "NHWC" || data_format == "NDHWC");
|
||||
|
||||
// transform Tensor
|
||||
DenseTensor transformed_X(X->type());
|
||||
DenseTensor transformed_dY(dY->type());
|
||||
DenseTensor transformed_ddX(X->type());
|
||||
|
||||
if (channel_last) {
|
||||
ResizeToChannelFirst<Context, T>(dev_ctx, X, &transformed_X);
|
||||
TransToChannelFirst<Context, T>(dev_ctx, X, &transformed_X);
|
||||
|
||||
ResizeToChannelFirst<Context, T>(dev_ctx, dY, &transformed_dY);
|
||||
TransToChannelFirst<Context, T>(dev_ctx, dY, &transformed_dY);
|
||||
|
||||
if (ddX) {
|
||||
ResizeToChannelFirst<Context, T>(dev_ctx, ddX, &transformed_ddX);
|
||||
TransToChannelFirst<Context, T>(dev_ctx, ddX, &transformed_ddX);
|
||||
}
|
||||
} else {
|
||||
transformed_X = *X;
|
||||
transformed_dY = *dY;
|
||||
if (ddX) {
|
||||
transformed_ddX = *ddX;
|
||||
}
|
||||
}
|
||||
|
||||
// update padding and dilation
|
||||
auto in_dims = transformed_X.dims();
|
||||
auto filter_dims = W.dims();
|
||||
|
||||
DDim in_data_dims = slice_ddim(in_dims, 2, in_dims.size());
|
||||
DDim filter_data_dims = slice_ddim(filter_dims, 2, filter_dims.size());
|
||||
std::vector<int> ksize = vectorize<int>(filter_data_dims);
|
||||
UpdatePaddingAndDilation(
|
||||
&paddings_, &dilations_, padding_algorithm, in_data_dims, strides, ksize);
|
||||
|
||||
const int64_t batch_size = transformed_X.dims()[0];
|
||||
std::vector<int64_t> filter_shape_vec(vectorize(W.dims()));
|
||||
std::vector<int64_t> output_shape_vec(vectorize(transformed_dY.dims()));
|
||||
|
||||
size_t data_dim = filter_shape_vec.size() - 2;
|
||||
std::vector<int64_t> col_shape_vec(1 + 2 * data_dim);
|
||||
// col_shape [in_channel/group, kh, kw, oh, ow]
|
||||
col_shape_vec[0] = transformed_X.dims()[1] / groups;
|
||||
for (size_t j = 0; j < data_dim; ++j) {
|
||||
col_shape_vec[j + 1] = filter_shape_vec[j + 2];
|
||||
col_shape_vec[j + data_dim + 1] = output_shape_vec[j + 2];
|
||||
}
|
||||
DDim col_shape(make_ddim(col_shape_vec));
|
||||
// col_matrix_shape [in_channel/group * kh * kw, oh * ow]
|
||||
DDim col_matrix_shape = flatten_to_2d(col_shape, data_dim + 1);
|
||||
// input_shape [Cin, H, W]
|
||||
DDim input_shape =
|
||||
slice_ddim(transformed_X.dims(), 1, transformed_X.dims().size());
|
||||
// filter_matrix_shape [Cout, Cin * kh * kw]
|
||||
DDim filter_matrix_shape = {W.dims()[0], W.numel() / W.dims()[0]};
|
||||
|
||||
W.Resize(filter_matrix_shape);
|
||||
DDim output_matrix_shape = {
|
||||
transformed_dY.dims()[1],
|
||||
transformed_dY.numel() /
|
||||
(transformed_dY.dims()[0] * transformed_dY.dims()[1])};
|
||||
int64_t in_step = transformed_X.dims()[1] / groups;
|
||||
int64_t out_step = transformed_dY.dims()[1] / groups;
|
||||
|
||||
bool is_expand = IsExpand(filter_shape_vec, strides, paddings_, dilations_);
|
||||
DenseTensor col;
|
||||
DenseTensor col_matrix;
|
||||
if (is_expand) {
|
||||
col.Resize(col_shape);
|
||||
dev_ctx.template Alloc<T>(&col);
|
||||
col_matrix.ShareDataWith(col);
|
||||
col_matrix.Resize(col_matrix_shape);
|
||||
}
|
||||
|
||||
funcs::SetConstant<Context, T> set_zero;
|
||||
auto blas = funcs::GetBlas<Context, T>(dev_ctx);
|
||||
|
||||
// dx convolution double grad: gemm + col2im(col2vol)
|
||||
// dx = ddw * dy ==> dx(N, Cin, H, W), ddw(Cout, Cin, kh, kw), dy(N, Cout,
|
||||
// oH, oW)
|
||||
if (dX && ddW_in) {
|
||||
DenseTensor ddW;
|
||||
ddW.ShareDataWith(*ddW_in).Resize(filter_matrix_shape);
|
||||
dev_ctx.template Alloc<T>(dX);
|
||||
|
||||
DenseTensor transformed_dX(dX->type());
|
||||
|
||||
if (channel_last) {
|
||||
ResizeToChannelFirst<Context, T>(dev_ctx, dX, &transformed_dX);
|
||||
|
||||
} else {
|
||||
transformed_dX = *dX;
|
||||
}
|
||||
// if is_expand is false, the operation of set_zero is unnecessary
|
||||
// because math::matmul will reset dx
|
||||
if (is_expand) {
|
||||
set_zero(dev_ctx, &transformed_dX, static_cast<T>(0));
|
||||
}
|
||||
funcs::Col2ImFunctor<funcs::ColFormat::CFO, Context, T> col2im;
|
||||
funcs::Col2VolFunctor<Context, T> col2vol;
|
||||
|
||||
for (int64_t i = 0; i < batch_size; i++) {
|
||||
DenseTensor dy_batch =
|
||||
transformed_dY.Slice(i, i + 1).Resize(output_matrix_shape);
|
||||
DenseTensor dx_batch = transformed_dX.Slice(i, i + 1).Resize(input_shape);
|
||||
for (int g = 0; g < groups; g++) {
|
||||
// gemm
|
||||
DenseTensor dy_slice = dy_batch.Slice(g * out_step, (g + 1) * out_step);
|
||||
DenseTensor ddw_slice = ddW.Slice(g * out_step, (g + 1) * out_step);
|
||||
DenseTensor dx_slice = dx_batch.Slice(g * in_step, (g + 1) * in_step);
|
||||
if (!is_expand) {
|
||||
col_matrix.ShareDataWith(dx_slice);
|
||||
col_matrix.Resize(col_matrix_shape);
|
||||
}
|
||||
blas.MatMul(
|
||||
ddw_slice, true, dy_slice, false, T(1.0), &col_matrix, T(0.0));
|
||||
|
||||
if (is_expand && data_dim == 2U) {
|
||||
col2im(dev_ctx,
|
||||
col,
|
||||
dilations_,
|
||||
strides,
|
||||
std::vector<int>{
|
||||
paddings_[0], paddings_[2], paddings_[1], paddings_[3]},
|
||||
&dx_slice);
|
||||
} else if (is_expand && data_dim == 3U) {
|
||||
col2vol(dev_ctx, col, dilations_, strides, paddings_, &dx_slice);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (channel_last) {
|
||||
TransToChannelLast<Context, T>(dev_ctx, &transformed_dX, dX);
|
||||
}
|
||||
}
|
||||
|
||||
// dw = ddx * dy ==> dw(Cout, Cin, kh, kw), ddx(N, Cin, H, W), dy(N, Cout,
|
||||
// oH, oW)
|
||||
// dw convolution double grad: im2col(vol2col) + gemm
|
||||
if (dW && ddX) {
|
||||
dev_ctx.template Alloc<T>(dW);
|
||||
set_zero(dev_ctx, dW, static_cast<T>(0));
|
||||
DenseTensor dW_arr = *dW;
|
||||
dW_arr.Resize(filter_matrix_shape);
|
||||
funcs::Im2ColFunctor<funcs::ColFormat::CFO, Context, T> im2col;
|
||||
funcs::Vol2ColFunctor<Context, T> vol2col;
|
||||
for (int i = 0; i < batch_size; ++i) {
|
||||
DenseTensor dy_batch =
|
||||
transformed_dY.Slice(i, i + 1).Resize(output_matrix_shape);
|
||||
DenseTensor ddx_batch =
|
||||
transformed_ddX.Slice(i, i + 1).Resize(input_shape);
|
||||
for (int g = 0; g < groups; ++g) {
|
||||
// im2col
|
||||
DenseTensor dy_slice = dy_batch.Slice(g * out_step, (g + 1) * out_step);
|
||||
DenseTensor ddx_slice = ddx_batch.Slice(g * in_step, (g + 1) * in_step);
|
||||
if (!is_expand) {
|
||||
col.ShareDataWith(ddx_slice);
|
||||
col_matrix.ShareDataWith(col);
|
||||
col_matrix.Resize(col_matrix_shape);
|
||||
} else if (data_dim == 2U) {
|
||||
im2col(dev_ctx,
|
||||
ddx_slice,
|
||||
dilations_,
|
||||
strides,
|
||||
std::vector<int>{
|
||||
paddings_[0], paddings_[2], paddings_[1], paddings_[3]},
|
||||
&col);
|
||||
} else if (data_dim == 3U) {
|
||||
vol2col(dev_ctx, ddx_slice, dilations_, strides, paddings_, &col);
|
||||
}
|
||||
|
||||
DenseTensor dw_slice = dW_arr.Slice(g * out_step, (g + 1) * out_step);
|
||||
blas.MatMul(
|
||||
dy_slice, false, col_matrix, true, T(1.0), &dw_slice, T(1.0));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ddy = w * ddx + x * ddw ==> ddy(N, Cout, oH, oW), x/ddx(N, Cin, H, W),
|
||||
// w/ddw(Cout, Cin, kh, kw)
|
||||
// ddy convolution double grad: im2col(vol2col) + gemm
|
||||
if (ddY) {
|
||||
dev_ctx.template Alloc<T>(ddY);
|
||||
|
||||
DenseTensor transformed_ddY(ddY->type());
|
||||
if (channel_last) {
|
||||
ResizeToChannelFirst<Context, T>(dev_ctx, ddY, &transformed_ddY);
|
||||
} else {
|
||||
transformed_ddY = *ddY;
|
||||
}
|
||||
|
||||
set_zero(dev_ctx, &transformed_ddY, static_cast<T>(0));
|
||||
funcs::Im2ColFunctor<funcs::ColFormat::CFO, Context, T> im2col;
|
||||
funcs::Vol2ColFunctor<Context, T> vol2col;
|
||||
for (int i = 0; i < batch_size; ++i) {
|
||||
DenseTensor ddy_batch =
|
||||
transformed_ddY.Slice(i, i + 1).Resize(output_matrix_shape);
|
||||
for (int g = 0; g < groups; ++g) {
|
||||
// gemm
|
||||
DenseTensor ddy_slice =
|
||||
ddy_batch.Slice(g * out_step, (g + 1) * out_step);
|
||||
|
||||
if (ddX) {
|
||||
DenseTensor ddx_batch =
|
||||
transformed_ddX.Slice(i, i + 1).Resize(input_shape);
|
||||
DenseTensor ddx_slice =
|
||||
ddx_batch.Slice(g * in_step, (g + 1) * in_step);
|
||||
if (!is_expand) {
|
||||
col.ShareDataWith(ddx_slice);
|
||||
col_matrix.ShareDataWith(col);
|
||||
col_matrix.Resize(col_matrix_shape);
|
||||
} else if (data_dim == 2U) {
|
||||
im2col(dev_ctx,
|
||||
ddx_slice,
|
||||
dilations_,
|
||||
strides,
|
||||
std::vector<int>{
|
||||
paddings_[0], paddings_[2], paddings_[1], paddings_[3]},
|
||||
&col);
|
||||
} else if (data_dim == 3U) {
|
||||
vol2col(dev_ctx, ddx_slice, dilations_, strides, paddings_, &col);
|
||||
}
|
||||
DenseTensor w_slice = W.Slice(g * out_step, (g + 1) * out_step);
|
||||
blas.MatMul(
|
||||
w_slice, false, col_matrix, false, T(1.0), &ddy_slice, T(0.0));
|
||||
}
|
||||
|
||||
if (ddW_in) {
|
||||
DenseTensor x_batch =
|
||||
transformed_X.Slice(i, i + 1).Resize(input_shape);
|
||||
DenseTensor x_slice = x_batch.Slice(g * in_step, (g + 1) * in_step);
|
||||
|
||||
DenseTensor ddW;
|
||||
ddW.ShareDataWith(*ddW_in).Resize(filter_matrix_shape);
|
||||
if (!is_expand) {
|
||||
col.ShareDataWith(x_slice);
|
||||
col_matrix.ShareDataWith(col);
|
||||
col_matrix.Resize(col_matrix_shape);
|
||||
} else if (data_dim == 2U) {
|
||||
im2col(dev_ctx,
|
||||
x_slice,
|
||||
dilations_,
|
||||
strides,
|
||||
std::vector<int>{
|
||||
paddings_[0], paddings_[2], paddings_[1], paddings_[3]},
|
||||
&col);
|
||||
} else if (data_dim == 3U) {
|
||||
vol2col(dev_ctx, x_slice, dilations_, strides, paddings_, &col);
|
||||
}
|
||||
|
||||
// gemm
|
||||
DenseTensor ddw_slice = ddW.Slice(g * out_step, (g + 1) * out_step);
|
||||
blas.MatMul(
|
||||
ddw_slice, false, col_matrix, false, T(1.0), &ddy_slice, T(1.0));
|
||||
}
|
||||
}
|
||||
}
|
||||
if (channel_last) {
|
||||
TransToChannelLast<Context, T>(dev_ctx, &transformed_ddY, ddY);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,184 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/conv_kernel.h"
|
||||
#include "paddle/phi/kernels/cpu/conv_util.h"
|
||||
#include "paddle/phi/kernels/full_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/batch_norm_utils.h"
|
||||
#include "paddle/phi/kernels/funcs/blas/blas.h"
|
||||
#include "paddle/phi/kernels/funcs/im2col.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
#include "paddle/phi/kernels/funcs/vol2col.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ConvKernelImpl(const Context& dev_ctx,
|
||||
const DenseTensor& input,
|
||||
const DenseTensor& filter,
|
||||
const std::vector<int>& strides,
|
||||
const std::vector<int>& paddings,
|
||||
const std::string& padding_algorithm,
|
||||
int groups,
|
||||
const std::vector<int>& dilations,
|
||||
const std::string& data_format,
|
||||
DenseTensor* output) {
|
||||
std::vector<int> paddings_ = paddings;
|
||||
std::vector<int> dilations_ = dilations;
|
||||
DenseTensor filter_ = filter;
|
||||
if (input.numel() == 0 || filter.numel() == 0) {
|
||||
Full<T, Context>(dev_ctx, output->dims(), 0, output);
|
||||
return;
|
||||
}
|
||||
// The filter will be reshaped in the calculations,
|
||||
// so here use an assignment operation,
|
||||
// that avoids modifying the variable in the Scope.
|
||||
dev_ctx.template Alloc<T>(output);
|
||||
|
||||
const bool channel_last = (data_format == "NHWC" || data_format == "NDHWC");
|
||||
|
||||
DenseTensor transformed_input(input.type());
|
||||
DenseTensor transformed_output(output->type());
|
||||
|
||||
if (channel_last) {
|
||||
ResizeToChannelFirst<Context, T>(dev_ctx, &input, &transformed_input);
|
||||
TransToChannelFirst<Context, T>(dev_ctx, &input, &transformed_input);
|
||||
|
||||
ResizeToChannelFirst<Context, T>(dev_ctx, output, &transformed_output);
|
||||
|
||||
} else {
|
||||
transformed_input = input;
|
||||
transformed_output = *output;
|
||||
}
|
||||
|
||||
// update padding and dilation
|
||||
auto trans_in_dims = transformed_input.dims();
|
||||
auto filter_dims = filter.dims();
|
||||
|
||||
DDim in_data_dims = slice_ddim(trans_in_dims, 2, trans_in_dims.size());
|
||||
DDim filter_data_dims = slice_ddim(filter_dims, 2, filter_dims.size());
|
||||
|
||||
std::vector<int> ksize = vectorize<int>(filter_data_dims);
|
||||
UpdatePaddingAndDilation(
|
||||
&paddings_, &dilations_, padding_algorithm, in_data_dims, strides, ksize);
|
||||
|
||||
const int64_t batch_size = transformed_input.dims()[0];
|
||||
|
||||
// filter_shape_vec:
|
||||
// {k_o, k_i, k_h, k_w} or {k_o, k_i, k_d, k_h, k_w}
|
||||
std::vector<int64_t> filter_shape_vec(vectorize(filter.dims()));
|
||||
|
||||
// output_shape_vec:
|
||||
// {o_n, o_c, o_h, o_w} or {o_n, o_c, o_d, o_h, o_w}
|
||||
std::vector<int64_t> output_shape_vec(vectorize(transformed_output.dims()));
|
||||
|
||||
// use col_shape in the im2col calculation
|
||||
// col_shape_vec:
|
||||
// {i_c/g, k_h, k_w, o_h, o_w} or {i_c/g, k_d, k_h, k_w,
|
||||
// o_d,o_h, o_w}
|
||||
size_t data_dim = filter_shape_vec.size() - 2;
|
||||
|
||||
std::vector<int64_t> col_shape_vec(1 + 2 * data_dim);
|
||||
col_shape_vec[0] = trans_in_dims[1] / groups;
|
||||
for (size_t j = 0; j < data_dim; ++j) {
|
||||
col_shape_vec[j + 1] = filter_shape_vec[j + 2];
|
||||
col_shape_vec[j + 1 + data_dim] = output_shape_vec[j + 2];
|
||||
}
|
||||
|
||||
DDim col_shape(make_ddim(col_shape_vec));
|
||||
|
||||
// use col_matrix_shape in the gemm calculation
|
||||
// size:
|
||||
// (i_c/g * k_h * k_w, o_h * o_w) or (i_c/g * k_d * k_h * k_w, o_d * o_h *
|
||||
// o_w)
|
||||
|
||||
DDim col_matrix_shape = flatten_to_2d(col_shape, data_dim);
|
||||
|
||||
bool is_expand = IsExpand(filter_shape_vec, strides, paddings_, dilations_);
|
||||
|
||||
DenseTensor col;
|
||||
// col_matrix shares the same piece of data with col,
|
||||
// but will be reshaped into a two-dimensional matrix shape
|
||||
// to call the matrix multiplication interface.
|
||||
DenseTensor col_matrix;
|
||||
if (is_expand) {
|
||||
// col = dev_ctx.AllocateTmpTensor<T, Context>(col_shape, dev_ctx);
|
||||
col.Resize(col_shape);
|
||||
dev_ctx.template Alloc<T>(&col);
|
||||
col_matrix.ShareDataWith(col);
|
||||
col_matrix.Resize(col_matrix_shape);
|
||||
}
|
||||
|
||||
DDim in_matrix_shape =
|
||||
slice_ddim(transformed_input.dims(), 1, transformed_input.dims().size());
|
||||
|
||||
DDim filter_matrix_shape = {filter.dims()[0],
|
||||
filter.numel() / filter.dims()[0]};
|
||||
filter_.Resize(filter_matrix_shape);
|
||||
|
||||
DDim output_matrix_shape = {
|
||||
transformed_output.dims()[1],
|
||||
transformed_output.numel() /
|
||||
(transformed_output.dims()[0] * transformed_output.dims()[1])};
|
||||
|
||||
// convolution operator: im2col(or vol2col) + gemm
|
||||
int64_t in_step = transformed_input.dims()[1] / groups;
|
||||
int64_t out_step = transformed_output.dims()[1] / groups;
|
||||
|
||||
funcs::Im2ColFunctor<funcs::ColFormat::CFO, Context, T> im2col;
|
||||
funcs::Vol2ColFunctor<Context, T> vol2col;
|
||||
|
||||
auto blas = funcs::GetBlas<Context, T>(dev_ctx);
|
||||
for (int64_t i = 0; i < batch_size; i++) {
|
||||
DenseTensor in_batch =
|
||||
transformed_input.Slice(i, i + 1).Resize(in_matrix_shape);
|
||||
DenseTensor out_batch =
|
||||
transformed_output.Slice(i, i + 1).Resize(output_matrix_shape);
|
||||
|
||||
for (int g = 0; g < groups; g++) {
|
||||
DenseTensor in_slice = in_batch.Slice(g * in_step, (g + 1) * in_step);
|
||||
|
||||
if (!is_expand) {
|
||||
col.ShareDataWith(in_slice);
|
||||
col_matrix.ShareDataWith(col);
|
||||
col_matrix.Resize(col_matrix_shape);
|
||||
} else if (data_dim == 2U) {
|
||||
im2col(dev_ctx,
|
||||
in_slice,
|
||||
dilations_,
|
||||
strides,
|
||||
std::vector<int>{
|
||||
paddings_[0], paddings_[2], paddings_[1], paddings_[3]},
|
||||
&col);
|
||||
|
||||
} else if (data_dim == 3U) {
|
||||
vol2col(dev_ctx, in_slice, dilations_, strides, paddings_, &col);
|
||||
}
|
||||
|
||||
// gemm
|
||||
DenseTensor out_slice = out_batch.Slice(g * out_step, (g + 1) * out_step);
|
||||
DenseTensor filter_slice =
|
||||
filter_.Slice(g * out_step, (g + 1) * out_step);
|
||||
blas.MatMul(
|
||||
filter_slice, false, col_matrix, false, T(1.0), &out_slice, T(0.0));
|
||||
}
|
||||
}
|
||||
if (channel_last) {
|
||||
TransToChannelLast<Context, T>(dev_ctx, &transformed_output, output);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,387 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/common/ddim.h"
|
||||
#include "paddle/common/layout.h"
|
||||
#include "paddle/phi/kernels/conv_transpose_grad_kernel.h"
|
||||
#include "paddle/phi/kernels/cpu/conv_util.h"
|
||||
#include "paddle/phi/kernels/full_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/blas/blas.h"
|
||||
#include "paddle/phi/kernels/funcs/concat_and_split_functor.h"
|
||||
#include "paddle/phi/kernels/funcs/im2col.h"
|
||||
#include "paddle/phi/kernels/funcs/slice.h"
|
||||
#include "paddle/phi/kernels/funcs/vol2col.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ConvTransposeGradRawKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& filter,
|
||||
const DenseTensor& dout,
|
||||
const std::vector<int>& strides,
|
||||
const std::vector<int>& paddings,
|
||||
const std::string& padding_algorithm,
|
||||
int groups,
|
||||
const std::vector<int>& dilations,
|
||||
const std::string& data_format,
|
||||
DenseTensor* dx,
|
||||
DenseTensor* dfilter) {
|
||||
const DataLayout data_layout = StringToDataLayout(data_format);
|
||||
// For filter, we do not use const pointer because we will do reshape,
|
||||
// but we should avoid modifying its value.
|
||||
DenseTensor filter_ = filter;
|
||||
|
||||
if ((!dx) && (!dfilter)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// 0-size
|
||||
if (x.numel() == 0) {
|
||||
if (dx) dev_ctx.template Alloc<T>(dx);
|
||||
if (dfilter) {
|
||||
Full<T, Context>(dev_ctx, dfilter->dims(), 0, dfilter);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (filter.numel() == 0) {
|
||||
if (dfilter) dev_ctx.template Alloc<T>(dfilter);
|
||||
if (dx) {
|
||||
Full<T, Context>(dev_ctx, dx->dims(), 0, dx);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
std::vector<int> paddings_ = paddings;
|
||||
std::vector<int> dilations_ = dilations;
|
||||
|
||||
auto x_dims = x.dims();
|
||||
auto filter_dims = filter_.dims();
|
||||
auto dout_dims = dout.dims();
|
||||
const int batch_size = static_cast<int>(x.dims()[0]);
|
||||
|
||||
DDim in_data_dims;
|
||||
if (data_layout != DataLayout::NHWC) {
|
||||
in_data_dims = slice_ddim(x_dims, 2, x_dims.size());
|
||||
} else {
|
||||
in_data_dims = slice_ddim(x_dims, 1, x_dims.size() - 1);
|
||||
}
|
||||
DDim filter_data_dims = slice_ddim(filter_dims, 2, filter_dims.size());
|
||||
std::vector<int> ksize = vectorize<int>(filter_data_dims);
|
||||
UpdatePaddingAndDilation(
|
||||
&paddings_, &dilations_, padding_algorithm, in_data_dims, strides, ksize);
|
||||
|
||||
// x_shape_vec: {n, c, h, w} or {n, c, d, h, w} for channel_first
|
||||
// x_shape_vec: {n, h, w, c} or {n, d, h, w, c} for channel_last
|
||||
std::vector<int64_t> x_shape_vec = vectorize(x.dims());
|
||||
// filter_shape_vec: {i_c, o_c, k_h, k_w} or {i_c, o_c, k_d, k_h, k_w}
|
||||
std::vector<int64_t> filter_shape_vec = vectorize(filter_.dims());
|
||||
|
||||
// use col_shape in the im2col and col2im (or vol2col and col2vol)
|
||||
// calculation
|
||||
// col_shape_vec: {o_c, k_h, k_w, h, w} or {o_c, k_d, k_h, k_w, d, h, w} for
|
||||
size_t data_dim = filter_shape_vec.size() - 2;
|
||||
std::vector<int64_t> col_shape_vec(1 + 2 * data_dim);
|
||||
if (data_layout != DataLayout::NHWC) {
|
||||
col_shape_vec[0] = dout_dims[1];
|
||||
for (size_t j = 0; j < data_dim; ++j) {
|
||||
col_shape_vec[j + 1] = filter_shape_vec[j + 2];
|
||||
col_shape_vec[j + 1 + data_dim] = x_shape_vec[j + 2];
|
||||
}
|
||||
} else {
|
||||
col_shape_vec[0] = dout_dims[dout_dims.size() - 1];
|
||||
for (size_t j = 0; j < data_dim; ++j) {
|
||||
col_shape_vec[j + 1] = filter_shape_vec[j + 2];
|
||||
col_shape_vec[j + 1 + data_dim] = x_shape_vec[j + 1];
|
||||
}
|
||||
}
|
||||
DDim col_shape(make_ddim(col_shape_vec));
|
||||
|
||||
// use col_matrix_shape in the gemm calculation
|
||||
// size: (o_c * k_h * k_w, h * w) or (o_c * k_d * k_h * k_w, d * h * w)
|
||||
DDim col_matrix_shape = flatten_to_2d(col_shape, data_dim + 1);
|
||||
|
||||
// output size: (o_c, o_h, o_w) or (o_c, o_d, o_h, o_w) for channel_first
|
||||
// output size: (o_h, o_w, o_c) or (o_d, o_h, o_w, o_c) for channel_last
|
||||
DDim output_shape = slice_ddim(dout.dims(), 1, dout.dims().size());
|
||||
|
||||
// x matrix size: (i_c, h * w) or (i_c, d * h * w) for channel_first
|
||||
// x matrix size: (h * w, i_c) or (d * h * w, i_c) for channel_last
|
||||
DDim x_matrix_shape;
|
||||
if (data_layout != DataLayout::NHWC) {
|
||||
x_matrix_shape = {x_dims[1], col_matrix_shape[1]};
|
||||
} else {
|
||||
x_matrix_shape = {col_matrix_shape[1], x_dims[x_dims.size() - 1]};
|
||||
}
|
||||
|
||||
// filter size: (i_c, o_c/g * k_h * k_w) or (i_c, o_c/g * k_d * k_h * k_w)
|
||||
DDim filter_matrix_shape;
|
||||
if (data_layout != DataLayout::NHWC) {
|
||||
filter_matrix_shape = {x_dims[1], col_matrix_shape[0] / groups};
|
||||
} else {
|
||||
filter_matrix_shape = {x_dims[x_dims.size() - 1],
|
||||
col_matrix_shape[0] / groups};
|
||||
}
|
||||
filter_.Resize(filter_matrix_shape);
|
||||
|
||||
int in_step = (data_layout != DataLayout::NHWC
|
||||
? static_cast<int>(x_dims[1]) / groups
|
||||
: static_cast<int>(x_dims[x_dims.size() - 1]) / groups);
|
||||
int col_step = static_cast<int>(col_matrix_shape[0]) / groups;
|
||||
|
||||
// convolution transpose grad on x:
|
||||
// im2col + gemm (similar to conv-forward)
|
||||
// x need to compute gradient
|
||||
auto blas = funcs::GetBlas<Context, T>(dev_ctx);
|
||||
if (dx || dfilter) {
|
||||
DenseTensor col;
|
||||
col.Resize(col_shape);
|
||||
dev_ctx.template Alloc<T>(&col);
|
||||
// col_matrix shares the same piece of data with col,
|
||||
// but will be reshaped into a two-dimensional matrix shape
|
||||
// to call the matrix multiplication interface.
|
||||
DenseTensor col_matrix;
|
||||
col_matrix.ShareDataWith(col);
|
||||
col_matrix.Resize(col_matrix_shape);
|
||||
|
||||
DenseTensor dfilter_;
|
||||
funcs::SetConstant<Context, T> set_zero;
|
||||
|
||||
funcs::Im2ColFunctor<funcs::ColFormat::CFO, Context, T> im2col;
|
||||
funcs::Vol2ColFunctor<Context, T> vol2col;
|
||||
funcs::ConcatFunctor<Context, T> concat_functor;
|
||||
|
||||
if (dx) {
|
||||
dev_ctx.template Alloc<T>(dx);
|
||||
set_zero(dev_ctx, dx, static_cast<T>(0));
|
||||
}
|
||||
if (dfilter) { // dfilter_ size (i_c, o_c/g, k_h, k_w)
|
||||
dev_ctx.template Alloc<T>(dfilter);
|
||||
set_zero(dev_ctx, dfilter, static_cast<T>(0));
|
||||
dfilter_ = *dfilter;
|
||||
dfilter_.Resize(filter_matrix_shape);
|
||||
}
|
||||
|
||||
size_t D = x.dims().size();
|
||||
for (int i = 0; i < batch_size; i++) {
|
||||
// batch with size (o_c, o_h, o_w) or (o_c, o_d, o_h, o_w) for
|
||||
// channel_first
|
||||
// batch with size (o_h, o_w, o_c) or (o_d, o_h, o_w, o_c) for
|
||||
// channel_last
|
||||
DenseTensor dout_batch = dout.Slice(i, i + 1).Resize(output_shape);
|
||||
|
||||
if (data_dim == 2U) {
|
||||
// im2col: dy -> col matrix
|
||||
// from (o_c, o_h, o_w) to (o_c * k_h * k_w, i_h * i_w) for
|
||||
// channel_first
|
||||
// from (o_h, o_w, o_c) to (o_c * k_h * k_w, i_h * i_w) for
|
||||
// channel_last
|
||||
im2col(dev_ctx,
|
||||
dout_batch,
|
||||
dilations_,
|
||||
strides,
|
||||
std::vector<int>{
|
||||
paddings_[0], paddings_[2], paddings_[1], paddings_[3]},
|
||||
&col,
|
||||
data_layout);
|
||||
} else if (data_dim == 3U) {
|
||||
// vol2col: dy -> col_matrix
|
||||
// from (o_c, o_d, o_h, o_w) to (o_c * k_d * k_h * k_w, i_d * i_h *
|
||||
// i_w) for channel_first
|
||||
// from (o_d, o_h, o_w, o_c) to (i_d * i_h * i_w, o_c * k_d * k_h *
|
||||
// k_w) for channel_last
|
||||
vol2col(dev_ctx,
|
||||
dout_batch,
|
||||
dilations_,
|
||||
strides,
|
||||
paddings_,
|
||||
&col,
|
||||
data_layout);
|
||||
}
|
||||
if (dx) {
|
||||
// batch with size (i_c, i_h, i_w) or (i_h, i_w, i_c)
|
||||
DenseTensor dx_batch = dx->Slice(i, i + 1).Resize(x_matrix_shape);
|
||||
|
||||
// gemm: dx = filter * dy
|
||||
// (i_c, o_c * k_h * k_w) * (o_c * k_h * k_w, i_h * i_w) -> (i_c, i_h
|
||||
// * i_w)
|
||||
// or
|
||||
// (i_c, o_c * k_d * k_h * k_w) * (o_c * k_d * k_h * k_w, i_d * i_h *
|
||||
// i_w) -> (i_c,
|
||||
// i_d, i_h, i_w)
|
||||
// gemm: dx = dy^T * filter^T for channel_last
|
||||
|
||||
std::vector<DenseTensor> dx_batch_vec;
|
||||
for (int g = 0; g < groups; g++) {
|
||||
// dx_slice: (i_c/g, i_h * i_w) or (i_c/g, i_d * i_h * i_w)
|
||||
// for channel_first
|
||||
// dx_slice: (i_h * i_w, i_c/g) or (i_d * i_h * i_w, i_c/g)
|
||||
// for channel_last
|
||||
// filter_slice: (i_c/g, o_c/g * k_h * k_w)
|
||||
DenseTensor filter_slice =
|
||||
filter_.Slice(g * in_step, (g + 1) * in_step);
|
||||
// col_matrix_slice: (o_c/g * k_h * k_w, h * w) or (o_c/g * k_d *
|
||||
// k_h * k_w, d * h * w)
|
||||
DenseTensor col_matrix_slice =
|
||||
col_matrix.Slice(g * col_step, (g + 1) * col_step);
|
||||
if (data_layout != DataLayout::NHWC) {
|
||||
DenseTensor dx_slice =
|
||||
dx_batch.Slice(g * in_step, (g + 1) * in_step);
|
||||
blas.MatMul(filter_slice,
|
||||
false,
|
||||
col_matrix_slice,
|
||||
false,
|
||||
static_cast<T>(1.0),
|
||||
&dx_slice,
|
||||
static_cast<T>(0.0));
|
||||
} else {
|
||||
DenseTensor dx_slice;
|
||||
funcs::Slice<Context, T, 2>(dev_ctx,
|
||||
&dx_batch,
|
||||
&dx_slice,
|
||||
g * in_step,
|
||||
(g + 1) * in_step,
|
||||
1);
|
||||
blas.MatMul(col_matrix_slice,
|
||||
true,
|
||||
filter_slice,
|
||||
true,
|
||||
static_cast<T>(1.0),
|
||||
&dx_slice,
|
||||
static_cast<T>(0.0));
|
||||
DDim dx_slice_shape;
|
||||
if (data_dim == 2U) {
|
||||
dx_slice_shape = {x_dims[1], x_dims[2], in_step};
|
||||
} else {
|
||||
dx_slice_shape = {x_dims[1], x_dims[2], x_dims[3], in_step};
|
||||
}
|
||||
dx_slice = dx_slice.Resize(dx_slice_shape);
|
||||
dx_batch_vec.push_back(dx_slice);
|
||||
}
|
||||
}
|
||||
if (data_layout == DataLayout::NHWC) {
|
||||
concat_functor(
|
||||
dev_ctx, dx_batch_vec, static_cast<int>(D - 2), &dx_batch);
|
||||
}
|
||||
}
|
||||
if (dfilter) {
|
||||
// x batch: (i_c, i_h * i_w) or (i_h, i_w * i_c)
|
||||
DenseTensor in_batch = x.Slice(i, i + 1).Resize(x_matrix_shape);
|
||||
// gemm: d_filter = x * dy^T
|
||||
// (i_c, i_h * i_w) * (i_h * i_w, o_c * k_h * k_w) -> (i_c, o_c * k_h
|
||||
// * k_w)
|
||||
// or
|
||||
// (i_c, i_d * i_h * i_w) * (i_d * i_h * i_w, o_c * k_d * k_h * k_w)
|
||||
// -> (i_c, o_c * k_d *
|
||||
// k_h * k_w)
|
||||
// gemm: d_filter = x^T * dy^T for channel_last
|
||||
|
||||
for (int g = 0; g < groups; g++) {
|
||||
DenseTensor dfilter_slice =
|
||||
dfilter_.Slice(g * in_step, (g + 1) * in_step);
|
||||
DenseTensor col_matrix_slice =
|
||||
col_matrix.Slice(g * col_step, (g + 1) * col_step);
|
||||
if (data_layout != DataLayout::NHWC) {
|
||||
DenseTensor in_batch_slice =
|
||||
in_batch.Slice(g * in_step, (g + 1) * in_step);
|
||||
blas.MatMul(in_batch_slice,
|
||||
false,
|
||||
col_matrix_slice,
|
||||
true,
|
||||
static_cast<T>(1.0),
|
||||
&dfilter_slice,
|
||||
static_cast<T>(1.0));
|
||||
} else {
|
||||
DenseTensor in_batch_slice;
|
||||
funcs::Slice<Context, T, 2>(dev_ctx,
|
||||
&in_batch,
|
||||
&in_batch_slice,
|
||||
g * in_step,
|
||||
(g + 1) * in_step,
|
||||
1);
|
||||
blas.MatMul(in_batch_slice,
|
||||
true,
|
||||
col_matrix_slice,
|
||||
true,
|
||||
static_cast<T>(1.0),
|
||||
&dfilter_slice,
|
||||
static_cast<T>(1.0));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void Conv2dTransposeGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& filter,
|
||||
const DenseTensor& dout,
|
||||
const std::vector<int>& strides,
|
||||
const std::vector<int>& paddings,
|
||||
const std::vector<int>& output_padding UNUSED,
|
||||
const IntArray& output_size UNUSED,
|
||||
const std::string& padding_algorithm,
|
||||
int groups,
|
||||
const std::vector<int>& dilations,
|
||||
const std::string& data_format,
|
||||
DenseTensor* dx,
|
||||
DenseTensor* dfilter) {
|
||||
ConvTransposeGradRawKernel<T, Context>(dev_ctx,
|
||||
x,
|
||||
filter,
|
||||
dout,
|
||||
strides,
|
||||
paddings,
|
||||
padding_algorithm,
|
||||
groups,
|
||||
dilations,
|
||||
data_format,
|
||||
dx,
|
||||
dfilter);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void Conv3dTransposeGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& filter,
|
||||
const DenseTensor& dout,
|
||||
const std::vector<int>& strides,
|
||||
const std::vector<int>& paddings,
|
||||
const std::vector<int>& output_padding UNUSED,
|
||||
const std::vector<int>& output_size UNUSED,
|
||||
const std::string& padding_algorithm,
|
||||
int groups,
|
||||
const std::vector<int>& dilations,
|
||||
const std::string& data_format,
|
||||
DenseTensor* dx,
|
||||
DenseTensor* dfilter) {
|
||||
ConvTransposeGradRawKernel<T, Context>(dev_ctx,
|
||||
x,
|
||||
filter,
|
||||
dout,
|
||||
strides,
|
||||
paddings,
|
||||
padding_algorithm,
|
||||
groups,
|
||||
dilations,
|
||||
data_format,
|
||||
dx,
|
||||
dfilter);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,286 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/common/ddim.h"
|
||||
#include "paddle/common/layout.h"
|
||||
#include "paddle/phi/kernels/conv_transpose_kernel.h"
|
||||
#include "paddle/phi/kernels/cpu/conv_util.h"
|
||||
#include "paddle/phi/kernels/full_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/blas/blas.h"
|
||||
#include "paddle/phi/kernels/funcs/concat_and_split_functor.h"
|
||||
#include "paddle/phi/kernels/funcs/im2col.h"
|
||||
#include "paddle/phi/kernels/funcs/slice.h"
|
||||
#include "paddle/phi/kernels/funcs/vol2col.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ConvTransposeRawKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& filter,
|
||||
const std::vector<int>& strides,
|
||||
const std::vector<int>& paddings,
|
||||
const std::string& padding_algorithm,
|
||||
int groups,
|
||||
const std::vector<int>& dilations,
|
||||
const std::string& data_format,
|
||||
DenseTensor* out) {
|
||||
if (x.numel() == 0 || filter.numel() == 0) {
|
||||
Full<T, Context>(dev_ctx, out->dims(), 0, out);
|
||||
return;
|
||||
}
|
||||
const DataLayout data_layout = StringToDataLayout(data_format);
|
||||
// The filter will be reshaped, so it should not be constant
|
||||
DenseTensor filter_ = filter;
|
||||
std::vector<int> paddings_ = paddings;
|
||||
std::vector<int> dilations_ = dilations;
|
||||
|
||||
auto x_dims = x.dims();
|
||||
auto filter_dims = filter_.dims();
|
||||
auto out_dims = out->dims();
|
||||
const int batch_size = static_cast<int>(x.dims()[0]);
|
||||
|
||||
DDim in_data_dims;
|
||||
if (data_layout != DataLayout::NHWC) {
|
||||
in_data_dims = slice_ddim(x_dims, 2, x_dims.size());
|
||||
} else {
|
||||
in_data_dims = slice_ddim(x_dims, 1, x_dims.size() - 1);
|
||||
}
|
||||
DDim filter_data_dims = slice_ddim(filter_dims, 2, filter_dims.size());
|
||||
std::vector<int> ksize = vectorize<int>(filter_data_dims);
|
||||
UpdatePaddingAndDilation(
|
||||
&paddings_, &dilations_, padding_algorithm, in_data_dims, strides, ksize);
|
||||
|
||||
// x_shape_vec: {n, c, h, w} or {n, c, d, h, w} for channel_first
|
||||
// x_shape_vec: {n, h, w, c} or {n, d, h, w, c} for channel_last
|
||||
std::vector<int64_t> x_shape_vec = vectorize(x.dims());
|
||||
// filter_shape_vec: {k_o, k_i, k_h, k_w} or {k_o, k_i, k_d, k_h, k_w}
|
||||
std::vector<int64_t> filter_shape_vec = vectorize(filter_.dims());
|
||||
|
||||
// use col_shape in the im2col and col2im (or vol2col and col2vol)
|
||||
// calculation
|
||||
// col_shape_vec: {o_c/g, k_h, k_w, h, w} or {o_c/g, k_d, k_h, k_w, d, h, w}
|
||||
size_t data_dim = filter_shape_vec.size() - 2;
|
||||
std::vector<int64_t> col_shape_vec(1 + 2 * data_dim);
|
||||
if (data_layout != DataLayout::NHWC) {
|
||||
col_shape_vec[0] = out_dims[1] / groups;
|
||||
for (size_t j = 0; j < data_dim; ++j) {
|
||||
col_shape_vec[j + 1] = filter_shape_vec[j + 2];
|
||||
col_shape_vec[j + 1 + data_dim] = x_shape_vec[j + 2];
|
||||
}
|
||||
} else {
|
||||
col_shape_vec[0] = out_dims[out_dims.size() - 1] / groups;
|
||||
for (size_t j = 0; j < data_dim; ++j) {
|
||||
col_shape_vec[j + 1] = filter_shape_vec[j + 2];
|
||||
col_shape_vec[j + 1 + data_dim] = x_shape_vec[j + 1];
|
||||
}
|
||||
}
|
||||
DDim col_shape(make_ddim(col_shape_vec));
|
||||
|
||||
// use col_matrix_shape in the gemm calculation
|
||||
// size: (o_c/g * k_h * k_w, h * w) or (o_c/g * k_d * k_h * k_w, d * h * w)
|
||||
DDim col_matrix_shape = flatten_to_2d(col_shape, data_dim + 1);
|
||||
|
||||
DenseTensor col;
|
||||
col.Resize(col_shape);
|
||||
dev_ctx.template Alloc<T>(&col);
|
||||
// col_matrix shares the same piece of data with col,
|
||||
// but will be reshaped into a two-dimensional matrix shape
|
||||
// to call the matrix multiplication interface.
|
||||
DenseTensor col_matrix;
|
||||
col_matrix.ShareDataWith(col);
|
||||
col_matrix.Resize(col_matrix_shape);
|
||||
|
||||
// out size: (o_c, o_h, o_w) or (o_c, o_d, o_h, o_w) for channel_first
|
||||
// out size: (o_h, o_w, o_c) or (o_d, o_h, o_w, o_c) for channel_last
|
||||
DDim out_shape = slice_ddim(out->dims(), 1, out->dims().size());
|
||||
|
||||
// x matrix size: (i_c, h * w) or (i_c, d * h * w) for channel_first
|
||||
// x matrix size: (h * w, i_c) or (d * h * w, i_c) for channel_last
|
||||
DDim x_matrix_shape;
|
||||
if (data_layout != DataLayout::NHWC) {
|
||||
x_matrix_shape = {x_dims[1], col_matrix_shape[1]};
|
||||
} else {
|
||||
x_matrix_shape = {col_matrix_shape[1], x_dims[x_dims.size() - 1]};
|
||||
}
|
||||
|
||||
// filter size: (i_c, o_c/g * k_h * k_w) or (i_c, o_c/g * k_d * k_h * k_w)
|
||||
DDim filter_matrix_shape;
|
||||
if (data_layout != DataLayout::NHWC) {
|
||||
filter_matrix_shape = {x_dims[1], col_matrix_shape[0]};
|
||||
} else {
|
||||
filter_matrix_shape = {x_dims[x_dims.size() - 1], col_matrix_shape[0]};
|
||||
}
|
||||
filter_.Resize(filter_matrix_shape);
|
||||
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
|
||||
funcs::SetConstant<Context, T> set_zero;
|
||||
|
||||
auto blas = funcs::GetBlas<Context, T>(dev_ctx);
|
||||
set_zero(dev_ctx, out, static_cast<T>(0));
|
||||
|
||||
int in_step = (data_layout != DataLayout::NHWC
|
||||
? static_cast<int>(x_dims[1]) / groups
|
||||
: static_cast<int>(x_dims[x_dims.size() - 1]) / groups);
|
||||
|
||||
int out_step =
|
||||
(data_layout != DataLayout::NHWC
|
||||
? static_cast<int>(out_dims[1]) / groups
|
||||
: static_cast<int>(out_dims[out_dims.size() - 1]) / groups);
|
||||
funcs::Col2ImFunctor<funcs::ColFormat::CFO, Context, T> col2im;
|
||||
funcs::Col2VolFunctor<Context, T> col2vol;
|
||||
funcs::ConcatFunctor<Context, T> concat_functor;
|
||||
|
||||
// convolution transpose: gemm + col2im or col2vol (similar to conv-backward
|
||||
// on x)
|
||||
size_t D = x.dims().size();
|
||||
for (int i = 0; i < batch_size; i++) {
|
||||
// batch with size (i_c, h * w) or (i_c, d * h * w) for channel_first
|
||||
// batch with size (h * w, i_c) or (d * h * w, i_c) for channel_last
|
||||
DenseTensor x_batch = x.Slice(i, i + 1).Resize(x_matrix_shape);
|
||||
|
||||
// out size: (o_c, o_h, o_w) or (o_c, o_d, o_h, o_w) for channel_first
|
||||
// out size: (o_h, o_w, o_c) or (o_d, o_h, o_w, o_c) for channel_last
|
||||
DenseTensor out_batch = out->Slice(i, i + 1).Resize(out_shape);
|
||||
|
||||
std::vector<DenseTensor> out_batch_vec;
|
||||
for (int g = 0; g < groups; g++) {
|
||||
int64_t start = g * in_step;
|
||||
int64_t end = (g + 1) * in_step;
|
||||
int axes = (data_layout != DataLayout::NHWC ? 0 : 1);
|
||||
DenseTensor filter_slice = filter_.Slice(g * in_step, (g + 1) * in_step);
|
||||
DenseTensor in_slice, out_slice;
|
||||
|
||||
// col_matrix = filter_slice * x_slice
|
||||
// of shape (o_c/g * k_h * k_w, h * w)
|
||||
// or (o_c/g * k_d * k_h * k_w, d * h * w)
|
||||
if (data_layout != DataLayout::NHWC) {
|
||||
in_slice = x_batch.Slice(g * in_step, (g + 1) * in_step);
|
||||
out_slice = out_batch.Slice(g * out_step, (g + 1) * out_step);
|
||||
blas.MatMul(filter_slice,
|
||||
true,
|
||||
in_slice,
|
||||
false,
|
||||
static_cast<T>(1.0),
|
||||
&col_matrix,
|
||||
static_cast<T>(0.0));
|
||||
} else {
|
||||
funcs::Slice<Context, T, 2>(
|
||||
dev_ctx, &x_batch, &in_slice, start, end, axes);
|
||||
start = g * out_step;
|
||||
end = (g + 1) * out_step;
|
||||
axes = D - 2;
|
||||
if (D == 4U) {
|
||||
funcs::Slice<Context, T, 3>(
|
||||
dev_ctx, &out_batch, &out_slice, start, end, axes);
|
||||
} else if (D == 5U) {
|
||||
funcs::Slice<Context, T, 4>(
|
||||
dev_ctx, &out_batch, &out_slice, start, end, axes);
|
||||
}
|
||||
blas.MatMul(filter_slice,
|
||||
true,
|
||||
in_slice,
|
||||
true,
|
||||
static_cast<T>(1.0),
|
||||
&col_matrix,
|
||||
static_cast<T>(0.0));
|
||||
}
|
||||
|
||||
if (data_dim == 2U) {
|
||||
// col2im: col_matrix -> dy from (o_c/g * k_h * k_w, h * w) to (o_c/g,
|
||||
// o_h, o_w) or (o_h, o_w, o_c/g)
|
||||
col2im(dev_ctx,
|
||||
col,
|
||||
dilations_,
|
||||
strides,
|
||||
std::vector<int>{
|
||||
paddings_[0], paddings_[2], paddings_[1], paddings_[3]},
|
||||
&out_slice,
|
||||
data_layout);
|
||||
} else if (data_dim == 3U) {
|
||||
// col2vol: col_matrix -> dy from (o_c/g * k_d * k_h * k_w, d * h * w)
|
||||
// to (o_c/g, o_d, o_h, o_w) or (o_d, o_h, o_w, o_c/g)
|
||||
col2vol(dev_ctx,
|
||||
col,
|
||||
dilations_,
|
||||
strides,
|
||||
paddings_,
|
||||
&out_slice,
|
||||
data_layout);
|
||||
}
|
||||
if (data_layout == DataLayout::NHWC) {
|
||||
out_batch_vec.push_back(out_slice);
|
||||
}
|
||||
}
|
||||
if (data_layout == DataLayout::NHWC) {
|
||||
concat_functor(
|
||||
dev_ctx, out_batch_vec, static_cast<int>(D - 2), &out_batch);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void Conv2dTransposeKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& filter,
|
||||
const std::vector<int>& strides,
|
||||
const std::vector<int>& paddings,
|
||||
const std::vector<int>& output_padding UNUSED,
|
||||
const IntArray& output_size UNUSED,
|
||||
const std::string& padding_algorithm,
|
||||
int groups,
|
||||
const std::vector<int>& dilations,
|
||||
const std::string& data_format,
|
||||
DenseTensor* out) {
|
||||
ConvTransposeRawKernel<T, Context>(dev_ctx,
|
||||
x,
|
||||
filter,
|
||||
strides,
|
||||
paddings,
|
||||
padding_algorithm,
|
||||
groups,
|
||||
dilations,
|
||||
data_format,
|
||||
out);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void Conv3dTransposeKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& filter,
|
||||
const std::vector<int>& strides,
|
||||
const std::vector<int>& paddings,
|
||||
const std::vector<int>& output_padding UNUSED,
|
||||
const std::vector<int>& output_size UNUSED,
|
||||
const std::string& padding_algorithm,
|
||||
int groups,
|
||||
const std::vector<int>& dilations,
|
||||
const std::string& data_format,
|
||||
DenseTensor* out) {
|
||||
ConvTransposeRawKernel<T, Context>(dev_ctx,
|
||||
x,
|
||||
filter,
|
||||
strides,
|
||||
paddings,
|
||||
padding_algorithm,
|
||||
groups,
|
||||
dilations,
|
||||
data_format,
|
||||
out);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,110 @@
|
||||
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/crop_grad_kernel.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "paddle/phi/core/tensor_utils.h"
|
||||
#include "paddle/phi/kernels/full_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/eigen_function.h"
|
||||
namespace phi {
|
||||
|
||||
template <typename Context, typename T, size_t D>
|
||||
void CropTensorGradFunction(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& out_grad,
|
||||
const IntArray& offsets,
|
||||
DenseTensor* x_grad) {
|
||||
if (x_grad != nullptr) {
|
||||
x_grad->Resize(x.dims());
|
||||
dev_ctx.template Alloc<T>(x_grad);
|
||||
|
||||
auto offsets_vec = offsets.GetData();
|
||||
std::array<std::pair<int64_t, int64_t>, D> paddings;
|
||||
for (size_t i = 0; i < D; ++i) {
|
||||
paddings[i].first = offsets_vec[i];
|
||||
paddings[i].second =
|
||||
x_grad->dims()[i] - out_grad.dims()[i] - offsets_vec[i];
|
||||
}
|
||||
auto x_grad_tensor = EigenTensor<T, D>::From(*x_grad);
|
||||
auto out_grad_tensor = EigenTensor<T, D>::From(out_grad);
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
|
||||
funcs::EigenPad<std::decay_t<decltype(place)>, T, D>::Eval(
|
||||
place, x_grad_tensor, out_grad_tensor, paddings, static_cast<T>(0));
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void CropGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& out_grad,
|
||||
const DenseTensor& x,
|
||||
const IntArray& offsets,
|
||||
DenseTensor* x_grad) {
|
||||
// x[3, 5], shape[2, 0], out[2, 0]
|
||||
if (out_grad.numel() == 0 && x_grad != nullptr) {
|
||||
Full<T, Context>(dev_ctx, x_grad->dims(), 0, x_grad);
|
||||
return;
|
||||
}
|
||||
size_t rank = out_grad.dims().size();
|
||||
PADDLE_ENFORCE_GE(
|
||||
rank,
|
||||
1,
|
||||
errors::InvalidArgument(
|
||||
"The number of dimensions of the input 'Out@GRAD' for "
|
||||
"Op(crop_tensor_grad) must be greater than or equal to 1, but the "
|
||||
"value received is %d.",
|
||||
rank));
|
||||
PADDLE_ENFORCE_LE(
|
||||
rank,
|
||||
6,
|
||||
errors::InvalidArgument(
|
||||
"The number of dimensions of the input 'Out@GRAD' for "
|
||||
"Op(crop_tensor_grad) must be less than or equal to 6, but the "
|
||||
"value received is %d.",
|
||||
rank));
|
||||
switch (rank) {
|
||||
case 1:
|
||||
CropTensorGradFunction<Context, T, 1>(
|
||||
dev_ctx, out_grad, x, offsets, x_grad);
|
||||
break;
|
||||
case 2:
|
||||
CropTensorGradFunction<Context, T, 2>(
|
||||
dev_ctx, out_grad, x, offsets, x_grad);
|
||||
break;
|
||||
case 3:
|
||||
CropTensorGradFunction<Context, T, 3>(
|
||||
dev_ctx, out_grad, x, offsets, x_grad);
|
||||
break;
|
||||
case 4:
|
||||
CropTensorGradFunction<Context, T, 4>(
|
||||
dev_ctx, out_grad, x, offsets, x_grad);
|
||||
break;
|
||||
case 5:
|
||||
CropTensorGradFunction<Context, T, 5>(
|
||||
dev_ctx, out_grad, x, offsets, x_grad);
|
||||
break;
|
||||
case 6:
|
||||
CropTensorGradFunction<Context, T, 6>(
|
||||
dev_ctx, out_grad, x, offsets, x_grad);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,188 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/crop_kernel.h"
|
||||
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "paddle/phi/common/int_array.h"
|
||||
#include "paddle/phi/core/tensor_utils.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/eigen_function.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
static DDim ValidateShape(const std::vector<int64_t>& shape,
|
||||
const std::vector<int64_t>& offsets,
|
||||
const DDim& in_dims) {
|
||||
auto in_dim_size = in_dims.size();
|
||||
auto shape_size = shape.size();
|
||||
PADDLE_ENFORCE_EQ(
|
||||
in_dim_size,
|
||||
shape_size,
|
||||
errors::InvalidArgument(
|
||||
"The number of elements (%d) for shape of Op(crop_tensor) should be "
|
||||
"equal to the number of dimensions (%d) of the input tensor.",
|
||||
shape_size,
|
||||
in_dim_size));
|
||||
std::vector<int64_t> output_shape(shape.size(), 0);
|
||||
for (size_t i = 0; i < shape.size(); ++i) {
|
||||
if (shape[i] <= 0 && in_dims[i] > 0) {
|
||||
PADDLE_ENFORCE_NE(shape[i],
|
||||
0,
|
||||
errors::InvalidArgument(
|
||||
"The value (%d) of the %uth element for shape of "
|
||||
"Op(crop_tensor) should not be zero.",
|
||||
shape[i],
|
||||
i));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
shape[i],
|
||||
-1,
|
||||
errors::InvalidArgument("When the value (%d) of the %uth "
|
||||
"element for shape of Op(crop_tensor)"
|
||||
" is negative, only -1 is supported.",
|
||||
shape[i],
|
||||
i));
|
||||
output_shape[i] = in_dims[i] - offsets[i];
|
||||
} else {
|
||||
output_shape[i] = static_cast<int64_t>(shape[i]);
|
||||
}
|
||||
}
|
||||
|
||||
return make_ddim(output_shape);
|
||||
}
|
||||
|
||||
template <typename Context, typename T, size_t D>
|
||||
void CropTensorFunction(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const IntArray& shape,
|
||||
const IntArray& offsets,
|
||||
DenseTensor* out) {
|
||||
auto x_dims = x.dims();
|
||||
auto rank = x.dims().size();
|
||||
auto out_dims = out->dims();
|
||||
|
||||
auto shape_vec = shape.GetData();
|
||||
|
||||
if (shape_vec.size() == 0) {
|
||||
for (int i = 0; i < out_dims.size(); ++i) {
|
||||
shape_vec.push_back(out_dims[i]);
|
||||
}
|
||||
}
|
||||
|
||||
auto offsets_vec = offsets.GetData();
|
||||
|
||||
PADDLE_ENFORCE_EQ(
|
||||
rank,
|
||||
static_cast<int>(offsets_vec.size()),
|
||||
errors::InvalidArgument("The number of elements (%d) for "
|
||||
"input 'Offsets' must be equal to "
|
||||
"the number of dimensions (%d) "
|
||||
"of the input tensor.",
|
||||
static_cast<int>(offsets_vec.size()),
|
||||
rank));
|
||||
|
||||
out_dims = ValidateShape(shape_vec, offsets_vec, x.dims());
|
||||
out->Resize(out_dims);
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
for (size_t i = 0; i < offsets_vec.size(); ++i) {
|
||||
PADDLE_ENFORCE_GE(
|
||||
offsets_vec[i],
|
||||
0,
|
||||
errors::InvalidArgument("The offsets (%d) of the %uth elements of"
|
||||
" Op(crop_tensor) "
|
||||
"should be greater than or "
|
||||
"equal to 0.",
|
||||
offsets_vec[i],
|
||||
i));
|
||||
|
||||
PADDLE_ENFORCE_LE(offsets_vec[i] + shape_vec[i],
|
||||
x_dims[i],
|
||||
errors::InvalidArgument(
|
||||
"The sum of the %uth elements of "
|
||||
"offsets (%d) and shape (%d) of Op(crop_tensor) "
|
||||
"should be less than or "
|
||||
"equal to the size of %uth dimension of the input.",
|
||||
i,
|
||||
offsets_vec[i],
|
||||
shape_vec[i],
|
||||
i));
|
||||
}
|
||||
|
||||
auto x_tensor = EigenTensor<T, D>::From(x);
|
||||
auto out_tensor = EigenTensor<T, D>::From(*out);
|
||||
Eigen::DSizes<int64_t, D> e_offsets;
|
||||
Eigen::DSizes<int64_t, D> e_shape;
|
||||
for (size_t i = 0; i < D; ++i) {
|
||||
e_offsets[i] = offsets_vec[i];
|
||||
e_shape[i] = out->dims()[i];
|
||||
}
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
funcs::EigenSlice<std::decay_t<decltype(place)>, T, D>::Eval(
|
||||
place, out_tensor, x_tensor, e_offsets, e_shape);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void CropKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const IntArray& shape,
|
||||
const IntArray& offsets,
|
||||
DenseTensor* out) {
|
||||
if (out && out->numel() == 0) {
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
return;
|
||||
}
|
||||
int rank = x.dims().size();
|
||||
PADDLE_ENFORCE_GE(
|
||||
rank,
|
||||
1,
|
||||
errors::InvalidArgument(
|
||||
"The number of dimensions of the input 'x' for "
|
||||
"Op(crop_tensor) must be greater than or equal to 1, but the "
|
||||
"value received is %d.",
|
||||
rank));
|
||||
PADDLE_ENFORCE_LE(
|
||||
rank,
|
||||
6,
|
||||
errors::InvalidArgument(
|
||||
"The number of dimensions of the input 'x' for "
|
||||
"Op(crop_tensor) must be less than or equal to 6, but the "
|
||||
"value received is %d.",
|
||||
rank));
|
||||
switch (rank) {
|
||||
case 1:
|
||||
CropTensorFunction<Context, T, 1>(dev_ctx, x, shape, offsets, out);
|
||||
break;
|
||||
case 2:
|
||||
CropTensorFunction<Context, T, 2>(dev_ctx, x, shape, offsets, out);
|
||||
break;
|
||||
case 3:
|
||||
CropTensorFunction<Context, T, 3>(dev_ctx, x, shape, offsets, out);
|
||||
break;
|
||||
case 4:
|
||||
CropTensorFunction<Context, T, 4>(dev_ctx, x, shape, offsets, out);
|
||||
break;
|
||||
case 5:
|
||||
CropTensorFunction<Context, T, 5>(dev_ctx, x, shape, offsets, out);
|
||||
break;
|
||||
case 6:
|
||||
CropTensorFunction<Context, T, 6>(dev_ctx, x, shape, offsets, out);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,285 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/core/tensor_utils.h"
|
||||
#include "paddle/phi/kernels/funcs/cross_entropy.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
#include "paddle/phi/kernels/funcs/math.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void CrossEntropyOpKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& label,
|
||||
bool soft_label,
|
||||
int ignore_index,
|
||||
DenseTensor* out) {
|
||||
auto* labels = &label;
|
||||
auto* y = out;
|
||||
dev_ctx.template Alloc<T>(y);
|
||||
|
||||
int rank = x.dims().size();
|
||||
auto label_dims = labels->dims();
|
||||
DenseTensor x_2d = ReshapeToMatrix(x, rank - 1);
|
||||
DenseTensor labels_2d, y_2d;
|
||||
if (label_dims.size() < rank) {
|
||||
labels_2d.ShareDataWith(*labels);
|
||||
labels_2d.Resize({common::product(label_dims), 1});
|
||||
|
||||
y_2d.ShareDataWith(*y);
|
||||
y_2d.Resize({common::product(y->dims()), 1});
|
||||
|
||||
} else {
|
||||
labels_2d = ReshapeToMatrix(*labels, rank - 1);
|
||||
y_2d = ReshapeToMatrix(*y, rank - 1);
|
||||
}
|
||||
|
||||
// TODO(large-tensor): downstream functors may still use int
|
||||
int64_t axis_dim = x.dims()[rank - 1];
|
||||
|
||||
funcs::CrossEntropyFunctor<Context, T>()(
|
||||
dev_ctx, &y_2d, &x_2d, &labels_2d, soft_label, ignore_index, axis_dim);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
class XeSoftLabelGradFunctor {
|
||||
public:
|
||||
XeSoftLabelGradFunctor(T* dx,
|
||||
const T* dy, // NOLINT
|
||||
const T* x, // NOLINT
|
||||
const T* label, // NOLINT
|
||||
size_t num_classes)
|
||||
: dx_(dx), dy_(dy), x_(x), label_(label), num_classes_(num_classes) {}
|
||||
|
||||
HOSTDEVICE void operator()(size_t i) {
|
||||
auto row_ids = i / num_classes_;
|
||||
dx_[i] = -label_[i] * dy_[row_ids] / x_[i];
|
||||
}
|
||||
|
||||
private:
|
||||
T* dx_;
|
||||
const T* dy_;
|
||||
const T* x_;
|
||||
const T* label_;
|
||||
size_t num_classes_;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
class XeGradFunctor {
|
||||
public:
|
||||
XeGradFunctor(T* dx,
|
||||
const T* dy, // NOLINT
|
||||
const T* x, // NOLINT
|
||||
const int64_t* label, // NOLINT
|
||||
size_t num_classes,
|
||||
size_t ignore_index)
|
||||
: dx_(dx),
|
||||
dy_(dy),
|
||||
x_(x),
|
||||
label_(label),
|
||||
num_classes_(num_classes),
|
||||
ignore_index_(ignore_index) {}
|
||||
|
||||
HOSTDEVICE void operator()(size_t sample_id) {
|
||||
auto x_is_true_offset = sample_id * num_classes_ + label_[sample_id];
|
||||
for (size_t x_offset = sample_id * num_classes_;
|
||||
x_offset < (sample_id + 1) * num_classes_;
|
||||
++x_offset) {
|
||||
dx_[x_offset] = (x_offset != x_is_true_offset ||
|
||||
label_[sample_id] == static_cast<int64_t>(ignore_index_))
|
||||
? static_cast<T>(0)
|
||||
: -dy_[sample_id] / x_[x_offset];
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
T* dx_;
|
||||
const T* dy_;
|
||||
const T* x_;
|
||||
const int64_t* label_;
|
||||
size_t num_classes_;
|
||||
size_t ignore_index_;
|
||||
};
|
||||
|
||||
template <typename T, typename Context>
|
||||
void CrossEntropyGradientOpKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& label,
|
||||
const DenseTensor& out_grad,
|
||||
bool soft_label,
|
||||
int ignore_index,
|
||||
DenseTensor* x_grad) {
|
||||
auto* dy = &out_grad;
|
||||
auto* dx = x_grad;
|
||||
T* dx_data = dev_ctx.template Alloc<T>(dx);
|
||||
|
||||
// Following computation only depends on the last dimension size. So it's
|
||||
// unnecessary to convert tensors to 2-D views.
|
||||
int rank = x.dims().size();
|
||||
int64_t class_num = x.dims()[rank - 1];
|
||||
if (soft_label) {
|
||||
XeSoftLabelGradFunctor<T> functor(dx_data,
|
||||
dy->data<T>(),
|
||||
x.data<T>(),
|
||||
label.data<T>(),
|
||||
static_cast<size_t>(class_num));
|
||||
funcs::ForRange<Context> for_range(dev_ctx,
|
||||
static_cast<size_t>(dx->numel()));
|
||||
for_range(functor);
|
||||
} else {
|
||||
XeGradFunctor<T> functor(dx_data,
|
||||
dy->data<T>(),
|
||||
x.data<T>(),
|
||||
label.data<int64_t>(),
|
||||
static_cast<size_t>(class_num),
|
||||
static_cast<size_t>(ignore_index));
|
||||
funcs::ForRange<Context> for_range(dev_ctx,
|
||||
static_cast<size_t>(dy->numel()));
|
||||
for_range(functor);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
struct HardLabelCrossEntropyForwardFunctor {
|
||||
HardLabelCrossEntropyForwardFunctor(const T* x,
|
||||
T* y,
|
||||
T* match_x,
|
||||
const int64_t* label,
|
||||
int64_t ignore_index,
|
||||
int64_t feature_size)
|
||||
: x_(x),
|
||||
y_(y),
|
||||
match_x_(match_x),
|
||||
label_(label),
|
||||
ignore_index_(ignore_index),
|
||||
feature_size_(feature_size) {}
|
||||
|
||||
HOSTDEVICE void operator()(int64_t idx) const {
|
||||
auto label = label_[idx];
|
||||
if (label != ignore_index_) {
|
||||
// don't update to PADDLE_ENFORCE_GE and PADDLE_ENFORCE_LT cause
|
||||
// can't use common::errors::InvalidArgument in HOSTDEVICE
|
||||
PADDLE_ENFORCE(label >= 0 && label < feature_size_,
|
||||
"Variable value (label) of "
|
||||
"OP(fluid.layers.cross_entropy) expected >= 0 "
|
||||
"and < %ld, but got %ld. Please check label value.",
|
||||
feature_size_,
|
||||
label);
|
||||
|
||||
auto match_x = x_[idx * feature_size_ + label];
|
||||
y_[idx] = -funcs::TolerableValue<T>()(funcs::real_log(match_x));
|
||||
match_x_[idx] = match_x;
|
||||
} else {
|
||||
y_[idx] = 0;
|
||||
match_x_[idx] = 0; // any value is ok
|
||||
}
|
||||
}
|
||||
|
||||
const T* x_;
|
||||
T* y_;
|
||||
T* match_x_;
|
||||
const int64_t* label_;
|
||||
int64_t ignore_index_;
|
||||
int64_t feature_size_;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct HardLabelCrossEntropyBackwardFunctor {
|
||||
HardLabelCrossEntropyBackwardFunctor(T* dx,
|
||||
const T* dy,
|
||||
const T* match_x,
|
||||
const int64_t* label,
|
||||
int64_t ignore_index,
|
||||
int64_t feature_size)
|
||||
: dx_(dx),
|
||||
dy_(dy),
|
||||
match_x_(match_x),
|
||||
label_(label),
|
||||
ignore_index_(ignore_index),
|
||||
feature_size_(feature_size) {}
|
||||
|
||||
HOSTDEVICE void operator()(int64_t idx) const {
|
||||
auto row_idx = idx / feature_size_;
|
||||
auto col_idx = idx % feature_size_;
|
||||
auto label = label_[row_idx];
|
||||
if (label == col_idx && label != ignore_index_) {
|
||||
dx_[idx] = -dy_[row_idx] / match_x_[row_idx];
|
||||
} else {
|
||||
dx_[idx] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
T* dx_;
|
||||
const T* dy_;
|
||||
const T* match_x_;
|
||||
const int64_t* label_;
|
||||
int64_t ignore_index_;
|
||||
int64_t feature_size_;
|
||||
};
|
||||
|
||||
template <typename T, typename Context>
|
||||
void CrossEntropyOpKernel2(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& label,
|
||||
int ignore_index,
|
||||
DenseTensor* out,
|
||||
DenseTensor* x_shape,
|
||||
DenseTensor* match_x) {
|
||||
auto* y = out;
|
||||
|
||||
auto& x_dims = x.dims();
|
||||
auto feature_size = x_dims[x_dims.size() - 1];
|
||||
auto batch_size = common::product(x.dims()) / feature_size;
|
||||
|
||||
auto* p_x = x.data<T>();
|
||||
auto* p_label = label.data<int64_t>();
|
||||
auto* p_y = dev_ctx.template Alloc<T>(y);
|
||||
auto* p_match_x = dev_ctx.template Alloc<T>(match_x);
|
||||
|
||||
funcs::ForRange<Context> for_range(dev_ctx, batch_size);
|
||||
for_range(HardLabelCrossEntropyForwardFunctor<T>(
|
||||
p_x, p_y, p_match_x, p_label, ignore_index, feature_size));
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void CrossEntropyGradientOpKernel2(const Context& dev_ctx,
|
||||
const DenseTensor& x_shape,
|
||||
const DenseTensor& label,
|
||||
const DenseTensor& match_x,
|
||||
const DenseTensor& out_grad,
|
||||
int ignore_index,
|
||||
DenseTensor* x_grad) {
|
||||
auto* dx = x_grad;
|
||||
auto* dy = &out_grad;
|
||||
|
||||
auto* p_dx = dev_ctx.template Alloc<T>(dx);
|
||||
auto* p_dy = dy->data<T>();
|
||||
auto* p_match_x = match_x.data<T>();
|
||||
auto* p_label = label.data<int64_t>();
|
||||
|
||||
int rank = dx->dims().size();
|
||||
int64_t feature_size = dx->dims()[rank - 1];
|
||||
int64_t batch_size = common::product(dx->dims()) / feature_size;
|
||||
|
||||
funcs::ForRange<Context> for_range(dev_ctx, batch_size * feature_size);
|
||||
for_range(HardLabelCrossEntropyBackwardFunctor<T>(
|
||||
p_dx, p_dy, p_match_x, p_label, ignore_index, feature_size));
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,113 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/core/lod_utils.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void CTCAlignKernel(const Context& dev_ctx,
|
||||
const DenseTensor& input,
|
||||
const optional<DenseTensor>& input_length,
|
||||
int blank,
|
||||
bool merge_repeated,
|
||||
int padding_value,
|
||||
DenseTensor* output,
|
||||
DenseTensor* output_length) {
|
||||
T* output_data = dev_ctx.template Alloc<T>(output);
|
||||
auto input_dims = vectorize<int>(input.dims());
|
||||
const T* input_data = input.data<T>();
|
||||
|
||||
// support tensor input, no lod information
|
||||
if (input.lod().empty()) {
|
||||
size_t padding_value_new = static_cast<size_t>(padding_value);
|
||||
const T* input_length_data = input_length.get().data<T>();
|
||||
|
||||
T* output_length_data = dev_ctx.template Alloc<T>(output_length);
|
||||
|
||||
for (size_t batch_id = 0; batch_id < (unsigned)input_dims[0]; batch_id++) {
|
||||
T prev_token = -1;
|
||||
size_t output_idx = 0;
|
||||
for (size_t i = 0; i < (unsigned)input_length_data[batch_id]; i++) {
|
||||
size_t input_ind = batch_id * input_dims[1] + i;
|
||||
if ((unsigned)input_data[input_ind] != (unsigned)blank &&
|
||||
!(merge_repeated && input_data[input_ind] == prev_token)) {
|
||||
output_data[batch_id * input_dims[1] + output_idx] =
|
||||
input_data[input_ind];
|
||||
++output_idx;
|
||||
}
|
||||
prev_token = input_data[input_ind];
|
||||
}
|
||||
output_length_data[batch_id] = output_idx;
|
||||
for (size_t j = output_idx; j < (unsigned)input_dims[1]; j++)
|
||||
output_data[batch_id * input_dims[1] + j] = padding_value_new;
|
||||
}
|
||||
} else {
|
||||
const size_t level = 0;
|
||||
auto input_lod = ToAbsOffset(input.lod());
|
||||
|
||||
// check input dims and lod
|
||||
PADDLE_ENFORCE_EQ(
|
||||
input_dims[0],
|
||||
static_cast<int64_t>(input_lod[level].back()),
|
||||
common::errors::InvalidArgument(
|
||||
"The first dimension %d of CTCAlign operator Input(Input) should "
|
||||
"be equal to "
|
||||
"the sum of all sequences' lengths %d.",
|
||||
input_dims[0],
|
||||
static_cast<int64_t>(input_lod[level].back())));
|
||||
|
||||
const size_t num_sequences = input_lod[level].size() - 1;
|
||||
|
||||
// merge repeated tokens and delete blank
|
||||
size_t output_idx = 0;
|
||||
std::vector<size_t> output_lod0(1, 0);
|
||||
for (size_t seq_idx = 0; seq_idx < num_sequences; ++seq_idx) {
|
||||
T prev_token = -1;
|
||||
for (size_t i = input_lod[level][seq_idx];
|
||||
i < input_lod[level][seq_idx + 1];
|
||||
++i) {
|
||||
if ((unsigned)input_data[i] != (unsigned)blank &&
|
||||
!(merge_repeated && input_data[i] == prev_token)) {
|
||||
output_data[output_idx] = input_data[i];
|
||||
++output_idx;
|
||||
}
|
||||
prev_token = input_data[i];
|
||||
}
|
||||
output_lod0.push_back(output_idx);
|
||||
}
|
||||
|
||||
// set output lod
|
||||
LegacyLoD output_lod;
|
||||
output_lod.push_back(output_lod0);
|
||||
output->set_lod(output_lod);
|
||||
// resize output dims
|
||||
output->Resize({static_cast<int64_t>(output_lod0.back()), 1});
|
||||
// for empty sequence
|
||||
if (output_lod0.back() == 0) {
|
||||
output->Resize({1, 1});
|
||||
output_data = dev_ctx.template Alloc<T>(output);
|
||||
output_data[0] = -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,135 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T>
|
||||
void CvmComputeKernel(const bool use_cvm,
|
||||
const int64_t item_width,
|
||||
const T** X,
|
||||
T** Y) {
|
||||
const auto cvm_offset = use_cvm ? 0 : 2;
|
||||
|
||||
std::memcpy(*Y, *X + cvm_offset, (item_width - cvm_offset) * sizeof(T));
|
||||
|
||||
if (use_cvm) {
|
||||
(*Y)[0] = log((*Y)[0] + 1);
|
||||
(*Y)[1] = log((*Y)[1] + 1) - (*Y)[0];
|
||||
}
|
||||
|
||||
(*X) += item_width;
|
||||
(*Y) += item_width - cvm_offset;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void CvmGradComputeKernel(const bool use_cvm,
|
||||
const int64_t item_width,
|
||||
const T& CVM,
|
||||
const T** DY,
|
||||
T** DX) {
|
||||
const auto cvm_offset = use_cvm ? 0 : 2;
|
||||
|
||||
std::memcpy(*DX + cvm_offset, *DY, (item_width - cvm_offset) * sizeof(T));
|
||||
|
||||
(*DX)[0] = (&CVM)[0];
|
||||
(*DX)[1] = (&CVM)[1];
|
||||
|
||||
(*DX) += item_width;
|
||||
(*DY) += item_width - cvm_offset;
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void CVMOpKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x_in,
|
||||
const DenseTensor& cvm,
|
||||
bool use_cvm,
|
||||
DenseTensor* out) {
|
||||
const auto* x = &x_in;
|
||||
const T* x_data = x->data<T>();
|
||||
|
||||
auto batch_size = x->dims()[0];
|
||||
auto item_size = x->numel() / batch_size;
|
||||
|
||||
auto* y = out;
|
||||
T* y_data = dev_ctx.template Alloc<T>(y);
|
||||
|
||||
// for Input X do not have Lod Information.
|
||||
if (x->NumLevels() == 0) {
|
||||
if (use_cvm) {
|
||||
for (int i = 0; i < batch_size; i++) {
|
||||
int64_t cursor = i * item_size;
|
||||
y_data[cursor] = log(x_data[cursor] + 1);
|
||||
y_data[cursor + 1] = log(x_data[cursor + 1] + 1) - y_data[cursor];
|
||||
for (int j = 2; j < item_size; j++) {
|
||||
y_data[cursor + j] = x_data[cursor + j];
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (int i = 0; i < batch_size; i++) {
|
||||
CvmComputeKernel(use_cvm, item_size, &x_data, &y_data);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
auto lod = x->lod()[0];
|
||||
for (size_t i = 0; i < lod.size() - 1; ++i) {
|
||||
for (size_t j = 0; j < lod[i + 1] - lod[i]; ++j) {
|
||||
CvmComputeKernel(use_cvm, item_size, &x_data, &y_data);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void CVMGradOpKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x_in,
|
||||
const DenseTensor& cvm_in,
|
||||
const DenseTensor& out_grad,
|
||||
bool use_cvm,
|
||||
DenseTensor* x_grad) {
|
||||
auto* dx = x_grad;
|
||||
T* dx_data = dev_ctx.template Alloc<T>(dx);
|
||||
|
||||
const DenseTensor* cvm = &cvm_in;
|
||||
const T* cvm_data = cvm->data<T>();
|
||||
|
||||
const auto* dOut = &out_grad;
|
||||
const T* dout_data = dOut->data<T>();
|
||||
|
||||
auto offset = 2;
|
||||
auto batch_size = dx->dims()[0];
|
||||
auto item_size = dx->numel() / batch_size;
|
||||
|
||||
// for Input X do not have Lod Information.
|
||||
if (dx->NumLevels() == 0) {
|
||||
for (int x = 0; x < batch_size; ++x) {
|
||||
CvmGradComputeKernel(use_cvm, item_size, *cvm_data, &dout_data, &dx_data);
|
||||
cvm_data += offset;
|
||||
}
|
||||
} else {
|
||||
auto lod = dx->lod()[0];
|
||||
int seq_num = static_cast<int>(lod.size()) - 1;
|
||||
for (int i = 0; i < seq_num; ++i) {
|
||||
for (size_t j = 0; j < lod[i + 1] - lod[i]; ++j) {
|
||||
CvmGradComputeKernel(
|
||||
use_cvm, item_size, *cvm_data, &dout_data, &dx_data);
|
||||
}
|
||||
cvm_data += offset;
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,118 @@
|
||||
// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/core/tensor_utils.h"
|
||||
#include "paddle/phi/kernels/funcs/tensor_formatter.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
const char kForward[] = "FORWARD";
|
||||
const char kBackward[] = "BACKWARD";
|
||||
|
||||
template <typename Context>
|
||||
void ShadowFeedKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
int dst_place_type,
|
||||
DenseTensor* out) {
|
||||
Place target_place;
|
||||
switch (dst_place_type) {
|
||||
case 0: // CPUPlace
|
||||
target_place = CPUPlace();
|
||||
break;
|
||||
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
|
||||
case 1: // CUDAPlace
|
||||
target_place = GPUPlace(backends::gpu::GetCurrentDeviceId());
|
||||
break;
|
||||
#elif defined(PADDLE_WITH_XPU)
|
||||
case 1: // XPUPlace
|
||||
target_place = XPUPlace(backends::xpu::GetXPUCurrentDeviceId());
|
||||
break;
|
||||
#elif defined(PADDLE_WITH_CUSTOM_DEVICE)
|
||||
case 1: // CustomPlace
|
||||
target_place = dev_ctx.GetPlace();
|
||||
break;
|
||||
#endif
|
||||
default:
|
||||
PADDLE_THROW(errors::Unimplemented("dst_place_type: %d is not supported.",
|
||||
dst_place_type));
|
||||
break;
|
||||
}
|
||||
|
||||
if (!(x.has_allocation())) {
|
||||
if (target_place == CPUPlace()) {
|
||||
dev_ctx.HostAlloc(out, out->dtype());
|
||||
} else {
|
||||
dev_ctx.Alloc(out, out->dtype());
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (x.place() == target_place) {
|
||||
out->ShareDataWith(x);
|
||||
out->set_lod(x.lod());
|
||||
} else {
|
||||
Copy<Context>(dev_ctx, x, target_place, true, out);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Context>
|
||||
void ShadowFeedTensorsKernel(const Context& dev_ctx,
|
||||
const std::vector<const DenseTensor*>& xs,
|
||||
int dst_place_type,
|
||||
std::vector<DenseTensor*> outs) {
|
||||
for (size_t i = 0; i < xs.size(); ++i) {
|
||||
ShadowFeedKernel<Context>(dev_ctx, *(xs[i]), dst_place_type, outs[i]);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Context>
|
||||
void PrintKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
int first_n,
|
||||
const std::string& message,
|
||||
int summarize,
|
||||
bool print_tensor_name,
|
||||
bool print_tensor_type,
|
||||
bool print_tensor_shape,
|
||||
bool print_tensor_layout,
|
||||
bool print_tensor_lod,
|
||||
const std::string& print_phase,
|
||||
bool is_forward,
|
||||
DenseTensor* out) {
|
||||
Copy<Context>(dev_ctx, x, dev_ctx.GetPlace(), true, out);
|
||||
out->set_lod(x.lod());
|
||||
|
||||
if ((is_forward && print_phase == kBackward) ||
|
||||
(!is_forward && print_phase == kForward)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO(phlrain): support first_n using a input tensor
|
||||
// if (first_n > 0 && ++times_ > first_n) return;
|
||||
|
||||
// TODO(phlrain): support printed_var_name
|
||||
funcs::TensorFormatter formatter;
|
||||
const std::string& name = print_tensor_name ? "var" : "";
|
||||
formatter.SetPrintTensorType(print_tensor_type);
|
||||
formatter.SetPrintTensorShape(print_tensor_shape);
|
||||
formatter.SetPrintTensorLod(print_tensor_lod);
|
||||
formatter.SetPrintTensorLayout(print_tensor_layout);
|
||||
formatter.SetSummarize(summarize);
|
||||
formatter.Print(x, name, message);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,36 @@
|
||||
// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
#include "glog/logging.h"
|
||||
#include "paddle/common/flags.h"
|
||||
#include "paddle/phi/kernels/cast_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
|
||||
COMMON_DECLARE_bool(check_nan_inf);
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void CheckModelNanInfKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
int flag,
|
||||
DenseTensor* out) {
|
||||
CastKernel<T>(dev_ctx, x, x.dtype(), out);
|
||||
VLOG(6) << "model_check_nan_inf: Change FLAGS_check_nan_inf "
|
||||
<< FLAGS_check_nan_inf << " to " << flag;
|
||||
FLAGS_check_nan_inf = flag;
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,49 @@
|
||||
// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/decayed_adagrad_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void DecayedAdagradDenseKernel(const Context& dev_ctx,
|
||||
const DenseTensor& param_t,
|
||||
const DenseTensor& grad_t,
|
||||
const DenseTensor& moment_t,
|
||||
const DenseTensor& learning_rate,
|
||||
float decay,
|
||||
float epsilon,
|
||||
DenseTensor* param_out_t,
|
||||
DenseTensor* moment_out_t) {
|
||||
dev_ctx.template Alloc<T>(param_out_t);
|
||||
dev_ctx.template Alloc<T>(moment_out_t);
|
||||
|
||||
auto param = EigenVector<T>::Flatten(param_t);
|
||||
auto grad = EigenVector<T>::Flatten(grad_t);
|
||||
auto moment = EigenVector<T>::Flatten(moment_t);
|
||||
auto lr = EigenVector<T>::Flatten(learning_rate);
|
||||
|
||||
auto param_out = EigenVector<T>::Flatten(*param_out_t);
|
||||
auto moment_out = EigenVector<T>::Flatten(*moment_out_t);
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
|
||||
moment_out.device(place) = decay * moment + (1 - decay) * grad * grad;
|
||||
Eigen::DSizes<int, 1> m_dsize(moment_out_t->numel());
|
||||
param_out.device(place) =
|
||||
param - lr.broadcast(m_dsize) * grad / (moment_out.sqrt() + epsilon);
|
||||
}
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,437 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/common/hostdevice.h"
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/empty_kernel.h"
|
||||
#include "paddle/phi/kernels/full_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/blas/blas.h"
|
||||
#include "paddle/phi/kernels/funcs/deformable_conv_functor.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T>
|
||||
HOSTDEVICE T DmcnGetGradientWeight(T argmax_h,
|
||||
T argmax_w,
|
||||
const int h,
|
||||
const int w,
|
||||
const int height,
|
||||
const int width) {
|
||||
if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 ||
|
||||
argmax_w >= width) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int argmax_h_low = floor(argmax_h);
|
||||
int argmax_w_low = floor(argmax_w);
|
||||
int argmax_h_high = argmax_h_low + 1;
|
||||
int argmax_w_high = argmax_w_low + 1;
|
||||
|
||||
T weight = 0;
|
||||
|
||||
weight = (h == argmax_h_low && w == argmax_w_low)
|
||||
? (h + 1 - argmax_h) * (w + 1 - argmax_w)
|
||||
: weight;
|
||||
weight = (h == argmax_h_low && w == argmax_w_high)
|
||||
? (h + 1 - argmax_h) * (argmax_w + 1 - w)
|
||||
: weight;
|
||||
weight = (h == argmax_h_high && w == argmax_w_low)
|
||||
? (argmax_h + 1 - h) * (w + 1 - argmax_w)
|
||||
: weight;
|
||||
weight = (h == argmax_h_high && w == argmax_w_high)
|
||||
? (argmax_h + 1 - h) * (argmax_w + 1 - w)
|
||||
: weight;
|
||||
|
||||
return weight;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
HOSTDEVICE T DmcnGetCoordinateWeight(T argmax_h,
|
||||
T argmax_w,
|
||||
const int height,
|
||||
const int width,
|
||||
const T* im_data,
|
||||
const int data_width,
|
||||
const int bp_dir) {
|
||||
if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 ||
|
||||
argmax_w >= width) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int argmax_h_low = floor(argmax_h);
|
||||
int argmax_w_low = floor(argmax_w);
|
||||
int argmax_h_high = argmax_h_low + 1;
|
||||
int argmax_w_high = argmax_w_low + 1;
|
||||
|
||||
T weight = 0;
|
||||
|
||||
if (bp_dir == 0) {
|
||||
weight += (argmax_h_low >= 0 && argmax_w_low >= 0)
|
||||
? -1 * (argmax_w_low + 1 - argmax_w) *
|
||||
im_data[argmax_h_low * data_width + argmax_w_low]
|
||||
: 0;
|
||||
|
||||
weight += (argmax_h_low >= 0 && argmax_w_high <= width - 1)
|
||||
? -1 * (argmax_w - argmax_w_low) *
|
||||
im_data[argmax_h_low * data_width + argmax_w_high]
|
||||
: 0;
|
||||
|
||||
weight += (argmax_h_high <= height - 1 && argmax_w_low >= 0)
|
||||
? (argmax_w_low + 1 - argmax_w) *
|
||||
im_data[argmax_h_high * data_width + argmax_w_low]
|
||||
: 0;
|
||||
weight += (argmax_h_high <= height - 1 && argmax_w_high <= width - 1)
|
||||
? (argmax_w - argmax_w_low) *
|
||||
im_data[argmax_h_high * data_width + argmax_w_high]
|
||||
: 0;
|
||||
} else if (bp_dir == 1) {
|
||||
weight += (argmax_h_low >= 0 && argmax_w_low >= 0)
|
||||
? -1 * (argmax_h_low + 1 - argmax_h) *
|
||||
im_data[argmax_h_low * data_width + argmax_w_low]
|
||||
: 0;
|
||||
weight += (argmax_h_low >= 0 && argmax_w_high <= width - 1)
|
||||
? (argmax_h_low + 1 - argmax_h) *
|
||||
im_data[argmax_h_low * data_width + argmax_w_high]
|
||||
: 0;
|
||||
weight += (argmax_h_high <= height - 1 && argmax_w_low >= 0)
|
||||
? -1 * (argmax_h - argmax_h_low) *
|
||||
im_data[argmax_h_high * data_width + argmax_w_low]
|
||||
: 0;
|
||||
weight += (argmax_h_high <= height - 1 && argmax_w_high <= width - 1)
|
||||
? (argmax_h - argmax_h_low) *
|
||||
im_data[argmax_h_high * data_width + argmax_w_high]
|
||||
: 0;
|
||||
}
|
||||
|
||||
return weight;
|
||||
}
|
||||
|
||||
template <typename T, typename Context, typename IndexT>
|
||||
void ModulatedDeformableCol2imCoord(const Context& dev_ctx,
|
||||
const T* data_col,
|
||||
const T* data_im,
|
||||
const T* data_offset,
|
||||
const T* data_mask,
|
||||
const std::vector<int64_t>& im_shape,
|
||||
const std::vector<int64_t>& col_shape,
|
||||
const std::vector<int64_t>& kernel_shape,
|
||||
const std::vector<int>& paddings,
|
||||
const std::vector<int>& strides,
|
||||
const std::vector<int>& dilations,
|
||||
const int deformable_groups,
|
||||
T* grad_offset,
|
||||
T* grad_mask);
|
||||
|
||||
template <typename T, typename Context, typename IndexT>
|
||||
void ModulatedDeformableCol2im(const Context& dev_ctx,
|
||||
const T* data_col,
|
||||
const T* data_offset,
|
||||
const T* data_mask,
|
||||
const std::vector<int64_t>& im_shape,
|
||||
const std::vector<int64_t>& col_shape,
|
||||
const std::vector<int64_t>& kernel_shape,
|
||||
const std::vector<int>& pad,
|
||||
const std::vector<int>& stride,
|
||||
const std::vector<int>& dilation,
|
||||
const int deformable_group,
|
||||
T* grad_im);
|
||||
|
||||
template <typename T, typename Context, typename IndexT>
|
||||
void FilterGradAddup(const Context& dev_ctx,
|
||||
const int64_t nthreads,
|
||||
const int64_t n,
|
||||
const int64_t height,
|
||||
const int64_t width,
|
||||
const T* dweight_3d,
|
||||
T* filter_grad);
|
||||
|
||||
template <typename T, typename Context>
|
||||
void DeformableConvGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& offset,
|
||||
const DenseTensor& filter,
|
||||
const optional<DenseTensor>& mask,
|
||||
const DenseTensor& out_grad,
|
||||
const std::vector<int>& strides,
|
||||
const std::vector<int>& paddings,
|
||||
const std::vector<int>& dilations,
|
||||
int deformable_groups,
|
||||
int groups,
|
||||
int im2col_step,
|
||||
DenseTensor* dx,
|
||||
DenseTensor* offset_grad,
|
||||
DenseTensor* filter_grad,
|
||||
DenseTensor* mask_grad) {
|
||||
if (x.numel() == 0 || filter.numel() == 0) {
|
||||
if (dx) Full<T, Context>(dev_ctx, dx->dims(), 0, dx);
|
||||
if (offset_grad)
|
||||
Full<T, Context>(dev_ctx, offset_grad->dims(), 0, offset_grad);
|
||||
if (filter_grad)
|
||||
Full<T, Context>(dev_ctx, filter_grad->dims(), 0, filter_grad);
|
||||
if (mask_grad) Full<T, Context>(dev_ctx, mask_grad->dims(), 0, mask_grad);
|
||||
return;
|
||||
}
|
||||
|
||||
const int batch_size = static_cast<int>(x.dims()[0]);
|
||||
|
||||
DDim input_shape = slice_ddim(x.dims(), 1, x.dims().size());
|
||||
std::vector<int64_t> input_shape_vec = vectorize(input_shape);
|
||||
std::vector<int64_t> filter_shape_vec(vectorize(filter.dims()));
|
||||
std::vector<int64_t> output_shape_vec(vectorize(out_grad.dims()));
|
||||
|
||||
std::vector<int64_t> col_buffer_shape_vec(filter_shape_vec.size());
|
||||
col_buffer_shape_vec[0] = x.dims()[1] * filter.dims()[2] * filter.dims()[3];
|
||||
col_buffer_shape_vec[1] = im2col_step;
|
||||
for (size_t j = 0; j < filter_shape_vec.size() - 2; ++j) {
|
||||
col_buffer_shape_vec[j + 2] = output_shape_vec[j + 2];
|
||||
}
|
||||
std::vector<int64_t> output_buffer_shape_vec(1);
|
||||
output_buffer_shape_vec[0] = batch_size * output_shape_vec[1] *
|
||||
output_shape_vec[2] * output_shape_vec[3];
|
||||
|
||||
DenseTensor col_buffer = Empty<T>(dev_ctx, col_buffer_shape_vec);
|
||||
DenseTensor output_buffer;
|
||||
output_buffer.ShareDataWith(out_grad).Resize(
|
||||
make_ddim(output_buffer_shape_vec));
|
||||
|
||||
int64_t M =
|
||||
input_shape_vec[0] / groups * filter_shape_vec[2] * filter_shape_vec[3];
|
||||
int64_t N = im2col_step * output_shape_vec[2] * output_shape_vec[3];
|
||||
int64_t K = output_shape_vec[1] / groups;
|
||||
|
||||
DDim weight_3d_shape = {groups, K, M};
|
||||
DDim out_grad_4d_shape = {batch_size / im2col_step, groups, K, N};
|
||||
DDim col_buffer_3d_shape = {groups, M, N};
|
||||
DDim filter_grad_shape = {groups, K, M};
|
||||
|
||||
DenseTensor weight_3d;
|
||||
weight_3d.ShareDataWith(filter).Resize(weight_3d_shape);
|
||||
DenseTensor out_grad_4d;
|
||||
out_grad_4d.ShareDataWith(output_buffer).Resize(out_grad_4d_shape);
|
||||
DenseTensor col_buffer_3d;
|
||||
col_buffer_3d.ShareDataWith(col_buffer).Resize(col_buffer_3d_shape);
|
||||
|
||||
funcs::SetConstant<Context, T> set_zero;
|
||||
auto blas = funcs::GetBlas<Context, T>(dev_ctx);
|
||||
|
||||
int64_t input_dim = x.numel() / x.dims()[0];
|
||||
int64_t input_offset_dim = offset.numel() / offset.dims()[0];
|
||||
int64_t input_mask_dim = mask ? mask->numel() / mask->dims()[0] : 0;
|
||||
|
||||
if (filter_grad) {
|
||||
Full<T>(dev_ctx, filter_grad_shape, 0, filter_grad);
|
||||
}
|
||||
|
||||
if (dx) {
|
||||
dev_ctx.template Alloc<T>(dx);
|
||||
set_zero(dev_ctx, dx, static_cast<T>(0));
|
||||
}
|
||||
|
||||
if (offset_grad) {
|
||||
dev_ctx.template Alloc<T>(offset_grad);
|
||||
set_zero(dev_ctx, offset_grad, static_cast<T>(0));
|
||||
|
||||
if (mask_grad) {
|
||||
dev_ctx.template Alloc<T>(mask_grad);
|
||||
set_zero(dev_ctx, mask_grad, static_cast<T>(0));
|
||||
}
|
||||
}
|
||||
|
||||
bool using_int32_index =
|
||||
(x.numel() <= std::numeric_limits<int>::max()) &&
|
||||
(offset.numel() <= std::numeric_limits<int>::max()) &&
|
||||
(filter.numel() <= std::numeric_limits<int>::max()) &&
|
||||
(mask ? mask->numel() <= std::numeric_limits<int>::max() : true) &&
|
||||
(out_grad.numel() <= std::numeric_limits<int>::max());
|
||||
|
||||
for (int i = 0; i < batch_size / im2col_step; ++i) {
|
||||
DenseTensor out_grad_3d = out_grad_4d.Slice(i, i + 1).Resize(
|
||||
slice_ddim(out_grad_4d.dims(), 1, out_grad_4d.dims().size()));
|
||||
for (int g = 0; g < groups; ++g) {
|
||||
DenseTensor weight_3d_slice = weight_3d.Slice(g, g + 1).Resize(
|
||||
slice_ddim(weight_3d.dims(), 1, weight_3d.dims().size()));
|
||||
DenseTensor out_grad_3d_slice = out_grad_3d.Slice(g, g + 1).Resize(
|
||||
slice_ddim(out_grad_3d.dims(), 1, out_grad_3d.dims().size()));
|
||||
DenseTensor col_buffer_3d_slice = col_buffer_3d.Slice(g, g + 1).Resize(
|
||||
slice_ddim(col_buffer_3d.dims(), 1, col_buffer_3d.dims().size()));
|
||||
blas.MatMul(weight_3d_slice,
|
||||
true,
|
||||
out_grad_3d_slice,
|
||||
false,
|
||||
T(1.0),
|
||||
&col_buffer_3d_slice,
|
||||
T(0.0));
|
||||
}
|
||||
col_buffer.Resize(col_buffer_shape_vec);
|
||||
|
||||
T* col_buffer_ptr = col_buffer.data<T>();
|
||||
const T* input_ptr = x.data<T>();
|
||||
const T* offset_ptr = offset.data<T>();
|
||||
const T* mask_data_ptr =
|
||||
mask ? mask->data<T>() + i * im2col_step * input_mask_dim : nullptr;
|
||||
if (offset_grad) {
|
||||
T* offset_grad_ptr = offset_grad->data<T>();
|
||||
T* mask_grad_data_ptr =
|
||||
mask_grad ? mask_grad->data<T>() + i * im2col_step * input_mask_dim
|
||||
: nullptr;
|
||||
// get grad of offset and mask
|
||||
if (using_int32_index) {
|
||||
ModulatedDeformableCol2imCoord<T, Context, int>(
|
||||
dev_ctx,
|
||||
col_buffer_ptr,
|
||||
input_ptr + i * im2col_step * input_dim,
|
||||
offset_ptr + i * im2col_step * input_offset_dim,
|
||||
mask_data_ptr,
|
||||
input_shape_vec,
|
||||
col_buffer_shape_vec,
|
||||
filter_shape_vec,
|
||||
paddings,
|
||||
strides,
|
||||
dilations,
|
||||
deformable_groups,
|
||||
offset_grad_ptr + i * im2col_step * input_offset_dim,
|
||||
mask_grad_data_ptr);
|
||||
} else {
|
||||
ModulatedDeformableCol2imCoord<T, Context, int64_t>(
|
||||
dev_ctx,
|
||||
col_buffer_ptr,
|
||||
input_ptr + i * im2col_step * input_dim,
|
||||
offset_ptr + i * im2col_step * input_offset_dim,
|
||||
mask_data_ptr,
|
||||
input_shape_vec,
|
||||
col_buffer_shape_vec,
|
||||
filter_shape_vec,
|
||||
paddings,
|
||||
strides,
|
||||
dilations,
|
||||
deformable_groups,
|
||||
offset_grad_ptr + i * im2col_step * input_offset_dim,
|
||||
mask_grad_data_ptr);
|
||||
}
|
||||
}
|
||||
if (dx) {
|
||||
T* dx_ptr = dx->data<T>();
|
||||
// get grad of input
|
||||
if (using_int32_index) {
|
||||
ModulatedDeformableCol2im<T, Context, int>(
|
||||
dev_ctx,
|
||||
col_buffer_ptr,
|
||||
offset_ptr + i * im2col_step * input_offset_dim,
|
||||
mask_data_ptr,
|
||||
input_shape_vec,
|
||||
col_buffer_shape_vec,
|
||||
filter_shape_vec,
|
||||
paddings,
|
||||
strides,
|
||||
dilations,
|
||||
deformable_groups,
|
||||
dx_ptr + i * im2col_step * input_dim);
|
||||
} else {
|
||||
ModulatedDeformableCol2im<T, Context, int64_t>(
|
||||
dev_ctx,
|
||||
col_buffer_ptr,
|
||||
offset_ptr + i * im2col_step * input_offset_dim,
|
||||
mask_data_ptr,
|
||||
input_shape_vec,
|
||||
col_buffer_shape_vec,
|
||||
filter_shape_vec,
|
||||
paddings,
|
||||
strides,
|
||||
dilations,
|
||||
deformable_groups,
|
||||
dx_ptr + i * im2col_step * input_dim);
|
||||
}
|
||||
|
||||
dx->Resize(x.dims());
|
||||
}
|
||||
if (using_int32_index) {
|
||||
funcs::ModulatedDeformableIm2col<T, Context, int>(
|
||||
dev_ctx,
|
||||
input_ptr + i * im2col_step * input_dim,
|
||||
offset_ptr + i * im2col_step * input_offset_dim,
|
||||
mask_data_ptr,
|
||||
input_shape_vec,
|
||||
col_buffer_shape_vec,
|
||||
filter_shape_vec,
|
||||
paddings,
|
||||
strides,
|
||||
dilations,
|
||||
deformable_groups,
|
||||
col_buffer_ptr);
|
||||
} else {
|
||||
funcs::ModulatedDeformableIm2col<T, Context, int64_t>(
|
||||
dev_ctx,
|
||||
input_ptr + i * im2col_step * input_dim,
|
||||
offset_ptr + i * im2col_step * input_offset_dim,
|
||||
mask_data_ptr,
|
||||
input_shape_vec,
|
||||
col_buffer_shape_vec,
|
||||
filter_shape_vec,
|
||||
paddings,
|
||||
strides,
|
||||
dilations,
|
||||
deformable_groups,
|
||||
col_buffer_ptr);
|
||||
}
|
||||
|
||||
col_buffer_3d.Resize(col_buffer_3d_shape);
|
||||
|
||||
if (filter_grad) {
|
||||
DenseTensor dweight_3d = Empty<T>(
|
||||
dev_ctx, {filter_grad_shape.Get(), filter_grad_shape.size()});
|
||||
for (int g = 0; g < groups; ++g) {
|
||||
DenseTensor out_grad_3d_slice = out_grad_3d.Slice(g, g + 1).Resize(
|
||||
slice_ddim(out_grad_3d.dims(), 1, out_grad_3d.dims().size()));
|
||||
DenseTensor col_buffer_3d_slice = col_buffer_3d.Slice(g, g + 1).Resize(
|
||||
slice_ddim(col_buffer_3d.dims(), 1, col_buffer_3d.dims().size()));
|
||||
DenseTensor dweight_3d_slice = dweight_3d.Slice(g, g + 1).Resize(
|
||||
slice_ddim(dweight_3d.dims(), 1, dweight_3d.dims().size()));
|
||||
|
||||
blas.MatMul(out_grad_3d_slice,
|
||||
false,
|
||||
col_buffer_3d_slice,
|
||||
true,
|
||||
T(1.0),
|
||||
&dweight_3d_slice,
|
||||
T(0.0));
|
||||
}
|
||||
|
||||
// update grad of weights
|
||||
if (using_int32_index) {
|
||||
FilterGradAddup<T, Context, int>(dev_ctx,
|
||||
dweight_3d.numel(),
|
||||
groups,
|
||||
K,
|
||||
M,
|
||||
dweight_3d.data<T>(),
|
||||
filter_grad->data<T>());
|
||||
} else {
|
||||
FilterGradAddup<T, Context, int64_t>(dev_ctx,
|
||||
dweight_3d.numel(),
|
||||
groups,
|
||||
K,
|
||||
M,
|
||||
dweight_3d.data<T>(),
|
||||
filter_grad->data<T>());
|
||||
}
|
||||
}
|
||||
}
|
||||
if (filter_grad) {
|
||||
filter_grad->Resize(filter.dims());
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,187 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/common/hostdevice.h"
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/empty_kernel.h"
|
||||
#include "paddle/phi/kernels/full_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/blas/blas.h"
|
||||
#include "paddle/phi/kernels/funcs/deformable_conv_functor.h"
|
||||
#include "paddle/phi/kernels/transpose_kernel.h"
|
||||
#include "paddle/utils/optional.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void DeformableConvKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& offset,
|
||||
const DenseTensor& filter,
|
||||
const optional<DenseTensor>& mask,
|
||||
const std::vector<int>& strides,
|
||||
const std::vector<int>& paddings,
|
||||
const std::vector<int>& dilations,
|
||||
int deformable_groups,
|
||||
int groups,
|
||||
int im2col_step,
|
||||
DenseTensor* out) {
|
||||
if (x.numel() == 0 || filter.numel() == 0) {
|
||||
Full<T, Context>(dev_ctx, out->dims(), 0, out);
|
||||
return;
|
||||
}
|
||||
|
||||
const int64_t batch_size = static_cast<int64_t>(x.dims()[0]);
|
||||
|
||||
int64_t temp_step = std::min<int64_t>(64, batch_size);
|
||||
if (batch_size % temp_step == 0) {
|
||||
im2col_step = temp_step;
|
||||
}
|
||||
|
||||
std::vector<int64_t> filter_shape_vec(vectorize(filter.dims()));
|
||||
std::vector<int64_t> output_shape_vec(vectorize(out->dims()));
|
||||
|
||||
// col_shape_vec: {c_i * k_h * k_w, im2col_step, o_h, o_w}
|
||||
std::vector<int64_t> col_buffer_shape_vec(filter_shape_vec.size());
|
||||
col_buffer_shape_vec[0] = x.dims()[1] * filter.dims()[2] * filter.dims()[3];
|
||||
col_buffer_shape_vec[1] = im2col_step;
|
||||
for (size_t j = 0; j < filter_shape_vec.size() - 2; ++j) {
|
||||
col_buffer_shape_vec[j + 2] = output_shape_vec[j + 2];
|
||||
}
|
||||
|
||||
std::vector<int64_t> output_buffer_shape_vec(1);
|
||||
output_buffer_shape_vec[0] = batch_size * output_shape_vec[1] *
|
||||
output_shape_vec[2] * output_shape_vec[3];
|
||||
|
||||
DenseTensor col_buffer = Empty<T>(dev_ctx, col_buffer_shape_vec);
|
||||
DenseTensor output_buffer = Empty<T>(dev_ctx, output_buffer_shape_vec);
|
||||
|
||||
int64_t M = output_shape_vec[1] / groups;
|
||||
int64_t N = im2col_step * output_shape_vec[2] * output_shape_vec[3];
|
||||
int64_t K = x.dims()[1] * filter_shape_vec[2] * filter_shape_vec[3] / groups;
|
||||
|
||||
DenseTensor weight_3d;
|
||||
weight_3d.ShareDataWith(filter).Resize({groups, M, K});
|
||||
|
||||
DenseTensor col_buffer_3d;
|
||||
col_buffer_3d.ShareDataWith(col_buffer).Resize({groups, K, N});
|
||||
|
||||
DenseTensor output_4d;
|
||||
output_4d.ShareDataWith(output_buffer)
|
||||
.Resize({batch_size / im2col_step, groups, M, N});
|
||||
|
||||
DDim input_shape = slice_ddim(x.dims(), 1, x.dims().size());
|
||||
std::vector<int64_t> input_shape_vec = vectorize(input_shape);
|
||||
|
||||
int64_t input_dim = x.numel() / x.dims()[0];
|
||||
int64_t input_offset_dim = offset.numel() / offset.dims()[0];
|
||||
int64_t input_mask_dim = mask ? mask->numel() / mask->dims()[0] : 0;
|
||||
|
||||
const T* input_ptr = x.data<T>();
|
||||
const T* offset_ptr = offset.data<T>();
|
||||
const T* mask_ptr = mask ? mask->data<T>() : nullptr;
|
||||
T* col_buffer_ptr = col_buffer.data<T>();
|
||||
|
||||
auto blas = funcs::GetBlas<Context, T>(dev_ctx);
|
||||
|
||||
bool using_int32_index =
|
||||
(x.numel() <= std::numeric_limits<int>::max()) &&
|
||||
(offset.numel() <= std::numeric_limits<int>::max()) &&
|
||||
(filter.numel() <= std::numeric_limits<int>::max()) &&
|
||||
(mask ? mask->numel() <= std::numeric_limits<int>::max() : true) &&
|
||||
(out->numel() <= std::numeric_limits<int>::max());
|
||||
|
||||
for (int64_t i = 0; i < batch_size / im2col_step; ++i) {
|
||||
const T* temp_mask_ptr =
|
||||
mask_ptr ? mask_ptr + i * im2col_step * input_mask_dim : nullptr;
|
||||
if (using_int32_index) {
|
||||
funcs::ModulatedDeformableIm2col<T, Context, int>(
|
||||
dev_ctx,
|
||||
input_ptr + i * im2col_step * input_dim,
|
||||
offset_ptr + i * im2col_step * input_offset_dim,
|
||||
temp_mask_ptr,
|
||||
input_shape_vec,
|
||||
col_buffer_shape_vec,
|
||||
filter_shape_vec,
|
||||
paddings,
|
||||
strides,
|
||||
dilations,
|
||||
deformable_groups,
|
||||
col_buffer_ptr);
|
||||
} else {
|
||||
funcs::ModulatedDeformableIm2col<T, Context, int64_t>(
|
||||
dev_ctx,
|
||||
input_ptr + i * im2col_step * input_dim,
|
||||
offset_ptr + i * im2col_step * input_offset_dim,
|
||||
temp_mask_ptr,
|
||||
input_shape_vec,
|
||||
col_buffer_shape_vec,
|
||||
filter_shape_vec,
|
||||
paddings,
|
||||
strides,
|
||||
dilations,
|
||||
deformable_groups,
|
||||
col_buffer_ptr);
|
||||
}
|
||||
|
||||
DenseTensor output_3d = output_4d.Slice(i, i + 1).Resize(slice_ddim(
|
||||
output_4d.dims(),
|
||||
1,
|
||||
output_4d.dims().size())); // group * C/group * (im2step * H * W)
|
||||
|
||||
// get the product of pixel and weight
|
||||
for (int g = 0; g < groups; ++g) {
|
||||
DenseTensor weight_3d_slice = weight_3d.Slice(g, g + 1).Resize(
|
||||
slice_ddim(weight_3d.dims(), 1, weight_3d.dims().size()));
|
||||
DenseTensor col_buffer_3d_slice = col_buffer_3d.Slice(g, g + 1).Resize(
|
||||
slice_ddim(col_buffer_3d.dims(), 1, col_buffer_3d.dims().size()));
|
||||
DenseTensor output_3d_slice = output_3d.Slice(g, g + 1).Resize(
|
||||
slice_ddim(output_3d.dims(),
|
||||
1,
|
||||
output_3d.dims().size())); // C * ((im2col_step)*H*W))
|
||||
blas.MatMul(weight_3d_slice,
|
||||
false,
|
||||
col_buffer_3d_slice,
|
||||
false,
|
||||
T(1.0),
|
||||
&output_3d_slice,
|
||||
T(0.0));
|
||||
}
|
||||
}
|
||||
|
||||
// swap axis to get the right result when im2col_step is greater than 1
|
||||
if (im2col_step > 1) {
|
||||
std::vector<int> axis(4);
|
||||
axis[0] = 0;
|
||||
axis[1] = 2;
|
||||
axis[2] = 1;
|
||||
axis[3] = 3;
|
||||
|
||||
DenseTensor real_output_buffer = Transpose<T, Context>(
|
||||
dev_ctx,
|
||||
output_4d.Resize(
|
||||
make_ddim({batch_size / im2col_step,
|
||||
output_shape_vec[1],
|
||||
im2col_step,
|
||||
output_shape_vec[2] * output_shape_vec[3]})),
|
||||
axis);
|
||||
|
||||
out->ShareDataWith(real_output_buffer).Resize(output_shape_vec);
|
||||
} else {
|
||||
out->ShareDataWith(output_buffer).Resize(output_shape_vec);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,35 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
|
||||
namespace phi {
|
||||
template <typename T, typename Context>
|
||||
void DependKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const std::vector<const DenseTensor*>& dep,
|
||||
DenseTensor* out) {
|
||||
auto x_name = &x;
|
||||
auto out_name = out;
|
||||
PADDLE_ENFORCE_EQ(x_name,
|
||||
out_name,
|
||||
common::errors::PreconditionNotMet(
|
||||
"Input(X) and Output(Out) variable should be the "
|
||||
"same, but got Input is %s and Output is %s.",
|
||||
x_name,
|
||||
out_name));
|
||||
}
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,215 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "glog/logging.h"
|
||||
#include "paddle/phi/common/amp_type_traits.h"
|
||||
|
||||
#include "paddle/phi/common/type_traits.h"
|
||||
#include "paddle/phi/core/tensor_utils.h"
|
||||
#include "paddle/phi/kernels/cast_kernel.h"
|
||||
#include "paddle/phi/kernels/complex_kernel.h"
|
||||
#include "paddle/phi/kernels/determinant_grad_kernel.h"
|
||||
#include "paddle/phi/kernels/elementwise_multiply_kernel.h"
|
||||
#include "paddle/phi/kernels/empty_kernel.h"
|
||||
#include "paddle/phi/kernels/full_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
#include "paddle/phi/kernels/funcs/matrix_inverse.h"
|
||||
#include "paddle/phi/kernels/funcs/unsqueeze.h"
|
||||
#include "paddle/phi/kernels/transpose_kernel.h"
|
||||
|
||||
namespace phi {
|
||||
namespace detail {
|
||||
|
||||
// epsilon_ should be smaller if linalg.det achieves higher precision
|
||||
template <typename T>
|
||||
struct FoundZeroEpsilon {
|
||||
// default for float16
|
||||
static constexpr T value() { return static_cast<T>(1e-3f); }
|
||||
};
|
||||
|
||||
template <>
|
||||
struct FoundZeroEpsilon<float> {
|
||||
static constexpr float value() { return 1e-5f; }
|
||||
};
|
||||
|
||||
template <>
|
||||
struct FoundZeroEpsilon<double> {
|
||||
static constexpr double value() { return 1e-12; }
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct FoundZeroFunctor {
|
||||
using RealType = dtype::Real<T>;
|
||||
|
||||
FoundZeroFunctor(const T* x, int64_t numel, bool* res)
|
||||
: x_(x), numel_(numel), res_(res) {}
|
||||
|
||||
HOSTDEVICE void operator()(size_t idx) const {
|
||||
if (*res_ || idx >= static_cast<size_t>(numel_)) {
|
||||
// found a singular matrix
|
||||
return;
|
||||
}
|
||||
if (abs(x_[idx]) < FoundZeroEpsilon<RealType>::value()) {
|
||||
*res_ = true;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
const T* x_;
|
||||
int64_t numel_;
|
||||
bool* res_;
|
||||
};
|
||||
|
||||
template <typename T, typename Context>
|
||||
inline bool CheckMatrixInvertible(const Context& dev_ctx,
|
||||
const DenseTensor* det) {
|
||||
auto numel = det->numel();
|
||||
|
||||
DenseTensor dev_tensor = Empty<bool, Context>(dev_ctx, {1});
|
||||
|
||||
// set false
|
||||
funcs::SetConstant<Context, bool> zero;
|
||||
zero(dev_ctx, &dev_tensor, false);
|
||||
|
||||
// find whether zero
|
||||
funcs::ForRange<Context> for_range(dev_ctx, numel);
|
||||
FoundZeroFunctor<T> functor(det->data<T>(), numel, dev_tensor.data<bool>());
|
||||
for_range(functor);
|
||||
|
||||
// copy to host
|
||||
DenseTensor cpu_tensor;
|
||||
Copy<Context>(dev_ctx, dev_tensor, CPUPlace(), false, &cpu_tensor);
|
||||
|
||||
// if founded zero, the matrix is not invertible
|
||||
// else the matrix is invertible
|
||||
auto* res = cpu_tensor.data<bool>();
|
||||
return !(*res);
|
||||
}
|
||||
|
||||
} // namespace detail
|
||||
|
||||
template <typename T, typename Context>
|
||||
void DeterminantGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& out,
|
||||
const DenseTensor& out_grad,
|
||||
DenseTensor* x_grad) {
|
||||
if (x_grad && x_grad->numel() == 0) {
|
||||
dev_ctx.template Alloc<T>(x_grad);
|
||||
return;
|
||||
}
|
||||
auto input_dims_size = x.dims().size();
|
||||
if (input_dims_size > 2) {
|
||||
PADDLE_ENFORCE_EQ(
|
||||
out_grad.dims().size() + 2,
|
||||
input_dims_size,
|
||||
common::errors::InvalidArgument(
|
||||
"The grad tensor of det dims size should be 2 less than"
|
||||
" input tensor's, but here differ %d",
|
||||
input_dims_size - out_grad.dims().size()));
|
||||
} else if (input_dims_size == 2) {
|
||||
// input dims size 2 and grad dims size 0 is possible
|
||||
PADDLE_ENFORCE_EQ(
|
||||
out_grad.dims().size(),
|
||||
0,
|
||||
common::errors::InvalidArgument(
|
||||
"The grad tensor of det dims size should be 2 less than"
|
||||
" input tensor's, but here differ %d",
|
||||
input_dims_size - out_grad.dims().size()));
|
||||
} else {
|
||||
// checked in forward, pass
|
||||
}
|
||||
|
||||
// Check Whether the matrix is invertible
|
||||
// (matrix A not invertible) == (det(A)=0)
|
||||
if (!detail::CheckMatrixInvertible<T, Context>(dev_ctx, &out)) {
|
||||
// The matrix is not invertible
|
||||
VLOG(3) << "The input matrix not invertible!";
|
||||
x_grad->Resize(x.dims());
|
||||
Full<T>(dev_ctx, vectorize(x.dims()), static_cast<T>(0.0f), x_grad);
|
||||
return;
|
||||
}
|
||||
|
||||
using MPType = typename MPTypeTrait<T>::Type;
|
||||
|
||||
// The matrix is invertible
|
||||
// let |A| = Determinant(A)
|
||||
// Ref to https://people.maths.ox.ac.uk/gilesm/files/NA-08-01.pdf
|
||||
// we set d|A| = unsqueeze(dA * |A|.conj(), [-1, -2]) *
|
||||
// inverse(A).conj().transpose(-2, -1)
|
||||
|
||||
// First: inverse(A)
|
||||
DenseTensor transpose_inverse_A;
|
||||
{
|
||||
DenseTensor inverse_A;
|
||||
// A must be square matrices!
|
||||
inverse_A.Resize(x.dims());
|
||||
dev_ctx.template Alloc<MPType>(&inverse_A);
|
||||
|
||||
funcs::MatrixInverseFunctor<Context, MPType> mat_inv;
|
||||
if constexpr (!std::is_same_v<MPType, T>) {
|
||||
auto x_mp =
|
||||
Cast<T, Context>(dev_ctx, x, CppTypeToDataType<MPType>::Type());
|
||||
mat_inv(dev_ctx, x_mp, &inverse_A);
|
||||
} else {
|
||||
mat_inv(dev_ctx, x, &inverse_A);
|
||||
}
|
||||
|
||||
auto conj_inverse_A = Conj<MPType>(dev_ctx, inverse_A);
|
||||
VLOG(3) << "inverse(A).conj() dims: " << conj_inverse_A.dims();
|
||||
|
||||
// Second: inverse(A).conj().transpose(-2, -1)
|
||||
transpose_inverse_A = TransposeLast2Dim<MPType>(dev_ctx, conj_inverse_A);
|
||||
VLOG(3) << "(dA * |A|).transpose(-2, -1) dims: "
|
||||
<< transpose_inverse_A.dims();
|
||||
}
|
||||
|
||||
DenseTensor mul_unsqueezed;
|
||||
{
|
||||
DenseTensor mul_dA_detA;
|
||||
// Third: dA * |A|.conj()
|
||||
if constexpr (!std::is_same_v<MPType, T>) {
|
||||
auto out_mp =
|
||||
Cast<T, Context>(dev_ctx, out, CppTypeToDataType<MPType>::Type());
|
||||
auto out_grad_mp = Cast<T, Context>(
|
||||
dev_ctx, out_grad, CppTypeToDataType<MPType>::Type());
|
||||
|
||||
auto conj_out_mp = Conj<MPType>(dev_ctx, out_mp);
|
||||
mul_dA_detA = Multiply<MPType>(dev_ctx, out_grad_mp, conj_out_mp);
|
||||
} else {
|
||||
auto conj_out = Conj<T>(dev_ctx, out);
|
||||
mul_dA_detA = Multiply<T>(dev_ctx, out_grad, conj_out);
|
||||
}
|
||||
VLOG(3) << "dA * |A| dims: " << mul_dA_detA.dims();
|
||||
|
||||
// Fourth: unsqueeze(dA * |A|, [-1, -2])
|
||||
auto unsqueeze1 = funcs::Unsqueeze(mul_dA_detA, -1);
|
||||
mul_unsqueezed = funcs::Unsqueeze(unsqueeze1, -2);
|
||||
VLOG(3) << "unsqueezed(dA * |A|) dims: " << mul_unsqueezed.dims();
|
||||
}
|
||||
|
||||
// Finally: unsqueeze(dA * |A|) * inverse(A)
|
||||
auto res_mp = Multiply<MPType>(dev_ctx, mul_unsqueezed, transpose_inverse_A);
|
||||
VLOG(3) << "unsqueeze(dA * |A|) * inverse(A) dims: " << res_mp.dims();
|
||||
|
||||
x_grad->Resize(x.dims());
|
||||
VLOG(3) << "d|A| dims: " << x_grad->dims();
|
||||
|
||||
Copy(dev_ctx, res_mp, dev_ctx.GetPlace(), false, x_grad);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,168 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <Eigen/Dense>
|
||||
#include <Eigen/LU>
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <vector>
|
||||
|
||||
#include "glog/logging.h"
|
||||
#include "paddle/phi/common/amp_type_traits.h"
|
||||
|
||||
#include "paddle/phi/core/enforce.h"
|
||||
#include "paddle/phi/core/tensor_utils.h"
|
||||
#include "paddle/phi/kernels/determinant_kernel.h"
|
||||
|
||||
namespace phi {
|
||||
namespace detail {
|
||||
template <typename T>
|
||||
class EigenMatrix {};
|
||||
|
||||
template <>
|
||||
class EigenMatrix<float16> {
|
||||
public:
|
||||
using MatrixType = Eigen::Matrix<float16, Eigen::Dynamic, Eigen::Dynamic>;
|
||||
};
|
||||
|
||||
template <>
|
||||
class EigenMatrix<float> {
|
||||
public:
|
||||
using MatrixType = Eigen::MatrixXf;
|
||||
};
|
||||
|
||||
template <>
|
||||
class EigenMatrix<double> {
|
||||
public:
|
||||
using MatrixType = Eigen::MatrixXd;
|
||||
};
|
||||
|
||||
inline int64_t GetBatchCount(const DDim dims) {
|
||||
int64_t batch_count = 1;
|
||||
auto dim_size = dims.size();
|
||||
PADDLE_ENFORCE_GE(
|
||||
dim_size,
|
||||
2,
|
||||
common::errors::InvalidArgument(
|
||||
"the input matrix dimension size should greater than 2."));
|
||||
|
||||
// Cumulative multiplying each dimension until the last 2 to get the batch
|
||||
// count,
|
||||
// for example a tensor with shape [3,3,3,3], the batch count of matrices is
|
||||
// 9.
|
||||
for (int64_t i = 0; i < dims.size() - 2; i++) {
|
||||
batch_count *= dims[i];
|
||||
}
|
||||
|
||||
return batch_count;
|
||||
}
|
||||
} // namespace detail
|
||||
|
||||
template <typename T, typename Context>
|
||||
struct DeterminantFunctor {
|
||||
void operator()(const Context& dev_ctx,
|
||||
const DenseTensor& input,
|
||||
int64_t rank,
|
||||
int64_t batch_count,
|
||||
DenseTensor* output) {
|
||||
std::vector<T> input_vec;
|
||||
std::vector<T> output_vec;
|
||||
TensorToVector(input, dev_ctx, &input_vec);
|
||||
using MPType = typename MPTypeTrait<T>::Type;
|
||||
for (int64_t i = 0; i < batch_count; ++i) { // maybe can be parallel
|
||||
auto begin_iter = input_vec.begin() + i * rank * rank;
|
||||
auto end_iter = input_vec.begin() + (i + 1) * rank * rank;
|
||||
std::vector<T> sub_vec(begin_iter,
|
||||
end_iter); // get every square matrix data
|
||||
typename detail::EigenMatrix<T>::MatrixType matrix(rank, rank);
|
||||
for (int64_t i = 0; i < rank; ++i) {
|
||||
for (int64_t j = 0; j < rank; ++j) {
|
||||
matrix(i, j) = sub_vec[rank * i + j];
|
||||
}
|
||||
}
|
||||
output_vec.push_back(
|
||||
static_cast<T>(matrix.template cast<MPType>().determinant()));
|
||||
}
|
||||
TensorFromVector(output_vec, dev_ctx, output);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T, typename Context>
|
||||
struct DeterminantFunctor<dtype::complex<T>, Context> {
|
||||
void operator()(const Context& dev_ctx,
|
||||
const DenseTensor& input,
|
||||
int64_t rank,
|
||||
int64_t batch_count,
|
||||
DenseTensor* output) {
|
||||
using MatrixType =
|
||||
Eigen::Matrix<std::complex<T>, Eigen::Dynamic, Eigen::Dynamic>;
|
||||
std::vector<dtype::complex<T>> input_vec;
|
||||
std::vector<dtype::complex<T>> output_vec;
|
||||
TensorToVector(input, dev_ctx, &input_vec);
|
||||
for (int64_t i = 0; i < batch_count; ++i) { // maybe can be parallel
|
||||
auto begin_iter = input_vec.begin() + i * rank * rank;
|
||||
auto end_iter = input_vec.begin() + (i + 1) * rank * rank;
|
||||
std::vector<dtype::complex<T>> sub_vec(
|
||||
begin_iter,
|
||||
end_iter); // get every square matrix data
|
||||
MatrixType matrix(rank, rank);
|
||||
for (int64_t i = 0; i < rank; ++i) {
|
||||
for (int64_t j = 0; j < rank; ++j) {
|
||||
matrix(i, j) = static_cast<std::complex<T>>(sub_vec[rank * i + j]);
|
||||
}
|
||||
}
|
||||
output_vec.push_back(
|
||||
static_cast<dtype::complex<T>>(matrix.determinant()));
|
||||
}
|
||||
TensorFromVector(output_vec, dev_ctx, output);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T, typename Context>
|
||||
void DeterminantKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
DenseTensor* out) {
|
||||
if (out && out->numel() == 0) {
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
return;
|
||||
}
|
||||
auto input_dim = vectorize(x.dims());
|
||||
auto input_dim_size = input_dim.size();
|
||||
|
||||
auto batch_count = detail::GetBatchCount(x.dims());
|
||||
VLOG(10) << "input dim:" << x.dims();
|
||||
PADDLE_ENFORCE_GE(
|
||||
input_dim_size,
|
||||
2,
|
||||
common::errors::InvalidArgument("the input matrix dimension size should "
|
||||
"greater than or equal to 2."));
|
||||
PADDLE_ENFORCE_EQ(input_dim[input_dim_size - 1],
|
||||
input_dim[input_dim_size - 2],
|
||||
common::errors::InvalidArgument(
|
||||
"the input matrix should be square matrix."));
|
||||
auto rank = input_dim[input_dim_size - 1]; // square matrix length
|
||||
DeterminantFunctor<T, Context>()(dev_ctx, x, rank, batch_count, out);
|
||||
auto output_dims = slice_ddim(x.dims(), 0, input_dim_size - 2);
|
||||
if (input_dim_size > 2) {
|
||||
out->Resize(output_dims);
|
||||
} else {
|
||||
// when input is a two-dimension matrix, The det value is a number.
|
||||
out->Resize({});
|
||||
}
|
||||
VLOG(10) << "output dim:" << out->dims();
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,49 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
#include "glog/logging.h"
|
||||
#include "paddle/phi/kernels/clip_by_norm_kernel.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void DGCClipByNormKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x_in,
|
||||
const DenseTensor& current_step_in,
|
||||
float max_norm,
|
||||
float rampup_begin_step,
|
||||
DenseTensor* out) {
|
||||
if (static_cast<int>(rampup_begin_step) < 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto current_step_tensor = ¤t_step_in;
|
||||
auto* current_step = current_step_tensor->data<T>();
|
||||
|
||||
VLOG(10) << "current_step:" << *current_step
|
||||
<< ", rampup_begin_step:" << rampup_begin_step;
|
||||
|
||||
if (static_cast<int>(*current_step) < static_cast<int>(rampup_begin_step)) {
|
||||
VLOG(10) << "current_step:" << *current_step
|
||||
<< " < rampup_begin_step:" << rampup_begin_step
|
||||
<< " so doesn't use dgc_clip_by_norm";
|
||||
return;
|
||||
}
|
||||
|
||||
auto* x = &x_in;
|
||||
auto* y = out;
|
||||
return ClipByNormKernel<T>(dev_ctx, *x, max_norm, y);
|
||||
}
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,113 @@
|
||||
// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "glog/logging.h"
|
||||
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/eigen_function.h"
|
||||
#include "paddle/phi/kernels/momentum_kernel.h"
|
||||
#include "paddle/phi/kernels/sgd_kernel.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void DGCMomentumKernel(const Context& dev_ctx,
|
||||
const DenseTensor& param,
|
||||
const DenseTensor& grad,
|
||||
const DenseTensor& velocity,
|
||||
const DenseTensor& learning_rate,
|
||||
const DenseTensor& master_param,
|
||||
const DenseTensor& current_step_tensor,
|
||||
const DenseTensor& nranks_tensor,
|
||||
float mu,
|
||||
bool use_nesterov,
|
||||
const std::string& regularization_method,
|
||||
float regularization_coeff,
|
||||
bool multi_precision,
|
||||
float rescale_grad,
|
||||
float rampup_begin_step,
|
||||
DenseTensor* param_out,
|
||||
DenseTensor* velocity_out,
|
||||
DenseTensor* master_param_out,
|
||||
DenseTensor* grad_out) {
|
||||
if (static_cast<int>(rampup_begin_step) < 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto* current_step = current_step_tensor.data<T>();
|
||||
|
||||
// nranks
|
||||
const int nranks = static_cast<int>(*nranks_tensor.data<float>());
|
||||
PADDLE_ENFORCE_GT(
|
||||
nranks,
|
||||
1,
|
||||
common::errors::InvalidArgument(
|
||||
"DGC is not useful when num_trainers <= 1, but now nranks=%d",
|
||||
nranks));
|
||||
|
||||
auto grad_e = EigenVector<T>::Flatten(grad);
|
||||
auto grad_out_e = EigenVector<T>::Flatten(*grad_out);
|
||||
|
||||
auto& eigen_ctx = *dev_ctx.eigen_device();
|
||||
|
||||
// NOTE. In dgc_op we multi grad with nranks, so we need /nranks here.
|
||||
grad_out_e.device(eigen_ctx) = (1.0 / nranks) * grad_e;
|
||||
|
||||
VLOG(10) << "current_step:" << *current_step
|
||||
<< ", rampup_begin_step:" << rampup_begin_step;
|
||||
|
||||
if (static_cast<int>(*current_step) < static_cast<int>(rampup_begin_step)) {
|
||||
VLOG(10) << " so use momentum optimizer";
|
||||
|
||||
optional<DenseTensor> master_param_opt(paddle::none);
|
||||
|
||||
MomentumDenseKernel<T>(dev_ctx,
|
||||
param,
|
||||
grad,
|
||||
velocity,
|
||||
learning_rate,
|
||||
master_param_opt,
|
||||
mu,
|
||||
use_nesterov,
|
||||
regularization_method,
|
||||
regularization_coeff,
|
||||
multi_precision,
|
||||
rescale_grad,
|
||||
param_out,
|
||||
velocity_out,
|
||||
master_param_out);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
VLOG(10) << " so use sgd optimizer";
|
||||
|
||||
optional<DenseTensor> master_param_opt(paddle::none);
|
||||
if (multi_precision) {
|
||||
master_param_opt = master_param;
|
||||
}
|
||||
|
||||
SGDDenseKernel<T>(dev_ctx,
|
||||
param,
|
||||
learning_rate,
|
||||
grad,
|
||||
master_param_opt,
|
||||
multi_precision,
|
||||
param_out,
|
||||
master_param_out);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,132 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#if defined(__NVCC__) || defined(__HIPCC__)
|
||||
#include <thrust/device_vector.h>
|
||||
#include <thrust/host_vector.h>
|
||||
#endif
|
||||
|
||||
#include "paddle/phi/kernels/diag_embed_kernel.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T>
|
||||
struct DiagEmbedFunctor {
|
||||
DiagEmbedFunctor(const T* input,
|
||||
int64_t numel,
|
||||
const int64_t* dim,
|
||||
int64_t offset,
|
||||
int64_t dims_size,
|
||||
T* output,
|
||||
const int64_t* strides)
|
||||
: input_(input),
|
||||
numel_(numel),
|
||||
dim_(dim),
|
||||
offset_(offset),
|
||||
dims_size_(dims_size),
|
||||
output_(output),
|
||||
strides_(strides) {}
|
||||
|
||||
HOSTDEVICE void operator()(size_t idx) const {
|
||||
int64_t position = 0;
|
||||
auto numel = numel_;
|
||||
int64_t num = idx;
|
||||
for (int64_t i = 0; i < dims_size_; i++) {
|
||||
numel = numel / dim_[i];
|
||||
position += num / numel * strides_[i];
|
||||
num = num % numel;
|
||||
}
|
||||
output_[position + offset_] = input_[idx];
|
||||
}
|
||||
|
||||
const T* input_;
|
||||
int64_t numel_;
|
||||
const int64_t* dim_;
|
||||
int64_t offset_;
|
||||
int64_t dims_size_;
|
||||
T* output_;
|
||||
const int64_t* strides_;
|
||||
};
|
||||
|
||||
template <typename T, typename Context>
|
||||
void DiagEmbedKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
int offset,
|
||||
int dim1,
|
||||
int dim2,
|
||||
DenseTensor* out) {
|
||||
auto* input_data = x.data<T>();
|
||||
T* out_data = dev_ctx.template Alloc<T>(out);
|
||||
if (out && out->numel() == 0) {
|
||||
return;
|
||||
}
|
||||
funcs::SetConstant<Context, T> set_zero;
|
||||
|
||||
set_zero(dev_ctx, out, static_cast<T>(0.0));
|
||||
|
||||
auto out_dims = out->dims();
|
||||
int dim1_ = dim1 < 0 ? out_dims.size() + dim1 : dim1;
|
||||
int dim2_ = dim2 < 0 ? out_dims.size() + dim2 : dim2;
|
||||
auto stride = common::stride(out_dims);
|
||||
int64_t diag_size;
|
||||
int64_t storage_offset = 0;
|
||||
if (offset >= 0) {
|
||||
int64_t dim = out_dims[dim2_] - offset;
|
||||
diag_size = std::max<int64_t>(std::min(out_dims[dim1_], dim), 0);
|
||||
} else {
|
||||
int64_t dim = out_dims[dim1_] + offset;
|
||||
diag_size = std::max<int64_t>(std::min(dim, out_dims[dim2_]), 0);
|
||||
}
|
||||
if (diag_size == 0) {
|
||||
// skip
|
||||
} else if (offset >= 0) {
|
||||
storage_offset += offset * stride[dim2_];
|
||||
} else {
|
||||
storage_offset -= offset * stride[dim1_];
|
||||
}
|
||||
auto strides = vectorize(stride);
|
||||
strides.erase(strides.begin() + std::max(dim1_, dim2_));
|
||||
strides.erase(strides.begin() + std::min(dim1_, dim2_));
|
||||
strides.push_back(stride[dim1_] + stride[dim2_]);
|
||||
const auto dims = vectorize(x.dims());
|
||||
|
||||
#if defined(__NVCC__) || defined(__HIPCC__)
|
||||
thrust::device_vector<int64_t> dims_vec(dims);
|
||||
const int64_t* dims_arr = thrust::raw_pointer_cast(dims_vec.data());
|
||||
thrust::device_vector<int64_t> strides_vec(strides);
|
||||
const int64_t* strides_arr = thrust::raw_pointer_cast(strides_vec.data());
|
||||
#else
|
||||
const int64_t* dims_arr = dims.data();
|
||||
const int64_t* strides_arr = strides.data();
|
||||
#endif
|
||||
|
||||
funcs::ForRange<Context> for_range(dev_ctx, x.numel());
|
||||
DiagEmbedFunctor<T> functor(input_data,
|
||||
x.numel(),
|
||||
dims_arr,
|
||||
storage_offset,
|
||||
dims.size(),
|
||||
out_data,
|
||||
strides_arr);
|
||||
for_range(functor);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,64 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <unsupported/Eigen/SpecialFunctions>
|
||||
|
||||
#include "paddle/phi/common/amp_type_traits.h"
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T>
|
||||
struct DigammaGradFunctor {
|
||||
DigammaGradFunctor(const T* dout, const T* x, T* output, int64_t numel)
|
||||
: dout_(dout), x_(x), output_(output), numel_(numel) {}
|
||||
|
||||
HOSTDEVICE void operator()(int64_t idx) const {
|
||||
using MPType = typename MPTypeTrait<T>::Type;
|
||||
const MPType mp_dout = static_cast<MPType>(dout_[idx]);
|
||||
const MPType mp_x = static_cast<MPType>(x_[idx]);
|
||||
output_[idx] =
|
||||
static_cast<T>(mp_dout * Eigen::numext::polygamma(MPType(1), mp_x));
|
||||
}
|
||||
|
||||
private:
|
||||
const T* dout_;
|
||||
const T* x_;
|
||||
T* output_;
|
||||
int64_t numel_;
|
||||
};
|
||||
|
||||
template <typename T, typename Context>
|
||||
void DigammaGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& out_grad,
|
||||
DenseTensor* x_grad) {
|
||||
dev_ctx.template Alloc<T>(x_grad);
|
||||
if (x_grad && x_grad->numel() == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto* dout_data = out_grad.data<T>();
|
||||
auto* x_data = x.data<T>();
|
||||
auto* dx_data = x_grad->data<T>();
|
||||
auto numel = out_grad.numel();
|
||||
funcs::ForRange<Context> for_range(dev_ctx, numel);
|
||||
DigammaGradFunctor<T> functor(dout_data, x_data, dx_data, numel);
|
||||
for_range(functor);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,62 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <unsupported/Eigen/SpecialFunctions>
|
||||
|
||||
#include "paddle/phi/common/amp_type_traits.h"
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T>
|
||||
struct DigammaFunctor {
|
||||
DigammaFunctor(const T* input, T* output, int64_t numel)
|
||||
: input_(input), output_(output), numel_(numel) {}
|
||||
|
||||
HOSTDEVICE void operator()(int64_t idx) const {
|
||||
using MPType = typename MPTypeTrait<T>::Type;
|
||||
const MPType mp_input = static_cast<MPType>(input_[idx]);
|
||||
MPType eigen_out = Eigen::numext::digamma(mp_input);
|
||||
constexpr MPType mp_inf = std::numeric_limits<MPType>::infinity();
|
||||
MPType mp_out =
|
||||
mp_input == 0 ? (std::signbit(mp_input) ? mp_inf : -mp_inf) : eigen_out;
|
||||
output_[idx] = static_cast<T>(mp_out);
|
||||
}
|
||||
|
||||
private:
|
||||
const T* input_;
|
||||
T* output_;
|
||||
int64_t numel_;
|
||||
};
|
||||
|
||||
template <typename T, typename Context>
|
||||
void DigammaKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
DenseTensor* out) {
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
if (out && out->numel() == 0) {
|
||||
return;
|
||||
}
|
||||
auto* x_data = x.data<T>();
|
||||
auto* out_data = out->data<T>();
|
||||
auto numel = x.numel();
|
||||
funcs::ForRange<Context> for_range(dev_ctx, numel);
|
||||
DigammaFunctor<T> functor(x_data, out_data, numel);
|
||||
for_range(functor);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,326 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cmath>
|
||||
#include <random>
|
||||
#include "paddle/phi/backends/cpu/cpu_context.h"
|
||||
#include "paddle/phi/backends/gpu/gpu_context.h"
|
||||
#include "paddle/phi/common/amp_type_traits.h"
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/cpu/elementwise.h"
|
||||
#include "paddle/phi/kernels/dirichlet_kernel.h"
|
||||
#include "paddle/phi/kernels/elementwise_divide_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/broadcast_function.h"
|
||||
#include "paddle/phi/kernels/funcs/elementwise_functor.h"
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
#include "paddle/phi/kernels/funcs/reduce_function.h"
|
||||
#include "paddle/phi/kernels/funcs/reduce_functor.h"
|
||||
#include "paddle/phi/kernels/reduce_sum_kernel.h"
|
||||
|
||||
// ROCM hcc doesn't work well with using std:: in kernel functions
|
||||
#if defined(PADDLE_WITH_CUDA)
|
||||
#define COMPAT_LOG log
|
||||
#define COMPAT_POW pow
|
||||
#define COMPAT_SQRT sqrt
|
||||
#else
|
||||
#define COMPAT_LOG std::log
|
||||
#define COMPAT_POW std::pow
|
||||
#define COMPAT_SQRT std::sqrt
|
||||
#endif
|
||||
|
||||
#ifdef PADDLE_WITH_CUDA
|
||||
#include <curand_kernel.h>
|
||||
#endif
|
||||
#ifdef PADDLE_WITH_HIP
|
||||
#include <hiprand_kernel.h>
|
||||
#endif
|
||||
|
||||
#if defined(PADDLE_WITH_CUDA)
|
||||
using COMPAT_RANDSTATEPHILOX4_32_10_T = curandStatePhilox4_32_10_t;
|
||||
#define COMPAT_RAND_INIT curand_init
|
||||
#define COMPAT_RAND_UNIFORM curand_uniform
|
||||
#define COMPAT_RAND_NORMAL curand_normal
|
||||
#elif defined(PADDLE_WITH_HIP)
|
||||
using COMPAT_RANDSTATEPHILOX4_32_10_T = hiprandStatePhilox4_32_10_t;
|
||||
#define COMPAT_RAND_INIT hiprand_init
|
||||
#define COMPAT_RAND_UNIFORM hiprand_uniform
|
||||
#define COMPAT_RAND_NORMAL hiprand_normal
|
||||
#endif
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename ScalarT, typename SamplerT>
|
||||
struct BaseSampler {
|
||||
SamplerT sampler_;
|
||||
HOSTDEVICE BaseSampler(const SamplerT& sampler) : sampler_(sampler) {}
|
||||
HOSTDEVICE ScalarT sample() {
|
||||
// Sometimes convert float to float16/bfloat16
|
||||
return static_cast<ScalarT>(sampler_());
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Context, typename T>
|
||||
struct GammaSampler {
|
||||
void operator()(const Context& dev_ctx,
|
||||
const DenseTensor& alpha,
|
||||
DenseTensor* out);
|
||||
};
|
||||
|
||||
template <typename Context, typename T>
|
||||
struct DirichletSampler {
|
||||
void operator()(const Context& dev_ctx,
|
||||
const DenseTensor& alpha,
|
||||
DenseTensor* out);
|
||||
};
|
||||
|
||||
// `sample_gamma` is d from Numpy's distributions.c, and add support for
|
||||
// paddle data type and code style.
|
||||
// Source MIT licensed:
|
||||
/* Copyright 2005 Robert Kern (robert.kern@gmail.com)
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included
|
||||
* in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
template <typename ScalarT,
|
||||
typename AccscalarT,
|
||||
typename UniformSamplerT,
|
||||
typename NormalSamplerT>
|
||||
HOSTDEVICE ScalarT
|
||||
sample_gamma(ScalarT alpha,
|
||||
BaseSampler<AccscalarT, UniformSamplerT> standard_uniform,
|
||||
BaseSampler<AccscalarT, NormalSamplerT> standard_normal) {
|
||||
using MPTypeScalar = typename MPTypeTrait<ScalarT>::Type;
|
||||
using MPTypeAccscalar = typename MPTypeTrait<AccscalarT>::Type;
|
||||
|
||||
MPTypeAccscalar mp_scale = static_cast<MPTypeAccscalar>(1.0f);
|
||||
MPTypeScalar mp_alpha = static_cast<MPTypeScalar>(alpha);
|
||||
|
||||
// Boost alpha for higher acceptance probability.
|
||||
if (mp_alpha < 1.0f) {
|
||||
if (mp_alpha == 0.f) return static_cast<ScalarT>(0.f);
|
||||
MPTypeAccscalar mp_sample =
|
||||
static_cast<MPTypeAccscalar>(standard_uniform.sample());
|
||||
mp_scale *= COMPAT_POW(1 - mp_sample, 1.0f / mp_alpha);
|
||||
mp_alpha += 1.0f;
|
||||
}
|
||||
|
||||
// This implements the acceptance-rejection method of Marsaglia and Tsang
|
||||
// (2000)
|
||||
// doi:10.1145/358407.358414
|
||||
const MPTypeAccscalar d = mp_alpha - 1.0f / 3.0f;
|
||||
const MPTypeAccscalar c = 1.0f / COMPAT_SQRT(9.0f * d);
|
||||
for (;;) {
|
||||
MPTypeAccscalar x, y;
|
||||
do {
|
||||
x = static_cast<MPTypeAccscalar>(standard_normal.sample());
|
||||
y = 1.0f + c * x;
|
||||
} while (y <= 0);
|
||||
const MPTypeAccscalar v = y * y * y;
|
||||
const MPTypeAccscalar u =
|
||||
1 - static_cast<MPTypeAccscalar>(standard_uniform.sample());
|
||||
const MPTypeAccscalar xx = x * x;
|
||||
if (u < 1.0f - 0.0331f * xx * xx)
|
||||
return static_cast<ScalarT>(mp_scale * d * v);
|
||||
if (COMPAT_LOG(u) < 0.5f * xx + d * (1.0f - v + COMPAT_LOG(v)))
|
||||
return static_cast<ScalarT>(mp_scale * d * v);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename UniformSamplerT, typename NormalSamplerT>
|
||||
struct GammaCPUFunctor {
|
||||
GammaCPUFunctor(const T* alpha,
|
||||
T* gamma,
|
||||
BaseSampler<T, UniformSamplerT> uniform,
|
||||
BaseSampler<T, NormalSamplerT> normal)
|
||||
: alpha_(alpha), gamma_(gamma), uniform_(uniform), normal_(normal) {}
|
||||
|
||||
HOST void operator()(int64_t index) {
|
||||
auto sample = sample_gamma<T, T, UniformSamplerT, NormalSamplerT>(
|
||||
alpha_[index], uniform_, normal_);
|
||||
gamma_[index] = std::max(std::numeric_limits<T>::min(), sample);
|
||||
}
|
||||
|
||||
const T* alpha_;
|
||||
T* gamma_;
|
||||
BaseSampler<T, UniformSamplerT> uniform_;
|
||||
BaseSampler<T, NormalSamplerT> normal_;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct GammaSampler<CPUContext, T> {
|
||||
void operator()(const CPUContext& dev_ctx,
|
||||
const DenseTensor& alpha,
|
||||
DenseTensor* out) {
|
||||
auto generator = dev_ctx.GetGenerator()->GetCPUEngine();
|
||||
|
||||
auto uniform = [&generator]() -> T {
|
||||
std::uniform_real_distribution<T> u(0.0, 1.0);
|
||||
return u(*generator);
|
||||
};
|
||||
BaseSampler<T, decltype(uniform)> standard_uniform(uniform);
|
||||
|
||||
auto normal = [&generator]() {
|
||||
std::normal_distribution<T> n(0.0, 1.0);
|
||||
return n(*generator);
|
||||
};
|
||||
BaseSampler<T, decltype(normal)> standard_normal(normal);
|
||||
|
||||
GammaCPUFunctor<T, decltype(uniform), decltype(normal)> gamma_functor(
|
||||
alpha.data<T>(), out->data<T>(), standard_uniform, standard_normal);
|
||||
funcs::ForRange<CPUContext> for_range(dev_ctx, out->numel());
|
||||
for_range(gamma_functor);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct DirichletSampler<CPUContext, T> {
|
||||
void operator()(const CPUContext& dev_ctx,
|
||||
const DenseTensor& alpha,
|
||||
DenseTensor* out) {
|
||||
// sample from K gamma distributions, where K=alpha.numel()
|
||||
DenseTensor gamma_samples;
|
||||
gamma_samples.Resize(alpha.dims());
|
||||
dev_ctx.template Alloc<T>(&gamma_samples);
|
||||
|
||||
GammaSampler<CPUContext, T> gamma_sampler;
|
||||
gamma_sampler(dev_ctx, alpha, &gamma_samples);
|
||||
|
||||
// normalize them into a simplex, along the last axis
|
||||
DenseTensor gamma_sum;
|
||||
auto new_shape = gamma_samples.dims();
|
||||
new_shape[new_shape.size() - 1] = 1;
|
||||
gamma_sum.Resize(new_shape);
|
||||
dev_ctx.template Alloc<T>(&gamma_sum);
|
||||
|
||||
funcs::ReduceKernelImpl<CPUContext, T, T, funcs::SumFunctor>(
|
||||
dev_ctx,
|
||||
gamma_samples,
|
||||
&gamma_sum,
|
||||
{new_shape.size() - 1},
|
||||
true,
|
||||
false);
|
||||
|
||||
funcs::ElementwiseCompute<funcs::DivideFunctor<T>, T>(
|
||||
dev_ctx, gamma_samples, gamma_sum, funcs::DivideFunctor<T>(), out);
|
||||
}
|
||||
};
|
||||
|
||||
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
|
||||
template <typename T>
|
||||
struct GammaCUDAFunctor {
|
||||
GammaCUDAFunctor(const T* alpha, T* gamma, uint64_t seed, uint64_t offset)
|
||||
: alpha_(alpha), gamma_(gamma), seed_(seed), offset_(offset) {}
|
||||
|
||||
DEVICE void operator()(int64_t index) {
|
||||
// curand initialization
|
||||
COMPAT_RANDSTATEPHILOX4_32_10_T state;
|
||||
COMPAT_RAND_INIT(
|
||||
/*seed=*/seed_, /*subsequence=*/index, /*offset=*/offset_, &state);
|
||||
|
||||
// sample
|
||||
auto uniform_lambda = [&state]() { return COMPAT_RAND_UNIFORM(&state); };
|
||||
BaseSampler<T, decltype(uniform_lambda)> standard_uniform(uniform_lambda);
|
||||
auto normal_lambda = [&state]() { return COMPAT_RAND_NORMAL(&state); };
|
||||
BaseSampler<T, decltype(normal_lambda)> standard_normal(normal_lambda);
|
||||
|
||||
auto sample =
|
||||
sample_gamma<T, T, decltype(uniform_lambda), decltype(normal_lambda)>(
|
||||
alpha_[index], standard_uniform, standard_normal);
|
||||
gamma_[index] = std::max(std::numeric_limits<T>::min(), sample);
|
||||
}
|
||||
|
||||
const T* alpha_;
|
||||
T* gamma_;
|
||||
const uint64_t seed_;
|
||||
const uint64_t offset_;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct GammaSampler<GPUContext, T> {
|
||||
void operator()(const GPUContext& dev_ctx,
|
||||
const DenseTensor& alpha,
|
||||
DenseTensor* out) {
|
||||
auto p_gen = dev_ctx.GetGenerator();
|
||||
auto seed_and_offset = p_gen->IncrementOffset(10); // hard-coded offset
|
||||
auto seed = seed_and_offset.first;
|
||||
auto offset = seed_and_offset.second;
|
||||
|
||||
GammaCUDAFunctor<T> gamma_functor(
|
||||
alpha.data<T>(), out->data<T>(), seed, offset);
|
||||
funcs::ForRange<GPUContext> for_range(dev_ctx, out->numel());
|
||||
for_range(gamma_functor);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct DirichletSampler<GPUContext, T> {
|
||||
void operator()(const GPUContext& dev_ctx,
|
||||
const DenseTensor& alpha,
|
||||
DenseTensor* out) {
|
||||
// sample from K gamma distributions, where K=alpha.numel()
|
||||
DenseTensor gamma_samples;
|
||||
gamma_samples.Resize(alpha.dims());
|
||||
dev_ctx.template Alloc<T>(&gamma_samples);
|
||||
|
||||
GammaSampler<GPUContext, T> gamma_sampler;
|
||||
gamma_sampler(dev_ctx, alpha, &gamma_samples);
|
||||
|
||||
// normalize them into a simplex, along the last axis
|
||||
DenseTensor gamma_sum;
|
||||
auto new_shape = gamma_samples.dims();
|
||||
new_shape[new_shape.size() - 1] = 1;
|
||||
gamma_sum.Resize(new_shape);
|
||||
dev_ctx.template Alloc<T>(&gamma_sum);
|
||||
|
||||
SumRawKernel<T, GPUContext>(dev_ctx,
|
||||
gamma_samples,
|
||||
{new_shape.size() - 1},
|
||||
true,
|
||||
false,
|
||||
gamma_sum.dtype(),
|
||||
&gamma_sum);
|
||||
DivideKernel<T, GPUContext>(dev_ctx, gamma_samples, gamma_sum, out);
|
||||
}
|
||||
};
|
||||
#endif
|
||||
|
||||
template <typename T, typename Context>
|
||||
void DirichletKernel(const Context& dev_ctx,
|
||||
const DenseTensor& alpha,
|
||||
DenseTensor* out) {
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
DirichletSampler<Context, T> sampler;
|
||||
sampler(dev_ctx, alpha, out);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,73 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/complex_kernel.h"
|
||||
#include "paddle/phi/kernels/elementwise_divide_kernel.h"
|
||||
#include "paddle/phi/kernels/elementwise_multiply_kernel.h"
|
||||
#include "paddle/phi/kernels/elementwise_subtract_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/diag_functor.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
#include "paddle/phi/kernels/funcs/unsqueeze.h"
|
||||
#include "paddle/phi/kernels/matmul_kernel.h"
|
||||
#include "paddle/phi/kernels/transpose_kernel.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void EighGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& out_w,
|
||||
const DenseTensor& out_v,
|
||||
const DenseTensor& dout_w,
|
||||
const DenseTensor& dout_v,
|
||||
DenseTensor* dx) {
|
||||
dev_ctx.template Alloc<T>(dx);
|
||||
if (out_v.numel() == 0) {
|
||||
return;
|
||||
}
|
||||
auto& dims = out_v.dims();
|
||||
const int m = dims[dims.size() - 1];
|
||||
DenseTensor tV = TransposeLast2Dim<T>(dev_ctx, Conj<T>(dev_ctx, out_v));
|
||||
DenseTensor W = Subtract<dtype::Real<T>>(
|
||||
dev_ctx, funcs::Unsqueeze(out_w, -2), funcs::Unsqueeze(out_w, -1));
|
||||
DenseTensor result = Matmul<T>(dev_ctx, tV, dout_v);
|
||||
result.Resize(dims);
|
||||
dev_ctx.template Alloc<T>(&result);
|
||||
|
||||
std::vector<int> out_shape = vectorize<int>(dims);
|
||||
DenseTensor constant;
|
||||
constant.Resize(out_shape);
|
||||
dev_ctx.template Alloc<T>(&constant);
|
||||
funcs::SetConstant<Context, T>()(dev_ctx, &constant, T(0.5));
|
||||
result = Subtract<T>(
|
||||
dev_ctx, result, Conj<T>(dev_ctx, TransposeLast2Dim<T>(dev_ctx, result)));
|
||||
result = Multiply<T>(dev_ctx, result, constant);
|
||||
if (result.type() != W.type()) {
|
||||
auto x_vector = EigenVector<T>::Flatten(result);
|
||||
auto y_vector = EigenVector<dtype::Real<T>>::Flatten(W);
|
||||
auto out_vector = EigenVector<T>::Flatten(result);
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
out_vector.device(place) = x_vector / y_vector;
|
||||
} else {
|
||||
result = Divide<T>(dev_ctx, result, W);
|
||||
}
|
||||
result =
|
||||
funcs::DiagFill<T, dtype::Real<T>>(dev_ctx, m, m, m, 0, dout_w, result);
|
||||
*dx = Matmul<T>(dev_ctx, out_v, Matmul<T>(dev_ctx, result, tV));
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,54 @@
|
||||
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/eigvalsh_grad_kernel.h"
|
||||
|
||||
#include "paddle/phi/kernels/complex_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/complex_functors.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/matmul_kernel.h"
|
||||
#include "paddle/phi/kernels/transpose_kernel.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void EigvalshGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& out_v,
|
||||
const DenseTensor& out_w_grad,
|
||||
const std::string& uplo UNUSED,
|
||||
bool is_test UNUSED,
|
||||
DenseTensor* x_grad) {
|
||||
if (x_grad->numel() == 0) {
|
||||
dev_ctx.template Alloc<T>(x_grad);
|
||||
return;
|
||||
}
|
||||
auto tV = TransposeLast2Dim<T>(dev_ctx, Conj<T>(dev_ctx, out_v));
|
||||
|
||||
x_grad->Resize(out_v.dims());
|
||||
dev_ctx.template Alloc<T>(x_grad);
|
||||
|
||||
auto output_v_vector = EigenVector<T>::Flatten(out_v);
|
||||
auto output_w_grad_vector = EigenVector<dtype::Real<T>>::Flatten(out_w_grad);
|
||||
auto result_vector = EigenVector<T>::Flatten(*x_grad);
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
std::vector<int> broadcast_factor;
|
||||
broadcast_factor.push_back(out_v.dims().at(out_v.dims().size() - 1));
|
||||
result_vector.device(place) =
|
||||
output_v_vector * output_w_grad_vector.broadcast(broadcast_factor);
|
||||
|
||||
*x_grad = Matmul<T>(dev_ctx, *x_grad, tV);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,45 @@
|
||||
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/eigvalsh_kernel.h"
|
||||
|
||||
#include "paddle/phi/kernels/funcs/values_vectors_functor.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void EigvalshKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const std::string& uplo,
|
||||
bool is_test,
|
||||
DenseTensor* out_w,
|
||||
DenseTensor* out_v) {
|
||||
if (x.numel() == 0) {
|
||||
auto x_dim = x.dims();
|
||||
auto w_dim = slice_ddim(x_dim, 0, x_dim.size() - 1);
|
||||
out_w->Resize(w_dim);
|
||||
out_v->Resize(x_dim);
|
||||
dev_ctx.template Alloc<T>(out_w);
|
||||
dev_ctx.template Alloc<T>(out_v);
|
||||
return;
|
||||
}
|
||||
bool is_lower = (uplo == "L");
|
||||
funcs::MatrixEighFunctor<Context, T> functor;
|
||||
if (is_test) {
|
||||
functor(dev_ctx, x, out_w, nullptr, is_lower, false);
|
||||
} else {
|
||||
functor(dev_ctx, x, out_w, out_v, is_lower, true);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,283 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include "glog/logging.h"
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/complex_kernel.h"
|
||||
#include "paddle/phi/kernels/full_kernel.h"
|
||||
#include "paddle/phi/kernels/impl/einsum_kernel_impl.h"
|
||||
#include "paddle/phi/kernels/tile_grad_kernel.h"
|
||||
#include "paddle/phi/kernels/tile_kernel.h"
|
||||
#include "paddle/utils/string/string_helper.h"
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
DenseTensor PerformTileAndReduction(const Context& dev_ctx,
|
||||
const LabelMap& label2type,
|
||||
const LabelMap& label2shape,
|
||||
const std::vector<int64_t>& broadcast_shape,
|
||||
const std::vector<int64_t> x_shape,
|
||||
std::string equ, // value pass
|
||||
DenseTensor& t) { // NOLINT
|
||||
auto tmp_label = equ;
|
||||
auto tmp_union = unique_labels(tmp_label);
|
||||
auto op_label = std::string(tmp_union.begin(), tmp_union.end());
|
||||
VLOG(5) << "Start PerformTileAndReduction equation " << equ
|
||||
<< " with operand shape: "
|
||||
<< paddle::string::join_strings(vectorize<int64_t>(t.dims()), ",");
|
||||
DenseTensor ret;
|
||||
std::vector<int64_t> repeat_times;
|
||||
std::vector<int64_t> resize_dims;
|
||||
std::vector<int64_t> recover_shape;
|
||||
std::vector<int64_t> t_shape = vectorize<int64_t>(t.dims());
|
||||
for (size_t i = 0; i < op_label.size(); i++) {
|
||||
int c = op_label[i];
|
||||
if (label2type[c] == LabelType::Reduction) {
|
||||
repeat_times.push_back(label2shape[c]);
|
||||
resize_dims.push_back(1);
|
||||
recover_shape.push_back(label2shape[c]);
|
||||
t_shape.insert(t_shape.begin() + i, 1);
|
||||
} else {
|
||||
resize_dims.push_back(label2shape[c]);
|
||||
repeat_times.push_back(1);
|
||||
recover_shape.push_back(label2shape[c]);
|
||||
}
|
||||
}
|
||||
PADDLE_ENFORCE_EQ(op_label.size(),
|
||||
t_shape.size(),
|
||||
common::errors::InvalidArgument(
|
||||
"Input shape size doesn't match label nums, input "
|
||||
"shape size: `%d`, but got label nums: `%d`",
|
||||
t_shape.size(),
|
||||
op_label.size()));
|
||||
for (size_t i = 0; i < op_label.size(); i++) {
|
||||
int c = op_label[i];
|
||||
if (label2type[c] == LabelType::Contraction &&
|
||||
t_shape[i] != label2shape[c]) {
|
||||
repeat_times[i] = label2shape[c];
|
||||
resize_dims[i] = 1;
|
||||
}
|
||||
}
|
||||
t.Resize(resize_dims);
|
||||
DenseTensor after_tile;
|
||||
if (std::all_of(repeat_times.begin(), repeat_times.end(), [](int64_t x) {
|
||||
return x == 1;
|
||||
})) {
|
||||
after_tile = t;
|
||||
} else {
|
||||
VLOG(4) << "do TileKernel with repeat_times="
|
||||
<< paddle::string::join_strings(repeat_times, ",");
|
||||
TileKernel<T, Context>(dev_ctx, t, repeat_times, &after_tile);
|
||||
}
|
||||
ret = after_tile;
|
||||
VLOG(5) << "PermformTileAndReduction: recover shape: "
|
||||
<< paddle::string::join_strings(recover_shape, ",");
|
||||
ret.Resize(recover_shape);
|
||||
|
||||
// undiagonalize by einsum equation. only contain undiagonal operations.
|
||||
DenseTensor undiagonal_out;
|
||||
if (op_label != equ) {
|
||||
VLOG(5) << "Undiagonal by einsum with args: " << op_label + "->" + equ;
|
||||
EinsumInferKernel<T, Context>(
|
||||
dev_ctx, {&ret}, op_label + "->" + equ, &undiagonal_out);
|
||||
} else {
|
||||
undiagonal_out = ret;
|
||||
}
|
||||
|
||||
// call TileGradKernel to reverse broadcast operation.
|
||||
VLOG(5) << "After diagonalize, we have tensor with shape: "
|
||||
<< paddle::string::join_strings(
|
||||
vectorize<int64_t>(undiagonal_out.dims()), ',');
|
||||
repeat_times.clear();
|
||||
for (size_t i = 0; i < x_shape.size(); ++i) {
|
||||
VLOG(4) << "broadcast shape is " << broadcast_shape[i] << ", x_shape is "
|
||||
<< x_shape[i];
|
||||
repeat_times.push_back(broadcast_shape[i] / x_shape[i]);
|
||||
}
|
||||
bool is_all_ones = std::all_of(repeat_times.begin(),
|
||||
repeat_times.end(),
|
||||
[](int64_t x) { return x == 1; });
|
||||
if (is_all_ones) {
|
||||
VLOG(4) << "don't need broadcast recover, we just return undiagonal_out.";
|
||||
return undiagonal_out;
|
||||
}
|
||||
DenseTensor tmp_x;
|
||||
DenseTensor broadcast_out;
|
||||
tmp_x.Resize(x_shape);
|
||||
broadcast_out.Resize(x_shape);
|
||||
TileGradKernel<T, Context>(
|
||||
dev_ctx, tmp_x, undiagonal_out, repeat_times, &broadcast_out);
|
||||
VLOG(5) << "After broadcast recover, we have tensor with shape: "
|
||||
<< paddle::string::join_strings(
|
||||
vectorize<int64_t>(broadcast_out.dims()), ',');
|
||||
return broadcast_out;
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void EinsumGradKernel(const Context& dev_ctx,
|
||||
const std::vector<const DenseTensor*>& x,
|
||||
const std::vector<const DenseTensor*>& inner_cache,
|
||||
const DenseTensor& out_grad,
|
||||
const std::string& equation,
|
||||
std::vector<DenseTensor*> x_grad) {
|
||||
VLOG(5) << "Start EinsumGradKernel:";
|
||||
bool has_zero_size_tensor = out_grad.numel() == 0;
|
||||
for (auto& i : x_grad) {
|
||||
if (i != nullptr) {
|
||||
if (i->numel() == 0) {
|
||||
has_zero_size_tensor = true;
|
||||
}
|
||||
Full<T, Context>(dev_ctx, i->dims(), 0, i);
|
||||
}
|
||||
}
|
||||
if (has_zero_size_tensor) return;
|
||||
LabelMap labelshape(0);
|
||||
LabelMap labeltype(LabelType::Reduction);
|
||||
std::vector<LabelMap> label2perms(x.size(), LabelMap(-1));
|
||||
std::vector<char> all_labels; // order: ABO, AO, BO, AB, Reduce
|
||||
std::vector<std::vector<int64_t>> broadcast_shapes(2);
|
||||
std::vector<int64_t> output_dims;
|
||||
|
||||
std::vector<DDim> input_dims;
|
||||
for (auto& i : x) {
|
||||
input_dims.push_back(i->dims());
|
||||
}
|
||||
std::vector<std::string> input_strs;
|
||||
std::string right;
|
||||
ParseEinsumEquation(equation,
|
||||
input_dims,
|
||||
&labelshape,
|
||||
&labeltype,
|
||||
&all_labels,
|
||||
&label2perms,
|
||||
&broadcast_shapes,
|
||||
&output_dims,
|
||||
&right,
|
||||
&input_strs);
|
||||
|
||||
VLOG(4) << "After grad parse einsum equation.";
|
||||
|
||||
auto gather_labels_except_reduction = [&labeltype](std::string all) {
|
||||
std::string res("");
|
||||
for (auto c : all)
|
||||
if (labeltype[static_cast<int>(c)] != LabelType::Reduction) res += c;
|
||||
auto tmp_unique = unique_labels(res);
|
||||
return std::string(tmp_unique.begin(), tmp_unique.end());
|
||||
};
|
||||
if (x.size() == 1) { // Unary
|
||||
auto splits = paddle::string::split_string(equation, "->");
|
||||
auto left = splits[0];
|
||||
right = splits[1];
|
||||
auto new_equation = right + "->" + gather_labels_except_reduction(left);
|
||||
auto new_operands = std::vector<const DenseTensor*>();
|
||||
new_operands.push_back(&out_grad);
|
||||
DenseTensor before_tile;
|
||||
VLOG(5) << "new_equation is " << new_equation;
|
||||
EinsumInferKernel<T, Context>(
|
||||
dev_ctx, new_operands, new_equation, &before_tile);
|
||||
*(x_grad[0]) =
|
||||
PerformTileAndReduction<T, Context>(dev_ctx,
|
||||
labeltype,
|
||||
labelshape,
|
||||
broadcast_shapes[0],
|
||||
vectorize<int64_t>(x[0]->dims()),
|
||||
left,
|
||||
before_tile);
|
||||
#ifndef PADDLE_WITH_XPU // xpu is not support conj now, we just disable it.
|
||||
*(x_grad[0]) = Conj<T, Context>(dev_ctx, *x_grad[0]);
|
||||
#endif
|
||||
} else {
|
||||
auto splits = paddle::string::split_string(equation, "->");
|
||||
auto left = splits[0];
|
||||
auto ops = paddle::string::split_string(left, ",");
|
||||
right = splits[1];
|
||||
auto equation_for_A =
|
||||
ops[1] + "," + right + "->" + gather_labels_except_reduction(ops[0]);
|
||||
auto equation_for_B =
|
||||
right + "," + ops[0] + "->" + gather_labels_except_reduction(ops[1]);
|
||||
auto operands_for_A = std::vector<const DenseTensor*>();
|
||||
auto operands_for_B = std::vector<const DenseTensor*>();
|
||||
DenseTensor dA, dB;
|
||||
#ifndef PADDLE_WITH_XPU // xpu is not support conj now, we just disable it.
|
||||
auto out_grad_conj = Conj<T, Context>(dev_ctx, out_grad);
|
||||
#else
|
||||
auto out_grad_conj = out_grad;
|
||||
#endif
|
||||
// dA = einsum(B, dC)
|
||||
operands_for_A.push_back(x[1]);
|
||||
operands_for_A.push_back(&out_grad_conj);
|
||||
// dB = einsum(dC, A)
|
||||
operands_for_B.push_back(&out_grad_conj);
|
||||
operands_for_B.push_back(x[0]);
|
||||
|
||||
std::vector<DenseTensor> cache(3); // set empty; TA, TB, TdC
|
||||
if (inner_cache.size() >
|
||||
0) { // for compatibility, we can load and run v2.3 EinsumOp.
|
||||
cache[0].ShareBufferWith(*(inner_cache[0]));
|
||||
cache[1].ShareBufferWith(*(inner_cache[1]));
|
||||
}
|
||||
EinsumKernelImpl<T, Context>(dev_ctx,
|
||||
all_labels,
|
||||
labelshape,
|
||||
operands_for_A,
|
||||
equation_for_A,
|
||||
&dA,
|
||||
{&cache[1], &cache[2]},
|
||||
false);
|
||||
|
||||
EinsumKernelImpl<T, Context>(dev_ctx,
|
||||
all_labels,
|
||||
labelshape,
|
||||
operands_for_B,
|
||||
equation_for_B,
|
||||
&dB,
|
||||
{&cache[2], &cache[0]},
|
||||
false);
|
||||
|
||||
// release the cache tensor dTC to save memory right now. they are useless
|
||||
// now.
|
||||
cache.clear();
|
||||
if (x_grad[0]) {
|
||||
*(x_grad[0]) =
|
||||
PerformTileAndReduction<T, Context>(dev_ctx,
|
||||
labeltype,
|
||||
labelshape,
|
||||
broadcast_shapes[0],
|
||||
vectorize<int64_t>(x[0]->dims()),
|
||||
ops[0],
|
||||
dA);
|
||||
VLOG(4) << "After call dA";
|
||||
#ifndef PADDLE_WITH_XPU // xpu is not support conj now, we just disable it.
|
||||
*(x_grad[0]) = Conj<T, Context>(dev_ctx, *x_grad[0]);
|
||||
#endif
|
||||
}
|
||||
if (x_grad[1]) {
|
||||
*(x_grad[1]) =
|
||||
PerformTileAndReduction<T, Context>(dev_ctx,
|
||||
labeltype,
|
||||
labelshape,
|
||||
broadcast_shapes[1],
|
||||
vectorize<int64_t>(x[1]->dims()),
|
||||
ops[1],
|
||||
dB);
|
||||
#ifndef PADDLE_WITH_XPU // xpu is not support conj now, we just disable it.
|
||||
*(x_grad[1]) = Conj<T, Context>(dev_ctx, *x_grad[1]);
|
||||
#endif
|
||||
VLOG(4) << "After call dA";
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,780 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <set>
|
||||
|
||||
#include "glog/logging.h"
|
||||
#include "paddle/common/flags.h"
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/diagonal_kernel.h"
|
||||
#include "paddle/phi/kernels/fill_diagonal_tensor_kernel.h"
|
||||
#include "paddle/phi/kernels/full_kernel.h"
|
||||
#include "paddle/phi/kernels/matmul_kernel.h"
|
||||
#include "paddle/phi/kernels/reduce_sum_kernel.h"
|
||||
#include "paddle/phi/kernels/tile_kernel.h"
|
||||
#include "paddle/phi/kernels/transpose_kernel.h"
|
||||
#include "paddle/utils/string/string_helper.h"
|
||||
|
||||
PD_DECLARE_bool(einsum_opt);
|
||||
|
||||
namespace phi {
|
||||
|
||||
// check the validation of the Einsum equation.
|
||||
// 1. the label must between 'a' - 'z'.
|
||||
// 2. the dim of the same label must be same.
|
||||
// 3. the broad cast dims in two operands is broadcastable.
|
||||
// 4. there must exist '->' and the default output is complete in python.
|
||||
// may be we can skip validation check in C++ and just put it in python.
|
||||
inline static void ValidationCheck(const std::string& equation) {
|
||||
auto n_part = paddle::string::split_string(equation, "->").size();
|
||||
PADDLE_ENFORCE_EQ(n_part,
|
||||
2,
|
||||
common::errors::InvalidArgument(
|
||||
"Required at least one `->` in equation of EinsumOp."));
|
||||
size_t pos;
|
||||
auto trimmed_equ = equation;
|
||||
if ((pos = trimmed_equ.find("->", 0)) != std::string::npos) {
|
||||
trimmed_equ.replace(pos, 2, "");
|
||||
}
|
||||
auto is_valid_char = [](char c) {
|
||||
if (c >= 'a' && c <= 'z') return true;
|
||||
if (c == ',') return true;
|
||||
return false;
|
||||
};
|
||||
for (auto c : trimmed_equ) {
|
||||
if (!is_valid_char(c))
|
||||
PADDLE_THROW(common::errors::InvalidArgument(
|
||||
"Found invalid char in equation. Einsum only accept `a`-`z` and `...`"
|
||||
"but get:`%c`",
|
||||
c));
|
||||
}
|
||||
}
|
||||
|
||||
enum LabelType {
|
||||
ALL_TYPE = 0,
|
||||
Batch = 1, // ABO
|
||||
AO, // AO -- free label
|
||||
BO, // BO -- free label
|
||||
Contraction, // AB
|
||||
Reduction, // A, B
|
||||
};
|
||||
|
||||
// map a label('a' - 'z') -> int64_t, O(1) speed.
|
||||
class LabelMap {
|
||||
constexpr static int N =
|
||||
26 + 1; // 'a' - 'z' + '.', '.' is for broadcast dims
|
||||
int64_t default_value;
|
||||
int64_t map[N];
|
||||
|
||||
public:
|
||||
explicit LabelMap(int64_t default_value = 0) {
|
||||
this->default_value = default_value;
|
||||
for (size_t i = 0; i < N; ++i) map[i] = default_value;
|
||||
}
|
||||
int64_t& operator[](int label) {
|
||||
int i = label - 'a';
|
||||
return map[i];
|
||||
}
|
||||
int64_t operator[](int label) const {
|
||||
int i = label - 'a';
|
||||
return map[i];
|
||||
}
|
||||
bool exist(char label) { return !is_default(label); }
|
||||
|
||||
private:
|
||||
// non-exist is present by is_default
|
||||
bool is_default(char label) {
|
||||
return (*this)[static_cast<int>(label)] == default_value;
|
||||
}
|
||||
};
|
||||
|
||||
inline std::string label_to_string(const std::vector<char>& all_labels,
|
||||
const LabelMap& label2type) {
|
||||
std::string str;
|
||||
for (int a : all_labels) {
|
||||
std::stringstream ss;
|
||||
ss << label2type[a];
|
||||
str += ss.str();
|
||||
}
|
||||
return str;
|
||||
}
|
||||
|
||||
template <typename CharIterable1, typename CharIterable2>
|
||||
inline std::vector<char> union_labels(const CharIterable1& a,
|
||||
const CharIterable2& b) {
|
||||
LabelMap counter(0);
|
||||
std::vector<char> res;
|
||||
auto f = [&](char c) {
|
||||
if (counter[static_cast<int>(c)] == 0) {
|
||||
res.push_back(c);
|
||||
}
|
||||
counter[static_cast<int>(c)] += 1;
|
||||
};
|
||||
std::for_each(a.begin(), a.end(), f);
|
||||
std::for_each(b.begin(), b.end(), f);
|
||||
return res;
|
||||
}
|
||||
|
||||
template <typename CharIterable>
|
||||
inline std::vector<char> unique_labels(const CharIterable& a) {
|
||||
return union_labels(a, CharIterable());
|
||||
}
|
||||
|
||||
// Apply transforms to all_labels and get another all_labels
|
||||
inline std::vector<char> TransformLabelsOrder(
|
||||
const std::vector<char>& all_labels,
|
||||
const LabelMap& type,
|
||||
std::vector<LabelType> new_order) {
|
||||
std::vector<char> ret;
|
||||
for (auto cnt_type : new_order) {
|
||||
std::vector<char> tmp;
|
||||
for (int c : all_labels) {
|
||||
if (type[c] == cnt_type) tmp.push_back(c);
|
||||
}
|
||||
ret.insert(ret.end(), tmp.begin(), tmp.end());
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
inline static void GlobalInfo(const std::vector<std::string>& op_labels,
|
||||
const std::string& right,
|
||||
LabelMap* label2type,
|
||||
std::vector<char>* sorted_labels) {
|
||||
std::vector<char> all;
|
||||
LabelMap counter(0);
|
||||
for (auto& ch : right) { // char
|
||||
int c = ch;
|
||||
(*label2type)[c] = LabelType::BO;
|
||||
}
|
||||
|
||||
for (auto& op : op_labels) {
|
||||
for (auto& ch : unique_labels(op)) { // char
|
||||
int c = ch;
|
||||
if (!counter.exist(c)) {
|
||||
all.push_back(ch);
|
||||
}
|
||||
counter[c] += 1;
|
||||
if ((*label2type)[c] != LabelType::BO && counter[c] == 2)
|
||||
(*label2type)[c] = LabelType::Contraction;
|
||||
else if (counter[c] == 2)
|
||||
(*label2type)[c] = LabelType::Batch;
|
||||
}
|
||||
}
|
||||
|
||||
// BO is represent Free, so we need find the AO.
|
||||
for (int c : op_labels[0]) {
|
||||
if ((*label2type)[c] == LabelType::BO) (*label2type)[c] = LabelType::AO;
|
||||
}
|
||||
|
||||
if (sorted_labels->size()) {
|
||||
std::set<char> exist(all.begin(), all.end());
|
||||
all.clear();
|
||||
std::for_each(
|
||||
sorted_labels->begin(), sorted_labels->end(), [&exist, &all](char c) {
|
||||
if (exist.count(c)) all.push_back(c);
|
||||
});
|
||||
}
|
||||
|
||||
*sorted_labels = TransformLabelsOrder(all,
|
||||
*label2type,
|
||||
{LabelType::Batch,
|
||||
LabelType::AO,
|
||||
LabelType::BO,
|
||||
LabelType::Contraction,
|
||||
LabelType::Reduction});
|
||||
|
||||
VLOG(5) << "GlobalInfo: sorted_labels after: "
|
||||
<< paddle::string::join_strings(*sorted_labels, ",");
|
||||
}
|
||||
|
||||
inline static void InferLabelShape(
|
||||
const std::vector<std::string>& op_labels,
|
||||
const std::vector<DDim>& inputs,
|
||||
LabelMap* labelshape,
|
||||
std::vector<std::vector<int64_t>>* broadcast_shapes,
|
||||
LabelMap* labeltype) {
|
||||
LabelMap labelshape_copy = *labelshape;
|
||||
VLOG(5) << "Start InferLabelShape";
|
||||
for (size_t i = 0; i < op_labels.size(); ++i) {
|
||||
auto& op_str = op_labels[i];
|
||||
auto& op_dim = inputs[i];
|
||||
VLOG(5) << "i = " << i << " op_str " << op_str << " op_dim " << op_dim;
|
||||
int dim_ptr = 0;
|
||||
for (auto& c : op_str) {
|
||||
if (!labelshape->exist(c) || abs((*labelshape)[c]) == 1) {
|
||||
VLOG(5)
|
||||
<< "if (!labelshape->exist(c) || abs((*labelshape)[c]) == 1) c = "
|
||||
<< c << " (*labelshape)[c] " << (*labelshape)[c]
|
||||
<< " op_dim[dim_ptr] " << op_dim[dim_ptr];
|
||||
(*labelshape)[c] = static_cast<int>(op_dim[dim_ptr]);
|
||||
} else if (abs(op_dim[dim_ptr]) != 1) {
|
||||
VLOG(5) << "if (abs(op_dim[dim_ptr]) != 1) c = " << c
|
||||
<< " (*labelshape)[c] " << (*labelshape)[c]
|
||||
<< " op_dim[dim_ptr] " << op_dim[dim_ptr];
|
||||
PADDLE_ENFORCE_EQ(
|
||||
(*labelshape)[c],
|
||||
op_dim[dim_ptr],
|
||||
common::errors::InvalidArgument(
|
||||
"Same label have different shapes for label: `%c`", c));
|
||||
}
|
||||
dim_ptr++;
|
||||
}
|
||||
}
|
||||
for (size_t i = 0; i < op_labels.size(); ++i) {
|
||||
for (auto& c : op_labels[i]) {
|
||||
// Note: When broadcasting is involved, ensure the gradient is calculated
|
||||
// with respect to the broadcasted shape. For example, in
|
||||
// einsum("ij,ij->j", x(2,2), y(1,2)), y is broadcast to (2,2). The
|
||||
// gradient calculation for x must use this broadcasted shape of y.
|
||||
if (labelshape_copy.exist(c) && labelshape_copy[c] > (*labelshape)[c]) {
|
||||
// Strict check for the situation.
|
||||
PADDLE_ENFORCE_EQ(
|
||||
(*labelshape)[c] == 1 && ((*labeltype)[c] == LabelType::AO ||
|
||||
(*labeltype)[c] == LabelType::BO),
|
||||
true,
|
||||
common::errors::InvalidArgument(
|
||||
"Broadcast dims must be 1 for label: `%c`", c));
|
||||
(*labelshape)[c] = labelshape_copy[c];
|
||||
}
|
||||
(*broadcast_shapes)[i].push_back((*labelshape)[c]);
|
||||
}
|
||||
}
|
||||
for (size_t i = 0; i < op_labels.size(); ++i) {
|
||||
VLOG(5) << "InferLabelShape: After broadcast shape is:"
|
||||
<< paddle::string::join_strings((*broadcast_shapes)[i], ",");
|
||||
}
|
||||
}
|
||||
|
||||
template <class CharIterable>
|
||||
inline static void InferLabelPerm(const CharIterable& op,
|
||||
LabelMap* label2perm) {
|
||||
int cur = 0;
|
||||
for (int c : op) {
|
||||
if (!label2perm->exist(
|
||||
c)) // can appear repeatedly. we just record the first position.
|
||||
(*label2perm)[c] = cur;
|
||||
cur += 1;
|
||||
}
|
||||
}
|
||||
|
||||
inline static void InferOutputDims(const std::string& right,
|
||||
const LabelMap& labelshape,
|
||||
std::vector<int64_t>* output_dims) {
|
||||
for (int c : right) {
|
||||
output_dims->push_back(labelshape[c]);
|
||||
}
|
||||
}
|
||||
//
|
||||
inline static void ParseEinsumEquation(
|
||||
const std::string& equation,
|
||||
const std::vector<DDim>& inputs,
|
||||
LabelMap* labelshape,
|
||||
LabelMap* labeltype,
|
||||
std::vector<char>* all_labels,
|
||||
std::vector<LabelMap>* label2perms,
|
||||
std::vector<std::vector<int64_t>>* broadcast_shapes,
|
||||
std::vector<int64_t>* output_dims,
|
||||
std::string* right,
|
||||
std::vector<std::string>* input_strs) {
|
||||
VLOG(5) << "Start ParseEinsumEquation " << equation;
|
||||
auto results = paddle::string::split_string(equation, "->");
|
||||
auto left = results[0];
|
||||
*right = results[1];
|
||||
auto op_labels = paddle::string::split_string(left, ",");
|
||||
// split_string("i,") -> ["i", ""], we push back a "".
|
||||
// split_string("->") -> [], we push back a "".
|
||||
if (op_labels.empty()) op_labels.emplace_back("");
|
||||
GlobalInfo(op_labels, *right, labeltype, all_labels);
|
||||
InferLabelShape(op_labels, inputs, labelshape, broadcast_shapes, labeltype);
|
||||
VLOG(5) << "Einsum Infershape: right:" << *right;
|
||||
VLOG(5) << "Einsum Infershape: left :"
|
||||
<< paddle::string::join_strings(op_labels, '\n');
|
||||
InferOutputDims(*right, *labelshape, output_dims);
|
||||
for (size_t i = 0; i < inputs.size(); ++i) {
|
||||
InferLabelPerm(op_labels[i], &((*label2perms)[i]));
|
||||
(*input_strs).push_back(std::move(op_labels[i]));
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
std::vector<T> GetLabelIndexByType(const std::vector<char>& all_labels,
|
||||
const LabelMap& type,
|
||||
const LabelMap& perm,
|
||||
LabelType filter) {
|
||||
std::vector<T> res;
|
||||
for (T c : all_labels) {
|
||||
if ((filter == LabelType::ALL_TYPE || type[c] == filter) && perm[c] != -1) {
|
||||
res.push_back(perm[c]);
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
std::vector<T> GetShapeByType(const std::vector<char>& all_labels,
|
||||
const LabelMap& type,
|
||||
const LabelMap& perm,
|
||||
const LabelMap& label2shape,
|
||||
std::set<LabelType> filter) {
|
||||
std::vector<T> res;
|
||||
for (T c : all_labels) {
|
||||
if ((filter.count(LabelType::ALL_TYPE) ||
|
||||
filter.count(LabelType(type[c]))) &&
|
||||
perm[c] != -1) {
|
||||
res.push_back(label2shape[c]);
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
inline static std::vector<int> perm_moveto(int n, int from, int to) {
|
||||
// a permutation means moving `from` to `to`.
|
||||
/*
|
||||
f => t permutation
|
||||
--------------------
|
||||
0 1 2 3 4 5
|
||||
5 => 2 : 0 2 5 2 3 4
|
||||
2 => 5 : 0 1 3 4 5 2
|
||||
we can conclude the following rules.
|
||||
*/
|
||||
if (from < 0) from = n + from;
|
||||
if (to < 0) to = n + to;
|
||||
std::vector<int> res(n);
|
||||
for (int i = 0; i < n; ++i) {
|
||||
res[i] = i;
|
||||
}
|
||||
res[to] = from;
|
||||
auto offset = from > to ? -1 : 1;
|
||||
auto start = from > to ? to + 1 : from;
|
||||
auto end = from > to ? from : to - 1;
|
||||
for (int i = start; i <= end; ++i) {
|
||||
res[i] += offset;
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
DenseTensor Undiagonal(const Context& dev_ctx,
|
||||
const DenseTensor& tensor,
|
||||
size_t insert_pos,
|
||||
size_t axis) {
|
||||
// tensor with shape (3, 4, 5, 2, 1), insert_pos = 5, axis = 2.
|
||||
// output is (3, 4, 5, 2, 1, 5)
|
||||
VLOG(5) << "Start undiagonal with args: insert_pos = " << insert_pos
|
||||
<< ", axis = " << axis;
|
||||
std::vector<int64_t> shape(tensor.dims().size() + 1);
|
||||
int point = 0; // point to the tensor.dims()
|
||||
for (size_t i = 0; i < shape.size(); ++i) {
|
||||
if (i == insert_pos)
|
||||
shape[i] = tensor.dims()[axis];
|
||||
else
|
||||
shape[i] = tensor.dims()[point++];
|
||||
}
|
||||
auto zeros = Full<T, Context>(dev_ctx, shape, 0);
|
||||
auto diags = Transpose<T, Context>(
|
||||
dev_ctx, tensor, perm_moveto(tensor.dims().size(), axis, -1));
|
||||
return FillDiagonalTensor<T, Context>(
|
||||
dev_ctx, zeros, diags, 0, insert_pos, axis + (insert_pos <= axis));
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
DenseTensor PerformUndiagonal(const Context& dev_ctx,
|
||||
const DenseTensor& tensor,
|
||||
const std::string& equ) {
|
||||
// if the equ is 'iijjkij', then the tensor must be 'ijk', so we have enough
|
||||
// information to do un-diagonal with equ.
|
||||
auto res = tensor;
|
||||
LabelMap label2perm(-1);
|
||||
InferLabelPerm(equ, &label2perm);
|
||||
// Un-Diagonal
|
||||
int tot = equ.size();
|
||||
int cur = tot - 1;
|
||||
for (auto it = equ.rbegin(); it != equ.rend(); ++it) {
|
||||
char c = *it;
|
||||
if (cur != label2perm[c]) {
|
||||
// do diagonal, followed by movedim().
|
||||
auto insert_pos = cur - tot + res.dims().size() + 1;
|
||||
res = Undiagonal<T, Context>(dev_ctx, res, insert_pos, label2perm[c]);
|
||||
}
|
||||
--cur;
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
DenseTensor PerformDiagonalAndReduction(
|
||||
const Context& dev_ctx,
|
||||
const DenseTensor& tensor,
|
||||
const std::string& equ,
|
||||
const LabelMap& label2perm,
|
||||
const std::vector<char>& all_labels,
|
||||
const std::vector<int64_t>& broadcast_shape,
|
||||
const LabelMap& label2type) {
|
||||
auto res = tensor;
|
||||
int tot = equ.size();
|
||||
// tiling tensor for broadcast
|
||||
std::vector<int64_t> repeat_times;
|
||||
auto tensor_origin_shape = vectorize(tensor.dims());
|
||||
for (size_t i = 0; i < tensor_origin_shape.size(); ++i) {
|
||||
VLOG(4) << "broadcast shape is " << broadcast_shape[i]
|
||||
<< ", tensor shape is " << tensor_origin_shape[i];
|
||||
repeat_times.push_back(broadcast_shape[i] / tensor_origin_shape[i]);
|
||||
}
|
||||
DenseTensor after_tile;
|
||||
bool is_all_ones = std::all_of(repeat_times.begin(),
|
||||
repeat_times.end(),
|
||||
[](int64_t x) { return x == 1; });
|
||||
if (!is_all_ones) {
|
||||
TileKernel<T, Context>(dev_ctx, res, repeat_times, &after_tile);
|
||||
res = after_tile;
|
||||
}
|
||||
// Diagonal
|
||||
int cur = tot - 1;
|
||||
for (auto it = equ.rbegin(); it != equ.rend(); ++it) {
|
||||
char c = *it;
|
||||
if (cur != label2perm[c]) {
|
||||
// do diagonal, followed by movedim().
|
||||
VLOG(5) << "Do diagonal with shape="
|
||||
<< paddle::string::join_strings(vectorize<int64_t>(res.dims()),
|
||||
',')
|
||||
<< ", axis1=" << cur << ", axis2=" << label2perm[c];
|
||||
res = Diagonal<T, Context>(dev_ctx, res, 0, cur, label2perm[c]);
|
||||
res = Transpose<T, Context>(
|
||||
dev_ctx, res, perm_moveto(res.dims().size(), -1, label2perm[c]));
|
||||
}
|
||||
--cur;
|
||||
}
|
||||
// reduction
|
||||
auto indices = GetLabelIndexByType<int64_t>(
|
||||
all_labels, label2type, label2perm, LabelType::Reduction);
|
||||
VLOG(5) << "call PerformDiagonalAndReduction: with axis: "
|
||||
<< paddle::string::join_strings(indices, ",");
|
||||
if (indices.empty()) return res;
|
||||
return Sum<T, Context>(dev_ctx, res, IntArray(indices), res.dtype(), true);
|
||||
}
|
||||
|
||||
inline bool is_no_need_transpose(const std::vector<int>& axis) {
|
||||
for (size_t i = 0; i < axis.size(); ++i) {
|
||||
if (i != static_cast<size_t>(axis[i])) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
DenseTensor PerformTranspose(const Context& dev_ctx,
|
||||
const DenseTensor& tensor,
|
||||
const LabelMap& label2perm,
|
||||
const std::vector<char>& all_labels,
|
||||
const LabelMap& label2type) {
|
||||
auto axis = GetLabelIndexByType<int>(
|
||||
all_labels, label2type, label2perm, LabelType::ALL_TYPE);
|
||||
VLOG(5) << "PerformTranspose: " << paddle::string::join_strings(axis, ",");
|
||||
if (is_no_need_transpose(axis)) {
|
||||
return tensor;
|
||||
}
|
||||
auto ret = Transpose<T, Context>(dev_ctx, tensor, axis);
|
||||
VLOG(5) << "PerformTranspose: do_transpose()";
|
||||
return ret;
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
DenseTensor PerformContraction(
|
||||
const Context& dev_ctx,
|
||||
const std::vector<const DenseTensor*>& operands,
|
||||
const std::vector<std::string>& input_strs,
|
||||
const std::vector<LabelMap>& label2perm,
|
||||
const std::vector<char>& all_labels,
|
||||
const LabelMap& label2type,
|
||||
const LabelMap& label2shape,
|
||||
const std::vector<std::vector<int64_t>>& broadcast_shapes,
|
||||
std::vector<DenseTensor*> cache,
|
||||
bool use_cache) {
|
||||
auto all_valid = LabelMap(1);
|
||||
auto recover_dim = GetShapeByType<int64_t>(
|
||||
all_labels, label2type, all_valid, label2shape, {LabelType::Batch});
|
||||
auto preprocess = [&](const DenseTensor& t,
|
||||
const LabelMap& perm,
|
||||
const std::vector<int64_t>& broadcast,
|
||||
int operand_idx) -> DenseTensor {
|
||||
// reshape
|
||||
auto frees = GetShapeByType<int64_t>(all_labels,
|
||||
label2type,
|
||||
perm,
|
||||
label2shape,
|
||||
{LabelType::AO, LabelType::BO});
|
||||
auto conts = GetShapeByType<int64_t>(
|
||||
all_labels, label2type, perm, label2shape, {LabelType::Contraction});
|
||||
std::vector<char> reordered_all_labels = all_labels;
|
||||
if (operand_idx == 1) {
|
||||
reordered_all_labels = TransformLabelsOrder(all_labels,
|
||||
label2type,
|
||||
{LabelType::Batch,
|
||||
LabelType::Contraction,
|
||||
LabelType::AO,
|
||||
LabelType::BO,
|
||||
LabelType::Reduction});
|
||||
}
|
||||
// reduction
|
||||
DenseTensor trans_t;
|
||||
if (use_cache && cache[operand_idx] != nullptr &&
|
||||
cache[operand_idx]->IsInitialized()) {
|
||||
trans_t.ShareBufferWith(*(cache[operand_idx]));
|
||||
VLOG(5) << "Cache Used!";
|
||||
} else {
|
||||
auto reduct_t =
|
||||
PerformDiagonalAndReduction<T, Context>(dev_ctx,
|
||||
t,
|
||||
input_strs[operand_idx],
|
||||
perm,
|
||||
all_labels,
|
||||
broadcast_shapes[operand_idx],
|
||||
label2type);
|
||||
trans_t = PerformTranspose<T, Context>(
|
||||
dev_ctx, reduct_t, perm, reordered_all_labels, label2type);
|
||||
if (cache[operand_idx] != nullptr) {
|
||||
std::vector<int64_t> broadcast_shapes_restore(
|
||||
broadcast_shapes[operand_idx].size());
|
||||
|
||||
auto contraction_dim1 =
|
||||
[&](const std::vector<int64_t>& broadcast_shapes,
|
||||
const std::vector<int64_t>& original_shapes) -> bool {
|
||||
bool found = false;
|
||||
for (size_t i = 0; i < broadcast_shapes.size(); ++i) {
|
||||
if (broadcast_shapes[i] != original_shapes[i] &&
|
||||
label2type[input_strs[operand_idx][i]] ==
|
||||
LabelType::Contraction) {
|
||||
broadcast_shapes_restore[i] = original_shapes[i];
|
||||
found = true;
|
||||
} else {
|
||||
broadcast_shapes_restore[i] = broadcast_shapes[i];
|
||||
}
|
||||
}
|
||||
return found;
|
||||
};
|
||||
if (!contraction_dim1(broadcast_shapes[operand_idx],
|
||||
vectorize<int64_t>(t.dims()))) {
|
||||
cache[operand_idx]->ShareBufferWith(trans_t);
|
||||
cache[operand_idx]->Resize(trans_t.dims());
|
||||
VLOG(5) << "Set dims of cache[" << operand_idx
|
||||
<< "]: " << trans_t.dims();
|
||||
} else {
|
||||
auto reduct_t_for_cache =
|
||||
PerformDiagonalAndReduction<T, Context>(dev_ctx,
|
||||
t,
|
||||
input_strs[operand_idx],
|
||||
perm,
|
||||
all_labels,
|
||||
broadcast_shapes_restore,
|
||||
label2type);
|
||||
DenseTensor trans_t_for_cache;
|
||||
trans_t_for_cache = PerformTranspose<T, Context>(dev_ctx,
|
||||
reduct_t_for_cache,
|
||||
perm,
|
||||
reordered_all_labels,
|
||||
label2type);
|
||||
cache[operand_idx]->ShareBufferWith(trans_t_for_cache);
|
||||
cache[operand_idx]->Resize(trans_t_for_cache.dims());
|
||||
VLOG(5) << "Set dims of cache[" << operand_idx
|
||||
<< "]: " << trans_t_for_cache.dims();
|
||||
}
|
||||
}
|
||||
}
|
||||
auto mul_dims = GetShapeByType<int64_t>(
|
||||
all_labels, label2type, perm, label2shape, {LabelType::Batch});
|
||||
recover_dim.insert(recover_dim.end(), frees.begin(), frees.end());
|
||||
if (operand_idx == 0) {
|
||||
mul_dims.push_back(std::accumulate(
|
||||
frees.begin(), frees.end(), 1, std::multiplies<int64_t>()));
|
||||
mul_dims.push_back(std::accumulate(
|
||||
conts.begin(), conts.end(), 1, std::multiplies<int64_t>()));
|
||||
} else {
|
||||
mul_dims.push_back(std::accumulate(
|
||||
conts.begin(), conts.end(), 1, std::multiplies<int64_t>()));
|
||||
mul_dims.push_back(std::accumulate(
|
||||
frees.begin(), frees.end(), 1, std::multiplies<int64_t>()));
|
||||
}
|
||||
VLOG(5) << "PerformContraction: mul_dims: "
|
||||
<< paddle::string::join_strings(mul_dims, ",");
|
||||
trans_t.Resize(mul_dims);
|
||||
return trans_t;
|
||||
};
|
||||
|
||||
// Reduction, Reshape and Matmul
|
||||
DenseTensor after_contraction;
|
||||
if (operands.size() == 2) {
|
||||
auto trans_a =
|
||||
preprocess(*(operands[0]), label2perm[0], broadcast_shapes[0], 0);
|
||||
auto trans_b =
|
||||
preprocess(*(operands[1]), label2perm[1], broadcast_shapes[1], 1);
|
||||
after_contraction =
|
||||
Matmul<T, Context>(dev_ctx, trans_a, trans_b, false, false);
|
||||
} else if (operands.size() == 1) {
|
||||
after_contraction =
|
||||
preprocess(*(operands[0]), label2perm[0], broadcast_shapes[0], 0);
|
||||
}
|
||||
if (recover_dim.empty()) recover_dim.push_back(1);
|
||||
VLOG(5) << "PerformContraction: recover_dim: "
|
||||
<< paddle::string::join_strings(recover_dim, ",");
|
||||
after_contraction.Resize(recover_dim);
|
||||
return after_contraction;
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
DenseTensor TransposeToOutput(const Context& dev_ctx,
|
||||
const DenseTensor& to_trans,
|
||||
const std::vector<char>& right,
|
||||
const std::vector<char>& all_labels) {
|
||||
std::vector<int> axis;
|
||||
for (char c : right) {
|
||||
auto it = std::find(all_labels.begin(), all_labels.end(), c);
|
||||
PADDLE_ENFORCE_NE(it,
|
||||
all_labels.end(),
|
||||
common::errors::InvalidArgument("Must in all_labels."));
|
||||
axis.push_back(it - all_labels.begin());
|
||||
}
|
||||
if (is_no_need_transpose(axis)) {
|
||||
return to_trans;
|
||||
}
|
||||
VLOG(5) << "call TransposeToOutput: with axis: "
|
||||
<< paddle::string::join_strings(axis, ",")
|
||||
<< " to trans dims is: " << to_trans.dims();
|
||||
auto output = Transpose<T, Context>(dev_ctx, to_trans, axis);
|
||||
VLOG(5) << "After Transpose.";
|
||||
return output;
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void EinsumKernelImpl(const Context& dev_ctx,
|
||||
const std::vector<char>& forward_all_labels,
|
||||
const LabelMap& forward_label_shape,
|
||||
const std::vector<const DenseTensor*>& inputs,
|
||||
const std::string& equation,
|
||||
DenseTensor* out,
|
||||
std::vector<DenseTensor*> cache,
|
||||
bool is_forward = true) {
|
||||
VLOG(5) << "Start EinsumKernelImpl with inputs(" << inputs.size() << "): ";
|
||||
for (auto& i : inputs) {
|
||||
VLOG(5) << " inputs [ " << i << " ].shape=" << i->dims();
|
||||
}
|
||||
ValidationCheck(equation);
|
||||
// collect the following information to prepare einsum.
|
||||
LabelMap labelshape(0);
|
||||
LabelMap labeltype(LabelType::Reduction);
|
||||
std::vector<LabelMap> label2perms(inputs.size(), LabelMap(-1));
|
||||
std::vector<char> all_labels; // order: ABO, AO, BO, AB, Reduce
|
||||
std::vector<std::vector<int64_t>> broadcast_shapes(2);
|
||||
std::vector<int64_t> output_dims;
|
||||
|
||||
std::vector<DDim> input_dims;
|
||||
for (auto& i : inputs) {
|
||||
input_dims.push_back(i->dims());
|
||||
}
|
||||
std::vector<std::string> input_strs;
|
||||
std::string right;
|
||||
if (!is_forward) {
|
||||
all_labels = forward_all_labels;
|
||||
labelshape = forward_label_shape;
|
||||
}
|
||||
ParseEinsumEquation(equation,
|
||||
input_dims,
|
||||
&labelshape,
|
||||
&labeltype,
|
||||
&all_labels,
|
||||
&label2perms,
|
||||
&broadcast_shapes,
|
||||
&output_dims,
|
||||
&right,
|
||||
&input_strs);
|
||||
if (inputs.size() > 2) {
|
||||
PADDLE_THROW(common::errors::InvalidArgument(
|
||||
"EinsumOp kernel only support len(operands) between (0, 2]. Use "
|
||||
"opt_einsum first to convert multi-variable to binary-variable."));
|
||||
}
|
||||
auto after_contraction = PerformContraction<T, Context>(dev_ctx,
|
||||
inputs,
|
||||
input_strs,
|
||||
label2perms,
|
||||
all_labels,
|
||||
labeltype,
|
||||
labelshape,
|
||||
broadcast_shapes,
|
||||
cache,
|
||||
!is_forward);
|
||||
*out = TransposeToOutput<T, Context>(
|
||||
dev_ctx, after_contraction, unique_labels(right), all_labels);
|
||||
*out = PerformUndiagonal<T, Context>(dev_ctx, *out, right);
|
||||
out->Resize(output_dims);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void EinsumKernel(const Context& dev_ctx,
|
||||
const std::vector<const DenseTensor*>& inputs,
|
||||
const std::string& equation,
|
||||
DenseTensor* out,
|
||||
std::vector<DenseTensor*> cache,
|
||||
std::vector<DenseTensor*> xshape UNUSED) {
|
||||
for (const auto& input : inputs) {
|
||||
if (input->numel() == 0) {
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
if (out->numel() > 0) {
|
||||
std::vector<int64_t> vec_dims = vectorize(out->dims());
|
||||
Full<T, Context>(dev_ctx, IntArray(vec_dims), static_cast<T>(0), out);
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
std::vector<char> tmp;
|
||||
LabelMap labelshape_holder;
|
||||
// for the sake of compatibility, we may load and run v2.3 EinsumOp. Output
|
||||
// may have nullptr and the cache.size() is not equal to inputs.size(). refer
|
||||
// to BuildPhiKernelContext for details.
|
||||
int diff = inputs.size() - cache.size();
|
||||
for (int i = 0; i < diff; ++i) {
|
||||
cache.push_back(nullptr);
|
||||
}
|
||||
EinsumKernelImpl<T, Context>(dev_ctx,
|
||||
tmp,
|
||||
labelshape_holder,
|
||||
inputs,
|
||||
equation,
|
||||
out,
|
||||
cache,
|
||||
/*forward=*/true);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void EinsumInferKernel(const Context& dev_ctx,
|
||||
const std::vector<const DenseTensor*>& inputs,
|
||||
const std::string& equation,
|
||||
DenseTensor* out) {
|
||||
std::vector<char> place_holder;
|
||||
LabelMap labelshape_holder;
|
||||
std::vector<DenseTensor*> cache_tensor(
|
||||
inputs.size()); // set empty; TA, TB, TdC
|
||||
for (size_t i = 0; i < inputs.size(); ++i) {
|
||||
cache_tensor[i] = nullptr;
|
||||
}
|
||||
EinsumKernelImpl<T, Context>(dev_ctx,
|
||||
place_holder,
|
||||
labelshape_holder,
|
||||
inputs,
|
||||
equation,
|
||||
out,
|
||||
cache_tensor,
|
||||
true);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,84 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/elementwise_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/elementwise_base.h"
|
||||
#include "paddle/phi/kernels/funcs/elementwise_functor.h"
|
||||
#if defined(__NVCC__) || defined(__HIPCC__) || defined(__xpu__)
|
||||
#include "paddle/phi/kernels/funcs/broadcast_function.h"
|
||||
#endif
|
||||
|
||||
namespace phi {
|
||||
|
||||
#define DEFINE_CPU_ELEMENTWISE_OP(name) \
|
||||
template <typename T, typename Context> \
|
||||
void name##RawKernel(const Context& dev_ctx, \
|
||||
const DenseTensor& x, \
|
||||
const DenseTensor& y, \
|
||||
int axis, \
|
||||
DenseTensor* out) { \
|
||||
dev_ctx.template Alloc<T>(out); \
|
||||
if (x.dims() == y.dims()) { \
|
||||
SameDimsElementwiseCompute<SameDims##name##Functor<CPUContext, T>>()( \
|
||||
dev_ctx, x, y, out); \
|
||||
} else { \
|
||||
auto x_dims = x.dims(); \
|
||||
auto y_dims = y.dims(); \
|
||||
if (x_dims.size() >= y_dims.size()) { \
|
||||
funcs::ElementwiseCompute<funcs::name##Functor<T>, T>( \
|
||||
dev_ctx, x, y, funcs::name##Functor<T>(), out, axis); \
|
||||
} else { \
|
||||
funcs::ElementwiseCompute<funcs::Inverse##name##Functor<T>, T>( \
|
||||
dev_ctx, x, y, funcs::Inverse##name##Functor<T>(), out, axis); \
|
||||
} \
|
||||
} \
|
||||
}
|
||||
|
||||
#define DEFINE_CUDA_ELEMENTWISE_OP(name) \
|
||||
template <typename T, typename Context> \
|
||||
void name##RawKernel(const Context& dev_ctx, \
|
||||
const DenseTensor& x, \
|
||||
const DenseTensor& y, \
|
||||
int axis, \
|
||||
DenseTensor* out) { \
|
||||
std::vector<const DenseTensor*> inputs = {&x, &y}; \
|
||||
std::vector<DenseTensor*> outputs = {out}; \
|
||||
dev_ctx.template Alloc<T>(out); \
|
||||
funcs::BroadcastKernel<T>( \
|
||||
dev_ctx, inputs, &outputs, funcs::name##Functor<T>(), axis); \
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void FMaxKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& y,
|
||||
DenseTensor* out) {
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
funcs::ElementwiseCompute<funcs::FMaxFunctor<T>, T>(
|
||||
dev_ctx, x, y, funcs::FMaxFunctor<T>(), out);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void FMinKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& y,
|
||||
DenseTensor* out) {
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
funcs::ElementwiseCompute<funcs::FMinFunctor<T>, T>(
|
||||
dev_ctx, x, y, funcs::FMinFunctor<T>(), out);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,40 @@
|
||||
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/backends/all_context.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/kernels/erf_grad_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/eigen_function.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ErfGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& out_grad,
|
||||
DenseTensor* x_grad) {
|
||||
dev_ctx.template Alloc<T>(x_grad);
|
||||
|
||||
auto eigen_x = EigenVector<T>::Flatten(x);
|
||||
auto eigen_dout = EigenVector<T>::Flatten(out_grad);
|
||||
auto eigen_dx = EigenVector<T>::Flatten(*x_grad);
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
funcs::EigenErfGrad<std::decay_t<decltype(place)>, T>::Eval(
|
||||
place, eigen_dx, eigen_x, eigen_dout);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,36 @@
|
||||
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/backends/all_context.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/kernels/erf_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/eigen_function.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ErfKernel(const Context& dev_ctx, const DenseTensor& x, DenseTensor* out) {
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
|
||||
auto eigen_out = EigenVector<T>::Flatten(*out);
|
||||
auto eigen_in = EigenVector<T>::Flatten(x);
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
funcs::EigenErf<std::decay_t<decltype(place)>, T>::Eval(
|
||||
place, eigen_out, eigen_in);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,39 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ErfinvGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& out,
|
||||
const DenseTensor& out_grad,
|
||||
DenseTensor* x_grad) {
|
||||
dev_ctx.template Alloc<T>(x_grad);
|
||||
if (x_grad && x_grad->numel() == 0) {
|
||||
return;
|
||||
}
|
||||
auto eigen_out = EigenVector<T>::Flatten(out);
|
||||
auto eigen_dout = EigenVector<T>::Flatten(out_grad);
|
||||
auto eigen_dx = EigenVector<T>::Flatten(*x_grad);
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
T half_sqrt_pi = static_cast<T>(1 / M_2_SQRTPI);
|
||||
eigen_dx.device(place) = half_sqrt_pi * eigen_dout * eigen_out.square().exp();
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,142 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/core/tensor_utils.h"
|
||||
#include "paddle/phi/kernels/full_kernel.h"
|
||||
#include "paddle/phi/kernels/impl/expand_as_kernel_impl.h"
|
||||
namespace phi {
|
||||
template <typename Context, typename T, int Dims>
|
||||
void ExpandAsBackward(const Context& dev_ctx,
|
||||
const DenseTensor& out_grad,
|
||||
const std::vector<int64_t>& reshape_dims_vec,
|
||||
const std::vector<int>& reduce_dims_vec,
|
||||
DenseTensor* in_grad) {
|
||||
size_t reshape_size = reshape_dims_vec.size();
|
||||
size_t reduce_size = reduce_dims_vec.size();
|
||||
dev_ctx.template Alloc<T>(in_grad);
|
||||
auto x_grad = EigenVector<T>::Flatten(*in_grad);
|
||||
Eigen::DSizes<int64_t, Dims * 2> reshape_dims;
|
||||
for (size_t i = 0; i < reshape_size; ++i) {
|
||||
reshape_dims[i] = reshape_dims_vec[i];
|
||||
}
|
||||
Eigen::DSizes<int64_t, Dims> reduce_dims;
|
||||
for (size_t i = 0; i < reduce_size; ++i) {
|
||||
reduce_dims[i] = reduce_dims_vec[i];
|
||||
}
|
||||
auto out_grad0 = EigenVector<T>::Flatten(out_grad);
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
funcs::EigenBroadcastGrad<std::decay_t<decltype(place)>, T, Dims>::Eval(
|
||||
place, x_grad, out_grad0, reduce_dims, reshape_dims);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ExpandAsGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& out_grad,
|
||||
const std::vector<int64_t>& target_shape,
|
||||
DenseTensor* in_grad) {
|
||||
if (out_grad.numel() == 0) {
|
||||
Full<T, Context>(dev_ctx, in_grad->dims(), 0, in_grad);
|
||||
return;
|
||||
}
|
||||
auto x_dims = x.dims();
|
||||
auto out_grad_dims = out_grad.dims();
|
||||
std::vector<int64_t> real_target_shape = vectorize<int64_t>(out_grad_dims);
|
||||
|
||||
if (in_grad->dims() == out_grad_dims) {
|
||||
Copy(dev_ctx, out_grad, dev_ctx.GetPlace(), false, in_grad);
|
||||
return;
|
||||
}
|
||||
|
||||
auto vec_in_dims = vectorize<int64_t>(x_dims);
|
||||
auto diff = real_target_shape.size() - vec_in_dims.size();
|
||||
vec_in_dims.insert(vec_in_dims.begin(), diff, 1);
|
||||
std::vector<int64_t> repeat_times(vec_in_dims.size());
|
||||
for (size_t i = 0; i < vec_in_dims.size(); ++i) {
|
||||
repeat_times[i] = real_target_shape[i] / vec_in_dims[i];
|
||||
}
|
||||
std::vector<int64_t> reshape_dims_vec;
|
||||
std::vector<int> reduce_dims_vec;
|
||||
for (size_t i = 0; i < repeat_times.size(); ++i) {
|
||||
reduce_dims_vec.push_back(reshape_dims_vec.size());
|
||||
reshape_dims_vec.push_back(repeat_times[i]);
|
||||
reshape_dims_vec.push_back(vec_in_dims[i]);
|
||||
}
|
||||
|
||||
int dims = reduce_dims_vec.size();
|
||||
|
||||
PADDLE_ENFORCE_GE(
|
||||
dims,
|
||||
0,
|
||||
errors::InvalidArgument("The rank of the input 'Out@GRAD' for "
|
||||
"expand_as_v2_grad op must be greater than or "
|
||||
"equal to 0, but the value received is %d.",
|
||||
dims));
|
||||
PADDLE_ENFORCE_LE(
|
||||
dims,
|
||||
MAX_RANK_SUPPORTED,
|
||||
errors::InvalidArgument("The rank of the input 'Out@GRAD' for "
|
||||
"expand_as_v2_grad op must be less than or equal "
|
||||
"to %d, but the value received is %d.",
|
||||
MAX_RANK_SUPPORTED,
|
||||
dims));
|
||||
switch (dims) {
|
||||
case 0:
|
||||
ExpandAsBackward<Context, T, 0>(
|
||||
dev_ctx, out_grad, reshape_dims_vec, reduce_dims_vec, in_grad);
|
||||
break;
|
||||
case 1:
|
||||
ExpandAsBackward<Context, T, 1>(
|
||||
dev_ctx, out_grad, reshape_dims_vec, reduce_dims_vec, in_grad);
|
||||
break;
|
||||
case 2:
|
||||
ExpandAsBackward<Context, T, 2>(
|
||||
dev_ctx, out_grad, reshape_dims_vec, reduce_dims_vec, in_grad);
|
||||
break;
|
||||
case 3:
|
||||
ExpandAsBackward<Context, T, 3>(
|
||||
dev_ctx, out_grad, reshape_dims_vec, reduce_dims_vec, in_grad);
|
||||
break;
|
||||
case 4:
|
||||
ExpandAsBackward<Context, T, 4>(
|
||||
dev_ctx, out_grad, reshape_dims_vec, reduce_dims_vec, in_grad);
|
||||
break;
|
||||
case 5:
|
||||
ExpandAsBackward<Context, T, 5>(
|
||||
dev_ctx, out_grad, reshape_dims_vec, reduce_dims_vec, in_grad);
|
||||
break;
|
||||
case 6:
|
||||
ExpandAsBackward<Context, T, 6>(
|
||||
dev_ctx, out_grad, reshape_dims_vec, reduce_dims_vec, in_grad);
|
||||
break;
|
||||
case 7:
|
||||
ExpandAsBackward<Context, T, 7>(
|
||||
dev_ctx, out_grad, reshape_dims_vec, reduce_dims_vec, in_grad);
|
||||
break;
|
||||
case 8:
|
||||
ExpandAsBackward<Context, T, 8>(
|
||||
dev_ctx, out_grad, reshape_dims_vec, reduce_dims_vec, in_grad);
|
||||
break;
|
||||
default:
|
||||
PADDLE_THROW(errors::InvalidArgument(
|
||||
"Only support tensor with rank being between 1 and %d. But "
|
||||
"received tensor's rank = %d.",
|
||||
MAX_RANK_SUPPORTED,
|
||||
dims));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <vector>
|
||||
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/eigen_function.h"
|
||||
#define MAX_RANK_SUPPORTED 8
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename Context, typename T, int Rank>
|
||||
void ExpandAs(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const std::vector<int64_t>& target_shape,
|
||||
DenseTensor* out) {
|
||||
auto in_dims = x.dims();
|
||||
auto vec_in_dims = vectorize<int>(in_dims);
|
||||
auto diff = target_shape.size() - vec_in_dims.size();
|
||||
vec_in_dims.insert(vec_in_dims.begin(), diff, 1);
|
||||
std::vector<int64_t> repeat_times(vec_in_dims.size());
|
||||
if (Rank == 0) {
|
||||
Copy<Context>(dev_ctx, x, dev_ctx.GetPlace(), false, out);
|
||||
return;
|
||||
}
|
||||
for (size_t i = 0; i < vec_in_dims.size(); ++i) {
|
||||
if (target_shape[i] == 0) {
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
return;
|
||||
}
|
||||
if (i < diff) {
|
||||
PADDLE_ENFORCE_GT(
|
||||
target_shape[i],
|
||||
0,
|
||||
errors::InvalidArgument(
|
||||
"The expanded size (%d) for non-existing dimensions must be "
|
||||
"positive for expand_as_v2 op.",
|
||||
target_shape[i]));
|
||||
repeat_times[i] = target_shape[i];
|
||||
} else if (target_shape[i] > 0) {
|
||||
if (vec_in_dims[i] != 1) {
|
||||
PADDLE_ENFORCE_EQ(
|
||||
vec_in_dims[i],
|
||||
target_shape[i],
|
||||
errors::InvalidArgument(
|
||||
"The value (%d) of the non-singleton dimension does not match"
|
||||
" the corresponding value (%d) in shape for expand_as_v2 op.",
|
||||
vec_in_dims[i],
|
||||
target_shape[i]));
|
||||
repeat_times[i] = 1;
|
||||
} else {
|
||||
repeat_times[i] = target_shape[i];
|
||||
}
|
||||
} else {
|
||||
PADDLE_ENFORCE_EQ(
|
||||
target_shape[i],
|
||||
-1,
|
||||
errors::InvalidArgument(
|
||||
"When the value in shape is negative for expand_as_v2 op, "
|
||||
"only -1 is supported, but the value received is %d.",
|
||||
target_shape[i]));
|
||||
repeat_times[i] = 1;
|
||||
}
|
||||
}
|
||||
Eigen::DSizes<int64_t, Rank> bcast_dims;
|
||||
for (size_t i = 0; i < repeat_times.size(); ++i) {
|
||||
bcast_dims[i] = repeat_times[i];
|
||||
}
|
||||
|
||||
DDim new_in_dims = make_ddim(vec_in_dims);
|
||||
DDim out_dims = make_ddim(target_shape);
|
||||
|
||||
out->Resize(out_dims);
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
auto x0 = EigenTensor<T, Rank>::From(x, new_in_dims);
|
||||
auto y = EigenTensor<T, Rank>::From(*out, out_dims);
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
funcs::EigenBroadcast<std::decay_t<decltype(place)>, T, Rank>::Eval(
|
||||
place, y, x0, bcast_dims);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ExpandAsKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const optional<DenseTensor>& y,
|
||||
const std::vector<int64_t>& target_shape,
|
||||
DenseTensor* out) {
|
||||
if (x.numel() == 0 || (y.get_ptr() && y.get_ptr()->numel() == 0)) {
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
return;
|
||||
}
|
||||
auto rank = x.dims().size();
|
||||
auto target_rank = target_shape.size();
|
||||
PADDLE_ENFORCE_GE(target_rank,
|
||||
rank,
|
||||
errors::InvalidArgument(
|
||||
"The rank (%d) of the input 'target_tensor' for "
|
||||
"expand_as_v2 op must be greater than or equal to "
|
||||
"the rank (%d) of the input 'x'.",
|
||||
target_rank,
|
||||
rank));
|
||||
PADDLE_ENFORCE_GE(
|
||||
rank,
|
||||
0,
|
||||
errors::InvalidArgument("The rank (%d) of the input 'x' for "
|
||||
"expand_as_v2 op must be positive.",
|
||||
rank));
|
||||
PADDLE_ENFORCE_LE(target_rank,
|
||||
MAX_RANK_SUPPORTED,
|
||||
errors::InvalidArgument(
|
||||
"The rank (%d) of the input 'target_tensor' for "
|
||||
"expand_as_v2 op must be less than or equal to %d.",
|
||||
target_rank,
|
||||
MAX_RANK_SUPPORTED));
|
||||
|
||||
std::vector<int64_t> real_target_shape = target_shape;
|
||||
if (y.get_ptr()) {
|
||||
real_target_shape = vectorize<int64_t>(y.get_ptr()->dims());
|
||||
}
|
||||
|
||||
switch (target_rank) {
|
||||
case 0:
|
||||
ExpandAs<Context, T, 0>(dev_ctx, x, real_target_shape, out);
|
||||
break;
|
||||
case 1:
|
||||
ExpandAs<Context, T, 1>(dev_ctx, x, real_target_shape, out);
|
||||
break;
|
||||
case 2:
|
||||
ExpandAs<Context, T, 2>(dev_ctx, x, real_target_shape, out);
|
||||
break;
|
||||
case 3:
|
||||
ExpandAs<Context, T, 3>(dev_ctx, x, real_target_shape, out);
|
||||
break;
|
||||
case 4:
|
||||
ExpandAs<Context, T, 4>(dev_ctx, x, real_target_shape, out);
|
||||
break;
|
||||
case 5:
|
||||
ExpandAs<Context, T, 5>(dev_ctx, x, real_target_shape, out);
|
||||
break;
|
||||
case 6:
|
||||
ExpandAs<Context, T, 6>(dev_ctx, x, real_target_shape, out);
|
||||
break;
|
||||
case 7:
|
||||
ExpandAs<Context, T, 7>(dev_ctx, x, real_target_shape, out);
|
||||
break;
|
||||
case 8:
|
||||
ExpandAs<Context, T, 8>(dev_ctx, x, real_target_shape, out);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,187 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/core/tensor_utils.h"
|
||||
#include "paddle/phi/kernels/cast_kernel.h"
|
||||
#include "paddle/phi/kernels/full_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/eigen_function.h"
|
||||
#include "paddle/phi/kernels/impl/expand_kernel_impl.h"
|
||||
|
||||
namespace phi {
|
||||
template <typename Context, typename T, int Dims>
|
||||
void ExpandBackward(const Context& dev_ctx,
|
||||
const DenseTensor& out_grad,
|
||||
const std::vector<int>& reshape_dims_vec,
|
||||
const std::vector<int>& reduce_dims_vec,
|
||||
DenseTensor* in_grad) {
|
||||
size_t reshape_size = reshape_dims_vec.size();
|
||||
size_t reduce_size = reduce_dims_vec.size();
|
||||
dev_ctx.template Alloc<T>(in_grad);
|
||||
in_grad->data<T>();
|
||||
|
||||
if constexpr (std::is_same_v<T, dtype::float16> ||
|
||||
std::is_same_v<T, dtype::bfloat16>) {
|
||||
const DenseTensor out_grad_fp32 =
|
||||
Cast<T, Context>(dev_ctx, out_grad, DataType::FLOAT32);
|
||||
DenseTensor in_grad_fp32;
|
||||
in_grad_fp32.Resize(in_grad->dims());
|
||||
dev_ctx.template Alloc<float>(&in_grad_fp32);
|
||||
|
||||
auto x_grad = EigenVector<float>::Flatten(in_grad_fp32);
|
||||
Eigen::DSizes<int64_t, Dims * 2> reshape_dims;
|
||||
for (size_t i = 0; i < reshape_size; ++i) {
|
||||
reshape_dims[i] = reshape_dims_vec[i];
|
||||
}
|
||||
Eigen::DSizes<int64_t, Dims> reduce_dims;
|
||||
for (size_t i = 0; i < reduce_size; ++i) {
|
||||
reduce_dims[i] = reduce_dims_vec[i];
|
||||
}
|
||||
const auto out_grad0 = EigenVector<float>::Flatten(out_grad_fp32);
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
funcs::EigenBroadcastGrad<std::decay_t<decltype(place)>, float, Dims>::Eval(
|
||||
place, x_grad, out_grad0, reduce_dims, reshape_dims);
|
||||
|
||||
if constexpr (std::is_same_v<T, dtype::float16>) {
|
||||
CastKernel<float, Context>(
|
||||
dev_ctx, in_grad_fp32, DataType::FLOAT16, in_grad);
|
||||
} else {
|
||||
CastKernel<float, Context>(
|
||||
dev_ctx, in_grad_fp32, DataType::BFLOAT16, in_grad);
|
||||
}
|
||||
} else {
|
||||
auto x_grad = EigenVector<T>::Flatten(*in_grad);
|
||||
Eigen::DSizes<int64_t, Dims * 2> reshape_dims;
|
||||
for (size_t i = 0; i < reshape_size; ++i) {
|
||||
reshape_dims[i] = reshape_dims_vec[i];
|
||||
}
|
||||
Eigen::DSizes<int64_t, Dims> reduce_dims;
|
||||
for (size_t i = 0; i < reduce_size; ++i) {
|
||||
reduce_dims[i] = reduce_dims_vec[i];
|
||||
}
|
||||
auto out_grad0 = EigenVector<T>::Flatten(out_grad);
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
funcs::EigenBroadcastGrad<std::decay_t<decltype(place)>, T, Dims>::Eval(
|
||||
place, x_grad, out_grad0, reduce_dims, reshape_dims);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ExpandGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& out_grad,
|
||||
const IntArray& shape,
|
||||
DenseTensor* in_grad) {
|
||||
auto x_dims = x.dims();
|
||||
auto out_grad_dims = out_grad.dims();
|
||||
std::vector<int64_t> expand_shape = vectorize<int64_t>(out_grad_dims);
|
||||
|
||||
if (x.numel() == 0 || out_grad.numel() == 0 ||
|
||||
(in_grad && in_grad->numel() == 0)) {
|
||||
dev_ctx.template Alloc<T>(in_grad);
|
||||
if (in_grad->numel() != 0) {
|
||||
Full<T, Context>(dev_ctx, in_grad->dims(), 0, in_grad);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (in_grad->dims() == out_grad_dims) {
|
||||
Copy(dev_ctx, out_grad, dev_ctx.GetPlace(), false, in_grad);
|
||||
return;
|
||||
}
|
||||
auto vec_in_dims = vectorize<int64_t>(x_dims);
|
||||
auto diff = expand_shape.size() - vec_in_dims.size();
|
||||
vec_in_dims.insert(vec_in_dims.begin(), diff, 1);
|
||||
// 1. reshape_dims_vec is the broadcast parameter.
|
||||
// 2. reduce_dims_vec is the dimension parameter to compute gradients. For
|
||||
// each dimension expanded, the gradients should be summed to original
|
||||
// size.
|
||||
std::vector<int> repeat_times(vec_in_dims.size());
|
||||
for (size_t i = 0; i < vec_in_dims.size(); ++i) {
|
||||
repeat_times[i] = expand_shape[i] / vec_in_dims[i];
|
||||
}
|
||||
std::vector<int> reshape_dims_vec;
|
||||
std::vector<int> reduce_dims_vec;
|
||||
for (size_t i = 0; i < repeat_times.size(); ++i) {
|
||||
reduce_dims_vec.push_back(reshape_dims_vec.size());
|
||||
reshape_dims_vec.push_back(repeat_times[i]);
|
||||
reshape_dims_vec.push_back(vec_in_dims[i]);
|
||||
}
|
||||
|
||||
int dims = reduce_dims_vec.size();
|
||||
|
||||
PADDLE_ENFORCE_GE(dims,
|
||||
0,
|
||||
common::errors::InvalidArgument(
|
||||
"The rank of the input 'Out@GRAD' for "
|
||||
"expand_v2_grad op must be greater than or "
|
||||
"equal to 0, but the value received is %d.",
|
||||
dims));
|
||||
PADDLE_ENFORCE_LE(dims,
|
||||
MAX_RANK_SUPPORTED,
|
||||
common::errors::InvalidArgument(
|
||||
"The rank of the input 'Out@GRAD' for "
|
||||
"expand_v2_grad op must be less than or equal "
|
||||
"to %d, but the value received is %d.",
|
||||
MAX_RANK_SUPPORTED,
|
||||
dims));
|
||||
switch (dims) {
|
||||
case 0:
|
||||
ExpandBackward<Context, T, 1>(
|
||||
dev_ctx, out_grad, reshape_dims_vec, reduce_dims_vec, in_grad);
|
||||
break;
|
||||
case 1:
|
||||
ExpandBackward<Context, T, 1>(
|
||||
dev_ctx, out_grad, reshape_dims_vec, reduce_dims_vec, in_grad);
|
||||
break;
|
||||
case 2:
|
||||
ExpandBackward<Context, T, 2>(
|
||||
dev_ctx, out_grad, reshape_dims_vec, reduce_dims_vec, in_grad);
|
||||
break;
|
||||
case 3:
|
||||
ExpandBackward<Context, T, 3>(
|
||||
dev_ctx, out_grad, reshape_dims_vec, reduce_dims_vec, in_grad);
|
||||
break;
|
||||
case 4:
|
||||
ExpandBackward<Context, T, 4>(
|
||||
dev_ctx, out_grad, reshape_dims_vec, reduce_dims_vec, in_grad);
|
||||
break;
|
||||
case 5:
|
||||
ExpandBackward<Context, T, 5>(
|
||||
dev_ctx, out_grad, reshape_dims_vec, reduce_dims_vec, in_grad);
|
||||
break;
|
||||
case 6:
|
||||
ExpandBackward<Context, T, 6>(
|
||||
dev_ctx, out_grad, reshape_dims_vec, reduce_dims_vec, in_grad);
|
||||
break;
|
||||
case 7:
|
||||
ExpandBackward<Context, T, 7>(
|
||||
dev_ctx, out_grad, reshape_dims_vec, reduce_dims_vec, in_grad);
|
||||
break;
|
||||
case 8:
|
||||
ExpandBackward<Context, T, 8>(
|
||||
dev_ctx, out_grad, reshape_dims_vec, reduce_dims_vec, in_grad);
|
||||
break;
|
||||
default:
|
||||
PADDLE_THROW(common::errors::InvalidArgument(
|
||||
"Only support tensor with rank being between 1 and %d. But "
|
||||
"received tensor's rank = %d.",
|
||||
MAX_RANK_SUPPORTED,
|
||||
dims));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,178 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <vector>
|
||||
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/eigen_function.h"
|
||||
#define MAX_RANK_SUPPORTED 8
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename Context, typename T, int Rank>
|
||||
void Expand(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const IntArray& shape,
|
||||
DenseTensor* out) {
|
||||
auto in_dims = x.dims();
|
||||
auto expand_shape = shape.GetData();
|
||||
auto vec_in_dims = vectorize<int64_t>(in_dims);
|
||||
auto diff = expand_shape.size() - vec_in_dims.size();
|
||||
vec_in_dims.insert(vec_in_dims.begin(), diff, 1);
|
||||
std::vector<int> repeat_times(vec_in_dims.size());
|
||||
if (Rank == 0) {
|
||||
Copy<Context>(dev_ctx, x, dev_ctx.GetPlace(), false, out);
|
||||
return;
|
||||
}
|
||||
for (size_t i = 0; i < vec_in_dims.size(); ++i) {
|
||||
if (i < diff) {
|
||||
PADDLE_ENFORCE_GE(
|
||||
expand_shape[i],
|
||||
0,
|
||||
common::errors::InvalidArgument(
|
||||
"The expanded size (%d) for non-existing dimensions must be "
|
||||
"positive for expand_v2 op.",
|
||||
expand_shape[i]));
|
||||
repeat_times[i] = expand_shape[i];
|
||||
} else if (expand_shape[i] == 0) {
|
||||
PADDLE_ENFORCE_EQ(
|
||||
vec_in_dims[i] == 1 || vec_in_dims[i] == expand_shape[i],
|
||||
true,
|
||||
common::errors::InvalidArgument(
|
||||
"The value (%d) of the non-singleton dimension does not match"
|
||||
" the corresponding value (%d) in shape for expand_v2 op.",
|
||||
vec_in_dims[i],
|
||||
expand_shape[i]));
|
||||
repeat_times[i] = 0;
|
||||
} else if (expand_shape[i] > 0) {
|
||||
if (vec_in_dims[i] != 1) {
|
||||
PADDLE_ENFORCE_EQ(
|
||||
vec_in_dims[i],
|
||||
expand_shape[i],
|
||||
common::errors::InvalidArgument(
|
||||
"The value (%d) of the non-singleton dimension does not match"
|
||||
" the corresponding value (%d) in shape for expand_v2 op.",
|
||||
vec_in_dims[i],
|
||||
expand_shape[i]));
|
||||
repeat_times[i] = 1;
|
||||
} else {
|
||||
repeat_times[i] = expand_shape[i];
|
||||
}
|
||||
} else if (expand_shape[i] == -1) {
|
||||
repeat_times[i] = 1;
|
||||
}
|
||||
}
|
||||
Eigen::DSizes<int64_t, Rank> bcast_dims;
|
||||
for (size_t i = 0; i < repeat_times.size(); ++i) {
|
||||
bcast_dims[i] = repeat_times[i];
|
||||
}
|
||||
|
||||
DDim new_in_dims = make_ddim(vec_in_dims);
|
||||
DDim out_dims(new_in_dims);
|
||||
for (size_t i = 0; i < repeat_times.size(); ++i) {
|
||||
if (repeat_times[i] == 0) {
|
||||
out_dims[i] = 0;
|
||||
} else if (expand_shape[i] == -1) {
|
||||
out_dims[i] = new_in_dims[i];
|
||||
} else {
|
||||
out_dims[i] *= repeat_times[i];
|
||||
}
|
||||
}
|
||||
|
||||
out->Resize(out_dims);
|
||||
auto x0 = EigenTensor<T, Rank>::From(x, new_in_dims);
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
out->data<T>();
|
||||
|
||||
auto y = EigenTensor<T, Rank>::From(*out, out_dims);
|
||||
auto& place = *dev_ctx.eigen_device();
|
||||
funcs::EigenBroadcast<std::decay_t<decltype(place)>, T, Rank>::Eval(
|
||||
place, y, x0, bcast_dims);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void ExpandKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const IntArray& shape,
|
||||
DenseTensor* out) {
|
||||
auto rank = x.dims().size();
|
||||
auto expand_shape = shape.GetData();
|
||||
auto shape_size = expand_shape.size();
|
||||
PADDLE_ENFORCE_GE(
|
||||
rank,
|
||||
0,
|
||||
common::errors::InvalidArgument(
|
||||
"The rank of the input 'X' for expand_v2 op must be positive, "
|
||||
"but the value received is %d.",
|
||||
rank));
|
||||
PADDLE_ENFORCE_LE(
|
||||
rank,
|
||||
MAX_RANK_SUPPORTED,
|
||||
common::errors::InvalidArgument(
|
||||
"The rank of the input 'X' for expand_v2 op must be less than "
|
||||
"or equal to %d, but the value received is %d.",
|
||||
MAX_RANK_SUPPORTED,
|
||||
rank));
|
||||
PADDLE_ENFORCE_GE(
|
||||
shape_size,
|
||||
rank,
|
||||
common::errors::InvalidArgument(
|
||||
"The number (%d) of elements of 'shape' for expand_v2 op must be "
|
||||
"greater than or equal to the rank (%d) of the input 'X'.",
|
||||
shape_size,
|
||||
rank));
|
||||
PADDLE_ENFORCE_LE(
|
||||
shape_size,
|
||||
MAX_RANK_SUPPORTED,
|
||||
common::errors::InvalidArgument(
|
||||
"The number (%d) of elements of 'shape' for expand_v2 op must be "
|
||||
"less than or equal to %d.",
|
||||
shape_size,
|
||||
MAX_RANK_SUPPORTED));
|
||||
rank = std::max(rank, static_cast<int>(shape_size));
|
||||
switch (rank) {
|
||||
case 0:
|
||||
Expand<Context, T, 0>(dev_ctx, x, shape, out);
|
||||
break;
|
||||
case 1:
|
||||
Expand<Context, T, 1>(dev_ctx, x, shape, out);
|
||||
break;
|
||||
case 2:
|
||||
Expand<Context, T, 2>(dev_ctx, x, shape, out);
|
||||
break;
|
||||
case 3:
|
||||
Expand<Context, T, 3>(dev_ctx, x, shape, out);
|
||||
break;
|
||||
case 4:
|
||||
Expand<Context, T, 4>(dev_ctx, x, shape, out);
|
||||
break;
|
||||
case 5:
|
||||
Expand<Context, T, 5>(dev_ctx, x, shape, out);
|
||||
break;
|
||||
case 6:
|
||||
Expand<Context, T, 6>(dev_ctx, x, shape, out);
|
||||
break;
|
||||
case 7:
|
||||
Expand<Context, T, 7>(dev_ctx, x, shape, out);
|
||||
break;
|
||||
case 8:
|
||||
Expand<Context, T, 8>(dev_ctx, x, shape, out);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,56 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/common/scalar.h"
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T>
|
||||
struct EyeFunctor {
|
||||
EyeFunctor(int64_t num_columns, T* output)
|
||||
: num_columns_(num_columns), output_(output) {}
|
||||
|
||||
HOSTDEVICE void operator()(size_t idx) const {
|
||||
output_[idx * num_columns_ + idx] = static_cast<T>(1);
|
||||
}
|
||||
|
||||
int64_t num_columns_;
|
||||
T* output_;
|
||||
};
|
||||
|
||||
template <typename T, typename Context>
|
||||
void EyeKernel(const Context& dev_ctx,
|
||||
const Scalar& num_rows,
|
||||
const Scalar& num_columns,
|
||||
DataType dtype UNUSED,
|
||||
DenseTensor* out) {
|
||||
auto columns = num_columns.to<int64_t>();
|
||||
auto rows = num_rows.to<int64_t>();
|
||||
if (columns == -1) {
|
||||
columns = rows;
|
||||
}
|
||||
T* out_data = dev_ctx.template Alloc<T>(out);
|
||||
funcs::SetConstant<Context, T> set_zero;
|
||||
set_zero(dev_ctx, out, static_cast<T>(0));
|
||||
int64_t num_eyes = (std::min)(rows, columns);
|
||||
funcs::ForRange<Context> for_range(dev_ctx, num_eyes);
|
||||
EyeFunctor<T> functor(columns, out_data);
|
||||
for_range(functor);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,86 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/fake_dequantize_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/fake_dequantize_functor.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void FakeDequantizeMaxAbsKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& scale,
|
||||
float max_range,
|
||||
DenseTensor* out) {
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
funcs::DequantizeFunctor<Context, T>()(
|
||||
dev_ctx, &x, &scale, static_cast<T>(max_range), out);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void FakeChannelWiseDequantizeMaxAbsKernel(
|
||||
const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const std::vector<const DenseTensor*>& scales,
|
||||
const std::vector<int>& quant_bits,
|
||||
int quant_axis,
|
||||
int x_num_col_dims,
|
||||
DenseTensor* out) {
|
||||
int max_range = 1;
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
int scale_num = scales.size();
|
||||
if (scale_num == 1) {
|
||||
PADDLE_ENFORCE_EQ(
|
||||
scales[0]->numel(),
|
||||
x.dims()[quant_axis],
|
||||
common::errors::PreconditionNotMet(
|
||||
"The number of first scale values must be the same with "
|
||||
"quant_axis dimension value of Input(X) when the `Scales` has "
|
||||
"only one element, but %ld != %ld here.",
|
||||
scales[0]->numel(),
|
||||
x.dims()[quant_axis]));
|
||||
max_range *= (std::pow(2, quant_bits[0] - 1) - 1);
|
||||
} else if (scale_num == 2) {
|
||||
PADDLE_ENFORCE_EQ(
|
||||
scales[0]->numel(),
|
||||
x.dims()[x_num_col_dims],
|
||||
common::errors::PreconditionNotMet(
|
||||
"The number of first scale values must be the same with "
|
||||
"corresponding dimension value of Input(X) when the `Scales` "
|
||||
"has two elements, but %ld != %ld here.",
|
||||
scales[0]->numel(),
|
||||
x.dims()[1]));
|
||||
PADDLE_ENFORCE_EQ(scales[1]->numel(),
|
||||
1,
|
||||
common::errors::PreconditionNotMet(
|
||||
"The second scale tensor should only have one "
|
||||
"value at now, but it has %ld values here.",
|
||||
scales[1]->numel()));
|
||||
max_range *= (std::pow(2, quant_bits[0] - 1) - 1) *
|
||||
(std::pow(2, quant_bits[1] - 1) - 1);
|
||||
}
|
||||
funcs::ChannelDequantizeFunctor<Context, T>()(
|
||||
dev_ctx,
|
||||
&x,
|
||||
(const_cast<std::vector<const DenseTensor*>*>(&scales))->data(),
|
||||
scale_num,
|
||||
static_cast<T>(max_range),
|
||||
quant_axis,
|
||||
x_num_col_dims,
|
||||
out);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,231 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/fake_quantize_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/fake_quantize_functor.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void FakeQuantizeRangeAbsMaxKernel(const Context &dev_ctx,
|
||||
const DenseTensor &x,
|
||||
const DenseTensor &in_scale,
|
||||
const optional<DenseTensor> &iter,
|
||||
int window_size,
|
||||
int bit_length,
|
||||
bool is_test,
|
||||
int round_type,
|
||||
DenseTensor *out,
|
||||
DenseTensor *out_scale,
|
||||
DenseTensor *out_scales) {
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
int bin_cnt = std::pow(2, bit_length - 1) - 1;
|
||||
|
||||
// testing
|
||||
if (is_test) {
|
||||
funcs::ClipAndFakeQuantFunctor<Context, T>()(
|
||||
dev_ctx, x, in_scale, bin_cnt, round_type, out);
|
||||
return;
|
||||
}
|
||||
|
||||
// training
|
||||
dev_ctx.template Alloc<T>(out_scale);
|
||||
|
||||
DenseTensor cur_scale;
|
||||
cur_scale.Resize({1});
|
||||
T *cur_scale_data = dev_ctx.template Alloc<T>(&cur_scale);
|
||||
funcs::FindAbsMaxFunctor<Context, T>()(
|
||||
dev_ctx, x.data<T>(), x.numel(), cur_scale_data);
|
||||
funcs::FindRangeAbsMaxFunctor<Context, T>()(dev_ctx,
|
||||
cur_scale,
|
||||
in_scale,
|
||||
iter.get(),
|
||||
window_size,
|
||||
out_scales,
|
||||
out_scale);
|
||||
funcs::ClipAndFakeQuantFunctor<Context, T>()(
|
||||
dev_ctx, x, *out_scale, bin_cnt, round_type, out);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void FakeQuantizeAbsMaxKernel(const Context &dev_ctx,
|
||||
const DenseTensor &x,
|
||||
int bit_length,
|
||||
int round_type,
|
||||
DenseTensor *out,
|
||||
DenseTensor *out_scale) {
|
||||
T *out_s = dev_ctx.template Alloc<T>(out_scale);
|
||||
int bin_cnt = std::pow(2, bit_length - 1) - 1;
|
||||
const T *in_data = x.data<T>();
|
||||
funcs::FindAbsMaxFunctor<Context, T> find_abs_max_functor;
|
||||
find_abs_max_functor(dev_ctx, in_data, x.numel(), out_s);
|
||||
|
||||
funcs::ClipAndFakeQuantFunctor<Context, T> clip_and_fake_quant_functor;
|
||||
clip_and_fake_quant_functor(dev_ctx, x, *out_scale, bin_cnt, round_type, out);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void FakeQuantOrWithDequantMovingAverageAbsMaxKernel(
|
||||
const Context &dev_ctx,
|
||||
const DenseTensor &x,
|
||||
const DenseTensor &in_scale,
|
||||
const optional<DenseTensor> &in_accum,
|
||||
const optional<DenseTensor> &in_state,
|
||||
float moving_rate,
|
||||
int bit_length,
|
||||
bool is_test,
|
||||
int round_type,
|
||||
DenseTensor *out,
|
||||
DenseTensor *out_scale,
|
||||
DenseTensor *out_state,
|
||||
DenseTensor *out_accum) {
|
||||
int bin_cnt = std::pow(2, bit_length - 1) - 1;
|
||||
|
||||
// testing
|
||||
if (is_test) {
|
||||
funcs::ClipAndFakeQuantFunctor<Context, T>()(
|
||||
dev_ctx, x, in_scale, bin_cnt, round_type, out);
|
||||
return;
|
||||
}
|
||||
|
||||
// training
|
||||
DenseTensor tmp_scale;
|
||||
tmp_scale.Resize(common::make_dim(1));
|
||||
T *cur_scale_data = dev_ctx.template Alloc<T>(&tmp_scale);
|
||||
|
||||
funcs::FindAbsMaxFunctor<Context, T>()(
|
||||
dev_ctx, x.data<T>(), x.numel(), cur_scale_data);
|
||||
|
||||
funcs::FindMovingAverageAbsMaxFunctor<Context, T>()(dev_ctx,
|
||||
in_accum.get(),
|
||||
in_state.get(),
|
||||
cur_scale_data,
|
||||
moving_rate,
|
||||
out_state,
|
||||
out_accum,
|
||||
out_scale);
|
||||
|
||||
funcs::ClipAndFakeQuantFunctor<Context, T>()(
|
||||
dev_ctx, x, *out_scale, bin_cnt, round_type, out);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void FakeChannelWiseQuantizeAbsMaxKernel(const Context &dev_ctx,
|
||||
const DenseTensor &x,
|
||||
int bit_length,
|
||||
int round_type,
|
||||
int quant_axis,
|
||||
bool is_test,
|
||||
DenseTensor *out,
|
||||
DenseTensor *out_scale) {
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
int bin_cnt = std::pow(2, bit_length - 1) - 1;
|
||||
|
||||
if (!is_test) {
|
||||
T *out_scale_data = dev_ctx.template Alloc<T>(out_scale);
|
||||
funcs::FindChannelAbsMaxFunctor<Context, T>()(
|
||||
dev_ctx, x, quant_axis, out_scale_data);
|
||||
}
|
||||
funcs::ChannelClipAndFakeQuantFunctor<Context, T>()(
|
||||
dev_ctx, x, *out_scale, bin_cnt, round_type, quant_axis, out);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void FakeChannelWiseQuantizeDequantizeAbsMaxKernel(const Context &dev_ctx,
|
||||
const DenseTensor &x,
|
||||
int bit_length,
|
||||
int round_type,
|
||||
int quant_axis,
|
||||
DenseTensor *out,
|
||||
DenseTensor *out_scale) {
|
||||
T *out_scale_data = dev_ctx.template Alloc<T>(out_scale);
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
int bin_cnt = std::pow(2, bit_length - 1) - 1;
|
||||
|
||||
funcs::FindChannelAbsMaxFunctor<Context, T>()(
|
||||
dev_ctx, x, quant_axis, out_scale_data);
|
||||
|
||||
funcs::ChannelClipFakeQuantDequantFunctor<Context, T>()(
|
||||
dev_ctx, x, *out_scale, bin_cnt, round_type, quant_axis, out);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void FakeQuantizeDequantizeMovingAverageAbsMaxKernel(
|
||||
const Context &dev_ctx,
|
||||
const DenseTensor &x,
|
||||
const DenseTensor &in_scale,
|
||||
const optional<DenseTensor> &in_accum,
|
||||
const optional<DenseTensor> &in_state,
|
||||
float moving_rate,
|
||||
int bit_length,
|
||||
bool is_test,
|
||||
int round_type,
|
||||
DenseTensor *out,
|
||||
DenseTensor *out_scale,
|
||||
DenseTensor *out_state,
|
||||
DenseTensor *out_accum) {
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
int bin_cnt = std::pow(2, bit_length - 1) - 1;
|
||||
|
||||
// testing
|
||||
if (is_test) {
|
||||
funcs::ClipAndFakeQuantDequantFunctor<Context, T>()(
|
||||
dev_ctx, x, in_scale, bin_cnt, round_type, out);
|
||||
return;
|
||||
}
|
||||
|
||||
// training
|
||||
|
||||
DenseTensor tmp_scale;
|
||||
tmp_scale.Resize(common::make_dim(1));
|
||||
T *cur_scale_data = dev_ctx.template Alloc<T>(&tmp_scale);
|
||||
funcs::FindAbsMaxFunctor<Context, T>()(
|
||||
dev_ctx, x.data<T>(), x.numel(), cur_scale_data);
|
||||
|
||||
dev_ctx.template Alloc<T>(out_state);
|
||||
dev_ctx.template Alloc<T>(out_accum);
|
||||
dev_ctx.template Alloc<T>(out_scale);
|
||||
|
||||
funcs::FindMovingAverageAbsMaxFunctor<Context, T>()(dev_ctx,
|
||||
in_accum.get(),
|
||||
in_state.get(),
|
||||
cur_scale_data,
|
||||
moving_rate,
|
||||
out_state,
|
||||
out_accum,
|
||||
out_scale);
|
||||
|
||||
funcs::ClipAndFakeQuantDequantFunctor<Context, T>()(
|
||||
dev_ctx, x, *out_scale, bin_cnt, round_type, out);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void FakeQuantizeDequantizeAbsMaxKernel(const Context &dev_ctx,
|
||||
const DenseTensor &x,
|
||||
int bit_length,
|
||||
int round_type,
|
||||
DenseTensor *out,
|
||||
DenseTensor *out_scale) {
|
||||
T *out_s = dev_ctx.template Alloc<T>(out_scale);
|
||||
int bin_cnt = std::pow(2, bit_length - 1) - 1;
|
||||
const T *in_data = x.data<T>();
|
||||
funcs::FindAbsMaxFunctor<Context, T>()(dev_ctx, in_data, x.numel(), out_s);
|
||||
|
||||
funcs::ClipAndFakeQuantDequantFunctor<Context, T>()(
|
||||
dev_ctx, x, *out_scale, bin_cnt, round_type, out);
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,67 @@
|
||||
// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/funcs/common_shape.h"
|
||||
#include "paddle/phi/kernels/funcs/fc_functor.h"
|
||||
|
||||
namespace phi {
|
||||
namespace fusion {
|
||||
template <typename T, typename Context>
|
||||
void FCKernel(const Context& dev_ctx,
|
||||
const DenseTensor& input,
|
||||
const DenseTensor& w,
|
||||
const optional<DenseTensor>& bias,
|
||||
const int in_num_col_dims,
|
||||
const std::string& activation_type,
|
||||
const bool padding_weights,
|
||||
DenseTensor* out) {
|
||||
bool with_relu = (activation_type == "relu") ? true : false;
|
||||
|
||||
auto w_dims = w.dims();
|
||||
|
||||
std::vector<int64_t> output_dims;
|
||||
funcs::FCOutputSize(
|
||||
input.dims(), w_dims, output_dims, in_num_col_dims, padding_weights);
|
||||
out->Resize(output_dims);
|
||||
out->set_lod(input.lod());
|
||||
|
||||
auto out_dims = out->dims();
|
||||
auto w_dims0 = padding_weights ? w_dims[0] - 4 : w_dims[0];
|
||||
auto w_dims1 = padding_weights ? w_dims[1] - 4 : w_dims[1];
|
||||
int M = common::product(out_dims) / w_dims1;
|
||||
|
||||
const T* input_data = input.data<T>();
|
||||
const T* w_data = w.data<T>();
|
||||
auto* output_data = dev_ctx.template Alloc<T>(out, out->numel() * sizeof(T));
|
||||
|
||||
funcs::FCFunctor<Context, T> fc;
|
||||
fc(dev_ctx,
|
||||
M,
|
||||
w_dims1,
|
||||
w_dims0,
|
||||
input_data,
|
||||
w_data,
|
||||
output_data,
|
||||
bias ? bias->data<T>() : NULL,
|
||||
with_relu,
|
||||
padding_weights);
|
||||
}
|
||||
} // namespace fusion
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,31 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "glog/logging.h"
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/utils/optional.h"
|
||||
|
||||
namespace phi {
|
||||
template <typename T, typename Context>
|
||||
void FetchBarrierKernel(const Context &dev_ctx,
|
||||
const optional<std::vector<const DenseTensor *>> &x
|
||||
UNUSED,
|
||||
int trainer_id UNUSED,
|
||||
const std::vector<std::string> &endpoints UNUSED,
|
||||
std::vector<DenseTensor *> out UNUSED) {
|
||||
VLOG(5) << "FetchBarrier Sync, do not need now";
|
||||
}
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,42 @@
|
||||
// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/core/tensor_utils.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void FetchKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
DenseTensor* out) {
|
||||
if (!x.IsInitialized()) {
|
||||
return;
|
||||
}
|
||||
Copy(dev_ctx, x, CPUPlace(), true, out);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void FetchArrayKernel(const Context& dev_ctx,
|
||||
const TensorArray& x,
|
||||
TensorArray* out) {
|
||||
out->resize(x.size());
|
||||
for (size_t i = 0; i < x.size(); ++i) {
|
||||
Copy(dev_ctx, x[i], CPUPlace(), true, &(out->at(i)));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,121 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
#include "paddle/phi/kernels/fft_grad_kernel.h"
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "paddle/common/ddim.h"
|
||||
#include "paddle/phi/common/data_type.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/core/tensor_meta.h"
|
||||
#include "paddle/phi/kernels/complex_kernel.h"
|
||||
#include "paddle/phi/kernels/empty_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/fft.h"
|
||||
#include "paddle/phi/kernels/funcs/fft_fill_conj.h"
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
#include "paddle/phi/kernels/pad_kernel.h"
|
||||
|
||||
namespace phi {
|
||||
template <typename T, typename Context>
|
||||
void FFTC2CGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& out_grad,
|
||||
const std::vector<int64_t>& axes,
|
||||
const std::string& normalization,
|
||||
bool forward,
|
||||
DenseTensor* x_grad) {
|
||||
dev_ctx.template Alloc<T>(x_grad);
|
||||
if (x_grad && x_grad->numel() == 0) {
|
||||
return;
|
||||
}
|
||||
auto norm_type = funcs::get_norm_from_string(normalization, forward);
|
||||
funcs::FFTC2CFunctor<Context, T, T> fft_c2c_func;
|
||||
fft_c2c_func(dev_ctx, out_grad, x_grad, axes, norm_type, !forward);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void FFTR2CGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const DenseTensor& out_grad,
|
||||
const std::vector<int64_t>& axes,
|
||||
const std::string& normalization,
|
||||
bool forward,
|
||||
bool onesided,
|
||||
DenseTensor* x_grad) {
|
||||
using R = typename T::value_type;
|
||||
DenseTensor complex_x_grad = EmptyLike<T>(dev_ctx, x);
|
||||
dev_ctx.template Alloc<R>(x_grad);
|
||||
if (x_grad && x_grad->numel() == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto norm_type = funcs::get_norm_from_string(normalization, forward);
|
||||
funcs::FFTC2CFunctor<Context, T, T> fft_c2c_func;
|
||||
|
||||
if (!onesided) {
|
||||
fft_c2c_func(dev_ctx, out_grad, &complex_x_grad, axes, norm_type, !forward);
|
||||
} else {
|
||||
DenseTensor full_dy;
|
||||
DenseTensorMeta full_dy_meta(out_grad.type(), x_grad->dims());
|
||||
full_dy.set_meta(full_dy_meta);
|
||||
auto zero_length = static_cast<int>(full_dy.dims().at(axes.back()) -
|
||||
out_grad.dims().at(axes.back()));
|
||||
auto rank = out_grad.dims().size();
|
||||
std::vector<int> pads(rank * 2, 0);
|
||||
pads[axes.back() * 2 + 1] = zero_length;
|
||||
PadKernel<T>(dev_ctx, out_grad, pads, static_cast<float>(0.0), &full_dy);
|
||||
fft_c2c_func(dev_ctx, full_dy, &complex_x_grad, axes, norm_type, !forward);
|
||||
}
|
||||
RealKernel<T>(dev_ctx, complex_x_grad, x_grad);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void FFTC2RGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& out_grad,
|
||||
const std::vector<int64_t>& axes,
|
||||
const std::string& normalization,
|
||||
bool forward,
|
||||
int64_t last_dim_size UNUSED,
|
||||
DenseTensor* x_grad) {
|
||||
using C = dtype::complex<T>;
|
||||
dev_ctx.template Alloc<C>(x_grad);
|
||||
if (x_grad && x_grad->numel() == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto norm_type = funcs::get_norm_from_string(normalization, forward);
|
||||
|
||||
funcs::FFTR2CFunctor<Context, T, C> fft_r2c_func;
|
||||
fft_r2c_func(dev_ctx, out_grad, x_grad, axes, norm_type, !forward);
|
||||
|
||||
const int64_t double_length =
|
||||
out_grad.dims()[axes.back()] - x_grad->dims()[axes.back()];
|
||||
int64_t stride_to_last_axis = 1;
|
||||
auto ddim = x_grad->dims();
|
||||
for (int i = ddim.size() - 2; i >= axes.back(); --i) {
|
||||
stride_to_last_axis *= ddim[i + 1];
|
||||
}
|
||||
int64_t stride_second_to_last_axis = stride_to_last_axis * ddim[axes.back()];
|
||||
funcs::FFTFillConjGradFunctor<C> func(x_grad->data<C>(),
|
||||
axes.back(),
|
||||
stride_second_to_last_axis,
|
||||
stride_to_last_axis,
|
||||
double_length);
|
||||
size_t limit = x_grad->numel();
|
||||
funcs::ForRange<Context> for_range(dev_ctx, limit);
|
||||
for_range(func);
|
||||
}
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,102 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
#include "paddle/phi/kernels/fft_kernel.h"
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "paddle/common/ddim.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/kernels/empty_kernel.h"
|
||||
#include "paddle/phi/kernels/full_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/fft.h"
|
||||
#include "paddle/phi/kernels/funcs/fft_fill_conj.h"
|
||||
namespace phi {
|
||||
template <typename T, typename Context>
|
||||
void FFTC2CKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const std::vector<int64_t>& axes,
|
||||
const std::string& normalization,
|
||||
bool forward,
|
||||
DenseTensor* out) {
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
if (x.numel() == 0) {
|
||||
/*
|
||||
This will return 0:
|
||||
>>> scipy.fft.fft2(np.random.random([3, 0, 1, 2]), s=(1, 2), axes=(0, 1),
|
||||
norm='backward')
|
||||
array([[[[0.-0.j, 0.-0.j]],
|
||||
[[0.-0.j, 0.-0.j]]]])
|
||||
*/
|
||||
Full<T, Context>(dev_ctx, out->dims(), 0, out);
|
||||
return;
|
||||
}
|
||||
const auto norm_type = funcs::get_norm_from_string(normalization, forward);
|
||||
funcs::FFTC2CFunctor<Context, T, T> fft_c2c_func;
|
||||
fft_c2c_func(dev_ctx, x, out, axes, norm_type, forward);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void FFTC2RKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const std::vector<int64_t>& axes,
|
||||
const std::string& normalization,
|
||||
bool forward,
|
||||
int64_t last_dim_size UNUSED,
|
||||
DenseTensor* out) {
|
||||
using R = typename T::value_type; // get real type
|
||||
dev_ctx.template Alloc<R>(out);
|
||||
if (x.numel() == 0) {
|
||||
Full<R, Context>(dev_ctx, out->dims(), 0, out);
|
||||
return;
|
||||
}
|
||||
const auto norm_type = funcs::get_norm_from_string(normalization, forward);
|
||||
funcs::FFTC2RFunctor<Context, T, R> fft_c2r_func;
|
||||
fft_c2r_func(dev_ctx, x, out, axes, norm_type, forward);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void FFTR2CKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x,
|
||||
const std::vector<int64_t>& axes,
|
||||
const std::string& normalization,
|
||||
bool forward,
|
||||
bool onesided,
|
||||
DenseTensor* out) {
|
||||
using C = dtype::complex<T>;
|
||||
dev_ctx.template Alloc<C>(out);
|
||||
if (x.numel() == 0) {
|
||||
Full<C, Context>(dev_ctx, out->dims(), 0, out);
|
||||
return;
|
||||
}
|
||||
auto norm_type = funcs::get_norm_from_string(normalization, forward);
|
||||
funcs::FFTR2CFunctor<Context, T, C> fft_r2c_func;
|
||||
|
||||
if (onesided) {
|
||||
fft_r2c_func(dev_ctx, x, out, axes, norm_type, forward);
|
||||
} else {
|
||||
DDim onesided_out_shape = x.dims();
|
||||
const int64_t last_fft_axis = axes.back();
|
||||
const int64_t onesided_last_axis_size =
|
||||
out->dims().at(last_fft_axis) / 2 + 1;
|
||||
onesided_out_shape[last_fft_axis] = onesided_last_axis_size;
|
||||
DenseTensor onesided_out =
|
||||
Empty<C, Context>(dev_ctx, vectorize(onesided_out_shape));
|
||||
fft_r2c_func(dev_ctx, x, &onesided_out, axes, norm_type, forward);
|
||||
funcs::FFTFillConj<Context, C>(dev_ctx, &onesided_out, out, axes);
|
||||
}
|
||||
}
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,38 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/fill_grad_kernel.h"
|
||||
|
||||
#include "paddle/phi/common/scalar.h"
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void FillGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& out_grad UNUSED,
|
||||
const Scalar& value UNUSED,
|
||||
DenseTensor* in_grad) {
|
||||
if (in_grad) {
|
||||
dev_ctx.template Alloc<T>(in_grad);
|
||||
|
||||
funcs::SetConstant<Context, T> functor;
|
||||
functor(dev_ctx, in_grad, T(0));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,47 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "paddle/phi/kernels/fill_kernel.h"
|
||||
|
||||
#include "paddle/phi/common/scalar.h"
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void FillKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x UNUSED,
|
||||
const Scalar& value,
|
||||
DenseTensor* out) {
|
||||
double fill_var = value.to<double>();
|
||||
|
||||
PADDLE_ENFORCE_EQ(
|
||||
std::isnan(fill_var),
|
||||
false,
|
||||
common::errors::InvalidArgument("fill value should not be NaN,"
|
||||
" but received NaN"));
|
||||
|
||||
dev_ctx.template Alloc<T>(out);
|
||||
if (out->numel() == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
funcs::SetConstant<Context, T> functor;
|
||||
functor(dev_ctx, out, value.to<T>());
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,62 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
#include <vector>
|
||||
|
||||
#include "paddle/phi/kernels/empty_kernel.h"
|
||||
#include "paddle/phi/kernels/flatten_grad_kernel.h"
|
||||
#include "paddle/phi/kernels/flatten_kernel.h"
|
||||
#include "paddle/phi/kernels/funcs/blas/blas.h"
|
||||
#include "paddle/phi/kernels/funcs/flatten2_utils.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void Flatten2Kernel(const Context &dev_ctx,
|
||||
const DenseTensor &x,
|
||||
int axis,
|
||||
DenseTensor *out,
|
||||
DenseTensor *x_shape) {
|
||||
auto &axes = axis;
|
||||
|
||||
auto *in = &x;
|
||||
auto x_dims = in->dims();
|
||||
|
||||
auto out_dims = make_ddim(funcs::GetOutputShape(axes, x_dims));
|
||||
|
||||
dev_ctx.Alloc(out, x.dtype());
|
||||
Copy(dev_ctx, *in, dev_ctx.GetPlace(), false, out);
|
||||
out->Resize(out_dims);
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void Flatten2GradKernel(const Context &dev_ctx,
|
||||
const DenseTensor &x,
|
||||
const DenseTensor &x_shape,
|
||||
const DenseTensor &out_grad,
|
||||
int axis,
|
||||
DenseTensor *x_grad) {
|
||||
auto *d_x = x_grad;
|
||||
auto *d_out = &out_grad;
|
||||
|
||||
auto xshape_dims = x_shape.dims();
|
||||
auto x_dims = slice_ddim(xshape_dims, 1, xshape_dims.size());
|
||||
|
||||
dev_ctx.Alloc(x_grad, out_grad.dtype());
|
||||
Copy(dev_ctx, *d_out, dev_ctx.GetPlace(), false, d_x);
|
||||
d_x->Resize(x_dims);
|
||||
}
|
||||
} // namespace phi
|
||||
@@ -0,0 +1,71 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/kernels/funcs/im2col.h"
|
||||
#include "paddle/phi/kernels/funcs/math_function.h"
|
||||
#include "paddle/phi/kernels/funcs/unfold_functor.h"
|
||||
|
||||
namespace phi {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void FoldGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& x UNUSED,
|
||||
const DenseTensor& out_grad,
|
||||
const std::vector<int>& output_sizes,
|
||||
const std::vector<int>& kernel_sizes,
|
||||
const std::vector<int>& strides,
|
||||
const std::vector<int>& paddings,
|
||||
const std::vector<int>& dilations,
|
||||
DenseTensor* x_grad) {
|
||||
dev_ctx.template Alloc<T>(x_grad);
|
||||
|
||||
if (!x_grad) return;
|
||||
|
||||
const auto& x_dims = x_grad->dims();
|
||||
const int64_t batch_size = x_dims[0];
|
||||
|
||||
int output_height = (output_sizes[0] + 2 * paddings[0] -
|
||||
(dilations[0] * (kernel_sizes[0] - 1) + 1)) /
|
||||
strides[0] +
|
||||
1;
|
||||
int output_width = (output_sizes[1] + 2 * paddings[1] -
|
||||
(dilations[1] * (kernel_sizes[1] - 1) + 1)) /
|
||||
strides[1] +
|
||||
1;
|
||||
|
||||
int64_t n_input_plane = x_dims[1];
|
||||
int64_t n_output_plane = n_input_plane / (kernel_sizes[0] * kernel_sizes[1]);
|
||||
|
||||
DDim out_shape =
|
||||
make_ddim({n_output_plane, output_sizes[0], output_sizes[1]});
|
||||
DDim input_matrix_shape = make_ddim(
|
||||
{1, kernel_sizes[0], kernel_sizes[1], output_height, output_width});
|
||||
|
||||
funcs::Im2ColFunctor<funcs::ColFormat::CFO, Context, T> im2col;
|
||||
|
||||
for (int64_t i = 0; i < batch_size; i++) {
|
||||
DenseTensor out_grad_batch = out_grad.Slice(i, i + 1).Resize(out_shape);
|
||||
DenseTensor x_grad_batch =
|
||||
x_grad->Slice(i, i + 1).Resize(input_matrix_shape);
|
||||
im2col(
|
||||
dev_ctx, out_grad_batch, dilations, strides, paddings, &x_grad_batch);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace phi
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user