mirrors/paddlepaddle--paddle
1
0
Fork 0
Code Issues Pull Requests Actions 2 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:40:42 +08:00
commit e25996e7db
15472 changed files with 3536181 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
paddle/phi/kernels/cpu/abs_grad_kernel.cc
+45
View File
@@ -0,0 +1,45 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/common/type_traits.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/complex_functors.h"
#include "paddle/phi/kernels/impl/abs_grad_kernel_impl.h"
using phi::dtype::complex;
PD_REGISTER_KERNEL(abs_grad,
CPU,
ALL_LAYOUT,
phi::AbsGradKernel,
float,
double,
int,
int64_t,
complex<float>,
complex<double>) {
kernel->InputAt(1).SetDataType(phi::dtype::ToReal(kernel_key.dtype()));
}
PD_REGISTER_KERNEL(abs_double_grad,
CPU,
ALL_LAYOUT,
phi::AbsDoubleGradKernel,
float,
double,
int,
int64_t,
complex<float>,
complex<double>) {
kernel->InputAt(1).SetDataType(phi::dtype::ToReal(kernel_key.dtype()));
}
paddle/phi/kernels/cpu/abs_kernel.cc
+52
View File
@@ -0,0 +1,52 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/abs_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/complex_functors.h"
#include "paddle/phi/kernels/funcs/for_range.h"
namespace phi {
template <typename T, typename Context>
PADDLE_API void AbsKernel(const Context& dev_ctx,
const DenseTensor& x,
DenseTensor* out) {
auto numel = x.numel();
auto* x_data = x.data<T>();
dev_ctx.template Alloc<dtype::Real<T>>(
out, size_t(x.numel() * sizeof(dtype::Real<T>)));
auto* out_data = out->data<dtype::Real<T>>();
funcs::ForRange<Context> for_range(dev_ctx, numel);
funcs::AbsFunctor<T> functor(x_data, out_data, numel);
for_range(functor);
}
} // namespace phi
PD_REGISTER_KERNEL(abs,
CPU,
ALL_LAYOUT,
phi::AbsKernel,
float,
double,
int,
int64_t,
phi::complex64,
phi::complex128) {
kernel->OutputAt(0).SetDataType(phi::dtype::ToReal(kernel_key.dtype()));
}
paddle/phi/kernels/cpu/accuracy_check_kernel.cc
+37
View File
@@ -0,0 +1,37 @@
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/accuracy_check_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/dense_tensor.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/accuracy_check_kernel_impl.h"
PD_REGISTER_KERNEL(accuracy_check,
CPU,
ALL_LAYOUT,
phi::AccuracyCheckKernel,
float,
double,
int,
int64_t,
uint8_t,
int8_t,
int16_t,
bool,
phi::float16,
phi::bfloat16,
phi::complex64,
phi::complex128) {}
paddle/phi/kernels/cpu/accuracy_kernel.cc
+102
View File
@@ -0,0 +1,102 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/accuracy_kernel.h"
#include <algorithm>
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
namespace phi {
template <typename T, typename Context>
void AccuracyKernel(const Context& dev_ctx,
const DenseTensor& inference,
const DenseTensor& indices,
const DenseTensor& label,
DenseTensor* accuracy,
DenseTensor* correct,
DenseTensor* total) {
int* correct_data = dev_ctx.template Alloc<int>(correct);
int* total_data = dev_ctx.template Alloc<int>(total);
float* accuracy_data = dev_ctx.template Alloc<float>(accuracy);
const int64_t* indices_data = indices.data<int64_t>();
const int64_t* label_data = label.data<int64_t>();
PADDLE_ENFORCE_EQ(
inference.dims().size(),
2,
common::errors::InvalidArgument(
"Rank(Input) of AccuracyOp must be 2, with shape "
"[sample_number, class_dim], But received rank(Input) is %d",
inference.dims().size()));
size_t num_samples = inference.dims()[0];
size_t class_dim = inference.dims()[1];
*accuracy_data = 0.0f;
PADDLE_ENFORCE_GT(label.dims().size(),
0,
common::errors::InvalidArgument(
"Rank(Label) of AccuracyOp must greater than 0, "
"But received rank(Label) is %d",
label.dims().size()));
PADDLE_ENFORCE_GE(label.dims()[0],
inference.dims()[0],
common::errors::InvalidArgument(
"num_samples(%d) of Label should less than "
"or equal to num_samples(%d) of Input",
label.dims()[0],
num_samples));
if (num_samples == 0) {
return;
}
int num_correct = 0;
// assume inference is already the topk of the output
for (size_t i = 0; i < num_samples; ++i) {
PADDLE_ENFORCE_GE(
label_data[i],
0,
common::errors::InvalidArgument(
"label of AccuracyOp must >= 0, But received label[%d] is %d",
i,
label_data[i]));
for (size_t j = 0; j < class_dim; ++j) {
if (indices_data[i * class_dim + j] == label_data[i]) {
++num_correct;
break;
}
}
}
*correct_data = num_correct;
*total_data = static_cast<int>(num_samples);
*accuracy_data =
static_cast<float>(num_correct) / static_cast<float>(num_samples);
}
} // namespace phi
// TODO(add supported dtype.)
PD_REGISTER_KERNEL(
accuracy, CPU, ALL_LAYOUT, phi::AccuracyKernel, float, double) {
kernel->InputAt(1).SetDataType(phi::DataType::INT64);
kernel->InputAt(2).SetDataType(phi::DataType::INT64);
kernel->OutputAt(0).SetDataType(phi::DataType::FLOAT32);
kernel->OutputAt(1).SetDataType(phi::DataType::INT32);
kernel->OutputAt(2).SetDataType(phi::DataType::INT32);
}
paddle/phi/kernels/cpu/activation_grad_kernel.cc
+598
View File
@@ -0,0 +1,598 @@
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/phi/kernels/activation_grad_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/full_kernel.h"
#include "paddle/phi/kernels/impl/activation_grad_impl.h"
namespace phi {
#define DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(name, functor_class) \
template <typename T, typename Context> \
void name##GradKernel(const Context& dev_ctx, \
const DenseTensor& x, \
const DenseTensor& dout, \
DenseTensor* dx) { \
funcs::functor_class<T> functor; \
ActivationGradImpl<T, Context, funcs::functor_class<T>>( \
dev_ctx, &x, nullptr, &dout, dx, functor); \
}
#define DEFINE_CPU_ACT_GRAD_KERNEL_WITH_ONE_ATTRS_DEPX( \
name, functor_class, attr) \
template <typename T, typename Context> \
void name##GradKernel(const Context& dev_ctx, \
const DenseTensor& x, \
const DenseTensor& dout, \
float attr, \
DenseTensor* dx) { \
funcs::functor_class<T> functor; \
auto attrs = functor.GetAttrs(); \
*(attrs[0].second) = attr; \
ActivationGradImpl<T, Context, funcs::functor_class<T>>( \
dev_ctx, &x, nullptr, &dout, dx, functor); \
}
#define DEFINE_CPU_ACT_GRAD_KERNEL_WITH_ONE_DOUBLE_ATTRS_DEPX( \
name, functor_class, attr) \
template <typename T, typename Context> \
void name##GradKernel(const Context& dev_ctx, \
const DenseTensor& x, \
const DenseTensor& dout, \
double attr, \
DenseTensor* dx) { \
funcs::functor_class<T> functor; \
auto attrs = functor.GetAttrs(); \
*(attrs[0].second) = attr; \
ActivationGradImpl<T, Context, funcs::functor_class<T>>( \
dev_ctx, &x, nullptr, &dout, dx, functor); \
}
#define DEFINE_CPU_ACT_GRAD_KERNEL_WITH_TWO_ATTRS_DEPX( \
name, functor_class, attr1, attr2) \
template <typename T, typename Context> \
void name##GradKernel(const Context& dev_ctx, \
const DenseTensor& x, \
const DenseTensor& dout, \
float attr1, \
float attr2, \
DenseTensor* dx) { \
funcs::functor_class<T> functor; \
auto attrs = functor.GetAttrs(); \
*(attrs[0].second) = attr1; \
*(attrs[1].second) = attr2; \
ActivationGradImpl<T, Context, funcs::functor_class<T>>( \
dev_ctx, &x, nullptr, &dout, dx, functor); \
}
#define DEFINE_CPU_ACT_GRAD_KERNEL_WITH_TWO_DOUBLE_ATTRS_DEPX( \
name, functor_class, attr1, attr2) \
template <typename T, typename Context> \
void name##GradKernel(const Context& dev_ctx, \
const DenseTensor& x, \
const DenseTensor& dout, \
double attr1, \
double attr2, \
DenseTensor* dx) { \
funcs::functor_class<T> functor; \
auto attrs = functor.GetAttrs(); \
*(attrs[0].second) = attr1; \
*(attrs[1].second) = attr2; \
ActivationGradImpl<T, Context, funcs::functor_class<T>>( \
dev_ctx, &x, nullptr, &dout, dx, functor); \
}
#define DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPOUT(name, functor_class) \
template <typename T, typename Context> \
void name##GradKernel(const Context& dev_ctx, \
const DenseTensor& out, \
const DenseTensor& dout, \
DenseTensor* dx) { \
funcs::functor_class<T> functor; \
ActivationGradImpl<T, Context, funcs::functor_class<T>>( \
dev_ctx, nullptr, &out, &dout, dx, functor); \
}
#define DEFINE_CPU_ACT_GRAD_KERNEL_WITH_ONE_ATTRS_DEPOUT( \
name, functor_class, attr) \
template <typename T, typename Context> \
void name##GradKernel(const Context& dev_ctx, \
const DenseTensor& out, \
const DenseTensor& dout, \
float attr, \
DenseTensor* dx) { \
funcs::functor_class<T> functor; \
auto attrs = functor.GetAttrs(); \
*(attrs[0].second) = attr; \
ActivationGradImpl<T, Context, funcs::functor_class<T>>( \
dev_ctx, nullptr, &out, &dout, dx, functor); \
}
#define DEFINE_CPU_ACT_GRAD_KERNEL_WITH_TWO_ATTRS_DEPOUT( \
name, functor_class, attr1, attr2) \
template <typename T, typename Context> \
void name##GradKernel(const Context& dev_ctx, \
const DenseTensor& out, \
const DenseTensor& dout, \
float attr1, \
float attr2, \
DenseTensor* dx) { \
funcs::functor_class<T> functor; \
auto attrs = functor.GetAttrs(); \
*(attrs[0].second) = attr1; \
*(attrs[1].second) = attr2; \
ActivationGradImpl<T, Context, funcs::functor_class<T>>( \
dev_ctx, nullptr, &out, &dout, dx, functor); \
}
#define DEFINE_CPU_ACTIVATION_GRAD_KERNEL_NODEP(name, functor_class) \
template <typename T, typename Context> \
void name##GradKernel( \
const Context& dev_ctx, const DenseTensor& dout, DenseTensor* dx) { \
funcs::functor_class<T> functor; \
ActivationGradImpl<T, Context, funcs::functor_class<T>>( \
dev_ctx, nullptr, nullptr, &dout, dx, functor); \
}
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(Cos, CosGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(Tan, TanGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(Acos, AcosGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(Sin, SinGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(Asin, AsinGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(Atan, AtanGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(Sinh, SinhGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(Cosh, CoshGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(Asinh, AsinhGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(Acosh, AcoshGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(Atanh, AtanhGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(TanhShrink, TanhShrinkGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(Square, SquareGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPOUT(Exp, ExpGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPOUT(Expm1, Expm1GradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPOUT(Reciprocal, ReciprocalGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPOUT(Sqrt, SqrtGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPOUT(Rsqrt, RsqrtGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPOUT(Relu6, Relu6GradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(Softsign, SoftsignGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(LogSigmoid, LogSigmoidGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(Log, LogGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(Log2, Log2GradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(Log10, Log10GradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(Log1p, Log1pGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPX(Swish, SwishGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPOUT(Relu, ReluGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPOUT(Tanh, TanhGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_DEPOUT(Sigmoid, SigmoidGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_NODEP(Rint, ZeroGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_NODEP(Round, ZeroGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_NODEP(Floor, ZeroGradFunctor);
DEFINE_CPU_ACTIVATION_GRAD_KERNEL_NODEP(Ceil, ZeroGradFunctor);
DEFINE_CPU_ACT_GRAD_KERNEL_WITH_ONE_DOUBLE_ATTRS_DEPX(LeakyRelu,
LeakyReluGradFunctor,
alpha);
DEFINE_CPU_ACT_GRAD_KERNEL_WITH_ONE_ATTRS_DEPX(SoftShrink,
SoftShrinkGradFunctor,
lambda);
DEFINE_CPU_ACT_GRAD_KERNEL_WITH_ONE_ATTRS_DEPX(HardShrink,
HardShrinkGradFunctor,
threshold);
DEFINE_CPU_ACT_GRAD_KERNEL_WITH_ONE_ATTRS_DEPX(Mish,
MishGradFunctor,
threshold);
DEFINE_CPU_ACT_GRAD_KERNEL_WITH_ONE_ATTRS_DEPX(Celu, CELUGradFunctor, alpha);
DEFINE_CPU_ACT_GRAD_KERNEL_WITH_TWO_ATTRS_DEPX(HardTanh,
HardTanhGradFunctor,
t_min,
t_max);
DEFINE_CPU_ACT_GRAD_KERNEL_WITH_TWO_ATTRS_DEPX(STanh,
STanhGradFunctor,
scale_a,
scale_b);
DEFINE_CPU_ACT_GRAD_KERNEL_WITH_TWO_DOUBLE_ATTRS_DEPX(Softplus,
SoftplusGradFunctor,
beta,
threshold);
DEFINE_CPU_ACT_GRAD_KERNEL_WITH_TWO_ATTRS_DEPOUT(HardSigmoid,
HardSigmoidGradFunctor,
slope,
offset);
DEFINE_CPU_ACT_GRAD_KERNEL_WITH_TWO_ATTRS_DEPX(ThresholdedRelu,
ThresholdedReluGradFunctor,
threshold,
value);
template <typename T, typename Context>
void SiluGradKernel(const Context& dev_ctx,
const DenseTensor& x,
const DenseTensor& out,
const DenseTensor& dout,
DenseTensor* dx) {
funcs::SiluGradFunctor<T> functor;
ActivationGradImpl<T, Context, funcs::SiluGradFunctor<T>>(
dev_ctx, &x, &out, &dout, dx, functor);
}
template <typename T, typename Context>
void EluGradKernel(const Context& dev_ctx,
const DenseTensor& x,
const DenseTensor& out,
const DenseTensor& dout,
float alpha,
DenseTensor* dx) {
dev_ctx.template Alloc<T>(dx);
if (dx->numel() == 0) {
return;
}
auto x_flatten =
EigenVector<T>::Flatten(GET_DATA_SAFELY(&x, "Input", "X", "elu_grad"));
auto out_flatten = EigenVector<T>::Flatten(
GET_DATA_SAFELY(&out, "Input", "Out", "elu_grad"));
auto dout_flatten = EigenVector<T>::Flatten(
GET_DATA_SAFELY(&dout, "Input", "dOut", "elu_grad"));
auto dx_flatten =
EigenVector<T>::Flatten(GET_DATA_SAFELY(dx, "Output", "dX", "elu_grad"));
auto* place = dev_ctx.eigen_device();
if (alpha > 0) {
funcs::ELUGradFunctor<T> functor;
functor.alpha = alpha;
functor(*place, x_flatten, out_flatten, dout_flatten, dx_flatten);
} else {
funcs::ELUGradNegativeAlphaFunctor<T> functor;
functor.alpha = alpha;
functor(*place, x_flatten, out_flatten, dout_flatten, dx_flatten);
}
}
template <typename T, typename Context>
void HardSwishGradKernel(const Context& dev_ctx,
const DenseTensor& x,
const DenseTensor& dout,
DenseTensor* dx) {
funcs::HardSwishGradFunctor<T> functor;
float threshold = 6;
float scale = 6;
float offset = 3;
auto attrs = functor.GetAttrs();
*(attrs[0].second) = threshold;
*(attrs[1].second) = scale;
*(attrs[2].second) = offset;
ActivationGradImpl<T, Context, funcs::HardSwishGradFunctor<T>>(
dev_ctx, &x, nullptr, &dout, dx, functor);
}
template <typename T, typename Context>
void PowGradKernel(const Context& dev_ctx,
const DenseTensor& x,
const DenseTensor& dout,
const Scalar& factor,
DenseTensor* dx) {
if (factor.to<float>() == 0) {
std::vector<int64_t> vec_dims = vectorize(dx->dims());
Full<T, Context>(dev_ctx, vec_dims, static_cast<T>(0), dx);
return;
}
PADDLE_ENFORCE_NOT_NULL(
dx,
errors::InvalidArgument("The output DenseTensor dx can not be nullptr"));
dev_ctx.template Alloc<T>(dx);
auto dout_flatten = EigenVector<T>::Flatten(
GET_DATA_SAFELY(&dout, "Input", "Out@GRAD", "PowGrad"));
auto dx_flatten = EigenVector<T>::Flatten(
GET_DATA_SAFELY(dx, "Output", "X@GRAD", "PowGrad"));
auto x_flatten =
EigenVector<T>::Flatten(GET_DATA_SAFELY(&x, "Input", "X", "PowGrad"));
auto* place = dev_ctx.eigen_device();
funcs::PowGradFunctor<T> functor;
auto attrs = functor.GetAttrs();
*(attrs[0].second) = factor.to<float>();
functor(*place, x_flatten, nullptr, dout_flatten, dx_flatten);
}
} // namespace phi
PD_REGISTER_KERNEL(
relu_grad, CPU, ALL_LAYOUT, phi::ReluGradKernel, float, double) {}
#define PD_REGISTER_ACTIVATION_GRAD_KERNEL(name, func) \
PD_REGISTER_KERNEL(name, CPU, ALL_LAYOUT, phi::func, float, double) {}
#define PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(name, func) \
PD_REGISTER_KERNEL(name, \
CPU, \
ALL_LAYOUT, \
phi::func, \
float, \
double, \
phi::complex64, \
phi::complex128) {}
#define PD_REGISTER_ACTIVATION_DOUBLE_GRAD_KERNEL(name, func) \
PD_REGISTER_KERNEL( \
name, CPU, ALL_LAYOUT, phi::func, float, double, phi::float16) {}
#define PD_REGISTER_ACTIVATION_DOUBLE_GRAD_KERNEL_WITH_COMPLEX(name, func) \
PD_REGISTER_KERNEL(name, \
CPU, \
ALL_LAYOUT, \
phi::func, \
float, \
double, \
phi::float16, \
phi::complex64, \
phi::complex128) {}
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(sin_grad, SinGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(cos_grad, CosGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(tan_grad, TanGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(acos_grad, AcosGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(asin_grad, AsinGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(atan_grad, AtanGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(sinh_grad, SinhGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(cosh_grad, CoshGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(asinh_grad, AsinhGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(acosh_grad, AcoshGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(atanh_grad, AtanhGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(tanh_grad, TanhGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(hardtanh_grad, HardTanhGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(leaky_relu_grad, LeakyReluGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(thresholded_relu_grad,
ThresholdedReluGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(relu6_grad, Relu6GradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(softshrink_grad, SoftShrinkGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(hard_shrink_grad, HardShrinkGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(tanh_shrink_grad, TanhShrinkGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(elu_grad, EluGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(silu_grad, SiluGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(mish_grad, MishGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(stanh_grad, STanhGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(reciprocal_grad,
ReciprocalGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(sqrt_grad, SqrtGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(rsqrt_grad, RsqrtGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(softplus_grad,
SoftplusGradKernel)
PD_REGISTER_ACTIVATION_DOUBLE_GRAD_KERNEL(relu_double_grad,
ReluDoubleGradKernel)
PD_REGISTER_ACTIVATION_DOUBLE_GRAD_KERNEL_WITH_COMPLEX(tanh_double_grad,
TanhDoubleGradKernel)
PD_REGISTER_ACTIVATION_DOUBLE_GRAD_KERNEL(leaky_relu_double_grad,
LeakyReluDoubleGradKernel)
PD_REGISTER_ACTIVATION_DOUBLE_GRAD_KERNEL(elu_double_grad, EluDoubleGradKernel)
PD_REGISTER_ACTIVATION_DOUBLE_GRAD_KERNEL(sqrt_double_grad,
SqrtDoubleGradKernel)
PD_REGISTER_ACTIVATION_DOUBLE_GRAD_KERNEL(rsqrt_double_grad,
RsqrtDoubleGradKernel)
PD_REGISTER_ACTIVATION_DOUBLE_GRAD_KERNEL_WITH_COMPLEX(softplus_double_grad,
SoftplusDoubleGradKernel)
PD_REGISTER_KERNEL(tanh_triple_grad,
CPU,
ALL_LAYOUT,
phi::TanhTripleGradKernel,
float,
double,
phi::float16,
phi::complex64,
phi::complex128) {}
PD_REGISTER_KERNEL(exp_grad,
CPU,
ALL_LAYOUT,
phi::ExpGradKernel,
float,
double,
int,
int64_t,
phi::complex64,
phi::complex128) {}
PD_REGISTER_KERNEL(expm1_grad,
CPU,
ALL_LAYOUT,
phi::Expm1GradKernel,
float,
double,
phi::float16,
phi::complex64,
phi::complex128) {}
PD_REGISTER_KERNEL(
logit_grad, CPU, ALL_LAYOUT, phi::LogitGradKernel, float, double) {}
PD_REGISTER_KERNEL(square_grad,
CPU,
ALL_LAYOUT,
phi::SquareGradKernel,
float,
double,
int,
int64_t,
phi::complex64,
phi::complex128) {}
PD_REGISTER_KERNEL(square_double_grad,
CPU,
ALL_LAYOUT,
phi::SquareDoubleGradKernel,
float,
double,
phi::float16,
int,
int64_t,
phi::complex64,
phi::complex128) {}
PD_REGISTER_KERNEL(sin_double_grad,
CPU,
ALL_LAYOUT,
phi::SinDoubleGradKernel,
float,
double,
phi::float16,
int,
int64_t,
phi::complex64,
phi::complex128) {}
PD_REGISTER_KERNEL(sin_triple_grad,
CPU,
ALL_LAYOUT,
phi::SinTripleGradKernel,
float,
double,
phi::float16,
int,
int64_t,
phi::complex64,
phi::complex128) {}
PD_REGISTER_KERNEL(cos_double_grad,
CPU,
ALL_LAYOUT,
phi::CosDoubleGradKernel,
float,
double,
phi::float16,
int,
int64_t,
phi::complex64,
phi::complex128) {}
PD_REGISTER_KERNEL(cos_triple_grad,
CPU,
ALL_LAYOUT,
phi::CosTripleGradKernel,
float,
double,
phi::float16,
int,
int64_t,
phi::complex64,
phi::complex128) {}
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(softsign_grad,
SoftsignGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(sigmoid_grad, SigmoidGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(sigmoid_double_grad,
SigmoidDoubleGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(sigmoid_triple_grad,
SigmoidTripleGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(hardsigmoid_grad, HardSigmoidGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(logsigmoid_grad,
LogSigmoidGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(log_grad, LogGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(log2_grad, Log2GradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(log10_grad, Log10GradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(log1p_grad, Log1pGradKernel)
PD_REGISTER_ACTIVATION_DOUBLE_GRAD_KERNEL_WITH_COMPLEX(log_double_grad,
LogDoubleGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(hardswish_grad,
HardSwishGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(swish_grad, SwishGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(celu_grad, CeluGradKernel)
PD_REGISTER_ACTIVATION_DOUBLE_GRAD_KERNEL(celu_double_grad,
CeluDoubleGradKernel)
PD_REGISTER_KERNEL(rint_grad,
CPU,
ALL_LAYOUT,
phi::RintGradKernel,
float,
double,
int,
int64_t) {}
PD_REGISTER_KERNEL(round_grad,
CPU,
ALL_LAYOUT,
phi::RoundGradKernel,
float,
double,
int,
int64_t,
phi::complex64,
phi::complex128) {}
PD_REGISTER_KERNEL(pow_grad,
CPU,
ALL_LAYOUT,
phi::PowGradKernel,
float,
double,
int,
int64_t,
phi::complex64,
phi::complex128) {}
PD_REGISTER_KERNEL(pow_double_grad,
CPU,
ALL_LAYOUT,
phi::PowDoubleGradKernel,
float,
double,
int,
int64_t,
phi::complex64,
phi::complex128) {}
PD_REGISTER_KERNEL(pow_triple_grad,
CPU,
ALL_LAYOUT,
phi::PowTripleGradKernel,
float,
double,
int,
int64_t,
phi::complex64,
phi::complex128) {}
PD_REGISTER_KERNEL(ceil_grad,
CPU,
ALL_LAYOUT,
phi::CeilGradKernel,
float,
double,
uint8_t,
int8_t,
int16_t,
int,
int64_t,
phi::float16,
phi::bfloat16) {}
PD_REGISTER_KERNEL(floor_grad,
CPU,
ALL_LAYOUT,
phi::FloorGradKernel,
float,
double,
uint8_t,
int8_t,
int16_t,
int,
int64_t,
phi::float16,
phi::bfloat16) {}
paddle/phi/kernels/cpu/activation_kernel.cc
+484
View File
@@ -0,0 +1,484 @@
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/phi/kernels/activation_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/full_kernel.h"
#include "paddle/phi/kernels/funcs/activation_functor.h"
#include "paddle/phi/kernels/impl/activation_impl.h"
namespace phi {
#define DEFINE_CPU_ACTIVATION_KERNEL(name, functor_class) \
template <typename T, typename Context> \
void name##Kernel( \
const Context& dev_ctx, const DenseTensor& x, DenseTensor* out) { \
funcs::functor_class<T> functor; \
ActivationImpl<T, T, Context, funcs::functor_class<T>>( \
dev_ctx, x, out, functor); \
}
#define DEFINE_CPU_ACTIVATION_KERNEL_WITH_INT_IN_FLOAT_OUT(name, \
functor_class) \
template <typename T, typename Context> \
void name##Kernel( \
const Context& dev_ctx, const DenseTensor& x, DenseTensor* out) { \
funcs::functor_class<T> functor; \
using U = \
typename std::conditional_t<std::is_integral<T>::value, float, T>; \
ActivationImpl<T, U, Context, funcs::functor_class<T>>( \
dev_ctx, x, out, functor); \
}
#define DEFINE_CPU_ACT_KERNEL_WITH_ONE_ATTRS(name, functor_class, attr) \
template <typename T, typename Context> \
void name##Kernel(const Context& dev_ctx, \
const DenseTensor& x, \
float attr, \
DenseTensor* out) { \
funcs::functor_class<T> functor; \
auto attrs = functor.GetAttrs(); \
*(attrs[0].second) = attr; \
ActivationImpl<T, T, Context, funcs::functor_class<T>>( \
dev_ctx, x, out, functor); \
}
#define DEFINE_CPU_ACT_KERNEL_WITH_ONE_DOUBLE_ATTRS(name, functor_class, attr) \
template <typename T, typename Context> \
void name##Kernel(const Context& dev_ctx, \
const DenseTensor& x, \
double attr, \
DenseTensor* out) { \
funcs::functor_class<T> functor; \
auto attrs = functor.GetAttrs(); \
*(attrs[0].second) = attr; \
ActivationImpl<T, T, Context, funcs::functor_class<T>>( \
dev_ctx, x, out, functor); \
}
#define DEFINE_CPU_ACT_KERNEL_WITH_TWO_ATTRS( \
name, functor_class, attr1, attr2) \
template <typename T, typename Context> \
void name##Kernel(const Context& dev_ctx, \
const DenseTensor& x, \
float attr1, \
float attr2, \
DenseTensor* out) { \
funcs::functor_class<T> functor; \
auto attrs = functor.GetAttrs(); \
*(attrs[0].second) = attr1; \
*(attrs[1].second) = attr2; \
ActivationImpl<T, T, Context, funcs::functor_class<T>>( \
dev_ctx, x, out, functor); \
}
#define DEFINE_CPU_ACT_KERNEL_WITH_TWO_DOUBLE_ATTRS( \
name, functor_class, attr1, attr2) \
template <typename T, typename Context> \
void name##Kernel(const Context& dev_ctx, \
const DenseTensor& x, \
double attr1, \
double attr2, \
DenseTensor* out) { \
funcs::functor_class<T> functor; \
auto attrs = functor.GetAttrs(); \
*(attrs[0].second) = attr1; \
*(attrs[1].second) = attr2; \
ActivationImpl<T, T, Context, funcs::functor_class<T>>( \
dev_ctx, x, out, functor); \
}
// Specialized Sin kernel with vectorized Sleef implementation for float/double
// This ensures high precision computation
template <typename T, typename Context>
void SinKernel(const Context& dev_ctx, const DenseTensor& x, DenseTensor* out) {
if constexpr (std::is_same<T, float>::value ||
std::is_same<T, double>::value) {
// Use vectorized Sleef path for float/double for high precision
VectorizedSinImpl<T, Context>(dev_ctx, x, out);
} else {
// Use generic path for other types (float16, bfloat16, complex)
funcs::SinFunctor<T> functor;
ActivationImpl<T, T, Context, funcs::SinFunctor<T>>(
dev_ctx, x, out, functor);
}
}
// Specialized Cos kernel with vectorized Sleef implementation for float/double
template <typename T, typename Context>
void CosKernel(const Context& dev_ctx, const DenseTensor& x, DenseTensor* out) {
if constexpr (std::is_same<T, float>::value ||
std::is_same<T, double>::value) {
// Use vectorized Sleef path for float/double for high precision
VectorizedCosImpl<T, Context>(dev_ctx, x, out);
} else {
// Use generic path for other types (float16, bfloat16, complex)
funcs::CosFunctor<T> functor;
ActivationImpl<T, T, Context, funcs::CosFunctor<T>>(
dev_ctx, x, out, functor);
}
}
DEFINE_CPU_ACTIVATION_KERNEL(Tan, TanFunctor)
DEFINE_CPU_ACTIVATION_KERNEL(Asin, AsinFunctor)
DEFINE_CPU_ACTIVATION_KERNEL(Atan, AtanFunctor)
DEFINE_CPU_ACTIVATION_KERNEL(Acos, AcosFunctor)
DEFINE_CPU_ACTIVATION_KERNEL(Sinh, SinhFunctor)
DEFINE_CPU_ACTIVATION_KERNEL(Cosh, CoshFunctor)
DEFINE_CPU_ACTIVATION_KERNEL(Asinh, AsinhFunctor)
DEFINE_CPU_ACTIVATION_KERNEL(Acosh, AcoshFunctor)
DEFINE_CPU_ACTIVATION_KERNEL(Atanh, AtanhFunctor)
DEFINE_CPU_ACTIVATION_KERNEL(Relu, ReluCPUFunctor)
DEFINE_CPU_ACTIVATION_KERNEL(Tanh, TanhFunctor)
DEFINE_CPU_ACTIVATION_KERNEL(TanhShrink, TanhShrinkFunctor)
DEFINE_CPU_ACTIVATION_KERNEL(Silu, SiluFunctor)
DEFINE_CPU_ACTIVATION_KERNEL(Reciprocal, ReciprocalFunctor)
DEFINE_CPU_ACTIVATION_KERNEL(Square, SquareFunctor)
DEFINE_CPU_ACTIVATION_KERNEL(Sqrt, SqrtFunctor)
DEFINE_CPU_ACTIVATION_KERNEL(Rsqrt, RsqrtFunctor)
DEFINE_CPU_ACTIVATION_KERNEL(Softsign, SoftsignFunctor)
DEFINE_CPU_ACTIVATION_KERNEL(Sigmoid, SigmoidFunctor)
DEFINE_CPU_ACTIVATION_KERNEL(LogSigmoid, LogSigmoidFunctor)
DEFINE_CPU_ACTIVATION_KERNEL(Floor, FloorFunctor)
DEFINE_CPU_ACTIVATION_KERNEL(Ceil, CeilFunctor)
DEFINE_CPU_ACTIVATION_KERNEL(Negative, NegativeFunctor)
DEFINE_CPU_ACTIVATION_KERNEL(Rint, RintFunctor)
DEFINE_CPU_ACTIVATION_KERNEL_WITH_INT_IN_FLOAT_OUT(Log, LogFunctor)
DEFINE_CPU_ACTIVATION_KERNEL_WITH_INT_IN_FLOAT_OUT(Log2, Log2Functor)
DEFINE_CPU_ACTIVATION_KERNEL_WITH_INT_IN_FLOAT_OUT(Log10, Log10Functor)
DEFINE_CPU_ACTIVATION_KERNEL_WITH_INT_IN_FLOAT_OUT(Log1p, Log1pFunctor)
DEFINE_CPU_ACTIVATION_KERNEL_WITH_INT_IN_FLOAT_OUT(Expm1, Expm1Functor)
// Specialized Exp kernel with vectorized MKL VML/Sleef implementation
// for float/double. This ensures high precision computation.
template <typename T, typename Context>
void ExpKernel(const Context& dev_ctx, const DenseTensor& x, DenseTensor* out) {
if constexpr (std::is_same<T, float>::value ||
std::is_same<T, double>::value) {
// Use vectorized MKL VML/Sleef path for float/double for high precision
VectorizedExpImpl<T, Context>(dev_ctx, x, out);
} else {
// Use generic path for other types (int, int64, float16, complex)
funcs::ExpFunctor<T> functor;
using U = typename std::conditional_t<std::is_integral<T>::value, float, T>;
ActivationImpl<T, U, Context, funcs::ExpFunctor<T>>(
dev_ctx, x, out, functor);
}
}
DEFINE_CPU_ACT_KERNEL_WITH_ONE_DOUBLE_ATTRS(LeakyRelu, LeakyReluFunctor, alpha)
DEFINE_CPU_ACT_KERNEL_WITH_ONE_ATTRS(Mish, MishFunctor, threshold)
DEFINE_CPU_ACT_KERNEL_WITH_ONE_ATTRS(HardShrink, HardShrinkFunctor, threshold)
DEFINE_CPU_ACT_KERNEL_WITH_ONE_ATTRS(SoftShrink, SoftShrinkFunctor, lambda)
DEFINE_CPU_ACT_KERNEL_WITH_ONE_ATTRS(Elu, ELUFunctor, alpha)
DEFINE_CPU_ACT_KERNEL_WITH_ONE_ATTRS(Celu, CELUFunctor, alpha)
DEFINE_CPU_ACT_KERNEL_WITH_TWO_ATTRS(HardTanh, HardTanhFunctor, t_min, t_max)
DEFINE_CPU_ACT_KERNEL_WITH_TWO_ATTRS(STanh, STanhFunctor, scale_a, scale_b)
DEFINE_CPU_ACT_KERNEL_WITH_TWO_DOUBLE_ATTRS(Softplus,
SoftplusFunctor,
beta,
threshold)
DEFINE_CPU_ACT_KERNEL_WITH_TWO_ATTRS(HardSigmoid,
HardSigmoidFunctor,
slope,
offset)
DEFINE_CPU_ACT_KERNEL_WITH_TWO_ATTRS(ThresholdedRelu,
ThresholdedReluFunctor,
threshold,
value)
template <typename T, typename Context>
void HardSwishKernel(const Context& dev_ctx,
const DenseTensor& x,
DenseTensor* out) {
funcs::HardSwishFunctor<T> functor;
float threshold = 6;
float scale = 6;
float offset = 3;
auto attrs = functor.GetAttrs();
*(attrs[0].second) = threshold;
*(attrs[1].second) = scale;
*(attrs[2].second) = offset;
ActivationImpl<T, T, Context, funcs::HardSwishFunctor<T>>(
dev_ctx, x, out, functor);
}
template <typename T, typename Context>
void SwishKernel(const Context& dev_ctx,
const DenseTensor& x,
DenseTensor* out) {
funcs::SwishFunctor<T> functor;
auto attrs = functor.GetAttrs();
*(attrs[0].second) = 1.0;
ActivationImpl<T, T, Context, funcs::SwishFunctor<T>>(
dev_ctx, x, out, functor);
}
template <typename T, typename Context>
void Relu6Kernel(const Context& dev_ctx,
const DenseTensor& x,
DenseTensor* out) {
funcs::Relu6Functor<T> functor;
auto attrs = functor.GetAttrs();
*(attrs[0].second) = 6.0;
ActivationImpl<T, T, Context, funcs::Relu6Functor<T>>(
dev_ctx, x, out, functor);
}
template <typename T, typename Context>
void RoundKernel(const Context& dev_ctx,
const DenseTensor& x,
const int decimals,
DenseTensor* out) {
funcs::RoundFunctor<T> functor;
auto attrs = functor.GetAttrs();
*(attrs[0].second) = decimals;
ActivationImpl<T, T, Context, funcs::RoundFunctor<T>>(
dev_ctx, x, out, functor);
}
template <typename T, typename Context>
void PowKernel(const Context& dev_ctx,
const DenseTensor& x,
const Scalar& factor,
DenseTensor* out) {
PADDLE_ENFORCE_NOT_NULL(
out, errors::InvalidArgument("Output Out should not be nullptr"));
dev_ctx.template Alloc<T>(out);
if (factor.to<float>() == 0) {
std::vector<int64_t> vec_dims = vectorize(out->dims());
Full<T, Context>(dev_ctx, vec_dims, static_cast<T>(1), out);
return;
}
if (factor.to<float>() == 1) {
Copy<Context>(dev_ctx, x, dev_ctx.GetPlace(), false, out);
return;
}
auto x_flatten =
EigenVector<T>::Flatten(GET_DATA_SAFELY(&x, "Input", "X", "Activation"));
auto out_flatten = EigenVector<T>::Flatten(
GET_DATA_SAFELY(out, "Output", "Out", "Activation"));
auto* place = dev_ctx.eigen_device();
funcs::PowFunctor<T> functor;
auto attrs = functor.GetAttrs();
*(attrs[0].second) = factor.to<float>();
functor(*place, x_flatten, out_flatten);
}
} // namespace phi
PD_REGISTER_KERNEL(relu, CPU, ALL_LAYOUT, phi::ReluKernel, float, double) {}
#define PD_REGISTER_ACTIVATION_KERNEL(name, func) \
PD_REGISTER_KERNEL(name, CPU, ALL_LAYOUT, phi::func, float, double) {}
#define PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(name, func) \
PD_REGISTER_KERNEL(name, \
CPU, \
ALL_LAYOUT, \
phi::func, \
float, \
double, \
phi::complex64, \
phi::complex128) {}
PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(sin, SinKernel)
PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(cos, CosKernel)
PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(tan, TanKernel)
PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(acos, AcosKernel)
PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(asin, AsinKernel)
PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(atan, AtanKernel)
PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(sinh, SinhKernel)
PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(cosh, CoshKernel)
PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(asinh, AsinhKernel)
PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(acosh, AcoshKernel)
PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(atanh, AtanhKernel)
PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(tanh, TanhKernel)
PD_REGISTER_ACTIVATION_KERNEL(hardtanh, HardTanhKernel)
PD_REGISTER_ACTIVATION_KERNEL(leaky_relu, LeakyReluKernel)
PD_REGISTER_ACTIVATION_KERNEL(thresholded_relu, ThresholdedReluKernel)
PD_REGISTER_ACTIVATION_KERNEL(hard_shrink, HardShrinkKernel)
PD_REGISTER_ACTIVATION_KERNEL(softshrink, SoftShrinkKernel)
PD_REGISTER_ACTIVATION_KERNEL(tanh_shrink, TanhShrinkKernel)
PD_REGISTER_ACTIVATION_KERNEL(elu, EluKernel)
PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(silu, SiluKernel)
PD_REGISTER_ACTIVATION_KERNEL(mish, MishKernel)
PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(stanh, STanhKernel)
PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(reciprocal, ReciprocalKernel)
PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(sqrt, SqrtKernel)
PD_REGISTER_ACTIVATION_KERNEL(rsqrt, RsqrtKernel)
PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(softplus, SoftplusKernel)
PD_REGISTER_ACTIVATION_KERNEL(logit, LogitKernel)
PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(softsign, SoftsignKernel)
PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(sigmoid, SigmoidKernel)
PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(logsigmoid, LogSigmoidKernel)
PD_REGISTER_ACTIVATION_KERNEL(hardsigmoid, HardSigmoidKernel)
PD_REGISTER_ACTIVATION_KERNEL(swish, SwishKernel)
PD_REGISTER_ACTIVATION_KERNEL(relu6, Relu6Kernel)
PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(hardswish, HardSwishKernel)
PD_REGISTER_ACTIVATION_KERNEL(celu, CeluKernel)
PD_REGISTER_KERNEL(
rint, CPU, ALL_LAYOUT, phi::RintKernel, int, int64_t, float, double) {}
PD_REGISTER_KERNEL(round,
CPU,
ALL_LAYOUT,
phi::RoundKernel,
int,
int64_t,
float,
double,
phi::complex64,
phi::complex128) {}
PD_REGISTER_KERNEL(exp,
CPU,
ALL_LAYOUT,
phi::ExpKernel,
float,
double,
int,
int64_t,
phi::float16,
phi::complex64,
phi::complex128) {}
PD_REGISTER_KERNEL(expm1,
CPU,
ALL_LAYOUT,
phi::Expm1Kernel,
float,
double,
int,
int64_t,
phi::float16,
phi::complex64,
phi::complex128) {}
PD_REGISTER_KERNEL(square,
CPU,
ALL_LAYOUT,
phi::SquareKernel,
float,
double,
int,
int64_t,
phi::complex64,
phi::complex128) {}
PD_REGISTER_KERNEL(log,
CPU,
ALL_LAYOUT,
phi::LogKernel,
float,
double,
int,
int64_t,
phi::float16,
phi::bfloat16,
phi::complex64,
phi::complex128) {}
PD_REGISTER_KERNEL(log2,
CPU,
ALL_LAYOUT,
phi::Log2Kernel,
float,
double,
int,
int64_t,
phi::float16,
phi::bfloat16,
phi::complex64,
phi::complex128) {}
PD_REGISTER_KERNEL(log10,
CPU,
ALL_LAYOUT,
phi::Log10Kernel,
float,
double,
int,
int64_t,
phi::float16,
phi::bfloat16,
phi::complex64,
phi::complex128) {}
PD_REGISTER_KERNEL(log1p,
CPU,
ALL_LAYOUT,
phi::Log1pKernel,
float,
double,
int,
int64_t,
phi::float16,
phi::bfloat16,
phi::complex64,
phi::complex128) {}
PD_REGISTER_KERNEL(negative,
CPU,
ALL_LAYOUT,
phi::NegativeKernel,
float,
double,
int16_t,
int,
int64_t,
phi::complex64,
phi::complex128) {}
PD_REGISTER_KERNEL(pow,
CPU,
ALL_LAYOUT,
phi::PowKernel,
float,
double,
int,
int64_t,
phi::complex64,
phi::complex128) {}
PD_REGISTER_KERNEL(ceil,
CPU,
ALL_LAYOUT,
phi::CeilKernel,
float,
double,
uint8_t,
int8_t,
int16_t,
int,
int64_t,
phi::float16,
phi::bfloat16) {}
PD_REGISTER_KERNEL(floor,
CPU,
ALL_LAYOUT,
phi::FloorKernel,
float,
double,
uint8_t,
int8_t,
int16_t,
int,
int64_t,
phi::float16,
phi::bfloat16) {}
paddle/phi/kernels/cpu/adadelta_kernel.cc
+22
View File
@@ -0,0 +1,22 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/adadelta_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/adadelta_kernel_impl.h"
PD_REGISTER_KERNEL(
adadelta, CPU, ALL_LAYOUT, phi::AdadeltaKernel, float, double) {}
paddle/phi/kernels/cpu/adagrad_kernel.cc
+118
View File
@@ -0,0 +1,118 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/adagrad_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/math_function.h"
#include "paddle/phi/kernels/funcs/selected_rows_functor.h"
#include "paddle/phi/kernels/impl/adagrad_kernel_impl.h"
namespace phi {
namespace {
size_t FindPos(const std::vector<int64_t>& rows, int64_t value) {
return std::find(rows.begin(), rows.end(), value) - rows.begin();
}
} // namespace
template <typename T>
struct DenseAdagradFunctor<CPUContext, T> {
void operator()(const CPUContext& dev_ctx,
const DenseTensor& param_t,
const DenseTensor& grad_t,
const DenseTensor& moment_t,
const DenseTensor& learning_rate,
const optional<DenseTensor>& master_param,
float epsilon_t,
bool multi_precision,
DenseTensor* param_out_tensor,
DenseTensor* moment_out_tensor,
DenseTensor* master_param_outs) {
dev_ctx.template Alloc<T>(param_out_tensor);
dev_ctx.template Alloc<T>(moment_out_tensor);
T epsilon = static_cast<T>(epsilon_t);
auto param = EigenVector<T>::Flatten(param_t);
auto grad = EigenVector<T>::Flatten(grad_t);
auto moment = EigenVector<T>::Flatten(moment_t);
auto param_out = EigenVector<T>::Flatten(*param_out_tensor);
auto moment_out = EigenVector<T>::Flatten(*moment_out_tensor);
auto place = *dev_ctx.eigen_device();
moment_out.device(place) = moment + grad * grad;
Eigen::DSizes<int, 1> m_dsize(static_cast<int>(moment_out_tensor->numel()));
auto* lr = learning_rate.data<T>();
param_out.device(place) =
param - lr[0] * grad / (moment_out.sqrt() + epsilon);
}
};
template <typename T>
struct SparseAdagradFunctor<CPUContext, T> {
void operator()(const CPUContext& dev_ctx,
const SelectedRows& grad,
const DenseTensor& learning_rate,
T epsilon,
DenseTensor* moment,
DenseTensor* param) {
// 1. g_m.rows = set(g.rows)
auto grad_width = grad.value().dims()[1];
funcs::scatter::MergeAdd<CPUContext, T> merge_func;
auto grad_merge = merge_func(dev_ctx, grad);
auto& merge_rows = grad_merge.rows();
auto* grad_merge_data = grad_merge.mutable_value()->template data<T>();
// 2. m += g_m * g_m
auto grad_square = SquareSelectedRows<CPUContext, T>(dev_ctx, grad_merge);
funcs::SelectedRowsAddToTensor<CPUContext, T> functor;
functor(dev_ctx, grad_square, moment);
// 3. update parameter
auto* lr = learning_rate.data<T>();
auto* param_data = param->data<T>();
auto* moment_data = moment->data<T>();
for (size_t i = 0; i < merge_rows.size(); i++) {
for (int64_t j = 0; j < grad_width; j++) {
param_data[merge_rows[i] * grad_width + j] -=
lr[0] * grad_merge_data[i * grad_width + j] /
(std::sqrt(moment_data[merge_rows[i] * grad_width + j]) + epsilon);
}
}
}
};
template struct SparseAdagradFunctor<CPUContext, float>;
template struct SparseAdagradFunctor<CPUContext, double>;
template struct DenseAdagradFunctor<CPUContext, float>;
template struct DenseAdagradFunctor<CPUContext, double>;
} // namespace phi
PD_REGISTER_KERNEL(
adagrad, CPU, ALL_LAYOUT, phi::AdagradDenseKernel, float, double) {}
PD_REGISTER_KERNEL(adagrad_dense_param_sparse_grad,
CPU,
ALL_LAYOUT,
phi::AdagradSparseKernel,
float,
double) {}
paddle/phi/kernels/cpu/adam_kernel.cc
+316
View File
@@ -0,0 +1,316 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/adam_kernel.h"
#include <vector>
#include "glog/logging.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/core/tensor_utils.h"
#include "paddle/phi/kernels/funcs/adam_functors.h"
#include "paddle/phi/kernels/funcs/jit/kernels.h"
PD_DECLARE_int32(inner_op_parallelism);
namespace phi {
template <typename T, typename Context>
PADDLE_API void AdamDenseKernel(const Context& dev_ctx,
const DenseTensor& param,
const DenseTensor& grad,
const DenseTensor& learning_rate,
const DenseTensor& moment1,
const DenseTensor& moment2,
const optional<DenseTensor>& moment2_max,
const DenseTensor& beta1_pow,
const DenseTensor& beta2_pow,
const optional<DenseTensor>& master_param,
const optional<DenseTensor>& skip_update,
const Scalar& beta1,
const Scalar& beta2,
const Scalar& epsilon,
bool lazy_mode,
int64_t min_row_size_to_use_multithread,
bool multi_precision,
bool use_global_beta_pow,
bool amsgrad,
DenseTensor* param_out,
DenseTensor* moment1_out,
DenseTensor* moment2_out,
DenseTensor* moment2_max_out,
DenseTensor* beta1_pow_out,
DenseTensor* beta2_pow_out,
DenseTensor* master_param_outs) {
VLOG(4) << "use_global_beta_pow:" << use_global_beta_pow;
bool skip_update_ = false;
if (skip_update.is_initialized()) {
PADDLE_ENFORCE_EQ(
skip_update->numel(),
1,
errors::InvalidArgument("Input(SkipUpdate) size must be 1, but get %d",
skip_update->numel()));
std::vector<bool> skip_update_vec;
TensorToVector(*skip_update, dev_ctx, &skip_update_vec);
skip_update_ = skip_update_vec[0];
}
// skip_update=true, just copy input to output, and TensorCopy will call
// mutable_data
if (skip_update_) {
VLOG(4) << "Adam skip update";
Copy(dev_ctx, param, dev_ctx.GetPlace(), false, param_out);
Copy(dev_ctx, moment1, dev_ctx.GetPlace(), false, moment1_out);
Copy(dev_ctx, moment2, dev_ctx.GetPlace(), false, moment2_out);
if (amsgrad) {
Copy(dev_ctx,
moment2_max.get(),
dev_ctx.GetPlace(),
false,
moment2_max_out);
}
if (!use_global_beta_pow) {
Copy(dev_ctx, beta1_pow, beta1_pow.place(), false, beta1_pow_out);
Copy(dev_ctx, beta2_pow, beta2_pow.place(), false, beta2_pow_out);
}
return;
}
T beta1_ = beta1.to<T>();
T beta2_ = beta2.to<T>();
T epsilon_ = epsilon.to<T>();
VLOG(3) << "beta1_pow.numel() : " << beta1_pow.numel();
VLOG(3) << "beta2_pow.numel() : " << beta2_pow.numel();
VLOG(3) << "param.numel(): " << param.numel();
PADDLE_ENFORCE_EQ(
beta1_pow_out->numel(),
1,
errors::InvalidArgument("beta1 pow output size should be 1, but received "
"value is:%d.",
beta1_pow_out->numel()));
PADDLE_ENFORCE_EQ(
beta2_pow_out->numel(),
1,
errors::InvalidArgument("beta2 pow output size should be 1, but received "
"value is:%d.",
beta2_pow_out->numel()));
T beta1_p = beta1_pow.data<T>()[0];
T beta2_p = beta2_pow.data<T>()[0];
if (!use_global_beta_pow) {
dev_ctx.template Alloc<T>(beta1_pow_out)[0] = beta1_ * beta1_p;
dev_ctx.template Alloc<T>(beta2_pow_out)[0] = beta2_ * beta2_p;
}
T* param_out_ptr = dev_ctx.template Alloc<T>(param_out);
T* mom1_out_ptr = dev_ctx.template Alloc<T>(moment1_out);
T* mom2_out_ptr = dev_ctx.template Alloc<T>(moment2_out);
T* mom2_max_out_ptr =
amsgrad ? dev_ctx.template Alloc<T>(moment2_max_out) : nullptr;
T learning_rate_ = static_cast<T>(learning_rate.data<double>()[0]) *
(sqrt(1 - beta2_p) / (1 - beta1_p));
T eps = epsilon_ * sqrt(1 - beta2_p);
jit::adam_attr_t attr(beta1_, beta2_, amsgrad);
int64_t numel = param.numel();
const T* param_ptr = param.data<T>();
const T* mom1_ptr = moment1.data<T>();
const T* mom2_ptr = moment2.data<T>();
const T* mom2_max_ptr = amsgrad ? moment2_max.get().data<T>() : nullptr;
const T* grad_ptr = grad.data<T>();
auto adam = jit::KernelFuncs<jit::AdamTuple<T>, CPUPlace>::Cache().At(attr);
static constexpr int64_t chunk_size = 512;
#ifdef PADDLE_WITH_MKLML
#pragma omp parallel for
#endif
for (int64_t i = 0; i < numel / chunk_size; ++i) {
const int64_t offset = i * chunk_size;
const T* mom2_max_in_data = amsgrad ? mom2_max_ptr + offset : nullptr;
T* mom2_max_out_data = amsgrad ? mom2_max_out_ptr + offset : nullptr;
adam(beta1_,
beta2_,
-learning_rate_,
eps,
chunk_size,
grad_ptr + offset,
mom1_ptr + offset,
mom2_ptr + offset,
mom2_max_in_data,
param_ptr + offset,
mom1_out_ptr + offset,
mom2_out_ptr + offset,
mom2_max_out_data,
param_out_ptr + offset,
amsgrad);
}
if (numel % chunk_size != 0) {
const int64_t offset = (numel / chunk_size) * chunk_size;
const int64_t tail_numel = numel % chunk_size;
const T* mom2_max_in_data = amsgrad ? mom2_max_ptr + offset : nullptr;
T* mom2_max_out_data = amsgrad ? mom2_max_out_ptr + offset : nullptr;
adam(beta1_,
beta2_,
-learning_rate_,
eps,
tail_numel,
grad_ptr + offset,
mom1_ptr + offset,
mom2_ptr + offset,
mom2_max_in_data,
param_ptr + offset,
mom1_out_ptr + offset,
mom2_out_ptr + offset,
mom2_max_out_data,
param_out_ptr + offset,
amsgrad);
}
}
template <typename T, typename Context>
void MergedAdamKernel(
const Context& dev_ctx,
const std::vector<const DenseTensor*>& param,
const std::vector<const DenseTensor*>& grad,
const std::vector<const DenseTensor*>& learning_rate,
const std::vector<const DenseTensor*>& moment1,
const std::vector<const DenseTensor*>& moment2,
const optional<std::vector<const DenseTensor*>>& moment2_max,
const std::vector<const DenseTensor*>& beta1_pow,
const std::vector<const DenseTensor*>& beta2_pow,
const optional<std::vector<const DenseTensor*>>& master_param,
const Scalar& beta1,
const Scalar& beta2,
const Scalar& epsilon,
bool multi_precision,
bool use_global_beta_pow,
bool amsgrad,
std::vector<DenseTensor*> param_out,
std::vector<DenseTensor*> moment1_out,
std::vector<DenseTensor*> moment2_out,
std::vector<DenseTensor*> moment2_max_out,
std::vector<DenseTensor*> beta1_pow_out,
std::vector<DenseTensor*> beta2_pow_out,
std::vector<DenseTensor*> master_param_out) {
size_t param_num = param.size();
PADDLE_ENFORCE_EQ(
param_num,
grad.size(),
errors::InvalidArgument("The size of Input(grad) must be equal to "
"Input(param), but got the size of Input(grad) "
"is %d, the size of Input(param) is %d.",
grad.size(),
param_num));
PADDLE_ENFORCE_EQ(
param_num,
learning_rate.size(),
errors::InvalidArgument(
"The size of Input(learning_rate) must be equal to "
"Input(param), but got the size of Input(learning_rate) "
"is %d, the size of Input(param) is %d.",
learning_rate.size(),
param_num));
PADDLE_ENFORCE_EQ(param_num,
moment1.size(),
errors::InvalidArgument(
"The size of Input(moment1) must be equal to "
"Input(param), but got the size of Input(moment1) "
"is %d, the size of Input(param) is %d.",
moment1.size(),
param_num));
PADDLE_ENFORCE_EQ(param_num,
moment2.size(),
errors::InvalidArgument(
"The size of Input(moment2) must be equal to "
"Input(param), but got the size of Input(moment2) "
"is %d, the size of Input(param) is %d.",
moment2.size(),
param_num));
PADDLE_ENFORCE_EQ(param_num,
beta1_pow.size(),
errors::InvalidArgument(
"The size of Input(beta1_pow) must be equal to "
"Input(param), but got the size of Input(beta1_pow) "
"is %d, the size of Input(param) is %d.",
beta1_pow.size(),
param_num));
PADDLE_ENFORCE_EQ(param_num,
beta2_pow.size(),
errors::InvalidArgument(
"The size of Input(beta2_pow) must be equal to "
"Input(param), but got the size of Input(beta2_pow) "
"is %d, the size of Input(param) is %d.",
beta2_pow.size(),
param_num));
T beta1_ = beta1.to<T>();
T beta2_ = beta2.to<T>();
T epsilon_ = epsilon.to<T>();
for (size_t idx = 0; idx < param_num; idx++) {
const T* mom2_max_in_data =
amsgrad ? moment2_max.get()[idx]->data<T>() : nullptr;
T* mom2_max_out_data =
amsgrad ? dev_ctx.template Alloc<T>(moment2_max_out[idx]) : nullptr;
const T lr_val = static_cast<T>(learning_rate[idx]->data<double>()[0]);
funcs::AdamFunctor<T, funcs::CPUAdam> functor(
beta1_,
beta2_,
epsilon_,
beta1_pow[idx]->data<T>(),
beta2_pow[idx]->data<T>(),
moment1[idx]->data<T>(),
dev_ctx.template Alloc<T>(moment1_out[idx]),
moment2[idx]->data<T>(),
dev_ctx.template Alloc<T>(moment2_out[idx]),
mom2_max_in_data,
mom2_max_out_data,
&lr_val,
grad[idx]->data<T>(),
param[idx]->data<T>(),
dev_ctx.template Alloc<T>(param_out[idx]),
amsgrad);
functor(param[idx]->numel());
if (!use_global_beta_pow) {
dev_ctx.template Alloc<T>(beta1_pow_out[idx])[0] =
beta1_ * beta1_pow[idx]->data<T>()[0];
dev_ctx.template Alloc<T>(beta2_pow_out[idx])[0] =
beta2_ * beta2_pow[idx]->data<T>()[0];
}
}
}
} // namespace phi
PD_REGISTER_KERNEL(adam, CPU, ALL_LAYOUT, phi::AdamDenseKernel, float, double) {
kernel->InputAt(2).SetDataType(phi::DataType::FLOAT64);
}
PD_REGISTER_KERNEL(
merged_adam, CPU, ALL_LAYOUT, phi::MergedAdamKernel, float, double) {
kernel->InputAt(2).SetDataType(phi::DataType::FLOAT64);
}
paddle/phi/kernels/cpu/adamax_kernel.cc
+21
View File
@@ -0,0 +1,21 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/adamax_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/adamax_kernel_impl.h"
PD_REGISTER_KERNEL(adamax, CPU, ALL_LAYOUT, phi::AdamaxKernel, float, double) {}
paddle/phi/kernels/cpu/adamw_kernel.cc
+219
View File
@@ -0,0 +1,219 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/adamw_kernel.h"
#include <vector>
#include "glog/logging.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/core/tensor_utils.h"
#include "paddle/phi/kernels/adam_kernel.h"
#include "paddle/phi/kernels/funcs/adam_functors.h"
#include "paddle/phi/kernels/funcs/jit/kernels.h"
namespace phi {
template <typename T, typename Context>
PADDLE_API void AdamwDenseKernel(const Context& dev_ctx,
const DenseTensor& param,
const DenseTensor& grad,
const DenseTensor& learning_rate,
const DenseTensor& moment1,
const DenseTensor& moment2,
const optional<DenseTensor>& moment2_max,
const DenseTensor& beta1_pow,
const DenseTensor& beta2_pow,
const optional<DenseTensor>& master_param,
const optional<DenseTensor>& skip_update,
const Scalar& beta1,
const Scalar& beta2,
const Scalar& epsilon,
double lr_ratio,
double coeff,
bool with_decay,
bool lazy_mode,
int64_t min_row_size_to_use_multithread,
bool multi_precision,
bool use_global_beta_pow,
bool amsgrad,
DenseTensor* param_out,
DenseTensor* moment1_out,
DenseTensor* moment2_out,
DenseTensor* moment2_max_out,
DenseTensor* beta1_pow_out,
DenseTensor* beta2_pow_out,
DenseTensor* master_param_outs) {
bool skip_update_ = false;
if (skip_update.is_initialized()) {
PADDLE_ENFORCE_EQ(
skip_update->numel(),
1,
errors::InvalidArgument("Input(SkipUpdate) size must be 1, but get %d",
skip_update->numel()));
std::vector<bool> skip_update_vec;
TensorToVector(*skip_update, dev_ctx, &skip_update_vec);
skip_update_ = skip_update_vec[0];
}
VLOG(3) << "Skip update" << skip_update_;
if (skip_update_ || !with_decay) {
AdamDenseKernel<T, Context>(dev_ctx,
param,
grad,
learning_rate,
moment1,
moment2,
moment2_max,
beta1_pow,
beta2_pow,
master_param,
skip_update,
beta1,
beta2,
epsilon,
lazy_mode,
min_row_size_to_use_multithread,
multi_precision,
use_global_beta_pow,
amsgrad,
param_out,
moment1_out,
moment2_out,
moment2_max_out,
beta1_pow_out,
beta2_pow_out,
master_param_outs);
return;
}
T beta1_ = beta1.to<T>();
T beta2_ = beta2.to<T>();
T epsilon_ = epsilon.to<T>();
T coeff_ = static_cast<T>(coeff);
T lr_ratio_ = static_cast<T>(lr_ratio);
VLOG(3) << "beta1_pow.numel() : " << beta1_pow.numel();
VLOG(3) << "beta2_pow.numel() : " << beta2_pow.numel();
VLOG(3) << "param.numel(): " << param.numel();
PADDLE_ENFORCE_EQ(
beta1_pow_out->numel(),
1,
errors::InvalidArgument("beta1 pow output size should be 1, but received "
"value is:%d.",
beta1_pow_out->numel()));
PADDLE_ENFORCE_EQ(
beta2_pow_out->numel(),
1,
errors::InvalidArgument("beta2 pow output size should be 1, but received "
"value is:%d.",
beta2_pow_out->numel()));
T beta1_p = beta1_pow.data<T>()[0];
T beta2_p = beta2_pow.data<T>()[0];
if (!use_global_beta_pow) {
dev_ctx.template Alloc<T>(beta1_pow_out)[0] = beta1_ * beta1_p;
dev_ctx.template Alloc<T>(beta2_pow_out)[0] = beta2_ * beta2_p;
}
T* param_out_ptr = dev_ctx.template Alloc<T>(param_out);
T* mom1_out_ptr = dev_ctx.template Alloc<T>(moment1_out);
T* mom2_out_ptr = dev_ctx.template Alloc<T>(moment2_out);
T* mom2_max_out_ptr =
amsgrad ? dev_ctx.template Alloc<T>(moment2_max_out) : nullptr;
T old_lr = static_cast<T>(learning_rate.data<double>()[0]);
T learning_rate_ = static_cast<T>(learning_rate.data<double>()[0]) *
(sqrt(1 - beta2_p) / (1 - beta1_p));
T eps = epsilon_ * sqrt(1 - beta2_p);
jit::adamw_attr_t attr(beta1_, beta2_, coeff_, amsgrad);
int64_t numel = param.numel();
const T* param_ptr = param.data<T>();
const T* mom1_ptr = moment1.data<T>();
const T* mom2_ptr = moment2.data<T>();
const T* mom2_max_ptr = amsgrad ? moment2_max.get().data<T>() : nullptr;
const T* grad_ptr = grad.data<T>();
auto adamw = jit::KernelFuncs<jit::AdamWTuple<T>, CPUPlace>::Cache().At(attr);
static constexpr int64_t chunk_size = 512;
#ifdef PADDLE_WITH_MKLML
#pragma omp parallel for
#endif
for (int64_t i = 0; i < numel / chunk_size; ++i) {
const int64_t offset = i * chunk_size;
const T* mom2_max_in_data = amsgrad ? mom2_max_ptr + offset : nullptr;
T* mom2_max_out_data = amsgrad ? mom2_max_out_ptr + offset : nullptr;
adamw(beta1_,
beta2_,
-learning_rate_,
eps,
old_lr,
lr_ratio_,
coeff_,
chunk_size,
grad_ptr + offset,
mom1_ptr + offset,
mom2_ptr + offset,
mom2_max_in_data,
param_ptr + offset,
mom1_out_ptr + offset,
mom2_out_ptr + offset,
mom2_max_out_data,
param_out_ptr + offset,
amsgrad);
}
if (numel % chunk_size != 0) {
const int64_t offset = (numel / chunk_size) * chunk_size;
const int64_t tail_numel = numel % chunk_size;
const T* mom2_max_in_data = amsgrad ? mom2_max_ptr + offset : nullptr;
T* mom2_max_out_data = amsgrad ? mom2_max_out_ptr + offset : nullptr;
adamw(beta1_,
beta2_,
-learning_rate_,
eps,
old_lr,
lr_ratio_,
coeff_,
tail_numel,
grad_ptr + offset,
mom1_ptr + offset,
mom2_ptr + offset,
mom2_max_in_data,
param_ptr + offset,
mom1_out_ptr + offset,
mom2_out_ptr + offset,
mom2_max_out_data,
param_out_ptr + offset,
amsgrad);
}
}
} // namespace phi
PD_REGISTER_KERNEL(
adamw, CPU, ALL_LAYOUT, phi::AdamwDenseKernel, float, double) {
kernel->InputAt(2).SetDataType(phi::DataType::FLOAT64);
kernel->InputAt(9).SetBackend(phi::Backend::ALL_BACKEND);
}
paddle/phi/kernels/cpu/add_n_kernel.cc
+154
View File
@@ -0,0 +1,154 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/common/amp_type_traits.h"
#include "paddle/phi/kernels/impl/add_n_kernel_impl.h"
#include "glog/logging.h"
namespace phi {
template <typename T, typename Context>
void AddNKernel(const Context& dev_ctx,
const std::vector<const TensorBase*>& x,
DenseTensor* out) {
size_t in_num = x.size();
dev_ctx.template Alloc<T>(out);
if (out && out->numel() == 0) {
return;
}
bool in_place = false;
if (!x.empty() && x[0]->initialized() && DenseTensor::classof(x[0])) {
if ((static_cast<const DenseTensor*>(x[0]))->Holder() == out->Holder()) {
in_place = true;
if (in_num == 1) {
return;
}
}
}
// using MT to keep precision
using MT = typename MPTypeTrait<T>::Type;
auto& place = *dev_ctx.eigen_device();
if constexpr (std::is_same_v<MT, T>) {
// compute in out
auto result = EigenVector<T>::Flatten(*out);
if (!in_place) {
funcs::SetConstant<Context, T> constant_functor;
constant_functor(dev_ctx, out, static_cast<T>(0));
}
funcs::SelectedRowsAddToTensor<Context, T> functor;
size_t start = in_place ? 1 : 0;
for (size_t i = start; i < in_num; i++) {
if (DenseTensor::classof(x[i])) {
auto& in_t = *(static_cast<const DenseTensor*>(x[i]));
if (!in_t.initialized() || in_t.numel() == 0) {
continue;
}
auto in = EigenVector<T>::Flatten(in_t);
result.device(place) = result + in;
} else if (SelectedRows::classof(x[i])) {
auto& in_t = *(static_cast<const SelectedRows*>(x[i]));
functor(dev_ctx, in_t, out);
} else {
PADDLE_THROW(common::errors::InvalidArgument(
"Expected type of Input(X) of %d-th must be Tensor, "
"SelectedRows. But got "
"unsupported type: %s.",
i,
x[i]->type_info().name()));
}
}
} else {
// compute in temp_out by using MT
DenseTensor temp_out;
temp_out.Resize(out->dims());
dev_ctx.template Alloc<MT>(&temp_out);
auto result_mp = EigenVector<MT>::Flatten(temp_out);
// set temp_out
funcs::SetConstant<Context, MT> constant_functor;
if (in_place && DenseTensor::classof(x[0]) && x[0]->initialized()) {
auto& in_0 = *(static_cast<const DenseTensor*>(x[0]));
if (in_0.numel()) {
auto in_0_e = EigenVector<T>::Flatten(in_0).template cast<MT>();
result_mp.device(place) = in_0_e;
} else {
constant_functor(dev_ctx, &temp_out, static_cast<MT>(0));
}
} else {
constant_functor(dev_ctx, &temp_out, static_cast<MT>(0));
}
funcs::SelectedRowsAddToTensor<Context, MT> functor;
size_t start = in_place ? 1 : 0;
for (size_t i = start; i < in_num; i++) {
if (DenseTensor::classof(x[i])) {
auto& in_t = *(static_cast<const DenseTensor*>(x[i]));
if (!in_t.initialized() || in_t.numel() == 0) {
continue;
}
auto in = EigenVector<T>::Flatten(in_t).template cast<MT>();
result_mp.device(place) = result_mp + in;
} else if (SelectedRows::classof(x[i])) {
auto& in_t = *(static_cast<const SelectedRows*>(x[i]));
functor(dev_ctx, in_t, &temp_out);
} else {
PADDLE_THROW(common::errors::InvalidArgument(
"Expected type of Input(X) of %d-th must be Tensor, "
"SelectedRows. But got "
"unsupported type: %s.",
i,
x[i]->type_info().name()));
}
}
// cast back to T, and copy to out
auto result = EigenVector<T>::Flatten(*out);
result.device(place) = result_mp.template cast<T>();
}
}
} // namespace phi
PD_REGISTER_KERNEL(add_n,
CPU,
ALL_LAYOUT,
phi::AddNKernel,
float,
double,
int,
phi::bfloat16,
phi::float16,
int64_t,
phi::complex64,
phi::complex128) {}
PD_REGISTER_KERNEL(add_n_array,
CPU,
ALL_LAYOUT,
phi::AddNArrayKernel,
float,
double,
int,
phi::bfloat16,
phi::float16,
int64_t,
phi::complex64,
phi::complex128) {}
paddle/phi/kernels/cpu/add_position_encoding_grad_kernel.cc
+44
View File
@@ -0,0 +1,44 @@
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/eigen/common.h"
namespace phi {
template <typename T, typename Context>
void AddPositionEncodingGradKernel(const Context& dev_ctx,
const DenseTensor& x_in,
const DenseTensor& out_grad,
float alpha,
float beta,
DenseTensor* x_grad) {
auto* dOut = &out_grad;
auto dout = EigenVector<T>::Flatten(*dOut);
auto* dX = x_grad;
dev_ctx.template Alloc<T>(dX);
auto dx = EigenVector<T>::Flatten(*dX);
auto* place = dev_ctx.eigen_device();
dx.device(*place) = dout * static_cast<T>(alpha);
}
} // namespace phi
PD_REGISTER_KERNEL(add_position_encoding_grad,
CPU,
ALL_LAYOUT,
phi::AddPositionEncodingGradKernel,
float,
double) {}
paddle/phi/kernels/cpu/add_position_encoding_kernel.cc
+104
View File
@@ -0,0 +1,104 @@
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/eigen/common.h"
namespace phi {
template <typename T, typename Context>
void AddPositionEncodingKernel(const Context& dev_ctx,
const DenseTensor& x_in,
float alpha,
float beta,
DenseTensor* out) {
auto* X = &x_in;
auto& x_lod = X->lod();
auto* src_ptr = X->data<T>();
auto* dst_ptr = dev_ctx.template Alloc<T>(out);
auto x_dim = X->dims();
int64_t batch_size = 0;
int64_t max_seq_len = 0;
int64_t enc_size = 0;
if (x_lod.empty()) {
PADDLE_ENFORCE_EQ(x_dim.size(),
3,
common::errors::InvalidArgument(
"The input(X)'s dimension of AddPositionEncodingOp "
"should be equal to "
"3, but received %d. ",
x_dim.size()));
batch_size = x_dim[0];
max_seq_len = x_dim[1];
enc_size = x_dim[2];
} else {
PADDLE_ENFORCE_EQ(x_dim.size(),
2,
common::errors::InvalidArgument(
"The input(X)'s dimension of AddPositionEncodingOp "
"should be equal to "
"2, but received %d. ",
x_dim.size()));
PADDLE_ENFORCE_EQ(x_lod.size(),
1,
common::errors::InvalidArgument(
"The input(X)'s lod level of AddPositionEncodingOp "
"should be equal to "
"1, but received %d. ",
x_lod.size()));
batch_size = x_lod[0].size() - 1;
max_seq_len = -1;
enc_size = x_dim[1];
}
PADDLE_ENFORCE_EQ(enc_size % 2,
0,
common::errors::InvalidArgument(
"The input(X)'s feature size of "
"AddPositionEncodingOp only support even, "
"but received an odd number: %d. ",
enc_size));
const int64_t half_size = enc_size / 2;
for (int64_t i = 0; i < batch_size; ++i) {
const int64_t max_length =
x_lod.empty() ? max_seq_len : x_lod[0][i + 1] - x_lod[0][i];
for (int64_t j = 0; j < max_length; ++j) {
for (int64_t k = 0; k < half_size; ++k) {
const double val =
(half_size > 1)
? j / pow(10000.0, static_cast<double>(k) / (half_size - 1))
: j / 10000.0;
dst_ptr[k] = src_ptr[k] * alpha + sin(val) * beta;
dst_ptr[half_size + k] =
src_ptr[half_size + k] * alpha + cos(val) * beta;
}
src_ptr += enc_size;
dst_ptr += enc_size;
}
}
}
} // namespace phi
PD_REGISTER_KERNEL(add_position_encoding,
CPU,
ALL_LAYOUT,
phi::AddPositionEncodingKernel,
float,
double) {}
paddle/phi/kernels/cpu/addmm_grad_kernel.cc
+22
View File
@@ -0,0 +1,22 @@
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/phi/kernels/addmm_grad_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/addmm_grad_kernel_impl.h"
PD_REGISTER_KERNEL(
addmm_grad, CPU, ALL_LAYOUT, phi::AddmmGradKernel, float, double) {}
paddle/phi/kernels/cpu/addmm_kernel.cc
+21
View File
@@ -0,0 +1,21 @@
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/phi/kernels/addmm_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/addmm_kernel_impl.h"
PD_REGISTER_KERNEL(addmm, CPU, ALL_LAYOUT, phi::AddmmKernel, float, double) {}
paddle/phi/kernels/cpu/affine_channel_grad_kernel.cc
+132
View File
@@ -0,0 +1,132 @@
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/affine_channel_grad_kernel.h"
#include <string>
#include <unordered_map>
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/eigen/common.h"
namespace phi {
template <typename T>
using EigenArrayMap =
Eigen::Map<Eigen::Array<T, Eigen::Dynamic, Eigen::Dynamic>>;
template <typename T>
using ConstEigenArrayMap =
Eigen::Map<const Eigen::Array<T, Eigen::Dynamic, Eigen::Dynamic>>;
template <typename T>
using EigenVectorArrayMap = Eigen::Map<Eigen::Array<T, Eigen::Dynamic, 1>>;
template <typename T>
using ConstEigenVectorArrayMap =
Eigen::Map<const Eigen::Array<T, Eigen::Dynamic, 1>>;
template <typename T, typename Context>
void AffineChannelGradKernel(const Context& dev_ctx,
const DenseTensor& x_in,
const DenseTensor& scale_in,
const DenseTensor& bias_in,
const DenseTensor& out_grad,
const std::string& data_layout,
DenseTensor* x_grad,
DenseTensor* scale_grad,
DenseTensor* bias_grad) {
auto* x = &x_in;
auto* scale = &scale_in;
auto* dy = &out_grad;
auto* dx = x_grad;
auto* dscale = scale_grad;
auto* dbias = bias_grad;
const DataLayout layout = StringToDataLayout(data_layout);
auto dims = x->dims();
int N = static_cast<int>(dims[0]);
int C = static_cast<int>(layout == DataLayout::NCHW ? dims[1]
: dims[dims.size() - 1]);
int HxW = static_cast<int>(x->numel() / N / C);
auto* dy_d = dy->data<T>();
auto* scale_d = scale->data<T>();
ConstEigenVectorArrayMap<T> scale_e(scale_d, C);
T* dx_d = dx ? dev_ctx.template Alloc<T>(dx) : nullptr;
T* dscale_d = dscale ? dev_ctx.template Alloc<T>(dscale) : nullptr;
T* dbias_d = dbias ? dev_ctx.template Alloc<T>(dbias) : nullptr;
EigenVectorArrayMap<T> dscale_e(dscale_d, C);
EigenVectorArrayMap<T> dbias_e(dbias_d, C);
if (layout == DataLayout::NCHW) {
// compute dscale and dbias
int64_t stride = static_cast<int64_t>(C) * HxW;
auto* original_dy_d = dy_d;
if (dscale && dbias) {
auto* x_d = x->data<T>();
for (int i = 0; i < N; i++) {
ConstEigenArrayMap<T> x_e(x_d, HxW, C);
ConstEigenArrayMap<T> dy_e(dy_d, HxW, C);
if (i == 0) {
dscale_e = (x_e * dy_e).colwise().sum();
} else {
dscale_e += (x_e * dy_e).colwise().sum();
}
if (i == 0) {
dbias_e = dy_e.colwise().sum();
} else {
dbias_e += dy_e.colwise().sum();
}
x_d += stride;
dy_d += stride;
}
}
// compute dx
if (dx) {
dy_d = original_dy_d;
for (int i = 0; i < N; i++) {
ConstEigenArrayMap<T> dy_e(dy_d, HxW, C);
EigenArrayMap<T> dx_e(dx_d, HxW, C);
dx_e = dy_e.rowwise() * scale_e.transpose();
dy_d += stride;
dx_d += stride;
}
}
} else {
int64_t num = static_cast<int64_t>(N) * HxW;
ConstEigenArrayMap<T> dy_e(dy_d, C, num);
// compute dscale and dbias
if (dscale && dbias) {
auto* x_d = x->data<T>();
ConstEigenArrayMap<T> x_e(x_d, C, num);
dscale_e = (x_e * dy_e).rowwise().sum();
dbias_e = dy_e.rowwise().sum();
}
// compute dx
if (dx) {
EigenArrayMap<T> dx_e(dx_d, C, num);
dx_e = dy_e.colwise() * scale_e;
}
}
}
} // namespace phi
PD_REGISTER_KERNEL(affine_channel_grad,
CPU,
ALL_LAYOUT,
phi::AffineChannelGradKernel,
float,
double) {}
paddle/phi/kernels/cpu/affine_channel_kernel.cc
+83
View File
@@ -0,0 +1,83 @@
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/affine_channel_kernel.h"
#include <string>
#include <unordered_map>
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/eigen/common.h"
namespace phi {
template <typename T>
using EigenArrayMap =
Eigen::Map<Eigen::Array<T, Eigen::Dynamic, Eigen::Dynamic>>;
template <typename T>
using ConstEigenArrayMap =
Eigen::Map<const Eigen::Array<T, Eigen::Dynamic, Eigen::Dynamic>>;
template <typename T>
using EigenVectorArrayMap = Eigen::Map<Eigen::Array<T, Eigen::Dynamic, 1>>;
template <typename T>
using ConstEigenVectorArrayMap =
Eigen::Map<const Eigen::Array<T, Eigen::Dynamic, 1>>;
template <typename T, typename Context>
void AffineChannelKernel(const Context& dev_ctx,
const DenseTensor& x_in,
const DenseTensor& scale_in,
const DenseTensor& bias_in,
const std::string& data_layout,
DenseTensor* out) {
auto* x = &x_in;
auto* scale = &scale_in;
auto* bias = &bias_in;
auto* y = out;
dev_ctx.template Alloc<T>(y);
const DataLayout layout = StringToDataLayout(data_layout);
auto dims = x->dims();
int N = static_cast<int>(dims[0]);
int C = static_cast<int>(layout == DataLayout::NCHW ? dims[1]
: dims[dims.size() - 1]);
int HxW = static_cast<int>(x->numel() / N / C);
auto* scale_d = scale->data<T>();
auto* bias_d = bias->data<T>();
ConstEigenVectorArrayMap<T> a_e(scale_d, C);
ConstEigenVectorArrayMap<T> b_e(bias_d, C);
auto* x_d = x->data<T>();
auto* y_d = y->data<T>();
if (layout == DataLayout::NCHW) {
int64_t stride = static_cast<int64_t>(C) * HxW;
for (int i = 0; i < N; i++) {
ConstEigenArrayMap<T> x_e(x_d, HxW, C);
EigenArrayMap<T> y_e(y_d, HxW, C);
y_e = (x_e.rowwise() * a_e.transpose()).rowwise() + b_e.transpose();
x_d += stride;
y_d += stride;
}
} else {
int64_t num = static_cast<int64_t>(N) * HxW;
ConstEigenArrayMap<T> x_e(x_d, C, num);
EigenArrayMap<T> y_e(y_d, C, num);
y_e = (x_e.colwise() * a_e).colwise() + b_e;
}
}
} // namespace phi
PD_REGISTER_KERNEL(
affine_channel, CPU, ALL_LAYOUT, phi::AffineChannelKernel, float, double) {}
paddle/phi/kernels/cpu/affine_grid_grad_kernel.cc
+146
View File
@@ -0,0 +1,146 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/affine_grid_grad_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/common/int_array.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/affine_grid_utils.h"
namespace phi {
template <typename T>
struct Linspace<CPUContext, T> {
void operator()(T start,
T end,
int count,
bool align_corners,
DenseTensor* numbers,
const CPUContext& dev_ctx) {
numbers->Resize({count});
T* number_data = dev_ctx.template Alloc<T>(numbers);
T slice = (end - start) / (T)(count - 1);
if (!align_corners) {
slice = (end - start) / (T)count;
start *= (T)(count - 1) / (T)count;
}
for (int i = 0; i < count; ++i) {
number_data[i] = start + (T)i * slice;
}
}
};
template <typename T, typename Context>
void AffineGridGrad4DKernel(const Context& dev_ctx,
const DenseTensor& output_grad,
const IntArray& outputShape,
bool align_corners,
DenseTensor* input_grad) {
auto& theta_grad = input_grad;
int n = static_cast<int>(output_grad.dims()[0]);
auto& size_attr = outputShape.GetData();
int h = 0;
int w = 0;
h = static_cast<int>(size_attr[2]);
w = static_cast<int>(size_attr[3]);
theta_grad->Resize({n, 2, 3});
dev_ctx.template Alloc<T>(theta_grad);
funcs::SetConstant<Context, T>()(dev_ctx, theta_grad, static_cast<T>(0));
DenseTensor grid;
GetIdxMap4D<Context, T>(n, h, w, align_corners, &grid, dev_ctx);
// output = grid * theta.T
// TODO(wanghaoshuang): Refine batched matrix multiply
auto blas = funcs::GetBlas<Context, T>(dev_ctx);
for (int i = 0; i < n; ++i) {
DenseTensor sliced_grid = grid.Slice(i, i + 1).Resize(
{static_cast<int64_t>(h) * static_cast<int64_t>(w), 3});
DenseTensor sliced_out_grad = output_grad.Slice(i, i + 1).Resize(
{static_cast<int64_t>(h) * static_cast<int64_t>(w), 2});
DenseTensor sliced_theta_grad = theta_grad->Slice(i, i + 1).Resize({2, 3});
blas.MatMul(sliced_out_grad,
true,
sliced_grid,
false,
T(1),
&sliced_theta_grad,
T(0));
}
}
template <typename T, typename Context>
void AffineGridGrad5DKernel(const Context& dev_ctx,
const DenseTensor& output_grad,
const IntArray& outputShape,
bool align_corners,
DenseTensor* input_grad) {
auto& theta_grad = input_grad;
int n = static_cast<int>(output_grad.dims()[0]);
auto& size_attr = outputShape.GetData();
int d = 0;
int h = 0;
int w = 0;
d = static_cast<int>(size_attr[2]);
h = static_cast<int>(size_attr[3]);
w = static_cast<int>(size_attr[4]);
theta_grad->Resize({n, 3, 4});
dev_ctx.template Alloc<T>(theta_grad);
funcs::SetConstant<Context, T>()(dev_ctx, theta_grad, static_cast<T>(0));
DenseTensor grid;
GetIdxMap5D<Context, T>(n, d, h, w, align_corners, &grid, dev_ctx);
auto blas = funcs::GetBlas<Context, T>(dev_ctx);
for (int i = 0; i < n; ++i) {
DenseTensor sliced_grid = grid.Slice(i, i + 1).Resize(
{static_cast<int64_t>(d) * static_cast<int64_t>(h) *
static_cast<int64_t>(w),
4});
DenseTensor sliced_out_grad = output_grad.Slice(i, i + 1).Resize(
{static_cast<int64_t>(d) * static_cast<int64_t>(h) *
static_cast<int64_t>(w),
3});
DenseTensor sliced_theta_grad = theta_grad->Slice(i, i + 1).Resize({3, 4});
blas.MatMul(sliced_out_grad,
true,
sliced_grid,
false,
T(1),
&sliced_theta_grad,
T(0));
}
}
template <typename T, typename Context>
void AffineGridGradKernel(const Context& dev_ctx,
const DenseTensor& output_grad,
const IntArray& outputShape,
bool align_corners,
DenseTensor* input_grad) {
auto& size_attr = outputShape.GetData();
if (size_attr.size() == 4) {
AffineGridGrad4DKernel<T, Context>(
dev_ctx, output_grad, outputShape, align_corners, input_grad);
} else {
AffineGridGrad5DKernel<T, Context>(
dev_ctx, output_grad, outputShape, align_corners, input_grad);
}
}
} // namespace phi
PD_REGISTER_KERNEL(affine_grid_grad,
CPU,
ALL_LAYOUT,
phi::AffineGridGradKernel,
float,
double){};
paddle/phi/kernels/cpu/affine_grid_kernel.cc
+132
View File
@@ -0,0 +1,132 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/affine_grid_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/common/int_array.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/affine_grid_utils.h"
namespace phi {
template <typename T>
struct Linspace<CPUContext, T> {
void operator()(T start,
T end,
int count,
bool align_corners,
DenseTensor* numbers,
const CPUContext& dev_ctx) {
numbers->Resize({count});
T* number_data = dev_ctx.template Alloc<T>(numbers);
T slice = (end - start) / (T)(count - 1);
if (!align_corners) {
slice = (end - start) / (T)count;
start *= (T)(count - 1) / (T)count;
}
for (int i = 0; i < count; ++i) {
number_data[i] = start + (T)i * slice;
}
}
};
template <typename T, typename Context>
void AffineGrid4DKernel(const Context& dev_ctx,
const DenseTensor& input,
const IntArray& outputShape,
bool align_corners,
DenseTensor* output) {
auto* theta = &input;
int n = static_cast<int>(theta->dims()[0]);
auto& size_attr = outputShape.GetData();
int h = 0;
int w = 0;
h = static_cast<int>(size_attr[2]);
w = static_cast<int>(size_attr[3]);
output->Resize({n, h, w, 2});
dev_ctx.template Alloc<T>(output);
funcs::SetConstant<Context, T>()(dev_ctx, output, static_cast<T>(0));
DenseTensor grid;
GetIdxMap4D<Context, T>(n, h, w, align_corners, &grid, dev_ctx);
// output = grid * theta.T
// TODO(wanghaoshuang): Refine batched matrix multiply
auto blas = funcs::GetBlas<Context, T>(dev_ctx);
for (int i = 0; i < n; ++i) {
DenseTensor sliced_grid = grid.Slice(i, i + 1).Resize(
{static_cast<int64_t>(h) * static_cast<int64_t>(w), 3});
DenseTensor sliced_theta = theta->Slice(i, i + 1).Resize({2, 3});
DenseTensor sliced_out = output->Slice(i, i + 1).Resize(
{static_cast<int64_t>(h) * static_cast<int64_t>(w), 2});
blas.MatMul(
sliced_grid, false, sliced_theta, true, T(1), &sliced_out, T(0));
}
}
template <typename T, typename Context>
void AffineGrid5DKernel(const Context& dev_ctx,
const DenseTensor& input,
const IntArray& outputShape,
bool align_corners,
DenseTensor* output) {
auto* theta = &input;
int n = static_cast<int>(theta->dims()[0]);
auto& size_attr = outputShape.GetData();
int d = 0;
int h = 0;
int w = 0;
d = static_cast<int>(size_attr[2]);
h = static_cast<int>(size_attr[3]);
w = static_cast<int>(size_attr[4]);
output->Resize({n, d, h, w, 3});
dev_ctx.template Alloc<T>(output);
funcs::SetConstant<Context, T>()(dev_ctx, output, static_cast<T>(0));
DenseTensor grid;
GetIdxMap5D<Context, T>(n, d, h, w, align_corners, &grid, dev_ctx);
auto blas = funcs::GetBlas<Context, T>(dev_ctx);
for (int i = 0; i < n; ++i) {
DenseTensor sliced_grid = grid.Slice(i, i + 1).Resize(
{static_cast<int64_t>(d) * static_cast<int64_t>(h) *
static_cast<int64_t>(w),
4});
DenseTensor sliced_theta = theta->Slice(i, i + 1).Resize({3, 4});
DenseTensor sliced_out = output->Slice(i, i + 1).Resize(
{static_cast<int64_t>(d) * static_cast<int64_t>(h) *
static_cast<int64_t>(w),
3});
blas.MatMul(
sliced_grid, false, sliced_theta, true, T(1), &sliced_out, T(0));
}
}
template <typename T, typename Context>
void AffineGridKernel(const Context& dev_ctx,
const DenseTensor& input,
const IntArray& outputShape,
bool align_corners,
DenseTensor* output) {
auto& size_attr = outputShape.GetData();
if (size_attr.size() == 4) {
AffineGrid4DKernel<T, Context>(
dev_ctx, input, outputShape, align_corners, output);
} else {
AffineGrid5DKernel<T, Context>(
dev_ctx, input, outputShape, align_corners, output);
}
}
} // namespace phi
PD_REGISTER_KERNEL(
affine_grid, CPU, ALL_LAYOUT, phi::AffineGridKernel, float, double){};
paddle/phi/kernels/cpu/all_gather_kernel.cc
+111
View File
@@ -0,0 +1,111 @@
// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/all_gather_kernel.h"
#include "paddle/phi/backends/all_context.h"
#include "paddle/phi/core/kernel_registry.h"
#if defined(PADDLE_WITH_GLOO)
#include "paddle/phi/core/distributed/gloo_comm_context.h"
#endif
#ifdef PADDLE_WITH_CUSTOM_DEVICE
#include "paddle/phi/core/distributed/xccl_comm_context.h"
#endif
namespace phi {
template <typename T, typename Context>
void AllGatherKernel(const Context& dev_ctx,
const DenseTensor& x,
int nranks,
DenseTensor* out) {
#if defined(PADDLE_WITH_GLOO)
dev_ctx.template Alloc<T>(out);
auto out_dims = x.dims();
out_dims[0] *= nranks;
out->Resize(out_dims);
auto comm_ctx =
static_cast<distributed::GlooCommContext*>(dev_ctx.GetCommContext());
PADDLE_ENFORCE_EQ(
nranks,
comm_ctx->GetSize(),
errors::InvalidArgument(
"nranks: %s should equal to %s", nranks, comm_ctx->GetSize()));
comm_ctx->AllGather(out, x);
#else
PADDLE_THROW(errors::Unavailable(
"PaddlePaddle should compile with GLOO by setting WITH_GLOO=ON"));
#endif
}
#ifdef PADDLE_WITH_CUSTOM_DEVICE
template <typename T>
void AllGatherKernel(const CustomContext& dev_ctx,
const DenseTensor& x,
int nranks,
DenseTensor* out) {
dev_ctx.template Alloc<T>(out);
auto out_dims = x.dims();
out_dims[0] *= nranks;
out->Resize(out_dims);
auto comm_ctx =
static_cast<distributed::XCCLCommContext*>(dev_ctx.GetCommContext());
PADDLE_ENFORCE_EQ(
nranks,
comm_ctx->GetSize(),
errors::InvalidArgument(
"nranks: %s should equal to %s", nranks, comm_ctx->GetSize()));
comm_ctx->AllGather(out, x, dev_ctx.stream());
}
#endif
} // namespace phi
PD_REGISTER_KERNEL(all_gather,
CPU,
ALL_LAYOUT,
phi::AllGatherKernel,
float,
double,
int,
bool,
int8_t,
uint8_t,
int16_t,
int64_t,
phi::float16,
phi::complex64,
phi::complex128) {}
#ifdef PADDLE_WITH_CUSTOM_DEVICE
PD_REGISTER_KERNEL(all_gather,
Custom,
ALL_LAYOUT,
phi::AllGatherKernel,
float,
double,
int,
bool,
int8_t,
uint8_t,
int16_t,
int64_t,
phi::float16,
phi::complex64,
phi::complex128) {}
#endif
paddle/phi/kernels/cpu/all_reduce_kernel.cc
+103
View File
@@ -0,0 +1,103 @@
// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/all_reduce_kernel.h"
#include "paddle/phi/backends/all_context.h"
#include "paddle/phi/core/kernel_registry.h"
#if defined(PADDLE_WITH_GLOO)
#include "paddle/phi/core/distributed/gloo_comm_context.h"
#endif
#ifdef PADDLE_WITH_CUSTOM_DEVICE
#include "paddle/phi/core/distributed/xccl_comm_context.h"
#endif
namespace phi {
template <typename T, typename Context>
void AllReduceKernel(const Context& dev_ctx,
const DenseTensor& x,
int reduce_type,
DenseTensor* out) {
#if defined(PADDLE_WITH_GLOO)
out->Resize(x.dims());
dev_ctx.template Alloc<T>(out);
auto comm_ctx =
static_cast<distributed::GlooCommContext*>(dev_ctx.GetCommContext());
PADDLE_ENFORCE_NE(
comm_ctx,
nullptr,
errors::Unavailable("NCCLCommContext is nullptr, collective op should "
"has ring_id attr."));
comm_ctx->AllReduce(out, x, reduce_type);
#else
PADDLE_THROW(
errors::PreconditionNotMet("PaddlePaddle should compile with GPU."));
#endif
}
#ifdef PADDLE_WITH_CUSTOM_DEVICE
template <typename T>
void AllReduceKernel(const CustomContext& dev_ctx,
const DenseTensor& x,
int reduce_type,
DenseTensor* out) {
out->Resize(x.dims());
dev_ctx.template Alloc<T>(out);
auto comm_ctx =
static_cast<distributed::XCCLCommContext*>(dev_ctx.GetCommContext());
PADDLE_ENFORCE_NE(
comm_ctx,
nullptr,
errors::Unavailable("XCCLCommContext is nullptr, collective op should "
"has ring_id attr."));
comm_ctx->AllReduce(
out, x, ccl::ToXCCLReduceOp(reduce_type), dev_ctx.stream());
}
#endif
} // namespace phi
PD_REGISTER_KERNEL(all_reduce,
CPU,
ALL_LAYOUT,
phi::AllReduceKernel,
float,
double,
int,
bool,
int8_t,
uint8_t,
int16_t,
int64_t,
phi::float16) {}
#ifdef PADDLE_WITH_CUSTOM_DEVICE
PD_REGISTER_KERNEL(all_reduce,
Custom,
ALL_LAYOUT,
phi::AllReduceKernel,
float,
double,
int,
bool,
int8_t,
uint8_t,
int64_t,
phi::float16) {}
#endif
paddle/phi/kernels/cpu/all_to_all_kernel.cc
+97
View File
@@ -0,0 +1,97 @@
// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/all_to_all_kernel.h"
#include "paddle/phi/backends/all_context.h"
#include "paddle/phi/core/kernel_registry.h"
#ifdef PADDLE_WITH_CUSTOM_DEVICE
#include "paddle/phi/core/distributed/xccl_comm_context.h"
#endif
namespace phi {
template <typename T, typename Context>
void AllToAllKernel(const Context& dev_ctx UNUSED,
const DenseTensor& x UNUSED,
DenseTensor* out UNUSED) {
PADDLE_THROW(
errors::Unimplemented("Unimplemented cpu kernel for all_to_all."));
}
#ifdef PADDLE_WITH_CUSTOM_DEVICE
template <typename T>
void AllToAllKernel(const CustomContext& dev_ctx,
const DenseTensor& x,
DenseTensor* out) {
out->Resize(x.dims());
dev_ctx.template Alloc<T>(out);
auto comm_ctx =
static_cast<distributed::XCCLCommContext*>(dev_ctx.GetCommContext());
int nranks = comm_ctx->GetSize();
int rank = comm_ctx->GetRank();
int send_numel = x.numel() / nranks;
std::vector<void*> sendbuf, recvbuf;
std::vector<size_t> sendsize(send_numel, nranks);
std::vector<DataType> sendtype(x.dtype(), nranks);
for (auto i = 0; i < nranks; ++i) {
sendbuf.push_back(x.data<T>() + i * send_numel);
recvbuf.push_back(out->data<T>() + i * send_numel);
}
DeviceManager::CCLAllToAll(dev_ctx.GetPlace().GetDeviceType(),
const_cast<const void**>(sendbuf.data()),
sendsize.data(),
sendtype.data(),
recvbuf.data(),
sendsize.data(),
sendtype.data(),
rank,
nranks,
comm_ctx->GetXcclComm(),
dev_ctx.stream());
}
#endif
} // namespace phi
PD_REGISTER_KERNEL(all_to_all,
CPU,
ALL_LAYOUT,
phi::AllToAllKernel,
float,
double,
int,
bool,
int8_t,
uint8_t,
int16_t,
int64_t,
phi::float16) {}
#ifdef PADDLE_WITH_CUSTOM_DEVICE
PD_REGISTER_KERNEL(all_to_all,
Custom,
ALL_LAYOUT,
phi::AllToAllKernel,
float,
double,
int,
bool,
int8_t,
int16_t,
uint8_t,
int64_t,
phi::float16) {}
#endif
paddle/phi/kernels/cpu/allclose_kernel.cc
+107
View File
@@ -0,0 +1,107 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/allclose_kernel.h"
#include <cmath>
#include <type_traits>
#include "glog/logging.h"
#include "paddle/phi/core/enforce.h"
#include "paddle/phi/core/kernel_registry.h"
namespace phi {
template <typename T, typename Context>
void AllCloseKernel(const Context& dev_ctx,
const DenseTensor& x,
const DenseTensor& y,
const Scalar& rtol,
const Scalar& atol,
bool equal_nan,
DenseTensor* out) {
if (x.numel() == 0 || y.numel() == 0) {
auto* out_data = dev_ctx.template Alloc<bool>(out);
*out_data = true;
return;
}
double rtol_v = NAN, atol_v = NAN;
if (rtol.dtype() == DataType::FLOAT64) {
rtol_v = rtol.to<double>();
} else if (rtol.dtype() == DataType::FLOAT32) {
rtol_v = rtol.to<float>();
} else {
PADDLE_THROW(common::errors::InvalidArgument(
"Input (Rtol) type must be double or float, but get %s.",
rtol.dtype()));
}
if (atol.dtype() == DataType::FLOAT64) {
atol_v = atol.to<double>();
} else if (atol.dtype() == DataType::FLOAT32) {
atol_v = atol.to<float>();
} else {
PADDLE_THROW(common::errors::InvalidArgument(
"Input (Atol) type must be double or float, but get %s.",
atol.dtype()));
}
VLOG(3) << "rtol and atol is : " << rtol_v << " " << atol_v;
auto* in_a = x.data<T>();
auto* in_b = y.data<T>();
auto* out_data = dev_ctx.template Alloc<bool>(out);
*out_data = true;
auto num = x.numel();
if (std::is_same<T, int32_t>::value || std::is_same<T, int64_t>::value ||
std::is_same<T, bool>::value) {
for (int64_t i = 0; i < num; ++i) {
const double a = static_cast<double>(in_a[i]),
b = static_cast<double>(in_b[i]);
double left = (a > b ? a - b : b - a);
double right = atol_v + (b > 0 ? rtol_v * b : (-rtol_v) * b);
double diff = (left > right ? left - right : right - left);
bool val = a == b || left <= right || diff <= 1e-15;
*out_data &= val;
}
} else {
for (int64_t i = 0; i < num; ++i) {
const T a = in_a[i], b = in_b[i];
bool val = false;
if (std::isnan(static_cast<double>(a)) ||
std::isnan(static_cast<double>(b))) {
val = equal_nan && std::isnan(static_cast<double>(a)) ==
std::isnan(static_cast<double>(b));
} else {
T left = (a > b ? a - b : b - a);
T right = atol_v + (b > 0 ? rtol_v * b : (-rtol_v) * b);
T diff = (left > right ? left - right : right - left);
val = a == b || left <= right || diff <= 1e-15;
}
*out_data &= val;
}
}
}
} // namespace phi
PD_REGISTER_KERNEL(allclose,
CPU,
ALL_LAYOUT,
phi::AllCloseKernel,
float,
double,
bool,
int,
int64_t) {
kernel->OutputAt(0).SetDataType(phi::DataType::BOOL);
}
paddle/phi/kernels/cpu/amp_kernel.cc
+126
View File
@@ -0,0 +1,126 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/amp_kernel.h"
#include <cmath>
#include "paddle/phi/common/amp_type_traits.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/empty_kernel.h"
#include "paddle/phi/kernels/funcs/eigen/common.h"
#include "paddle/phi/kernels/impl/amp_kernel_impl.h"
#include "paddle/phi/kernels/isfinite_kernel.h"
#include "paddle/phi/kernels/reduce_all_kernel.h"
namespace phi {
// Utils
template <typename T, bool IsFoundInfOnCPU>
class UpdateLossScalingFunctor<CPUContext, T, IsFoundInfOnCPU> {
public:
void operator()(const CPUContext& dev_ctx UNUSED,
const bool* found_inf_data,
const T* pre_loss_scaling_data,
const int* good_in_data,
const int* bad_in_data,
const int incr_every_n_steps,
const int decr_every_n_nan_or_inf,
const float incr_ratio,
const float decr_ratio,
T* updated_loss_scaling_data,
int* good_out_data,
int* bad_out_data) const {
PADDLE_ENFORCE_EQ(
IsFoundInfOnCPU,
true,
common::errors::InvalidArgument(
"The Input(FoundInfinite) should be on the CPUPlace."));
Update<T>(found_inf_data,
pre_loss_scaling_data,
good_in_data,
bad_in_data,
incr_every_n_steps,
decr_every_n_nan_or_inf,
incr_ratio,
decr_ratio,
updated_loss_scaling_data,
good_out_data,
bad_out_data);
}
};
// Kernels
template <typename T, typename Context>
void CheckFiniteAndUnscaleKernel(const Context& dev_ctx,
const std::vector<const DenseTensor*>& xs,
const DenseTensor& scale,
std::vector<DenseTensor*> outs,
DenseTensor* found_infinite) {
const T* scale_data = scale.data<T>();
bool* found_inf_data = dev_ctx.template Alloc<bool>(found_infinite);
*found_inf_data = false;
DenseTensor is_finite = Empty<bool>(dev_ctx, {1});
bool* is_finite_data = is_finite.template data<bool>();
auto& dev = *dev_ctx.eigen_device();
T inverse_scale = 1.0 / *scale_data;
for (size_t i = 0; i < xs.size(); ++i) {
const auto* x = xs[i];
auto* out = outs[i];
dev_ctx.template Alloc<T>(out);
if (!(*found_inf_data)) {
DenseTensor tmp;
tmp.Resize(x->dims());
IsfiniteKernel<T, Context>(dev_ctx, *x, &tmp);
std::vector<int64_t> dims(x->dims().size());
std::iota(dims.begin(), dims.end(), 0);
AllKernel<bool, Context>(dev_ctx, tmp, dims, false, &is_finite);
*found_inf_data = !(*is_finite_data);
}
auto eigen_out = EigenVector<T>::Flatten(*out);
auto eigen_in = EigenVector<T>::Flatten(*x);
if (!(*found_inf_data)) {
eigen_out.device(dev) = eigen_in * inverse_scale;
} else {
eigen_out.device(dev) = eigen_in * static_cast<T>(0);
}
}
}
} // namespace phi
PD_REGISTER_KERNEL(check_finite_and_unscale,
CPU,
ALL_LAYOUT,
phi::CheckFiniteAndUnscaleKernel,
float,
double) {
kernel->OutputAt(1).SetDataType(phi::DataType::BOOL);
}
PD_REGISTER_KERNEL(update_loss_scaling,
CPU,
ALL_LAYOUT,
phi::UpdateLossScalingKernel,
float,
double) {
kernel->OutputAt(2).SetDataType(phi::DataType::INT32);
kernel->OutputAt(3).SetDataType(phi::DataType::INT32);
}
paddle/phi/kernels/cpu/angle_grad_kernel.cc
+30
View File
@@ -0,0 +1,30 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/angle_grad_kernel.h"
#include "paddle/phi/kernels/impl/angle_grad_kernel_impl.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
PD_REGISTER_KERNEL(angle_grad,
CPU,
ALL_LAYOUT,
phi::AngleGradKernel,
float,
double,
phi::complex64,
phi::complex128) {
kernel->InputAt(1).SetDataType(phi::dtype::ToReal(kernel_key.dtype()));
}
paddle/phi/kernels/cpu/angle_kernel.cc
+31
View File
@@ -0,0 +1,31 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/angle_kernel.h"
#include "paddle/phi/kernels/impl/angle_kernel_impl.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
PD_REGISTER_KERNEL(angle,
CPU,
ALL_LAYOUT,
phi::AngleKernel,
float,
double,
phi::complex64,
phi::complex128) {
kernel->OutputAt(0).SetDataType(phi::DataType::UNDEFINED);
}
paddle/phi/kernels/cpu/arange_kernel.cc
+125
View File
@@ -0,0 +1,125 @@
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/phi/kernels/arange_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/common/amp_type_traits.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/range_function.h"
namespace phi {
template <typename T, typename Context>
void ArangeFunc(const Context& dev_ctx,
const T& start_value,
const T& end_value,
const T& step_value,
DenseTensor* out) {
int64_t size = 0;
funcs::GetSize(start_value, end_value, step_value, &size);
out->Resize({size});
T* out_data = dev_ctx.template Alloc<T>(out);
T value = start_value;
for (int64_t i = 0; i < size; ++i) {
out_data[i] = value;
value += step_value;
}
}
template <typename T, typename Context>
void ArangeTensorKernel(const Context& dev_ctx,
const DenseTensor& start,
const DenseTensor& end,
const DenseTensor& step,
DenseTensor* out) {
int64_t size = 0;
using MPType = typename phi::dtype::MPTypeTrait<T>::Type;
bool any_float = phi::IsFloatingType(start.dtype()) ||
phi::IsFloatingType(end.dtype()) ||
phi::IsFloatingType(step.dtype());
Scalar start_scalar(start);
Scalar end_scalar(end);
Scalar step_scalar(step);
if (any_float) {
double sv = start_scalar.to<double>();
double ev = end_scalar.to<double>();
double stv = step_scalar.to<double>();
funcs::GetSize<double>(sv, ev, stv, &size);
} else {
int64_t sv = start_scalar.to<int64_t>();
int64_t ev = end_scalar.to<int64_t>();
int64_t stv = step_scalar.to<int64_t>();
funcs::GetSize<int64_t>(sv, ev, stv, &size);
}
MPType start_value = start_scalar.to<MPType>();
MPType step_value = step_scalar.to<MPType>();
out->Resize({size});
T* out_data = dev_ctx.template Alloc<T>(out);
MPType value = start_value;
for (int64_t i = 0; i < size; ++i) {
out_data[i] = static_cast<T>(value);
value += step_value;
}
}
template <typename T, typename Context>
void ArangeKernel(const Context& dev_ctx,
const Scalar& start,
const Scalar& end,
const Scalar& step,
DenseTensor* out) {
bool any_float = phi::IsFloatingType(start.dtype()) ||
phi::IsFloatingType(end.dtype()) ||
phi::IsFloatingType(step.dtype());
int64_t size = 0;
using MPType = typename phi::dtype::MPTypeTrait<T>::Type;
if (any_float) {
double sv = start.to<double>();
double ev = end.to<double>();
double stv = step.to<double>();
funcs::GetSize<double>(sv, ev, stv, &size);
} else {
int64_t sv = start.to<int64_t>();
int64_t ev = end.to<int64_t>();
int64_t stv = step.to<int64_t>();
funcs::GetSize<int64_t>(sv, ev, stv, &size);
}
MPType start_value = start.to<MPType>();
MPType step_value = step.to<MPType>();
out->Resize({size});
T* out_data = dev_ctx.template Alloc<T>(out);
MPType value = start_value;
for (int64_t i = 0; i < size; ++i) {
out_data[i] = static_cast<T>(value);
value += step_value;
}
}
} // namespace phi
PD_REGISTER_KERNEL(arange_tensor,
CPU,
ALL_LAYOUT,
phi::ArangeTensorKernel,
float,
double,
int,
int64_t) {}
PD_REGISTER_KERNEL(
arange, CPU, ALL_LAYOUT, phi::ArangeKernel, float, double, int, int64_t) {}
paddle/phi/kernels/cpu/arg_min_max_kernel.cc
+224
View File
@@ -0,0 +1,224 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/arg_min_max_kernel.h"
#include "paddle/common/ddim.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/core/utils/data_type.h"
#include "paddle/phi/kernels/funcs/eigen/common.h"
#include "paddle/phi/kernels/funcs/math_function.h"
namespace phi {
enum ArgMinMaxType { kArgMin, kArgMax };
template <typename Context,
typename T,
typename Tout,
int64_t Rank,
ArgMinMaxType argMinMaxValue>
struct ArgMinMaxFunctor {};
#define DECLARE_ARG_MIN_MAX_FUNCTOR(eigen_op_type, enum_argminmax_value) \
template <typename Context, typename T, typename Tout, int64_t Rank> \
struct ArgMinMaxFunctor<Context, T, Tout, Rank, enum_argminmax_value> { \
void operator()(const Context& dev_ctx, \
const DenseTensor& in, \
DenseTensor* out, \
DDim x_dims, \
DDim out_dims, \
int64_t axis, \
bool keepdims, \
bool flatten) { \
auto in_eigen = EigenTensor<T, Rank>::From(in, x_dims); \
if (flatten) { \
auto out_eigen = EigenTensor<Tout, 0>::From(*out, out_dims); \
out_eigen.device(*(dev_ctx.eigen_device())) = \
in_eigen.eigen_op_type(axis).template cast<Tout>(); \
} else { \
if (keepdims) { \
auto out_eigen = EigenTensor<Tout, Rank>::From(*out, out_dims); \
out_eigen.device(*(dev_ctx.eigen_device())) = \
in_eigen.eigen_op_type(axis).template cast<Tout>(); \
} else { \
auto out_eigen = EigenTensor<Tout, Rank - 1>::From(*out, out_dims); \
out_eigen.device(*(dev_ctx.eigen_device())) = \
in_eigen.eigen_op_type(axis).template cast<Tout>(); \
} \
} \
} \
}
DECLARE_ARG_MIN_MAX_FUNCTOR(argmin, ArgMinMaxType::kArgMin);
DECLARE_ARG_MIN_MAX_FUNCTOR(argmax, ArgMinMaxType::kArgMax);
template <typename Context, typename T, ArgMinMaxType EnumArgMinMaxValue>
struct VisitDataArgMinMaxFunctor {
const Context& dev_ctx;
const DenseTensor& x;
int64_t axis;
bool keepdims;
bool flatten;
DenseTensor* out;
explicit VisitDataArgMinMaxFunctor(const Context& dev_ctx,
const DenseTensor& x,
int64_t axis,
bool keepdims,
bool flatten,
DenseTensor* out)
: dev_ctx(dev_ctx),
x(x),
axis(axis),
keepdims(keepdims),
flatten(flatten),
out(out) {}
template <typename Tout>
void apply() const {
dev_ctx.template Alloc<Tout>(out);
if (x.numel() == 0) return;
// if flatten, will construct the new dims for the calculation
DDim x_dims;
DDim out_dims;
int new_axis = axis;
if (flatten) {
// always reduce 1D -> 0D
x_dims = make_ddim({x.numel()});
out_dims = make_ddim({});
new_axis = 0;
} else {
x_dims = x.dims();
out_dims = out->dims();
if (axis < 0) new_axis = axis + x_dims.size();
}
#define CALL_ARG_MINMAX_FUNCTOR(rank) \
ArgMinMaxFunctor<Context, T, Tout, rank, EnumArgMinMaxValue> functor##rank; \
functor##rank(dev_ctx, x, out, x_dims, out_dims, new_axis, keepdims, flatten)
switch (x_dims.size()) {
case 0:
funcs::set_constant(dev_ctx, out, static_cast<Tout>(0));
return;
case 1:
CALL_ARG_MINMAX_FUNCTOR(1);
break;
case 2:
CALL_ARG_MINMAX_FUNCTOR(2);
break;
case 3:
CALL_ARG_MINMAX_FUNCTOR(3);
break;
case 4:
CALL_ARG_MINMAX_FUNCTOR(4);
break;
case 5:
CALL_ARG_MINMAX_FUNCTOR(5);
break;
case 6:
CALL_ARG_MINMAX_FUNCTOR(6);
break;
default:
PADDLE_ENFORCE_LE(
x_dims.size(),
6,
common::errors::InvalidArgument(
"%s operator doesn't supports tensors whose ranks are greater "
"than 6.",
(EnumArgMinMaxValue == kArgMin ? "argmin" : "argmax")));
break;
#undef CALL_ARG_MINMAX_FUNCTOR
}
}
};
template <typename Context, typename T, ArgMinMaxType EnumArgMinMaxValue>
void ArgMinMaxKernel(const Context& dev_ctx,
const DenseTensor& x,
const Scalar& axis,
bool keepdims,
bool flatten,
DataType dtype,
DenseTensor* out) {
PADDLE_ENFORCE_GE(
x.numel(),
0,
common::errors::InvalidArgument(
"argmin/argmax input numel must > 0, bug got %d", x.numel()));
if (dtype == DataType::UNDEFINED) {
VisitDataTypeTiny(
DataType::INT64,
VisitDataArgMinMaxFunctor<Context, T, EnumArgMinMaxValue>(
dev_ctx, x, axis.to<int64_t>(), keepdims, flatten, out));
return;
}
VisitDataTypeTiny(
dtype,
VisitDataArgMinMaxFunctor<Context, T, EnumArgMinMaxValue>(
dev_ctx, x, axis.to<int64_t>(), keepdims, flatten, out));
}
template <typename T, typename Context>
void ArgMinKernel(const Context& dev_ctx,
const DenseTensor& x,
const Scalar& axis,
bool keepdims,
bool flatten,
DataType dtype,
DenseTensor* out) {
ArgMinMaxKernel<Context, T, ArgMinMaxType::kArgMin>(
dev_ctx, x, axis, keepdims, flatten, dtype, out);
}
template <typename T, typename Context>
void ArgMaxKernel(const Context& dev_ctx,
const DenseTensor& x,
const Scalar& axis,
bool keepdims,
bool flatten,
DataType dtype,
DenseTensor* out) {
ArgMinMaxKernel<Context, T, ArgMinMaxType::kArgMax>(
dev_ctx, x, axis, keepdims, flatten, dtype, out);
}
} // namespace phi
PD_REGISTER_KERNEL(argmin,
CPU,
ALL_LAYOUT,
phi::ArgMinKernel,
float,
double,
int32_t,
int64_t,
int16_t,
uint8_t) {
kernel->OutputAt(0).SetDataType(phi::DataType::UNDEFINED);
}
PD_REGISTER_KERNEL(argmax,
CPU,
ALL_LAYOUT,
phi::ArgMaxKernel,
float,
double,
int32_t,
int64_t,
int16_t,
uint8_t) {
kernel->OutputAt(0).SetDataType(phi::DataType::UNDEFINED);
}
paddle/phi/kernels/cpu/argsort_grad_kernel.cc
+144
View File
@@ -0,0 +1,144 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/argsort_grad_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/eigen/common.h"
#include "paddle/phi/kernels/funcs/eigen/eigen_function.h"
#include "paddle/phi/kernels/transpose_kernel.h"
namespace phi {
template <typename T, typename Type>
static void FullAssign(Type input_height,
Type input_width,
int input_dim,
const DenseTensor* input,
const DenseTensor* indices,
T* t_out) {
#ifdef PADDLE_WITH_MKLML
#pragma omp parallel for
#endif
for (Type i = 0; i < input_height; ++i) {
if (input_dim == 1) {
auto e_input = EigenVector<T>::Flatten(*input);
auto e_indices = EigenVector<Type>::Flatten(*indices);
for (Type j = 0; j < input_width; ++j) {
t_out[i * input_width + e_indices(j)] = e_input(j);
}
} else {
auto e_input = EigenMatrix<T>::Reshape(*input, input_dim - 1);
auto e_indices = EigenMatrix<Type>::Reshape(*indices, input_dim - 1);
for (Type j = 0; j < input_width; ++j) {
t_out[i * input_width + e_indices(i, j)] = e_input(i, j);
}
}
}
}
template <typename T, typename Context>
void ArgsortGradKernel(const Context& dev_ctx,
const DenseTensor& indices,
const DenseTensor& input,
const DenseTensor& out_grad,
int axis,
bool descending UNUSED,
bool stable UNUSED,
DenseTensor* in_grad) {
auto in_dims = indices.dims();
auto rank = input.dims().size();
axis = (axis < 0) ? (in_dims.size() + axis) : axis;
dev_ctx.template Alloc<T>(in_grad);
auto dxt = EigenVector<T>::Flatten(*in_grad);
auto& place = *dev_ctx.eigen_device();
dxt.device(place) = dxt.constant(static_cast<T>(0));
if (out_grad.numel() == 0) return;
if (rank == 0) {
Copy<Context>(dev_ctx, out_grad, dev_ctx.GetPlace(), false, in_grad);
return;
}
// Do full assign
if (axis == -1 || axis + 1 == in_dims.size()) {
const int64_t input_height =
common::product(slice_ddim(in_dims, 0, in_dims.size() - 1));
const int64_t input_width = in_dims[in_dims.size() - 1];
FullAssign<T, int64_t>(input_height,
input_width,
in_dims.size(),
&out_grad,
&indices,
in_grad->data<T>());
} else {
// If not full assign do transpose
std::vector<int> trans;
for (int i = 0; i < axis; i++) {
trans.push_back(i);
}
trans.push_back(in_dims.size() - 1);
for (int i = axis + 1; i < in_dims.size() - 1; i++) {
trans.push_back(i);
}
trans.push_back(axis);
DDim trans_dims(in_dims);
for (size_t i = 0; i < trans.size(); i++) {
trans_dims[static_cast<int>(i)] = in_dims[trans[i]];
}
DenseTensor trans_dO;
trans_dO.Resize(trans_dims);
dev_ctx.template Alloc<T>(&trans_dO);
DenseTensor trans_ind;
trans_ind.Resize(trans_dims);
dev_ctx.template Alloc<int64_t>(&trans_ind);
TransposeKernel<T, Context>(dev_ctx, out_grad, trans, &trans_dO);
TransposeKernel<int64_t, Context>(dev_ctx, indices, trans, &trans_ind);
const int64_t input_height =
common::product(slice_ddim(trans_dims, 0, trans_dims.size() - 1));
const int64_t input_width = trans_dims[trans_dims.size() - 1];
DenseTensor tmp_out;
tmp_out.Resize(trans_dims);
T* t_out = dev_ctx.template Alloc<T>(&tmp_out);
FullAssign<T, int64_t>(input_height,
input_width,
in_dims.size(),
&trans_dO,
&trans_ind,
t_out);
// transpose back
TransposeKernel<T, Context>(dev_ctx, tmp_out, trans, in_grad);
}
}
} // namespace phi
PD_REGISTER_KERNEL(argsort_grad,
CPU,
ALL_LAYOUT,
phi::ArgsortGradKernel,
float,
double,
phi::float16,
phi::bfloat16,
uint8_t,
int16_t,
int,
int64_t) {}
paddle/phi/kernels/cpu/argsort_kernel.cc
+197
View File
@@ -0,0 +1,197 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/argsort_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/eigen/common.h"
#include "paddle/phi/kernels/funcs/eigen/eigen_function.h"
#include "paddle/phi/kernels/funcs/math_function.h"
#include "paddle/phi/kernels/transpose_kernel.h"
namespace phi {
template <typename T, typename Type>
static void FullSort(Type input_height,
Type input_width,
int input_dim,
const DenseTensor* input,
T* t_out,
Type* t_indices,
bool descending,
bool stable) {
#ifdef PADDLE_WITH_MKLML
#pragma omp parallel for
#endif
for (Type i = 0; i < input_height; ++i) {
std::vector<std::pair<T, Type>> col_vec;
col_vec.reserve(input_width);
if (input_dim == 1) {
auto e_input = EigenVector<T>::Flatten(*input);
for (Type j = 0; j < input_width; ++j) {
col_vec.push_back(std::pair<T, Type>(e_input(j), j));
}
} else {
auto e_input = EigenMatrix<T>::Reshape(*input, input_dim - 1);
for (Type j = 0; j < input_width; ++j) {
col_vec.push_back(std::pair<T, Type>(e_input(i, j), j));
}
}
if (stable) {
std::stable_sort(
col_vec.begin(),
col_vec.end(),
[&](const std::pair<T, Type>& l, const std::pair<T, Type>& r) {
if (descending)
return (std::isnan(static_cast<double>(l.first)) &&
!std::isnan(static_cast<double>(r.first))) ||
(l.first > r.first);
else
return (!std::isnan(static_cast<double>(l.first)) &&
std::isnan(static_cast<double>(r.first))) ||
(l.first < r.first);
});
} else {
std::sort(col_vec.begin(),
col_vec.end(),
[&](const std::pair<T, Type>& l, const std::pair<T, Type>& r) {
if (descending)
return (std::isnan(static_cast<double>(l.first)) &&
!std::isnan(static_cast<double>(r.first))) ||
(l.first > r.first);
else
return (!std::isnan(static_cast<double>(l.first)) &&
std::isnan(static_cast<double>(r.first))) ||
(l.first < r.first);
});
}
for (Type j = 0; j < input_width; ++j) {
t_out[i * input_width + j] = col_vec[j].first;
t_indices[i * input_width + j] = col_vec[j].second;
}
}
}
template <typename T, typename Context>
void ArgsortKernel(const Context& dev_ctx,
const DenseTensor& input,
int axis,
bool descending,
bool stable,
DenseTensor* output,
DenseTensor* indices) {
auto in_dims = input.dims();
auto rank = in_dims.size();
if (input.numel() == 0) {
output->Resize(in_dims);
indices->Resize(in_dims);
dev_ctx.template Alloc<T>(output);
dev_ctx.template Alloc<int64_t>(indices);
return;
}
axis = (axis < 0) ? (in_dims.size() + axis) : axis;
T* out_data = dev_ctx.template Alloc<T>(output);
// For 0D Tensor
if (rank == 0) {
Copy<Context>(dev_ctx, input, dev_ctx.GetPlace(), false, output);
dev_ctx.template Alloc<int64_t>(indices);
funcs::set_constant(dev_ctx, indices, static_cast<int64_t>(0));
return;
}
// Do full sort
if (axis == -1 || axis + 1 == in_dims.size()) {
const int64_t input_height =
common::product(slice_ddim(in_dims, 0, in_dims.size() - 1));
const int64_t input_width = in_dims[in_dims.size() - 1];
int64_t* ids_data = dev_ctx.template Alloc<int64_t>(indices);
FullSort<T, int64_t>(input_height,
input_width,
in_dims.size(),
&input,
out_data,
ids_data,
descending,
stable);
} else {
// If not full sort do transpose
std::vector<int> trans;
for (int i = 0; i < axis; i++) {
trans.push_back(i);
}
trans.push_back(in_dims.size() - 1);
for (int i = axis + 1; i < in_dims.size() - 1; i++) {
trans.push_back(i);
}
trans.push_back(axis);
DDim trans_dims(in_dims);
for (size_t i = 0; i < trans.size(); i++) {
trans_dims[static_cast<int>(i)] = in_dims[trans[i]];
}
DenseTensor trans_inp;
trans_inp.Resize(trans_dims);
dev_ctx.template Alloc<T>(&trans_inp);
// Do transpose
TransposeKernel<T, Context>(dev_ctx, input, trans, &trans_inp);
const int64_t input_height =
common::product(slice_ddim(trans_dims, 0, trans_dims.size() - 1));
const int64_t input_width = trans_dims[trans_dims.size() - 1];
DenseTensor tmp_out;
tmp_out.Resize(trans_dims);
T* t_out = dev_ctx.template Alloc<T>(&tmp_out);
DenseTensor tmp_indices;
tmp_indices.Resize(trans_dims);
auto* t_ind = dev_ctx.template Alloc<int64_t>(&tmp_indices);
FullSort<T, int64_t>(input_height,
input_width,
in_dims.size(),
&trans_inp,
t_out,
t_ind,
descending,
stable);
dev_ctx.template Alloc<int64_t>(indices);
TransposeKernel<int64_t, Context>(dev_ctx, tmp_indices, trans, indices);
// transpose back
TransposeKernel<T, Context>(dev_ctx, tmp_out, trans, output);
}
}
} // namespace phi
PD_REGISTER_KERNEL(argsort,
CPU,
ALL_LAYOUT,
phi::ArgsortKernel,
float,
double,
int,
int64_t,
int16_t,
uint8_t,
phi::float16,
phi::bfloat16) {
kernel->OutputAt(1).SetDataType(phi::DataType::INT64);
}
paddle/phi/kernels/cpu/as_complex_kernel.cc
+25
View File
@@ -0,0 +1,25 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/as_complex_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/common/type_traits.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/as_complex_impl.h"
PD_REGISTER_KERNEL(
as_complex, CPU, ALL_LAYOUT, phi::AsComplexKernel, float, double) {
kernel->OutputAt(0).SetDataType(phi::dtype::ToComplex(kernel_key.dtype()));
}
paddle/phi/kernels/cpu/as_real_kernel.cc
+28
View File
@@ -0,0 +1,28 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/as_real_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/as_real_impl.h"
PD_REGISTER_KERNEL(as_real,
CPU,
ALL_LAYOUT,
phi::AsRealKernel,
phi::complex64,
phi::complex128) {
kernel->OutputAt(0).SetDataType(phi::DataType::UNDEFINED);
}
paddle/phi/kernels/cpu/asgd_kernel.cc
+73
View File
@@ -0,0 +1,73 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/asgd_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/eigen/common.h"
#include "paddle/phi/kernels/funcs/jit/kernels.h"
namespace phi {
template <typename T, typename Context>
void ASGDKernelCPUImpl(const Context& dev_ctx,
const DenseTensor& param,
const DenseTensor& grad,
const DenseTensor& learning_rate,
const DenseTensor& d,
const DenseTensor& y,
const DenseTensor& n,
DenseTensor* param_out,
DenseTensor* d_out,
DenseTensor* y_out) {
auto param_eigen = EigenVector<T>::Flatten(param);
auto grad_eigen = EigenVector<T>::Flatten(grad);
auto d_eigen = EigenVector<T>::Flatten(d);
auto y_eigen = EigenVector<T>::Flatten(y);
auto param_out_eigen = EigenVector<T>::Flatten(*param_out);
auto d_out_eigen = EigenVector<T>::Flatten(*d_out);
auto y_out_eigen = EigenVector<T>::Flatten(*y_out);
T learning_rate_T = learning_rate.data<T>()[0];
T n_T = n.data<T>()[0];
d_out_eigen = d_eigen - y_eigen + grad_eigen;
y_out_eigen = grad_eigen;
param_out_eigen = param_eigen - (learning_rate_T / n_T) * d_out_eigen;
}
template <typename T, typename Context>
void ASGDKernel(const Context& dev_ctx,
const DenseTensor& param,
const DenseTensor& grad,
const DenseTensor& learning_rate,
const DenseTensor& d,
const DenseTensor& y,
const DenseTensor& n,
const optional<DenseTensor>& master_param UNUSED,
bool multi_precision UNUSED,
DenseTensor* param_out,
DenseTensor* d_out,
DenseTensor* y_out,
DenseTensor* master_param_out UNUSED) {
dev_ctx.template Alloc<T>(param_out);
dev_ctx.template Alloc<T>(d_out);
dev_ctx.template Alloc<T>(y_out);
ASGDKernelCPUImpl<T, Context>(
dev_ctx, param, grad, learning_rate, d, y, n, param_out, d_out, y_out);
}
} // namespace phi
PD_REGISTER_KERNEL(asgd, CPU, ALL_LAYOUT, phi::ASGDKernel, float, double) {}
paddle/phi/kernels/cpu/assign_pos_kernel.cc
+34
View File
@@ -0,0 +1,34 @@
// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/assign_pos_kernel.h"
#include "paddle/common/errors.h"
#include "paddle/phi/core/kernel_registry.h"
namespace phi {
template <typename T, typename Context>
void AssignPosKernel(const Context& dev_ctx,
const DenseTensor& x,
const DenseTensor& cum_count,
const DenseTensor& eff_num_len,
DenseTensor* out) {
PADDLE_THROW(common::errors::Unavailable(
"Do not support assign pos op for cpu kernel now."));
}
} // namespace phi
PD_REGISTER_KERNEL(
assign_pos, CPU, ALL_LAYOUT, phi::AssignPosKernel, int, int64_t) {}
paddle/phi/kernels/cpu/atan2_grad_kernel.cc
+25
View File
@@ -0,0 +1,25 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/atan2_grad_kernel_impl.h"
PD_REGISTER_KERNEL(atan2_grad,
CPU,
ALL_LAYOUT,
phi::Atan2GradKernel,
float,
double,
phi::float16) {}
paddle/phi/kernels/cpu/atan2_kernel.cc
+29
View File
@@ -0,0 +1,29 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/atan2_kernel_impl.h"
PD_REGISTER_KERNEL(atan2,
CPU,
ALL_LAYOUT,
phi::Atan2Kernel,
float,
double,
phi::float16,
int,
int64_t) {
kernel->OutputAt(0).SetDataType(phi::DataType::UNDEFINED);
}
paddle/phi/kernels/cpu/attention_lstm_kernel.cc
+233
View File
@@ -0,0 +1,233 @@
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <string>
#include "paddle/phi/backends/cpu/cpu_info.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/blas/blas.h"
#include "paddle/phi/kernels/funcs/cpu_vec.h"
#include "paddle/phi/kernels/funcs/fc_functor.h"
#include "paddle/utils/optional.h"
namespace phi {
// y[i] = (x[i] + bias[0]) > 0 ? (x[i] + bias[0]) : 0;
template <typename T>
inline void bias_relu(const int n, const T* x, const T* bias, T* y) {
if (bias) {
funcs::vec_add_bias<T, backends::cpu::avx>(n, *bias, x, y);
funcs::vec_relu<T, backends::cpu::avx>(n, y, y);
} else {
funcs::vec_relu<T, backends::cpu::avx>(n, x, y);
}
}
template <typename T>
inline void vec_softmax(const int n, const T* x, T* y) {
T scalar = x[0];
// max
for (int i = 1; i < n; ++i) {
scalar = scalar < x[i] ? x[i] : scalar;
}
funcs::vec_add_bias<T, backends::cpu::avx>(n, -scalar, x, y); // sub
funcs::vec_exp<T>(n, y, y); // exp
// sum
scalar = T(0);
for (int i = 0; i < n; ++i) {
scalar += y[i];
}
funcs::vec_scal<T>(n, static_cast<T>(1) / scalar, y); // scale
}
template <typename T, typename Context>
void AttentionLSTMKernel(const Context& dev_ctx,
const DenseTensor& x_in,
const DenseTensor& c0_in,
const optional<DenseTensor>& h0_in,
const DenseTensor& attention_weight_in,
const optional<DenseTensor>& attention_bias_in,
const optional<DenseTensor>& attention_scalar_in,
const optional<DenseTensor>& attention_scalar_bias_in,
const DenseTensor& lstm_weight_in,
const DenseTensor& lstm_bias_in,
const std::string& gate_activation,
const std::string& cell_activation,
const std::string& candidate_activation,
DenseTensor* hidden,
DenseTensor* cell,
DenseTensor* attentioned_x,
DenseTensor* attention_fc_out,
DenseTensor* lstm_x,
DenseTensor* lstm_out) {
auto* x = &x_in;
auto* h0 = h0_in.get_ptr();
auto* c0 = &c0_in;
auto* atten_w = &attention_weight_in;
auto* atten_b = attention_bias_in.get_ptr();
auto* atten_scalar = attention_scalar_in.get_ptr();
auto* atten_scalar_bias = attention_scalar_bias_in.get_ptr();
auto* lstm_w = &lstm_weight_in;
auto* lstm_b = &lstm_bias_in;
auto* hidden_out = hidden;
auto* cell_out = cell;
auto* atted_x = attentioned_x;
auto* fc_out = attention_fc_out;
// some shape should be reshape here since infershape can not get lod info
auto x_lod = x->lod();
const int N = static_cast<int>(x_lod[0].size() - 1); // batch size
auto x_dims = x->dims(); // T x M
auto w_dims = lstm_w->dims(); // (D+M) x 4D
const int total_T = static_cast<int>(x_dims[0]);
const int M = static_cast<int>(x_dims[1]); // x frame size
const int D = static_cast<int>(w_dims[1] / 4); // gate frame size
const int D2 = static_cast<int>(D * 2);
const int D3 = static_cast<int>(D * 3);
const int D4 = static_cast<int>(w_dims[1]);
int max_seq_len = static_cast<int>(x_lod[0][1]);
for (int i = 1; i < N; ++i) {
int len = static_cast<int>(x_lod[0][i + 1] - x_lod[0][i]);
max_seq_len = max_seq_len < len ? len : max_seq_len;
}
PADDLE_ENFORCE_EQ(
x_lod.size(),
1UL,
common::errors::InvalidArgument("Input(X)'s lod size must be 1."));
PADDLE_ENFORCE_EQ(
c0->dims()[0],
N,
common::errors::InvalidArgument("C0 dims should be %d x %d.", N, D));
fc_out->Resize({max_seq_len, 1});
std::function<void(const int, const T*, T*)> act_gate, act_cell, act_cand;
auto& act_gate_str = gate_activation;
auto& act_cell_str = cell_activation;
auto& act_cand_str = candidate_activation;
if (backends::cpu::MayIUse(backends::cpu::avx)) {
funcs::VecActivations<T, backends::cpu::avx> act_functor;
act_gate = act_functor(act_gate_str);
act_cell = act_functor(act_cell_str);
act_cand = act_functor(act_cand_str);
} else {
funcs::VecActivations<T, backends::cpu::isa_any> act_functor;
act_gate = act_functor(act_gate_str);
act_cell = act_functor(act_cell_str);
act_cand = act_functor(act_cand_str);
}
const T* x_data = x->data<T>();
const T* h0_data = h0 ? h0->data<T>() : NULL;
const T* c0_data = c0->data<T>();
const T* lstm_w_data = lstm_w->data<T>();
const T* lstm_b_data = lstm_b->data<T>();
const T* atten_w_data = atten_w->data<T>();
const T* atten_b_data = atten_b ? atten_b->data<T>() : NULL;
const T* atten_scalar_data = atten_scalar ? atten_scalar->data<T>() : NULL;
const T* atten_scalar_bias_data =
atten_scalar_bias ? atten_scalar_bias->data<T>() : NULL;
T* hidden_out_data = dev_ctx.template Alloc<T>(hidden_out);
T* cell_out_data = dev_ctx.template Alloc<T>(cell_out);
T* atted_x_data = dev_ctx.template Alloc<T>(atted_x);
T* fc_out_data = dev_ctx.template Alloc<T>(fc_out);
T* lstm_x_data = dev_ctx.template Alloc<T>(lstm_x);
T* lstm_out_data = dev_ctx.template Alloc<T>(lstm_out);
// x(TxM) * fc (Mx1) part of atten_wgt(M+D)x1
auto blas = funcs::GetBlas<CPUContext, T>(dev_ctx);
funcs::FCFunctor<Context, T> fc;
fc(dev_ctx, total_T, 1, M, x_data, atten_w_data, atted_x_data, atten_b_data);
const T* cur_atten_x_data = atted_x_data;
const T* cur_x_data = x_data;
const T* prev_cell_data = NULL;
const T* prev_hidden_data = NULL;
T* cur_cell_out_data = cell_out_data;
T* cur_hidden_out_data = hidden_out_data;
for (int i = 0; i < N; ++i) {
int seq_len = static_cast<int>(x_lod[0][i + 1] - x_lod[0][i]);
prev_cell_data = c0_data + i * D;
prev_hidden_data = h0_data ? h0_data + i * D : NULL;
for (int step = 0; step < seq_len; ++step) {
/// 1. compute attention vector
// 1a. prev_cell(1xD) * fc(D) rest part of atten_wgt
T prev_cell_bias = blas.DOT(D, prev_cell_data, atten_w_data + M);
// 1b. add cell bias and relu
bias_relu<T>(seq_len, cur_atten_x_data, &prev_cell_bias, fc_out_data);
// 1c. fc scalar
if (atten_scalar_data) {
blas.SCAL(seq_len, *atten_scalar_data, fc_out_data);
bias_relu<T>(seq_len, fc_out_data, atten_scalar_bias_data, fc_out_data);
}
// 1d. softmax
vec_softmax<T>(seq_len, fc_out_data, fc_out_data);
// mul x(seq_len*M) and sum pool
fc(dev_ctx, 1, M, seq_len, fc_out_data, cur_x_data, lstm_x_data);
/// 2. compute LSTM step
// lstm weight : concat[forget , input , output , tilde]
// shape : (D + M) x (4 * D)
// fc inputX(1xM) * weightX(M*(4D)) => 1 x 4D
blas.MatMul(1, D4, M, lstm_x_data, lstm_w_data + D * D4, lstm_out_data);
if (prev_hidden_data) {
blas.GEMM(CblasNoTrans,
CblasNoTrans,
1,
D4,
D,
static_cast<T>(1),
prev_hidden_data,
D,
lstm_w_data,
D4,
static_cast<T>(1),
lstm_out_data,
D4);
}
// since input is 1xM, so can use add bias
blas.VADD(D4, lstm_b_data, lstm_out_data, lstm_out_data);
// gate act: sigmoid
act_gate(D3, lstm_out_data, lstm_out_data);
// candidate act: tanh
act_cand(D, lstm_out_data + D3, lstm_out_data + D3);
// a = forget * prev_cell
blas.VMUL(D, lstm_out_data, prev_cell_data, lstm_out_data);
// b = input * tilde
blas.VMUL(D, lstm_out_data + D, lstm_out_data + D3, lstm_out_data + D);
// cell_out = a + b
blas.VADD(D, lstm_out_data, lstm_out_data + D, cur_cell_out_data);
// state act tanh(cell_out) * output_gate
act_cell(D, cur_cell_out_data, lstm_out_data);
blas.VMUL(D, lstm_out_data, lstm_out_data + D2, cur_hidden_out_data);
prev_hidden_data = cur_hidden_out_data;
prev_cell_data = cur_cell_out_data;
cur_cell_out_data = cur_cell_out_data + D;
cur_hidden_out_data = cur_hidden_out_data + D;
}
cur_x_data = cur_x_data + seq_len * M;
cur_atten_x_data = cur_atten_x_data + seq_len;
}
}
} // namespace phi
PD_REGISTER_KERNEL(
attention_lstm, CPU, ALL_LAYOUT, phi::AttentionLSTMKernel, float, double) {}
paddle/phi/kernels/cpu/auc_kernel.cc
+250
View File
@@ -0,0 +1,250 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/auc_kernel.h"
#include <glog/logging.h>
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
namespace phi {
inline static double trapezoidArea(double X1, double X2, double Y1, double Y2) {
return (X1 > X2 ? (X1 - X2) : (X2 - X1)) * (Y1 + Y2) / 2.0;
}
inline static size_t compute_max_bytes(int64_t *dest,
const long *src, // NOLINT
const int num_thresholds,
const int slide_steps) {
return reinterpret_cast<const char *>(src + (num_thresholds + 1) *
(slide_steps + 1)) -
reinterpret_cast<const char *>(dest);
}
template <typename T>
void statAuc(const DenseTensor &label,
const DenseTensor &predict,
const int num_thresholds,
const int slide_steps,
int64_t *origin_stat_pos,
int64_t *origin_stat_neg,
const bool is_fake_data) {
size_t batch_size = predict.dims()[0];
size_t inference_width = predict.dims()[1];
const T *inference_data = predict.data<T>();
const auto *label_data = label.data<int64_t>();
const int bucket_length = num_thresholds + 1;
if (slide_steps == 0) {
for (size_t i = 0; i < batch_size; i++) {
// if predict_data[i] has dim of 2, then predict_data[i][1] is pos prob
// if predict_data[i] has dim of 1, then predict_data[i][0] is pos prob
auto predict_data =
inference_data[i * inference_width + (inference_width - 1)];
PADDLE_ENFORCE_LE(predict_data,
1,
common::errors::PreconditionNotMet(
"The predict data must less or equal 1."));
PADDLE_ENFORCE_GE(predict_data,
0,
common::errors::PreconditionNotMet(
"The predict data must gather or equal 0."));
uint32_t binIdx = static_cast<uint32_t>(predict_data * num_thresholds);
if (label_data[i] > 0) {
origin_stat_pos[binIdx] += 1;
} else if (label_data[i] == 0) {
origin_stat_neg[binIdx] += 1;
}
}
return;
}
// the last number of origin_stat_pos store the index should be used in
// current step
int cur_step_index =
static_cast<int>(origin_stat_pos[(slide_steps + 1) * bucket_length]) %
slide_steps;
int cur_step_begin = cur_step_index * bucket_length;
int sum_step_begin = slide_steps * bucket_length;
for (int i = 0; i < bucket_length; ++i) {
origin_stat_pos[sum_step_begin + i] -= origin_stat_pos[cur_step_begin + i];
origin_stat_neg[sum_step_begin + i] -= origin_stat_neg[cur_step_begin + i];
}
std::memset(
origin_stat_pos + cur_step_begin, 0, bucket_length * sizeof(int64_t));
std::memset(
origin_stat_neg + cur_step_begin, 0, bucket_length * sizeof(int64_t));
for (size_t i = 0; i < batch_size; i++) {
// if predict_data[i] has dim of 2, then predict_data[i][1] is pos prob
// if predict_data[i] has dim of 1, then predict_data[i][0] is pos prob
auto predict_data =
inference_data[i * inference_width + (inference_width - 1)];
PADDLE_ENFORCE_LE(predict_data,
1,
common::errors::PreconditionNotMet(
"The predict data must less or equal 1."));
PADDLE_ENFORCE_GE(predict_data,
0,
common::errors::PreconditionNotMet(
"The predict data must gather or equal 0."));
uint32_t binIdx = static_cast<uint32_t>(predict_data * num_thresholds);
if (label_data[i] > 0) {
origin_stat_pos[cur_step_begin + binIdx] += 1;
} else if (label_data[i] == 0) {
origin_stat_neg[cur_step_begin + binIdx] += 1;
}
}
if (!is_fake_data) {
for (int i = 0; i < bucket_length; ++i) {
origin_stat_pos[sum_step_begin + i] +=
origin_stat_pos[cur_step_begin + i];
origin_stat_neg[sum_step_begin + i] +=
origin_stat_neg[cur_step_begin + i];
}
}
}
inline static void calcAuc(const int64_t *stat_pos,
const int64_t *stat_neg,
int num_thresholds,
double *auc) {
*auc = 0.0f;
double totPos = 0.0;
double totNeg = 0.0;
double totPosPrev = 0.0;
double totNegPrev = 0.0;
int idx = num_thresholds;
while (idx >= 0) {
totPosPrev = totPos;
totNegPrev = totNeg;
totPos += static_cast<double>(stat_pos[idx]);
totNeg += static_cast<double>(stat_neg[idx]);
*auc += trapezoidArea(totNeg, totNegPrev, totPos, totPosPrev);
--idx;
}
if (totPos > 0.0 && totNeg > 0.0) {
*auc = *auc / totPos / totNeg;
}
}
template <typename T, typename Context>
void AucKernel(const Context &dev_ctx,
const DenseTensor &input,
const DenseTensor &label,
const DenseTensor &stat_pos,
const DenseTensor &stat_neg,
const optional<DenseTensor> &ins_tag_weight,
const std::string &curve,
int num_thresholds,
int slide_steps,
DenseTensor *auc,
DenseTensor *stat_pos_out,
DenseTensor *stat_neg_out) {
// Only use output var for now, make sure it's persistable and
// not cleaned up for each batch.
auto *origin_stat_pos = dev_ctx.template Alloc<int64_t>(stat_pos_out);
auto *origin_stat_neg = dev_ctx.template Alloc<int64_t>(stat_neg_out);
auto *auc_value = dev_ctx.template Alloc<double>(auc);
// Just for pass UT, since UT's input & output cannot be set same var
auto *stat_pos_in_tensor = &stat_pos;
auto *stat_neg_in_tensor = &stat_neg;
auto *pos_in_data = stat_pos.data<int64_t>();
auto *neg_in_data = stat_neg.data<int64_t>();
bool is_fake_data = false;
if (ins_tag_weight.get_ptr() != nullptr) {
const auto *ins_tag_weight_data = ins_tag_weight->data<float>();
VLOG(4) << "auc ins_tag_weight = " << ins_tag_weight_data[0];
if (ins_tag_weight_data[0] == 0) {
is_fake_data = true;
}
}
size_t required_bytes =
((1 + slide_steps) * (num_thresholds + 1) + (slide_steps > 0 ? 1 : 0)) *
sizeof(int64_t);
if (stat_pos_in_tensor != stat_pos_out) {
size_t max_bytes = compute_max_bytes(
origin_stat_pos,
reinterpret_cast<const long *>(pos_in_data), // NOLINT
num_thresholds,
slide_steps);
PADDLE_ENFORCE_LE(required_bytes,
max_bytes,
common::errors::PreconditionNotMet(
"The number of bytes to be copied %d must be less "
"than or equal to the maximum number of bytes %d. ",
required_bytes,
max_bytes));
memcpy(
origin_stat_pos,
pos_in_data,
((1 + slide_steps) * (num_thresholds + 1) + (slide_steps > 0 ? 1 : 0)) *
sizeof(int64_t));
}
if (stat_neg_in_tensor != stat_neg_out) {
size_t max_bytes = compute_max_bytes(
origin_stat_neg,
reinterpret_cast<const long *>(neg_in_data), // NOLINT
num_thresholds,
slide_steps);
PADDLE_ENFORCE_LE(required_bytes,
max_bytes,
common::errors::PreconditionNotMet(
"The number of bytes to be copied %d must be less "
"than or equal to the maximum number of bytes %d. ",
required_bytes,
max_bytes));
memcpy(
origin_stat_neg,
neg_in_data,
((1 + slide_steps) * (num_thresholds + 1) + (slide_steps > 0 ? 1 : 0)) *
sizeof(int64_t));
}
// when calculate global_auc && is fake data, just do nothing
if (slide_steps == 0 && is_fake_data) {
return;
}
statAuc<T>(label,
input,
num_thresholds,
slide_steps,
origin_stat_pos,
origin_stat_neg,
is_fake_data);
int sum_offset = slide_steps * (num_thresholds + 1);
calcAuc(origin_stat_pos + sum_offset,
origin_stat_neg + sum_offset,
num_thresholds,
auc_value);
if (slide_steps) {
origin_stat_pos[(slide_steps + 1) * (num_thresholds + 1)] += 1;
origin_stat_neg[(slide_steps + 1) * (num_thresholds + 1)] += 1;
}
}
} // namespace phi
PD_REGISTER_KERNEL(auc, CPU, ALL_LAYOUT, phi::AucKernel, float) {
kernel->OutputAt(0).SetDataType(phi::DataType::FLOAT64);
kernel->OutputAt(1).SetDataType(phi::DataType::INT64);
kernel->OutputAt(2).SetDataType(phi::DataType::INT64);
}
paddle/phi/kernels/cpu/average_accumulates_kernel.cc
+60
View File
@@ -0,0 +1,60 @@
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/phi/kernels/average_accumulates_kernel.h"
#include "paddle/phi/kernels/impl/average_accumulates_kernel_impl.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
namespace phi {
template <>
void GetAccumulators<CPUContext>(const CPUContext& dev_ctx,
const DenseTensor& in_num_accumulates,
const DenseTensor& in_old_num_accumulates,
const DenseTensor& in_num_updates,
int64_t* num_updates,
int64_t* num_accumulates,
int64_t* old_num_accumulates) {
*old_num_accumulates = in_old_num_accumulates.data<int64_t>()[0];
*num_accumulates = in_num_accumulates.data<int64_t>()[0];
*num_updates = in_num_updates.data<int64_t>()[0];
}
template <>
void SetAccumulators<CPUContext>(const CPUContext& dev_ctx,
int64_t num_updates,
int64_t num_accumulates,
int64_t old_num_accumulates,
DenseTensor* out_num_accumulates,
DenseTensor* out_old_num_accumulates,
DenseTensor* out_num_updates) {
out_old_num_accumulates->data<int64_t>()[0] = old_num_accumulates;
out_num_accumulates->data<int64_t>()[0] = num_accumulates;
out_num_updates->data<int64_t>()[0] = num_updates;
}
} // namespace phi
PD_REGISTER_KERNEL(average_accumulates,
CPU,
ALL_LAYOUT,
phi::AverageAccumulatesKernel,
float,
double) {
kernel->OutputAt(3).SetDataType(phi::DataType::INT64);
kernel->OutputAt(4).SetDataType(phi::DataType::INT64);
kernel->OutputAt(5).SetDataType(phi::DataType::INT64);
}
paddle/phi/kernels/cpu/baddbmm_grad_kernel.cc
+22
View File
@@ -0,0 +1,22 @@
/* Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/phi/kernels/baddbmm_grad_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/baddbmm_grad_kernel_impl.h"
PD_REGISTER_KERNEL(
baddbmm_grad, CPU, ALL_LAYOUT, phi::BaddbmmGradKernel, float, double) {}
paddle/phi/kernels/cpu/baddbmm_kernel.cc
+22
View File
@@ -0,0 +1,22 @@
/* Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/phi/kernels/baddbmm_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/baddbmm_kernel_impl.h"
PD_REGISTER_KERNEL(
baddbmm, CPU, ALL_LAYOUT, phi::BaddbmmKernel, float, double) {}
paddle/phi/kernels/cpu/barrier_kernel.cc
+45
View File
@@ -0,0 +1,45 @@
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/barrier_kernel.h"
#include "paddle/phi/core/dense_tensor.h"
#include "paddle/phi/core/distributed/comm_context_manager.h"
#include "paddle/phi/core/kernel_registry.h"
#if defined(PADDLE_WITH_GLOO)
#include "paddle/phi/core/distributed/gloo_comm_context.h"
#endif
namespace phi {
template <typename T, typename Context>
void BarrierKernel(const Context &dev_ctx,
const DenseTensor &x,
DenseTensor *out) {
#if defined(PADDLE_WITH_GLOO)
auto comm_ctx =
static_cast<distributed::GlooCommContext *>(dev_ctx.GetCommContext());
PADDLE_ENFORCE_NE(comm_ctx,
nullptr,
::common::errors::Unavailable(
"NCCLCommContext is nullptr, collective op should "
"has ring_id attr."));
comm_ctx->Barrier();
#else
PADDLE_THROW(common::errors::Unavailable(
"PaddlePaddle should compile with GLOO by setting WITH_GLOO=ON"));
#endif
}
} // namespace phi
PD_REGISTER_KERNEL(barrier, CPU, ALL_LAYOUT, phi::BarrierKernel, int) {}
paddle/phi/kernels/cpu/batch_fc_kernel.cc
+35
View File
@@ -0,0 +1,35 @@
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/core/kernel_registry.h"
namespace phi {
template <typename T, typename Context>
void BatchFCKernel(const Context &dev_ctx,
const DenseTensor &input,
const DenseTensor &w,
const DenseTensor &bias,
DenseTensor *out) {
PADDLE_ENFORCE_EQ(
(dev_ctx.GetPlace().GetType() == AllocationType::GPU) ||
(dev_ctx.GetPlace().GetType() == AllocationType::CUSTOM),
true,
common::errors::Unimplemented("BatchFC only supports GPU now."));
}
} // namespace phi
PD_REGISTER_KERNEL(
batch_fc, CPU, ALL_LAYOUT, phi::BatchFCKernel, float, double) {}
paddle/phi/kernels/cpu/batch_norm_grad_kernel.cc
+693
View File
@@ -0,0 +1,693 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "glog/logging.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/batch_norm_kernel.h"
#include "paddle/phi/kernels/full_kernel.h"
#include "paddle/phi/kernels/funcs/batch_norm_utils.h"
#include "paddle/phi/kernels/funcs/eigen/common.h"
#include "paddle/phi/kernels/funcs/math_function.h"
namespace phi {
template <typename T>
using EigenArrayMap =
Eigen::Map<Eigen::Array<T, Eigen::Dynamic, Eigen::Dynamic>>;
template <typename T>
using ConstEigenArrayMap =
Eigen::Map<const Eigen::Array<T, Eigen::Dynamic, Eigen::Dynamic>>;
template <typename T>
using EigenVectorArrayMap = Eigen::Map<Eigen::Array<T, Eigen::Dynamic, 1>>;
template <typename T>
using ConstEigenVectorArrayMap =
Eigen::Map<const Eigen::Array<T, Eigen::Dynamic, 1>>;
template <typename T, typename Context>
void BatchNormGradFunctor(const Context& dev_ctx,
const DenseTensor& x,
const optional<DenseTensor>& scale,
const optional<DenseTensor>& bias,
const optional<DenseTensor>& mean,
const optional<DenseTensor>& variance,
const DenseTensor& saved_mean,
const DenseTensor& saved_variance,
const optional<DenseTensor>& reserve_space,
const DenseTensor& y_grad,
float momentum,
float epsilon,
const std::string& data_layout_str,
bool is_test,
bool use_global_stats,
bool trainable_statistics,
bool is_inplace,
DenseTensor* x_grad,
DenseTensor* scale_grad,
DenseTensor* bias_grad) {
const auto* d_y = &y_grad;
DataLayout data_layout = StringToDataLayout(data_layout_str);
auto* d_x = x_grad;
auto* d_scale = scale_grad;
auto* d_bias = bias_grad;
use_global_stats = is_test || use_global_stats;
// batch_norm with inplace as false will take X as grad input, which
// is same as cuDNN batch_norm backward calculation, batch_norm
// with inplace as true only take Y as input and X should be calculate
// by inverse operation of batch_norm on Y
if (is_inplace) {
if (d_x) {
PADDLE_ENFORCE_EQ(d_x,
d_y,
common::errors::InvalidArgument(
"X@GRAD and Y@GRAD inplaced in non-inplace mode"));
}
} else {
if (d_x) {
PADDLE_ENFORCE_NE(d_x,
d_y,
common::errors::InvalidArgument(
"X@GRAD and Y@GRAD inplaced in non-inplace mode"));
}
}
// Get the size for each dimension.
// NCHW [batch_size, in_channels, in_height, in_width]
const auto& x_dims = x.dims();
PADDLE_ENFORCE_GE(
x_dims.size(),
2,
common::errors::InvalidArgument(
"The size of input X's dimensions should be larger than 1."
"But received: the size of input X's dimensions is [%d]",
x_dims.size()));
PADDLE_ENFORCE_LE(
x_dims.size(),
5,
common::errors::InvalidArgument(
"The size of input X's dimensions should be less than 6."
"But received: the size of input X's dimensions is [%d]",
x_dims.size()));
const int64_t N = x_dims[0];
const int64_t C =
data_layout == DataLayout::NCHW ? x_dims[1] : x_dims[x_dims.size() - 1];
const int64_t sample_size = x.numel() / N / C;
const int64_t num_batch_channels = N * C;
const int64_t num_batch_spatial = N * sample_size;
// input dimension is 2 and the format is NCHW. The input can be regarded as
// NHWC format
if (x_dims.size() == 2 && data_layout == DataLayout::NCHW) {
data_layout = DataLayout::NHWC;
}
// init output
if (d_x) {
dev_ctx.template Alloc<T>(d_x);
}
const T* mean_data = nullptr;
const T* inv_var_data = nullptr;
DenseTensor inv_var_tensor;
if (use_global_stats) {
const auto* running_mean = mean.get_ptr();
const auto* running_variance = variance.get_ptr();
mean_data = running_mean->data<T>();
inv_var_tensor.Resize({C});
T* running_inv_var_data = dev_ctx.template Alloc<T>(&inv_var_tensor);
EigenVectorArrayMap<T> inv_var_tmp(running_inv_var_data, C);
ConstEigenVectorArrayMap<T> var_arr(running_variance->data<T>(), C);
inv_var_tmp = (var_arr + epsilon).sqrt().inverse();
inv_var_data = running_inv_var_data;
} else {
mean_data = saved_mean.data<T>();
inv_var_data = saved_variance.data<T>();
}
ConstEigenVectorArrayMap<T> mean_arr(mean_data, C);
ConstEigenVectorArrayMap<T> inv_var_arr(inv_var_data, C);
T* d_bias_data = nullptr;
T* d_scale_data = nullptr;
if (d_scale && d_bias) {
d_bias_data = dev_ctx.template Alloc<T>(d_bias);
d_scale_data = dev_ctx.template Alloc<T>(d_scale);
}
// d_bias = np.sum(d_y, axis=0)
// d_scale = np.sum((X - mean) / inv_std * dy, axis=0)
// d_x = (1. / N) * scale * inv_var * (N * d_y - np.sum(d_y, axis=0)
// - (X - mean) * inv_var * inv_var * np.sum(d_y * (X - mean), axis=0))
EigenVectorArrayMap<T> d_bias_arr(d_bias_data, C);
EigenVectorArrayMap<T> d_scale_arr(d_scale_data, C);
if (d_scale && d_bias) {
d_bias_arr.setZero();
d_scale_arr.setZero();
}
if (d_x && num_batch_spatial == 1 && !use_global_stats) {
Copy(dev_ctx, *d_y, dev_ctx.GetPlace(), false, d_x);
return;
}
auto* Scale = scale.get_ptr();
auto* Bias = bias.get_ptr();
Eigen::Array<T, Eigen::Dynamic, 1> scale_arr(C);
Eigen::Array<T, Eigen::Dynamic, 1> bias_arr(C);
if (Scale) {
scale_arr = ConstEigenVectorArrayMap<T>(Scale->data<T>(), C);
} else {
scale_arr.setOnes();
}
if (Bias) {
bias_arr = ConstEigenVectorArrayMap<T>(Bias->data<T>(), C);
} else {
bias_arr.setZero();
}
int64_t scale_coeff = use_global_stats ? 1 : num_batch_spatial;
const auto scale_inv_var_nhw = scale_arr * inv_var_arr / scale_coeff;
DenseTensor dy_sum;
dy_sum.Resize({C});
auto dy_sum_data = dev_ctx.template Alloc<T>(&dy_sum);
EigenVectorArrayMap<T> dy_sum_arr(dy_sum_data, C);
DenseTensor dy_mul_x_sub_mean_mul_invstd_sum;
dy_mul_x_sub_mean_mul_invstd_sum.Resize({C});
auto dy_mul_x_sub_mean_mul_invstd_sum_data =
dev_ctx.template Alloc<T>(&dy_mul_x_sub_mean_mul_invstd_sum);
EigenVectorArrayMap<T> dy_mul_x_sub_mean_mul_invstd_sum_arr(
dy_mul_x_sub_mean_mul_invstd_sum_data, C);
dy_sum_arr.setZero();
dy_mul_x_sub_mean_mul_invstd_sum_arr.setZero();
// inplace calculation
// Y: ((x - est_mean) * (inv_var) * scale + bias
// formula transform ====>
// (x * inv_var * scale) + (bias - est_mean * inv_var * scale)
// X: (y - bias) / scale / (inv_var) + est_mean
// formula transform ====>
// (y - bias) / (scale * inv_var) + est_mean
switch (data_layout) {
case DataLayout::NCHW: {
if (is_inplace) {
auto px = x;
EigenArrayMap<T> x_data(
dev_ctx.template Alloc<T>(&px), sample_size, num_batch_channels);
ConstEigenArrayMap<T> y_data(
x.data<T>(), sample_size, num_batch_channels);
for (int64_t nc = 0; nc < num_batch_channels; ++nc) {
x_data.col(nc) = (y_data.col(nc) - bias_arr(nc % C)) /
scale_inv_var_nhw(nc % C) / scale_coeff +
mean_arr(nc % C);
}
}
ConstEigenArrayMap<T> x_arr(x.data<T>(), sample_size, num_batch_channels);
ConstEigenArrayMap<T> d_y_arr(
d_y->data<T>(), sample_size, num_batch_channels);
for (int64_t nc = 0; nc < num_batch_channels; ++nc) {
int c = nc % C;
dy_sum_arr(c) += d_y_arr.col(nc).sum();
dy_mul_x_sub_mean_mul_invstd_sum_arr(c) +=
((x_arr.col(nc) - mean_arr(c)) * inv_var_arr(c) * d_y_arr.col(nc))
.sum();
}
if (d_scale && d_bias) {
d_bias_arr = dy_sum_arr;
d_scale_arr = dy_mul_x_sub_mean_mul_invstd_sum_arr;
}
if (d_x) {
EigenArrayMap<T> d_x_arr(
dev_ctx.template Alloc<T>(d_x), sample_size, num_batch_channels);
if (!use_global_stats) {
for (int64_t nc = 0; nc < num_batch_channels; ++nc) {
int c = nc % C;
d_x_arr.col(nc) =
scale_inv_var_nhw(c) *
(d_y_arr.col(nc) * num_batch_spatial - dy_sum_arr(c) -
(x_arr.col(nc) - mean_arr[c]) *
dy_mul_x_sub_mean_mul_invstd_sum_arr(c) * inv_var_arr(c));
}
} else {
for (int64_t nc = 0; nc < num_batch_channels; ++nc) {
int c = nc % C;
d_x_arr.col(nc) = scale_inv_var_nhw(c) * d_y_arr.col(nc);
}
}
}
break;
}
case DataLayout::NHWC: {
if (is_inplace) {
auto px = x;
EigenArrayMap<T> x_data(
dev_ctx.template Alloc<T>(&px), C, num_batch_spatial);
ConstEigenArrayMap<T> y_data(x.data<T>(), C, num_batch_spatial);
for (int64_t nhw = 0; nhw < num_batch_spatial; nhw++) {
x_data.col(nhw) =
(y_data.col(nhw) - bias_arr) / scale_inv_var_nhw / scale_coeff +
mean_arr;
}
}
ConstEigenArrayMap<T> x_arr(x.data<T>(), C, num_batch_spatial);
ConstEigenArrayMap<T> d_y_arr(d_y->data<T>(), C, num_batch_spatial);
for (int64_t nhw = 0; nhw < num_batch_spatial; ++nhw) {
dy_sum_arr += d_y_arr.col(nhw);
dy_mul_x_sub_mean_mul_invstd_sum_arr +=
(x_arr.col(nhw) - mean_arr) * inv_var_arr * d_y_arr.col(nhw);
}
if (d_scale && d_bias) {
d_bias_arr = dy_sum_arr;
d_scale_arr = dy_mul_x_sub_mean_mul_invstd_sum_arr;
}
if (d_x) {
EigenArrayMap<T> d_x_arr(
dev_ctx.template Alloc<T>(d_x), C, num_batch_spatial);
if (!use_global_stats) {
for (int64_t nhw = 0; nhw < num_batch_spatial; ++nhw) {
d_x_arr.col(nhw) =
scale_inv_var_nhw *
(d_y_arr.col(nhw) * num_batch_spatial - dy_sum_arr -
(x_arr.col(nhw) - mean_arr) *
dy_mul_x_sub_mean_mul_invstd_sum_arr * inv_var_arr);
}
} else {
for (int64_t nhw = 0; nhw < num_batch_spatial; ++nhw) {
d_x_arr.col(nhw) = scale_inv_var_nhw * d_y_arr.col(nhw);
}
}
}
break;
}
default:
PADDLE_THROW(common::errors::InvalidArgument("Unknown storage order: %s",
data_layout_str));
}
}
template <typename T, typename Context>
void BatchNormGradKernel(const Context& dev_ctx,
const DenseTensor& x,
const optional<DenseTensor>& scale,
const optional<DenseTensor>& bias,
const optional<DenseTensor>& mean,
const optional<DenseTensor>& variance,
const DenseTensor& saved_mean,
const DenseTensor& saved_variance,
const optional<DenseTensor>& reserve_space,
const DenseTensor& y_grad,
float momentum,
float epsilon,
const std::string& data_layout,
bool is_test,
bool use_global_stats,
bool trainable_statistics,
DenseTensor* x_grad,
DenseTensor* scale_grad,
DenseTensor* bias_grad) {
if (x.numel() == 0) {
dev_ctx.template Alloc<T>(x_grad);
if (scale_grad)
Full<T, Context>(dev_ctx, scale_grad->dims(), 0, scale_grad);
if (bias_grad) Full<T, Context>(dev_ctx, bias_grad->dims(), 0, bias_grad);
return;
}
BatchNormGradFunctor<T, Context>(dev_ctx,
x,
scale,
bias,
mean,
variance,
saved_mean,
saved_variance,
reserve_space,
y_grad,
momentum,
epsilon,
data_layout,
is_test,
use_global_stats,
trainable_statistics,
false,
x_grad,
scale_grad,
bias_grad);
}
template <typename T, typename Context>
void BatchNormDoubleGradKernel(const Context& dev_ctx,
const DenseTensor& x,
const optional<DenseTensor>& scale,
const optional<DenseTensor>& mean,
const optional<DenseTensor>& variance,
const DenseTensor& saved_mean,
const DenseTensor& saved_variance,
const DenseTensor& y_grad,
const optional<DenseTensor>& x_grad_grad,
const optional<DenseTensor>& scale_grad_grad,
const optional<DenseTensor>& bias_grad_grad,
float momentum,
float epsilon,
const std::string& data_layout_str,
bool is_test,
bool use_global_stats,
bool trainable_statistics,
DenseTensor* x_grad,
DenseTensor* scale_grad,
DenseTensor* y_grad_grad) {
const auto* X = &x;
const auto* Scale = scale.get_ptr();
const auto* dY = &y_grad;
const auto* Saved_mean = &saved_mean;
const auto* Saved_variance = &saved_variance;
PADDLE_ENFORCE_EQ(is_test,
false,
common::errors::InvalidArgument(
"`is_test = True` CANNOT be used in train program. If "
"you want to use global status in pre_train model, "
"please set `use_global_stats = True`"));
const auto data_layout = StringToDataLayout(data_layout_str);
const auto* ddX = x_grad_grad.get_ptr();
const auto* ddScale = scale_grad_grad.get_ptr();
const auto* ddBias = bias_grad_grad.get_ptr();
auto* dX = x_grad;
auto* dScale = scale_grad;
auto* ddY = y_grad_grad;
dev_ctx.template Alloc<T>(dX);
dev_ctx.template Alloc<T>(ddY);
const auto& x_dims = X->dims();
const int64_t C =
data_layout == DataLayout::NCHW ? x_dims[1] : x_dims[x_dims.size() - 1];
const int64_t sample_size = X->numel() / C;
funcs::SetConstant<Context, T> set_constant;
const T* mean_data = Saved_mean->data<T>();
const T* inv_var_data = Saved_variance->data<T>();
DenseTensor inv_var_tensor;
if (use_global_stats) {
const auto* running_mean = mean.get_ptr();
const auto* running_variance = variance.get_ptr();
mean_data = running_mean->data<T>();
inv_var_tensor.Resize({C});
T* running_inv_var_data = dev_ctx.template Alloc<T>(&inv_var_tensor);
EigenVectorArrayMap<T> inv_var_tmp(running_inv_var_data, C);
ConstEigenVectorArrayMap<T> var_arr(running_variance->data<T>(), C);
inv_var_tmp = (var_arr + epsilon).sqrt().inverse();
inv_var_data = running_inv_var_data;
}
// transpose NCHW -> NHWC for easy calculate
DenseTensor transformed_x(X->type());
DenseTensor transformed_dy(dY->type());
DenseTensor transformed_ddx(ddX->type());
DenseTensor transformed_dx(dX->type());
DenseTensor transformed_ddy(ddY->type());
if (data_layout == DataLayout::NCHW && x_dims.size() > 2) {
VLOG(3) << "Transform batchnorm output from NCHW to NHWC";
// Input DenseTensor
ResizeToChannelLast<Context, T>(dev_ctx, X, &transformed_x);
TransToChannelLast<Context, T>(dev_ctx, X, &transformed_x);
ResizeToChannelLast<Context, T>(dev_ctx, dY, &transformed_dy);
TransToChannelLast<Context, T>(dev_ctx, dY, &transformed_dy);
ResizeToChannelLast<Context, T>(dev_ctx, ddX, &transformed_ddx);
TransToChannelLast<Context, T>(dev_ctx, ddX, &transformed_ddx);
// Output DenseTensor
ResizeToChannelLast<Context, T>(dev_ctx, dX, &transformed_dx);
ResizeToChannelLast<Context, T>(dev_ctx, ddY, &transformed_ddy);
} else {
transformed_x.ShareDataWith(*X);
transformed_dy.ShareDataWith(*dY);
transformed_ddx.ShareDataWith(*ddX);
transformed_dx.ShareDataWith(*dX);
transformed_ddy.ShareDataWith(*ddY);
}
ConstEigenArrayMap<T> x_arr(transformed_x.data<T>(), C, sample_size);
ConstEigenVectorArrayMap<T> mean_arr(mean_data, C);
ConstEigenVectorArrayMap<T> inv_var_arr(inv_var_data, C);
DenseTensor mean_tile;
mean_tile.Resize({C, sample_size});
EigenArrayMap<T> mean_tile_data(
dev_ctx.template Alloc<T>(&mean_tile), C, sample_size);
DenseTensor inv_var_tile;
inv_var_tile.Resize({C, sample_size});
EigenArrayMap<T> inv_var_tile_data(
dev_ctx.template Alloc<T>(&inv_var_tile), C, sample_size);
mean_tile_data = mean_arr.replicate(1, sample_size);
inv_var_tile_data = inv_var_arr.replicate(1, sample_size);
DenseTensor Scale_data;
if (!Scale) {
Scale_data.Resize({C});
dev_ctx.template Alloc<T>(&Scale_data);
set_constant(dev_ctx, &Scale_data, static_cast<T>(1));
}
ConstEigenVectorArrayMap<T> scale_arr(
Scale ? Scale->data<T>() : Scale_data.data<T>(), C);
DenseTensor scale_tile;
scale_tile.Resize({C, sample_size});
EigenArrayMap<T> scale_tile_data(
dev_ctx.template Alloc<T>(&scale_tile), C, sample_size);
scale_tile_data = scale_arr.replicate(1, sample_size);
ConstEigenArrayMap<T> dy_arr(transformed_dy.data<T>(), C, sample_size);
ConstEigenArrayMap<T> ddx_arr(transformed_ddx.data<T>(), C, sample_size);
DenseTensor x_sub_mean_mul_invstd;
x_sub_mean_mul_invstd.Resize({C, sample_size});
EigenArrayMap<T> x_sub_mean_mul_invstd_arr(
dev_ctx.template Alloc<T>(&x_sub_mean_mul_invstd), C, sample_size);
x_sub_mean_mul_invstd_arr = (x_arr - mean_tile_data) * inv_var_tile_data;
if (dX) {
dev_ctx.template Alloc<T>(dX);
EigenArrayMap<T> dx_arr(
dev_ctx.template Alloc<T>(&transformed_dx), C, sample_size);
dx_arr.setZero();
if (use_global_stats) {
// math: dx = (ddscale * dy) * inv_var
if (ddScale) {
ConstEigenVectorArrayMap<T> ddscale_arr(ddScale->data<T>(), C);
DenseTensor ddscale_tile;
ddscale_tile.Resize({C, sample_size});
EigenArrayMap<T> ddscale_tile_data(
dev_ctx.template Alloc<T>(&ddscale_tile), C, sample_size);
ddscale_tile_data = ddscale_arr.replicate(1, sample_size);
dx_arr = dy_arr * ddscale_tile_data * inv_var_tile_data;
}
} else {
// math: dx = scale * ((x - mean) * inv_var / NxHxW * (np.mean(ddx,
// axis=(n,h,w)) *
// np.sum(dy, axis=(n,h,w)) -
// np.sum(dy * ddx, axis=(n,h,w)) + 3 * np.mean(dy * (x -
// mean),
// axis=(n,h,w)) * inv_var.pow(2) *
// np.sum(ddx * (x - mean), axis=(n,h,w))) + inv_var.pow(3) /
// NxHxW *
// np.sum(ddx * (x - mean)) *
// (np.mean(dy, axis=(n,h,w)) - dy) + inv_var.pow(3) / NxHxW *
// np.sum(dy,
// axis=(n,h,w)) * (x - mean) *
// (np.mean(ddx, axis=(n,h,w)) - ddx)) + ddr * (dy * inv_var -
// inv_var
// *
// np.mean(dy, axis=(n,h,w)) -
// inv_var.pow(3) * (x - mean) * np.mean(dy * (x - mean),
// axis=(n,h,w)))
if (ddX) {
dx_arr +=
(x_sub_mean_mul_invstd_arr * inv_var_tile_data * inv_var_tile_data /
sample_size)
.colwise() *
(ddx_arr.rowwise().sum() * dy_arr.rowwise().sum() / sample_size -
(dy_arr * ddx_arr).rowwise().sum() +
3. * (dy_arr * x_sub_mean_mul_invstd_arr).rowwise().sum() *
(ddx_arr * x_sub_mean_mul_invstd_arr).rowwise().sum() /
sample_size);
dx_arr += (inv_var_tile_data * inv_var_tile_data).colwise() *
(ddx_arr * x_sub_mean_mul_invstd_arr).rowwise().sum() /
sample_size * (dy_arr.rowwise().sum() / sample_size - dy_arr);
dx_arr += (inv_var_tile_data * inv_var_tile_data).colwise() *
(dy_arr * x_sub_mean_mul_invstd_arr).rowwise().sum() /
sample_size *
(ddx_arr.rowwise().sum() / sample_size - ddx_arr);
dx_arr = scale_tile_data * dx_arr;
}
if (ddScale) {
ConstEigenVectorArrayMap<T> ddscale_arr(ddScale->data<T>(), C);
DenseTensor ddscale_tile;
ddscale_tile.Resize({C, sample_size});
EigenArrayMap<T> ddscale_tile_data(
dev_ctx.template Alloc<T>(&ddscale_tile), C, sample_size);
ddscale_tile_data = ddscale_arr.replicate(1, sample_size);
dx_arr +=
(dy_arr * inv_var_tile_data -
(dy_arr.rowwise().sum().replicate(1, sample_size) / sample_size) *
inv_var_tile_data -
x_sub_mean_mul_invstd_arr * inv_var_tile_data *
(dy_arr * x_sub_mean_mul_invstd_arr)
.rowwise()
.sum()
.replicate(1, sample_size) /
sample_size) *
ddscale_tile_data;
}
}
if (data_layout == DataLayout::NCHW) {
VLOG(3) << "Transform batchnorm output from NHWC to NCHW";
TransToChannelFirst<Context, T>(dev_ctx, &transformed_dx, dX);
}
}
if (dScale) {
EigenVectorArrayMap<T> dscale_arr(dev_ctx.template Alloc<T>(dScale), C);
dscale_arr.setZero();
if (use_global_stats) {
// math: dscale = np.sum(ddx * dy, axis=(n,h,w)) * inv_var
if (ddX) {
dscale_arr = (ddx_arr * dy_arr * inv_var_tile_data).rowwise().sum();
}
} else {
// math: dscale = inv_var * (dy - np.mean(dy, axis=(n,h,w) - (x-mean) *
// inv_var.pow(2) * np.mean(dy * (x-mean), axis=(n,h,w)))) *
// ddx
if (ddX) {
DenseTensor first_grad;
first_grad.Resize({C, sample_size});
EigenArrayMap<T> first_grad_arr(
dev_ctx.template Alloc<T>(&first_grad), C, sample_size);
first_grad_arr.setZero();
first_grad_arr +=
inv_var_tile_data *
(dy_arr -
dy_arr.rowwise().sum().replicate(1, sample_size) / sample_size -
x_sub_mean_mul_invstd_arr *
(dy_arr * x_sub_mean_mul_invstd_arr)
.rowwise()
.sum()
.replicate(1, sample_size) /
sample_size);
dscale_arr = (first_grad_arr * ddx_arr).rowwise().sum();
}
}
}
if (ddY) {
dev_ctx.template Alloc<T>(ddY);
EigenArrayMap<T> ddy_arr(
dev_ctx.template Alloc<T>(&transformed_ddy), C, sample_size);
ddy_arr.setZero();
if (use_global_stats) { // NOLINT
// math: ddy = r * ddx * inv_var + ddbias +
// ddscale * (x - mean) * inv_var
if (ddX) {
ddy_arr = scale_tile_data * ddx_arr * inv_var_tile_data;
}
} else {
// math: ddy = (x - mean) * inv_var * ddscale + ddbias +
// scale * inv_var * (ddx - (x - mean) * inv_var.pow(2) *
// np.mean(ddx * (x - mean), axis=(n,h,w)))
if (ddX) {
ddy_arr +=
scale_tile_data * inv_var_tile_data *
(ddx_arr -
ddx_arr.rowwise().sum().replicate(1, sample_size) / sample_size -
x_sub_mean_mul_invstd_arr *
(ddx_arr * x_sub_mean_mul_invstd_arr)
.rowwise()
.sum()
.replicate(1, sample_size) /
sample_size);
}
}
if (ddScale) {
ConstEigenVectorArrayMap<T> ddscale_arr(ddScale->data<T>(), C);
DenseTensor ddscale_tile;
ddscale_tile.Resize({C, sample_size});
EigenArrayMap<T> ddscale_tile_data(
dev_ctx.template Alloc<T>(&ddscale_tile), C, sample_size);
ddscale_tile_data = ddscale_arr.replicate(1, sample_size);
ddy_arr += x_sub_mean_mul_invstd_arr * ddscale_tile_data;
}
if (ddBias) {
ConstEigenVectorArrayMap<T> ddbias_arr(ddBias->data<T>(), C);
DenseTensor ddbias_tile;
ddbias_tile.Resize({C, sample_size});
EigenArrayMap<T> ddbias_tile_data(
dev_ctx.template Alloc<T>(&ddbias_tile), C, sample_size);
ddbias_tile_data = ddbias_arr.replicate(1, sample_size);
ddy_arr += ddbias_tile_data;
}
if (data_layout == DataLayout::NCHW) {
VLOG(3) << "Transform batchnorm output from NHWC to NCHW";
TransToChannelFirst<Context, T>(dev_ctx, &transformed_ddy, ddY);
}
}
}
} // namespace phi
PD_DECLARE_BN_GRAD_FUNCTOR(float, CPU);
PD_DECLARE_BN_GRAD_FUNCTOR(double, CPU);
PD_REGISTER_KERNEL(
batch_norm_grad, CPU, ALL_LAYOUT, phi::BatchNormGradKernel, float, double) {
}
PD_REGISTER_KERNEL(batch_norm_double_grad,
CPU,
ALL_LAYOUT,
phi::BatchNormDoubleGradKernel,
float,
double) {}
paddle/phi/kernels/cpu/batch_norm_kernel.cc
+287
View File
@@ -0,0 +1,287 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/batch_norm_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/eigen/common.h"
namespace phi {
template <typename T>
using EigenArrayMap =
Eigen::Map<Eigen::Array<T, Eigen::Dynamic, Eigen::Dynamic>>;
template <typename T>
using ConstEigenArrayMap =
Eigen::Map<const Eigen::Array<T, Eigen::Dynamic, Eigen::Dynamic>>;
template <typename T>
using EigenVectorArrayMap = Eigen::Map<Eigen::Array<T, Eigen::Dynamic, 1>>;
template <typename T>
using ConstEigenVectorArrayMap =
Eigen::Map<const Eigen::Array<T, Eigen::Dynamic, 1>>;
template <typename T, typename Context>
void BatchNormKernel(const Context& dev_ctx,
const DenseTensor& x,
const DenseTensor& mean,
const DenseTensor& variance,
const optional<DenseTensor>& scale,
const optional<DenseTensor>& bias,
bool is_test,
float momentum,
float epsilon,
const std::string& data_layout_str,
bool use_global_stats,
bool trainable_statistics,
DenseTensor* y,
DenseTensor* mean_out,
DenseTensor* variance_out,
DenseTensor* saved_mean,
DenseTensor* saved_variance,
DenseTensor* reserve_space) {
if (x.numel() == 0) {
dev_ctx.template Alloc<T>(y);
if (mean_out) dev_ctx.template Alloc<T>(mean_out);
if (variance_out) dev_ctx.template Alloc<T>(variance_out);
if (saved_mean) dev_ctx.template Alloc<T>(saved_mean);
if (saved_variance) dev_ctx.template Alloc<T>(saved_variance);
if (reserve_space) {
// infermeta dim is -1.
reserve_space->Resize({0});
dev_ctx.template Alloc<T>(reserve_space);
}
return;
}
bool test_mode = is_test && (!trainable_statistics);
bool global_stats = test_mode || use_global_stats;
auto data_layout = StringToDataLayout(data_layout_str);
const auto& x_dims = x.dims();
PADDLE_ENFORCE_GE(
x_dims.size(),
2,
common::errors::InvalidArgument(
"The size of input X's dimensions should be larger than 1."
"But received: the size of input X's dimensions is [%d]",
x_dims.size()));
PADDLE_ENFORCE_LE(
x_dims.size(),
5,
common::errors::InvalidArgument(
"The size of input X's dimensions should be less than 6."
"But received: the size of input X's dimensions is [%d]",
x_dims.size()));
const int64_t N = x_dims[0];
const int64_t C =
data_layout == DataLayout::NCHW ? x_dims[1] : x_dims[x_dims.size() - 1];
const int64_t sample_size = x.numel() / N / C;
const int64_t num_batch_channels = static_cast<int64_t>(N) * C;
const int64_t num_batch_spatial = static_cast<int64_t>(N) * sample_size;
// alloc memory
dev_ctx.template Alloc<T>(y);
dev_ctx.template Alloc<T>(mean_out);
dev_ctx.template Alloc<T>(variance_out);
dev_ctx.template Alloc<T>(saved_mean);
dev_ctx.template Alloc<T>(saved_variance);
if (reserve_space != nullptr) {
reserve_space->Resize({0});
dev_ctx.template Alloc<T>(reserve_space);
}
// input dimension is 2 and the format is NCHW. The input can be regarded
// as NHWC format
if (x_dims.size() == 2 && data_layout == DataLayout::NCHW) {
data_layout = DataLayout::NHWC;
}
if (!global_stats) {
// saved_xx is use just in this batch of data
EigenVectorArrayMap<T> saved_mean_e(dev_ctx.template Alloc<T>(saved_mean),
C);
EigenVectorArrayMap<T> saved_variance_e(
dev_ctx.template Alloc<T>(saved_variance), C);
saved_mean_e.setZero();
saved_variance_e.setZero();
EigenVectorArrayMap<uint8_t> reserve_space_e(
dev_ctx.template Alloc<uint8_t>(reserve_space), 0);
reserve_space_e.setZero();
EigenVectorArrayMap<T> running_mean_arr(dev_ctx.template Alloc<T>(mean_out),
C);
EigenVectorArrayMap<T> running_var_arr(
dev_ctx.template Alloc<T>(variance_out), C);
if (num_batch_spatial == 1) {
// Only 1 element in normalization dimension,
// we skip the batch norm calculation, let y = x.
Copy(dev_ctx, x, dev_ctx.GetPlace(), false, y);
return;
}
switch (data_layout) {
case DataLayout::NCHW: {
ConstEigenArrayMap<T> x_arr(
x.data<T>(), sample_size, num_batch_channels);
for (int64_t nc = 0; nc < num_batch_channels; ++nc) {
saved_mean_e(nc % C) += x_arr.col(nc).sum();
}
saved_mean_e /= num_batch_spatial;
for (int64_t nc = 0; nc < num_batch_channels; ++nc) {
saved_variance_e(nc % C) +=
(x_arr.col(nc) - saved_mean_e(nc % C)).matrix().squaredNorm();
}
saved_variance_e /= num_batch_spatial;
break;
}
case DataLayout::NHWC: {
ConstEigenArrayMap<T> x_arr(x.data<T>(), C, num_batch_spatial);
for (int64_t i = 0; i < num_batch_spatial; ++i) {
saved_mean_e += x_arr.col(i);
}
saved_mean_e /= num_batch_spatial;
for (int64_t i = 0; i < num_batch_spatial; ++i) {
saved_variance_e +=
(x_arr.col(i) - saved_mean_e) * (x_arr.col(i) - saved_mean_e);
}
saved_variance_e /= num_batch_spatial;
break;
}
default:
PADDLE_THROW(common::errors::InvalidArgument(
"Unknown storage order: %s", data_layout_str));
}
// if MomentumTensor is set, use MomentumTensor value, momentum
// is only used in this training branch
running_mean_arr =
running_mean_arr * momentum + saved_mean_e * (1. - momentum);
running_var_arr =
running_var_arr * momentum + saved_variance_e * (1. - momentum);
} else {
const auto* est_mean = &mean;
const auto* est_var = &variance;
PADDLE_ENFORCE_EQ(
est_mean->dims().size(),
1UL,
common::errors::InvalidArgument(
"The size of mean's dimensions must equal to 1."
"But received: the size of mean's dimensions mean is [%d],"
"the dimensions of mean is [%s].",
est_mean->dims().size(),
est_mean->dims()));
PADDLE_ENFORCE_EQ(
est_var->dims().size(),
1UL,
common::errors::InvalidArgument(
"The size of variance's dimensions must equal to 1."
"But received: the size of variance's dimensions is [%d],"
"the dimensions of variance is [%s].",
est_var->dims().size(),
est_var->dims()));
PADDLE_ENFORCE_EQ(
est_mean->dims()[0],
C,
common::errors::InvalidArgument(
"The first dimension of mean must equal to the number of "
"Channels, which is [%d]. But received: the first dimension "
"of mean is [%d], the dimensions of mean is [%s].",
C,
est_mean->dims()[0],
est_mean->dims()));
PADDLE_ENFORCE_EQ(
est_var->dims()[0],
C,
common::errors::InvalidArgument(
"The first dimension of variance must equal to the number "
"of Channels, which is [%d]. But received: the first dimension of "
"variance is [%d], the dimensions of variance is [%s].",
C,
est_var->dims()[0],
est_var->dims()));
}
// use SavedMean and SavedVariance to do normalize
Eigen::Array<T, Eigen::Dynamic, 1> inv_std(C);
if (global_stats) { // NOLINT
ConstEigenVectorArrayMap<T> var_arr(variance.data<T>(), C);
inv_std = (var_arr + epsilon).sqrt().inverse();
} else {
EigenVectorArrayMap<T> saved_inv_std(saved_variance->data<T>(), C);
// inverse SavedVariance first, gradient will use it too.
saved_inv_std = (saved_inv_std + epsilon).inverse().sqrt();
inv_std = saved_inv_std;
}
ConstEigenVectorArrayMap<T> mean_arr(
global_stats ? mean.data<T>() : saved_mean->data<T>(), C);
// ((x - est_mean) * (inv_var) * scale + bias
// formula transform ====>
// (x * inv_var * scale) + (bias - est_mean * inv_var * scale)
auto* Scale = scale.get_ptr();
auto* Bias = bias.get_ptr();
Eigen::Array<T, Eigen::Dynamic, 1> new_scale(C);
Eigen::Array<T, Eigen::Dynamic, 1> new_bias(C);
if (Scale && Bias) { // NOLINT
ConstEigenVectorArrayMap<T> scale_arr(Scale->data<T>(), C);
ConstEigenVectorArrayMap<T> bias_arr(Bias->data<T>(), C);
new_scale = inv_std * scale_arr;
new_bias = bias_arr - mean_arr * inv_std * scale_arr;
} else if (Scale) {
ConstEigenVectorArrayMap<T> scale_arr(Scale->data<T>(), C);
new_scale = inv_std * scale_arr;
new_bias = -(mean_arr * inv_std * scale_arr);
} else if (Bias) {
ConstEigenVectorArrayMap<T> bias_arr(Bias->data<T>(), C);
new_scale = inv_std;
new_bias = bias_arr - mean_arr * inv_std;
} else {
new_scale = inv_std;
new_bias = -(mean_arr * inv_std);
}
switch (data_layout) {
case DataLayout::NCHW: {
EigenArrayMap<T> y_arr(
dev_ctx.template Alloc<T>(y), sample_size, num_batch_channels);
ConstEigenArrayMap<T> x_arr(x.data<T>(), sample_size, num_batch_channels);
for (int64_t nc = 0; nc < num_batch_channels; ++nc) {
y_arr.col(nc) = x_arr.col(nc) * new_scale(nc % C) + new_bias(nc % C);
}
break;
}
case DataLayout::NHWC: {
EigenArrayMap<T>(dev_ctx.template Alloc<T>(y), C, num_batch_spatial) =
(ConstEigenArrayMap<T>(x.data<T>(), C, num_batch_spatial).colwise() *
new_scale)
.colwise() +
new_bias;
break;
}
default:
PADDLE_THROW(common::errors::InvalidArgument("Unknown storage order: %d",
data_layout));
}
}
} // namespace phi
PD_REGISTER_KERNEL(
batch_norm, CPU, ALL_LAYOUT, phi::BatchNormKernel, float, double) {
kernel->OutputAt(5).SetDataType(phi::DataType::UINT8);
}
paddle/phi/kernels/cpu/bce_loss_grad_kernel.cc
+48
View File
@@ -0,0 +1,48 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/bce_loss_grad_kernel.h"
#include <algorithm> // for max
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
namespace phi {
template <typename T, typename Context>
void BCELossGradKernel(const Context& dev_ctx,
const DenseTensor& input,
const DenseTensor& label,
const DenseTensor& out_grad,
DenseTensor* input_grad) {
auto dx_data = dev_ctx.template Alloc<T>(input_grad);
auto dout_data = out_grad.data<T>();
auto x_data = input.data<T>();
auto label_data = label.data<T>();
int x_numel = static_cast<int>(input.numel());
// dx = dout * ((x - label)/(x - x^2))
for (int i = 0; i < x_numel; ++i) {
dx_data[i] =
dout_data[i] * ((x_data[i] - label_data[i]) /
std::max((static_cast<T>(1) - x_data[i]) * x_data[i],
static_cast<T>(1e-12)));
}
}
} // namespace phi
PD_REGISTER_KERNEL(
bce_loss_grad, CPU, ALL_LAYOUT, phi::BCELossGradKernel, float, double) {}
paddle/phi/kernels/cpu/bce_loss_kernel.cc
+59
View File
@@ -0,0 +1,59 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/bce_loss_kernel.h"
#include <algorithm> // for max
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/math.h"
namespace phi {
template <typename T, typename Context>
void BCELossKernel(const Context& dev_ctx,
const DenseTensor& input,
const DenseTensor& label,
DenseTensor* out) {
auto x_data = input.data<T>();
auto label_data = label.data<T>();
auto out_data = dev_ctx.template Alloc<T>(out);
auto x_numel = input.numel();
// out = -(label * ln(x) + (1 - label) * ln(1 - x)) = (label - 1) * ln(1 -
// x) - label * ln(x)
for (int64_t i = 0; i < x_numel; ++i) {
PADDLE_ENFORCE_GE(
x_data[i],
static_cast<T>(0),
common::errors::InvalidArgument(
"Illegal input, input must be greater than or equal to 0"));
PADDLE_ENFORCE_LE(
x_data[i],
static_cast<T>(1),
common::errors::InvalidArgument(
"Illegal input, input must be less than or equal to 1"));
out_data[i] =
(label_data[i] - static_cast<T>(1)) *
std::max(funcs::real_log(static_cast<T>(1) - x_data[i]),
(T)(-100)) -
label_data[i] * std::max(funcs::real_log(x_data[i]), (T)(-100));
}
}
} // namespace phi
PD_REGISTER_KERNEL(
bce_loss, CPU, ALL_LAYOUT, phi::BCELossKernel, float, double) {}
paddle/phi/kernels/cpu/beam_search_decode_kernel.cc
+30
View File
@@ -0,0 +1,30 @@
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/common/data_type.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/beam_search_decode_kernel_impl.h"
PD_REGISTER_KERNEL(beam_search_decode,
CPU,
ALL_LAYOUT,
phi::BeamSearchDecodeOpKernel,
float,
double,
phi::float16,
int,
int64_t) {
kernel->OutputAt(0).SetDataType(phi::DataType::INT64);
kernel->OutputAt(1).SetDataType(phi::DataType::INT64);
}
paddle/phi/kernels/cpu/beam_search_kernel.cc
+25
View File
@@ -0,0 +1,25 @@
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/beam_search_kernel_impl.h"
PD_REGISTER_KERNEL(beam_search,
CPU,
ALL_LAYOUT,
phi::BeamSearchOpKernel,
float,
double,
int,
int64_t) {}
paddle/phi/kernels/cpu/bernoulli_kernel.cc
+57
View File
@@ -0,0 +1,57 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/bernoulli_kernel.h"
#include <random>
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
namespace phi {
template <typename T>
inline T BernoulliFunctor(T p, T rand) {
PADDLE_ENFORCE_LE(p,
1.0,
common::errors::OutOfRange(
"The probability should be <= 1, but got %f", p));
PADDLE_ENFORCE_GE(p,
0.0,
common::errors::OutOfRange(
"The probability should be >= 0, but got %f", p));
return static_cast<T>(rand < p);
}
template <typename T, typename Context>
void BernoulliKernel(const Context& dev_ctx,
const DenseTensor& x,
DenseTensor* out) {
auto numel = x.numel();
auto* x_data = x.data<T>();
T* out_data = dev_ctx.template Alloc<T>(out);
std::uniform_real_distribution<T> dist(0.0, 1.0);
auto gen_ptr = dev_ctx.GetGenerator();
auto engine = gen_ptr->GetCPUEngine();
for (int64_t i = 0; i < numel; ++i) {
out_data[i] = BernoulliFunctor(x_data[i], dist(*engine));
}
}
} // namespace phi
PD_REGISTER_KERNEL(
bernoulli, CPU, ALL_LAYOUT, phi::BernoulliKernel, float, double) {}
paddle/phi/kernels/cpu/bilinear_grad_kernel.cc
+21
View File
@@ -0,0 +1,21 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/bilinear_grad_kernel.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/bilinear_grad_kernel_impl.h"
PD_REGISTER_KERNEL(
bilinear_grad, CPU, ALL_LAYOUT, phi::BilinearGradKernel, float, double) {}
paddle/phi/kernels/cpu/bilinear_kernel.cc
+21
View File
@@ -0,0 +1,21 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/bilinear_kernel.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/bilinear_kernel_impl.h"
PD_REGISTER_KERNEL(
bilinear, CPU, ALL_LAYOUT, phi::BilinearKernel, float, double) {}
paddle/phi/kernels/cpu/bincount_kernel.cc
+119
View File
@@ -0,0 +1,119 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/bincount_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/full_kernel.h"
#include "paddle/phi/kernels/funcs/math_function.h"
namespace phi {
template <typename Context, typename T, typename InputT>
void BincountInner(const Context& dev_ctx,
const DenseTensor& x,
const optional<DenseTensor>& weights,
int64_t minlength,
DenseTensor* out) {
const DenseTensor* input = &x;
DenseTensor* output = out;
const InputT* input_data = input->data<InputT>();
auto input_numel = input->numel();
if (input_data == nullptr) {
DDim out_dim{minlength};
output->Resize(out_dim);
// Since minlength may >0 , so fill with 0.
Full<int64_t, Context>(dev_ctx, output->dims(), 0, output);
return;
}
PADDLE_ENFORCE_GE(
*std::min_element(input_data, input_data + input_numel),
static_cast<InputT>(0),
common::errors::InvalidArgument(
"The elements in input tensor must be non-negative ints"));
int64_t output_size = static_cast<int64_t>(*std::max_element(
input_data, input_data + input_numel)) +
1L;
output_size = std::max(output_size, minlength);
DDim out_dim{output_size};
output->Resize(out_dim);
bool has_weights = weights.is_initialized();
if (has_weights) {
const T* weights_data = weights->data<T>();
if (weights->dtype() == DataType::FLOAT32) {
float* output_data = dev_ctx.template Alloc<float>(output);
funcs::SetConstant<Context, float>()(
dev_ctx, output, static_cast<float>(0));
for (int64_t i = 0; i < input_numel; i++) {
output_data[input_data[i]] += static_cast<float>(weights_data[i]);
}
} else {
double* output_data = dev_ctx.template Alloc<double>(output);
funcs::SetConstant<Context, double>()(
dev_ctx, output, static_cast<double>(0));
for (int64_t i = 0; i < input_numel; i++) {
output_data[input_data[i]] += static_cast<double>(weights_data[i]);
}
}
} else {
int64_t* output_data = dev_ctx.template Alloc<int64_t>(output);
funcs::SetConstant<Context, int64_t>()(
dev_ctx, output, static_cast<int64_t>(0));
for (int64_t i = 0; i < input_numel; i++) {
output_data[input_data[i]] += 1L;
}
}
}
template <typename T, typename Context>
void BincountKernel(const Context& dev_ctx,
const DenseTensor& x,
const optional<DenseTensor>& weights,
const Scalar& minlength,
DenseTensor* out) {
int64_t int_minlength = minlength.to<int64_t>();
PADDLE_ENFORCE_GE(int_minlength,
0,
common::errors::InvalidArgument(
"The minlength should be greater than or equal to 0."
"But received minlength is %d",
int_minlength));
if (x.dtype() == DataType::INT32) {
BincountInner<Context, T, int>(dev_ctx, x, weights, int_minlength, out);
} else if (x.dtype() == DataType::INT64) {
BincountInner<Context, T, int64_t>(dev_ctx, x, weights, int_minlength, out);
}
}
} // namespace phi
PD_REGISTER_KERNEL(bincount,
CPU,
ALL_LAYOUT,
phi::BincountKernel,
float,
double,
int,
int64_t) {
kernel->OutputAt(0).SetDataType(phi::DataType::UNDEFINED);
}
paddle/phi/kernels/cpu/binomial_kernel.cc
+44
View File
@@ -0,0 +1,44 @@
// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/binomial_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/binomial_functor.h"
namespace phi {
template <typename T, typename Context>
void BinomialKernel(const Context& dev_ctx,
const DenseTensor& count,
const DenseTensor& prob,
DenseTensor* out) {
auto numel = count.numel();
auto* count_data = count.data<T>();
auto* prob_data = prob.data<T>();
int64_t* out_data = dev_ctx.template Alloc<int64_t>(out);
for (int64_t i = 0; i < numel; ++i) {
out_data[i] =
funcs::BinomialFunctor<T>(dev_ctx, count_data[i], prob_data[i]);
}
}
} // namespace phi
PD_REGISTER_KERNEL(
binomial, CPU, ALL_LAYOUT, phi::BinomialKernel, float, double) {
kernel->OutputAt(0).SetDataType(phi::DataType::INT64);
}
paddle/phi/kernels/cpu/bipartite_match_kernel.cc
+216
View File
@@ -0,0 +1,216 @@
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/math_function.h"
namespace phi {
template <class T>
bool DistPairDescend(std::tuple<int, int, T> pair1,
std::tuple<int, int, T> pair2) {
return std::get<2>(pair1) > std::get<2>(pair2);
}
// The match_indices must be initialized to -1 at first.
// The match_dist must be initialized to 0 at first.
template <typename T>
void BipartiteMatch(const DenseTensor& dist,
int* match_indices,
T* match_dist) {
PADDLE_ENFORCE_EQ(
dist.dims().size(),
2,
common::errors::InvalidArgument("The rank of dist must be 2."));
int64_t row = dist.dims()[0];
int64_t col = dist.dims()[1];
auto* dist_data = dist.data<T>();
// Test result: When row==130 the speed of these two methods almost the same
if (row >= 130) {
std::vector<std::tuple<int, int, T>> match_pair;
for (int64_t i = 0; i < row; ++i) {
for (int64_t j = 0; j < col; ++j) {
match_pair.push_back(std::make_tuple(i, j, dist_data[i * col + j]));
}
}
std::sort(match_pair.begin(), match_pair.end(), DistPairDescend<T>);
std::vector<int> row_indices(row, -1);
int64_t idx = 0;
for (int64_t k = 0; k < row * col; ++k) {
int64_t i = std::get<0>(match_pair[k]);
int64_t j = std::get<1>(match_pair[k]);
T dist = std::get<2>(match_pair[k]);
if (idx >= row) {
break;
}
if (match_indices[j] == -1 && row_indices[i] == -1 && dist > 0) {
match_indices[j] = static_cast<int>(i);
row_indices[i] = static_cast<int>(j);
match_dist[j] = dist;
idx += 1;
}
}
} else {
constexpr T kEPS = static_cast<T>(1e-6);
std::vector<int> row_pool;
for (int i = 0; i < row; ++i) {
row_pool.push_back(i);
}
while (!row_pool.empty()) {
int max_idx = -1;
int max_row_idx = -1;
T max_dist = -1;
for (int64_t j = 0; j < col; ++j) {
if (match_indices[j] != -1) {
continue;
}
for (auto m : row_pool) {
// distance is 0 between m-th row and j-th column
if (dist_data[m * col + j] < kEPS) {
continue;
}
if (dist_data[m * col + j] > max_dist) {
max_idx = static_cast<int>(j);
max_row_idx = m;
max_dist = dist_data[m * col + j];
}
}
}
if (max_idx == -1) {
// Cannot find good match.
break;
} else {
PADDLE_ENFORCE_EQ(
match_indices[max_idx],
-1,
common::errors::InvalidArgument(
"The match_indices must be initialized to -1 at [%d].",
max_idx));
match_indices[max_idx] = max_row_idx;
match_dist[max_idx] = max_dist;
// Erase the row index.
row_pool.erase(
std::find(row_pool.begin(), row_pool.end(), max_row_idx));
}
}
}
}
template <typename T>
void ArgMaxMatch(const DenseTensor& dist,
int* match_indices,
T* match_dist,
T overlap_threshold) {
constexpr T kEPS = static_cast<T>(1e-6);
int64_t row = dist.dims()[0];
int64_t col = dist.dims()[1];
auto* dist_data = dist.data<T>();
for (int64_t j = 0; j < col; ++j) {
if (match_indices[j] != -1) {
// the j-th column has been matched to one entity.
continue;
}
int max_row_idx = -1;
T max_dist = -1;
for (int i = 0; i < row; ++i) {
T dist = dist_data[i * col + j];
if (dist < kEPS) {
// distance is 0 between m-th row and j-th column
continue;
}
if (dist >= overlap_threshold && dist > max_dist) {
max_row_idx = i;
max_dist = dist;
}
}
if (max_row_idx != -1) {
PADDLE_ENFORCE_EQ(
match_indices[j],
-1,
common::errors::InvalidArgument(
"The match_indices must be initialized to -1 at [%d].", j));
match_indices[j] = max_row_idx;
match_dist[j] = max_dist;
}
}
}
template <typename T, typename Context>
void BipartiteMatchKernel(const Context& dev_ctx,
const DenseTensor& dist_mat_in,
const std::string& match_type,
float dist_threshold,
DenseTensor* col_to_row_match_indices,
DenseTensor* col_to_row_match_dist) {
auto* dist_mat = &dist_mat_in;
auto* match_indices = col_to_row_match_indices;
auto* match_dist = col_to_row_match_dist;
auto col = dist_mat->dims()[1];
int64_t n = dist_mat->lod().empty()
? 1
: static_cast<int64_t>(dist_mat->lod().back().size() - 1);
if (!dist_mat->lod().empty()) {
PADDLE_ENFORCE_EQ(
dist_mat->lod().size(),
1UL,
common::errors::InvalidArgument("Only support 1 level of LoD."));
}
match_indices->Resize({n, col});
dev_ctx.template Alloc<int>(match_indices);
match_dist->Resize({n, col});
dev_ctx.template Alloc<T>(match_dist);
funcs::SetConstant<CPUContext, int> iset;
iset(dev_ctx, match_indices, static_cast<int>(-1));
funcs::SetConstant<CPUContext, T> tset;
tset(dev_ctx, match_dist, static_cast<T>(0));
int* indices = match_indices->data<int>();
T* dist = match_dist->data<T>();
auto type = match_type;
auto threshold = dist_threshold;
if (n == 1) {
BipartiteMatch<T>(*dist_mat, indices, dist);
if (type == "per_prediction") {
ArgMaxMatch<T>(*dist_mat, indices, dist, threshold);
}
} else {
auto lod = dist_mat->lod().back();
for (size_t i = 0; i < lod.size() - 1; ++i) {
if (lod[i + 1] > lod[i]) {
DenseTensor one_ins = dist_mat->Slice(static_cast<int64_t>(lod[i]),
static_cast<int64_t>(lod[i + 1]));
BipartiteMatch<T>(one_ins, indices + i * col, dist + i * col);
if (type == "per_prediction") {
ArgMaxMatch<T>(one_ins, indices + i * col, dist + i * col, threshold);
}
}
}
}
}
} // namespace phi
PD_REGISTER_KERNEL(bipartite_match,
CPU,
ALL_LAYOUT,
phi::BipartiteMatchKernel,
float,
double) {
kernel->OutputAt(0).SetDataType(phi::DataType::INT32);
}
paddle/phi/kernels/cpu/bitwise_kernel.cc
+157
View File
@@ -0,0 +1,157 @@
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/phi/kernels/bitwise_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/bitwise_functors.h"
#include "paddle/phi/kernels/funcs/elementwise_base.h"
#include "paddle/phi/common/transform.h"
namespace phi {
#define DEFINE_BITWISE_KERNEL(op_type) \
template <typename T, typename Context> \
void Bitwise##op_type##Kernel(const Context& dev_ctx, \
const DenseTensor& x, \
const DenseTensor& y, \
DenseTensor* out) { \
funcs::Bitwise##op_type##Functor<T> func; \
funcs::ElementwiseCompute<funcs::Bitwise##op_type##Functor<T>, T>( \
dev_ctx, x, y, func, out); \
}
DEFINE_BITWISE_KERNEL(And)
DEFINE_BITWISE_KERNEL(Or)
DEFINE_BITWISE_KERNEL(Xor)
#undef DEFINE_BITWISE_KERNEL
#define DEFINE_BITWISE_KERNEL_WITH_INVERSE(op_type) \
template <typename T, typename Context> \
void Bitwise##op_type##Kernel(const Context& dev_ctx, \
const DenseTensor& x, \
const DenseTensor& y, \
bool is_arithmetic, \
DenseTensor* out) { \
auto x_dims = x.dims(); \
auto y_dims = y.dims(); \
if (x_dims.size() >= y_dims.size()) { \
if (is_arithmetic) { \
funcs::Bitwise##op_type##ArithmeticFunctor<T> func; \
funcs::ElementwiseCompute< \
funcs::Bitwise##op_type##ArithmeticFunctor<T>, \
T>(dev_ctx, x, y, func, out); \
} else { \
funcs::Bitwise##op_type##LogicFunctor<T> func; \
funcs::ElementwiseCompute<funcs::Bitwise##op_type##LogicFunctor<T>, \
T>(dev_ctx, x, y, func, out); \
} \
} else { \
if (is_arithmetic) { \
funcs::InverseBitwise##op_type##ArithmeticFunctor<T> inv_func; \
funcs::ElementwiseCompute< \
funcs::InverseBitwise##op_type##ArithmeticFunctor<T>, \
T>(dev_ctx, x, y, inv_func, out); \
} else { \
funcs::InverseBitwise##op_type##LogicFunctor<T> inv_func; \
funcs::ElementwiseCompute< \
funcs::InverseBitwise##op_type##LogicFunctor<T>, \
T>(dev_ctx, x, y, inv_func, out); \
} \
} \
}
DEFINE_BITWISE_KERNEL_WITH_INVERSE(LeftShift)
DEFINE_BITWISE_KERNEL_WITH_INVERSE(RightShift)
#undef DEFINE_BITWISE_KERNEL_WITH_INVERSE
template <typename T, typename Context>
void BitwiseNotKernel(const Context& dev_ctx,
const DenseTensor& x,
DenseTensor* out) {
const T* x_data = x.data<T>();
T* out_data = dev_ctx.template Alloc<T>(out);
size_t numel = x.numel();
funcs::BitwiseNotFunctor<T> func;
Transform<Context> trans;
trans(dev_ctx, x_data, x_data + numel, out_data, func);
}
} // namespace phi
PD_REGISTER_KERNEL(bitwise_and,
CPU,
ALL_LAYOUT,
phi::BitwiseAndKernel,
bool,
uint8_t,
int8_t,
int16_t,
int,
int64_t) {}
PD_REGISTER_KERNEL(bitwise_or,
CPU,
ALL_LAYOUT,
phi::BitwiseOrKernel,
bool,
uint8_t,
int8_t,
int16_t,
int,
int64_t) {}
PD_REGISTER_KERNEL(bitwise_xor,
CPU,
ALL_LAYOUT,
phi::BitwiseXorKernel,
bool,
uint8_t,
int8_t,
int16_t,
int,
int64_t) {}
PD_REGISTER_KERNEL(bitwise_not,
CPU,
ALL_LAYOUT,
phi::BitwiseNotKernel,
bool,
uint8_t,
int8_t,
int16_t,
int,
int64_t) {}
PD_REGISTER_KERNEL(bitwise_left_shift,
CPU,
ALL_LAYOUT,
phi::BitwiseLeftShiftKernel,
uint8_t,
int8_t,
int16_t,
int,
int64_t) {}
PD_REGISTER_KERNEL(bitwise_right_shift,
CPU,
ALL_LAYOUT,
phi::BitwiseRightShiftKernel,
uint8_t,
int8_t,
int16_t,
int,
int64_t) {}
paddle/phi/kernels/cpu/bmm_grad_kernel.cc
+22
View File
@@ -0,0 +1,22 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/bmm_grad_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/bmm_grad_kernel_impl.h"
PD_REGISTER_KERNEL(
bmm_grad, CPU, ALL_LAYOUT, phi::BmmGradKernel, float, double) {}
paddle/phi/kernels/cpu/bmm_kernel.cc
+21
View File
@@ -0,0 +1,21 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/bmm_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/bmm_kernel_impl.h"
PD_REGISTER_KERNEL(bmm, CPU, ALL_LAYOUT, phi::BmmKernel, float, double) {}
paddle/phi/kernels/cpu/box_clip_kernel.cc
+19
View File
@@ -0,0 +1,19 @@
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/box_clip_kernel_impl.h"
PD_REGISTER_KERNEL(
box_clip, CPU, ALL_LAYOUT, phi::BoxClipKernel, float, double) {}
paddle/phi/kernels/cpu/box_coder_kernel.cc
+288
View File
@@ -0,0 +1,288 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/box_coder_kernel.h"
#include <array>
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/full_kernel.h"
#include "paddle/phi/kernels/funcs/math_function.h"
#include "paddle/phi/kernels/impl/box_coder.h"
namespace phi {
template <typename T>
void EncodeCenterSize(const DenseTensor *target_box,
const DenseTensor *prior_box,
const DenseTensor *prior_box_var,
const bool normalized,
const std::vector<float> variance,
T *output) {
int64_t row = target_box->dims()[0];
int64_t col = prior_box->dims()[0];
int64_t len = prior_box->dims()[1];
#ifdef PADDLE_WITH_MKLML
#pragma omp parallel for collapse(2)
#endif
for (int64_t i = 0; i < row; ++i) {
for (int64_t j = 0; j < col; ++j) {
auto *target_box_data = target_box->data<T>();
auto *prior_box_data = prior_box->data<T>();
size_t offset = i * col * len + j * len;
T prior_box_width = prior_box_data[j * len + 2] -
prior_box_data[j * len] + (normalized == false);
T prior_box_height = prior_box_data[j * len + 3] -
prior_box_data[j * len + 1] + (normalized == false);
T prior_box_center_x = prior_box_data[j * len] + prior_box_width / 2;
T prior_box_center_y = prior_box_data[j * len + 1] + prior_box_height / 2;
T target_box_center_x =
(target_box_data[i * len + 2] + target_box_data[i * len]) / 2;
T target_box_center_y =
(target_box_data[i * len + 3] + target_box_data[i * len + 1]) / 2;
T target_box_width = target_box_data[i * len + 2] -
target_box_data[i * len] + (normalized == false);
T target_box_height = target_box_data[i * len + 3] -
target_box_data[i * len + 1] +
(normalized == false);
output[offset] =
(target_box_center_x - prior_box_center_x) / prior_box_width;
output[offset + 1] =
(target_box_center_y - prior_box_center_y) / prior_box_height;
output[offset + 2] =
std::log(std::fabs(target_box_width / prior_box_width));
output[offset + 3] =
std::log(std::fabs(target_box_height / prior_box_height));
}
}
if (prior_box_var) {
const T *prior_box_var_data = prior_box_var->data<T>();
#ifdef PADDLE_WITH_MKLML
#pragma omp parallel for collapse(3)
#endif
for (int64_t i = 0; i < row; ++i) {
for (int64_t j = 0; j < col; ++j) {
for (int k = 0; k < 4; ++k) {
size_t offset = i * col * len + j * len;
int prior_var_offset = static_cast<int>(j * len);
output[offset + k] /= prior_box_var_data[prior_var_offset + k];
}
}
}
} else if (!(variance.empty())) {
#ifdef PADDLE_WITH_MKLML
#pragma omp parallel for collapse(3)
#endif
for (int64_t i = 0; i < row; ++i) {
for (int64_t j = 0; j < col; ++j) {
for (int k = 0; k < 4; ++k) {
size_t offset = i * col * len + j * len;
output[offset + k] /= static_cast<T>(variance[k]);
}
}
}
}
}
template <typename T, int axis, int var_size>
void DecodeCenterSize(const DenseTensor *target_box,
const DenseTensor *prior_box,
const DenseTensor *prior_box_var,
const bool normalized,
std::vector<float> variance,
T *output) {
int64_t row = target_box->dims()[0];
int64_t col = target_box->dims()[1];
int64_t len = target_box->dims()[2];
#ifdef PADDLE_WITH_MKLML
#pragma omp parallel for collapse(2)
#endif
for (int64_t i = 0; i < row; ++i) {
for (int64_t j = 0; j < col; ++j) {
auto *target_box_data = target_box->data<T>();
auto *prior_box_data = prior_box->data<T>();
std::array<T, 4> var_data{1., 1., 1., 1.};
T *var_ptr = var_data.data();
size_t offset = i * col * len + j * len;
int prior_box_offset = axis == 0 ? j * len : i * len;
T prior_box_width = prior_box_data[prior_box_offset + 2] -
prior_box_data[prior_box_offset] +
(normalized == false);
T prior_box_height = prior_box_data[prior_box_offset + 3] -
prior_box_data[prior_box_offset + 1] +
(normalized == false);
T prior_box_center_x =
prior_box_data[prior_box_offset] + prior_box_width / 2;
T prior_box_center_y =
prior_box_data[prior_box_offset + 1] + prior_box_height / 2;
T target_box_center_x = 0, target_box_center_y = 0;
T target_box_width = 0, target_box_height = 0;
int prior_var_offset = axis == 0 ? j * len : i * len;
if (var_size == 2) {
std::memcpy(var_ptr,
prior_box_var->data<T>() + prior_var_offset,
4 * sizeof(T));
} else if (var_size == 1) {
var_ptr = reinterpret_cast<T *>(variance.data());
}
T box_var_x = *var_ptr;
T box_var_y = *(var_ptr + 1);
T box_var_w = *(var_ptr + 2);
T box_var_h = *(var_ptr + 3);
target_box_center_x =
box_var_x * target_box_data[offset] * prior_box_width +
prior_box_center_x;
target_box_center_y =
box_var_y * target_box_data[offset + 1] * prior_box_height +
prior_box_center_y;
target_box_width =
std::exp(box_var_w * target_box_data[offset + 2]) * prior_box_width;
target_box_height =
std::exp(box_var_h * target_box_data[offset + 3]) * prior_box_height;
output[offset] = target_box_center_x - target_box_width / 2;
output[offset + 1] = target_box_center_y - target_box_height / 2;
output[offset + 2] =
target_box_center_x + target_box_width / 2 - (normalized == false);
output[offset + 3] =
target_box_center_y + target_box_height / 2 - (normalized == false);
}
}
}
template <typename T, typename Context>
void BoxCoderKernel(const Context &dev_ctx,
const DenseTensor &prior_box,
const optional<DenseTensor> &prior_box_var,
const DenseTensor &target_box,
const std::string &code_type_str,
bool normalized,
int axis,
const std::vector<float> &variance,
DenseTensor *output_box) {
// prior_box and prior_box_var have the same shape, so do not judge
// prior_box_var
if (prior_box.numel() == 0 || target_box.numel() == 0) {
Full<T, Context>(dev_ctx, output_box->dims(), 0, output_box);
return;
}
if (!target_box.lod().empty()) {
PADDLE_ENFORCE_EQ(target_box.lod().size(),
1UL,
common::errors::InvalidArgument(
"Input(TargetBox) of BoxCoder operator "
"supports LoD with only one level. But received "
"level = %d",
target_box.lod().size()));
}
if (prior_box_var) {
PADDLE_ENFORCE_EQ(variance.empty(),
true,
common::errors::InvalidArgument(
"Input 'PriorBoxVar' and attribute 'variance' "
"of BoxCoder operator should not be used at the "
"same time."));
}
if (!(variance.empty())) {
PADDLE_ENFORCE_EQ(static_cast<int>(variance.size()),
4,
common::errors::InvalidArgument(
"Size of attribute 'variance' of BoxCoder "
"operator should be 4. But received "
"size = %d",
variance.size()));
}
auto code_type = funcs::GetBoxCodeType(code_type_str);
auto row = target_box.dims()[0];
auto col = prior_box.dims()[0];
if (code_type == funcs::BoxCodeType::kDecodeCenterSize) {
col = target_box.dims()[1];
}
auto len = prior_box.dims()[1];
output_box->Resize({row, col, len});
dev_ctx.template Alloc<T>(output_box);
T *output = output_box->data<T>();
if (code_type == funcs::BoxCodeType::kEncodeCenterSize) {
EncodeCenterSize<T>(&target_box,
&prior_box,
prior_box_var.get_ptr(),
normalized,
variance,
output);
} else if (code_type == funcs::BoxCodeType::kDecodeCenterSize) {
if (prior_box_var) {
if (axis == 0) {
DecodeCenterSize<T, 0, 2>(&target_box,
&prior_box,
prior_box_var.get_ptr(),
normalized,
variance,
output);
} else {
DecodeCenterSize<T, 1, 2>(&target_box,
&prior_box,
prior_box_var.get_ptr(),
normalized,
variance,
output);
}
} else if (!(variance.empty())) {
if (axis == 0) {
DecodeCenterSize<T, 0, 1>(&target_box,
&prior_box,
prior_box_var.get_ptr(),
normalized,
variance,
output);
} else {
DecodeCenterSize<T, 1, 1>(&target_box,
&prior_box,
prior_box_var.get_ptr(),
normalized,
variance,
output);
}
} else {
if (axis == 0) {
DecodeCenterSize<T, 0, 0>(&target_box,
&prior_box,
prior_box_var.get_ptr(),
normalized,
variance,
output);
} else {
DecodeCenterSize<T, 1, 0>(&target_box,
&prior_box,
prior_box_var.get_ptr(),
normalized,
variance,
output);
}
}
}
}
} // namespace phi
PD_REGISTER_KERNEL(
box_coder, CPU, ALL_LAYOUT, phi::BoxCoderKernel, float, double) {}
paddle/phi/kernels/cpu/broadcast_kernel.cc
+68
View File
@@ -0,0 +1,68 @@
// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/broadcast_kernel.h"
#include "paddle/phi/backends/all_context.h"
#include "paddle/phi/core/kernel_registry.h"
#if defined(PADDLE_WITH_GLOO)
#include "paddle/phi/core/distributed/gloo_comm_context.h"
#endif
namespace phi {
template <typename T, typename Context>
void BroadcastKernel(const Context& dev_ctx,
const DenseTensor& x,
int root,
DenseTensor* out) {
PADDLE_ENFORCE_GT(x.numel(),
0,
common::errors::InvalidArgument(
"Tensor need be broadcast must not empty."));
#if defined(PADDLE_WITH_GLOO)
dev_ctx.template Alloc<T>(out);
auto comm_context =
static_cast<distributed::GlooCommContext*>(dev_ctx.GetCommContext());
PADDLE_ENFORCE_NE(
comm_context,
nullptr,
errors::Unavailable("NCCLCommContext is nullptr, collective op should "
"has ring_id attr."));
comm_context->Broadcast(out, x, root);
#else
PADDLE_THROW(errors::Unavailable(
"PaddlePaddle should compile with GLOO by setting WITH_GLOO=ON"));
#endif
}
} // namespace phi
PD_REGISTER_KERNEL(broadcast,
CPU,
ALL_LAYOUT,
phi::BroadcastKernel,
float,
double,
int,
bool,
int8_t,
uint8_t,
int16_t,
int64_t,
phi::float16,
phi::complex64,
phi::complex128) {}
paddle/phi/kernels/cpu/broadcast_tensors_grad_kernel.cc
+209
View File
@@ -0,0 +1,209 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/broadcast_tensors_grad_kernel.h"
#include <vector>
#include "paddle/phi/core/dense_tensor.h"
#include "paddle/phi/core/enforce.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/full_kernel.h"
#include "paddle/phi/kernels/funcs/eigen/common.h"
#include "paddle/phi/kernels/funcs/eigen/eigen_function.h"
#include "paddle/phi/kernels/funcs/math_function.h"
#define SWITCH_RESHAPE_DIMS(n) \
case n: { \
Eigen::DSizes<int64_t, n> reshape_dims; \
for (size_t i = 0; i < reshape_dims_vec.size(); ++i) { \
reshape_dims[i] = reshape_dims_vec[i]; \
} \
dX.device(place) = \
dOut.reshape(reshape_dims).sum(reduce_dims).reshape(dX.dimensions()); \
break; \
}
#define UPPER_SWITCH_REDUCE_DIMS(m) \
case m: { \
Eigen::DSizes<int64_t, m> reduce_dims; \
for (size_t i = 0; i < reduce_dims_vec.size(); ++i) { \
reduce_dims[i] = reduce_dims_vec[i]; \
} \
switch (reshape_size) {
#define LOWER_SWITCH_REDUCE_DIMS \
default: { \
PADDLE_THROW(errors::InvalidArgument( \
"Detected reshape size: %d out of range. " \
"Minimum value should be larger than reduce size %d. " \
"While maximum supported is: 5", \
reshape_size, \
reduce_size)); \
} \
} \
break; \
}
namespace phi {
template <typename T, typename Context>
void BroadcastTensorsGradKernel(const Context& dev_ctx,
const std::vector<const DenseTensor*>& inputs,
const std::vector<const DenseTensor*>& dout,
std::vector<DenseTensor*> dx) {
(void)inputs;
// Find reduce dimensions
const auto& in_tensors = dout;
auto& out_tensors = dx;
size_t num_ins = in_tensors.size();
if (dout[0] && dout[0]->numel() == 0) {
for (auto dx : out_tensors) {
if (dx) Full<T, Context>(dev_ctx, dx->dims(), 0, dx);
}
return;
}
PADDLE_ENFORCE_GT(
num_ins,
1,
errors::InvalidArgument(
"Expected at least 2 input tensors, but only received %d.",
in_tensors.size()));
PADDLE_ENFORCE_EQ(num_ins,
out_tensors.size(),
errors::InvalidArgument(
"BroadcastTensorsOp expects equal number of inputs and "
"outputs, but received: %d inputs v.s %d outputs",
num_ins,
out_tensors.size()));
// For each In-Out tensor pair,
// Prepare and apply broadcast dims array
for (size_t i = 0; i < num_ins; i++) {
const auto* input_tensor = in_tensors[i];
auto* output_tensor = out_tensors[i];
const auto& input_dims = input_tensor->dims();
const auto& output_dims = output_tensor->dims();
int in_rank = input_dims.size();
int out_rank = output_dims.size();
// BroadcastTensorsGrad is simply a reduce_sum along broadcasted axes
// Here we perform the following Eigen operations:
// dOut(Flattened) -> reshape(reshape_dims) -> reduce(reduce_dims) ->
// reshape(dX_shape) -> dX
// Note the last "reshape(dX_shape)" will be performed implicitly,
// and we only need to collect reduce_dims and reshape_dims
std::vector<int> reduce_dims_vec;
std::vector<int> reshape_dims_vec;
for (int j = 0; j < in_rank; j++) {
int out_axis = out_rank - j - 1;
int in_axis = in_rank - j - 1;
reshape_dims_vec.push_back(static_cast<int>(input_dims[j]));
if (out_axis < 0 || output_dims[out_axis] != input_dims[in_axis]) {
reduce_dims_vec.push_back(in_axis);
}
}
size_t reduce_size = reduce_dims_vec.size();
size_t reshape_size = reshape_dims_vec.size();
bool just_copy = (reduce_dims_vec.empty());
dev_ctx.template Alloc<T>(output_tensor);
if (just_copy) {
// If this turns out to be a No-Op, simply perform a tensor copy
Copy(dev_ctx, *input_tensor, dev_ctx.GetPlace(), false, output_tensor);
} else {
PADDLE_ENFORCE_GE(
reduce_dims_vec.size(),
1,
errors::InvalidArgument("The number of dimensions of the input "
"'Out@GRAD' for Op(broadcast_tensors)"
" must be greater than or equal to 1, but "
"the value received is %d.",
reduce_dims_vec.size()));
PADDLE_ENFORCE_LE(
reduce_dims_vec.size(),
5,
errors::InvalidArgument(
"The number of dimensions of the input 'Out@GRAD' "
"for Op(broadcast_tensors) must be less than or equal "
"to 5, but the value received is %d.",
reduce_dims_vec.size()));
// Overall:
// dOut(Flattened) -> reshape(reshape_dims) -> reduce(reduce_dims) ->
// reshape(dX_shape) -> dX
auto dX = EigenVector<T>::Flatten(*output_tensor);
auto dOut = EigenVector<T>::Flatten(*input_tensor);
auto& place = *dev_ctx.eigen_device();
// Expand ReduceSize and ReshapeSize into static values
switch (reduce_size) {
UPPER_SWITCH_REDUCE_DIMS(1)
SWITCH_RESHAPE_DIMS(1)
SWITCH_RESHAPE_DIMS(2)
SWITCH_RESHAPE_DIMS(3)
SWITCH_RESHAPE_DIMS(4)
SWITCH_RESHAPE_DIMS(5)
LOWER_SWITCH_REDUCE_DIMS
UPPER_SWITCH_REDUCE_DIMS(2)
SWITCH_RESHAPE_DIMS(2)
SWITCH_RESHAPE_DIMS(3)
SWITCH_RESHAPE_DIMS(4)
SWITCH_RESHAPE_DIMS(5)
LOWER_SWITCH_REDUCE_DIMS
UPPER_SWITCH_REDUCE_DIMS(3)
SWITCH_RESHAPE_DIMS(3)
SWITCH_RESHAPE_DIMS(4)
SWITCH_RESHAPE_DIMS(5)
LOWER_SWITCH_REDUCE_DIMS
UPPER_SWITCH_REDUCE_DIMS(4)
SWITCH_RESHAPE_DIMS(4)
SWITCH_RESHAPE_DIMS(5)
LOWER_SWITCH_REDUCE_DIMS
UPPER_SWITCH_REDUCE_DIMS(5)
SWITCH_RESHAPE_DIMS(5)
LOWER_SWITCH_REDUCE_DIMS
default: {
PADDLE_THROW(
errors::InvalidArgument("Detected reduce size: %d out of range. "
"While maximum supported is: 5",
reduce_size));
}
}
}
}
}
} // namespace phi
PD_REGISTER_KERNEL(broadcast_tensors_grad,
CPU,
ALL_LAYOUT,
phi::BroadcastTensorsGradKernel,
int,
int64_t,
float,
double,
phi::float16,
phi::complex64,
phi::complex128) {}
paddle/phi/kernels/cpu/broadcast_tensors_kernel.cc
+31
View File
@@ -0,0 +1,31 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/broadcast_tensors_kernel.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/broadcast_tensors_kernel_impl.h"
PD_REGISTER_KERNEL(broadcast_tensors,
CPU,
ALL_LAYOUT,
phi::BroadcastTensorsKernel,
bool,
int,
int64_t,
float,
double,
phi::float16,
phi::complex64,
phi::complex128) {}
paddle/phi/kernels/cpu/c_concat_kernel.cc
+44
View File
@@ -0,0 +1,44 @@
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/c_concat_kernel.h"
#include "paddle/phi/core/kernel_registry.h"
namespace phi {
template <typename T, typename Context>
void CConcatKernel(const Context &dev_ctx UNUSED,
const DenseTensor &x UNUSED,
int rank UNUSED,
int nranks UNUSED,
int ring_id UNUSED,
bool use_calc_stream UNUSED,
bool use_model_parallel UNUSED,
DenseTensor *out UNUSED) {
PADDLE_THROW(common::errors::Unavailable(
"Do not support c_concat for cpu kernel now."));
}
} // namespace phi
PD_REGISTER_KERNEL(c_concat,
CPU,
ALL_LAYOUT,
phi::CConcatKernel,
float,
double,
int,
int64_t,
phi::float16) {}
paddle/phi/kernels/cpu/c_embedding_grad_kernel.cc
+101
View File
@@ -0,0 +1,101 @@
// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/c_embedding_grad_kernel.h"
#include "glog/logging.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
namespace phi {
template <typename TIds, typename TData>
void UpdateEmbedding(const TIds* ids,
size_t ids_len,
int64_t start_idx,
TData* table,
int64_t height,
int64_t width,
const TData* out) {
for (size_t i = 0; i < ids_len; i++) {
TIds id = ids[i];
int64_t local = id - start_idx;
if (local >= 0 && local < height) {
for (int64_t w = 0; w < width; w++) {
table[local * width + w] += out[i * width + w];
}
}
}
}
template <typename T, typename Context>
void CEmbeddingGradKernel(const Context& dev_ctx,
const DenseTensor& w,
const DenseTensor& ids,
const DenseTensor& out_grad,
int64_t start_index,
DenseTensor* w_grad) {
w_grad->Resize(w.dims());
T* table_grad_data = dev_ctx.template Alloc<T>(w_grad);
size_t table_t_mem_size = w.numel() * sizeof(w_grad->dtype());
size_t table_grad_t_mem_size = w_grad->numel() * sizeof(w_grad->dtype());
VLOG(10) << "table_dims:" << w.dims()
<< ", table_t memory_size:" << table_t_mem_size
<< ", table_grad_t memory_size:" << table_grad_t_mem_size
<< ", start_index:" << start_index;
memset(table_grad_data, 0, table_grad_t_mem_size);
const T* d_output_data = out_grad.data<T>();
const int64_t height = w.dims()[0];
const int64_t width = w.dims()[1];
const auto& index_type = ids.dtype();
if (index_type == DataType::INT32) {
UpdateEmbedding(ids.data<int32_t>(),
ids.numel(),
start_index,
table_grad_data,
height,
width,
d_output_data);
} else if (index_type == DataType::INT64) {
UpdateEmbedding(ids.data<int64_t>(),
ids.numel(),
start_index,
table_grad_data,
height,
width,
d_output_data);
} else {
PADDLE_THROW(common::errors::Unavailable(
"CPU c_embedding ids only support int32 or int64."));
}
}
} // namespace phi
PD_REGISTER_KERNEL(c_embedding_grad,
CPU,
ALL_LAYOUT,
phi::CEmbeddingGradKernel,
float,
double,
phi::float16,
phi::complex64,
phi::complex128) {}
paddle/phi/kernels/cpu/c_embedding_kernel.cc
+90
View File
@@ -0,0 +1,90 @@
// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/c_embedding_kernel.h"
#include "glog/logging.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
namespace phi {
template <typename TIds, typename TData>
void GetIdsEmbedding(const TIds* ids,
size_t ids_len,
int64_t start_idx,
const TData* table,
int64_t height,
int64_t width,
TData* out) {
for (size_t i = 0; i < ids_len; i++) {
TIds id = ids[i];
int64_t local = id - start_idx;
if (local >= 0 && local < height) {
memcpy(out + i * width, table + local * width, width * sizeof(TData));
} else {
memset(out + i * width, 0, width * sizeof(TData));
}
}
}
template <typename T, typename Context>
void CEmbeddingKernel(const Context& dev_ctx,
const DenseTensor& w,
const DenseTensor& ids,
int64_t start_index,
int64_t vocab_size,
DenseTensor* out) {
VLOG(10) << "table_dims:" << w.dims();
const T* table_data = w.data<T>();
T* output_data = dev_ctx.template Alloc<T>(out);
const int64_t height = w.dims()[0];
const int64_t width = w.dims()[1];
const auto& index_type = ids.dtype();
if (index_type == DataType::INT32) {
GetIdsEmbedding(ids.data<int32_t>(),
ids.numel(),
start_index,
table_data,
height,
width,
output_data);
} else if (index_type == DataType::INT64) {
GetIdsEmbedding(ids.data<int64_t>(),
ids.numel(),
start_index,
table_data,
height,
width,
output_data);
} else {
PADDLE_THROW(common::errors::Unavailable(
"CPU c_embedding ids only support int32 or int64."));
}
}
} // namespace phi
PD_REGISTER_KERNEL(c_embedding,
CPU,
ALL_LAYOUT,
phi::CEmbeddingKernel,
float,
double,
phi::float16,
phi::complex64,
phi::complex128) {}
paddle/phi/kernels/cpu/c_identity_kernel.cc
+43
View File
@@ -0,0 +1,43 @@
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/phi/kernels/c_identity_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
namespace phi {
template <typename T, typename Context>
void CIdentityKernel(const Context& dev_ctx,
const DenseTensor& x,
int ring_id,
bool use_calc_stream,
bool use_model_parallel,
DenseTensor* out) {
PADDLE_THROW(
errors::Unavailable("Do not support c_identity for cpu kernel now."));
}
} // namespace phi
PD_REGISTER_KERNEL(c_identity,
CPU,
ALL_LAYOUT,
phi::CIdentityKernel,
float,
double,
int,
int64_t,
phi::float16) {}
paddle/phi/kernels/cpu/c_scatter_kernel.cc
+63
View File
@@ -0,0 +1,63 @@
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <algorithm>
#include <utility>
#include <vector>
#include "paddle/phi/core/dense_tensor.h"
#include "paddle/phi/core/kernel_registry.h"
#if defined(PADDLE_WITH_GLOO)
#include "paddle/phi/core/distributed/gloo_comm_context.h"
#endif
namespace phi {
template <typename T, typename Context>
void CScatterOpCPUKernel(const Context &dev_ctx,
const DenseTensor &x,
int ring_id,
int root,
int nranks,
bool use_calc_stream,
DenseTensor *out) {
#if defined(PADDLE_WITH_GLOO)
auto in = &x;
auto root_id = root;
auto comm_ctx =
static_cast<distributed::GlooCommContext *>(dev_ctx.GetCommContext());
PADDLE_ENFORCE_NE(comm_ctx,
nullptr,
::common::errors::Unavailable(
"NCCLCommContext is nullptr, collective op should "
"has ring_id attr."));
comm_ctx->Scatter(out, *in, root_id);
#else
PADDLE_THROW(common::errors::Unavailable(
"PaddlePaddle should compile with GLOO by setting WITH_GLOO=ON"));
#endif
}
} // namespace phi
PD_REGISTER_KERNEL(c_scatter,
CPU,
ALL_LAYOUT,
phi::CScatterOpCPUKernel,
float,
double,
int,
int64_t,
phi::float16) {}
paddle/phi/kernels/cpu/c_softmax_with_cross_entropy_kernel.cc
+40
View File
@@ -0,0 +1,40 @@
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/core/dense_tensor.h"
#include "paddle/phi/core/kernel_registry.h"
namespace phi {
template <typename T, typename Context>
void CSoftmaxWithCrossEntropyKernel(const Context &dev_ctx UNUSED,
const DenseTensor &logits UNUSED,
const DenseTensor &label UNUSED,
int64_t ignore_index UNUSED,
int rank UNUSED,
int nranks UNUSED,
DenseTensor *softmax UNUSED,
DenseTensor *loss UNUSED) {
PADDLE_THROW(common::errors::Unavailable(
"Do not support c_softmax_with_cross_entropy for cpu kernel now."));
}
} // namespace phi
PD_REGISTER_KERNEL(c_softmax_with_cross_entropy,
CPU,
ALL_LAYOUT,
phi::CSoftmaxWithCrossEntropyKernel,
float,
double,
phi::float16) {}
paddle/phi/kernels/cpu/c_softmax_with_multi_label_cross_entropy_kernel.cc
+44
View File
@@ -0,0 +1,44 @@
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/core/dense_tensor.h"
#include "paddle/phi/core/kernel_registry.h"
namespace phi {
template <typename T, typename Context>
void CSoftmaxWithMultiLabelCrossEntropyKernel(const Context &dev_ctx UNUSED,
const DenseTensor &logits UNUSED,
const DenseTensor &label UNUSED,
const DenseTensor &smooth_weight
UNUSED,
int64_t ignore_index UNUSED,
bool sum_multi_label_loss UNUSED,
int rank UNUSED,
int nranks UNUSED,
DenseTensor *softmax UNUSED,
DenseTensor *loss UNUSED) {
PADDLE_THROW(common::errors::Unavailable(
"Do not support c_softmax_with_multi_label_cross_entropy for cpu kernel "
"now."));
}
} // namespace phi
PD_REGISTER_KERNEL(c_softmax_with_multi_label_cross_entropy,
CPU,
ALL_LAYOUT,
phi::CSoftmaxWithMultiLabelCrossEntropyKernel,
float,
double,
phi::float16) {}
paddle/phi/kernels/cpu/c_split_kernel.cc
+41
View File
@@ -0,0 +1,41 @@
// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/c_split_kernel.h"
#include "paddle/phi/core/kernel_registry.h"
namespace phi {
template <typename T, typename Context>
void CSplitKernel(const Context& dev_ctx,
const DenseTensor& x,
int rank,
int nranks,
bool use_model_parallel,
DenseTensor* out) {
PADDLE_THROW(common::errors::Unavailable(
"Do not support c_split for cpu kernel now."));
}
} // namespace phi
PD_REGISTER_KERNEL(c_split,
CPU,
ALL_LAYOUT,
phi::CSplitKernel,
float,
double,
int,
int64_t,
phi::float16) {}
paddle/phi/kernels/cpu/cast_impl.h
+128
View File
@@ -0,0 +1,128 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/common/transform.h"
namespace phi {
template <typename InT, typename OutT>
struct CastOpTransformFunctor {
HOSTDEVICE OutT operator()(InT in) const { return static_cast<OutT>(in); }
};
template <>
struct CastOpTransformFunctor<::phi::dtype::float8_e5m2, ::phi::complex64> {
HOSTDEVICE ::phi::complex64 operator()(::phi::dtype::float8_e5m2 in) const {
return ::phi::complex64(static_cast<float>(in));
}
};
template <>
struct CastOpTransformFunctor<::phi::dtype::float8_e5m2, ::phi::complex128> {
HOSTDEVICE ::phi::complex128 operator()(::phi::dtype::float8_e5m2 in) const {
return ::phi::complex128(static_cast<double>(in));
}
};
template <>
struct CastOpTransformFunctor<::phi::dtype::float8_e4m3fn, ::phi::complex64> {
HOSTDEVICE ::phi::complex64 operator()(::phi::dtype::float8_e4m3fn in) const {
return ::phi::complex64(static_cast<float>(in));
}
};
template <>
struct CastOpTransformFunctor<::phi::dtype::float8_e4m3fn, ::phi::complex128> {
HOSTDEVICE ::phi::complex128 operator()(
::phi::dtype::float8_e4m3fn in) const {
return ::phi::complex128(static_cast<double>(in));
}
};
template <>
struct CastOpTransformFunctor<::phi::dtype::bfloat16, ::phi::complex64> {
HOSTDEVICE ::phi::complex64 operator()(::phi::dtype::bfloat16 in) const {
return ::phi::complex64(static_cast<float>(in));
}
};
template <>
struct CastOpTransformFunctor<::phi::dtype::bfloat16, ::phi::complex128> {
HOSTDEVICE ::phi::complex128 operator()(::phi::dtype::bfloat16 in) const {
return ::phi::complex128(static_cast<double>(in));
}
};
template <>
struct CastOpTransformFunctor<::phi::dtype::float16, ::phi::complex64> {
HOSTDEVICE ::phi::complex64 operator()(::phi::dtype::float16 in) const {
return ::phi::complex64(static_cast<float>(in));
}
};
template <>
struct CastOpTransformFunctor<::phi::dtype::float16, ::phi::complex128> {
HOSTDEVICE ::phi::complex128 operator()(::phi::dtype::float16 in) const {
return ::phi::complex128(static_cast<double>(in));
}
};
template <typename InT, typename OutT>
void CastKernelImpl(const CPUContext& dev_ctx,
const DenseTensor& x,
DataType out_dtype,
DenseTensor* out) {
auto* in_begin = x.data<InT>();
auto numel = x.numel();
auto* in_end = in_begin + numel;
auto* out_begin = dev_ctx.Alloc<OutT>(out);
out->set_type(out_dtype);
phi::Transform<CPUContext> trans;
trans(dev_ctx,
in_begin,
in_end,
out_begin,
CastOpTransformFunctor<InT, OutT>());
}
template <typename InT, typename OutT>
void CastInplaceKernelImpl(const CPUContext& dev_ctx,
const DenseTensor& x,
DataType out_dtype,
DenseTensor* out) {
auto numel = x.numel();
auto* in_begin = new InT[numel];
auto* in_end = in_begin + numel;
auto* data_origin = x.data<InT>();
memcpy(in_begin, data_origin, sizeof(InT) * numel);
auto* out_begin = dev_ctx.Alloc<OutT>(out);
out->set_type(out_dtype);
phi::Transform<CPUContext> trans;
trans(dev_ctx,
in_begin,
in_end,
out_begin,
CastOpTransformFunctor<InT, OutT>());
delete[] in_begin;
}
} // namespace phi
paddle/phi/kernels/cpu/cast_kernel.cc
+89
View File
@@ -0,0 +1,89 @@
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/cast_kernel.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/core/visit_type.h"
#include "paddle/phi/kernels/cpu/cast_impl.h"
namespace phi {
template <typename T, typename Context>
void CastKernel(const Context& dev_ctx,
const DenseTensor& x,
DataType out_dtype,
DenseTensor* out) {
if (x.dtype() == out_dtype) {
if (x.dims() == make_ddim({-1})) {
*out = x;
return;
}
if (!out->IsSharedWith(x)) {
Copy(dev_ctx, x, dev_ctx.GetPlace(), false, out);
}
return;
}
if (out->IsSharedWith(x)) {
PD_VISIT_ALL_CPU_TYPES(out_dtype, "CastInplaceKernelImpl", ([&] {
CastInplaceKernelImpl<T, data_t>(
dev_ctx, x, out_dtype, out);
}));
} else {
PD_VISIT_ALL_CPU_TYPES(out_dtype, "CastKernelImpl", ([&] {
CastKernelImpl<T, data_t>(
dev_ctx, x, out_dtype, out);
}));
}
}
#ifdef _WIN32
INSTANTIATE_CAST_KERNEL(float, CPUContext)
INSTANTIATE_CAST_KERNEL(double, CPUContext)
INSTANTIATE_CAST_KERNEL(int, CPUContext)
INSTANTIATE_CAST_KERNEL(int64_t, CPUContext)
INSTANTIATE_CAST_KERNEL(uint8_t, CPUContext)
INSTANTIATE_CAST_KERNEL(uint16_t, CPUContext)
INSTANTIATE_CAST_KERNEL(uint32_t, CPUContext)
INSTANTIATE_CAST_KERNEL(uint64_t, CPUContext)
INSTANTIATE_CAST_KERNEL(bool, CPUContext)
INSTANTIATE_CAST_KERNEL(int16_t, CPUContext)
INSTANTIATE_CAST_KERNEL(float16, CPUContext)
INSTANTIATE_CAST_KERNEL(bfloat16, CPUContext)
#endif
} // namespace phi
PD_REGISTER_KERNEL(cast,
CPU,
ALL_LAYOUT,
phi::CastKernel,
float,
double,
int,
int64_t,
int16_t,
bool,
int8_t,
uint8_t,
uint16_t,
uint32_t,
uint64_t,
phi::float8_e4m3fn,
phi::float8_e5m2,
phi::float16,
phi::bfloat16,
phi::complex64,
phi::complex128) {
kernel->OutputAt(0).SetDataType(phi::DataType::UNDEFINED);
}
paddle/phi/kernels/cpu/channel_shuffle_grad_kernel.cc
+26
View File
@@ -0,0 +1,26 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/channel_shuffle_grad_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/channel_shuffle_grad_kernel_impl.h"
PD_REGISTER_KERNEL(channel_shuffle_grad,
CPU,
ALL_LAYOUT,
phi::ChannelShuffleGradKernel,
float,
double) {}
paddle/phi/kernels/cpu/channel_shuffle_kernel.cc
+26
View File
@@ -0,0 +1,26 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/channel_shuffle_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/channel_shuffle_kernel_impl.h"
PD_REGISTER_KERNEL(channel_shuffle,
CPU,
ALL_LAYOUT,
phi::ChannelShuffleKernel,
float,
double) {}
paddle/phi/kernels/cpu/check_numerics_kernel.cc
+85
View File
@@ -0,0 +1,85 @@
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/phi/kernels/check_numerics_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/check_numerics_utils.h"
namespace phi {
template <typename T, typename Context>
void CheckNumericsKernel(const Context& dev_ctx,
const DenseTensor& tensor,
const std::string& op_type,
const std::string& var_name,
const int check_nan_inf_level,
const int stack_height_limit,
const std::string& output_dir,
DenseTensor* stats,
DenseTensor* values) {
// stats stores the checking result of num_nan, num_inf and num_zero.
stats->Resize({static_cast<int64_t>(3)});
int64_t* stats_ptr = dev_ctx.template Alloc<int64_t>(stats);
// values stores the max_value, min_value and mean_value.
values->Resize({static_cast<int64_t>(3)});
float* values_ptr = dev_ctx.template Alloc<float>(values);
if (tensor.numel() == 0) {
stats_ptr[0] = 0;
stats_ptr[1] = 0;
stats_ptr[2] = 0;
values_ptr[0] = static_cast<float>(0);
values_ptr[1] = static_cast<float>(0);
values_ptr[2] = static_cast<float>(0);
return;
}
std::string cpu_hint_str =
funcs::GetCpuHintString<T>(op_type, var_name, tensor.place());
funcs::CheckNumericsCpuImpl(tensor.data<T>(),
tensor.numel(),
cpu_hint_str,
check_nan_inf_level,
"cpu",
output_dir,
stats_ptr,
values_ptr);
}
#ifdef _WIN32
INSTANTIATE_CHECKNUMBERICS_KERNEL(float, CPUContext)
INSTANTIATE_CHECKNUMBERICS_KERNEL(double, CPUContext)
INSTANTIATE_CHECKNUMBERICS_KERNEL(float16, CPUContext)
INSTANTIATE_CHECKNUMBERICS_KERNEL(bfloat16, CPUContext)
INSTANTIATE_CHECKNUMBERICS_KERNEL(complex64, CPUContext)
INSTANTIATE_CHECKNUMBERICS_KERNEL(complex128, CPUContext)
INSTANTIATE_CHECKNUMBERICS_KERNEL(float8_e4m3fn, CPUContext)
INSTANTIATE_CHECKNUMBERICS_KERNEL(float8_e5m2, CPUContext)
#endif
} // namespace phi
PD_REGISTER_KERNEL(check_numerics,
CPU,
ALL_LAYOUT,
phi::CheckNumericsKernel,
float,
double,
phi::float16,
phi::bfloat16,
phi::complex64,
phi::complex128,
phi::float8_e4m3fn,
phi::float8_e5m2) {}
paddle/phi/kernels/cpu/cholesky_grad_kernel.cc
+22
View File
@@ -0,0 +1,22 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/cholesky_grad_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/cholesky_grad_kernel_impl.h"
PD_REGISTER_KERNEL(
cholesky_grad, CPU, ALL_LAYOUT, phi::CholeskyGradKernel, float, double) {}
paddle/phi/kernels/cpu/cholesky_kernel.cc
+86
View File
@@ -0,0 +1,86 @@
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/phi/kernels/cholesky_kernel.h"
#include "Eigen/Cholesky"
#include "Eigen/Core"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/eigen/common.h"
namespace phi {
template <typename T, typename Context>
void CholeskyKernel(const Context& dev_ctx,
const DenseTensor& x,
bool upper,
DenseTensor* out) {
using EigenMatrix =
Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>;
using InputMatrixMap = Eigen::Map<const EigenMatrix>;
using OutputMatrixMap = Eigen::Map<EigenMatrix>;
if (out->numel() == 0) {
dev_ctx.template Alloc<T>(out);
return;
}
auto& dims = x.dims();
int batch_count = 1;
for (int i = 0; i < dims.size() - 2; i++) {
batch_count *= static_cast<int>(dims[i]);
}
auto m = dims[dims.size() - 1];
const auto* x_data = x.data<T>();
auto* out_data = dev_ctx.template Alloc<T>(out);
// Cholesky decomposition for each matrix, maybe can use multi threads
for (int i = 0; i < batch_count; i++) {
auto input = InputMatrixMap(x_data + i * m * m, m, m);
auto output = OutputMatrixMap(out_data + i * m * m, m, m);
if (upper) {
Eigen::LLT<
Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>,
Eigen::UpLoType::Upper>
llt_decomposition(input);
PADDLE_ENFORCE_EQ(llt_decomposition.info(),
Eigen::Success,
errors::InvalidArgument(
"Cholesky decomposition was not successful. The "
"%d-th input matrice "
"might not be not be positive definite.",
i));
output = llt_decomposition.matrixU();
} else {
Eigen::LLT<
Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>,
Eigen::UpLoType::Lower>
llt_decomposition(input);
PADDLE_ENFORCE_EQ(llt_decomposition.info(),
Eigen::Success,
errors::InvalidArgument(
"Cholesky decomposition was not successful. The "
"%d-th input matrice "
"might not be not be positive definite.",
i));
output = llt_decomposition.matrixL();
}
}
}
} // namespace phi
PD_REGISTER_KERNEL(
cholesky, CPU, ALL_LAYOUT, phi::CholeskyKernel, float, double) {}
paddle/phi/kernels/cpu/cholesky_solve_grad_kernel.cc
+24
View File
@@ -0,0 +1,24 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/cholesky_solve_grad_kernel_impl.h"
PD_REGISTER_KERNEL(cholesky_solve_grad,
CPU,
ALL_LAYOUT,
phi::CholeskySolveGradKernel,
float,
double) {}
paddle/phi/kernels/cpu/cholesky_solve_kernel.cc
+41
View File
@@ -0,0 +1,41 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/lapack/lapack_function.h"
#include "paddle/phi/kernels/impl/cholesky_solve_kernel_impl.h"
namespace phi {
template <typename T>
class CholeskySolveFunctor<T, CPUContext> {
public:
void operator()(const CPUContext &dev_ctx,
bool upper,
int M,
int N,
T *Adata,
int lda,
T *Bdata,
int *devInfo) {
char uplo = upper ? 'U' : 'L';
funcs::lapackCholeskySolve<T>(uplo, M, N, Adata, lda, Bdata, lda, devInfo);
}
};
} // namespace phi
PD_REGISTER_KERNEL(
cholesky_solve, CPU, ALL_LAYOUT, phi::CholeskySolveKernel, float, double) {}
paddle/phi/kernels/cpu/chunk_eval_kernel.cc
+27
View File
@@ -0,0 +1,27 @@
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/chunk_eval_kernel.h"
#include "paddle/phi/kernels/impl/chunk_eval_kernel_impl.h"
PD_REGISTER_KERNEL(
chunk_eval, CPU, ALL_LAYOUT, phi::ChunkEvalKernel, float, int64_t) {
kernel->InputAt(0).SetDataType(phi::DataType::INT64);
kernel->InputAt(1).SetDataType(phi::DataType::INT64);
kernel->InputAt(2).SetDataType(phi::DataType::INT64);
kernel->OutputAt(3).SetDataType(phi::DataType::INT64);
kernel->OutputAt(4).SetDataType(phi::DataType::INT64);
kernel->OutputAt(5).SetDataType(phi::DataType::INT64);
}
paddle/phi/kernels/cpu/class_center_sample_kernel.cc
+129
View File
@@ -0,0 +1,129 @@
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <map>
#include <set>
#include <vector>
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/generator.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/class_center_sample_kernel.h"
namespace phi {
template <typename T, typename Context>
void ClassCenterSampleKernel(const Context& dev_ctx,
const DenseTensor& label,
int num_classes,
int num_samples,
int ring_id,
int rank,
int nranks,
bool fix_seed,
int seed,
DenseTensor* remapped_label,
DenseTensor* sampled_local_class_center) {
PADDLE_ENFORCE_GT(num_classes,
0,
errors::InvalidArgument(
"The value 'num_classes' for Op(class_center_sample) "
"must be greater than 0, "
"but the value given is %d.",
num_classes));
PADDLE_ENFORCE_GT(num_samples,
0,
errors::InvalidArgument(
"The value 'num_samples' for Op(class_center_sample) "
"must be greater than 0, "
"but the value given is %d.",
num_samples));
PADDLE_ENFORCE_LE(num_samples,
num_classes,
errors::InvalidArgument(
"The value 'num_samples' for Op(class_center_sample) "
"must be less than or equal to %d, "
"but the value given is %d.",
num_classes,
num_samples));
int64_t numel = label.numel();
auto* label_ptr = label.data<T>();
// get unique positive class center by ascending
std::set<T, std::less<T>> unique_label; // NOLINT
for (int64_t i = 0; i < numel; ++i) {
unique_label.insert(label_ptr[i]);
}
// construct a lookup table and get sampled_local_class_center
std::vector<T> actual_sampled;
std::map<T, T> new_class_dict;
T idx = 0;
for (auto& t : unique_label) {
new_class_dict[t] = idx;
actual_sampled.push_back(t);
idx++;
}
if (!fix_seed) {
std::random_device rnd;
seed = static_cast<int>(rnd());
}
std::uniform_int_distribution<T> dist(0, num_classes - 1);
std::shared_ptr<std::mt19937_64> engine;
if (seed) {
engine = std::make_shared<std::mt19937_64>();
engine->seed(seed);
} else {
engine = dev_ctx.GetGenerator()->GetCPUEngine();
}
// sample negative class center randomly
while (unique_label.size() < static_cast<size_t>(num_samples)) {
T neg = dist(*engine);
if (unique_label.find(neg) == unique_label.end()) {
unique_label.insert(neg);
// unorder for negative class center
actual_sampled.push_back(neg);
}
}
int actual_num_samples = unique_label.size();
sampled_local_class_center->Resize({actual_num_samples});
T* sampled_local_class_center_ptr =
dev_ctx.template Alloc<T>(sampled_local_class_center);
idx = 0;
for (auto& t : actual_sampled) {
sampled_local_class_center_ptr[idx] = t;
idx++;
}
// remap the input label to sampled class
auto* remapped_label_ptr = dev_ctx.template Alloc<T>(remapped_label);
for (int64_t i = 0; i < numel; ++i) {
remapped_label_ptr[i] = new_class_dict[label_ptr[i]];
}
}
} // namespace phi
PD_REGISTER_KERNEL(class_center_sample,
CPU,
ALL_LAYOUT,
phi::ClassCenterSampleKernel,
int64_t,
int) {}
paddle/phi/kernels/cpu/clip_by_norm_kernel.cc
+34
View File
@@ -0,0 +1,34 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/clip_by_norm_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/clip_by_norm_kernel_impl.h"
namespace phi {
template <typename T, typename Context>
void ClipByNormKernel(const Context& dev_ctx,
const DenseTensor& in,
float max_norm,
DenseTensor* output) {
ClipByNormFunctor<T, Context>(dev_ctx, in, max_norm, output);
}
} // namespace phi
PD_REGISTER_KERNEL(
clip_by_norm, CPU, ALL_LAYOUT, phi::ClipByNormKernel, float) {}
paddle/phi/kernels/cpu/clip_grad_kernel.cc
+28
View File
@@ -0,0 +1,28 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/clip_grad_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/clip_grad_kernel_impl.h"
PD_REGISTER_KERNEL(clip_grad,
CPU,
ALL_LAYOUT,
phi::ClipGradKernel,
float,
double,
int,
int64_t) {}
paddle/phi/kernels/cpu/clip_kernel.cc
+22
View File
@@ -0,0 +1,22 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/clip_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/clip_kernel_impl.h"
PD_REGISTER_KERNEL(
clip, CPU, ALL_LAYOUT, phi::ClipKernel, float, double, int, int64_t) {}
paddle/phi/kernels/cpu/collect_fpn_proposals_kernel.cc
+26
View File
@@ -0,0 +1,26 @@
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/collect_fpn_proposals_kernel_impl.h"
PD_REGISTER_KERNEL(collect_fpn_proposals,
CPU,
ALL_LAYOUT,
phi::CollectFpnProposalsOpKernel,
float,
double) {
kernel->InputAt(2).SetDataType(phi::DataType::INT32);
kernel->OutputAt(1).SetDataType(phi::DataType::INT32);
}
paddle/phi/kernels/cpu/compare_kernel.cc
+134
View File
@@ -0,0 +1,134 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/compare_kernel.h"
#include "paddle/phi/core/dense_tensor.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/elementwise_base.h"
#include "paddle/phi/kernels/impl/compare_kernel_impl.h"
namespace phi {
template <typename T,
typename Context,
typename Functor,
typename InverseFunctor>
inline void CompareKernelImpl(const Context& dev_ctx,
const DenseTensor& x,
const DenseTensor& y,
int axis,
DenseTensor* out) {
dev_ctx.template Alloc<bool>(out);
if (x.dims().size() >= y.dims().size()) {
funcs::ElementwiseCompute<Functor, T, bool>(
dev_ctx, x, y, Functor(), out, axis);
} else {
funcs::ElementwiseCompute<InverseFunctor, T, bool>(
dev_ctx, x, y, InverseFunctor(), out, axis);
}
}
template <typename T,
typename Context,
typename Functor,
typename InverseFunctor>
inline void InplaceCompareKernelImpl(const Context& dev_ctx,
const DenseTensor& x,
const DenseTensor& y,
int axis,
DenseTensor* out) {
auto x_origin = x;
out->set_type(DataType::BOOL);
dev_ctx.template Alloc<bool>(out);
if (x_origin.dims().size() >= y.dims().size()) {
funcs::ElementwiseCompute<Functor, T, bool>(
dev_ctx, x_origin, y, Functor(), out, axis);
} else {
funcs::ElementwiseCompute<InverseFunctor, T, bool>(
dev_ctx, x_origin, y, InverseFunctor(), out, axis);
}
}
template <typename T, typename Context, typename Functor>
inline void CompareAllKernelImpl(const Context& dev_ctx,
const DenseTensor& x,
const DenseTensor& y,
DenseTensor* out) {
bool* out_data = dev_ctx.template Alloc<bool>(out);
if (x.dims() != y.dims()) {
out_data[0] = false;
} else if (x.numel() == 0) { // shape equal and numel is 0, return true
out_data[0] = true;
} else {
DenseTensor tmp;
tmp.Resize(x.dims());
dev_ctx.template Alloc<bool>(&tmp);
if (x.numel() == 1 && y.numel() == 1) {
bool* tmp_data = tmp.data<bool>();
tmp_data[0] = Functor()(x.data<T>()[0], y.data<T>()[0]);
} else {
funcs::ElementwiseCompute<Functor, T, bool>(
dev_ctx, x, y, Functor(), &tmp, 0);
}
auto tmp_flat = EigenVector<bool>::Flatten(tmp);
auto out_es = EigenScalar<bool>::From(*out);
auto& place = *dev_ctx.eigen_device();
auto reduce_dim = Eigen::array<int, 1>({{0}});
out_es.device(place) = tmp_flat.all(reduce_dim);
}
}
} // namespace phi
PD_REGISTER_KERNEL(equal_all,
CPU,
ALL_LAYOUT,
phi::EqualAllKernel,
bool,
int,
int64_t,
float,
double) {
kernel->OutputAt(0).SetDataType(phi::DataType::BOOL);
}
#define PD_REGISTER_COMPLEX_COMPARE_KERNEL(name, func) \
PD_REGISTER_KERNEL(name, \
CPU, \
ALL_LAYOUT, \
phi::func##Kernel, \
bool, \
int, \
uint8_t, \
int8_t, \
int16_t, \
int64_t, \
phi::complex64, \
phi::complex128, \
float, \
double, \
phi::float16, \
phi::bfloat16) { \
kernel->OutputAt(0).SetDataType(phi::DataType::BOOL); \
}
PD_REGISTER_COMPLEX_COMPARE_KERNEL(less_than, LessThan)
PD_REGISTER_COMPLEX_COMPARE_KERNEL(less_equal, LessEqual)
PD_REGISTER_COMPLEX_COMPARE_KERNEL(greater_than, GreaterThan)
PD_REGISTER_COMPLEX_COMPARE_KERNEL(greater_equal, GreaterEqual)
PD_REGISTER_COMPLEX_COMPARE_KERNEL(equal, Equal)
PD_REGISTER_COMPLEX_COMPARE_KERNEL(not_equal, NotEqual)
paddle/phi/kernels/cpu/complex_grad_kernel.cc
+41
View File
@@ -0,0 +1,41 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/complex_grad_kernel.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/complex_grad_kernel_impl.h"
PD_REGISTER_KERNEL(real_grad,
CPU,
ALL_LAYOUT,
phi::RealGradKernel,
phi::complex64,
phi::complex128) {
kernel->InputAt(0).SetDataType(phi::dtype::ToReal(kernel_key.dtype()));
}
PD_REGISTER_KERNEL(imag_grad,
CPU,
ALL_LAYOUT,
phi::ImagGradKernel,
phi::complex64,
phi::complex128) {
kernel->InputAt(0).SetDataType(phi::dtype::ToReal(kernel_key.dtype()));
}
PD_REGISTER_KERNEL(
complex_grad, CPU, ALL_LAYOUT, phi::ComplexGradKernel, float, double) {
kernel->InputAt(2).SetDataType(phi::dtype::ToComplex(kernel_key.dtype()));
}
paddle/phi/kernels/cpu/complex_kernel.cc
+45
View File
@@ -0,0 +1,45 @@
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/complex_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/complex_kernel_impl.h"
PD_REGISTER_KERNEL(conj,
CPU,
ALL_LAYOUT,
phi::ConjKernel,
phi::complex64,
phi::complex128,
float,
double,
int,
int64_t) {}
PD_REGISTER_KERNEL(
real, CPU, ALL_LAYOUT, phi::RealKernel, phi::complex64, phi::complex128) {
kernel->OutputAt(0).SetDataType(phi::dtype::ToReal(kernel_key.dtype()));
}
PD_REGISTER_KERNEL(
imag, CPU, ALL_LAYOUT, phi::ImagKernel, phi::complex64, phi::complex128) {
kernel->OutputAt(0).SetDataType(phi::dtype::ToReal(kernel_key.dtype()));
}
PD_REGISTER_KERNEL(
complex, CPU, ALL_LAYOUT, phi::ComplexKernel, float, double) {
kernel->OutputAt(0).SetDataType(phi::dtype::ToComplex(kernel_key.dtype()));
}
paddle/phi/kernels/cpu/concat_grad_kernel.cc
+37
View File
@@ -0,0 +1,37 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/concat_grad_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/concat_grad_kernel_impl.h"
PD_REGISTER_KERNEL(concat_grad,
CPU,
ALL_LAYOUT,
phi::ConcatGradKernel,
double,
float,
bool,
int64_t,
int,
int8_t,
int16_t,
uint8_t,
phi::float16,
phi::float8_e4m3fn,
phi::float8_e5m2,
phi::complex64,
phi::complex128) {}
paddle/phi/kernels/cpu/concat_kernel.cc
+134
View File
@@ -0,0 +1,134 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/concat_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/common/scalar.h"
#include "paddle/phi/core/dense_tensor.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/core/lod_utils.h"
#include "paddle/phi/kernels/funcs/concat_and_split_functor.h"
#include "paddle/phi/kernels/funcs/concat_funcs.h"
#include "paddle/phi/kernels/funcs/strided_memcpy.h"
namespace phi {
template <typename T, typename Context>
void ConcatKernel(const Context& dev_ctx,
const std::vector<const DenseTensor*>& x,
const Scalar& axis_scalar,
DenseTensor* out) {
int64_t axis = axis_scalar.to<int64_t>();
axis = funcs::ComputeAxis(axis, x[0]->dims().size());
std::vector<DDim> x_dims;
x_dims.reserve(x.size());
for (auto item : x) {
x_dims.push_back(item->dims());
}
DDim out_dims = funcs::ComputeAndCheckShape(true, x_dims, axis);
out->Resize(out_dims);
dev_ctx.template Alloc<T>(out);
if (out->numel() == 0) {
return;
}
// If axis is 0, the lod of the output is not the same as inputs.
if (axis == 0 && !x[0]->lod().empty()) {
size_t lod_size_0 = x[0]->lod().size();
size_t lod_size = lod_size_0;
for (size_t i = 1; i < x.size(); ++i) {
if (!x[i]->lod().empty()) {
PADDLE_ENFORCE_EQ(
x[i]->lod().size(),
lod_size_0,
common::errors::Unimplemented(
"The lod level of all input DenseTensors should be same. "
"Maybe different lod level of input DenseTensors can concat,"
"it is not supported currently. The lod level of %dth input "
"is %d and first input is %d.",
i,
x[i]->lod().size(),
lod_size_0));
} else {
lod_size = 0;
break;
}
}
if (lod_size) {
auto* out_lod = out->mutable_lod();
for (size_t i = 1; i < x.size(); ++i) {
auto in_lod = ConvertToLengthBasedLegacyLoD(x[i]->lod());
AppendLegacyLoD(out_lod, in_lod);
}
}
}
// Sometimes direct copies will be faster, this maybe need deeply analysis.
if (axis == 0 && x.size() < 10) {
size_t output_offset = 0;
for (const auto* in : x) {
if (in->numel() == 0UL) {
continue;
}
auto in_stride = common::stride_numel(in->dims());
auto out_stride = common::stride_numel(out->dims());
funcs::StridedNumelCopyWithAxis<T, Context>(
dev_ctx,
axis,
out->data<T>() + output_offset,
out_stride,
in->data<T>(),
in_stride,
in_stride[static_cast<int>(axis)]);
output_offset += in_stride[static_cast<int>(axis)];
}
} else {
// TODO(chenweihang): concat functor support vector<DenseTensor*> input
std::vector<DenseTensor> inputs;
inputs.reserve(x.size());
for (auto item : x) {
if (item->numel() > 0) {
inputs.emplace_back(*item);
} else {
continue;
}
}
funcs::ConcatFunctor<Context, T> functor;
functor(dev_ctx, inputs, axis, out);
}
}
} // namespace phi
PD_REGISTER_KERNEL(concat,
CPU,
ALL_LAYOUT,
phi::ConcatKernel,
float,
double,
bool,
int64_t,
int,
uint8_t,
int8_t,
int16_t,
phi::float16,
phi::bfloat16,
phi::float8_e4m3fn,
phi::float8_e5m2,
phi::complex64,
phi::complex128) {}
paddle/phi/kernels/cpu/contiguous_kernel.cc
+303
View File
@@ -0,0 +1,303 @@
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/phi/kernels/contiguous_kernel.h"
#include <set>
#include <vector>
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/dense_tensor_iterator.h"
#include "paddle/phi/kernels/funcs/math_function.h"
#include "paddle/phi/kernels/impl/transpose_grad_kernel_impl.h"
#if defined(PADDLE_WITH_OPENMP)
#include <omp.h>
#endif
namespace phi {
inline int64_t DivUp(const int64_t& x, const int64_t& y) {
return (x + y - 1) / y;
}
inline void DealWithStride(const DenseTensorIterator& iter, int64_t* strides) {
for (int dim = 0; dim < iter.ndim(); dim++) {
for (int arg = 0; arg < iter.ntensors(); arg++) {
*strides++ = iter.strides(arg)[dim];
}
}
if (iter.ndim() < 2) {
std::fill_n(strides, (2 - iter.ndim()) * iter.ntensors(), 0);
}
}
template <typename T>
inline void FallbackContiguous(const DDim& input_dims,
const DDim& input_stride,
const int64_t numel,
const T* input_data,
T* output_data) {
int rank = input_dims.size();
auto dims = input_dims;
for (int64_t i = 0; i < numel; i++) {
int64_t input_offset = 0;
int64_t index_tmp = i;
for (int dim = rank - 1; dim >= 0; --dim) {
int64_t mod = index_tmp % dims[dim];
index_tmp = index_tmp / dims[dim];
input_offset += mod * input_stride[dim];
}
output_data[i] = input_data[input_offset];
}
}
inline bool OnlyTransposed(const DDim& shape,
const DDim& stride,
const uint64_t& offset) {
if (offset != 0) {
return false;
}
DDim x_stride = stride;
DDim x_shape = shape;
std::set<int> visited_idx;
for (int i = 0; i < stride.size(); i++) {
int64_t max_num = 0;
int max_idx = -1;
for (int j = 0; j < stride.size(); j++) {
if (visited_idx.count(j)) {
continue;
}
if (stride[j] < 1) {
return false;
}
if (stride[j] > max_num) {
max_num = stride[j];
max_idx = j;
}
}
if (max_idx == -1) {
return false;
}
if (i != 0 && x_stride[i - 1] == max_num) {
return false;
}
visited_idx.insert(max_idx);
x_stride[i] = max_num;
x_shape[i] = shape[max_idx];
}
if (DenseTensorMeta::calc_strides(x_shape) == x_stride) {
return true;
} else {
return false;
}
}
inline bool FastContiguousJudge(const std::vector<int64_t>& coalesce_shape,
const DenseTensor& input) {
if (coalesce_shape.size() > 3 || input.dims().size() == 0) return false;
auto x_meta = input.meta();
if (coalesce_shape.size() == 3 &&
!OnlyTransposed(x_meta.dims, x_meta.strides, x_meta.offset))
return false;
return true;
}
inline bool FastTransposeCopyValid(const DenseTensor& self,
const DenseTensor& src) {
constexpr int64_t MIN_NUMEL = 360;
return src.numel() != 0 && src.dims().size() == 2 && src.strides()[0] == 1 &&
src.strides()[1] == src.dims()[0] &&
self.dims().size() == src.dims().size() && self.numel() >= MIN_NUMEL;
}
template <typename T, typename Context>
void ContiguousKernel(const Context& dev_ctx,
const DenseTensor& input,
DenseTensor* out) {
DenseTensorMeta meta = input.meta();
meta.strides = meta.calc_strides(meta.dims);
meta.offset = 0;
out->set_meta(meta);
const T* input_data = input.data<T>();
T* output_data = dev_ctx.template Alloc<T>(out);
auto numel = input.numel();
if (numel == 0) {
return;
}
if (IsComplexType(input.dtype())) {
FallbackContiguous<T>(
input.dims(), input.strides(), numel, input_data, output_data);
return;
}
#if defined(_WIN32)
FallbackContiguous<T>(
input.dims(), input.strides(), numel, input_data, output_data);
return;
#else
if (FastTransposeCopyValid(*out, input)) {
constexpr int64_t TRANS_NUMEL = 60;
void* trans_buffer =
malloc(SizeOf(input.dtype()) * TRANS_NUMEL * TRANS_NUMEL);
const T* tmp_src_ptr = input_data;
T* tmp_out_ptr = output_data;
T* tmp_buf_ptr = reinterpret_cast<T*>(trans_buffer);
int64_t dim0 = out->dims()[0];
int64_t dim1 = out->dims()[1];
for (int64_t d0 = 0; d0 < dim0; d0 += TRANS_NUMEL) {
for (int64_t d1 = 0; d1 < dim1; d1 += TRANS_NUMEL) {
const T* src_ptr_inter = tmp_src_ptr + d0 + d1 * dim0;
T* out_ptr_inter = tmp_out_ptr + d1 + d0 * dim1;
int nr = std::min(dim0 - d0, TRANS_NUMEL);
int nc = std::min(dim1 - d1, TRANS_NUMEL);
for (int c = 0; c < nc; c++) {
memcpy(tmp_buf_ptr + c * TRANS_NUMEL,
src_ptr_inter + c * dim0,
nr * sizeof(T));
}
int rc_max = std::max(nr, nc);
int rc_min = std::min(nr, nc);
for (int r = 0; r < rc_max; r++) {
int end = std::min(r, rc_min);
for (int c = 0; c < end; c++) {
T tmp = tmp_buf_ptr[r + TRANS_NUMEL * c];
tmp_buf_ptr[r + TRANS_NUMEL * c] = tmp_buf_ptr[r * TRANS_NUMEL + c];
tmp_buf_ptr[r * TRANS_NUMEL + c] = tmp;
}
}
for (int r = 0; r < nr; r++) {
memcpy(out_ptr_inter + r * dim1,
tmp_buf_ptr + r * TRANS_NUMEL,
nc * sizeof(T));
}
}
}
free(trans_buffer);
} else {
#if defined(PADDLE_WITH_OPENMP)
DenseTensorIteratorConfig config;
config.add_output(*out);
config.add_const_input(input);
config.is_alloc_out_ = true;
DenseTensorIterator iter = config.build();
if (!FastContiguousJudge(iter.shape(), input)) {
FallbackContiguous<T>(
input.dims(), input.strides(), numel, input_data, output_data);
return;
}
std::vector<int64_t> tmp_strides(
iter.ntensors() * static_cast<size_t>(std::max(iter.ndim(), 2)));
DealWithStride(iter, tmp_strides.data());
std::vector<int64_t> out_stride(tmp_strides.begin() + iter.ntensors(),
tmp_strides.end());
const int64_t& iter_numel = iter.numel();
const char* in_ptr = reinterpret_cast<const char*>(input_data);
char* out_ptr = reinterpret_cast<char*>(output_data);
int64_t end = iter_numel;
int64_t begin = 0;
int64_t grain_size = 32768;
int64_t* whole_stride = tmp_strides.data();
#pragma omp parallel
{
int64_t num_threads = omp_get_num_threads();
if (grain_size > 0) {
num_threads = std::min(num_threads, DivUp((end - begin), grain_size));
}
int64_t tid = omp_get_thread_num();
int64_t chunk_size = DivUp((end - begin), num_threads);
int64_t begin_tid = begin + tid * chunk_size;
if (begin_tid < end) {
int64_t range_start = begin_tid;
int64_t range_end = std::min(end, chunk_size + begin_tid);
auto dimiter = DimIter(iter.shape(), range_start, range_end);
while (!dimiter.iter_to_end()) {
const auto v_ndim = dimiter.values.size();
const char* tmp_in_data = in_ptr;
char* tmp_out_data = out_ptr;
for (size_t dim = 0; dim < v_ndim; dim++) {
int64_t value = dimiter.values[dim];
tmp_out_data += value * whole_stride[dim * iter.ntensors() + 0];
tmp_in_data += value * whole_stride[dim * iter.ntensors() + 1];
}
auto step = dimiter.iter_for_step();
for (int64_t i = 0; i < step[1]; i++) {
for (int64_t j = 0; j < step[0]; j++) {
const char* real_in_ptr = tmp_in_data + j * whole_stride[1];
char* real_out_ptr = tmp_out_data + j * whole_stride[0];
*reinterpret_cast<T*>(real_out_ptr) =
*reinterpret_cast<const T*>(real_in_ptr);
}
tmp_in_data = tmp_in_data + out_stride[1];
tmp_out_data = tmp_out_data + out_stride[0];
}
dimiter.iter_to_next(step);
}
}
}
#else
FallbackContiguous<T>(
input.dims(), input.strides(), numel, input_data, output_data);
#endif
}
#endif
}
} // namespace phi
PD_REGISTER_KERNEL(contiguous,
CPU,
ALL_LAYOUT,
phi::ContiguousKernel,
bool,
uint8_t,
uint16_t,
uint32_t,
uint64_t,
int8_t,
int16_t,
int32_t,
int64_t,
float,
double,
phi::float16,
phi::bfloat16,
phi::complex64,
phi::complex128,
phi::float8_e4m3fn,
phi::float8_e5m2) {}
paddle/phi/kernels/cpu/conv_grad_kernel.cc
+137
View File
@@ -0,0 +1,137 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/conv_grad_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/conv_grad_kernel_impl.h"
namespace phi {
template <typename T, typename Context>
void DepthwiseConvGradKernel(const Context& dev_ctx,
const DenseTensor& input,
const DenseTensor& filter,
const DenseTensor& out_grad,
const std::vector<int>& strides,
const std::vector<int>& paddings,
const std::string& padding_algorithm,
int groups,
const std::vector<int>& dilations,
const std::string& data_format,
DenseTensor* input_grad,
DenseTensor* filter_grad) {
ConvGradKernel<T>(dev_ctx,
input,
filter,
out_grad,
strides,
paddings,
padding_algorithm,
dilations,
groups,
data_format,
input_grad,
filter_grad);
}
template <typename T, typename Context>
void Conv3DGradKernel(const Context& dev_ctx,
const DenseTensor& input,
const DenseTensor& filter,
const DenseTensor& out_grad,
const std::vector<int>& strides,
const std::vector<int>& paddings,
const std::string& padding_algorithm,
int groups,
const std::vector<int>& dilations,
const std::string& data_format,
DenseTensor* input_grad,
DenseTensor* filter_grad) {
ConvGradKernel<T>(dev_ctx,
input,
filter,
out_grad,
strides,
paddings,
padding_algorithm,
dilations,
groups,
data_format,
input_grad,
filter_grad);
}
template <typename T, typename Context>
void Conv3DDoubleGradKernel(const Context& dev_ctx,
const DenseTensor& input,
const DenseTensor& filter,
const DenseTensor& out_grad,
const optional<DenseTensor>& input_grad_grad,
const optional<DenseTensor>& filter_grad_grad,
const std::vector<int>& strides,
const std::vector<int>& paddings_t,
const std::string& padding_algorithm,
int groups,
const std::vector<int>& dilations_t,
const std::string& data_format,
DenseTensor* input_grad,
DenseTensor* filter_grad,
DenseTensor* out_grad_grad) {
ConvGradGradKernel<T>(dev_ctx,
input,
filter,
out_grad,
input_grad_grad,
filter_grad_grad,
strides,
paddings_t,
padding_algorithm,
dilations_t,
groups,
data_format,
input_grad,
filter_grad,
out_grad_grad);
}
} // namespace phi
PD_REGISTER_KERNEL(
conv2d_grad, CPU, ALL_LAYOUT, phi::ConvGradKernel, float, double) {}
PD_REGISTER_KERNEL(depthwise_conv2d_grad,
CPU,
ALL_LAYOUT,
phi::DepthwiseConvGradKernel,
float,
double) {}
PD_REGISTER_KERNEL(
conv3d_grad, CPU, ALL_LAYOUT, phi::Conv3DGradKernel, float, double) {}
PD_REGISTER_KERNEL(conv2d_double_grad,
CPU,
ALL_LAYOUT,
phi::ConvGradGradKernel,
float,
double) {}
PD_REGISTER_KERNEL(conv3d_double_grad,
CPU,
ALL_LAYOUT,
phi::Conv3DDoubleGradKernel,
float,
double) {}
paddle/phi/kernels/cpu/conv_kernel.cc
+103
View File
@@ -0,0 +1,103 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/conv_kernel.h"
#include "paddle/phi/backends/cpu/cpu_context.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/conv_kernel_impl.h"
namespace phi {
template <typename T, typename Context>
void ConvKernel(const Context& dev_ctx,
const DenseTensor& input,
const DenseTensor& filter,
const std::vector<int>& strides,
const std::vector<int>& paddings,
const std::string& padding_algorithm,
const std::vector<int>& dilations,
int groups,
const std::string& data_format,
DenseTensor* out) {
ConvKernelImpl<T>(dev_ctx,
input,
filter,
strides,
paddings,
padding_algorithm,
groups,
dilations,
data_format,
out);
}
template <typename T, typename Context>
void DepthwiseConvKernel(const Context& dev_ctx,
const DenseTensor& input,
const DenseTensor& filter,
const std::vector<int>& strides,
const std::vector<int>& paddings,
const std::string& padding_algorithm,
int groups,
const std::vector<int>& dilations,
const std::string& data_format,
DenseTensor* out) {
ConvKernelImpl<T>(dev_ctx,
input,
filter,
strides,
paddings,
padding_algorithm,
groups,
dilations,
data_format,
out);
}
template <typename T, typename Context>
void Conv3DKernel(const Context& dev_ctx,
const DenseTensor& input,
const DenseTensor& filter,
const std::vector<int>& strides,
const std::vector<int>& paddings,
const std::string& padding_algorithm,
int groups,
const std::vector<int>& dilations,
const std::string& data_format,
DenseTensor* out) {
ConvKernelImpl<T>(dev_ctx,
input,
filter,
strides,
paddings,
padding_algorithm,
groups,
dilations,
data_format,
out);
}
} // namespace phi
PD_REGISTER_KERNEL(conv2d, CPU, ALL_LAYOUT, phi::ConvKernel, float, double) {}
PD_REGISTER_KERNEL(depthwise_conv2d,
CPU,
ALL_LAYOUT,
phi::DepthwiseConvKernel,
float,
double) {}
PD_REGISTER_KERNEL(conv3d, CPU, ALL_LAYOUT, phi::Conv3DKernel, float, double) {}
paddle/phi/kernels/cpu/conv_transpose_grad_kernel.cc
+70
View File
@@ -0,0 +1,70 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/conv_transpose_grad_kernel.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/conv_transpose_grad_kernel_impl.h"
namespace phi {
template <typename T, typename Context>
void DepthwiseConv2dTransposeGradKernel(const Context& dev_ctx,
const DenseTensor& x,
const DenseTensor& filter,
const DenseTensor& dout,
const std::vector<int>& strides,
const std::vector<int>& paddings,
const std::vector<int>& output_padding,
const IntArray& output_size,
const std::string& padding_algorithm,
int groups,
const std::vector<int>& dilations,
const std::string& data_format,
DenseTensor* dx,
DenseTensor* dfilter) {
ConvTransposeGradRawKernel<T, Context>(dev_ctx,
x,
filter,
dout,
strides,
paddings,
padding_algorithm,
groups,
dilations,
data_format,
dx,
dfilter);
}
} // namespace phi
PD_REGISTER_KERNEL(conv2d_transpose_grad,
CPU,
ALL_LAYOUT,
phi::Conv2dTransposeGradKernel,
float,
double) {}
PD_REGISTER_KERNEL(conv3d_transpose_grad,
CPU,
ALL_LAYOUT,
phi::Conv3dTransposeGradKernel,
float,
double) {}
PD_REGISTER_KERNEL(depthwise_conv2d_transpose_grad,
CPU,
ALL_LAYOUT,
phi::DepthwiseConv2dTransposeGradKernel,
float,
double) {}
paddle/phi/kernels/cpu/conv_transpose_kernel.cc
+66
View File
@@ -0,0 +1,66 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/conv_transpose_kernel.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/impl/conv_transpose_kernel_impl.h"
namespace phi {
template <typename T, typename Context>
void DepthwiseConv2dTransposeKernel(const Context& dev_ctx,
const DenseTensor& x,
const DenseTensor& filter,
const std::vector<int>& strides,
const std::vector<int>& paddings,
const std::vector<int>& output_padding,
const IntArray& output_size,
const std::string& padding_algorithm,
int groups,
const std::vector<int>& dilations,
const std::string& data_format,
DenseTensor* out) {
ConvTransposeRawKernel<T, Context>(dev_ctx,
x,
filter,
strides,
paddings,
padding_algorithm,
groups,
dilations,
data_format,
out);
}
} // namespace phi
PD_REGISTER_KERNEL(conv2d_transpose,
CPU,
ALL_LAYOUT,
phi::Conv2dTransposeKernel,
float,
double) {}
PD_REGISTER_KERNEL(conv3d_transpose,
CPU,
ALL_LAYOUT,
phi::Conv3dTransposeKernel,
float,
double) {}
PD_REGISTER_KERNEL(depthwise_conv2d_transpose,
CPU,
ALL_LAYOUT,
phi::DepthwiseConv2dTransposeKernel,
float,
double) {}
paddle/phi/kernels/cpu/conv_util.h
+297
View File
@@ -0,0 +1,297 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "paddle/common/ddim.h"
#include "paddle/phi/core/meta_tensor.h"
namespace phi {
template <typename T = int>
inline void UpdatePaddingAndDilation(std::vector<T>* paddings,
std::vector<T>* dilation,
const std::string padding_algorithm,
const DDim data_dims,
const std::vector<T>& strides,
const std::vector<T>& ksize) {
// set padding size == data_dims.size() * 2
auto data_shape = vectorize<T>(data_dims);
if (static_cast<int>(paddings->size()) == data_dims.size()) {
for (int i = 0; i < data_dims.size(); ++i) {
T copy_pad = *(paddings->begin() + 2 * i);
paddings->insert(paddings->begin() + 2 * i + 1, copy_pad);
}
} else {
PADDLE_ENFORCE_EQ(
data_dims.size() * 2,
paddings->size(),
common::errors::InvalidArgument(
"Attribute padding's size should be the same or twice as the "
"input's dimension. "
"But received: padding's size is %d, padding is [%s]; input's "
"dimension is %d, input's shape is [%s].",
paddings->size(),
make_ddim(*paddings),
data_dims.size(),
data_dims));
}
// when padding_algorithm is "VALID" or "SAME"
if (padding_algorithm == "SAME") {
for (int i = 0; i < data_dims.size(); ++i) {
T out_size = (data_dims[i] + strides[i] - 1) / strides[i];
T pad_sum =
std::max((out_size - 1) * strides[i] + ksize[i] - data_shape[i],
static_cast<T>(0));
T pad_0 = pad_sum / 2;
T pad_1 = pad_sum - pad_0;
*(paddings->begin() + i * 2) = pad_0;
*(paddings->begin() + i * 2 + 1) = pad_1;
// dilation
*(dilation->begin() + i) = 1;
}
} else if (padding_algorithm == "VALID") {
for (auto it = paddings->begin(); it != paddings->end(); it++) {
*it = 0;
}
}
}
inline int ConvOutSize(int input_size,
int filter_size,
int dilation,
int pad_left,
int pad_right,
int stride) {
const int64_t dkernel =
static_cast<int64_t>(dilation) * (filter_size - 1) + 1;
const int64_t output_size =
(static_cast<int64_t>(input_size) + pad_left + pad_right - dkernel) /
stride +
1;
PADDLE_ENFORCE_GT(
output_size,
0,
common::errors::InvalidArgument(
"The output's size is expected to be greater than 0. "
"But received: output's size is %ld. The output's size is "
"computed by ((input_size + pad_left + pad_right - "
"(dilation * (filter_size - 1) + 1)) / stride + 1), where "
"input_size is %d, padding is (%d, %d), filter_size is %d, "
"dilation is %d, stride is %d.",
output_size,
input_size,
pad_left,
pad_right,
filter_size,
dilation,
stride));
PADDLE_ENFORCE_LE_INT_MAX(output_size, "conv output size");
return static_cast<int>(output_size);
}
inline std::vector<int64_t> ComputeOutputShape(
const MetaTensor& input,
const MetaTensor& filter,
const MetaTensor& bias,
const std::vector<int>& strides,
const std::vector<int>& paddings,
const std::string& padding_algorithm,
const std::vector<int>& dilations,
int groups,
const std::string& data_format,
bool channel_last,
MetaConfig config) {
auto in_dims = input.dims();
auto filter_dims = filter.dims();
int dilation_size = dilations.size();
for (int i = 0; i < dilation_size; ++i) {
PADDLE_ENFORCE_GT(
dilations[i],
0,
common::errors::InvalidArgument(
"The dilation of Op(Conv) should be larger than 0, but received "
"dilation is %d.",
dilations[i]));
}
PADDLE_ENFORCE_EQ(
in_dims.size() == 4 || in_dims.size() == 5,
true,
common::errors::InvalidArgument(
"The input of Op(Conv) should be a 4-D or 5-D Tensor. But "
"received: input's dimension is %u, input's shape is [%s].",
in_dims.size(),
in_dims));
PADDLE_ENFORCE_EQ(
in_dims.size(),
filter_dims.size(),
common::errors::InvalidArgument(
"The input's dimension and filter's dimension of "
"Op(Conv) should be equal. But received: the input's shape is "
"[%s], "
"the input's dimension is %d; the filter's shape is [%s], "
"the filter's dimension is %d.",
in_dims,
in_dims.size(),
filter_dims,
filter_dims.size()));
int stride_size = strides.size();
for (int i = 0; i < stride_size; ++i) {
PADDLE_ENFORCE_GT(
strides[i],
0,
common::errors::InvalidArgument(
"The stride of Op(Conv) should be larger than 0, but received "
"stride is %d.",
strides[i]));
}
PADDLE_ENFORCE_EQ(
in_dims.size(),
strides.size() + 2U,
common::errors::InvalidArgument(
"The difference of input's dimension and Attr(strides)'s "
"length must be equal to 2 for Op(Conv). "
"But received: input's dimension is %d, input's shape is [%s]; "
"Attr(stride)'s length is %d, Attr(stride) is [%s]; "
"difference of input's dimension and Attr(strides)'s length = %u.",
in_dims.size(),
in_dims,
strides.size(),
make_ddim(strides),
in_dims.size() - stride_size));
const auto input_channels =
channel_last ? in_dims[in_dims.size() - 1] : in_dims[1];
if (config.is_runtime) {
PADDLE_ENFORCE_EQ(
input_channels,
(channel_last ? filter_dims[filter_dims.size() - 1] : filter_dims[1]) *
groups,
common::errors::InvalidArgument(
"The number of input's channels should be equal to filter's "
"channels "
"* groups for Op(Conv). But received: the input's channels is %d, "
"the input's shape is [%s]; the filter's channels is %d, the "
"filter's shape is [%s]; the groups is %d, the data_format is %s. "
"The error may come from wrong data_format setting.",
input_channels,
in_dims,
channel_last ? filter_dims[filter_dims.size() - 1] : filter_dims[1],
filter_dims,
groups,
data_format));
PADDLE_ENFORCE_EQ(
filter_dims[0] % groups,
0,
common::errors::InvalidArgument(
"The number of output's channels (filter's first dimension) of "
"Op(Conv) should be divided by groups. But received: "
"the output channels is %d, the filter's shape is [%s], "
"the groups is %d.",
filter_dims[0],
filter_dims,
groups));
PADDLE_ENFORCE_GT(
filter_dims[0],
0,
common::errors::InvalidArgument(
"the size of filter at axis 0 should be greater than 0"));
}
DDim in_data_dims;
if (channel_last) {
in_data_dims = slice_ddim(in_dims, 1, in_dims.size() - 1);
} else {
in_data_dims = slice_ddim(in_dims, 2, in_dims.size());
}
DDim filter_data_dims;
if (channel_last) {
filter_data_dims = slice_ddim(filter_dims, 1, filter_dims.size() - 1);
} else {
filter_data_dims = slice_ddim(filter_dims, 2, filter_dims.size());
}
std::vector<int64_t> ksize = vectorize<int64_t>(filter_data_dims);
std::vector<int64_t> paddings_vec(paddings.begin(), paddings.end());
std::vector<int64_t> dilations_vec(dilations.begin(), dilations.end());
std::vector<int64_t> strides_vec(strides.begin(), strides.end());
phi::UpdatePaddingAndDilation(&paddings_vec,
&dilations_vec,
padding_algorithm,
in_data_dims,
strides_vec,
ksize);
std::vector<int64_t> output_shape({in_dims[0]});
if (!channel_last) {
output_shape.push_back(filter_dims[0]);
}
for (int i = 0; i < in_data_dims.size(); ++i) {
if (!config.is_runtime &&
(in_data_dims[i] <= 0 || filter_dims[i + 2] <= 0)) {
output_shape.push_back(-1);
} else {
PADDLE_ENFORCE_LE_INT_MAX(in_data_dims[i], "conv input size");
PADDLE_ENFORCE_LE_INT_MAX(filter_data_dims[i], "conv filter size");
PADDLE_ENFORCE_LE_INT_MAX(paddings_vec[2 * i], "conv padding left");
PADDLE_ENFORCE_LE_INT_MAX(paddings_vec[2 * i + 1], "conv padding right");
output_shape.push_back(
ConvOutSize(static_cast<int>(in_data_dims[i]),
static_cast<int>(filter_data_dims[i]),
static_cast<int>(dilations_vec[i]),
static_cast<int>(paddings_vec[2 * i]),
static_cast<int>(paddings_vec[2 * i + 1]),
static_cast<int>(strides_vec[i])));
}
}
if (channel_last) {
output_shape.push_back(filter_dims[0]);
}
return output_shape;
}
inline bool IsExpand(const std::vector<int64_t>& filter_dim,
const std::vector<int>& strides,
const std::vector<int>& paddings,
const std::vector<int>& dilations) {
bool filter_1 = true, strides_1 = true, padding_0 = true, dilation_1 = true;
for (size_t j = 0; j < strides.size(); ++j) {
filter_1 = filter_1 && (filter_dim[j + 2] == 1);
strides_1 = strides_1 && (strides[j] == 1);
padding_0 = padding_0 && (paddings[j] == 0);
dilation_1 = dilation_1 && (dilations[j] == 1);
}
if (paddings.size() != strides.size()) {
for (size_t j = 0; j < paddings.size(); ++j) {
padding_0 = padding_0 && (paddings[j] == 0);
}
}
return !(filter_1 && strides_1 && padding_0 && dilation_1);
}
} // namespace phi
paddle/phi/kernels/cpu/correlation_kernel.cc
+47
View File
@@ -0,0 +1,47 @@
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/correlation_funcs.h"
namespace phi {
template <typename T, typename Context>
void CorrelationKernel(const Context& dev_ctx,
const DenseTensor& input1,
const DenseTensor& input2,
int pad_size,
int kernel_size,
int max_displacement,
int stride1,
int stride2,
int corr_type_multiply,
DenseTensor* out) {
bool is_gpu_place = (dev_ctx.GetPlace().GetType() == AllocationType::GPU) ||
(dev_ctx.GetPlace().GetType() == AllocationType::CUSTOM);
PADDLE_ENFORCE_EQ(
is_gpu_place,
true,
common::errors::Unimplemented("Correlation only supports GPU now."));
}
} // namespace phi
PD_REGISTER_KERNEL(
correlation, CPU, ALL_LAYOUT, phi::CorrelationKernel, float, double) {}
paddle/phi/kernels/cpu/crf_decoding_kernel.cc
+165
View File
@@ -0,0 +1,165 @@
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/phi/kernels/crf_decoding_kernel.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/funcs/eigen/common.h"
#include "paddle/phi/kernels/funcs/jit/kernels.h"
#include "paddle/phi/kernels/funcs/math_function.h"
namespace phi {
template <typename T, typename Context>
void Decode(const Context& dev_ctx,
const DenseTensor& emission_weights,
const DenseTensor& transition_weights,
DenseTensor* decoded_path) {
auto emission_dims = emission_weights.dims();
const size_t seq_len = emission_dims[0];
const size_t tag_num = emission_dims[1];
const T* x = emission_weights.data<T>();
const T* w = transition_weights.data<T>();
int64_t* path = decoded_path->data<int64_t>();
// alpha is a memo table. An element alpha(k, v) records the score of the
// best sequence of tags from position 1 to position k with v being the end
// tag.
DenseTensor alpha;
alpha.Resize(emission_dims);
T* alpha_value = dev_ctx.template Alloc<T>(&alpha);
DenseTensor track;
track.Resize(emission_dims);
int* track_value = dev_ctx.template Alloc<int>(&track);
auto ker =
jit::KernelFuncs<jit::CRFDecodingTuple<T>, CPUPlace>::Cache().At(tag_num);
ker(static_cast<int>(seq_len), x, w, alpha_value, track_value, tag_num);
T max_score = -std::numeric_limits<T>::max();
int max_i = 0;
for (size_t i = 0; i < tag_num; ++i) {
T score = alpha_value[(seq_len - 1) * tag_num + i] + w[tag_num + i];
if (score > max_score) {
max_score = score;
max_i = i;
}
}
path[seq_len - 1] = max_i;
for (int k = seq_len - 1; k >= 1; --k) {
path[k - 1] = max_i = track_value[k * tag_num + max_i];
}
}
// Slice() needs to be used in *.cc files, otherwise there is a error in
// test/custom_runtime/extension_header_test.cc
template <typename T, typename Context>
void CRFDecodingOpKernel(const Context& dev_ctx,
const DenseTensor& emission,
const DenseTensor& transition,
const optional<DenseTensor>& label,
const optional<DenseTensor>& length,
DenseTensor* viterbi_path) {
auto* emission_weights = &emission;
auto* transition_weights = &transition;
auto* label_p = label.get_ptr();
auto* decoded_path = viterbi_path;
int64_t* path = dev_ctx.template Alloc<int64_t>(decoded_path);
funcs::SetConstant<Context, int64_t>()(dev_ctx, decoded_path, 0);
bool has_length = length.get_ptr() != nullptr;
if (has_length) {
auto* length_p = length.get_ptr();
const size_t seq_num = length_p->numel();
const int64_t* length_data = length_p->data<int64_t>();
auto in_dims = emission_weights->dims();
DenseTensor emission_weights_tmp = *emission_weights;
emission_weights_tmp.Resize(
make_ddim({in_dims[0] * in_dims[1], in_dims[2]}));
decoded_path->Resize({in_dims[0] * in_dims[1], 1});
for (size_t i = 0; i < seq_num; ++i) {
if (length_data[i] == 0) continue;
int64_t start_pos = i * in_dims[1];
int64_t end_pos = start_pos + static_cast<int64_t>(length_data[i]);
DenseTensor decoded_path_one_seq =
decoded_path->Slice(start_pos, end_pos);
Decode<T, Context>(dev_ctx,
emission_weights_tmp.Slice(start_pos, end_pos),
*transition_weights,
&decoded_path_one_seq);
}
decoded_path->Resize({in_dims[0], in_dims[1]});
if (label) {
const int64_t* label_value = label_p->data<int64_t>();
for (size_t i = 0; i < seq_num; ++i) {
for (int64_t j = 0; j < in_dims[1]; ++j) {
int64_t start_pos = i * in_dims[1];
if (j < length_data[i]) {
path[start_pos + j] =
label_value[start_pos + j] == path[start_pos + j] ? 1 : 0;
} else {
path[start_pos + j] = 0;
}
}
}
}
} else {
PADDLE_ENFORCE_EQ(emission_weights->NumLevels(),
1UL,
common::errors::InvalidArgument(
"The Input(Emission) should be a sequence with lod "
"level 1. But received: lod level %u.",
emission_weights->NumLevels()));
auto lod = emission_weights->lod();
PADDLE_ENFORCE_GT(
lod.size(),
0,
common::errors::InvalidArgument(
"Input(Emission) must be a sequence. But received: lod level %u.",
lod.size()));
const size_t level = 0;
const size_t seq_num = lod[level].size() - 1;
for (size_t i = 0; i < seq_num; ++i) {
if (lod[level][i] == lod[level][i + 1]) continue;
int64_t start_pos = static_cast<int64_t>(lod[level][i]);
int64_t end_pos = static_cast<int64_t>(lod[level][i + 1]);
DenseTensor decoded_path_one_seq =
decoded_path->Slice(start_pos, end_pos);
Decode<T, Context>(dev_ctx,
emission_weights->Slice(start_pos, end_pos),
*transition_weights,
&decoded_path_one_seq);
}
if (label) {
PADDLE_ENFORCE_EQ(label_p->NumLevels(),
1UL,
common::errors::InvalidArgument(
"The Input(label) should be a sequence with lod "
"level 1. But received: lod level %u.",
label_p->NumLevels()));
const int64_t* label_value = label_p->data<int64_t>();
size_t numel = label->numel();
for (size_t i = 0; i < numel; ++i) {
path[i] = label_value[i] == path[i] ? 1 : 0;
}
}
}
}
} // namespace phi
PD_REGISTER_KERNEL(
crf_decoding, CPU, ALL_LAYOUT, phi::CRFDecodingOpKernel, float, double) {
kernel->OutputAt(0).SetDataType(phi::DataType::INT64);
}

Some files were not shown because too many files have changed in this diff Show More