chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,342 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/phi/kernels/selected_rows/adam_kernel.h"
|
||||
|
||||
#include "glog/logging.h"
|
||||
|
||||
#include "paddle/phi/backends/gpu/gpu_context.h"
|
||||
#include "paddle/phi/common/amp_type_traits.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/core/tensor_utils.h"
|
||||
#include "paddle/phi/kernels/funcs/adam_functors.h"
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
#include "paddle/phi/kernels/funcs/selected_rows_functor.h"
|
||||
|
||||
namespace phi {
|
||||
namespace sr {
|
||||
|
||||
template <typename T>
|
||||
__global__ void UpdateBetaPow(T beta1,
|
||||
T beta2,
|
||||
const T* beta1_pow_,
|
||||
const T* beta2_pow_,
|
||||
T* beta1_pow_out,
|
||||
T* beta2_pow_out) {
|
||||
*beta1_pow_out = beta1 * beta1_pow_[0];
|
||||
*beta2_pow_out = beta2 * beta2_pow_[0];
|
||||
}
|
||||
|
||||
template <typename T, typename MT>
|
||||
__global__ void SparseAdamCUDAKernelREG(MT beta1,
|
||||
MT beta2,
|
||||
MT epsilon,
|
||||
const MT beta1_pow,
|
||||
const MT beta2_pow,
|
||||
const MT* mom1_,
|
||||
MT* mom1_out_,
|
||||
const MT* mom2_,
|
||||
MT* mom2_out_,
|
||||
const MT* mom2_max_,
|
||||
MT* mom2_max_out_,
|
||||
const double* lr_,
|
||||
const T* grad_,
|
||||
const T* param_,
|
||||
T* param_out_,
|
||||
const MT* master_param,
|
||||
MT* master_param_out,
|
||||
const int64_t* rows_,
|
||||
int64_t row_numel,
|
||||
int64_t row_count,
|
||||
bool lazy_mode,
|
||||
int ndim,
|
||||
bool amsgrad) {
|
||||
int64_t id =
|
||||
static_cast<int64_t>(blockIdx.x) * static_cast<int64_t>(blockDim.x) +
|
||||
static_cast<int64_t>(threadIdx.x);
|
||||
MT lr = static_cast<MT>(*lr_);
|
||||
|
||||
for (; id < ndim; id += blockDim.x * gridDim.x) {
|
||||
auto row_idx =
|
||||
funcs::BinarySearch<int64_t>(rows_, row_count, id / row_numel);
|
||||
if (lazy_mode && row_idx < 0) {
|
||||
return;
|
||||
} else {
|
||||
MT mom1 = mom1_[id];
|
||||
MT mom2 = mom2_[id];
|
||||
|
||||
MT p = master_param ? master_param[id] : static_cast<MT>(param_[id]);
|
||||
MT g = row_idx >= 0
|
||||
? static_cast<MT>(grad_[row_idx * row_numel + id % row_numel])
|
||||
: static_cast<MT>(0);
|
||||
mom1 = beta1 * mom1 + (static_cast<MT>(1.0) - beta1) * g;
|
||||
mom2 = beta2 * mom2 + (static_cast<MT>(1.0) - beta2) * g * g;
|
||||
|
||||
MT denom;
|
||||
if (amsgrad) {
|
||||
MT mom2_max = mom2_max_[id];
|
||||
MT moment2_max_ = std::max(mom2, mom2_max);
|
||||
mom2_max_out_[id] = moment2_max_;
|
||||
|
||||
denom = (sqrt(moment2_max_) / sqrt(static_cast<MT>(1.0) - beta2_pow)) +
|
||||
epsilon;
|
||||
} else {
|
||||
denom = (sqrt(mom2) / sqrt(static_cast<MT>(1.0) - beta2_pow)) + epsilon;
|
||||
}
|
||||
|
||||
p += (mom1 / denom) * (-(lr / (static_cast<MT>(1.0) - beta1_pow)));
|
||||
|
||||
// Write back to global memory
|
||||
mom1_out_[id] = mom1;
|
||||
mom2_out_[id] = mom2;
|
||||
param_out_[id] = static_cast<T>(p);
|
||||
if (master_param_out) {
|
||||
master_param_out[id] = p;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void AdamDenseParamSparseGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& param,
|
||||
const SelectedRows& grad,
|
||||
const DenseTensor& learning_rate,
|
||||
const DenseTensor& moment1,
|
||||
const DenseTensor& moment2,
|
||||
const optional<DenseTensor>& moment2_max,
|
||||
const DenseTensor& beta1_pow,
|
||||
const DenseTensor& beta2_pow,
|
||||
const optional<DenseTensor>& master_param,
|
||||
const optional<DenseTensor>& skip_update,
|
||||
const Scalar& beta1,
|
||||
const Scalar& beta2,
|
||||
const Scalar& epsilon,
|
||||
bool lazy_mode,
|
||||
int64_t min_row_size_to_use_multithread,
|
||||
bool multi_precision,
|
||||
bool use_global_beta_pow,
|
||||
bool amsgrad,
|
||||
DenseTensor* param_out,
|
||||
DenseTensor* moment1_out,
|
||||
DenseTensor* moment2_out,
|
||||
DenseTensor* moment2_max_out,
|
||||
DenseTensor* beta1_pow_out,
|
||||
DenseTensor* beta2_pow_out,
|
||||
DenseTensor* master_param_outs) {
|
||||
using MT = typename phi::dtype::MPTypeTrait<T>::Type;
|
||||
|
||||
VLOG(4) << "use_global_beta_pow:" << use_global_beta_pow;
|
||||
|
||||
bool skip_update_ = false;
|
||||
if (skip_update.is_initialized()) {
|
||||
PADDLE_ENFORCE_EQ(
|
||||
skip_update->numel(),
|
||||
1,
|
||||
errors::InvalidArgument("Input(SkipUpdate) size must be 1, but get %d",
|
||||
skip_update->numel()));
|
||||
std::vector<bool> skip_update_vec;
|
||||
TensorToVector(*skip_update, dev_ctx, &skip_update_vec);
|
||||
skip_update_ = skip_update_vec[0];
|
||||
}
|
||||
// skip_update=true, just copy input to output, and TensorCopy will call
|
||||
// mutable_data
|
||||
if (skip_update_) {
|
||||
VLOG(4) << "Adam skip update";
|
||||
phi::Copy(dev_ctx, param, dev_ctx.GetPlace(), false, param_out);
|
||||
phi::Copy(dev_ctx, moment1, dev_ctx.GetPlace(), false, moment1_out);
|
||||
phi::Copy(dev_ctx, moment2, dev_ctx.GetPlace(), false, moment2_out);
|
||||
if (amsgrad) {
|
||||
phi::Copy(dev_ctx,
|
||||
moment2_max.get(),
|
||||
dev_ctx.GetPlace(),
|
||||
false,
|
||||
moment2_max_out);
|
||||
}
|
||||
if (!use_global_beta_pow) {
|
||||
phi::Copy(dev_ctx, beta1_pow, beta1_pow.place(), false, beta1_pow_out);
|
||||
phi::Copy(dev_ctx, beta2_pow, beta2_pow.place(), false, beta2_pow_out);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
MT beta1_ = beta1.to<MT>();
|
||||
MT beta2_ = beta2.to<MT>();
|
||||
MT epsilon_ = epsilon.to<MT>();
|
||||
VLOG(3) << "beta1_pow.numel() : " << beta1_pow.numel()
|
||||
<< "beta2_pow.numel() : " << beta2_pow.numel();
|
||||
VLOG(3) << "param.numel(): " << param.numel();
|
||||
PADDLE_ENFORCE_EQ(
|
||||
beta1_pow_out->numel(),
|
||||
1,
|
||||
errors::InvalidArgument("beta1 pow output size should be 1, but received "
|
||||
"value is:%d.",
|
||||
beta1_pow_out->numel()));
|
||||
|
||||
PADDLE_ENFORCE_EQ(
|
||||
beta2_pow_out->numel(),
|
||||
1,
|
||||
errors::InvalidArgument("beta2 pow output size should be 1, but received "
|
||||
"value is:%d.",
|
||||
beta2_pow_out->numel()));
|
||||
|
||||
const MT* master_in_data =
|
||||
multi_precision ? master_param->data<MT>() : nullptr;
|
||||
MT* master_out_data =
|
||||
multi_precision ? dev_ctx.template Alloc<MT>(master_param_outs) : nullptr;
|
||||
|
||||
const MT* moment2_max_in_data =
|
||||
amsgrad ? moment2_max.get().data<MT>() : nullptr;
|
||||
MT* moment2_max_out_data =
|
||||
amsgrad ? dev_ctx.template Alloc<MT>(moment2_max_out) : nullptr;
|
||||
|
||||
if (grad.rows().size() == 0) {
|
||||
VLOG(3) << "grad row size is 0!!";
|
||||
return;
|
||||
}
|
||||
|
||||
std::vector<int64_t> cpu_rows(grad.rows().begin(), grad.rows().end());
|
||||
bool is_strict_sorted = true;
|
||||
for (size_t i = 1; i < cpu_rows.size(); ++i) {
|
||||
if (cpu_rows[i - 1] >= cpu_rows[i]) {
|
||||
is_strict_sorted = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
SelectedRows tmp_grad_merge;
|
||||
const SelectedRows* grad_merge_ptr;
|
||||
if (is_strict_sorted) {
|
||||
grad_merge_ptr = &grad;
|
||||
} else {
|
||||
// merge duplicated rows if any.
|
||||
// The rows of grad_merge have been sorted inside MergeAdd functor
|
||||
funcs::scatter::MergeAdd<Context, T> merge_func;
|
||||
merge_func(dev_ctx, grad, &tmp_grad_merge, true);
|
||||
grad_merge_ptr = &tmp_grad_merge;
|
||||
}
|
||||
auto& grad_merge = *grad_merge_ptr;
|
||||
auto& grad_tensor = grad_merge.value();
|
||||
const T* grad_data = grad_tensor.template data<T>();
|
||||
auto* grad_merge_rows = &grad_merge.rows();
|
||||
phi::MixVector<int64_t> mixv_grad_merge_rows(grad_merge_rows);
|
||||
const int64_t* rows = mixv_grad_merge_rows.Data(dev_ctx.GetPlace());
|
||||
auto row_numel = grad_tensor.numel() / grad_merge.rows().size();
|
||||
|
||||
if (beta1_pow.place() == CPUPlace() && beta2_pow.place() == CPUPlace()) {
|
||||
int threads = 512;
|
||||
int64_t ndim = param.numel();
|
||||
int64_t blocks = (ndim + threads - 1) / threads;
|
||||
|
||||
// NOTE(large-tensor): Kernel launch requires int type for grid dimension
|
||||
PADDLE_ENFORCE_LE_INT_MAX(blocks, "blocks");
|
||||
SparseAdamCUDAKernelREG<T, MT>
|
||||
<<<static_cast<int>(blocks), threads, 0, dev_ctx.stream()>>>(
|
||||
beta1_,
|
||||
beta2_,
|
||||
epsilon_,
|
||||
*beta1_pow.data<MT>(),
|
||||
*beta2_pow.data<MT>(),
|
||||
moment1.data<MT>(),
|
||||
dev_ctx.template Alloc<MT>(moment1_out),
|
||||
moment2.data<MT>(),
|
||||
dev_ctx.template Alloc<MT>(moment2_out),
|
||||
moment2_max_in_data,
|
||||
moment2_max_out_data,
|
||||
learning_rate.data<double>(),
|
||||
grad_data,
|
||||
param.data<T>(),
|
||||
dev_ctx.template Alloc<T>(param_out),
|
||||
master_in_data,
|
||||
master_out_data,
|
||||
rows,
|
||||
row_numel,
|
||||
grad_merge.rows().size(),
|
||||
lazy_mode,
|
||||
ndim,
|
||||
amsgrad);
|
||||
if (!use_global_beta_pow) {
|
||||
// Update with cpu
|
||||
dev_ctx.template HostAlloc<MT>(beta1_pow_out)[0] =
|
||||
beta1_ * beta1_pow.data<MT>()[0];
|
||||
dev_ctx.template HostAlloc<MT>(beta2_pow_out)[0] =
|
||||
beta2_ * beta2_pow.data<MT>()[0];
|
||||
}
|
||||
} else {
|
||||
funcs::SparseAdamFunctor<T, funcs::GPUAdam, MT> functor(
|
||||
beta1_,
|
||||
beta2_,
|
||||
epsilon_,
|
||||
beta1_pow.data<MT>(),
|
||||
beta2_pow.data<MT>(),
|
||||
moment1.data<MT>(),
|
||||
dev_ctx.template Alloc<MT>(moment1_out),
|
||||
moment2.data<MT>(),
|
||||
dev_ctx.template Alloc<MT>(moment2_out),
|
||||
moment2_max_in_data,
|
||||
moment2_max_out_data,
|
||||
learning_rate.data<double>(),
|
||||
grad_data,
|
||||
param.data<T>(),
|
||||
dev_ctx.template Alloc<T>(param_out),
|
||||
master_in_data,
|
||||
master_out_data,
|
||||
rows,
|
||||
row_numel,
|
||||
grad_merge.rows().size(),
|
||||
lazy_mode,
|
||||
amsgrad);
|
||||
|
||||
// FIXME(minqiyang): remove BinarySearch in GPU later
|
||||
funcs::ForRange<Context> for_range(dev_ctx, param.numel());
|
||||
for_range(functor);
|
||||
if (!use_global_beta_pow) {
|
||||
// update beta1 and beta2
|
||||
UpdateBetaPow<MT><<<1, 32, 0, dev_ctx.stream()>>>(
|
||||
beta1_,
|
||||
beta2_,
|
||||
beta1_pow.data<MT>(),
|
||||
beta2_pow.data<MT>(),
|
||||
dev_ctx.template Alloc<MT>(beta1_pow_out),
|
||||
dev_ctx.template Alloc<MT>(beta2_pow_out));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace sr
|
||||
} // namespace phi
|
||||
|
||||
PD_REGISTER_KERNEL(adam_dense_param_sparse_grad,
|
||||
GPU,
|
||||
ALL_LAYOUT,
|
||||
phi::sr::AdamDenseParamSparseGradKernel,
|
||||
float,
|
||||
double,
|
||||
phi::float16) {
|
||||
// Skip beta1_pow, beta2_pow, skip_update data transform
|
||||
kernel->InputAt(6).SetBackend(phi::Backend::ALL_BACKEND);
|
||||
kernel->InputAt(7).SetBackend(phi::Backend::ALL_BACKEND);
|
||||
kernel->InputAt(9).SetBackend(phi::Backend::ALL_BACKEND);
|
||||
|
||||
if (kernel_key.dtype() == phi::DataType::FLOAT16) {
|
||||
kernel->OutputAt(1).SetDataType(phi::DataType::FLOAT32);
|
||||
kernel->OutputAt(2).SetDataType(phi::DataType::FLOAT32);
|
||||
kernel->OutputAt(3).SetDataType(phi::DataType::FLOAT32);
|
||||
kernel->OutputAt(4).SetDataType(phi::DataType::FLOAT32);
|
||||
kernel->OutputAt(5).SetDataType(phi::DataType::FLOAT32);
|
||||
kernel->OutputAt(6).SetDataType(phi::DataType::FLOAT32);
|
||||
}
|
||||
kernel->OutputAt(4).SetBackend(phi::Backend::UNDEFINED);
|
||||
kernel->OutputAt(5).SetBackend(phi::Backend::UNDEFINED);
|
||||
}
|
||||
@@ -0,0 +1,367 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/phi/kernels/selected_rows/adamw_kernel.h"
|
||||
|
||||
#include <math.h> // for sqrt in CPU and CUDA
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "glog/logging.h"
|
||||
|
||||
#include "paddle/phi/backends/gpu/gpu_context.h"
|
||||
#include "paddle/phi/common/amp_type_traits.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/core/tensor_utils.h"
|
||||
#include "paddle/phi/kernels/funcs/adam_functors.h"
|
||||
#include "paddle/phi/kernels/funcs/for_range.h"
|
||||
#include "paddle/phi/kernels/funcs/selected_rows_functor.h"
|
||||
|
||||
namespace phi {
|
||||
namespace sr {
|
||||
|
||||
template <typename T>
|
||||
__global__ void UpdateAdamWBetaPow(T beta1,
|
||||
T beta2,
|
||||
const T* beta1_pow_,
|
||||
const T* beta2_pow_,
|
||||
T* beta1_pow_out,
|
||||
T* beta2_pow_out) {
|
||||
*beta1_pow_out = beta1 * beta1_pow_[0];
|
||||
*beta2_pow_out = beta2 * beta2_pow_[0];
|
||||
}
|
||||
|
||||
template <typename T, typename MT>
|
||||
__global__ void SparseAdamWCUDAKernelREG(MT beta1,
|
||||
MT beta2,
|
||||
MT epsilon,
|
||||
MT coeff,
|
||||
MT lr_ratio,
|
||||
const MT beta1_pow,
|
||||
const MT beta2_pow,
|
||||
const MT* mom1_,
|
||||
MT* mom1_out_,
|
||||
const MT* mom2_,
|
||||
MT* mom2_out_,
|
||||
const MT* mom2_max_,
|
||||
MT* mom2_max_out_,
|
||||
const MT* lr_,
|
||||
const T* grad_,
|
||||
const T* param_,
|
||||
T* param_out_,
|
||||
const MT* master_param,
|
||||
MT* master_param_out,
|
||||
const int64_t* rows_,
|
||||
int64_t row_numel,
|
||||
int64_t row_count,
|
||||
bool lazy_mode,
|
||||
int ndim,
|
||||
bool amsgrad) {
|
||||
int64_t id =
|
||||
static_cast<int64_t>(blockIdx.x) * static_cast<int64_t>(blockDim.x) +
|
||||
static_cast<int64_t>(threadIdx.x);
|
||||
MT lr = *lr_ * lr_ratio;
|
||||
|
||||
for (; id < ndim; id += blockDim.x * gridDim.x) {
|
||||
auto row_idx =
|
||||
funcs::BinarySearch<int64_t>(rows_, row_count, id / row_numel);
|
||||
if (lazy_mode && row_idx < 0) {
|
||||
return;
|
||||
} else {
|
||||
MT mom1 = static_cast<MT>(mom1_[id]);
|
||||
MT mom2 = static_cast<MT>(mom2_[id]);
|
||||
|
||||
MT p = master_param ? master_param[id] : static_cast<MT>(param_[id]);
|
||||
MT g = row_idx >= 0
|
||||
? static_cast<MT>(grad_[row_idx * row_numel + id % row_numel])
|
||||
: static_cast<MT>(0);
|
||||
|
||||
p *= (static_cast<MT>(1.0) - lr * coeff);
|
||||
|
||||
mom1 = beta1 * mom1 + (static_cast<MT>(1.0) - beta1) * g;
|
||||
mom2 = beta2 * mom2 + (static_cast<MT>(1.0) - beta2) * g * g;
|
||||
|
||||
MT denom;
|
||||
if (amsgrad) {
|
||||
MT mom2_max = static_cast<MT>(mom2_max_[id]);
|
||||
MT mom2_max_ = std::max(mom2, mom2_max);
|
||||
mom2_max_out_[id] = mom2_max_;
|
||||
|
||||
denom = (sqrt(mom2_max_) / sqrt(static_cast<MT>(1.0) - beta2_pow)) +
|
||||
epsilon;
|
||||
} else {
|
||||
denom = (sqrt(mom2) / sqrt(static_cast<MT>(1.0) - beta2_pow)) + epsilon;
|
||||
}
|
||||
|
||||
p += (mom1 / denom) * (-(lr / (static_cast<MT>(1.0) - beta1_pow)));
|
||||
|
||||
// Write back to global memory
|
||||
mom1_out_[id] = mom1;
|
||||
mom2_out_[id] = mom2;
|
||||
param_out_[id] = static_cast<T>(p);
|
||||
if (master_param_out) {
|
||||
master_param_out[id] = p;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void AdamwDenseParamSparseGradKernel(const Context& dev_ctx,
|
||||
const DenseTensor& param,
|
||||
const SelectedRows& grad,
|
||||
const DenseTensor& learning_rate,
|
||||
const DenseTensor& moment1,
|
||||
const DenseTensor& moment2,
|
||||
const optional<DenseTensor>& moment2_max,
|
||||
const DenseTensor& beta1_pow,
|
||||
const DenseTensor& beta2_pow,
|
||||
const optional<DenseTensor>& master_param,
|
||||
const optional<DenseTensor>& skip_update,
|
||||
const Scalar& beta1,
|
||||
const Scalar& beta2,
|
||||
const Scalar& epsilon,
|
||||
float lr_ratio,
|
||||
float coeff,
|
||||
bool with_decay,
|
||||
bool lazy_mode,
|
||||
int64_t min_row_size_to_use_multithread,
|
||||
bool multi_precision,
|
||||
bool use_global_beta_pow,
|
||||
bool amsgrad,
|
||||
DenseTensor* param_out,
|
||||
DenseTensor* moment1_out,
|
||||
DenseTensor* moment2_out,
|
||||
DenseTensor* moment2_max_out,
|
||||
DenseTensor* beta1_pow_out,
|
||||
DenseTensor* beta2_pow_out,
|
||||
DenseTensor* master_param_outs) {
|
||||
using MT = typename phi::dtype::MPTypeTrait<T>::Type;
|
||||
|
||||
VLOG(4) << "use_global_beta_pow:" << use_global_beta_pow;
|
||||
|
||||
MT coeff_ = static_cast<MT>(coeff);
|
||||
MT lr_ratio_ = static_cast<MT>(lr_ratio);
|
||||
|
||||
bool skip_update_ = false;
|
||||
if (skip_update.is_initialized()) {
|
||||
PADDLE_ENFORCE_EQ(
|
||||
skip_update->numel(),
|
||||
1,
|
||||
errors::InvalidArgument("Input(SkipUpdate) size must be 1, but get %d",
|
||||
skip_update->numel()));
|
||||
std::vector<bool> skip_update_vec;
|
||||
TensorToVector(*skip_update, dev_ctx, &skip_update_vec);
|
||||
skip_update_ = skip_update_vec[0];
|
||||
}
|
||||
|
||||
// skip_update=true, just copy input to output, and TensorCopy will call
|
||||
// mutable_data
|
||||
if (skip_update_) {
|
||||
VLOG(4) << "Adamw skip update";
|
||||
phi::Copy(dev_ctx, param, dev_ctx.GetPlace(), false, param_out);
|
||||
phi::Copy(dev_ctx, moment1, dev_ctx.GetPlace(), false, moment1_out);
|
||||
phi::Copy(dev_ctx, moment2, dev_ctx.GetPlace(), false, moment2_out);
|
||||
if (amsgrad) {
|
||||
phi::Copy(dev_ctx,
|
||||
moment2_max.get(),
|
||||
dev_ctx.GetPlace(),
|
||||
false,
|
||||
moment2_max_out);
|
||||
}
|
||||
if (!use_global_beta_pow) {
|
||||
phi::Copy(dev_ctx, beta1_pow, beta1_pow.place(), false, beta1_pow_out);
|
||||
phi::Copy(dev_ctx, beta2_pow, beta2_pow.place(), false, beta2_pow_out);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// if with_decay = false, coeff = 0
|
||||
if (!with_decay) {
|
||||
coeff_ = static_cast<MT>(0.0);
|
||||
}
|
||||
|
||||
MT beta1_ = beta1.to<MT>();
|
||||
MT beta2_ = beta2.to<MT>();
|
||||
MT epsilon_ = epsilon.to<MT>();
|
||||
VLOG(3) << "beta1_pow.numel() : " << beta1_pow.numel()
|
||||
<< "beta2_pow.numel() : " << beta2_pow.numel();
|
||||
VLOG(3) << "param.numel(): " << param.numel();
|
||||
PADDLE_ENFORCE_EQ(
|
||||
beta1_pow_out->numel(),
|
||||
1,
|
||||
errors::InvalidArgument("beta1 pow output size should be 1, but received "
|
||||
"value is:%d.",
|
||||
beta1_pow_out->numel()));
|
||||
|
||||
PADDLE_ENFORCE_EQ(
|
||||
beta2_pow_out->numel(),
|
||||
1,
|
||||
errors::InvalidArgument("beta2 pow output size should be 1, but received "
|
||||
"value is:%d.",
|
||||
beta2_pow_out->numel()));
|
||||
|
||||
const MT* master_in_data =
|
||||
multi_precision ? master_param->data<MT>() : nullptr;
|
||||
MT* master_out_data =
|
||||
multi_precision ? dev_ctx.template Alloc<MT>(master_param_outs) : nullptr;
|
||||
|
||||
const MT* moment2_max_in_data =
|
||||
amsgrad ? moment2_max.get().data<MT>() : nullptr;
|
||||
MT* moment2_max_out_data =
|
||||
amsgrad ? dev_ctx.template Alloc<MT>(moment2_max_out) : nullptr;
|
||||
|
||||
if (grad.rows().size() == 0) {
|
||||
VLOG(3) << "grad row size is 0!!";
|
||||
return;
|
||||
}
|
||||
|
||||
std::vector<int64_t> cpu_rows(grad.rows().begin(), grad.rows().end());
|
||||
bool is_strict_sorted = true;
|
||||
for (size_t i = 1; i < cpu_rows.size(); ++i) {
|
||||
if (cpu_rows[i - 1] >= cpu_rows[i]) {
|
||||
is_strict_sorted = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
SelectedRows tmp_grad_merge;
|
||||
const SelectedRows* grad_merge_ptr;
|
||||
if (is_strict_sorted) {
|
||||
grad_merge_ptr = &grad;
|
||||
} else {
|
||||
// merge duplicated rows if any.
|
||||
// The rows of grad_merge have been sorted inside MergeAdd functor
|
||||
funcs::scatter::MergeAdd<Context, T> merge_func;
|
||||
merge_func(dev_ctx, grad, &tmp_grad_merge, true);
|
||||
grad_merge_ptr = &tmp_grad_merge;
|
||||
}
|
||||
auto& grad_merge = *grad_merge_ptr;
|
||||
auto& grad_tensor = grad_merge.value();
|
||||
const T* grad_data = grad_tensor.template data<T>();
|
||||
auto* grad_merge_rows = &grad_merge.rows();
|
||||
phi::MixVector<int64_t> mixv_grad_merge_rows(grad_merge_rows);
|
||||
const int64_t* rows = mixv_grad_merge_rows.Data(dev_ctx.GetPlace());
|
||||
auto row_numel = grad_tensor.numel() / grad_merge.rows().size();
|
||||
|
||||
if (beta1_pow.place() == CPUPlace() && beta2_pow.place() == CPUPlace()) {
|
||||
int threads = 512;
|
||||
int64_t ndim = param.numel();
|
||||
int64_t blocks = (ndim + threads - 1) / threads;
|
||||
|
||||
// NOTE(large-tensor): Kernel launch requires int type for grid dimension
|
||||
PADDLE_ENFORCE_LE_INT_MAX(blocks, "blocks");
|
||||
SparseAdamWCUDAKernelREG<T, MT>
|
||||
<<<static_cast<int>(blocks), threads, 0, dev_ctx.stream()>>>(
|
||||
beta1_,
|
||||
beta2_,
|
||||
epsilon_,
|
||||
coeff_,
|
||||
lr_ratio_,
|
||||
*beta1_pow.data<MT>(),
|
||||
*beta2_pow.data<MT>(),
|
||||
moment1.data<MT>(),
|
||||
dev_ctx.template Alloc<MT>(moment1_out),
|
||||
moment2.data<MT>(),
|
||||
dev_ctx.template Alloc<MT>(moment2_out),
|
||||
moment2_max_in_data,
|
||||
moment2_max_out_data,
|
||||
learning_rate.data<MT>(),
|
||||
grad_data,
|
||||
param.data<T>(),
|
||||
dev_ctx.template Alloc<T>(param_out),
|
||||
master_in_data,
|
||||
master_out_data,
|
||||
rows,
|
||||
row_numel,
|
||||
grad_merge.rows().size(),
|
||||
lazy_mode,
|
||||
ndim,
|
||||
amsgrad);
|
||||
if (!use_global_beta_pow) {
|
||||
// Update with cpu
|
||||
dev_ctx.template HostAlloc<MT>(beta1_pow_out)[0] =
|
||||
beta1_ * beta1_pow.data<MT>()[0];
|
||||
dev_ctx.template HostAlloc<MT>(beta2_pow_out)[0] =
|
||||
beta2_ * beta2_pow.data<MT>()[0];
|
||||
}
|
||||
} else {
|
||||
funcs::SparseAdamWFunctor<T, funcs::GPUAdamW, MT> functor(
|
||||
beta1_,
|
||||
beta2_,
|
||||
epsilon_,
|
||||
coeff_,
|
||||
lr_ratio_,
|
||||
beta1_pow.data<MT>(),
|
||||
beta2_pow.data<MT>(),
|
||||
moment1.data<MT>(),
|
||||
dev_ctx.template Alloc<MT>(moment1_out),
|
||||
moment2.data<MT>(),
|
||||
dev_ctx.template Alloc<MT>(moment2_out),
|
||||
moment2_max_in_data,
|
||||
moment2_max_out_data,
|
||||
learning_rate.data<MT>(),
|
||||
grad_data,
|
||||
param.data<T>(),
|
||||
dev_ctx.template Alloc<T>(param_out),
|
||||
master_in_data,
|
||||
master_out_data,
|
||||
rows,
|
||||
row_numel,
|
||||
grad_merge.rows().size(),
|
||||
lazy_mode,
|
||||
amsgrad);
|
||||
|
||||
// FIXME(minqiyang): remove BinarySearch in GPU later
|
||||
funcs::ForRange<Context> for_range(dev_ctx, param.numel());
|
||||
for_range(functor);
|
||||
if (!use_global_beta_pow) {
|
||||
// update beta1 and beta2
|
||||
UpdateAdamWBetaPow<MT><<<1, 32, 0, dev_ctx.stream()>>>(
|
||||
beta1_,
|
||||
beta2_,
|
||||
beta1_pow.data<MT>(),
|
||||
beta2_pow.data<MT>(),
|
||||
dev_ctx.template Alloc<MT>(beta1_pow_out),
|
||||
dev_ctx.template Alloc<MT>(beta2_pow_out));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace sr
|
||||
} // namespace phi
|
||||
|
||||
PD_REGISTER_KERNEL(adamw_dense_param_sparse_grad,
|
||||
GPU,
|
||||
ALL_LAYOUT,
|
||||
phi::sr::AdamwDenseParamSparseGradKernel,
|
||||
float,
|
||||
double,
|
||||
phi::float16) {
|
||||
// Skip beta1_pow, beta2_pow, skip_update data transform
|
||||
kernel->InputAt(6).SetBackend(phi::Backend::ALL_BACKEND);
|
||||
kernel->InputAt(7).SetBackend(phi::Backend::ALL_BACKEND);
|
||||
kernel->InputAt(9).SetBackend(phi::Backend::ALL_BACKEND);
|
||||
|
||||
if (kernel_key.dtype() == phi::DataType::FLOAT16) {
|
||||
kernel->OutputAt(1).SetDataType(phi::DataType::FLOAT32);
|
||||
kernel->OutputAt(2).SetDataType(phi::DataType::FLOAT32);
|
||||
kernel->OutputAt(3).SetDataType(phi::DataType::FLOAT32);
|
||||
kernel->OutputAt(4).SetDataType(phi::DataType::FLOAT32);
|
||||
kernel->OutputAt(5).SetDataType(phi::DataType::FLOAT32);
|
||||
kernel->OutputAt(6).SetDataType(phi::DataType::FLOAT32);
|
||||
}
|
||||
kernel->OutputAt(4).SetBackend(phi::Backend::UNDEFINED);
|
||||
kernel->OutputAt(5).SetBackend(phi::Backend::UNDEFINED);
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/phi/kernels/selected_rows/impl/add_n_kernel_impl.h"
|
||||
|
||||
PD_REGISTER_KERNEL(add_n_sr,
|
||||
GPU,
|
||||
ALL_LAYOUT,
|
||||
phi::sr::AddNKernel,
|
||||
float,
|
||||
double,
|
||||
int,
|
||||
phi::bfloat16,
|
||||
phi::float16,
|
||||
int64_t) {}
|
||||
@@ -0,0 +1,26 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/phi/kernels/selected_rows/clip_by_norm_kernel.h"
|
||||
|
||||
#include "paddle/phi/backends/gpu/gpu_context.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/kernels/selected_rows/impl/clip_by_norm_kernel_impl.h"
|
||||
|
||||
PD_REGISTER_KERNEL(clip_by_norm_sr,
|
||||
GPU,
|
||||
ALL_LAYOUT,
|
||||
phi::sr::ClipByNormKernel,
|
||||
float,
|
||||
phi::float16) {}
|
||||
@@ -0,0 +1,29 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/phi/kernels/selected_rows/clip_kernel.h"
|
||||
|
||||
#include "paddle/phi/backends/gpu/gpu_context.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/kernels/selected_rows/impl/clip_kernel_impl.h"
|
||||
|
||||
PD_REGISTER_KERNEL(clip_sr,
|
||||
GPU,
|
||||
ALL_LAYOUT,
|
||||
phi::sr::ClipSparseKernel,
|
||||
float,
|
||||
double,
|
||||
int,
|
||||
int64_t,
|
||||
phi::float16) {}
|
||||
@@ -0,0 +1,19 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/kernels/selected_rows/impl/dgc_clip_by_norm_kernel_impl.h"
|
||||
PD_REGISTER_KERNEL(
|
||||
dgc_clip_by_norm_sr, GPU, ALL_LAYOUT, phi::sr::DGCClipByNormKernel, float) {
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/kernels/selected_rows/impl/ftrl_kernel_impl.h"
|
||||
|
||||
PD_REGISTER_KERNEL(ftrl_sr, GPU, ALL_LAYOUT, phi::sr::FTRLOpKernel, float) {}
|
||||
@@ -0,0 +1,24 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/phi/kernels/selected_rows/impl/get_tensor_from_selected_rows_kernel_impl.h"
|
||||
|
||||
PD_REGISTER_KERNEL(get_tensor_from_selected_rows,
|
||||
GPU,
|
||||
ALL_LAYOUT,
|
||||
phi::sr::GetTensorFromSelectedRowsKernel,
|
||||
float,
|
||||
double,
|
||||
int,
|
||||
int64_t) {}
|
||||
@@ -0,0 +1,29 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/phi/kernels/selected_rows/lamb_kernel.h"
|
||||
#include "paddle/phi/backends/gpu/gpu_context.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/kernels/selected_rows/impl/lamb_kernel_impl.h"
|
||||
|
||||
PD_REGISTER_KERNEL(lamb_sr,
|
||||
GPU,
|
||||
ALL_LAYOUT,
|
||||
phi::sr::LambKernel,
|
||||
phi::float16,
|
||||
float,
|
||||
double) {
|
||||
kernel->InputAt(5).SetBackend(phi::Backend::ALL_BACKEND);
|
||||
kernel->InputAt(6).SetBackend(phi::Backend::ALL_BACKEND);
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/phi/kernels/selected_rows/impl/load_kernel_impl.h"
|
||||
|
||||
PD_REGISTER_KERNEL(
|
||||
load_sr, GPU, ALL_LAYOUT, phi::sr::LoadSelectedRowsKernel, float) {}
|
||||
@@ -0,0 +1,201 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/phi/backends/gpu/gpu_primitives.h"
|
||||
#include "paddle/phi/common/memory_utils.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/core/selected_rows.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/funcs/selected_rows_functor.h"
|
||||
|
||||
namespace phi {
|
||||
namespace sr {
|
||||
|
||||
template <typename T, int BlockDimX, int BlockDimY, int GridDimX>
|
||||
__global__ void LookupTableGrad(T *table,
|
||||
const T *output,
|
||||
const int64_t *ids,
|
||||
const int64_t N,
|
||||
const int64_t K,
|
||||
const int64_t D) {
|
||||
int idx = threadIdx.x;
|
||||
int64_t idy = static_cast<int64_t>(blockIdx.x) +
|
||||
static_cast<int64_t>(threadIdx.y) * GridDimX;
|
||||
|
||||
while (idy < K) {
|
||||
int64_t id = ids[idy];
|
||||
PADDLE_ENFORCE(
|
||||
id >= 0,
|
||||
"Variable value (input) of OP(lookup_table_grad) "
|
||||
"expected >= 0 and < %ld, but got %ld. Please check input value.",
|
||||
N,
|
||||
id);
|
||||
PADDLE_ENFORCE(
|
||||
id < N,
|
||||
"Variable value (input) of OP(lookup_table_grad) "
|
||||
"expected >= 0 and < %ld, but got %ld. Please check input value.",
|
||||
N,
|
||||
id);
|
||||
const T *out = output + idy * D;
|
||||
T *tab = table + id * D;
|
||||
for (int i = idx; i < D; i += BlockDimX) {
|
||||
CudaAtomicAdd(&tab[i], out[i]);
|
||||
}
|
||||
idy += BlockDimY * GridDimX;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void LookupTableGradCUDAKernel(
|
||||
const Context &dev_ctx,
|
||||
const SelectedRows &w,
|
||||
const DenseTensor &ids_in,
|
||||
const DenseTensor &out_grad,
|
||||
bool is_sparse,
|
||||
bool is_distributed UNUSED,
|
||||
int64_t padding_idx,
|
||||
bool remote_prefetch UNUSED,
|
||||
const std::string &entry_config UNUSED,
|
||||
bool is_test,
|
||||
const std::string &entry UNUSED,
|
||||
const std::string &table_class UNUSED,
|
||||
const std::vector<std::string> &table_names UNUSED,
|
||||
int trainer_id UNUSED,
|
||||
bool grad_inplace UNUSED,
|
||||
const std::vector<std::string> &epmap UNUSED,
|
||||
const std::vector<int64_t> &height_sections UNUSED,
|
||||
DenseTensor *w_grad) {
|
||||
// Since paddings are not trainable and fixed in forward, the gradient of
|
||||
// paddings makes no sense and we don't deal with it in backward.
|
||||
|
||||
auto ids_t = &ids_in;
|
||||
auto d_output_t = &out_grad;
|
||||
auto d_table_t = w_grad;
|
||||
|
||||
int64_t N = d_table_t->dims()[0];
|
||||
int64_t D = d_table_t->dims()[1];
|
||||
int64_t K = ids_t->numel();
|
||||
const int64_t *ids = ids_t->data<int64_t>();
|
||||
const T *d_output = d_output_t->data<T>();
|
||||
T *d_table = dev_ctx.template Alloc<T>(d_table_t);
|
||||
|
||||
auto t = EigenVector<T>::Flatten(*d_table_t);
|
||||
t.device(*dev_ctx.eigen_device()) = t.constant(static_cast<T>(0));
|
||||
|
||||
#ifdef PADDLE_WITH_HIP
|
||||
dim3 threads(64, 4);
|
||||
#else
|
||||
dim3 threads(128, 8);
|
||||
#endif // PADDLE_WITH_HIP
|
||||
dim3 grids(8, 1);
|
||||
|
||||
#ifdef PADDLE_WITH_HIP
|
||||
LookupTableGrad<T, 64, 4, 8><<<grids, threads, 0, dev_ctx.stream()>>>(
|
||||
d_table, d_output, ids, N, K, D);
|
||||
#else
|
||||
LookupTableGrad<T, 128, 8, 8><<<grids, threads, 0, dev_ctx.stream()>>>(
|
||||
d_table, d_output, ids, N, K, D);
|
||||
#endif // PADDLE_WITH_HIP
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void LookupTableSparseGradCUDAKernel(
|
||||
const Context &dev_ctx,
|
||||
const SelectedRows &w,
|
||||
const DenseTensor &ids_in,
|
||||
const DenseTensor &out_grad,
|
||||
bool is_sparse,
|
||||
bool is_distributed UNUSED,
|
||||
int64_t padding_idx,
|
||||
bool remote_prefetch UNUSED,
|
||||
const std::string &entry_config UNUSED,
|
||||
bool is_test,
|
||||
const std::string &entry UNUSED,
|
||||
const std::string &table_class UNUSED,
|
||||
const std::vector<std::string> &table_names UNUSED,
|
||||
int trainer_id UNUSED,
|
||||
bool grad_inplace UNUSED,
|
||||
const std::vector<std::string> &epmap UNUSED,
|
||||
const std::vector<int64_t> &height_sections UNUSED,
|
||||
SelectedRows *w_grad) {
|
||||
// Since paddings are not trainable and fixed in forward, the gradient of
|
||||
// paddings makes no sense and we don't deal with it in backward.
|
||||
auto *ids = &ids_in;
|
||||
auto *table = &w;
|
||||
auto *d_output = &out_grad;
|
||||
auto *d_table = w_grad;
|
||||
|
||||
auto *ids_data = ids->data<int64_t>();
|
||||
int64_t ids_num = ids->numel();
|
||||
|
||||
auto stream = dev_ctx.stream();
|
||||
// copy GPU memory to CPU pinned memory
|
||||
Vector<int64_t> new_rows;
|
||||
new_rows.resize(ids_num);
|
||||
auto gpu_place = dev_ctx.GetPlace();
|
||||
|
||||
// TODO(yuyang18): Strange code here.
|
||||
phi::MixVector<int64_t> mixv_new_rows(&new_rows);
|
||||
phi::memory_utils::Copy(gpu_place,
|
||||
mixv_new_rows.CUDAMutableData(dev_ctx.GetPlace()),
|
||||
gpu_place,
|
||||
ids_data,
|
||||
ids_num * sizeof(int64_t),
|
||||
stream);
|
||||
mixv_new_rows.CopyToCPU();
|
||||
d_table->set_rows(new_rows);
|
||||
|
||||
auto *d_table_value = d_table->mutable_value();
|
||||
d_table_value->Resize({ids_num, table->dims()[1]});
|
||||
dev_ctx.template Alloc<T>(d_table_value);
|
||||
|
||||
auto *d_table_data = d_table_value->data<T>();
|
||||
auto *d_output_data = d_output->data<T>();
|
||||
auto d_output_dims = d_output->dims();
|
||||
auto d_output_dims_2d =
|
||||
common::flatten_to_2d(d_output_dims, d_output_dims.size() - 1);
|
||||
PADDLE_ENFORCE_EQ(d_table_value->dims(),
|
||||
d_output_dims_2d,
|
||||
common::errors::InvalidArgument(
|
||||
"ShapeError: The shape of lookup_table@Grad and "
|
||||
"output@Grad should be same. "
|
||||
"But received lookup_table@Grad's shape = [%s], "
|
||||
"output@Grad's shape = [%s].",
|
||||
d_table_value->dims(),
|
||||
d_output_dims_2d));
|
||||
phi::memory_utils::Copy(gpu_place,
|
||||
d_table_data,
|
||||
gpu_place,
|
||||
d_output_data,
|
||||
d_output->numel() * sizeof(T),
|
||||
stream);
|
||||
}
|
||||
} // namespace sr
|
||||
} // namespace phi
|
||||
|
||||
PD_REGISTER_KERNEL(lookup_table_grad_sr,
|
||||
GPU,
|
||||
ALL_LAYOUT,
|
||||
phi::sr::LookupTableGradCUDAKernel,
|
||||
float,
|
||||
double,
|
||||
phi::float16) {}
|
||||
|
||||
PD_REGISTER_KERNEL(lookup_table_sparse_grad_sr,
|
||||
GPU,
|
||||
ALL_LAYOUT,
|
||||
phi::sr::LookupTableSparseGradCUDAKernel,
|
||||
float,
|
||||
double,
|
||||
phi::float16) {}
|
||||
@@ -0,0 +1,134 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/phi/backends/gpu/gpu_primitives.h"
|
||||
#include "paddle/phi/common/memory_utils.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/core/selected_rows.h"
|
||||
#include "paddle/phi/kernels/funcs/eigen/common.h"
|
||||
#include "paddle/phi/kernels/funcs/selected_rows_functor.h"
|
||||
|
||||
namespace phi {
|
||||
namespace sr {
|
||||
template <typename T,
|
||||
int BlockDimX,
|
||||
int BlockDimY,
|
||||
int GridDimX,
|
||||
bool PaddingFlag>
|
||||
__global__ void LookupTable(T *output,
|
||||
const T *table,
|
||||
const int64_t *ids,
|
||||
const int64_t N,
|
||||
const int64_t K,
|
||||
const int64_t D,
|
||||
const int64_t padding_idx) {
|
||||
int idx = threadIdx.x;
|
||||
int64_t idy = static_cast<int64_t>(blockIdx.x) +
|
||||
static_cast<int64_t>(threadIdx.y) * GridDimX;
|
||||
|
||||
while (idy < K) {
|
||||
int64_t id = ids[idy];
|
||||
PADDLE_ENFORCE(
|
||||
id >= 0,
|
||||
"Variable value (input) of OP(lookup_table) "
|
||||
"expected >= 0 and < %ld, but got %ld. Please check input value.",
|
||||
N,
|
||||
id);
|
||||
PADDLE_ENFORCE(
|
||||
id < N,
|
||||
"Variable value (input) of OP(lookup_table) "
|
||||
"expected >= 0 and < %ld, but got %ld. Please check input value.",
|
||||
N,
|
||||
id);
|
||||
T *out = output + idy * D;
|
||||
const T *tab = table + id * D;
|
||||
for (int i = idx; i < D; i += BlockDimX) {
|
||||
if (PaddingFlag) {
|
||||
if (id == padding_idx)
|
||||
out[i] = static_cast<T>(0);
|
||||
else
|
||||
out[i] = tab[i];
|
||||
} else {
|
||||
out[i] = tab[i];
|
||||
}
|
||||
}
|
||||
idy += BlockDimY * GridDimX;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename Context>
|
||||
void LookupTableCUDAKernel(const Context &dev_ctx,
|
||||
const SelectedRows &w,
|
||||
const DenseTensor &ids_in,
|
||||
bool is_sparse,
|
||||
bool is_distributed UNUSED,
|
||||
int64_t padding_idx,
|
||||
bool remote_prefetch UNUSED,
|
||||
const std::string &entry_config UNUSED,
|
||||
bool is_test,
|
||||
const std::string &entry UNUSED,
|
||||
const std::string &table_class UNUSED,
|
||||
const std::vector<std::string> &table_names UNUSED,
|
||||
int trainer_id UNUSED,
|
||||
bool grad_inplace UNUSED,
|
||||
const std::vector<std::string> &epmap UNUSED,
|
||||
const std::vector<int64_t> &height_sections UNUSED,
|
||||
SelectedRows *out) {
|
||||
auto *table_t = &w;
|
||||
auto *ids_t = &ids_in;
|
||||
auto *output_t = out;
|
||||
|
||||
size_t N = table_t->dims()[0];
|
||||
size_t D = table_t->dims()[1];
|
||||
size_t K = ids_t->numel();
|
||||
|
||||
auto *ids = ids_t->data<int64_t>();
|
||||
auto *table = table_t->value().data<T>();
|
||||
auto *output = dev_ctx.template Alloc<T>(output_t);
|
||||
|
||||
#ifdef PADDLE_WITH_HIP
|
||||
dim3 threads(64, 4);
|
||||
#else
|
||||
dim3 threads(128, 8);
|
||||
#endif // PADDLE_WITH_HIP
|
||||
dim3 grids(8, 1);
|
||||
#ifdef PADDLE_WITH_HIP
|
||||
if (padding_idx == -1)
|
||||
LookupTable<T, 64, 4, 8, false><<<grids, threads, 0, dev_ctx.stream()>>>(
|
||||
output, table, ids, N, K, D, padding_idx);
|
||||
else
|
||||
LookupTable<T, 64, 4, 8, true><<<grids, threads, 0, dev_ctx.stream()>>>(
|
||||
output, table, ids, N, K, D, padding_idx);
|
||||
#else
|
||||
if (padding_idx == -1)
|
||||
LookupTable<T, 128, 8, 8, false><<<grids, threads, 0, dev_ctx.stream()>>>(
|
||||
output, table, ids, N, K, D, padding_idx);
|
||||
else
|
||||
LookupTable<T, 128, 8, 8, true><<<grids, threads, 0, dev_ctx.stream()>>>(
|
||||
output, table, ids, N, K, D, padding_idx);
|
||||
#endif // PADDLE_WITH_HIP
|
||||
}
|
||||
|
||||
} // namespace sr
|
||||
} // namespace phi
|
||||
|
||||
PD_REGISTER_KERNEL(lookup_table_sr,
|
||||
GPU,
|
||||
ALL_LAYOUT,
|
||||
phi::sr::LookupTableCUDAKernel,
|
||||
float,
|
||||
double,
|
||||
phi::float16,
|
||||
int8_t,
|
||||
int16_t) {}
|
||||
@@ -0,0 +1,31 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/phi/kernels/selected_rows/impl/save_kernel_impl.h"
|
||||
|
||||
PD_REGISTER_KERNEL(save_sr,
|
||||
GPU,
|
||||
ALL_LAYOUT,
|
||||
phi::SaveSelectedRowsKernel,
|
||||
float,
|
||||
double,
|
||||
int,
|
||||
uint8_t,
|
||||
int8_t,
|
||||
int16_t,
|
||||
int64_t,
|
||||
phi::float16,
|
||||
phi::bfloat16) {
|
||||
kernel->InputAt(0).SetBackend(phi::Backend::ALL_BACKEND);
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/phi/common/data_type.h"
|
||||
#include "paddle/phi/kernels/selected_rows/impl/share_data_kernel_impl.h"
|
||||
|
||||
PD_REGISTER_KERNEL(share_data_sr,
|
||||
GPU,
|
||||
ALL_LAYOUT,
|
||||
phi::sr::ShareDataKernel,
|
||||
bool,
|
||||
int,
|
||||
int8_t,
|
||||
uint8_t,
|
||||
int64_t,
|
||||
float,
|
||||
double,
|
||||
phi::float16) {}
|
||||
@@ -0,0 +1,60 @@
|
||||
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/phi/backends/gpu/gpu_context.h"
|
||||
#include "paddle/phi/core/enforce.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/kernels/funcs/uniform_random_functor.h"
|
||||
|
||||
namespace phi {
|
||||
namespace sr {
|
||||
|
||||
template <typename T, typename Context>
|
||||
void GPUUniformRandomKernel(const Context& dev_ctx,
|
||||
const SelectedRows& input,
|
||||
const std::vector<int>& shape,
|
||||
int input_dim_idx,
|
||||
int output_dim_idx,
|
||||
float min,
|
||||
float max,
|
||||
int seed,
|
||||
int diag_num,
|
||||
int diag_step,
|
||||
float diag_val,
|
||||
DataType dtype UNUSED,
|
||||
SelectedRows* out) {
|
||||
out->set_rows(input.rows());
|
||||
out->set_height(input.height());
|
||||
DenseTensor* tensor = out->mutable_value();
|
||||
dev_ctx.template Alloc<T>(tensor);
|
||||
funcs::UniformRandom<T>(reinterpret_cast<const GPUContext&>(dev_ctx),
|
||||
tensor,
|
||||
seed,
|
||||
min,
|
||||
max,
|
||||
diag_num,
|
||||
diag_step,
|
||||
diag_val);
|
||||
}
|
||||
|
||||
} // namespace sr
|
||||
} // namespace phi
|
||||
|
||||
PD_REGISTER_KERNEL(uniform_random_batch_size_like_sr,
|
||||
GPU,
|
||||
ALL_LAYOUT,
|
||||
phi::sr::GPUUniformRandomKernel,
|
||||
float,
|
||||
double,
|
||||
phi::bfloat16) {}
|
||||
Reference in New Issue
Block a user