wehub-resource-sync
460 lines
18 KiB
C++
460 lines
18 KiB
C++
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
#pragma once
|
|
|
|
#include <vector>
|
|
|
|
#include "paddle/common/enforce.h"
|
|
#include "paddle/phi/backends/dynload/warpctc.h"
|
|
#include "paddle/phi/core/dense_tensor.h"
|
|
#include "paddle/phi/core/lod_utils.h"
|
|
#include "paddle/phi/core/tensor_utils.h"
|
|
#include "paddle/phi/kernels/empty_kernel.h"
|
|
#include "paddle/phi/kernels/funcs/math_function.h"
|
|
#include "paddle/phi/kernels/funcs/sequence_padding.h"
|
|
#include "paddle/phi/kernels/funcs/sequence_scale.h"
|
|
#include "paddle/utils/optional.h"
|
|
|
|
namespace phi {
|
|
|
|
template <typename Context, typename T>
|
|
class ComputeCtcLossFunctor {
|
|
public:
|
|
ctcStatus_t operator()(const T* const activations,
|
|
T* gradients,
|
|
const int* const flat_labels,
|
|
const int* const label_lengths,
|
|
const int* const input_lengths,
|
|
int alphabet_size,
|
|
int minibatch,
|
|
T* costs,
|
|
void* workspace,
|
|
ctcOptions options) {
|
|
return CTC_STATUS_EXECUTION_FAILED;
|
|
}
|
|
};
|
|
|
|
template <typename Context>
|
|
class ComputeCtcLossFunctor<Context, float> {
|
|
public:
|
|
ctcStatus_t operator()(const float* const activations,
|
|
float* gradients,
|
|
const int* const flat_labels,
|
|
const int* const label_lengths,
|
|
const int* const input_lengths,
|
|
int alphabet_size,
|
|
int minibatch,
|
|
float* costs,
|
|
void* workspace,
|
|
ctcOptions options) {
|
|
return dynload::compute_ctc_loss(activations,
|
|
gradients,
|
|
flat_labels,
|
|
label_lengths,
|
|
input_lengths,
|
|
static_cast<int>(alphabet_size),
|
|
static_cast<int>(minibatch),
|
|
costs,
|
|
workspace,
|
|
options);
|
|
}
|
|
};
|
|
|
|
template <typename Context>
|
|
class ComputeCtcLossFunctor<Context, double> {
|
|
public:
|
|
ctcStatus_t operator()(const double* const activations,
|
|
double* gradients,
|
|
const int* const flat_labels,
|
|
const int* const label_lengths,
|
|
const int* const input_lengths,
|
|
int alphabet_size,
|
|
int minibatch,
|
|
double* costs,
|
|
void* workspace,
|
|
ctcOptions options) {
|
|
return dynload::compute_ctc_loss_double(activations,
|
|
gradients,
|
|
flat_labels,
|
|
label_lengths,
|
|
input_lengths,
|
|
static_cast<int>(alphabet_size),
|
|
static_cast<int>(minibatch),
|
|
costs,
|
|
workspace,
|
|
options);
|
|
}
|
|
};
|
|
|
|
template <typename Context, typename T>
|
|
class WarpCTCFunctor {
|
|
public:
|
|
/*
|
|
* \brief Compute the connectionist temporal classification loss,
|
|
* and optionally compute the gradient with respect to the inputs.
|
|
*
|
|
* If gradient is nullptr, it only computes the ctc loss,
|
|
* or computes both ctc loss and gradient.
|
|
*
|
|
* \param dev_ctx execution context of this functor
|
|
* \param input batch matrix of input probabilities, in
|
|
* max_sequence_length x num_sequences x
|
|
* sequence_width, (row-major) format
|
|
* \param gradient batch matrix of gradient, with the same shape as
|
|
* input.
|
|
* \param cpu_labels labels always in CPU memory.
|
|
* \param cpu_label_lengths length of all labels in CPU memory.
|
|
* \param cpu_input_lengths length of all sequences in CPU memory.
|
|
* \param sequence_width number of possible output symbols.
|
|
* \param num_sequences number of sequence.
|
|
* \param blank blank label used in ctc loss function.
|
|
* \param cpu_losss cost of each sequence in CPU memory.
|
|
*/
|
|
void operator()(const Context& dev_ctx,
|
|
const T* input,
|
|
T* gradient,
|
|
const int* cpu_labels,
|
|
const int* cpu_label_lengths,
|
|
const int* cpu_input_lengths,
|
|
const size_t sequence_width,
|
|
const size_t num_sequences,
|
|
const size_t blank,
|
|
T* cpu_loss) {
|
|
// Init warp-ctc options
|
|
init(dev_ctx, blank);
|
|
PADDLE_ENFORCE_LE_INT_MAX(sequence_width, "warpctc sequence width");
|
|
PADDLE_ENFORCE_LE_INT_MAX(num_sequences, "warpctc num sequences");
|
|
const int sequence_width_int = static_cast<int>(sequence_width);
|
|
const int num_sequences_int = static_cast<int>(num_sequences);
|
|
|
|
// Compute the required workspace size.
|
|
// There is no memory allocated operations within warp-ctc.
|
|
size_t workspace_bytes = 0;
|
|
ctcStatus_t status = CTC_STATUS_UNKNOWN_ERROR;
|
|
if (sizeof(T) == 4) {
|
|
status = dynload::get_workspace_size(cpu_label_lengths,
|
|
cpu_input_lengths,
|
|
sequence_width_int,
|
|
num_sequences_int,
|
|
options_,
|
|
&workspace_bytes);
|
|
} else {
|
|
status = dynload::get_workspace_size_double(cpu_label_lengths,
|
|
cpu_input_lengths,
|
|
sequence_width_int,
|
|
num_sequences_int,
|
|
options_,
|
|
&workspace_bytes);
|
|
}
|
|
PADDLE_ENFORCE_EQ(
|
|
CTC_STATUS_SUCCESS,
|
|
status,
|
|
errors::PreconditionNotMet(
|
|
"warp-ctc [version %d] Error in get_workspace_size: %s",
|
|
warpctc_version_,
|
|
dynload::ctcGetStatusString(status)));
|
|
PADDLE_ENFORCE_GT(
|
|
workspace_bytes,
|
|
0UL,
|
|
errors::InvalidArgument(
|
|
"Bytes of workspace got by warp-ctc function, "
|
|
"get_workspace_size() should be larger than 0, but received %d",
|
|
workspace_bytes));
|
|
|
|
size_t workspace_elements = workspace_bytes / sizeof(T) + 1UL;
|
|
DenseTensor workspace =
|
|
Empty<T, Context>(dev_ctx, {static_cast<int64_t>(workspace_elements)});
|
|
T* workspace_data = workspace.data<T>();
|
|
funcs::SetConstant<Context, T>()(dev_ctx, &workspace, static_cast<T>(0));
|
|
|
|
// compute loss and gradient
|
|
status = ComputeCtcLossFunctor<Context, T>()(input,
|
|
gradient,
|
|
cpu_labels,
|
|
cpu_label_lengths,
|
|
cpu_input_lengths,
|
|
sequence_width_int,
|
|
num_sequences_int,
|
|
cpu_loss,
|
|
workspace_data,
|
|
options_);
|
|
|
|
PADDLE_ENFORCE_EQ(
|
|
CTC_STATUS_SUCCESS,
|
|
status,
|
|
errors::PreconditionNotMet(
|
|
"warp-ctc [version %d] Error in get_workspace_size: %s",
|
|
warpctc_version_,
|
|
dynload::ctcGetStatusString(status)));
|
|
}
|
|
|
|
protected:
|
|
void init(const Context& dev_ctx, const size_t blank) {
|
|
warpctc_version_ = dynload::get_warpctc_version();
|
|
PADDLE_ENFORCE_LE_INT_MAX(blank, "warpctc blank_label");
|
|
|
|
if (dev_ctx.GetPlace().GetType() != AllocationType::CPU) {
|
|
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
|
|
options_.loc = CTC_GPU;
|
|
options_.stream = reinterpret_cast<const GPUContext&>(dev_ctx).stream();
|
|
#else
|
|
PADDLE_THROW(
|
|
errors::PreconditionNotMet("[warpctc init] GPU is not enabled."));
|
|
#endif
|
|
} else {
|
|
options_.loc = CTC_CPU;
|
|
options_.num_threads = 1;
|
|
}
|
|
|
|
options_.blank_label = static_cast<int>(blank);
|
|
}
|
|
|
|
private:
|
|
int warpctc_version_;
|
|
ctcOptions options_;
|
|
};
|
|
|
|
template <typename T, typename Context>
|
|
void WarpctcKernel(const Context& dev_ctx,
|
|
const DenseTensor& logits,
|
|
const DenseTensor& label,
|
|
const optional<DenseTensor>& logits_length,
|
|
const optional<DenseTensor>& labels_length,
|
|
int blank,
|
|
bool norm_by_times UNUSED,
|
|
DenseTensor* loss,
|
|
DenseTensor* warpctcgrad) {
|
|
size_t num_sequences, sequence_width, max_sequence_length;
|
|
Vector<size_t> logits_lod;
|
|
Vector<size_t> label_lod;
|
|
if (logits_length.is_initialized() && labels_length.is_initialized()) {
|
|
num_sequences = logits.dims()[1];
|
|
sequence_width = logits.dims()[2];
|
|
max_sequence_length = logits.dims()[0];
|
|
|
|
PADDLE_ENFORCE_GT(max_sequence_length,
|
|
0,
|
|
common::errors::InvalidArgument(
|
|
"The first dimension of Input(Logits) should be "
|
|
"greater than zero "
|
|
"but received %d. ",
|
|
max_sequence_length));
|
|
PADDLE_ENFORCE_GT(num_sequences,
|
|
0,
|
|
common::errors::InvalidArgument(
|
|
"The second dimension of Input(Logits) should be "
|
|
"greater than zero "
|
|
"but received %d. ",
|
|
num_sequences));
|
|
PADDLE_ENFORCE_GT(sequence_width,
|
|
0,
|
|
common::errors::InvalidArgument(
|
|
"The third dimension of Input(Logits) should be "
|
|
"greater than zero "
|
|
"but received %d. ",
|
|
sequence_width));
|
|
PADDLE_ENFORCE_LT(
|
|
num_sequences * sequence_width * max_sequence_length,
|
|
std::numeric_limits<int>::max(),
|
|
errors::InvalidArgument(
|
|
"The total number of elements in Input(Logits) should be less than "
|
|
"%d, "
|
|
"but received %d",
|
|
std::numeric_limits<int>::max(),
|
|
num_sequences * sequence_width * max_sequence_length));
|
|
|
|
DenseTensor logits_length_cpu;
|
|
DenseTensor labels_length_cpu;
|
|
Copy(dev_ctx, *logits_length, CPUPlace(), false, &logits_length_cpu);
|
|
Copy(dev_ctx, *labels_length, CPUPlace(), false, &labels_length_cpu);
|
|
|
|
logits_lod.push_back(0);
|
|
label_lod.push_back(0);
|
|
for (size_t i = 0; i < num_sequences; i++) {
|
|
logits_lod.push_back(logits_lod[i] +
|
|
logits_length_cpu.data<int64_t>()[i]);
|
|
label_lod.push_back(label_lod[i] + labels_length_cpu.data<int64_t>()[i]);
|
|
}
|
|
} else {
|
|
PADDLE_ENFORCE_GT(
|
|
logits.NumLevels(),
|
|
0UL,
|
|
common::errors::InvalidArgument("Input(Logits) Tensor of WarpCTC "
|
|
"does not contain LoD information."));
|
|
PADDLE_ENFORCE_GT(
|
|
label.NumLevels(),
|
|
0UL,
|
|
common::errors::InvalidArgument("Input(Label) Tensor of WarpCTC "
|
|
"does not contain LoD information."));
|
|
|
|
logits_lod = ToAbsOffset(logits.lod())[0];
|
|
auto logits_dims = logits.dims();
|
|
|
|
PADDLE_ENFORCE_GT(logits_dims[0],
|
|
0,
|
|
common::errors::InvalidArgument(
|
|
"The first dimension of Input(Logits) should be "
|
|
"greater than zero "
|
|
"but received %d. ",
|
|
logits_dims[0]));
|
|
|
|
PADDLE_ENFORCE_EQ(
|
|
logits_dims[0],
|
|
static_cast<int64_t>(logits_lod.back()),
|
|
common::errors::InvalidArgument(
|
|
"The first dimension of Input(Logits) should be equal to "
|
|
"the sum of all sequences' lengths = %d., but received %d. ",
|
|
static_cast<int64_t>(logits_lod.back()),
|
|
logits_dims[0]));
|
|
|
|
label_lod = ToAbsOffset(label.lod())[0];
|
|
auto label_dims = label.dims();
|
|
PADDLE_ENFORCE_EQ(label_dims[1],
|
|
1,
|
|
common::errors::InvalidArgument(
|
|
"The last dimension of Input(Label) should be 1, "
|
|
"but received %d",
|
|
label_dims[1]));
|
|
|
|
num_sequences = logits_lod.size() - 1;
|
|
PADDLE_ENFORCE_EQ(num_sequences,
|
|
label_lod.size() - 1,
|
|
common::errors::InvalidArgument(
|
|
"The number of sequences of Input(Logits) should be "
|
|
"equal to that of Input(Label) = %d, but received %d",
|
|
label_lod.size() - 1,
|
|
num_sequences));
|
|
|
|
sequence_width = logits.numel() / logits_dims[0];
|
|
max_sequence_length = funcs::MaximumSequenceLength(logits_lod);
|
|
}
|
|
|
|
auto loss_dims = make_ddim({static_cast<int64_t>(num_sequences), 1});
|
|
|
|
// warpctc needs sequences data stored in transposed padding format
|
|
DenseTensor warpctc_logits_tmp =
|
|
Empty<T, Context>(dev_ctx,
|
|
{static_cast<int64_t>(max_sequence_length),
|
|
static_cast<int64_t>(num_sequences),
|
|
static_cast<int64_t>(sequence_width)});
|
|
DenseTensor warpctc_logits(warpctc_logits_tmp);
|
|
|
|
if (logits_length.is_initialized()) {
|
|
Copy(dev_ctx, logits, dev_ctx.GetPlace(), true, &warpctc_logits);
|
|
} else {
|
|
DenseTensor cpu_pad_value;
|
|
cpu_pad_value.Resize({1});
|
|
T* pad_value_data = dev_ctx.template HostAlloc<T>(&cpu_pad_value);
|
|
*pad_value_data = static_cast<T>(0);
|
|
DenseTensor pad_value;
|
|
if (dev_ctx.GetPlace() == CPUPlace()) {
|
|
pad_value = cpu_pad_value;
|
|
} else {
|
|
Copy(dev_ctx, cpu_pad_value, dev_ctx.GetPlace(), true, &pad_value);
|
|
}
|
|
|
|
funcs::PaddingDenseTensorFunctor<Context, T>()(dev_ctx,
|
|
logits,
|
|
&warpctc_logits,
|
|
pad_value,
|
|
-1,
|
|
0,
|
|
false /* norm_by_times */,
|
|
funcs::kLengthBatchWidth);
|
|
}
|
|
|
|
const T* warpctc_logits_data = warpctc_logits.data<T>();
|
|
PADDLE_ENFORCE_LE_INT_MAX(label.dims()[1], "warpctc label pad length");
|
|
const int label_pad_length = static_cast<int>(label.dims()[1]);
|
|
|
|
std::vector<int> warpctc_label_lengths(num_sequences);
|
|
std::vector<int> warpctc_logits_lengths(num_sequences);
|
|
|
|
for (size_t i = 0; i < num_sequences; ++i) {
|
|
const size_t label_length = label_lod[i + 1] - label_lod[i];
|
|
const size_t logits_length = logits_lod[i + 1] - logits_lod[i];
|
|
PADDLE_ENFORCE_LE_INT_MAX(label_length, "warpctc label length");
|
|
PADDLE_ENFORCE_LE_INT_MAX(logits_length, "warpctc logits length");
|
|
warpctc_label_lengths[i] = static_cast<int>(label_length);
|
|
warpctc_logits_lengths[i] = static_cast<int>(logits_length);
|
|
}
|
|
|
|
// warpctc computes loss and gradient in one call, gradient data also stored
|
|
// in batch format
|
|
warpctcgrad->Resize(warpctc_logits.dims());
|
|
T* warpctcgrad_data = dev_ctx.template Alloc<T>(warpctcgrad);
|
|
|
|
funcs::SetConstant<Context, T>()(dev_ctx, warpctcgrad, static_cast<T>(0));
|
|
|
|
// warpctc accesses labels in CPU memory
|
|
DenseTensor warpctc_label;
|
|
if (logits_length.is_initialized()) {
|
|
warpctc_label.Resize(
|
|
{static_cast<int64_t>(funcs::TotalSequenceLength(label_lod)), 1});
|
|
dev_ctx.template HostAlloc<int>(&warpctc_label);
|
|
std::vector<Vector<size_t>> lod;
|
|
lod.push_back(label_lod);
|
|
warpctc_label.set_lod(lod);
|
|
|
|
if (dev_ctx.GetPlace() == CPUPlace()) {
|
|
funcs::UnpaddingDenseTensorFunctor<Context, int>()(
|
|
dev_ctx,
|
|
label,
|
|
&warpctc_label,
|
|
label_pad_length /*pad_seq_len*/,
|
|
0 /*lod_level*/,
|
|
false /*norm_by_times*/,
|
|
funcs::kBatchLengthWidth);
|
|
} else {
|
|
DenseTensor gpu_label;
|
|
gpu_label.Resize(
|
|
{static_cast<int64_t>(funcs::TotalSequenceLength(label_lod)), 1});
|
|
dev_ctx.template Alloc<int>(&gpu_label);
|
|
gpu_label.set_lod(lod);
|
|
funcs::UnpaddingDenseTensorFunctor<Context, int>()(
|
|
dev_ctx,
|
|
label,
|
|
&gpu_label,
|
|
label_pad_length /*pad_seq_len*/,
|
|
0 /*lod_level*/,
|
|
false /*norm_by_times*/,
|
|
funcs::kBatchLengthWidth);
|
|
Copy(dev_ctx, gpu_label, CPUPlace(), true, &warpctc_label);
|
|
}
|
|
} else {
|
|
Copy(dev_ctx, label, CPUPlace(), true, &warpctc_label);
|
|
}
|
|
|
|
const int* warpctc_label_data = warpctc_label.data<int>();
|
|
// warpctc stores loss in CPU memory
|
|
DenseTensor warpctc_loss;
|
|
warpctc_loss.Resize(loss_dims);
|
|
T* warpctc_loss_data = dev_ctx.template HostAlloc<T>(&warpctc_loss);
|
|
WarpCTCFunctor<Context, T>()(dev_ctx,
|
|
warpctc_logits_data,
|
|
warpctcgrad_data,
|
|
warpctc_label_data,
|
|
warpctc_label_lengths.data(),
|
|
warpctc_logits_lengths.data(),
|
|
sequence_width,
|
|
num_sequences,
|
|
blank,
|
|
warpctc_loss_data);
|
|
// Copy the loss back
|
|
Copy(dev_ctx, warpctc_loss, dev_ctx.GetPlace(), false, loss);
|
|
}
|
|
|
|
} // namespace phi
|