wehub-resource-sync
330 lines
10 KiB
C++
330 lines
10 KiB
C++
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
#pragma once
|
|
#include <cmath>
|
|
#include <string>
|
|
|
|
#include "paddle/phi/backends/cpu/cpu_context.h"
|
|
#include "paddle/phi/backends/gpu/gpu_context.h"
|
|
#include "paddle/phi/common/amp_type_traits.h"
|
|
#include "paddle/phi/common/data_type.h"
|
|
#include "paddle/phi/common/place.h"
|
|
#include "paddle/phi/core/dense_tensor.h"
|
|
|
|
// TODO(xiongkun): remove the header when decouple the memcpy function in phi.
|
|
#include "paddle/phi/common/memory_utils.h"
|
|
|
|
namespace phi {
|
|
template <typename Context, typename T>
|
|
struct GetTensorValue {
|
|
T operator()(const Context& dev_ctx, const DenseTensor& tensor) const;
|
|
};
|
|
|
|
template <typename Context, typename T>
|
|
struct IscloseFunctor {
|
|
void operator()(const Context& dev_ctx,
|
|
const DenseTensor& in,
|
|
const DenseTensor& other,
|
|
const float rtol,
|
|
const float atol,
|
|
bool equal_nan,
|
|
DenseTensor* output);
|
|
};
|
|
|
|
template <typename T>
|
|
struct GetTensorValue<CPUContext, T> {
|
|
T operator()(const CPUContext& dev_ctx, const DenseTensor& tensor) const {
|
|
return *(tensor.data<T>());
|
|
}
|
|
};
|
|
|
|
template <typename T>
|
|
struct IscloseFunctor<CPUContext, T> {
|
|
void operator()(const CPUContext& dev_ctx,
|
|
const DenseTensor& in,
|
|
const DenseTensor& other,
|
|
const double rtol,
|
|
const double atol,
|
|
bool equal_nan,
|
|
DenseTensor* output) {
|
|
auto* in_a = in.data<T>();
|
|
auto* in_b = other.data<T>();
|
|
auto* out_data = dev_ctx.template Alloc<bool>(output);
|
|
int64_t num = in.numel();
|
|
// *out_data = true;
|
|
for (int64_t i = 0; i < num; i++) {
|
|
out_data[i] = true;
|
|
}
|
|
for (int64_t i = 0; i < num; i++) {
|
|
const T a = in_a[i], b = in_b[i];
|
|
bool val;
|
|
if (std::isnan(a) || std::isnan(b)) {
|
|
val = equal_nan && std::isnan(a) == std::isnan(b);
|
|
} else {
|
|
T left = (a > b ? a - b : b - a);
|
|
T right = atol + (b > 0 ? rtol * b : (-rtol) * b);
|
|
T diff = (left > right ? left - right : right - left);
|
|
val = a == b || left <= right || diff <= 1e-15;
|
|
}
|
|
// *out_data &= val;
|
|
out_data[i] = val;
|
|
}
|
|
}
|
|
};
|
|
|
|
template <typename T>
|
|
struct IscloseFunctor<CPUContext, dtype::complex<T>> {
|
|
void operator()(const CPUContext& dev_ctx,
|
|
const DenseTensor& in,
|
|
const DenseTensor& other,
|
|
const double rtol,
|
|
const double atol,
|
|
bool equal_nan,
|
|
DenseTensor* output) {
|
|
auto* in_a = in.data<dtype::complex<T>>();
|
|
auto* in_b = other.data<dtype::complex<T>>();
|
|
auto* out_data = dev_ctx.template Alloc<bool>(output);
|
|
int64_t num = in.numel();
|
|
// *out_data = true;
|
|
for (int64_t i = 0; i < num; i++) {
|
|
out_data[i] = true;
|
|
}
|
|
for (int64_t i = 0; i < num; i++) {
|
|
const dtype::complex<T> a = in_a[i], b = in_b[i];
|
|
bool val;
|
|
if (std::isnan(a) || std::isnan(b)) {
|
|
val = equal_nan && std::isnan(a) == std::isnan(b);
|
|
} else {
|
|
T left = abs(a - b);
|
|
T right = atol + rtol * abs(b);
|
|
T diff = abs(left - right);
|
|
val = a == b || left <= right || diff <= 1e-15;
|
|
// *out_data &= val;
|
|
out_data[i] = val;
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
#if defined(__NVCC__) || defined(__HIPCC__)
|
|
template <typename T, typename IndexType>
|
|
__global__ void IscloseCUDAKernel(const T* in_data,
|
|
const T* other_data,
|
|
const double rtol,
|
|
const double atol,
|
|
bool equal_nan,
|
|
IndexType num,
|
|
bool* out_data) {
|
|
IndexType idx =
|
|
static_cast<IndexType>(blockIdx.x) * static_cast<IndexType>(blockDim.x) +
|
|
static_cast<IndexType>(threadIdx.x);
|
|
bool val;
|
|
using MPType = typename MPTypeTrait<T>::Type;
|
|
for (IndexType i = idx; i < num; i += blockDim.x * gridDim.x) {
|
|
const MPType a = static_cast<MPType>(in_data[i]);
|
|
const MPType b = static_cast<MPType>(other_data[i]);
|
|
if (isnan(a) || isnan(b)) {
|
|
val = equal_nan && isnan(a) == isnan(b);
|
|
} else {
|
|
MPType left = (a > b ? a - b : b - a);
|
|
MPType right = atol + (b > 0 ? rtol * b : (-rtol) * b);
|
|
MPType diff = (left > right ? left - right : right - left);
|
|
val = a == b || left <= right || diff <= 1e-15;
|
|
}
|
|
out_data[i] = val;
|
|
// if (!val) *out_data = false;
|
|
}
|
|
}
|
|
template <>
|
|
__global__ void IscloseCUDAKernel<complex64, unsigned int>(
|
|
const complex64* in_data,
|
|
const complex64* other_data,
|
|
const double rtol,
|
|
const double atol,
|
|
bool equal_nan,
|
|
unsigned int num,
|
|
bool* out_data) {
|
|
unsigned int idx =
|
|
static_cast<unsigned int>(blockIdx.x) * blockDim.x + threadIdx.x;
|
|
bool val;
|
|
for (unsigned int i = idx; i < num; i += blockDim.x * gridDim.x) {
|
|
const complex64 a = in_data[i];
|
|
const complex64 b = other_data[i];
|
|
if (isnan(a) || isnan(b)) {
|
|
val = equal_nan && isnan(a) == isnan(b);
|
|
} else {
|
|
float left = abs(a - b);
|
|
float right = atol + rtol * abs(b);
|
|
float diff = abs(left - right);
|
|
val = a == b || left <= right || diff <= 1e-15;
|
|
}
|
|
out_data[i] = val;
|
|
// if (!val) *out_data = false;
|
|
}
|
|
}
|
|
|
|
template <>
|
|
__global__ void IscloseCUDAKernel<complex64, int64_t>(
|
|
const complex64* in_data,
|
|
const complex64* other_data,
|
|
const double rtol,
|
|
const double atol,
|
|
bool equal_nan,
|
|
int64_t num,
|
|
bool* out_data) {
|
|
int64_t idx = static_cast<int64_t>(blockIdx.x) * blockDim.x + threadIdx.x;
|
|
bool val;
|
|
for (int64_t i = idx; i < num; i += blockDim.x * gridDim.x) {
|
|
const complex64 a = in_data[i];
|
|
const complex64 b = other_data[i];
|
|
if (isnan(a) || isnan(b)) {
|
|
val = equal_nan && isnan(a) == isnan(b);
|
|
} else {
|
|
float left = abs(a - b);
|
|
float right = atol + rtol * abs(b);
|
|
float diff = abs(left - right);
|
|
val = a == b || left <= right || diff <= 1e-15;
|
|
}
|
|
out_data[i] = val;
|
|
// if (!val) *out_data = false;
|
|
}
|
|
}
|
|
|
|
template <>
|
|
__global__ void IscloseCUDAKernel<complex128, unsigned int>(
|
|
const complex128* in_data,
|
|
const complex128* other_data,
|
|
const double rtol,
|
|
const double atol,
|
|
bool equal_nan,
|
|
unsigned int num,
|
|
bool* out_data) {
|
|
unsigned int idx =
|
|
static_cast<unsigned int>(blockIdx.x) * blockDim.x + threadIdx.x;
|
|
bool val;
|
|
for (unsigned int i = idx; i < num; i += blockDim.x * gridDim.x) {
|
|
const complex128 a = in_data[i];
|
|
const complex128 b = other_data[i];
|
|
if (isnan(a) || isnan(b)) {
|
|
val = equal_nan && isnan(a) == isnan(b);
|
|
} else {
|
|
double left = abs(a - b);
|
|
double right = atol + rtol * abs(b);
|
|
double diff = abs(left - right);
|
|
val = a == b || left <= right || diff <= 1e-15;
|
|
}
|
|
out_data[i] = val;
|
|
// if (!val) *out_data = false;
|
|
}
|
|
}
|
|
|
|
template <>
|
|
__global__ void IscloseCUDAKernel<complex128, int64_t>(
|
|
const complex128* in_data,
|
|
const complex128* other_data,
|
|
const double rtol,
|
|
const double atol,
|
|
bool equal_nan,
|
|
int64_t num,
|
|
bool* out_data) {
|
|
int64_t idx = static_cast<int64_t>(blockIdx.x) * blockDim.x + threadIdx.x;
|
|
bool val;
|
|
for (int64_t i = idx; i < num; i += blockDim.x * gridDim.x) {
|
|
const complex128 a = in_data[i];
|
|
const complex128 b = other_data[i];
|
|
if (isnan(a) || isnan(b)) {
|
|
val = equal_nan && isnan(a) == isnan(b);
|
|
} else {
|
|
double left = abs(a - b);
|
|
double right = atol + rtol * abs(b);
|
|
double diff = abs(left - right);
|
|
val = a == b || left <= right || diff <= 1e-15;
|
|
}
|
|
out_data[i] = val;
|
|
// if (!val) *out_data = false;
|
|
}
|
|
}
|
|
|
|
template <typename T>
|
|
struct GetTensorValue<GPUContext, T> {
|
|
T operator()(const GPUContext& dev_ctx, const DenseTensor& tensor) const {
|
|
const T* data = tensor.data<T>();
|
|
T value;
|
|
const auto gpu_place = dev_ctx.GetPlace();
|
|
memory_utils::Copy(
|
|
CPUPlace(), &value, gpu_place, data, sizeof(T), dev_ctx.stream());
|
|
return value;
|
|
}
|
|
};
|
|
|
|
template <typename T>
|
|
struct IscloseFunctor<GPUContext, T> {
|
|
void operator()(const GPUContext& dev_ctx,
|
|
const DenseTensor& in,
|
|
const DenseTensor& other,
|
|
const double rtol,
|
|
const double atol,
|
|
bool equal_nan,
|
|
DenseTensor* output) {
|
|
int64_t num = in.numel();
|
|
const T* in_data = in.data<T>();
|
|
const T* other_data = other.data<T>();
|
|
bool* out_data = dev_ctx.template Alloc<bool>(output);
|
|
int64_t block = 1024;
|
|
int64_t grid = (block - 1 + num) / block;
|
|
grid = (grid > block) ? block : grid;
|
|
#ifdef PADDLE_WITH_HIP
|
|
hipMemset(out_data, true, num * sizeof(bool));
|
|
#else
|
|
cudaMemset(out_data, true, num * sizeof(bool));
|
|
#endif
|
|
if (num + grid * block + 1 > std::numeric_limits<unsigned int>::max()) {
|
|
IscloseCUDAKernel<T, int64_t><<<grid, block, 0, dev_ctx.stream()>>>(
|
|
in_data, other_data, rtol, atol, equal_nan, num, out_data);
|
|
} else {
|
|
IscloseCUDAKernel<T, unsigned int><<<grid, block, 0, dev_ctx.stream()>>>(
|
|
in_data, other_data, rtol, atol, equal_nan, num, out_data);
|
|
}
|
|
}
|
|
};
|
|
#endif
|
|
|
|
template <typename T, typename Context>
|
|
void IscloseKernel(const Context& dev_ctx,
|
|
const DenseTensor& x,
|
|
const DenseTensor& y,
|
|
const Scalar& rtol,
|
|
const Scalar& atol,
|
|
bool equal_nan,
|
|
DenseTensor* out) {
|
|
if (x.numel() == 0) {
|
|
dev_ctx.template Alloc<bool>(out);
|
|
return;
|
|
}
|
|
PADDLE_ENFORCE_EQ(
|
|
atol.dtype(),
|
|
DataType::FLOAT64,
|
|
common::errors::InvalidArgument("Input(Atol) type must be double"));
|
|
|
|
PADDLE_ENFORCE_EQ(
|
|
rtol.dtype(),
|
|
DataType::FLOAT64,
|
|
common::errors::InvalidArgument("Input(Rtol) type must be double"));
|
|
|
|
IscloseFunctor<Context, T>()(
|
|
dev_ctx, x, y, rtol.to<double>(), atol.to<double>(), equal_nan, out);
|
|
}
|
|
} // namespace phi
|