// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include #include #include "paddle/phi/backends/cpu/cpu_context.h" #include "paddle/phi/backends/gpu/gpu_context.h" #include "paddle/phi/common/amp_type_traits.h" #include "paddle/phi/common/data_type.h" #include "paddle/phi/common/place.h" #include "paddle/phi/core/dense_tensor.h" // TODO(xiongkun): remove the header when decouple the memcpy function in phi. #include "paddle/phi/common/memory_utils.h" namespace phi { template struct GetTensorValue { T operator()(const Context& dev_ctx, const DenseTensor& tensor) const; }; template struct IscloseFunctor { void operator()(const Context& dev_ctx, const DenseTensor& in, const DenseTensor& other, const float rtol, const float atol, bool equal_nan, DenseTensor* output); }; template struct GetTensorValue { T operator()(const CPUContext& dev_ctx, const DenseTensor& tensor) const { return *(tensor.data()); } }; template struct IscloseFunctor { void operator()(const CPUContext& dev_ctx, const DenseTensor& in, const DenseTensor& other, const double rtol, const double atol, bool equal_nan, DenseTensor* output) { auto* in_a = in.data(); auto* in_b = other.data(); auto* out_data = dev_ctx.template Alloc(output); int64_t num = in.numel(); // *out_data = true; for (int64_t i = 0; i < num; i++) { out_data[i] = true; } for (int64_t i = 0; i < num; i++) { const T a = in_a[i], b = in_b[i]; bool val; if (std::isnan(a) || std::isnan(b)) { val = equal_nan && std::isnan(a) == std::isnan(b); } else { T left = (a > b ? a - b : b - a); T right = atol + (b > 0 ? rtol * b : (-rtol) * b); T diff = (left > right ? left - right : right - left); val = a == b || left <= right || diff <= 1e-15; } // *out_data &= val; out_data[i] = val; } } }; template struct IscloseFunctor> { void operator()(const CPUContext& dev_ctx, const DenseTensor& in, const DenseTensor& other, const double rtol, const double atol, bool equal_nan, DenseTensor* output) { auto* in_a = in.data>(); auto* in_b = other.data>(); auto* out_data = dev_ctx.template Alloc(output); int64_t num = in.numel(); // *out_data = true; for (int64_t i = 0; i < num; i++) { out_data[i] = true; } for (int64_t i = 0; i < num; i++) { const dtype::complex a = in_a[i], b = in_b[i]; bool val; if (std::isnan(a) || std::isnan(b)) { val = equal_nan && std::isnan(a) == std::isnan(b); } else { T left = abs(a - b); T right = atol + rtol * abs(b); T diff = abs(left - right); val = a == b || left <= right || diff <= 1e-15; // *out_data &= val; out_data[i] = val; } } } }; #if defined(__NVCC__) || defined(__HIPCC__) template __global__ void IscloseCUDAKernel(const T* in_data, const T* other_data, const double rtol, const double atol, bool equal_nan, IndexType num, bool* out_data) { IndexType idx = static_cast(blockIdx.x) * static_cast(blockDim.x) + static_cast(threadIdx.x); bool val; using MPType = typename MPTypeTrait::Type; for (IndexType i = idx; i < num; i += blockDim.x * gridDim.x) { const MPType a = static_cast(in_data[i]); const MPType b = static_cast(other_data[i]); if (isnan(a) || isnan(b)) { val = equal_nan && isnan(a) == isnan(b); } else { MPType left = (a > b ? a - b : b - a); MPType right = atol + (b > 0 ? rtol * b : (-rtol) * b); MPType diff = (left > right ? left - right : right - left); val = a == b || left <= right || diff <= 1e-15; } out_data[i] = val; // if (!val) *out_data = false; } } template <> __global__ void IscloseCUDAKernel( const complex64* in_data, const complex64* other_data, const double rtol, const double atol, bool equal_nan, unsigned int num, bool* out_data) { unsigned int idx = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; bool val; for (unsigned int i = idx; i < num; i += blockDim.x * gridDim.x) { const complex64 a = in_data[i]; const complex64 b = other_data[i]; if (isnan(a) || isnan(b)) { val = equal_nan && isnan(a) == isnan(b); } else { float left = abs(a - b); float right = atol + rtol * abs(b); float diff = abs(left - right); val = a == b || left <= right || diff <= 1e-15; } out_data[i] = val; // if (!val) *out_data = false; } } template <> __global__ void IscloseCUDAKernel( const complex64* in_data, const complex64* other_data, const double rtol, const double atol, bool equal_nan, int64_t num, bool* out_data) { int64_t idx = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; bool val; for (int64_t i = idx; i < num; i += blockDim.x * gridDim.x) { const complex64 a = in_data[i]; const complex64 b = other_data[i]; if (isnan(a) || isnan(b)) { val = equal_nan && isnan(a) == isnan(b); } else { float left = abs(a - b); float right = atol + rtol * abs(b); float diff = abs(left - right); val = a == b || left <= right || diff <= 1e-15; } out_data[i] = val; // if (!val) *out_data = false; } } template <> __global__ void IscloseCUDAKernel( const complex128* in_data, const complex128* other_data, const double rtol, const double atol, bool equal_nan, unsigned int num, bool* out_data) { unsigned int idx = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; bool val; for (unsigned int i = idx; i < num; i += blockDim.x * gridDim.x) { const complex128 a = in_data[i]; const complex128 b = other_data[i]; if (isnan(a) || isnan(b)) { val = equal_nan && isnan(a) == isnan(b); } else { double left = abs(a - b); double right = atol + rtol * abs(b); double diff = abs(left - right); val = a == b || left <= right || diff <= 1e-15; } out_data[i] = val; // if (!val) *out_data = false; } } template <> __global__ void IscloseCUDAKernel( const complex128* in_data, const complex128* other_data, const double rtol, const double atol, bool equal_nan, int64_t num, bool* out_data) { int64_t idx = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; bool val; for (int64_t i = idx; i < num; i += blockDim.x * gridDim.x) { const complex128 a = in_data[i]; const complex128 b = other_data[i]; if (isnan(a) || isnan(b)) { val = equal_nan && isnan(a) == isnan(b); } else { double left = abs(a - b); double right = atol + rtol * abs(b); double diff = abs(left - right); val = a == b || left <= right || diff <= 1e-15; } out_data[i] = val; // if (!val) *out_data = false; } } template struct GetTensorValue { T operator()(const GPUContext& dev_ctx, const DenseTensor& tensor) const { const T* data = tensor.data(); T value; const auto gpu_place = dev_ctx.GetPlace(); memory_utils::Copy( CPUPlace(), &value, gpu_place, data, sizeof(T), dev_ctx.stream()); return value; } }; template struct IscloseFunctor { void operator()(const GPUContext& dev_ctx, const DenseTensor& in, const DenseTensor& other, const double rtol, const double atol, bool equal_nan, DenseTensor* output) { int64_t num = in.numel(); const T* in_data = in.data(); const T* other_data = other.data(); bool* out_data = dev_ctx.template Alloc(output); int64_t block = 1024; int64_t grid = (block - 1 + num) / block; grid = (grid > block) ? block : grid; #ifdef PADDLE_WITH_HIP hipMemset(out_data, true, num * sizeof(bool)); #else cudaMemset(out_data, true, num * sizeof(bool)); #endif if (num + grid * block + 1 > std::numeric_limits::max()) { IscloseCUDAKernel<<>>( in_data, other_data, rtol, atol, equal_nan, num, out_data); } else { IscloseCUDAKernel<<>>( in_data, other_data, rtol, atol, equal_nan, num, out_data); } } }; #endif template void IscloseKernel(const Context& dev_ctx, const DenseTensor& x, const DenseTensor& y, const Scalar& rtol, const Scalar& atol, bool equal_nan, DenseTensor* out) { if (x.numel() == 0) { dev_ctx.template Alloc(out); return; } PADDLE_ENFORCE_EQ( atol.dtype(), DataType::FLOAT64, common::errors::InvalidArgument("Input(Atol) type must be double")); PADDLE_ENFORCE_EQ( rtol.dtype(), DataType::FLOAT64, common::errors::InvalidArgument("Input(Rtol) type must be double")); IscloseFunctor()( dev_ctx, x, y, rtol.to(), atol.to(), equal_nan, out); } } // namespace phi