chore: import upstream snapshot with attribution

test/cpp/fluid/platform/CMakeLists.txt

@@ -0,0 +1,134 @@
+add_subdirectory(device)
+add_subdirectory(profiler)
+
+cc_test(
+  enforce_test
+  SRCS enforce_test.cc
+  DEPS phi common)
+cc_test(
+  place_test
+  SRCS place_test.cc
+  DEPS glog phi common)
+cc_test(
+  errors_test
+  SRCS errors_test.cc
+  DEPS phi common)
+cc_test(
+  cpu_info_test
+  SRCS cpu_info_test.cc
+  DEPS phi common)
+cc_test(
+  os_info_test
+  SRCS os_info_test.cc
+  DEPS phi common)
+cc_test(
+  cpu_helper_test
+  SRCS cpu_helper_test.cc
+  DEPS phi common)
+cc_test(
+  init_test
+  SRCS init_test.cc
+  DEPS phi)
+
+if(WITH_GPU)
+  nv_test(
+    device_event_test
+    SRCS device_event_test.cc
+    DEPS phi common)
+  nv_test(
+    device_context_test
+    SRCS device_context_test.cu
+    DEPS phi common)
+  nv_test(
+    device_context_test_cuda_graph
+    SRCS device_context_test_cuda_graph.cu
+    DEPS phi common)
+endif()
+
+if(WITH_ROCM)
+  hip_test(
+    device_event_test
+    SRCS device_event_test.cc
+    DEPS phi common)
+  hip_test(
+    device_context_test
+    SRCS device_context_test.cu
+    DEPS phi common)
+endif()
+
+cc_test(
+  timer_test
+  SRCS timer_test.cc
+  DEPS phi common)
+
+cc_test(
+  lodtensor_printer_test
+  SRCS lodtensor_printer_test.cc
+  DEPS densetensor_printer)
+
+cc_test(
+  profiler_test
+  SRCS profiler_test.cc
+  DEPS phi common)
+cc_test(
+  float16_test
+  SRCS float16_test.cc
+  DEPS lod_tensor)
+cc_test(
+  bfloat16_test
+  SRCS bfloat16_test.cc
+  DEPS lod_tensor)
+cc_test(
+  complex_test
+  SRCS complex_test.cc
+  DEPS lod_tensor)
+
+if(WITH_GPU)
+  nv_test(
+    float16_gpu_test
+    SRCS float16_test.cu
+    DEPS lod_tensor)
+  nv_test(
+    bfloat16_gpu_test
+    SRCS bfloat16_test.cu
+    DEPS lod_tensor)
+  nv_test(
+    complex_gpu_test
+    SRCS complex_test.cu
+    DEPS lod_tensor)
+  nv_test(
+    test_limit_gpu_memory
+    SRCS test_limit_gpu_memory.cu
+    DEPS phi common)
+endif()
+
+if(WITH_ROCM)
+  hip_test(
+    float16_gpu_test
+    SRCS float16_test.cu
+    DEPS lod_tensor)
+  hip_test(
+    test_limit_gpu_memory
+    SRCS test_limit_gpu_memory.cu
+    DEPS phi common)
+endif()
+
+if(NOT APPLE AND NOT WIN32)
+  if(WITH_GPU OR WITH_ROCM)
+    cc_test(
+      device_code_test
+      SRCS device_code_test.cc
+      DEPS phi common lod_tensor)
+  endif()
+endif()
+
+cc_test(
+  init_phi_test
+  SRCS init_phi_test.cc
+  DEPS phi
+       common
+       init_phi
+       op_dialect
+       op_dialect_vjp
+       static_prim_api
+       primitive_backend_static_experimental)

test/cpp/fluid/platform/bfloat16_test.cc

@@ -0,0 +1,165 @@
+/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/common/bfloat16.h"
+
+#include "paddle/phi/kernels/funcs/eigen/extensions.h"
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/framework/lod_tensor.h"
+#include "paddle/fluid/platform/enforce.h"
+
+namespace paddle::platform {
+
+using bfloat16 = phi::dtype::bfloat16;
+using namespace phi::dtype;  // NOLINT
+
+TEST(bfloat16, conversion_cpu) {
+  // Conversion from float
+  EXPECT_EQ(bfloat16(1.0f).x, 0x3f80);
+  EXPECT_EQ(bfloat16(0.5f).x, 0x3f00);
+  EXPECT_EQ(bfloat16(0.33333f).x, 0x3eab);
+  EXPECT_EQ(bfloat16(0.0f).x, 0x0000);
+  EXPECT_EQ(bfloat16(-0.0f).x, 0x8000);
+  EXPECT_EQ(bfloat16(65504.0f).x, 0x4780);
+  EXPECT_EQ(bfloat16(65536.0f).x, 0x4780);
+
+  // Conversion from double
+  EXPECT_EQ(bfloat16(1.0).x, 0x3f80);
+  EXPECT_EQ(bfloat16(0.5).x, 0x3f00);
+  EXPECT_EQ(bfloat16(0.33333).x, 0x3eab);
+  EXPECT_EQ(bfloat16(0.0).x, 0x0000);
+  EXPECT_EQ(bfloat16(-0.0).x, 0x8000);
+  EXPECT_EQ(bfloat16(65504.0).x, 0x4780);
+  EXPECT_EQ(bfloat16(65536.0).x, 0x4780);
+
+  // Conversion from int
+  EXPECT_EQ(bfloat16(-1).x, 0xbf80);
+  EXPECT_EQ(bfloat16(0).x, 0x0000);
+  EXPECT_EQ(bfloat16(1).x, 0x3f80);
+  EXPECT_EQ(bfloat16(2).x, 0x4000);
+  EXPECT_EQ(bfloat16(3).x, 0x4040);
+
+  // Conversion from bool
+  EXPECT_EQ(bfloat16(true).x, 0x3f80);
+  EXPECT_EQ(bfloat16(false).x, 0x0000);
+
+  // Assignment operator
+  bfloat16 v_assign;
+  v_assign = bfloat16(0.f);
+  EXPECT_EQ(v_assign.x, 0x0000);
+  v_assign = 0.5f;
+  EXPECT_EQ(v_assign.x, 0x3f00);
+  v_assign = 0.33333;
+  EXPECT_EQ(v_assign.x, 0x3eab);
+  v_assign = -1;
+  EXPECT_EQ(v_assign.x, 0xbf80);
+
+  // Conversion operator
+  EXPECT_EQ(static_cast<float>(bfloat16(0.5f)), 0.5f);
+  EXPECT_NEAR(static_cast<double>(bfloat16(0.33333)), 0.33333, 0.01);
+  EXPECT_EQ(static_cast<int>(bfloat16(-1)), -1);
+  EXPECT_EQ(static_cast<bool>(bfloat16(true)), true);
+}
+
+TEST(bfloat16, arithmetic_cpu) {
+  EXPECT_NEAR(static_cast<float>(bfloat16(1) + bfloat16(1)), 2, 0.001);
+  EXPECT_EQ(static_cast<float>(bfloat16(5) + bfloat16(-5)), 0);
+  EXPECT_NEAR(
+      static_cast<float>(bfloat16(0.33333f) + bfloat16(0.66667f)), 1.0f, 0.01);
+  EXPECT_EQ(static_cast<float>(bfloat16(3) - bfloat16(5)), -2);
+  EXPECT_NEAR(static_cast<float>(bfloat16(0.66667f) - bfloat16(0.33333f)),
+              0.33334f,
+              0.01);
+  EXPECT_NEAR(static_cast<float>(bfloat16(3.3f) * bfloat16(2.0f)), 6.6f, 0.01);
+  EXPECT_NEAR(static_cast<float>(bfloat16(-2.1f) * bfloat16(-3.0f)), 6.3f, 0.1);
+  EXPECT_NEAR(
+      static_cast<float>(bfloat16(2.0f) / bfloat16(3.0f)), 0.66667f, 0.01);
+  EXPECT_EQ(static_cast<float>(bfloat16(1.0f) / bfloat16(2.0f)), 0.5f);
+  EXPECT_EQ(static_cast<float>(-bfloat16(512.0f)), -512.0f);
+  EXPECT_EQ(static_cast<float>(-bfloat16(-512.0f)), 512.0f);
+}
+
+TEST(bfloat16, comparison_cpu) {
+  EXPECT_TRUE(bfloat16(1.0f) == bfloat16(1.0f));
+  EXPECT_FALSE(bfloat16(-1.0f) == bfloat16(-0.5f));
+  EXPECT_TRUE(bfloat16(1.0f) != bfloat16(0.5f));
+  EXPECT_FALSE(bfloat16(-1.0f) != bfloat16(-1.0f));
+  EXPECT_TRUE(bfloat16(1.0f) < bfloat16(2.0f));
+  EXPECT_FALSE(bfloat16(-1.0f) < bfloat16(-1.0f));
+  EXPECT_TRUE(bfloat16(1.0f) <= bfloat16(1.0f));
+  EXPECT_TRUE(bfloat16(2.0f) > bfloat16(1.0f));
+  EXPECT_FALSE(bfloat16(-2.0f) > bfloat16(-2.0f));
+  EXPECT_TRUE(bfloat16(2.0f) >= bfloat16(2.0f));
+}
+
+TEST(bfloat16, dense_tensor_cpu) {
+  phi::DenseTensor dense_tensor;
+
+  std::vector<bfloat16> input_data = {
+      bfloat16(1.0f), bfloat16(0.5f), bfloat16(0.33333f), bfloat16(0.0f)};
+  EXPECT_EQ(input_data[0].x, 0x3f80);
+  EXPECT_EQ(input_data[1].x, 0x3f00);
+  EXPECT_EQ(input_data[2].x, 0x3eab);
+  EXPECT_EQ(input_data[3].x, 0x0000);
+
+  dense_tensor.Resize({4, 1});
+  dense_tensor.set_lod(phi::LegacyLoD({{0, 2, 4}}));
+  bfloat16* data_ptr = dense_tensor.mutable_data<bfloat16>(CPUPlace());
+
+  EXPECT_NE(data_ptr, nullptr);
+  EXPECT_EQ(input_data.size(), static_cast<size_t>(dense_tensor.numel()));
+  for (size_t i = 0; i < input_data.size(); ++i) {
+    data_ptr[i] = input_data[i];
+    EXPECT_EQ(data_ptr[i].x, input_data[i].x);
+  }
+}
+
+TEST(bfloat16, floating) {
+  // compile time assert.
+  PADDLE_ENFORCE_EQ(
+      std::is_floating_point<bfloat16>::value,
+      true,
+      common::errors::Fatal("std::is_floating_point with bfloat16 data type "
+                            "should be equal to true but it is not"));
+}
+
+TEST(bfloat16, print) {
+  bfloat16 a = bfloat16(1.0f);
+  std::cout << "a:" << a << std::endl;
+  std::stringstream ss1, ss2;
+  ss1 << a;
+  ss2 << 1.0f;
+  EXPECT_EQ(ss1.str(), ss2.str());
+}
+
+// CPU test
+TEST(bfloat16, isinf) {
+  bfloat16 a;
+  a.x = 0x7f80;
+  bfloat16 b = bfloat16(INFINITY);
+  bfloat16 c = static_cast<bfloat16>(INFINITY);
+  EXPECT_EQ(std::isinf(a), true);
+  EXPECT_EQ(std::isinf(b), true);
+  EXPECT_EQ(std::isinf(c), true);
+}
+
+TEST(bfloat16, isnan) {
+  bfloat16 a;
+  a.x = 0x7fff;
+  bfloat16 b = bfloat16(NAN);
+  bfloat16 c = static_cast<bfloat16>(NAN);
+  EXPECT_EQ(std::isnan(a), true);
+  EXPECT_EQ(std::isnan(b), true);
+  EXPECT_EQ(std::isnan(c), true);
+}
+
+}  // namespace paddle::platform

test/cpp/fluid/platform/bfloat16_test.cu

@@ -0,0 +1,132 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/common/bfloat16.h"
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include <iostream>
+
+#include "paddle/fluid/framework/lod_tensor.h"
+
+#if defined(PADDLE_CUDA_BF16)
+namespace paddle {
+namespace platform {
+
+using bfloat16 = phi::dtype::bfloat16;
+using namespace phi::dtype;  // NOLINT
+
+TEST(bfloat16, convert_float32_to_bfloat16_on_gpu) {
+  // Convert float32 to bfloat16
+  EXPECT_EQ((bfloat16(1.0f)).x, 0x3f80);
+  EXPECT_EQ((bfloat16(0.5f)).x, 0x3f00);
+  EXPECT_EQ((bfloat16(0.33333f)).x, 0x3eab);
+  EXPECT_EQ((bfloat16(0.0f)).x, 0x0000);
+  EXPECT_EQ((bfloat16(-0.0f)).x, 0x8000);
+  EXPECT_EQ((bfloat16(65536.0f)).x, 0x4780);
+}
+
+TEST(bfloat16, assignment_operator_on_gpu) {
+  // Assignment operator
+  bfloat16 v_assign;
+  v_assign = bfloat16(1.0f).to_nv_bfloat16();
+  EXPECT_EQ(v_assign.x, 0x3f80);
+  v_assign = 0.33333;
+  EXPECT_EQ(v_assign.x, 0x3eab);
+}
+
+TEST(bfloat16, convert_bfloat16_to_float32_on_gpu) {
+  // Conversion operator
+  EXPECT_EQ(static_cast<float>(bfloat16(0.5f)), 0.5f);
+  EXPECT_NEAR(static_cast<double>(bfloat16(0.33333)), 0.33333, 0.01);
+  EXPECT_EQ(static_cast<int>(bfloat16(-1)), -1);
+  EXPECT_EQ(static_cast<bool>(bfloat16(true)), true);
+}
+
+TEST(bfloat16, dense_tensor_on_gpu) {
+  phi::DenseTensor src_tensor;
+  phi::DenseTensor gpu_tensor;
+  phi::DenseTensor dst_tensor;
+
+  bfloat16 *src_ptr =
+      src_tensor.mutable_data<bfloat16>(common::make_ddim({2, 2}), CPUPlace());
+
+  bfloat16 arr[4] = {
+      bfloat16(1.0f), bfloat16(0.5f), bfloat16(0.33333f), bfloat16(0.0f)};
+  memcpy(src_ptr, arr, 4 * sizeof(bfloat16));
+
+  // CPU DenseTensor to GPU DenseTensor
+  phi::GPUPlace gpu_place(0);
+  phi::GPUContext gpu_ctx(gpu_place);
+  gpu_ctx.SetAllocator(paddle::memory::allocation::AllocatorFacade::Instance()
+                           .GetAllocator(gpu_place, gpu_ctx.stream())
+                           .get());
+  gpu_ctx.PartialInitWithAllocator();
+  framework::TensorCopy(src_tensor, gpu_place, gpu_ctx, &gpu_tensor);
+
+  // GPU DenseTensor to CPU DenseTensor
+  framework::TensorCopy(gpu_tensor, CPUPlace(), gpu_ctx, &dst_tensor);
+
+  // Sync before comparing DenseTensors
+  gpu_ctx.Wait();
+  const bfloat16 *dst_ptr = dst_tensor.data<bfloat16>();
+  ASSERT_NE(src_ptr, dst_ptr);
+  for (size_t i = 0; i < 4; ++i) {
+    EXPECT_EQ(src_ptr[i].x, dst_ptr[i].x);
+  }
+}
+
+TEST(bfloat16, isinf) {
+  bfloat16 a;
+  a.x = 0x7f80;
+  bfloat16 b = bfloat16(INFINITY);
+  bfloat16 c = static_cast<bfloat16>(INFINITY);
+  EXPECT_EQ(std::isinf(a), true);
+  EXPECT_EQ(std::isinf(b), true);
+  EXPECT_EQ(std::isinf(c), true);
+}
+
+TEST(bfloat16, isnan) {
+  bfloat16 a;
+  a.x = 0x7fff;
+  bfloat16 b = bfloat16(NAN);
+  bfloat16 c = static_cast<bfloat16>(NAN);
+  EXPECT_EQ(std::isnan(a), true);
+  EXPECT_EQ(std::isnan(b), true);
+  EXPECT_EQ(std::isnan(c), true);
+}
+
+TEST(bfloat16, cast) {
+  bfloat16 a;
+  a.x = 0x0070;
+  auto b = a;
+  {
+    // change semantic, keep the same value
+    bfloat16 c = reinterpret_cast<bfloat16 &>(reinterpret_cast<unsigned &>(b));
+    EXPECT_EQ(b, c);
+  }
+
+  {
+    // use uint32 low 16 bit store float16
+    uint32_t c = reinterpret_cast<uint32_t &>(b);
+    bfloat16 d;
+    d.x = c;
+    EXPECT_EQ(b, d);
+  }
+}
+
+}  // namespace platform
+}  // namespace paddle
+#endif

test/cpp/fluid/platform/complex_test.cc

@@ -0,0 +1,326 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/common/complex.h"
+
+#include <complex>
+
+#include "paddle/phi/kernels/funcs/eigen/extensions.h"
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/framework/lod_tensor.h"
+#include "paddle/fluid/platform/enforce.h"
+
+namespace paddle::platform {
+
+template <typename T>
+using complex = phi::dtype::complex<T>;
+
+TEST(complex, conversion_cpu) {
+  // *********** complex<float> *************
+  // float to complex<float>
+  EXPECT_EQ(complex<float>().real, 0.0f);
+  EXPECT_EQ(complex<float>().imag, 0.0f);
+
+  EXPECT_EQ(complex<float>(1.0f, 1.0f).real, 1.0f);
+  EXPECT_EQ(complex<float>(1.0f, 1.0f).imag, 1.0f);
+  EXPECT_EQ(complex<float>(0.0f, 1.0f).real, 0.0f);
+  EXPECT_EQ(complex<float>(0.0f, 1.0f).imag, 1.0f);
+
+  EXPECT_EQ(complex<float>(1.0f).real, 1.0f);
+  EXPECT_EQ(complex<float>(1.0f).imag, 0.0f);
+
+  // int to complex<float>
+  EXPECT_EQ(complex<float>(1).real, 1.0f);
+  EXPECT_EQ(complex<float>(0).real, 0.0f);
+  EXPECT_EQ(complex<float>(2).real, 2.0f);
+  EXPECT_EQ(complex<float>(-2).real, -2.0f);
+
+  // bool to complex
+  EXPECT_EQ(complex<float>(true).real, 1.0f);
+  EXPECT_EQ(complex<float>(true).imag, 0.0f);
+
+  // complex<double> to complex<float>
+  EXPECT_EQ(complex<float>(complex<double>(1.0, 2.0)).real, 1.0f);
+  EXPECT_EQ(complex<float>(complex<double>(1.0, 2.0)).imag, 2.0f);
+
+  // std::complex<float> to complex<float>
+  EXPECT_EQ(complex<float>(std::complex<float>(1.0f, 2.0f)).real, 1.0f);
+  EXPECT_EQ(complex<float>(std::complex<float>(1.0f, 2.0f)).imag, 2.0f);
+  EXPECT_EQ(complex<float>(std::complex<double>(1.0, 2.0)).real, 1.0f);
+  EXPECT_EQ(complex<float>(std::complex<double>(1.0, 2.0)).imag, 2.0f);
+
+  // Assignment operator
+  complex<float> c = 1.0f;
+  EXPECT_EQ(c.real, 1.0f);
+  EXPECT_EQ(c.imag, 0.0f);
+  c = complex<float>(2.0, 2.0);
+  EXPECT_EQ(c.real, 2.0f);
+  EXPECT_EQ(c.imag, 2.0f);
+
+  // Conversion operator
+  EXPECT_EQ(static_cast<float>(complex<float>(0.5f)), 0.5f);
+  EXPECT_NEAR(static_cast<double>(complex<float>(0.33333)), 0.33333, 0.01);
+  EXPECT_EQ(static_cast<int>(complex<float>(-1)), -1);
+  EXPECT_EQ(static_cast<bool>(complex<float>(true)), true);
+
+  // *********** complex<double> *************
+  // double to complex<double>
+  EXPECT_EQ(complex<double>().real, 0.0);
+  EXPECT_EQ(complex<double>().imag, 0.0);
+
+  EXPECT_EQ(complex<double>(1.0, 1.0).real, 1.0);
+  EXPECT_EQ(complex<double>(1.0, 1.0).imag, 1.0);
+  EXPECT_EQ(complex<double>(0.0, 1.0).real, 0.0);
+  EXPECT_EQ(complex<double>(0.0, 1.0).imag, 1.0);
+
+  EXPECT_EQ(complex<double>(1.0).real, 1.0);
+  EXPECT_EQ(complex<double>(1.0).imag, 0.0);
+
+  // int to complex<double>
+  EXPECT_EQ(complex<double>(1).real, 1.0);
+  EXPECT_EQ(complex<double>(0).real, 0.0);
+  EXPECT_EQ(complex<double>(2).real, 2.0);
+  EXPECT_EQ(complex<double>(-2).real, -2.0);
+
+  // bool to complex
+  EXPECT_EQ(complex<double>(true).real, 1.0);
+  EXPECT_EQ(complex<double>(true).imag, 0.0);
+
+  // complex<float> to complex<double>
+  EXPECT_EQ(complex<double>(complex<float>(1.0f, 2.0f)).real, 1.0);
+  EXPECT_EQ(complex<double>(complex<float>(1.0f, 2.0f)).imag, 2.0);
+
+  // std::complex<float> to complex<double>
+  EXPECT_EQ(complex<double>(std::complex<double>(1.0, 2.0)).real, 1.0);
+  EXPECT_EQ(complex<double>(std::complex<double>(1.0, 2.0)).imag, 2.0);
+  EXPECT_EQ(complex<double>(std::complex<double>(1.0, 2.0)).real, 1.0);
+  EXPECT_EQ(complex<double>(std::complex<double>(1.0, 2.0)).imag, 2.0);
+
+  // Assignment operator
+  complex<double> c1 = 1.0;
+  EXPECT_EQ(c1.real, 1.0);
+  EXPECT_EQ(c1.imag, 0.0);
+  c1 = complex<double>(2.0, 2.0);
+  EXPECT_EQ(c1.real, 2.0);
+  EXPECT_EQ(c1.imag, 2.0);
+
+  // Conversion operator
+  EXPECT_EQ(static_cast<double>(complex<double>(0.5)), 0.5);
+  EXPECT_NEAR(static_cast<double>(complex<double>(0.33333)), 0.33333, 0.01);
+  EXPECT_EQ(static_cast<int>(complex<double>(-1)), -1);
+  EXPECT_EQ(static_cast<bool>(complex<double>(true)), true);
+}
+
+TEST(bfloat16, comparison_cpu) {
+  // *********** complex<float> *************
+  EXPECT_TRUE(complex<float>(1.0f) == complex<float>(1.0f));
+  EXPECT_TRUE(complex<float>(1.0f, 2.0f) == complex<float>(1.0f, 2.0f));
+  EXPECT_FALSE(complex<float>(-1.0f) == complex<float>(-0.5f));
+  EXPECT_TRUE(complex<float>(1.0f) != complex<float>(0.5f));
+  EXPECT_FALSE(complex<float>(-1.0f) != complex<float>(-1.0f));
+  EXPECT_TRUE(complex<float>(1.0f) < complex<float>(2.0f));
+  EXPECT_FALSE(complex<float>(-1.0f) < complex<float>(-1.0f));
+  EXPECT_TRUE(complex<float>(1.0f) <= complex<float>(1.0f));
+  EXPECT_TRUE(complex<float>(2.0f) > complex<float>(1.0f));
+  EXPECT_FALSE(complex<float>(-2.0f) > complex<float>(-2.0f));
+  EXPECT_TRUE(complex<float>(2.0f) >= complex<float>(2.0f));
+
+  // *********** complex<double> *************
+  EXPECT_TRUE(complex<double>(1.0) == complex<double>(1.0));
+  EXPECT_TRUE(complex<double>(1.0, 2.0) == complex<double>(1.0, 2.0));
+  EXPECT_FALSE(complex<double>(-1.0) == complex<double>(-0.5f));
+  EXPECT_TRUE(complex<double>(1.0) != complex<double>(0.5f));
+  EXPECT_FALSE(complex<double>(-1.0) != complex<double>(-1.0));
+  EXPECT_TRUE(complex<double>(1.0) < complex<double>(2.0));
+  EXPECT_FALSE(complex<double>(-1.0) < complex<double>(-1.0));
+  EXPECT_TRUE(complex<double>(1.0) <= complex<double>(1.0));
+  EXPECT_TRUE(complex<double>(2.0) > complex<double>(1.0));
+  EXPECT_FALSE(complex<double>(-2.0) > complex<double>(-2.0));
+  EXPECT_TRUE(complex<double>(2.0) >= complex<double>(2.0));
+}
+
+TEST(complex, arithmetic_cpu) {
+  // *********** complex<float> *************
+  complex<float> a = complex<float>(1, 1) + complex<float>(1, 1);
+  EXPECT_NEAR(a.real, 2, 0.001);
+  EXPECT_NEAR(a.imag, 2, 0.001);
+
+  complex<float> b = complex<float>(-5, -5) + complex<float>(5, 5);
+  EXPECT_EQ(b.real, 0);
+  EXPECT_EQ(b.imag, 0);
+
+  complex<float> c =
+      complex<float>(0.33333f, 0.33333f) + complex<float>(0.66667f, 0.66667f);
+  EXPECT_NEAR(c.real, 1.0f, 0.01);
+  EXPECT_NEAR(c.imag, 1.0f, 0.01);
+
+  complex<float> d = complex<float>(3) - complex<float>(5);
+  EXPECT_EQ(d.real, -2);
+  EXPECT_EQ(d.imag, 0);
+
+  complex<float> e =
+      complex<float>(0.66667f, 0.66667f) - complex<float>(0.33333f, 0.33333f);
+  EXPECT_NEAR(e.real, 0.33334f, 0.01);
+  EXPECT_NEAR(e.imag, 0.33334f, 0.01);
+
+  complex<float> f = complex<float>(0.33f, 0.33f) * complex<float>(0.2f, 0.2f);
+  EXPECT_NEAR(f.real, 0.0f, 0.01);
+  EXPECT_NEAR(f.imag, 0.132f, 0.01);
+
+  complex<float> g = complex<float>(0.33f, 0.33f) / complex<float>(0.2f, 0.2f);
+  EXPECT_NEAR(g.real, 1.65f, 0.01);
+  EXPECT_NEAR(g.imag, 0.0f, 0.01);
+
+  complex<float> h = -complex<float>(0.33f, 0.33f);
+  EXPECT_NEAR(h.real, -0.33f, 0.01);
+  EXPECT_NEAR(h.imag, -0.33f, 0.01);
+  h = -complex<float>(-0.33f, -0.33f);
+  EXPECT_NEAR(h.real, 0.33f, 0.01);
+  EXPECT_NEAR(h.imag, 0.33f, 0.01);
+
+  complex<float> i = complex<float>(1.0, 1.0);
+  i += complex<float>(2.0, 2.0);
+  EXPECT_NEAR(i.real, 3.0f, 0.01);
+  EXPECT_NEAR(i.imag, 3.0f, 0.01);
+  i -= complex<float>(1.0, 1.0);
+  EXPECT_NEAR(i.real, 2.0f, 0.01);
+  EXPECT_NEAR(i.imag, 2.0f, 0.01);
+  i *= complex<float>(3, 2);
+  EXPECT_NEAR(i.real, 2.0f, 0.01);
+  EXPECT_NEAR(i.imag, 10.0f, 0.01);
+  i /= complex<float>(3, 2);
+  EXPECT_NEAR(i.real, 2.0f, 0.01);
+  EXPECT_NEAR(i.imag, 2.0f, 0.01);
+
+  // *********** complex<double> *************
+  complex<double> a1 = complex<double>(1, 1) + complex<double>(1, 1);
+  EXPECT_NEAR(a1.real, 2, 0.001);
+  EXPECT_NEAR(a1.imag, 2, 0.001);
+
+  complex<double> b1 = complex<double>(-5, -5) + complex<double>(5, 5);
+  EXPECT_EQ(b1.real, 0);
+  EXPECT_EQ(b1.imag, 0);
+
+  complex<double> c1 =
+      complex<double>(0.33333f, 0.33333f) + complex<double>(0.66667f, 0.66667f);
+  EXPECT_NEAR(c1.real, 1.0f, 0.01);
+  EXPECT_NEAR(c1.imag, 1.0f, 0.01);
+
+  complex<double> d1 = complex<double>(3) - complex<double>(5);
+  EXPECT_EQ(d1.real, -2);
+  EXPECT_EQ(d1.imag, 0);
+
+  complex<double> e1 =
+      complex<double>(0.66667f, 0.66667f) - complex<double>(0.33333f, 0.33333f);
+  EXPECT_NEAR(e1.real, 0.33334f, 0.01);
+  EXPECT_NEAR(e1.imag, 0.33334f, 0.01);
+
+  complex<double> f1 =
+      complex<double>(0.33f, 0.33f) * complex<double>(0.2f, 0.2f);
+  EXPECT_NEAR(f1.real, 0.0f, 0.01);
+  EXPECT_NEAR(f1.imag, 0.132f, 0.01);
+
+  complex<double> g1 =
+      complex<double>(0.33f, 0.33f) / complex<double>(0.2f, 0.2f);
+  EXPECT_NEAR(g1.real, 1.65f, 0.01);
+  EXPECT_NEAR(g1.imag, 0.0f, 0.01);
+
+  complex<double> h1 = -complex<double>(0.33f, 0.33f);
+  EXPECT_NEAR(h1.real, -0.33f, 0.01);
+  EXPECT_NEAR(h1.imag, -0.33f, 0.01);
+  h1 = -complex<double>(-0.33f, -0.33f);
+  EXPECT_NEAR(h1.real, 0.33f, 0.01);
+  EXPECT_NEAR(h1.imag, 0.33f, 0.01);
+
+  complex<double> i1 = complex<double>(1.0, 1.0);
+  i1 += complex<double>(2.0, 2.0);
+  EXPECT_NEAR(i1.real, 3.0f, 0.01);
+  EXPECT_NEAR(i1.imag, 3.0f, 0.01);
+  i1 -= complex<double>(1.0, 1.0);
+  EXPECT_NEAR(i1.real, 2.0f, 0.01);
+  EXPECT_NEAR(i1.imag, 2.0f, 0.01);
+  i1 *= complex<double>(3, 2);
+  EXPECT_NEAR(i1.real, 2.0f, 0.01);
+  EXPECT_NEAR(i1.imag, 10.0f, 0.01);
+  i1 /= complex<double>(3, 2);
+  EXPECT_NEAR(i1.real, 2.0f, 0.01);
+  EXPECT_NEAR(i1.imag, 2.0f, 0.01);
+}
+
+TEST(complex, print) {
+  complex<float> a(1.0f);
+  std::cout << a << std::endl;
+
+  complex<double> b(1.0);
+  std::cout << b << std::endl;
+}
+
+TEST(complex, isinf) {
+  // *********** complex<float> *************
+  complex<float> a;
+  a.real = static_cast<float>(INFINITY);
+  EXPECT_EQ(std::isinf(a), true);
+  a.imag = static_cast<float>(INFINITY);
+  EXPECT_EQ(std::isinf(a), true);
+
+  complex<float> b = static_cast<float>(INFINITY);
+  EXPECT_EQ(std::isinf(b), true);
+
+  complex<float> c(static_cast<float>(INFINITY), 0);
+  EXPECT_EQ(std::isinf(c), true);
+
+  // *********** complex<double> *************
+  complex<double> a1;
+  a1.real = static_cast<double>(INFINITY);
+  EXPECT_EQ(std::isinf(a1), true);
+  a1.imag = static_cast<double>(INFINITY);
+  EXPECT_EQ(std::isinf(a1), true);
+
+  complex<double> b1 = static_cast<double>(INFINITY);
+  EXPECT_EQ(std::isinf(b1), true);
+
+  complex<double> c1(static_cast<double>(INFINITY), 0);
+  EXPECT_EQ(std::isinf(c1), true);
+}
+
+TEST(complex, isnan) {
+  // *********** complex<float> *************
+  complex<float> a;
+  a.real = static_cast<float>(NAN);
+  EXPECT_EQ(std::isnan(a), true);
+  a.imag = static_cast<float>(NAN);
+  EXPECT_EQ(std::isnan(a), true);
+
+  complex<float> b = static_cast<float>(NAN);
+  EXPECT_EQ(std::isnan(b), true);
+
+  complex<float> c(static_cast<float>(NAN), 0);
+  EXPECT_EQ(std::isnan(c), true);
+
+  // *********** complex<double> *************
+  complex<double> a1;
+  a1.real = static_cast<double>(NAN);
+  EXPECT_EQ(std::isnan(a1), true);
+  a1.imag = static_cast<double>(NAN);
+  EXPECT_EQ(std::isnan(a1), true);
+
+  complex<double> b1 = static_cast<double>(NAN);
+  EXPECT_EQ(std::isnan(b1), true);
+
+  complex<double> c1(static_cast<double>(NAN), 0);
+  EXPECT_EQ(std::isnan(c1), true);
+}
+
+}  // namespace paddle::platform

test/cpp/fluid/platform/complex_test.cu

@@ -0,0 +1,364 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/common/complex.h"
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+#include <thrust/complex.h>
+
+#include <bitset>
+#include <iostream>
+
+#include "paddle/fluid/framework/lod_tensor.h"
+#include "paddle/fluid/framework/tensor_util.h"
+#include "paddle/fluid/platform/enforce.h"
+#include "paddle/phi/kernels/funcs/eigen/extensions.h"
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+namespace paddle {
+namespace platform {
+
+template <typename T>
+using complex = phi::dtype::complex<T>;
+
+TEST(complex, conversion_on_gpu) {
+  // *********** complex<float> *************
+  // thrust<float> from and to complex<float>
+  complex<float> a(1.0f, 2.0f);
+  EXPECT_EQ(complex<float>(thrust::complex<float>(a)).real, 1.0);
+  EXPECT_EQ(complex<float>(thrust::complex<float>(a)).imag, 2.0);
+
+  complex<double> a1(1.0, 2.0);
+  EXPECT_EQ(complex<double>(thrust::complex<double>(a1)).real, 1.0);
+  EXPECT_EQ(complex<double>(thrust::complex<double>(a1)).imag, 2.0);
+
+#if defined(PADDLE_WITH_HIP)
+  EXPECT_EQ(hipFloatComplex(a).real(), 1.0);
+  EXPECT_EQ(hipFloatComplex(a).imag(), 2.0);
+  EXPECT_EQ(hipDoubleComplex(a).real(), 1.0);
+  EXPECT_EQ(hipDoubleComplex(a).imag(), 2.0);
+
+  EXPECT_EQ(hipFloatComplex(a1).real(), 1.0);
+  EXPECT_EQ(hipFloatComplex(a1).imag(), 2.0);
+  EXPECT_EQ(hipDoubleComplex(a1).real(), 1.0);
+  EXPECT_EQ(hipDoubleComplex(a1).imag(), 2.0);
+#else
+  EXPECT_EQ(cuCrealf(cuFloatComplex(a)), 1.0);
+  EXPECT_EQ(cuCimagf(cuFloatComplex(a)), 2.0);
+  EXPECT_EQ(cuCreal(cuDoubleComplex(a)), 1.0);
+  EXPECT_EQ(cuCimag(cuDoubleComplex(a)), 2.0);
+
+  EXPECT_EQ(cuCrealf(cuFloatComplex(a1)), 1.0);
+  EXPECT_EQ(cuCimagf(cuFloatComplex(a1)), 2.0);
+  EXPECT_EQ(cuCreal(cuDoubleComplex(a1)), 1.0);
+  EXPECT_EQ(cuCimag(cuDoubleComplex(a1)), 2.0);
+#endif
+
+  EXPECT_EQ(complex<float>().real, 0.0f);
+  EXPECT_EQ(complex<float>().imag, 0.0f);
+
+  EXPECT_EQ(complex<float>(1.0f, 1.0f).real, 1.0f);
+  EXPECT_EQ(complex<float>(1.0f, 1.0f).imag, 1.0f);
+  EXPECT_EQ(complex<float>(0.0f, 1.0f).real, 0.0f);
+  EXPECT_EQ(complex<float>(0.0f, 1.0f).imag, 1.0f);
+
+  EXPECT_EQ(complex<float>(1.0f).real, 1.0f);
+  EXPECT_EQ(complex<float>(1.0f).imag, 0.0f);
+
+  // int to complex<float>
+  EXPECT_EQ(complex<float>(1).real, 1.0f);
+  EXPECT_EQ(complex<float>(0).real, 0.0f);
+  EXPECT_EQ(complex<float>(2).real, 2.0f);
+  EXPECT_EQ(complex<float>(-2).real, -2.0f);
+
+  // bool to complex
+  EXPECT_EQ(complex<float>(true).real, 1.0f);
+  EXPECT_EQ(complex<float>(true).imag, 0.0f);
+
+  // complex<double> to complex<float>
+  EXPECT_EQ(complex<float>(complex<double>(1.0, 2.0)).real, 1.0f);
+  EXPECT_EQ(complex<float>(complex<double>(1.0, 2.0)).imag, 2.0f);
+
+  // std::complex<float> to complex<float>
+  EXPECT_EQ(complex<float>(std::complex<float>(1.0f, 2.0f)).real, 1.0f);
+  EXPECT_EQ(complex<float>(std::complex<float>(1.0f, 2.0f)).imag, 2.0f);
+  EXPECT_EQ(complex<float>(std::complex<double>(1.0, 2.0)).real, 1.0f);
+  EXPECT_EQ(complex<float>(std::complex<double>(1.0, 2.0)).imag, 2.0f);
+
+  // Assignment operator
+  complex<float> c = 1.0f;
+  EXPECT_EQ(c.real, 1.0f);
+  EXPECT_EQ(c.imag, 0.0f);
+  c = complex<float>(2.0, 2.0);
+  EXPECT_EQ(c.real, 2.0f);
+  EXPECT_EQ(c.imag, 2.0f);
+
+  // Conversion operator
+  EXPECT_EQ(static_cast<float>(complex<float>(0.5f)), 0.5f);
+  EXPECT_NEAR(static_cast<double>(complex<float>(0.33333)), 0.33333, 0.01);
+  EXPECT_EQ(static_cast<int>(complex<float>(-1)), -1);
+  EXPECT_EQ(static_cast<bool>(complex<float>(true)), true);
+
+  // *********** complex<double> *************
+  // double to complex<double>
+  EXPECT_EQ(complex<double>().real, 0.0);
+  EXPECT_EQ(complex<double>().imag, 0.0);
+
+  EXPECT_EQ(complex<double>(1.0, 1.0).real, 1.0);
+  EXPECT_EQ(complex<double>(1.0, 1.0).imag, 1.0);
+  EXPECT_EQ(complex<double>(0.0, 1.0).real, 0.0);
+  EXPECT_EQ(complex<double>(0.0, 1.0).imag, 1.0);
+
+  EXPECT_EQ(complex<double>(1.0).real, 1.0);
+  EXPECT_EQ(complex<double>(1.0).imag, 0.0);
+
+  // int to complex<double>
+  EXPECT_EQ(complex<double>(1).real, 1.0);
+  EXPECT_EQ(complex<double>(0).real, 0.0);
+  EXPECT_EQ(complex<double>(2).real, 2.0);
+  EXPECT_EQ(complex<double>(-2).real, -2.0);
+
+  // bool to complex
+  EXPECT_EQ(complex<double>(true).real, 1.0);
+  EXPECT_EQ(complex<double>(true).imag, 0.0);
+
+  // complex<float> to complex<double>
+  EXPECT_EQ(complex<double>(complex<float>(1.0f, 2.0f)).real, 1.0);
+  EXPECT_EQ(complex<double>(complex<float>(1.0f, 2.0f)).imag, 2.0);
+
+  // std::complex<float> to complex<double>
+  EXPECT_EQ(complex<double>(std::complex<double>(1.0, 2.0)).real, 1.0);
+  EXPECT_EQ(complex<double>(std::complex<double>(1.0, 2.0)).imag, 2.0);
+  EXPECT_EQ(complex<double>(std::complex<double>(1.0, 2.0)).real, 1.0);
+  EXPECT_EQ(complex<double>(std::complex<double>(1.0, 2.0)).imag, 2.0);
+
+  // Assignment operator
+  complex<double> c1 = 1.0;
+  EXPECT_EQ(c1.real, 1.0);
+  EXPECT_EQ(c1.imag, 0.0);
+  c1 = complex<double>(2.0, 2.0);
+  EXPECT_EQ(c1.real, 2.0);
+  EXPECT_EQ(c1.imag, 2.0);
+
+  // Conversion operator
+  EXPECT_EQ(static_cast<double>(complex<double>(0.5)), 0.5);
+  EXPECT_NEAR(static_cast<double>(complex<double>(0.33333)), 0.33333, 0.01);
+  EXPECT_EQ(static_cast<int>(complex<double>(-1)), -1);
+  EXPECT_EQ(static_cast<bool>(complex<double>(true)), true);
+}
+
+TEST(bfloat16, comparison_cpu) {
+  // *********** complex<float> *************
+  EXPECT_TRUE(complex<float>(1.0f) == complex<float>(1.0f));
+  EXPECT_TRUE(complex<float>(1.0f, 2.0f) == complex<float>(1.0f, 2.0f));
+  EXPECT_FALSE(complex<float>(-1.0f) == complex<float>(-0.5f));
+  EXPECT_TRUE(complex<float>(1.0f) != complex<float>(0.5f));
+  EXPECT_FALSE(complex<float>(-1.0f) != complex<float>(-1.0f));
+  EXPECT_TRUE(complex<float>(1.0f) < complex<float>(2.0f));
+  EXPECT_FALSE(complex<float>(-1.0f) < complex<float>(-1.0f));
+  EXPECT_TRUE(complex<float>(1.0f) <= complex<float>(1.0f));
+  EXPECT_TRUE(complex<float>(2.0f) > complex<float>(1.0f));
+  EXPECT_FALSE(complex<float>(-2.0f) > complex<float>(-2.0f));
+  EXPECT_TRUE(complex<float>(2.0f) >= complex<float>(2.0f));
+
+  // *********** complex<double> *************
+  EXPECT_TRUE(complex<double>(1.0) == complex<double>(1.0));
+  EXPECT_TRUE(complex<double>(1.0, 2.0) == complex<double>(1.0, 2.0));
+  EXPECT_FALSE(complex<double>(-1.0) == complex<double>(-0.5f));
+  EXPECT_TRUE(complex<double>(1.0) != complex<double>(0.5f));
+  EXPECT_FALSE(complex<double>(-1.0) != complex<double>(-1.0));
+  EXPECT_TRUE(complex<double>(1.0) < complex<double>(2.0));
+  EXPECT_FALSE(complex<double>(-1.0) < complex<double>(-1.0));
+  EXPECT_TRUE(complex<double>(1.0) <= complex<double>(1.0));
+  EXPECT_TRUE(complex<double>(2.0) > complex<double>(1.0));
+  EXPECT_FALSE(complex<double>(-2.0) > complex<double>(-2.0));
+  EXPECT_TRUE(complex<double>(2.0) >= complex<double>(2.0));
+}
+
+TEST(complex, arithmetic_cpu) {
+  // *********** complex<float> *************
+  complex<float> a = complex<float>(1, 1) + complex<float>(1, 1);
+  EXPECT_NEAR(a.real, 2, 0.001);
+  EXPECT_NEAR(a.imag, 2, 0.001);
+
+  complex<float> b = complex<float>(-5, -5) + complex<float>(5, 5);
+  EXPECT_EQ(b.real, 0);
+  EXPECT_EQ(b.imag, 0);
+
+  complex<float> c =
+      complex<float>(0.33333f, 0.33333f) + complex<float>(0.66667f, 0.66667f);
+  EXPECT_NEAR(c.real, 1.0f, 0.01);
+  EXPECT_NEAR(c.imag, 1.0f, 0.01);
+
+  complex<float> d = complex<float>(3) - complex<float>(5);
+  EXPECT_EQ(d.real, -2);
+  EXPECT_EQ(d.imag, 0);
+
+  complex<float> e =
+      complex<float>(0.66667f, 0.66667f) - complex<float>(0.33333f, 0.33333f);
+  EXPECT_NEAR(e.real, 0.33334f, 0.01);
+  EXPECT_NEAR(e.imag, 0.33334f, 0.01);
+
+  complex<float> f = complex<float>(0.33f, 0.33f) * complex<float>(0.2f, 0.2f);
+  EXPECT_NEAR(f.real, 0.0f, 0.01);
+  EXPECT_NEAR(f.imag, 0.132f, 0.01);
+
+  complex<float> g = complex<float>(0.33f, 0.33f) / complex<float>(0.2f, 0.2f);
+  EXPECT_NEAR(g.real, 1.65f, 0.01);
+  EXPECT_NEAR(g.imag, 0.0f, 0.01);
+
+  complex<float> h = -complex<float>(0.33f, 0.33f);
+  EXPECT_NEAR(h.real, -0.33f, 0.01);
+  EXPECT_NEAR(h.imag, -0.33f, 0.01);
+  h = -complex<float>(-0.33f, -0.33f);
+  EXPECT_NEAR(h.real, 0.33f, 0.01);
+  EXPECT_NEAR(h.imag, 0.33f, 0.01);
+
+  complex<float> i = complex<float>(1.0, 1.0);
+  i += complex<float>(2.0, 2.0);
+  EXPECT_NEAR(i.real, 3.0f, 0.01);
+  EXPECT_NEAR(i.imag, 3.0f, 0.01);
+  i -= complex<float>(1.0, 1.0);
+  EXPECT_NEAR(i.real, 2.0f, 0.01);
+  EXPECT_NEAR(i.imag, 2.0f, 0.01);
+  i *= complex<float>(3, 2);
+  EXPECT_NEAR(i.real, 2.0f, 0.01);
+  EXPECT_NEAR(i.imag, 10.0f, 0.01);
+  i /= complex<float>(3, 2);
+  EXPECT_NEAR(i.real, 2.0f, 0.01);
+  EXPECT_NEAR(i.imag, 2.0f, 0.01);
+
+  // *********** complex<double> *************
+  complex<double> a1 = complex<double>(1, 1) + complex<double>(1, 1);
+  EXPECT_NEAR(a1.real, 2, 0.001);
+  EXPECT_NEAR(a1.imag, 2, 0.001);
+
+  complex<double> b1 = complex<double>(-5, -5) + complex<double>(5, 5);
+  EXPECT_EQ(b1.real, 0);
+  EXPECT_EQ(b1.imag, 0);
+
+  complex<double> c1 =
+      complex<double>(0.33333f, 0.33333f) + complex<double>(0.66667f, 0.66667f);
+  EXPECT_NEAR(c1.real, 1.0f, 0.01);
+  EXPECT_NEAR(c1.imag, 1.0f, 0.01);
+
+  complex<double> d1 = complex<double>(3) - complex<double>(5);
+  EXPECT_EQ(d1.real, -2);
+  EXPECT_EQ(d1.imag, 0);
+
+  complex<double> e1 =
+      complex<double>(0.66667f, 0.66667f) - complex<double>(0.33333f, 0.33333f);
+  EXPECT_NEAR(e1.real, 0.33334f, 0.01);
+  EXPECT_NEAR(e1.imag, 0.33334f, 0.01);
+
+  complex<double> f1 =
+      complex<double>(0.33f, 0.33f) * complex<double>(0.2f, 0.2f);
+  EXPECT_NEAR(f1.real, 0.0f, 0.01);
+  EXPECT_NEAR(f1.imag, 0.132f, 0.01);
+
+  complex<double> g1 =
+      complex<double>(0.33f, 0.33f) / complex<double>(0.2f, 0.2f);
+  EXPECT_NEAR(g1.real, 1.65f, 0.01);
+  EXPECT_NEAR(g1.imag, 0.0f, 0.01);
+
+  complex<double> h1 = -complex<double>(0.33f, 0.33f);
+  EXPECT_NEAR(h1.real, -0.33f, 0.01);
+  EXPECT_NEAR(h1.imag, -0.33f, 0.01);
+  h1 = -complex<double>(-0.33f, -0.33f);
+  EXPECT_NEAR(h1.real, 0.33f, 0.01);
+  EXPECT_NEAR(h1.imag, 0.33f, 0.01);
+
+  complex<double> i1 = complex<double>(1.0, 1.0);
+  i1 += complex<double>(2.0, 2.0);
+  EXPECT_NEAR(i1.real, 3.0f, 0.01);
+  EXPECT_NEAR(i1.imag, 3.0f, 0.01);
+  i1 -= complex<double>(1.0, 1.0);
+  EXPECT_NEAR(i1.real, 2.0f, 0.01);
+  EXPECT_NEAR(i1.imag, 2.0f, 0.01);
+  i1 *= complex<double>(3, 2);
+  EXPECT_NEAR(i1.real, 2.0f, 0.01);
+  EXPECT_NEAR(i1.imag, 10.0f, 0.01);
+  i1 /= complex<double>(3, 2);
+  EXPECT_NEAR(i1.real, 2.0f, 0.01);
+  EXPECT_NEAR(i1.imag, 2.0f, 0.01);
+}
+
+TEST(complex, print) {
+  complex<float> a(1.0f);
+  std::cout << a << std::endl;
+
+  complex<double> b(1.0);
+  std::cout << b << std::endl;
+}
+
+TEST(complex, isinf) {
+  // *********** complex<float> *************
+  complex<float> a;
+  a.real = static_cast<float>(INFINITY);
+  EXPECT_EQ(std::isinf(a), true);
+  a.imag = static_cast<float>(INFINITY);
+  EXPECT_EQ(std::isinf(a), true);
+
+  complex<float> b = static_cast<float>(INFINITY);
+  EXPECT_EQ(std::isinf(b), true);
+
+  complex<float> c(static_cast<float>(INFINITY), 0);
+  EXPECT_EQ(std::isinf(c), true);
+
+  // *********** complex<double> *************
+  complex<double> a1;
+  a1.real = static_cast<double>(INFINITY);
+  EXPECT_EQ(std::isinf(a1), true);
+  a1.imag = static_cast<double>(INFINITY);
+  EXPECT_EQ(std::isinf(a1), true);
+
+  complex<double> b1 = static_cast<double>(INFINITY);
+  EXPECT_EQ(std::isinf(b1), true);
+
+  complex<double> c1(static_cast<double>(INFINITY), 0);
+  EXPECT_EQ(std::isinf(c1), true);
+}
+
+TEST(complex, isnan) {
+  // *********** complex<float> *************
+  complex<float> a;
+  a.real = static_cast<float>(NAN);
+  EXPECT_EQ(std::isnan(a), true);
+  a.imag = static_cast<float>(NAN);
+  EXPECT_EQ(std::isnan(a), true);
+
+  complex<float> b = static_cast<float>(NAN);
+  EXPECT_EQ(std::isnan(b), true);
+
+  complex<float> c(static_cast<float>(NAN), 0);
+  EXPECT_EQ(std::isnan(c), true);
+
+  // *********** complex<double> *************
+  complex<double> a1;
+  a1.real = static_cast<double>(NAN);
+  EXPECT_EQ(std::isnan(a1), true);
+  a1.imag = static_cast<double>(NAN);
+  EXPECT_EQ(std::isnan(a1), true);
+
+  complex<double> b1 = static_cast<double>(NAN);
+  EXPECT_EQ(std::isnan(b1), true);
+
+  complex<double> c1(static_cast<double>(NAN), 0);
+  EXPECT_EQ(std::isnan(c1), true);
+}
+
+}  // namespace platform
+}  // namespace paddle
+#endif

test/cpp/fluid/platform/cpu_helper_test.cc

@@ -0,0 +1,22 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/core/platform/cpu_helper.h"
+
+#include "gtest/gtest.h"
+
+TEST(CpuHelper, SetNumThread) {
+  paddle::platform::SetNumThreads(1);
+  paddle::platform::SetNumThreads(4);
+}

test/cpp/fluid/platform/cpu_info_test.cc

@@ -0,0 +1,34 @@
+//  Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "paddle/phi/backends/cpu/cpu_info.h"
+
+#include <sstream>
+
+#include "gtest/gtest.h"
+#include "paddle/common/flags.h"
+#include "paddle/utils/string/printf.h"
+
+COMMON_DECLARE_double(fraction_of_cpu_memory_to_use);
+
+TEST(CpuMemoryUsage, Print) {
+  std::stringstream ss;
+  size_t memory_size =
+      phi::backends::cpu::CpuMaxAllocSize() / 1024 / 1024 / 1024;
+  float use_percent = FLAGS_fraction_of_cpu_memory_to_use * 100;  // NOLINT
+
+  std::cout << paddle::string::Sprintf("\n%.2f %% of CPU Memory Usage: %d GB\n",
+                                       use_percent,
+                                       memory_size)
+            << std::endl;
+}

test/cpp/fluid/platform/device/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_subdirectory(custom)
+add_subdirectory(gpu)

test/cpp/fluid/platform/device/custom/CMakeLists.txt

@@ -0,0 +1,3 @@
+if(WITH_CUSTOM_DEVICE)
+  paddle_test(custom_device_test SRCS custom_device_test.cc)
+endif()

test/cpp/fluid/platform/device/custom/custom_device_test.cc

@@ -0,0 +1,267 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include <array>
+#include <string>
+
+#include "paddle/fluid/framework/tensor_util.h"
+#include "paddle/fluid/platform/init.h"
+#include "paddle/phi/backends/custom/fake_cpu_device.h"
+#include "paddle/phi/backends/device_manager.h"
+#include "paddle/phi/common/memory_utils.h"
+#include "paddle/phi/core/memory/allocation/allocator_facade.h"
+#include "paddle/phi/core/platform/device_context.h"
+
+void RegisterDevice() {
+  CustomRuntimeParams runtime_params;
+  runtime_params.size = sizeof(CustomRuntimeParams);
+  auto device_interface = std::make_unique<C_DeviceInterface>();
+  runtime_params.interface = device_interface.get();
+  std::memset(runtime_params.interface, 0, sizeof(C_DeviceInterface));
+  runtime_params.interface->size = sizeof(C_DeviceInterface);
+
+  InitFakeCPUDevice(&runtime_params);
+  phi::LoadCustomRuntimeLib(
+      runtime_params, std::move(device_interface), "", nullptr);
+
+  std::vector<std::string> passes =
+      phi::CustomDevicePassManager::Instance()->GetCustomDevicePass();
+  EXPECT_EQ(passes[0], "fake_cpu_device_pass");
+}
+
+void InitDevice() {
+  RegisterDevice();
+  EXPECT_GT(static_cast<int>(phi::DeviceManager::GetAllDeviceTypes().size()),
+            0);
+  auto place = phi::CustomPlace(DEVICE_TYPE, 0);
+  auto device = phi::DeviceManager::GetDeviceWithPlace(place);
+  EXPECT_NE(device, nullptr);
+
+  std::vector<phi::Place> places;
+  auto device_types = phi::DeviceManager::GetAllDeviceTypes();
+  for (auto dev_type : device_types) {
+    auto devices = phi::DeviceManager::GetDeviceList(dev_type);
+    for (auto dev_id : devices) {
+      places.push_back(phi::PlaceHelper::CreatePlace(dev_type, dev_id));
+    }
+  }
+  EXPECT_GT(static_cast<int>(places.size()), 0);
+
+  phi::DeviceContextPool::Init(places);
+}
+
+void TestDeviceInterface(const phi::Place& place) {
+  std::cout << "TestDeviceInterface on " << place << std::endl;
+  if (phi::is_custom_place(place)) {
+    auto device = phi::DeviceManager::GetDeviceWithPlace(place);
+    auto dev_type = phi::PlaceHelper::GetDeviceType(place);
+    auto p1 =
+        device->MemoryAllocate(phi::DeviceManager::GetMinChunkSize(place));
+    EXPECT_NE(p1, nullptr);
+
+    phi::DeviceManager::SetDevice(place);
+    auto dev_id = phi::DeviceManager::GetDevice(dev_type);
+    EXPECT_EQ(dev_id, place.GetDeviceId());
+  }
+}
+
+void TestTensorMutableData(const phi::Place& place) {
+  std::cout << "TestTensorInitialization on " << place << std::endl;
+  phi::DenseTensor src_tensor;
+  float* p1 = nullptr;
+  float* p2 = nullptr;
+  // initialization
+  p1 = src_tensor.mutable_data<float>(common::make_ddim({1, 2, 3}), place);
+  auto p1_holder = src_tensor.Holder();
+  EXPECT_NE(p1, nullptr);
+  // set src_tensor a new dim with large size
+  // memory is supposed to be re-allocated
+  p2 = src_tensor.mutable_data<float>(common::make_ddim({3, 1024}), place);
+  auto p2_holder = src_tensor.Holder();
+  EXPECT_NE(p2, nullptr);
+  EXPECT_NE(p1_holder.get(), p2_holder.get());
+  // set src_tensor a new dim with same size
+  // memory block is supposed to be unchanged
+  p1 = src_tensor.mutable_data<float>(common::make_ddim({2, 2, 3}), place);
+  EXPECT_EQ(p1, p2);
+  // set src_tensor a new dim with smaller size
+  // memory block is supposed to be unchanged
+  p2 = src_tensor.mutable_data<float>(common::make_ddim({2, 2}), place);
+  EXPECT_EQ(p1, p2);
+}
+
+void TestTensorShareDataWith(const phi::Place& place) {
+  std::cout << "TestTensorShareDataWith on " << place << std::endl;
+  phi::DenseTensor src_tensor;
+  phi::DenseTensor dst_tensor;
+  src_tensor.mutable_data<int>(common::make_ddim({2, 3, 4}), place);
+  dst_tensor.ShareDataWith(src_tensor);
+  ASSERT_EQ(src_tensor.data<int>(), dst_tensor.data<int>());
+}
+
+void TestTensorUtils(const phi::Place& place) {
+  std::cout << "TestTensorUtils on " << place << std::endl;
+  if (phi::is_custom_place(place) == false) {
+    return;
+  }
+  phi::DenseTensor src_tensor;
+  phi::DenseTensor gpu_tensor;
+  phi::DenseTensor dst_tensor;
+
+  int* src_ptr =
+      src_tensor.mutable_data<int>(common::make_ddim({3, 3}), phi::CPUPlace());
+
+  std::array<int, 9> arr = {1, 2, 3, 4, 5, 6, 7, 8, 9};
+  memcpy(src_ptr, arr.data(), 9 * sizeof(int));
+
+  // CPU Tensor to GPU Tensor
+  phi::CustomContext gpu_ctx(place);
+  paddle::framework::TensorCopy(src_tensor, place, gpu_ctx, &gpu_tensor);
+#if 0
+  // GPU Tensor to CPU Tensor
+  auto cpu_place = new phi::CPUPlace();
+  paddle::framework::TensorCopy(gpu_tensor, *cpu_place, gpu_ctx, &dst_tensor);
+
+  // Sync before Compare Tensors
+  gpu_ctx.Wait();
+  const int* dst_ptr = dst_tensor.data<int>();
+  EXPECT_NE(src_ptr, dst_ptr);
+  for (size_t i = 0; i < 9; ++i) {
+    EXPECT_EQ(src_ptr[i], dst_ptr[i]);
+  }
+
+  // Copy the same tensor
+  paddle::framework::TensorCopy(gpu_tensor, place, gpu_ctx, &gpu_tensor);
+  gpu_ctx.Wait();
+  const int* dst_ptr_tmp = dst_tensor.data<int>();
+  EXPECT_NE(src_ptr, dst_ptr_tmp);
+  for (size_t i = 0; i < 9; ++i) {
+    EXPECT_EQ(src_ptr[i], dst_ptr_tmp[i]);
+  }
+
+  phi::DenseTensor slice_tensor = src_tensor.Slice(1, 2);
+
+  // CPU Slice Tensor to GPU Tensor
+  paddle::framework::TensorCopy(slice_tensor, place, gpu_ctx, &gpu_tensor);
+
+  // GPU Tensor to CPU Tensor
+  paddle::framework::TensorCopy(gpu_tensor, *cpu_place, gpu_ctx, &dst_tensor);
+
+  // Sync before Compare Slice Tensors
+  gpu_ctx.Wait();
+  const int* slice_ptr = slice_tensor.data<int>();
+  dst_ptr = dst_tensor.data<int>();
+  EXPECT_NE(dst_ptr, slice_ptr);
+  for (size_t i = 0; i < 3; ++i) {
+    EXPECT_EQ(dst_ptr[i], slice_ptr[i]);
+  }
+
+  EXPECT_TRUE(dst_tensor.layout() == src_tensor.layout());
+#endif
+}
+
+void TestCustomCCL(const phi::Place& place) {
+  std::cout << "TestCustomCCL on " << place << std::endl;
+  if (phi::is_custom_place(place) == false) {
+    return;
+  }
+  std::string dev_type = place.GetDeviceType();
+  phi::ccl::CCLComm comm;
+  phi::stream::Stream stream(place, nullptr);
+  phi::ccl::CCLRootId root_id;
+
+  phi::DeviceManager::CCLDestroyComm(dev_type, nullptr);
+  phi::DeviceManager::CCLGetUniqueId(dev_type, &root_id);
+  phi::DeviceManager::CCLCommInitRank(dev_type, 0, &root_id, 0, nullptr);
+  phi::DeviceManager::CCLBroadcast(dev_type,
+                                   nullptr,
+                                   0,
+                                   phi::DataType::FLOAT32,
+                                   0,
+                                   comm,
+                                   stream.raw_stream());
+  phi::DeviceManager::CCLAllReduce(dev_type,
+                                   nullptr,
+                                   nullptr,
+                                   0,
+                                   phi::DataType::FLOAT32,
+                                   phi::ccl::CCLReduceOp::SUM,
+                                   comm,
+                                   stream.raw_stream());
+  phi::DeviceManager::CCLReduce(dev_type,
+                                nullptr,
+                                nullptr,
+                                0,
+                                phi::DataType::FLOAT32,
+                                phi::ccl::CCLReduceOp::SUM,
+                                0,
+                                comm,
+                                stream.raw_stream());
+  phi::DeviceManager::CCLAllGather(dev_type,
+                                   nullptr,
+                                   nullptr,
+                                   0,
+                                   phi::DataType::FLOAT32,
+                                   comm,
+                                   stream.raw_stream());
+  phi::DeviceManager::CCLReduceScatter(dev_type,
+                                       nullptr,
+                                       nullptr,
+                                       0,
+                                       phi::DataType::FLOAT32,
+                                       phi::ccl::CCLReduceOp::SUM,
+                                       comm,
+                                       stream.raw_stream());
+  phi::DeviceManager::CCLGroupStart(dev_type);
+  phi::DeviceManager::CCLGroupEnd(dev_type);
+  phi::DeviceManager::CCLSend(dev_type,
+                              nullptr,
+                              0,
+                              phi::DataType::FLOAT32,
+                              0,
+                              comm,
+                              stream.raw_stream());
+  phi::DeviceManager::CCLRecv(dev_type,
+                              nullptr,
+                              0,
+                              phi::DataType::FLOAT32,
+                              0,
+                              comm,
+                              stream.raw_stream());
+}
+
+TEST(CustomDevice, Tensor) {
+  paddle::framework::InitMemoryMethod();
+  InitDevice();
+  auto dev_types = phi::DeviceManager::GetAllDeviceTypes();
+  for (const auto& dev_type : dev_types) {
+    std::cout << "Test on " << dev_type << std::endl;
+    EXPECT_GT(static_cast<int>(phi::DeviceManager::GetDeviceCount(dev_type)),
+              0);
+    auto place = phi::PlaceHelper::CreatePlace(dev_type);
+
+    TestDeviceInterface(place);
+    TestTensorMutableData(place);
+    TestTensorShareDataWith(place);
+    TestTensorUtils(place);
+    TestCustomCCL(place);
+  }
+}
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

test/cpp/fluid/platform/device/gpu/CMakeLists.txt

@@ -0,0 +1,17 @@
+if(WITH_GPU)
+  add_subdirectory(cuda)
+
+  nv_test(cuda_helper_test SRCS cuda_helper_test.cu)
+  nv_test(
+    cudnn_desc_test
+    SRCS cudnn_desc_test.cc
+    DEPS phi common)
+elseif(WITH_ROCM)
+  add_subdirectory(rocm)
+
+  hip_test(cuda_helper_test SRCS cuda_helper_test.cu)
+  hip_test(
+    cudnn_desc_test
+    SRCS cudnn_desc_test.cc
+    DEPS phi common)
+endif()

test/cpp/fluid/platform/device/gpu/cuda/CMakeLists.txt

@@ -0,0 +1,4 @@
+nv_test(
+  cudnn_helper_test
+  SRCS cudnn_helper_test.cc
+  DEPS phi common)

test/cpp/fluid/platform/device/gpu/cuda/cudnn_helper_test.cc

@@ -0,0 +1,163 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#define GOOGLE_GLOG_DLL_DECL
+
+#include <gtest/gtest.h>
+
+#include "paddle/phi/core/platform/device/gpu/gpu_dnn.h"
+
+TEST(CudnnHelper, ScopedTensorDescriptor) {
+  using phi::DataLayout;
+  using phi::backends::gpu::ScopedTensorDescriptor;
+
+  ScopedTensorDescriptor tensor_desc;
+  std::vector<int> shape = {2, 4, 6, 6};
+  auto desc = tensor_desc.descriptor<float>(DataLayout::NCHW, shape);
+
+  cudnnDataType_t type;
+  int nd;
+  std::vector<int> dims(4);
+  std::vector<int> strides(4);
+  phi::dynload::cudnnGetTensorNdDescriptor(
+      desc, 4, &type, &nd, dims.data(), strides.data());
+
+  EXPECT_EQ(nd, 4);
+  for (size_t i = 0; i < dims.size(); ++i) {
+    EXPECT_EQ(dims[i], shape[i]);
+  }
+  EXPECT_EQ(strides[3], 1);
+  EXPECT_EQ(strides[2], 6);
+  EXPECT_EQ(strides[1], 36);
+  EXPECT_EQ(strides[0], 144);
+
+  // test tensor5d: ScopedTensorDescriptor
+  ScopedTensorDescriptor tensor5d_desc;
+  std::vector<int> shape_5d = {2, 4, 6, 6, 6};
+  auto desc_5d = tensor5d_desc.descriptor<float>(DataLayout::NCDHW, shape_5d);
+
+  std::vector<int> dims_5d(5);
+  std::vector<int> strides_5d(5);
+  phi::dynload::cudnnGetTensorNdDescriptor(
+      desc_5d, 5, &type, &nd, dims_5d.data(), strides_5d.data());
+
+  EXPECT_EQ(nd, 5);
+  for (size_t i = 0; i < dims_5d.size(); ++i) {
+    EXPECT_EQ(dims_5d[i], shape_5d[i]);
+  }
+  EXPECT_EQ(strides_5d[4], 1);
+  EXPECT_EQ(strides_5d[3], 6);
+  EXPECT_EQ(strides_5d[2], 36);
+  EXPECT_EQ(strides_5d[1], 216);
+  EXPECT_EQ(strides_5d[0], 864);
+}
+
+TEST(CudnnHelper, ScopedFilterDescriptor) {
+  using phi::DataLayout;
+  using phi::backends::gpu::GetCudnnTensorFormat;
+  using phi::backends::gpu::ScopedFilterDescriptor;
+
+  ScopedFilterDescriptor filter_desc;
+  std::vector<int> shape = {2, 3, 3};
+  auto desc = filter_desc.descriptor<float>(DataLayout::NCHW, shape);
+
+  cudnnDataType_t type;
+  int nd;
+  cudnnTensorFormat_t format;
+  std::vector<int> kernel(3);
+  phi::dynload::cudnnGetFilterNdDescriptor(
+      desc, 3, &type, &format, &nd, kernel.data());
+
+  EXPECT_EQ(GetCudnnTensorFormat(DataLayout::NCHW), format);
+  EXPECT_EQ(nd, 3);
+  for (size_t i = 0; i < shape.size(); ++i) {
+    EXPECT_EQ(kernel[i], shape[i]);
+  }
+
+  ScopedFilterDescriptor filter_desc_4d;
+  std::vector<int> shape_4d = {2, 3, 3, 3};
+  auto desc_4d = filter_desc.descriptor<float>(DataLayout::NCDHW, shape_4d);
+
+  std::vector<int> kernel_4d(4);
+  phi::dynload::cudnnGetFilterNdDescriptor(
+      desc_4d, 4, &type, &format, &nd, kernel_4d.data());
+
+  EXPECT_EQ(GetCudnnTensorFormat(DataLayout::NCHW), format);
+  EXPECT_EQ(nd, 4);
+  for (size_t i = 0; i < shape_4d.size(); ++i) {
+    EXPECT_EQ(kernel_4d[i], shape_4d[i]);
+  }
+}
+
+TEST(CudnnHelper, ScopedConvolutionDescriptor) {
+  using phi::backends::gpu::ScopedConvolutionDescriptor;
+
+  ScopedConvolutionDescriptor conv_desc;
+  std::vector<int> src_pads = {2, 2, 2};
+  std::vector<int> src_strides = {1, 1, 1};
+  std::vector<int> src_dilations = {1, 1, 1};
+  auto desc = conv_desc.descriptor<float>(src_pads, src_strides, src_dilations);
+
+  cudnnDataType_t type;
+  cudnnConvolutionMode_t mode;
+  int nd;
+  std::vector<int> pads(3);
+  std::vector<int> strides(3);
+  std::vector<int> dilations(3);
+  phi::dynload::cudnnGetConvolutionNdDescriptor(desc,
+                                                3,
+                                                &nd,
+                                                pads.data(),
+                                                strides.data(),
+                                                dilations.data(),
+                                                &mode,
+                                                &type);
+
+  EXPECT_EQ(nd, 3);
+  for (size_t i = 0; i < src_pads.size(); ++i) {
+    EXPECT_EQ(pads[i], src_pads[i]);
+    EXPECT_EQ(strides[i], src_strides[i]);
+    EXPECT_EQ(dilations[i], src_dilations[i]);
+  }
+  EXPECT_EQ(mode, CUDNN_CROSS_CORRELATION);
+}
+
+TEST(CudnnHelper, ScopedPoolingDescriptor) {
+  using phi::backends::gpu::PoolingMode;
+  using phi::backends::gpu::ScopedPoolingDescriptor;
+
+  ScopedPoolingDescriptor pool_desc;
+  std::vector<int> src_kernel = {2, 2, 5};
+  std::vector<int> src_pads = {1, 1, 2};
+  std::vector<int> src_strides = {2, 2, 3};
+  auto desc = pool_desc.descriptor(
+      PoolingMode::kMaximum, src_kernel, src_pads, src_strides);
+
+  cudnnPoolingMode_t mode;
+  cudnnNanPropagation_t nan_t = CUDNN_PROPAGATE_NAN;
+  int nd;
+  std::vector<int> kernel(3);
+  std::vector<int> pads(3);
+  std::vector<int> strides(3);
+  phi::dynload::cudnnGetPoolingNdDescriptor(
+      desc, 3, &mode, &nan_t, &nd, kernel.data(), pads.data(), strides.data());
+
+  EXPECT_EQ(nd, 3);
+  for (size_t i = 0; i < src_pads.size(); ++i) {
+    EXPECT_EQ(kernel[i], src_kernel[i]);
+    EXPECT_EQ(pads[i], src_pads[i]);
+    EXPECT_EQ(strides[i], src_strides[i]);
+  }
+  EXPECT_EQ(mode, CUDNN_POOLING_MAX);
+}

test/cpp/fluid/platform/device/gpu/cuda_helper_test.cu

@@ -0,0 +1,310 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include <algorithm>
+#include <iostream>
+#ifdef _WIN32
+#include <numeric>
+#endif
+#include <random>
+
+#include "paddle/phi/backends/gpu/gpu_device_function.h"
+#include "paddle/phi/backends/gpu/gpu_primitives.h"
+#include "paddle/phi/common/float16.h"
+#include "paddle/phi/core/platform/device/gpu/gpu_helper.h"
+
+using phi::PADDLE_CUDA_NUM_THREADS;
+using phi::dtype::float16;
+
+template <typename T>
+__global__ void AddKernel(const T* data_a, T* data_b, size_t num) {
+  CUDA_KERNEL_LOOP(i, num) { phi::CudaAtomicAdd(&data_b[i], data_a[i]); }
+}
+
+template <typename T>
+struct AddFunctor {
+  T operator()(const T& a, const T& b) { return a + b; }
+};
+
+template <typename T>
+void TestCase(size_t num) {
+  T *in1, *in2, *out;
+  T *d_in1, *d_in2;
+  size_t size = sizeof(T) * num;
+#ifdef PADDLE_WITH_HIP
+  hipMalloc(reinterpret_cast<void**>(&d_in1), size);
+  hipMalloc(reinterpret_cast<void**>(&d_in2), size);
+#else
+  cudaMalloc(reinterpret_cast<void**>(&d_in1), size);
+  cudaMalloc(reinterpret_cast<void**>(&d_in2), size);
+#endif
+  in1 = reinterpret_cast<T*>(malloc(size));
+  in2 = reinterpret_cast<T*>(malloc(size));
+  out = reinterpret_cast<T*>(malloc(size));
+  std::minstd_rand engine;
+  std::uniform_real_distribution<double> dist(0.0, 1.0);
+  for (size_t i = 0; i < num; ++i) {
+    in1[i] = static_cast<T>(dist(engine));
+    in2[i] = static_cast<T>(dist(engine));
+  }
+#ifdef PADDLE_WITH_HIP
+  hipMemcpy(d_in1, in1, size, hipMemcpyHostToDevice);
+  hipMemcpy(d_in2, in2, size, hipMemcpyHostToDevice);
+  hipLaunchKernelGGL(HIP_KERNEL_NAME(AddKernel<T>),
+                     dim3(1),
+                     dim3(PADDLE_CUDA_NUM_THREADS),
+                     0,
+                     0,
+                     d_in1,
+                     d_in2,
+                     num);
+  hipDeviceSynchronize();
+  hipMemcpy(out, d_in2, size, hipMemcpyDeviceToHost);
+  hipDeviceSynchronize();
+#else
+  cudaMemcpy(d_in1, in1, size, cudaMemcpyHostToDevice);
+  cudaMemcpy(d_in2, in2, size, cudaMemcpyHostToDevice);
+  AddKernel<T><<<1, PADDLE_CUDA_NUM_THREADS>>>(d_in1, d_in2, num);
+  cudaDeviceSynchronize();
+  cudaMemcpy(out, d_in2, size, cudaMemcpyDeviceToHost);
+  cudaDeviceSynchronize();
+#endif
+  for (size_t i = 0; i < num; ++i) {
+    // NOTE(dzhwinter): the float16 add has small underflow/overflow
+    // so we use EXPECT_NEAR to check the result.
+    EXPECT_NEAR(static_cast<float>(out[i]),
+                static_cast<float>(AddFunctor<T>()(in1[i], in2[i])),
+                0.001);
+  }
+  free(in1);
+  free(in2);
+  free(out);
+#ifdef PADDLE_WITH_HIP
+  hipFree(d_in1);
+  hipFree(d_in2);
+#else
+  cudaFree(d_in1);
+  cudaFree(d_in2);
+#endif
+}
+
+// cuda primitives
+TEST(CudaAtomic, Add) {
+  TestCase<float>(static_cast<size_t>(10));
+  TestCase<float>(static_cast<size_t>(1024 * 1024));
+
+  TestCase<double>(static_cast<size_t>(10));
+  TestCase<double>(static_cast<size_t>(1024 * 1024));
+}
+
+TEST(CudaAtomic, float16) {
+  TestCase<float16>(static_cast<size_t>(1));
+  TestCase<float16>(static_cast<size_t>(2));
+  TestCase<float16>(static_cast<size_t>(3));
+
+  TestCase<float16>(static_cast<size_t>(10));
+  TestCase<float16>(static_cast<size_t>(1024 * 1024));
+}
+
+// unalignment of uint8
+void TestUnalign(size_t num, const int shift_bit) {
+  ASSERT_EQ(num % 2, 0);
+  float16 *in1, *in2, *out;
+  float16 *d_in1, *d_in2;
+  size_t size = sizeof(uint8_t) * (num + shift_bit);
+  size_t array_size = sizeof(float16) * (num / 2);
+
+#ifdef PADDLE_WITH_HIP
+  hipMalloc(reinterpret_cast<void**>(&d_in1), size);
+  hipMalloc(reinterpret_cast<void**>(&d_in2), size);
+#else
+  cudaMalloc(reinterpret_cast<void**>(&d_in1), size);
+  cudaMalloc(reinterpret_cast<void**>(&d_in2), size);
+#endif
+  in1 = reinterpret_cast<float16*>(malloc(size));
+  in2 = reinterpret_cast<float16*>(malloc(size));
+  out = reinterpret_cast<float16*>(malloc(size));
+
+  // right shift 1, mimic the unalignment of address
+  float16* r_in1 =
+      reinterpret_cast<float16*>(reinterpret_cast<uint8_t*>(in1) + shift_bit);
+  float16* r_in2 =
+      reinterpret_cast<float16*>(reinterpret_cast<uint8_t*>(in2) + shift_bit);
+
+  std::minstd_rand engine;
+  std::uniform_real_distribution<double> dist(0.0, 1.0);
+  for (size_t i = 0; i < num / 2; ++i) {
+    r_in1[i] = static_cast<float16>(dist(engine));
+    r_in2[i] = static_cast<float16>(dist(engine));
+  }
+#ifdef PADDLE_WITH_HIP
+  hipMemcpy(d_in1, r_in1, array_size, hipMemcpyHostToDevice);
+  hipMemcpy(d_in2, r_in2, array_size, hipMemcpyHostToDevice);
+  hipLaunchKernelGGL(HIP_KERNEL_NAME(AddKernel<float16>),
+                     dim3(1),
+                     dim3(PADDLE_CUDA_NUM_THREADS),
+                     0,
+                     0,
+                     d_in1,
+                     d_in2,
+                     num / 2);
+  hipDeviceSynchronize();
+  hipMemcpy(out, d_in2, array_size, hipMemcpyDeviceToHost);
+  hipDeviceSynchronize();
+#else
+  cudaMemcpy(d_in1, r_in1, array_size, cudaMemcpyHostToDevice);
+  cudaMemcpy(d_in2, r_in2, array_size, cudaMemcpyHostToDevice);
+  AddKernel<float16><<<1, PADDLE_CUDA_NUM_THREADS>>>(d_in1, d_in2, num / 2);
+  cudaDeviceSynchronize();
+  cudaMemcpy(out, d_in2, array_size, cudaMemcpyDeviceToHost);
+  cudaDeviceSynchronize();
+#endif
+  for (size_t i = 0; i < num / 2; ++i) {
+    // NOTE(dzhwinter): the float16 add has small truncate error.
+    // so we use EXPECT_NEAR to check the result.
+    EXPECT_NEAR(static_cast<float>(out[i]),
+                static_cast<float>(AddFunctor<float16>()(r_in1[i], r_in2[i])),
+                0.001);
+  }
+  free(in1);
+  free(in2);
+  free(out);
+#ifdef PADDLE_WITH_HIP
+  hipFree(d_in1);
+  hipFree(d_in2);
+#else
+  cudaFree(d_in1);
+  cudaFree(d_in2);
+#endif
+}
+
+TEST(CudaAtomic, float16Unalign) {
+  // same with float16 testcase
+  TestUnalign(static_cast<size_t>(2), /*shift_bit*/ 2);
+  TestUnalign(static_cast<size_t>(1024), /*shift_bit*/ 2);
+  TestUnalign(static_cast<size_t>(1024 * 1024), /*shift_bit*/ 2);
+
+  // shift the address.
+  TestUnalign(static_cast<size_t>(2), /*shift_bit*/ 1);
+  TestUnalign(static_cast<size_t>(1024), /*shift_bit*/ 1);
+  TestUnalign(static_cast<size_t>(1024 * 1024), /*shift_bit*/ 1);
+
+  TestUnalign(static_cast<size_t>(2), /*shift_bit*/ 3);
+  TestUnalign(static_cast<size_t>(1024), /*shift_bit*/ 3);
+  TestUnalign(static_cast<size_t>(1024 * 1024), /*shift_bit*/ 3);
+}
+
+// https://devblogs.nvidia.com/faster-parallel-reductions-kepler/
+template <typename T>
+static __forceinline__ __device__ T WarpReduceSum(T val) {
+  unsigned mask = 0u;
+  CREATE_SHFL_MASK(mask, true);
+  for (int offset = warpSize / 2; offset > 0; offset /= 2) {
+    val += phi::backends::gpu::CudaShuffleDownSync(mask, val, offset);
+  }
+  return val;
+}
+
+template <typename T>
+__forceinline__ __device__ T BlockReduce(T val) {
+  static __shared__ T shared[32];  // Shared mem for 32 partial sums
+  int lane = threadIdx.x % warpSize;
+  int wid = threadIdx.x / warpSize;
+
+  val = WarpReduceSum(val);  // Each warp performs partial reduction
+
+  if (lane == 0) shared[wid] = val;  // Write reduced value to shared memory
+
+  __syncthreads();  // Wait for all partial reductions
+
+  // read from shared memory only if that warp existed
+  val =
+      (threadIdx.x < blockDim.x / warpSize) ? shared[lane] : static_cast<T>(0);
+
+  if (wid == 0) val = WarpReduceSum(val);  // Final reduce within first warp
+
+  return val;
+}
+
+template <typename T>
+__global__ void DeviceReduceSum(T* in, T* out, size_t N) {
+  T sum(0);
+  CUDA_KERNEL_LOOP(i, N) { sum += in[i]; }
+  sum = BlockReduce<T>(sum);
+  __syncthreads();
+  if (threadIdx.x == 0) out[blockIdx.x] = sum;
+}
+
+template <typename T>
+void TestReduce(size_t num, float atol = 0.01) {
+  T* in1;
+  T *d_in1, *d_in2;
+  size_t size = sizeof(T) * num;
+#ifdef PADDLE_WITH_HIP
+  hipMalloc(reinterpret_cast<void**>(&d_in1), size);
+  hipMalloc(reinterpret_cast<void**>(&d_in2), sizeof(T));
+#else
+  cudaMalloc(reinterpret_cast<void**>(&d_in1), size);
+  cudaMalloc(reinterpret_cast<void**>(&d_in2), sizeof(T));
+#endif
+  in1 = reinterpret_cast<T*>(malloc(size));
+  std::minstd_rand engine;
+  std::uniform_real_distribution<double> dist(0.0, 1.0);
+  for (size_t i = 0; i < num; ++i) {
+    in1[i] = static_cast<T>(dist(engine));
+  }
+  auto out = std::accumulate(in1, in1 + num, static_cast<T>(0));
+#ifdef PADDLE_WITH_HIP
+  hipMemcpy(d_in1, in1, size, hipMemcpyHostToDevice);
+  hipDeviceSynchronize();
+  hipLaunchKernelGGL(HIP_KERNEL_NAME(DeviceReduceSum<T>),
+                     dim3(1),
+                     dim3(PADDLE_CUDA_NUM_THREADS),
+                     0,
+                     0,
+                     d_in1,
+                     d_in2,
+                     num);
+  hipMemcpy(in1, d_in2, sizeof(T), hipMemcpyDeviceToHost);
+  hipDeviceSynchronize();
+#else
+  cudaMemcpy(d_in1, in1, size, cudaMemcpyHostToDevice);
+  cudaDeviceSynchronize();
+  DeviceReduceSum<T><<<1, PADDLE_CUDA_NUM_THREADS>>>(d_in1, d_in2, num);
+  cudaMemcpy(in1, d_in2, sizeof(T), cudaMemcpyDeviceToHost);
+  cudaDeviceSynchronize();
+#endif
+  // NOTE(dzhwinter): the float16 add has small underflow/overflow
+  // so we use EXPECT_NEAR to check the result.
+  EXPECT_NEAR(static_cast<float>(in1[0]), static_cast<float>(out), atol);
+  free(in1);
+#ifdef PADDLE_WITH_HIP
+  hipFree(d_in1);
+  hipFree(d_in2);
+#else
+  cudaFree(d_in1);
+  cudaFree(d_in2);
+#endif
+}
+
+TEST(CudaShuffleSync, float16) {
+  TestReduce<float>(10);
+  TestReduce<float>(1000);
+
+  // float16 will overflow or accumulate truncate errors in big size.
+  TestReduce<float16>(10);
+  TestReduce<float16>(100, /*atol error*/ 1.0);
+}

test/cpp/fluid/platform/device/gpu/cudnn_desc_test.cc

@@ -0,0 +1,44 @@
+// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include "paddle/phi/core/platform/device/gpu/gpu_dnn.h"
+
+namespace paddle {
+namespace platform {
+
+TEST(TensorDescriptor, Empty) {
+  phi::backends::gpu::ActivationDescriptor a;
+  phi::backends::gpu::TensorDescriptor t;
+  phi::backends::gpu::TensorDescriptor t1;
+  phi::backends::gpu::TensorDescriptor *t11 =
+      new phi::backends::gpu::TensorDescriptor();
+  delete t11;
+  std::unique_ptr<phi::backends::gpu::TensorDescriptor> tt(
+      new phi::backends::gpu::TensorDescriptor());
+}
+
+TEST(TensorDescriptor, Normal) {
+  phi::DenseTensor tt;
+  tt.Resize({2, 3, 4});
+  tt.mutable_data<float>(phi::CPUPlace());
+
+  phi::backends::gpu::TensorDescriptor desc;
+  desc.set(tt);
+  EXPECT_TRUE(desc.desc() != nullptr);
+}
+
+}  // namespace platform
+}  // namespace paddle

test/cpp/fluid/platform/device/gpu/rocm/CMakeLists.txt

@@ -0,0 +1,4 @@
+hip_test(
+  miopen_helper_test
+  SRCS miopen_helper_test.cc
+  DEPS phi common)

test/cpp/fluid/platform/device/gpu/rocm/miopen_helper_test.cc

@@ -0,0 +1,92 @@
+/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#define GOOGLE_GLOG_DLL_DECL
+
+#include <gtest/gtest.h>
+
+#include "paddle/phi/core/platform/device/gpu/gpu_dnn.h"
+
+TEST(MIOpenHelper, ScopedTensorDescriptor) {
+  using phi::DataLayout;
+  using phi::backends::gpu::ScopedTensorDescriptor;
+
+  ScopedTensorDescriptor tensor_desc;
+  std::vector<int> shape = {2, 4, 6, 6};
+  auto desc = tensor_desc.descriptor<float>(DataLayout::NCHW, shape);
+
+  miopenDataType_t type;
+  int nd;
+  std::vector<int> dims(4);
+  std::vector<int> strides(4);
+  phi::dynload::miopenGetTensorDescriptor(
+      desc, &type, dims.data(), strides.data());
+  phi::dynload::miopenGetTensorDescriptorSize(desc, &nd);
+
+  EXPECT_EQ(nd, 4);
+  for (size_t i = 0; i < dims.size(); ++i) {
+    EXPECT_EQ(dims[i], shape[i]);
+  }
+  EXPECT_EQ(strides[3], 1);
+  EXPECT_EQ(strides[2], 6);
+  EXPECT_EQ(strides[1], 36);
+  EXPECT_EQ(strides[0], 144);
+
+  // test tensor5d: ScopedTensorDescriptor
+  ScopedTensorDescriptor tensor5d_desc;
+  std::vector<int> shape_5d = {2, 4, 6, 6, 6};
+  auto desc_5d = tensor5d_desc.descriptor<float>(DataLayout::NCDHW, shape_5d);
+
+  std::vector<int> dims_5d(5);
+  std::vector<int> strides_5d(5);
+  phi::dynload::miopenGetTensorDescriptor(
+      desc_5d, &type, dims_5d.data(), strides_5d.data());
+  phi::dynload::miopenGetTensorDescriptorSize(desc_5d, &nd);
+
+  EXPECT_EQ(nd, 5);
+  for (size_t i = 0; i < dims_5d.size(); ++i) {
+    EXPECT_EQ(dims_5d[i], shape_5d[i]);
+  }
+  EXPECT_EQ(strides_5d[4], 1);
+  EXPECT_EQ(strides_5d[3], 6);
+  EXPECT_EQ(strides_5d[2], 36);
+  EXPECT_EQ(strides_5d[1], 216);
+  EXPECT_EQ(strides_5d[0], 864);
+}
+
+TEST(MIOpenHelper, ScopedConvolutionDescriptor) {
+  using phi::backends::gpu::ScopedConvolutionDescriptor;
+
+  ScopedConvolutionDescriptor conv_desc;
+  std::vector<int> src_pads = {2, 2, 2};
+  std::vector<int> src_strides = {1, 1, 1};
+  std::vector<int> src_dilations = {1, 1, 1};
+  auto desc = conv_desc.descriptor<float>(src_pads, src_strides, src_dilations);
+
+  miopenConvolutionMode_t mode;
+  int nd;
+  std::vector<int> pads(3);
+  std::vector<int> strides(3);
+  std::vector<int> dilations(3);
+  phi::dynload::miopenGetConvolutionNdDescriptor(
+      desc, 3, &nd, pads.data(), strides.data(), dilations.data(), &mode);
+
+  EXPECT_EQ(nd, 3);
+  for (size_t i = 0; i < src_pads.size(); ++i) {
+    EXPECT_EQ(pads[i], src_pads[i]);
+    EXPECT_EQ(strides[i], src_strides[i]);
+    EXPECT_EQ(dilations[i], src_dilations[i]);
+  }
+  EXPECT_EQ(mode, miopenConvolution);
+}

test/cpp/fluid/platform/device_code_test.cc

@@ -0,0 +1,131 @@
+/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/backends/device_code.h"
+
+#include <utility>
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/framework/lod_tensor.h"
+#include "paddle/fluid/platform/init.h"
+
+#ifdef PADDLE_WITH_CUDA
+constexpr auto saxpy_code = R"(
+extern "C" __global__
+void saxpy_kernel(float a, float *x, float* y, float* z, size_t n) {
+  for (size_t tid = blockIdx.x * blockDim.x + threadIdx.x; tid < n;
+       tid += blockDim.x * gridDim.x) {
+    z[tid] = a * x[tid] + y[tid];
+  }
+}
+)";
+#endif
+
+#ifdef PADDLE_WITH_HIP
+constexpr auto saxpy_code = R"(
+#include <hip/hip_runtime.h>
+extern "C" __global__
+void saxpy_kernel(float a, float *x, float* y, float* z, size_t n) {
+  for (size_t tid = blockIdx.x * blockDim.x + threadIdx.x; tid < n;
+       tid += blockDim.x * gridDim.x) {
+    z[tid] = a * x[tid] + y[tid];
+  }
+}
+)";
+#endif
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+TEST(DeviceCode, cuda) {
+  if (!phi::dynload::HasNVRTC() || !phi::dynload::HasCUDADriver()) {
+    return;
+  }
+
+  paddle::framework::InitDevices({0});
+  phi::GPUPlace place = phi::GPUPlace(0);
+  phi::GPUDeviceCode code(place, "saxpy_kernel", saxpy_code);
+
+  phi::DenseTensor cpu_x;
+  phi::DenseTensor cpu_y;
+  phi::DenseTensor cpu_z;
+
+  float scale = 2;
+  auto dims = common::make_ddim(
+      {static_cast<int64_t>(256), static_cast<int64_t>(1024)});
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* cpu_ctx = reinterpret_cast<phi::CPUContext*>(pool.Get(phi::CPUPlace()));
+  cpu_x.Resize(dims);
+  cpu_ctx->template Alloc<float>(&cpu_x);
+  cpu_y.Resize(dims);
+  cpu_ctx->template Alloc<float>(&cpu_y);
+
+  size_t n = cpu_x.numel();
+  for (size_t i = 0; i < n; ++i) {
+    cpu_x.data<float>()[i] = static_cast<float>(i);
+  }
+  for (size_t i = 0; i < n; ++i) {
+    cpu_y.data<float>()[i] = static_cast<float>(0.5);
+  }
+
+  phi::DenseTensor x;
+  phi::DenseTensor y;
+  phi::DenseTensor z;
+
+  auto* dev_ctx = reinterpret_cast<phi::GPUContext*>(pool.Get(place));
+  x.Resize(dims);
+  float* x_data = dev_ctx->template Alloc<float>(&x);
+  y.Resize(dims);
+  float* y_data = dev_ctx->template Alloc<float>(&y);
+  z.Resize(dims);
+  float* z_data = dev_ctx->template Alloc<float>(&z);
+
+  paddle::framework::TensorCopySync(cpu_x, place, &x);
+  paddle::framework::TensorCopySync(cpu_y, place, &y);
+
+  EXPECT_EQ(code.Compile(), true);
+
+  std::vector<void*> args = {&scale, &x_data, &y_data, &z_data, &n};
+  code.SetNumThreads(1024);
+  code.SetWorkloadPerThread(1);
+  code.Launch(n, &args);
+
+  dev_ctx->Wait();
+
+  paddle::framework::TensorCopySync(z, phi::CPUPlace(), &cpu_z);
+  for (size_t i = 0; i < n; i++) {
+    EXPECT_EQ(cpu_z.data<float>()[i], static_cast<float>(i) * scale + 0.5);
+  }
+}
+
+TEST(DeviceCodePool, cuda) {
+  if (!phi::dynload::HasNVRTC() || !phi::dynload::HasCUDADriver()) {
+    return;
+  }
+
+  paddle::framework::InitDevices({0});
+  phi::GPUPlace place = phi::GPUPlace(0);
+  phi::DeviceCodePool& pool = phi::DeviceCodePool::Init({place});
+  size_t num_device_codes_before = pool.size(place);
+  EXPECT_EQ(num_device_codes_before, 0UL);
+
+  std::unique_ptr<phi::DeviceCode> code(
+      new phi::GPUDeviceCode(place, "saxpy_kernel", saxpy_code));
+  LOG(INFO) << "origin ptr: " << code.get();
+  pool.Set(std::move(code));
+  size_t num_device_codes_after = pool.size(place);
+  EXPECT_EQ(num_device_codes_after, 1UL);
+
+  phi::DeviceCode* code_get = pool.Get(place, "saxpy_kernel");
+  LOG(INFO) << "get ptr: " << code_get;
+}
+#endif

test/cpp/fluid/platform/device_context_test.cu

@@ -0,0 +1,161 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+#include <vector>
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/memory/allocation/allocator_facade.h"
+#include "paddle/phi/core/platform/device_context.h"
+
+TEST(Device, Init) {
+  using phi::DeviceContext;
+  using phi::GPUContext;
+  using phi::GPUPlace;
+
+  int count = paddle::platform::GetGPUDeviceCount();
+  for (int i = 0; i < count; i++) {
+    phi::GPUContext* device_context = new phi::GPUContext(GPUPlace(i));
+    device_context->SetAllocator(
+        paddle::memory::allocation::AllocatorFacade::Instance()
+            .GetAllocator(GPUPlace(i), device_context->stream())
+            .get());
+    device_context->SetHostAllocator(
+        paddle::memory::allocation::AllocatorFacade::Instance()
+            .GetAllocator(phi::CPUPlace())
+            .get());
+    device_context->SetZeroAllocator(
+        paddle::memory::allocation::AllocatorFacade::Instance()
+            .GetZeroAllocator(GPUPlace(i))
+            .get());
+    device_context->SetHostZeroAllocator(
+        paddle::memory::allocation::AllocatorFacade::Instance()
+            .GetZeroAllocator(phi::CPUPlace())
+            .get());
+    device_context->SetPinnedAllocator(
+        paddle::memory::allocation::AllocatorFacade::Instance()
+            .GetAllocator(phi::GPUPinnedPlace())
+            .get());
+    device_context->PartialInitWithAllocator();
+
+    Eigen::GpuDevice* gpu_device = device_context->eigen_device();
+    ASSERT_NE(nullptr, gpu_device);
+    delete device_context;
+  }
+}
+
+TEST(Device, GPUContext) {
+  using phi::GPUContext;
+  using phi::GPUPlace;
+
+  int count = paddle::platform::GetGPUDeviceCount();
+  for (int i = 0; i < count; i++) {
+    phi::GPUContext* device_context = new phi::GPUContext(GPUPlace(i));
+    device_context->SetAllocator(
+        paddle::memory::allocation::AllocatorFacade::Instance()
+            .GetAllocator(GPUPlace(i), device_context->stream())
+            .get());
+    device_context->SetHostAllocator(
+        paddle::memory::allocation::AllocatorFacade::Instance()
+            .GetAllocator(phi::CPUPlace())
+            .get());
+    device_context->SetZeroAllocator(
+        paddle::memory::allocation::AllocatorFacade::Instance()
+            .GetZeroAllocator(GPUPlace(i))
+            .get());
+    device_context->SetHostZeroAllocator(
+        paddle::memory::allocation::AllocatorFacade::Instance()
+            .GetZeroAllocator(phi::CPUPlace())
+            .get());
+    device_context->SetPinnedAllocator(
+        paddle::memory::allocation::AllocatorFacade::Instance()
+            .GetAllocator(phi::GPUPinnedPlace())
+            .get());
+    device_context->PartialInitWithAllocator();
+    Eigen::GpuDevice* gpu_device = device_context->eigen_device();
+    ASSERT_NE(nullptr, gpu_device);
+#ifdef PADDLE_WITH_HIP
+    miopenHandle_t cudnn_handle = device_context->cudnn_handle();
+#else
+    cudnnHandle_t cudnn_handle = device_context->cudnn_handle();
+#endif
+    ASSERT_NE(nullptr, cudnn_handle);
+#ifdef PADDLE_WITH_HIP
+    rocblas_handle cublas_handle = device_context->cublas_handle();
+#else
+    cublasHandle_t cublas_handle = device_context->cublas_handle();
+#endif
+    ASSERT_NE(nullptr, cublas_handle);
+    delete device_context;
+  }
+}
+
+TEST(Device, HostZeroAllocator) {
+  using phi::GPUPlace;
+
+  auto device_context = std::make_unique<phi::GPUContext>(GPUPlace(0));
+  device_context->SetAllocator(
+      paddle::memory::allocation::AllocatorFacade::Instance()
+          .GetAllocator(GPUPlace(0), device_context->stream())
+          .get());
+  device_context->SetHostAllocator(
+      paddle::memory::allocation::AllocatorFacade::Instance()
+          .GetAllocator(phi::CPUPlace())
+          .get());
+  device_context->SetZeroAllocator(
+      paddle::memory::allocation::AllocatorFacade::Instance()
+          .GetZeroAllocator(GPUPlace(0))
+          .get());
+  device_context->SetHostZeroAllocator(
+      paddle::memory::allocation::AllocatorFacade::Instance()
+          .GetZeroAllocator(phi::CPUPlace())
+          .get());
+  device_context->SetPinnedAllocator(
+      paddle::memory::allocation::AllocatorFacade::Instance()
+          .GetAllocator(phi::GPUPinnedPlace())
+          .get());
+  device_context->PartialInitWithAllocator();
+
+  phi::DenseTensor tensor;
+  tensor.Resize({0});
+  device_context->HostAlloc<float>(&tensor);
+  ASSERT_EQ(tensor.place().GetType(), phi::AllocationType::CPU);
+  ASSERT_EQ(tensor.numel(), 0);
+  ASSERT_EQ(tensor.dtype(), phi::DataType::FLOAT32);
+
+  phi::GPUContext gpu_context(GPUPlace(0));
+  gpu_context.SetHostZeroAllocator(&device_context->GetHostZeroAllocator());
+  gpu_context.HostAlloc<float>(&tensor);
+  ASSERT_EQ(tensor.place().GetType(), phi::AllocationType::CPU);
+}
+
+TEST(Device, DeviceContextPool) {
+  using phi::CPUPlace;
+  using phi::DeviceContextPool;
+  using phi::GPUContext;
+  using phi::GPUPlace;
+  using phi::Place;
+
+  DeviceContextPool& pool = DeviceContextPool::Instance();
+  auto cpu_dev_ctx1 = pool.Get(CPUPlace());
+  auto cpu_dev_ctx2 = pool.Get(CPUPlace());
+  ASSERT_EQ(cpu_dev_ctx2, cpu_dev_ctx1);
+
+  std::vector<Place> gpu_places;
+  int count = paddle::platform::GetGPUDeviceCount();
+  for (int i = 0; i < count; ++i) {
+    auto dev_ctx = pool.Get(GPUPlace(i));
+    ASSERT_NE(dev_ctx, nullptr);
+  }
+}

test/cpp/fluid/platform/device_context_test_cuda_graph.cu

@@ -0,0 +1,40 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+#include "cuda.h"          // NOLINT
+#include "cuda_runtime.h"  // NOLINT
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/phi/core/memory/allocation/allocator_facade.h"
+#include "paddle/phi/core/platform/cuda_graph_with_memory_pool.h"
+#include "paddle/phi/core/platform/device_context.h"
+
+#ifdef PADDLE_WITH_CUDA
+TEST(Device, DeviceContextWithCUDAGraph) {
+  using phi::DeviceContext;
+  using phi::DeviceContextPool;
+  using phi::GPUContext;
+  using phi::GPUPlace;
+  using phi::Place;
+
+  DeviceContextPool& pool = DeviceContextPool::Instance();
+  Place place = GPUPlace(0);
+  auto* dev_ctx = pool.Get(place);
+  paddle::platform::BeginCUDAGraphCapture(
+      place, cudaStreamCaptureMode::cudaStreamCaptureModeThreadLocal, 0);
+  ASSERT_EQ(dev_ctx->IsCUDAGraphAllocatorValid(), true);
+  dev_ctx->GetCUDAGraphAllocator();
+  paddle::platform::EndCUDAGraphCapture();
+  ASSERT_EQ(dev_ctx->IsCUDAGraphAllocatorValid(), false);
+}
+#endif

test/cpp/fluid/platform/device_context_xpu_test.cc

@@ -0,0 +1,52 @@
+/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+#include <vector>
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/phi/core/platform/device_context.h"
+
+TEST(Device, Init) {
+  using phi::DeviceContext;
+  using phi::XPUContext;
+  using phi::XPUPlace;
+
+  int count = paddle::platform::GetXPUDeviceCount();
+  for (int i = 0; i < count; i++) {
+    XPUContext* device_context = new XPUContext(XPUPlace(i));
+    xpu::Context* ctx = device_context->x_context();
+    ASSERT_NE(nullptr, ctx);
+    delete device_context;
+  }
+}
+
+TEST(Device, DeviceContextPool) {
+  using phi::CPUPlace;
+  using phi::DeviceContextPool;
+  using phi::Place;
+  using phi::XPUContext;
+  using phi::XPUPlace;
+
+  DeviceContextPool& pool = DeviceContextPool::Instance();
+  auto cpu_dev_ctx1 = pool.Get(CPUPlace());
+  auto cpu_dev_ctx2 = pool.Get(CPUPlace());
+  ASSERT_EQ(cpu_dev_ctx2, cpu_dev_ctx1);
+
+  std::vector<Place> xpu_places;
+  int count = paddle::platform::GetXPUDeviceCount();
+  for (int i = 0; i < count; ++i) {
+    auto dev_ctx = pool.Get(XPUPlace(i));
+    ASSERT_NE(dev_ctx, nullptr);
+  }
+}

test/cpp/fluid/platform/device_event_test.cc

@@ -0,0 +1,147 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/core/platform/device_event.h"
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/phi/common/place.h"
+
+using ::paddle::platform::kCPU;
+using ::paddle::platform::kCUDA;
+
+using paddle::platform::DeviceEvent;
+using phi::DeviceContextPool;
+
+#ifdef PADDLE_WITH_CUDA
+#include <cuda_runtime.h>
+
+TEST(DeviceEvent, CUDA) {
+  VLOG(1) << "In Test";
+  using phi::GPUPlace;
+
+  auto& pool = DeviceContextPool::Instance();
+  auto place = GPUPlace(0);
+  auto* context = static_cast<phi::GPUContext*>(pool.Get(place));
+
+  ASSERT_NE(context, nullptr);
+  // case 1. test for event_creator
+  DeviceEvent event(place, paddle::platform::GenerateDeviceEventFlag());
+  ASSERT_NE(event.GetEvent().get(), nullptr);
+  bool status = event.Query();
+  ASSERT_EQ(status, true);
+  // case 2. test for event_recorder
+  event.Record(context);
+  // case 3. test for event_finisher
+  event.Finish();
+  status = event.Query();
+  ASSERT_EQ(status, true);
+
+  // case 4. test for event_waiter
+  float *src_fp32, *dst_fp32;
+  int size = 1000000 * sizeof(float);
+  cudaMallocHost(reinterpret_cast<void**>(&src_fp32), size);
+  cudaMalloc(reinterpret_cast<void**>(&dst_fp32), size);
+  cudaMemcpyAsync(
+      dst_fp32, src_fp32, size, cudaMemcpyHostToDevice, context->stream());
+  event.Record(context);  // step 1. record it
+  status = event.Query();
+  ASSERT_EQ(status, false);
+
+  event.Wait(kCUDA, context);  // step 2. add streamWaitEvent
+  status = event.Query();
+  ASSERT_EQ(status, false);  // async
+
+  event.Wait(kCPU, context);  // step 3. EventSynchronize
+  status = event.Query();
+  ASSERT_EQ(status, true);  // sync
+
+  // release resource
+  cudaFree(dst_fp32);
+  cudaFreeHost(src_fp32);
+}
+#endif
+
+#ifdef PADDLE_WITH_HIP
+#include <hip/hip_runtime.h>
+
+TEST(DeviceEvent, CUDA) {
+  VLOG(1) << "In Test";
+  using phi::GPUPlace;
+
+  auto& pool = DeviceContextPool::Instance();
+  auto place = GPUPlace(0);
+  auto* context = static_cast<phi::GPUContext*>(pool.Get(place));
+
+  ASSERT_NE(context, nullptr);
+  // case 1. test for event_creator
+  DeviceEvent event(place, paddle::platform::GenerateDeviceEventFlag());
+  ASSERT_NE(event.GetEvent().get(), nullptr);
+  bool status = event.Query();
+  ASSERT_EQ(status, true);
+  // case 2. test for event_recorder
+  event.Record(context);
+  status = event.Query();
+  ASSERT_EQ(status, false);
+  // case 3. test for event_finisher
+  event.Finish();
+  status = event.Query();
+  ASSERT_EQ(status, true);
+
+  // case 4. test for event_waiter
+  float *src_fp32, *dst_fp32;
+  int size = 1000000 * sizeof(float);
+  hipMallocHost(reinterpret_cast<void**>(&src_fp32), size);
+  hipMalloc(reinterpret_cast<void**>(&dst_fp32), size);
+  hipMemcpyAsync(
+      dst_fp32, src_fp32, size, hipMemcpyHostToDevice, context->stream());
+  event.Record(context);  // step 1. record it
+  status = event.Query();
+  ASSERT_EQ(status, false);
+
+  event.Wait(kCUDA, context);  // step 2. add streamWaitEvent
+  status = event.Query();
+  ASSERT_EQ(status, false);  // async
+
+  event.Wait(kCPU, context);  // step 3. EventSynchronize
+  status = event.Query();
+  ASSERT_EQ(status, true);  // sync
+
+  // release resource
+  hipFree(dst_fp32);
+  hipFreeHost(src_fp32);
+}
+#endif
+
+TEST(DeviceEvent, CPU) {
+  using phi::CPUPlace;
+  auto place = CPUPlace();
+  DeviceEvent event(place, paddle::platform::GenerateDeviceEventFlag());
+  auto& pool = DeviceContextPool::Instance();
+  auto* context = pool.Get(place);
+
+  // TODO(Aurelius84): All DeviceContext should has Record/Wait
+  event.Record(context);
+  event.SetFinished();
+  bool status = event.Query();
+  ASSERT_EQ(status, true);
+
+  // test for Record again
+  event.Record(context);
+  status = event.Query();
+  ASSERT_EQ(status, false);  // SCHEDULED
+
+  event.Reset();
+  ASSERT_EQ(status, false);  // INITIALIZED
+}

test/cpp/fluid/platform/enforce_test.cc

@@ -0,0 +1,584 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/fluid/platform/enforce.h"
+
+#include <array>
+#include <list>
+
+#include "gtest/gtest.h"
+
+TEST(ENFORCE, OK) {
+  PADDLE_ENFORCE(true,
+                 common::errors::Unavailable(
+                     "PADDLE_ENFORCE is ok %d now %f.", 123, 0.345));
+  size_t val = 1;
+  const size_t limit = 10;
+  PADDLE_ENFORCE(val < limit,
+                 common::errors::Unavailable("PADDLE_ENFORCE tests failed."));
+}
+
+TEST(ENFORCE, FAILED) {
+  bool caught_exception = false;
+  try {
+    PADDLE_ENFORCE(false,
+                   common::errors::Unavailable(
+                       "PADDLE_ENFORCE won't work %d at all.", 123));
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("PADDLE_ENFORCE won't work 123 at all.") !=
+                std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+
+  caught_exception = false;
+  try {
+    PADDLE_ENFORCE(
+        false,
+        common::errors::Unavailable("PADDLE_ENFORCE won't work at all."));
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("PADDLE_ENFORCE won't work at all.") !=
+                std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+
+  caught_exception = false;
+  try {
+    PADDLE_ENFORCE(
+        false,
+        common::errors::Unavailable("PADDLE_ENFORCE won't work at all."));
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    EXPECT_NE(std::string(error.what()).find(" at "), 0UL);
+  }
+  EXPECT_TRUE(caught_exception);
+}
+
+TEST(ENFORCE, NO_ARG_OK) {
+  int a = 2;
+  int b = 2;
+  PADDLE_ENFORCE_EQ(
+      a, b, common::errors::Unavailable("PADDLE_ENFORCE_EQ tests failed."));
+  // test enforce with extra message.
+  PADDLE_ENFORCE_EQ(a,
+                    b,
+                    common::errors::Unavailable(
+                        "Some %s wrong in PADDLE_ENFORCE_EQ.", "info"));
+}
+
+TEST(ENFORCE_EQ, NO_EXTRA_MSG_FAIL) {
+  int a = 2;
+  bool caught_exception = false;
+  try {
+    PADDLE_ENFORCE_EQ(a,
+                      1 + 3,
+                      common::errors::InvalidArgument(
+                          "The result is not equal correct result."));
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("Expected a == 1 + 3, but received a:2 != 1 "
+                            "+ 3:4.") != std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+}
+
+TEST(ENFORCE_EQ, EXTRA_MSG_FAIL) {
+  int a = 2;
+  bool caught_exception = false;
+  try {
+    PADDLE_ENFORCE_EQ(a,
+                      1 + 3,
+                      common::errors::InvalidArgument(
+                          "The result is not equal correct result."));
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(
+        ex_msg.find("Expected a == 1 + 3, but received a:2 != 1 + 3:4.") !=
+        std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+}
+
+TEST(ENFORCE_NE, OK) {
+  PADDLE_ENFORCE_NE(
+      1, 2, common::errors::Unavailable("PADDLE_ENFORCE_NE tests failed."));
+  PADDLE_ENFORCE_NE(
+      1.0, 2UL, common::errors::Unavailable("PADDLE_ENFORCE_NE tests failed."));
+}
+TEST(ENFORCE_NE, FAIL) {
+  bool caught_exception = false;
+
+  try {
+    // 2UL here to check data type compatible
+    PADDLE_ENFORCE_NE(1.0,
+                      1UL,
+                      common::errors::Unavailable(
+                          "Expected 1.0 != 1UL, but received 1.0:1 == 1UL:1."));
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("Expected 1.0 != 1UL, but "
+                            "received 1.0:1 == 1UL:1.") != std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+}
+
+TEST(ENFORCE_GT, OK) {
+  PADDLE_ENFORCE_GT(
+      2, 1, common::errors::Unavailable("PADDLE_ENFORCE_GT tests failed."));
+}
+TEST(ENFORCE_GT, FAIL) {
+  bool caught_exception = false;
+  try {
+    PADDLE_ENFORCE_GT(1,
+                      2,
+                      common::errors::InvalidArgument(
+                          "Expected 1 > 2, but received 1:1 <= 2:2."));
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("Expected 1 > 2, but received 1:1 <= 2:2.") !=
+                std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+}
+
+TEST(ENFORCE_GE, OK) {
+  PADDLE_ENFORCE_GE(
+      2, 2, common::errors::Unavailable("PADDLE_ENFORCE_GE tests failed."));
+  PADDLE_ENFORCE_GE(
+      3, 2, common::errors::Unavailable("PADDLE_ENFORCE_GE tests failed."));
+  PADDLE_ENFORCE_GE(
+      3.21,
+      2.0,
+      common::errors::Unavailable("PADDLE_ENFORCE_GE tests failed."));
+}
+TEST(ENFORCE_GE, FAIL) {
+  bool caught_exception = false;
+  try {
+    PADDLE_ENFORCE_GE(1,
+                      2,
+                      common::errors::InvalidArgument(
+                          "Expected 1 >= 2, but received 1:1 < 2:2."));
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("Expected 1 >= 2, but received 1:1 < 2:2.") !=
+                std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+}
+
+TEST(ENFORCE_LE, OK) {
+  PADDLE_ENFORCE_LE(
+      1, 1, common::errors::Unavailable("PADDLE_ENFORCE_LE tests failed."));
+  PADDLE_ENFORCE_LE(
+      1UL, 1UL, common::errors::Unavailable("PADDLE_ENFORCE_LE tests failed."));
+  PADDLE_ENFORCE_LE(
+      2, 3, common::errors::Unavailable("PADDLE_ENFORCE_LE tests failed."));
+  PADDLE_ENFORCE_LE(
+      2UL, 3UL, common::errors::Unavailable("PADDLE_ENFORCE_LE tests failed."));
+  PADDLE_ENFORCE_LE(
+      2.0, 3.2, common::errors::Unavailable("PADDLE_ENFORCE_LE tests failed."));
+}
+TEST(ENFORCE_LE, FAIL) {
+  bool caught_exception = false;
+  try {
+    PADDLE_ENFORCE_GT(1,
+                      2,
+                      common::errors::InvalidArgument(
+                          "Expected 1 > 2, but received 1:1 <= 2:2."));
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("Expected 1 > 2, but received 1:1 <= 2:2.") !=
+                std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+}
+
+TEST(ENFORCE_LT, OK) {
+  PADDLE_ENFORCE_LT(
+      3, 10, common::errors::Unavailable("PADDLE_ENFORCE_LT tests failed."));
+  PADDLE_ENFORCE_LT(
+      2UL, 3UL, common::errors::Unavailable("PADDLE_ENFORCE_LT tests failed."));
+  PADDLE_ENFORCE_LT(
+      2, 3, common::errors::Unavailable("PADDLE_ENFORCE_LT tests failed."));
+}
+TEST(ENFORCE_LT, FAIL) {
+  bool caught_exception = false;
+  try {
+    PADDLE_ENFORCE_LT(
+        1UL,
+        0.12,
+        common::errors::InvalidArgument(
+            "Expected 1UL < 0.12, but received 1UL:1 >= 0.12:0.12."));
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("Expected 1UL < 0.12, but "
+                            "received 1UL:1 >= 0.12:0.12.") !=
+                std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+}
+
+TEST(ENFORCE_NOT_NULL, OK) {
+  int* a = new int;
+  PADDLE_ENFORCE_NOT_NULL(
+      a, common::errors::Unavailable("PADDLE_ENFORCE_NOT_NULL tests failed."));
+  delete a;
+}
+TEST(ENFORCE_NOT_NULL, FAIL) {
+  bool caught_exception = false;
+  try {
+    int* a = nullptr;
+    PADDLE_ENFORCE_NOT_NULL(
+        a, common::errors::Unavailable("The a should not be null."));
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("The a should not be null.") != std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+}
+
+struct Dims {
+  std::array<size_t, 4> dims_;
+
+  bool operator==(const Dims& o) const {
+    for (size_t i = 0; i < 4; ++i) {
+      if (dims_[i] != o.dims_[i]) return false;
+    }
+    return true;
+  }
+};
+
+std::ostream& operator<<(std::ostream& os, const Dims& d) {
+  for (size_t i = 0; i < 4; ++i) {
+    if (i == 0) {
+      os << "[";
+    }
+    os << d.dims_[i];
+    if (i == 4 - 1) {
+      os << "]";
+    } else {
+      os << ", ";
+    }
+  }
+  return os;
+}
+
+TEST(ENFORCE_USER_DEFINED_CLASS, EQ) {
+  Dims a{{1, 2, 3, 4}}, b{{1, 2, 3, 4}};
+  PADDLE_ENFORCE_EQ(
+      a, b, common::errors::Unavailable("PADDLE_ENFORCE_EQ tests failed."));
+}
+
+TEST(ENFORCE_USER_DEFINED_CLASS, NE) {
+  Dims a{{1, 2, 3, 4}}, b{{5, 6, 7, 8}};
+  bool caught_exception = false;
+  try {
+    PADDLE_ENFORCE_EQ(
+        a, b, common::errors::Unavailable("PADDLE_ENFORCE_EQ tests failed."));
+  } catch (paddle::platform::EnforceNotMet&) {
+    caught_exception = true;
+  }
+  EXPECT_TRUE(caught_exception);
+}
+
+TEST(EOF_EXCEPTION, THROW_EOF) {
+  bool caught_eof = false;
+  try {
+    PADDLE_THROW_EOF();
+  } catch (paddle::platform::EOFException& error) {
+    caught_eof = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("There is no next data.") != std::string::npos);
+  }
+  EXPECT_TRUE(caught_eof);
+}
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+template <typename T>
+bool CheckCudaStatusSuccess(T value, const std::string& msg = "success") {
+  PADDLE_ENFORCE_GPU_SUCCESS(value);
+  return true;
+}
+
+template <typename T>
+bool CheckCudaStatusFailure(T value, const std::string& msg) {
+  try {
+    PADDLE_ENFORCE_GPU_SUCCESS(value);
+    return false;
+  } catch (paddle::platform::EnforceNotMet& error) {
+    std::string ex_msg = error.what();
+    std::cout << ex_msg << std::endl;
+    return ex_msg.find(msg) != std::string::npos;
+  }
+}
+#ifdef PADDLE_WITH_HIP
+TEST(enforce, hip_success) {
+  EXPECT_TRUE(CheckCudaStatusSuccess(hipSuccess));
+  EXPECT_TRUE(CheckCudaStatusFailure(hipErrorInvalidValue, "Hip error"));
+  EXPECT_TRUE(CheckCudaStatusFailure(hipErrorOutOfMemory, "Hip error"));
+
+  EXPECT_TRUE(CheckCudaStatusSuccess(HIPRAND_STATUS_SUCCESS));
+  EXPECT_TRUE(
+      CheckCudaStatusFailure(HIPRAND_STATUS_VERSION_MISMATCH, "Hiprand error"));
+  EXPECT_TRUE(
+      CheckCudaStatusFailure(HIPRAND_STATUS_NOT_INITIALIZED, "Hiprand error"));
+
+  EXPECT_TRUE(CheckCudaStatusSuccess(miopenStatusSuccess));
+  EXPECT_TRUE(
+      CheckCudaStatusFailure(miopenStatusNotInitialized, "Miopen error"));
+  EXPECT_TRUE(CheckCudaStatusFailure(miopenStatusAllocFailed, "Miopen error"));
+
+  EXPECT_TRUE(CheckCudaStatusSuccess(rocblas_status_success));
+  EXPECT_TRUE(
+      CheckCudaStatusFailure(rocblas_status_invalid_handle, "Rocblas error"));
+  EXPECT_TRUE(
+      CheckCudaStatusFailure(rocblas_status_invalid_value, "Rocblas error"));
+  EXPECT_TRUE(CheckCudaStatusSuccess(HIPFFT_SUCCESS));
+  EXPECT_TRUE(CheckCudaStatusFailure(HIPFFT_INVALID_PLAN, "HIPFFT error"));
+  EXPECT_TRUE(CheckCudaStatusFailure(HIPFFT_ALLOC_FAILED, "HIPFFT error"));
+
+#if !defined(__APPLE__) && defined(PADDLE_WITH_RCCL)
+  EXPECT_TRUE(CheckCudaStatusSuccess(ncclSuccess));
+  EXPECT_TRUE(CheckCudaStatusFailure(ncclUnhandledCudaError, "Rccl error"));
+  EXPECT_TRUE(CheckCudaStatusFailure(ncclSystemError, "Rccl error"));
+#endif
+}
+#else
+TEST(enforce, cuda_success) {
+  EXPECT_TRUE(CheckCudaStatusSuccess(cudaSuccess));
+  EXPECT_TRUE(CheckCudaStatusFailure(cudaErrorInvalidValue, "CUDA error"));
+
+  EXPECT_TRUE(CheckCudaStatusFailure(cudaErrorMemoryAllocation, "CUDA error"));
+
+  EXPECT_TRUE(
+      CheckCudaStatusFailure(cudaErrorInsufficientDriver, "CUDA error"));
+  EXPECT_TRUE(
+      CheckCudaStatusFailure(cudaErrorContextIsDestroyed, "CUDA error"));
+
+  EXPECT_TRUE(CheckCudaStatusSuccess(CURAND_STATUS_SUCCESS));
+  EXPECT_TRUE(
+      CheckCudaStatusFailure(CURAND_STATUS_VERSION_MISMATCH, "CURAND error"));
+  EXPECT_TRUE(
+      CheckCudaStatusFailure(CURAND_STATUS_NOT_INITIALIZED, "CURAND error"));
+  EXPECT_TRUE(
+      CheckCudaStatusFailure(CURAND_STATUS_ARCH_MISMATCH, "CURAND error"));
+  EXPECT_TRUE(CheckCudaStatusFailure(CURAND_STATUS_LENGTH_NOT_MULTIPLE,
+                                     "CURAND error"));
+
+  EXPECT_TRUE(CheckCudaStatusSuccess(CUDNN_STATUS_SUCCESS));
+  EXPECT_TRUE(
+      CheckCudaStatusFailure(CUDNN_STATUS_NOT_INITIALIZED, "CUDNN error"));
+  EXPECT_TRUE(CheckCudaStatusFailure(CUDNN_STATUS_ALLOC_FAILED, "CUDNN error"));
+  EXPECT_TRUE(CheckCudaStatusFailure(CUDNN_STATUS_BAD_PARAM, "CUDNN error"));
+  EXPECT_TRUE(
+      CheckCudaStatusFailure(CUDNN_STATUS_LICENSE_ERROR, "CUDNN error"));
+
+  EXPECT_TRUE(CheckCudaStatusSuccess(CUBLAS_STATUS_SUCCESS));
+  EXPECT_TRUE(
+      CheckCudaStatusFailure(CUBLAS_STATUS_NOT_INITIALIZED, "CUBLAS error"));
+  EXPECT_TRUE(
+      CheckCudaStatusFailure(CUBLAS_STATUS_INVALID_VALUE, "CUBLAS error"));
+  EXPECT_TRUE(
+      CheckCudaStatusFailure(CUBLAS_STATUS_EXECUTION_FAILED, "CUBLAS error"));
+  EXPECT_TRUE(
+      CheckCudaStatusFailure(CUBLAS_STATUS_MAPPING_ERROR, "CUBLAS error"));
+
+  EXPECT_TRUE(CheckCudaStatusSuccess(CUSOLVER_STATUS_SUCCESS));
+  EXPECT_TRUE(CheckCudaStatusFailure(CUSOLVER_STATUS_NOT_INITIALIZED,
+                                     "CUSOLVER error"));
+  EXPECT_TRUE(
+      CheckCudaStatusFailure(CUSOLVER_STATUS_ALLOC_FAILED, "CUSOLVER error"));
+  EXPECT_TRUE(
+      CheckCudaStatusFailure(CUSOLVER_STATUS_INTERNAL_ERROR, "CUSOLVER error"));
+  EXPECT_TRUE(
+      CheckCudaStatusFailure(CUSOLVER_STATUS_INVALID_VALUE, "CUSOLVER error"));
+
+  EXPECT_TRUE(CheckCudaStatusSuccess(CUFFT_SUCCESS));
+  EXPECT_TRUE(CheckCudaStatusFailure(CUFFT_INVALID_PLAN, "CUFFT error"));
+  EXPECT_TRUE(CheckCudaStatusFailure(CUFFT_ALLOC_FAILED, "CUFFT error"));
+  EXPECT_TRUE(CheckCudaStatusFailure(CUFFT_INVALID_TYPE, "CUFFT error"));
+  EXPECT_TRUE(CheckCudaStatusFailure(CUFFT_INVALID_VALUE, "CUFFT error"));
+  EXPECT_TRUE(CheckCudaStatusFailure(CUFFT_INTERNAL_ERROR, "CUFFT error"));
+  EXPECT_TRUE(CheckCudaStatusFailure(CUFFT_EXEC_FAILED, "CUFFT error"));
+  EXPECT_TRUE(CheckCudaStatusFailure(CUFFT_SETUP_FAILED, "CUFFT error"));
+  EXPECT_TRUE(CheckCudaStatusFailure(CUFFT_INVALID_SIZE, "CUFFT error"));
+  EXPECT_TRUE(CheckCudaStatusFailure(CUFFT_UNALIGNED_DATA, "CUFFT error"));
+#ifdef CUFFT_INCOMPLETE_PARAMETER_LIST
+  EXPECT_TRUE(
+      CheckCudaStatusFailure(CUFFT_INCOMPLETE_PARAMETER_LIST, "CUFFT error"));
+#endif
+  EXPECT_TRUE(CheckCudaStatusFailure(CUFFT_INVALID_DEVICE, "CUFFT error"));
+#ifdef CUFFT_PARSE_ERROR
+  EXPECT_TRUE(CheckCudaStatusFailure(CUFFT_PARSE_ERROR, "CUFFT error"));
+#endif
+  EXPECT_TRUE(CheckCudaStatusFailure(CUFFT_NO_WORKSPACE, "CUFFT error"));
+  EXPECT_TRUE(CheckCudaStatusFailure(CUFFT_NOT_IMPLEMENTED, "CUFFT error"));
+#ifdef CUFFT_LICENSE_ERROR
+  EXPECT_TRUE(CheckCudaStatusFailure(CUFFT_LICENSE_ERROR, "CUFFT error"));
+#endif
+  EXPECT_TRUE(CheckCudaStatusFailure(CUFFT_NOT_SUPPORTED, "CUFFT error"));
+#if !defined(__APPLE__) && defined(PADDLE_WITH_NCCL)
+  EXPECT_TRUE(CheckCudaStatusSuccess(ncclSuccess));
+  EXPECT_TRUE(CheckCudaStatusFailure(ncclUnhandledCudaError, "NCCL error"));
+  EXPECT_TRUE(CheckCudaStatusFailure(ncclSystemError, "NCCL error"));
+  EXPECT_TRUE(CheckCudaStatusFailure(ncclInternalError,
+                                     "An internal check failed. This is either "
+                                     "a bug in NCCL or due to memory "
+                                     "corruption"));
+  EXPECT_TRUE(CheckCudaStatusFailure(ncclInvalidUsage,
+                                     "The call to NCCL is incorrect. This is "
+                                     "usually reflecting a programming error"));
+#endif
+}
+#endif
+#endif
+
+struct CannotToStringType {
+  explicit CannotToStringType(int num) : num_(num) {}
+
+  bool operator==(const CannotToStringType& other) const {
+    return num_ == other.num_;
+  }
+
+  bool operator!=(const CannotToStringType& other) const {
+    return num_ != other.num_;
+  }
+
+ private:
+  int num_;
+};
+
+TEST(enforce, cannot_to_string_type) {
+  static_assert(
+      !common::enforce::details::CanToString<CannotToStringType>::kValue,
+      "CannotToStringType must not be converted to string");
+  static_assert(common::enforce::details::CanToString<int>::kValue,
+                "int can be converted to string");
+  CannotToStringType obj1(3), obj2(4), obj3(3);
+
+  PADDLE_ENFORCE_NE(
+      obj1,
+      obj2,
+      common::errors::InvalidArgument("Object 1 is not equal to Object 2"));
+  PADDLE_ENFORCE_EQ(
+      obj1,
+      obj3,
+      common::errors::InvalidArgument("Object 1 is equal to Object 3"));
+
+  std::string msg = "Compare obj1 with obj2";
+  try {
+    PADDLE_ENFORCE_EQ(obj1, obj2, common::errors::InvalidArgument(msg));
+  } catch (paddle::platform::EnforceNotMet& error) {
+    std::string ex_msg = error.what();
+    LOG(INFO) << ex_msg;
+    EXPECT_TRUE(ex_msg.find(msg) != std::string::npos);
+    EXPECT_TRUE(
+        ex_msg.find("Expected obj1 == obj2, but received obj1 != obj2") !=
+        std::string::npos);
+  }
+
+  msg = "Compare x with y";
+  try {
+    int x = 3, y = 2;
+    PADDLE_ENFORCE_EQ(x, y, common::errors::InvalidArgument(msg));
+  } catch (paddle::platform::EnforceNotMet& error) {
+    std::string ex_msg = error.what();
+    LOG(INFO) << ex_msg;
+    EXPECT_TRUE(ex_msg.find(msg) != std::string::npos);
+    EXPECT_TRUE(ex_msg.find("Expected x == y, but received x:3 != y:2") !=
+                std::string::npos);
+  }
+
+  std::set<int> set;
+  PADDLE_ENFORCE_EQ(set.begin(),
+                    set.end(),
+                    common::errors::InvalidArgument(
+                        "The begin and end of set is not equal."));
+  set.insert(3);
+  PADDLE_ENFORCE_NE(
+      set.begin(),
+      set.end(),
+      common::errors::InvalidArgument("The begin and end of set is equal."));
+
+  std::list<float> list;
+  PADDLE_ENFORCE_EQ(list.begin(),
+                    list.end(),
+                    common::errors::InvalidArgument(
+                        "The begin and end of list is not equal."));
+  list.push_back(4);
+  PADDLE_ENFORCE_NE(
+      list.begin(),
+      list.end(),
+      common::errors::InvalidArgument("The begin and end of list is equal."));
+}
+
+TEST(GET_DATA_SAFELY_MACRO, SUCCESS) {
+  int* a = new int(10);  // NOLINT
+  GET_DATA_SAFELY(a, "Input", "X", "dummy");
+  delete a;
+}
+
+#ifndef _WIN32
+TEST(GET_DATA_SAFELY_MACRO, FAIL) {
+  bool caught_exception = false;
+  try {
+    int* a = nullptr;
+    GET_DATA_SAFELY(a, "Input", "X", "dummy");
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+  }
+  EXPECT_TRUE(caught_exception);
+}
+#endif
+
+TEST(OP_INOUT_CHECK_MACRO, SUCCESS) {
+  OP_INOUT_CHECK(true, "Input", "X", "dummy");
+}
+
+TEST(OP_INOUT_CHECK_MACRO, FAIL) {
+  bool caught_exception = false;
+  try {
+    OP_INOUT_CHECK(false, "Input", "X", "dummy");
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+  }
+  EXPECT_TRUE(caught_exception);
+}
+
+TEST(PADDLE_GET_SAFELY, SUCCESS) {
+  paddle::framework::Attribute attr;
+  attr = true;
+  bool rlt = PADDLE_GET(bool, attr);
+  EXPECT_EQ(rlt, true);
+}
+
+TEST(PADDLE_GET_SAFELY, FAIL) {
+  paddle::framework::Attribute attr;
+  attr = true;
+  bool caught_exception = false;
+  try {
+    PADDLE_GET(int, attr);
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+  }
+  EXPECT_TRUE(caught_exception);
+}

test/cpp/fluid/platform/errors_test.cc

@@ -0,0 +1,119 @@
+/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+
+licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/common/errors.h"
+
+#include <string>
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/platform/enforce.h"
+
+using namespace common::errors;  // NOLINT
+
+#define CHECK_PADDLE_THROW(EFUNC)                                          \
+  do {                                                                     \
+    bool caught_exception = false;                                         \
+    try {                                                                  \
+      PADDLE_THROW((EFUNC)("paddle throw test."));                         \
+    } catch (paddle::platform::EnforceNotMet & error) {                    \
+      caught_exception = true;                                             \
+      std::string ex_msg = error.what();                                   \
+      EXPECT_TRUE(ex_msg.find("paddle throw test.") != std::string::npos); \
+    }                                                                      \
+    EXPECT_TRUE(caught_exception);                                         \
+  } while (0)
+
+#define CHECK_PADDLE_ENFORCE(EFUNC)                                          \
+  do {                                                                       \
+    bool caught_exception = false;                                           \
+    try {                                                                    \
+      PADDLE_ENFORCE(false, (EFUNC)("paddle enforce test."));                \
+    } catch (paddle::platform::EnforceNotMet & error) {                      \
+      caught_exception = true;                                               \
+      std::string ex_msg = error.what();                                     \
+      EXPECT_TRUE(ex_msg.find("paddle enforce test.") != std::string::npos); \
+    }                                                                        \
+    EXPECT_TRUE(caught_exception);                                           \
+  } while (0)
+
+#define CHECK_PADDLE_ENFORCE_NOT_NULL(EFUNC)                             \
+  do {                                                                   \
+    bool caught_exception = false;                                       \
+    try {                                                                \
+      PADDLE_ENFORCE_NOT_NULL(nullptr,                                   \
+                              (EFUNC)("paddle enforce not null test.")); \
+    } catch (paddle::platform::EnforceNotMet & error) {                  \
+      caught_exception = true;                                           \
+      std::string ex_msg = error.what();                                 \
+      EXPECT_TRUE(ex_msg.find("paddle enforce not null test.") !=        \
+                  std::string::npos);                                    \
+    }                                                                    \
+    EXPECT_TRUE(caught_exception);                                       \
+  } while (0)
+
+#define CHECK_PADDLE_ENFORCE_EQ(EFUNC)                                \
+  do {                                                                \
+    bool caught_exception = false;                                    \
+    try {                                                             \
+      PADDLE_ENFORCE_EQ(1, 2, (EFUNC)("paddle enforce equal test.")); \
+    } catch (paddle::platform::EnforceNotMet & error) {               \
+      caught_exception = true;                                        \
+      std::string ex_msg = error.what();                              \
+      EXPECT_TRUE(ex_msg.find("paddle enforce equal test.") !=        \
+                  std::string::npos);                                 \
+    }                                                                 \
+    EXPECT_TRUE(caught_exception);                                    \
+  } while (0)
+
+#define CHECK_ALL_PADDLE_EXCEPTION_MACRO(EFUNC) \
+  do {                                          \
+    CHECK_PADDLE_THROW(EFUNC);                  \
+    CHECK_PADDLE_ENFORCE(EFUNC);                \
+    CHECK_PADDLE_ENFORCE_NOT_NULL(EFUNC);       \
+    CHECK_PADDLE_ENFORCE_EQ(EFUNC);             \
+  } while (0)
+
+TEST(Errors, InvalidArgument) {
+  CHECK_ALL_PADDLE_EXCEPTION_MACRO(InvalidArgument);
+}
+
+TEST(Errors, NotFound) { CHECK_ALL_PADDLE_EXCEPTION_MACRO(NotFound); }
+
+TEST(Errors, OutOfRange) { CHECK_ALL_PADDLE_EXCEPTION_MACRO(OutOfRange); }
+
+TEST(Errors, AlreadyExists) { CHECK_ALL_PADDLE_EXCEPTION_MACRO(AlreadyExists); }
+
+TEST(Errors, ResourceExhausted) {
+  CHECK_ALL_PADDLE_EXCEPTION_MACRO(ResourceExhausted);
+}
+
+TEST(Errors, PreconditionNotMet) {
+  CHECK_ALL_PADDLE_EXCEPTION_MACRO(PreconditionNotMet);
+}
+
+TEST(Errors, PermissionDenied) {
+  CHECK_ALL_PADDLE_EXCEPTION_MACRO(PermissionDenied);
+}
+
+TEST(Errors, ExecutionTimeout) {
+  CHECK_ALL_PADDLE_EXCEPTION_MACRO(ExecutionTimeout);
+}
+
+TEST(Errors, Unimplemented) { CHECK_ALL_PADDLE_EXCEPTION_MACRO(Unimplemented); }
+
+TEST(Errors, Unavailable) { CHECK_ALL_PADDLE_EXCEPTION_MACRO(Unavailable); }
+
+TEST(Errors, Fatal) { CHECK_ALL_PADDLE_EXCEPTION_MACRO(Fatal); }
+
+TEST(Errors, External) { CHECK_ALL_PADDLE_EXCEPTION_MACRO(External); }

test/cpp/fluid/platform/float16_test.cc

@@ -0,0 +1,170 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/common/float16.h"
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/framework/lod_tensor.h"
+#include "paddle/fluid/platform/enforce.h"
+#include "paddle/phi/kernels/funcs/eigen/extensions.h"
+
+namespace paddle {
+namespace platform {
+using float16 = phi::dtype::float16;
+using namespace phi::dtype;  // NOLINT
+
+TEST(float16, conversion_cpu) {
+  // Conversion from float
+  EXPECT_EQ(float16(1.0f).x, 0x3c00);
+  EXPECT_EQ(float16(0.5f).x, 0x3800);
+  EXPECT_EQ(float16(0.33333f).x, 0x3555);
+  EXPECT_EQ(float16(0.0f).x, 0x0000);
+  EXPECT_EQ(float16(-0.0f).x, 0x8000);
+  EXPECT_EQ(float16(65504.0f).x, 0x7bff);
+  EXPECT_EQ(float16(65536.0f).x, 0x7c00);
+
+  // Conversion from double
+  EXPECT_EQ(float16(1.0).x, 0x3c00);
+  EXPECT_EQ(float16(0.5).x, 0x3800);
+  EXPECT_EQ(float16(0.33333).x, 0x3555);
+  EXPECT_EQ(float16(0.0).x, 0x0000);
+  EXPECT_EQ(float16(-0.0).x, 0x8000);
+  EXPECT_EQ(float16(65504.0).x, 0x7bff);
+  EXPECT_EQ(float16(65536.0).x, 0x7c00);
+
+  // Conversion from int
+  EXPECT_EQ(float16(-1).x, 0xbc00);
+  EXPECT_EQ(float16(0).x, 0x0000);
+  EXPECT_EQ(float16(1).x, 0x3c00);
+  EXPECT_EQ(float16(2).x, 0x4000);
+  EXPECT_EQ(float16(3).x, 0x4200);
+
+  // Conversion from bool
+  EXPECT_EQ(float16(true).x, 0x3c00);
+  EXPECT_EQ(float16(false).x, 0x0000);
+
+  // Assignment operator
+  float16 v_assign;
+  v_assign = float16(0);
+  EXPECT_EQ(v_assign.x, 0x0000);
+  v_assign = 0.5f;
+  EXPECT_EQ(v_assign.x, 0x3800);
+  v_assign = 0.33333;
+  EXPECT_EQ(v_assign.x, 0x3555);
+  v_assign = -1;
+  EXPECT_EQ(v_assign.x, 0xbc00);
+  v_assign = true;
+  EXPECT_EQ(v_assign.x, 0x3c00);
+
+  // Conversion operator
+  EXPECT_EQ(static_cast<float>(float16(0.5f)), 0.5f);
+  EXPECT_NEAR(static_cast<double>(float16(0.33333)), 0.33333, 0.0001);
+  EXPECT_EQ(static_cast<int>(float16(-1)), -1);
+  EXPECT_EQ(static_cast<bool>(float16(true)), true);
+}
+
+TEST(float16, arithmetic_cpu) {
+  EXPECT_EQ(static_cast<float>(float16(1) + float16(1)), 2);
+  EXPECT_EQ(static_cast<float>(float16(5) + float16(-5)), 0);
+  EXPECT_NEAR(
+      static_cast<float>(float16(0.33333f) + float16(0.66667f)), 1.0f, 0.001);
+  EXPECT_EQ(static_cast<float>(float16(3) - float16(5)), -2);
+  EXPECT_NEAR(static_cast<float>(float16(0.66667f) - float16(0.33333f)),
+              0.33334f,
+              0.001);
+  EXPECT_NEAR(static_cast<float>(float16(3.3f) * float16(2.0f)), 6.6f, 0.01);
+  EXPECT_NEAR(static_cast<float>(float16(-2.1f) * float16(-3.0f)), 6.3f, 0.01);
+  EXPECT_NEAR(
+      static_cast<float>(float16(2.0f) / float16(3.0f)), 0.66667f, 0.001);
+  EXPECT_EQ(static_cast<float>(float16(1.0f) / float16(2.0f)), 0.5f);
+  EXPECT_EQ(static_cast<float>(-float16(512.0f)), -512.0f);
+  EXPECT_EQ(static_cast<float>(-float16(-512.0f)), 512.0f);
+}
+
+TEST(float16, comparison_cpu) {
+  EXPECT_TRUE(float16(1.0f) == float16(1.0f));
+  EXPECT_FALSE(float16(-1.0f) == float16(-0.5f));
+  EXPECT_TRUE(float16(1.0f) != float16(0.5f));
+  EXPECT_FALSE(float16(-1.0f) != float16(-1.0f));
+  EXPECT_TRUE(float16(1.0f) < float16(2.0f));
+  EXPECT_FALSE(float16(-1.0f) < float16(-1.0f));
+  EXPECT_TRUE(float16(1.0f) <= float16(1.0f));
+  EXPECT_TRUE(float16(2.0f) > float16(1.0f));
+  EXPECT_FALSE(float16(-2.0f) > float16(-2.0f));
+  EXPECT_TRUE(float16(2.0f) >= float16(2.0f));
+
+  EXPECT_TRUE(float16(0.0f) == float16(-0.0f));
+  EXPECT_TRUE(float16(0.0f) <= float16(-0.0f));
+  EXPECT_TRUE(float16(0.0f) >= float16(-0.0f));
+  EXPECT_FALSE(float16(0.0f) < float16(-0.0f));
+  EXPECT_FALSE(float16(-0.0f) < float16(0.0f));
+  EXPECT_FALSE(float16(0.0f) > float16(-0.0f));
+  EXPECT_FALSE(float16(-0.0f) > float16(0.0f));
+}
+
+TEST(float16, lod_tensor_cpu) {
+  phi::DenseTensor lod_tensor;
+
+  std::vector<float16> input_data = {
+      float16(1.0f), float16(0.5f), float16(0.33333f), float16(0.0f)};
+  EXPECT_EQ(input_data[0].x, 0x3c00);
+  EXPECT_EQ(input_data[1].x, 0x3800);
+  EXPECT_EQ(input_data[2].x, 0x3555);
+  EXPECT_EQ(input_data[3].x, 0x0000);
+
+  lod_tensor.Resize({4, 1});
+  lod_tensor.set_lod(phi::LegacyLoD({{0, 2, 4}}));
+  float16* data_ptr = lod_tensor.mutable_data<float16>(CPUPlace());
+
+  EXPECT_NE(data_ptr, nullptr);
+  EXPECT_EQ(input_data.size(), static_cast<size_t>(lod_tensor.numel()));
+  for (size_t i = 0; i < input_data.size(); ++i) {
+    data_ptr[i] = input_data[i];
+    EXPECT_EQ(data_ptr[i].x, input_data[i].x);
+  }
+}
+
+TEST(float16, floating) {
+  // compile time assert.
+  PADDLE_ENFORCE_EQ(
+      std::is_floating_point<float16>::value,
+      true,
+      common::errors::Unavailable("The float16 support in CPU failed."));
+}
+
+TEST(float16, print) {
+  float16 a = float16(1.0f);
+  std::cout << a << std::endl;
+}
+
+// CPU test
+TEST(float16, isinf) {
+  float16 a;
+  a.x = 0x7c00;
+  float16 b = float16(INFINITY);
+  float16 c = static_cast<float16>(INFINITY);
+  EXPECT_EQ(std::isinf(a), true);
+  EXPECT_EQ(std::isinf(b), true);
+  EXPECT_EQ(std::isinf(c), true);
+}
+
+TEST(float16, isnan) {
+  float16 a;
+  a.x = 0x7fff;
+  float16 b = float16(NAN);
+  float16 c = static_cast<float16>(NAN);
+  EXPECT_EQ(std::isnan(a), true);
+  EXPECT_EQ(std::isnan(b), true);
+  EXPECT_EQ(std::isnan(c), true);
+}
+
+}  // namespace platform
+}  // namespace paddle

test/cpp/fluid/platform/float16_test.cu

@@ -0,0 +1,426 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/common/float16.h"
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include <bitset>
+#include <iostream>
+
+#include "paddle/fluid/framework/lod_tensor.h"
+#include "paddle/fluid/framework/tensor_util.h"
+#include "paddle/fluid/platform/enforce.h"
+#include "paddle/phi/kernels/funcs/eigen/extensions.h"
+
+#define ARITHMETIC_KERNEL(op_type, sign)                                 \
+  __global__ void op_type(const half *in1, const half *in2, half *out) { \
+    out[0] = in1[0] sign in2[0];                                         \
+  }
+
+#define COMPOUND_KERNEL(op_type, sign) \
+  __global__ void op_type(half *in1, const half *in2) { in1[0] sign in2[0]; }
+
+#define COMPARISON_KERNEL(op_type, sign)                                 \
+  __global__ void op_type(const half *in1, const half *in2, bool *out) { \
+    out[0] = in1[0] sign in2[0];                                         \
+  }
+
+#ifdef PADDLE_WITH_HIP
+#define ARITHMETIC_KERNEL_LAUNCH(op_type)                                     \
+  void Test##op_type(float v_in1, float v_in2, float v_out) {                 \
+    LOG(INFO) << "Test " << #op_type << " on GPU!";                           \
+    half *in1, *in2, *out;                                                    \
+    half *d_in1, *d_in2, *d_out;                                              \
+    int size = sizeof(half);                                                  \
+    hipMalloc(reinterpret_cast<void **>(&d_in1), size);                       \
+    hipMalloc(reinterpret_cast<void **>(&d_in2), size);                       \
+    hipMalloc(reinterpret_cast<void **>(&d_out), size);                       \
+    in1 = reinterpret_cast<half *>(malloc(size));                             \
+    in2 = reinterpret_cast<half *>(malloc(size));                             \
+    out = reinterpret_cast<half *>(malloc(size));                             \
+    in1[0] = float16(v_in1).to_half();                                        \
+    in2[0] = float16(v_in2).to_half();                                        \
+    hipMemcpy(d_in1, in1, size, hipMemcpyHostToDevice);                       \
+    hipMemcpy(d_in2, in2, size, hipMemcpyHostToDevice);                       \
+    hipLaunchKernelGGL(op_type, dim3(1), dim3(1), 0, 0, d_in1, d_in2, d_out); \
+    hipMemcpy(out, d_out, size, hipMemcpyDeviceToHost);                       \
+    EXPECT_EQ(static_cast<float>(float16(out[0])), v_out);                    \
+    free(in1);                                                                \
+    free(in2);                                                                \
+    free(out);                                                                \
+    hipFree(d_in1);                                                           \
+    hipFree(d_in2);                                                           \
+    hipFree(d_out);                                                           \
+  }
+
+#define COMPOUND_KERNEL_LAUNCH(op_type)                                \
+  void Test##op_type(float v_in1, float v_in2, float v_out) {          \
+    LOG(INFO) << "Test " << #op_type << " on GPU!";                    \
+    half *in1, *in2;                                                   \
+    half *d_in1, *d_in2;                                               \
+    int size = sizeof(half);                                           \
+    hipMalloc(reinterpret_cast<void **>(&d_in1), size);                \
+    hipMalloc(reinterpret_cast<void **>(&d_in2), size);                \
+    in1 = reinterpret_cast<half *>(malloc(size));                      \
+    in2 = reinterpret_cast<half *>(malloc(size));                      \
+    in1[0] = float16(v_in1).to_half();                                 \
+    in2[0] = float16(v_in2).to_half();                                 \
+    hipMemcpy(d_in1, in1, size, hipMemcpyHostToDevice);                \
+    hipMemcpy(d_in2, in2, size, hipMemcpyHostToDevice);                \
+    hipLaunchKernelGGL(op_type, dim3(1), dim3(1), 0, 0, d_in1, d_in2); \
+    hipMemcpy(in1, d_in1, size, hipMemcpyDeviceToHost);                \
+    EXPECT_EQ(static_cast<float>(float16(in1[0])), v_out);             \
+    free(in1);                                                         \
+    free(in2);                                                         \
+    hipFree(d_in1);                                                    \
+    hipFree(d_in2);                                                    \
+  }
+
+#define COMPARISON_KERNEL_LAUNCH(op_type)                                     \
+  void Test##op_type(float v_in1, float v_in2, bool v_out) {                  \
+    LOG(INFO) << "Test " << #op_type << " on GPU!";                           \
+    half *in1, *in2;                                                          \
+    half *d_in1, *d_in2;                                                      \
+    bool *out, *d_out;                                                        \
+    int size = sizeof(half);                                                  \
+    hipMalloc(reinterpret_cast<void **>(&d_in1), size);                       \
+    hipMalloc(reinterpret_cast<void **>(&d_in2), size);                       \
+    hipMalloc(reinterpret_cast<void **>(&d_out), 1);                          \
+    in1 = reinterpret_cast<half *>(malloc(size));                             \
+    in2 = reinterpret_cast<half *>(malloc(size));                             \
+    out = reinterpret_cast<bool *>(malloc(1));                                \
+    in1[0] = float16(v_in1).to_half();                                        \
+    in2[0] = float16(v_in2).to_half();                                        \
+    hipMemcpy(d_in1, in1, size, hipMemcpyHostToDevice);                       \
+    hipMemcpy(d_in2, in2, size, hipMemcpyHostToDevice);                       \
+    hipLaunchKernelGGL(op_type, dim3(1), dim3(1), 0, 0, d_in1, d_in2, d_out); \
+    hipMemcpy(out, d_out, 1, hipMemcpyDeviceToHost);                          \
+    EXPECT_EQ(out[0], v_out);                                                 \
+    free(in1);                                                                \
+    free(in2);                                                                \
+    free(out);                                                                \
+    hipFree(d_in1);                                                           \
+    hipFree(d_in2);                                                           \
+    hipFree(d_out);                                                           \
+  }
+#else
+#define ARITHMETIC_KERNEL_LAUNCH(op_type)                     \
+  void Test##op_type(float v_in1, float v_in2, float v_out) { \
+    LOG(INFO) << "Test " << #op_type << " on GPU!";           \
+    half *in1, *in2, *out;                                    \
+    half *d_in1, *d_in2, *d_out;                              \
+    int size = sizeof(half);                                  \
+    cudaMalloc(reinterpret_cast<void **>(&d_in1), size);      \
+    cudaMalloc(reinterpret_cast<void **>(&d_in2), size);      \
+    cudaMalloc(reinterpret_cast<void **>(&d_out), size);      \
+    in1 = reinterpret_cast<half *>(malloc(size));             \
+    in2 = reinterpret_cast<half *>(malloc(size));             \
+    out = reinterpret_cast<half *>(malloc(size));             \
+    in1[0] = float16(v_in1).to_half();                        \
+    in2[0] = float16(v_in2).to_half();                        \
+    cudaMemcpy(d_in1, in1, size, cudaMemcpyHostToDevice);     \
+    cudaMemcpy(d_in2, in2, size, cudaMemcpyHostToDevice);     \
+    op_type<<<1, 1>>>(d_in1, d_in2, d_out);                   \
+    cudaMemcpy(out, d_out, size, cudaMemcpyDeviceToHost);     \
+    EXPECT_EQ(static_cast<float>(float16(out[0])), v_out);    \
+    free(in1);                                                \
+    free(in2);                                                \
+    free(out);                                                \
+    cudaFree(d_in1);                                          \
+    cudaFree(d_in2);                                          \
+    cudaFree(d_out);                                          \
+  }
+
+#define COMPOUND_KERNEL_LAUNCH(op_type)                       \
+  void Test##op_type(float v_in1, float v_in2, float v_out) { \
+    LOG(INFO) << "Test " << #op_type << " on GPU!";           \
+    half *in1, *in2;                                          \
+    half *d_in1, *d_in2;                                      \
+    int size = sizeof(half);                                  \
+    cudaMalloc(reinterpret_cast<void **>(&d_in1), size);      \
+    cudaMalloc(reinterpret_cast<void **>(&d_in2), size);      \
+    in1 = reinterpret_cast<half *>(malloc(size));             \
+    in2 = reinterpret_cast<half *>(malloc(size));             \
+    in1[0] = float16(v_in1).to_half();                        \
+    in2[0] = float16(v_in2).to_half();                        \
+    cudaMemcpy(d_in1, in1, size, cudaMemcpyHostToDevice);     \
+    cudaMemcpy(d_in2, in2, size, cudaMemcpyHostToDevice);     \
+    op_type<<<1, 1>>>(d_in1, d_in2);                          \
+    cudaMemcpy(in1, d_in1, size, cudaMemcpyDeviceToHost);     \
+    EXPECT_EQ(static_cast<float>(float16(in1[0])), v_out);    \
+    free(in1);                                                \
+    free(in2);                                                \
+    cudaFree(d_in1);                                          \
+    cudaFree(d_in2);                                          \
+  }
+
+#define COMPARISON_KERNEL_LAUNCH(op_type)                    \
+  void Test##op_type(float v_in1, float v_in2, bool v_out) { \
+    LOG(INFO) << "Test " << #op_type << " on GPU!";          \
+    half *in1, *in2;                                         \
+    half *d_in1, *d_in2;                                     \
+    bool *out, *d_out;                                       \
+    int size = sizeof(half);                                 \
+    cudaMalloc(reinterpret_cast<void **>(&d_in1), size);     \
+    cudaMalloc(reinterpret_cast<void **>(&d_in2), size);     \
+    cudaMalloc(reinterpret_cast<void **>(&d_out), 1);        \
+    in1 = reinterpret_cast<half *>(malloc(size));            \
+    in2 = reinterpret_cast<half *>(malloc(size));            \
+    out = reinterpret_cast<bool *>(malloc(1));               \
+    in1[0] = float16(v_in1).to_half();                       \
+    in2[0] = float16(v_in2).to_half();                       \
+    cudaMemcpy(d_in1, in1, size, cudaMemcpyHostToDevice);    \
+    cudaMemcpy(d_in2, in2, size, cudaMemcpyHostToDevice);    \
+    op_type<<<1, 1>>>(d_in1, d_in2, d_out);                  \
+    cudaMemcpy(out, d_out, 1, cudaMemcpyDeviceToHost);       \
+    EXPECT_EQ(out[0], v_out);                                \
+    free(in1);                                               \
+    free(in2);                                               \
+    free(out);                                               \
+    cudaFree(d_in1);                                         \
+    cudaFree(d_in2);                                         \
+    cudaFree(d_out);                                         \
+  }
+#endif
+
+namespace paddle {
+namespace platform {
+
+using float16 = phi::dtype::float16;
+using namespace phi::dtype;  // NOLINT
+
+#if defined(PADDLE_WITH_HIP)
+ARITHMETIC_KERNEL(Add, +)
+ARITHMETIC_KERNEL(Sub, -)
+ARITHMETIC_KERNEL(Mul, *)
+ARITHMETIC_KERNEL(Div, /)
+
+ARITHMETIC_KERNEL_LAUNCH(Add)
+ARITHMETIC_KERNEL_LAUNCH(Sub)
+ARITHMETIC_KERNEL_LAUNCH(Mul)
+ARITHMETIC_KERNEL_LAUNCH(Div)
+
+// Negative sign kernel
+__global__ void Neg(half *in) { in[0] = -in[0]; }
+
+void TestNeg(float v_in, float v_out) {
+  LOG(INFO) << "Test Neg on GPU!";
+  half *in, *d_in;
+  int size = sizeof(half);
+#ifdef PADDLE_WITH_HIP
+  hipMalloc(reinterpret_cast<void **>(&d_in), size);
+#else
+  cudaMalloc(reinterpret_cast<void **>(&d_in), size);
+#endif
+  in = reinterpret_cast<half *>(malloc(size));
+  in[0] = float16(v_in).to_half();
+#ifdef PADDLE_WITH_HIP
+  hipMemcpy(d_in, in, size, hipMemcpyHostToDevice);
+#else
+  cudaMemcpy(d_in, in, size, cudaMemcpyHostToDevice);
+#endif
+  Neg<<<1, 1>>>(d_in);
+#ifdef PADDLE_WITH_HIP
+  hipMemcpy(in, d_in, size, hipMemcpyDeviceToHost);
+#else
+  cudaMemcpy(in, d_in, size, cudaMemcpyDeviceToHost);
+#endif
+  EXPECT_EQ(static_cast<float>(float16(in[0])), v_out);
+  free(in);
+#ifdef PADDLE_WITH_HIP
+  hipFree(d_in);
+#else
+  cudaFree(d_in);
+#endif
+}
+
+COMPOUND_KERNEL(AddAssign, +=)
+COMPOUND_KERNEL(SubAssign, -=)
+COMPOUND_KERNEL(MulAssign, *=)
+COMPOUND_KERNEL(DivAssign, /=)
+
+COMPOUND_KERNEL_LAUNCH(AddAssign)
+COMPOUND_KERNEL_LAUNCH(SubAssign)
+COMPOUND_KERNEL_LAUNCH(MulAssign)
+COMPOUND_KERNEL_LAUNCH(DivAssign)
+
+COMPARISON_KERNEL(Equal, ==)
+COMPARISON_KERNEL(NotEqual, !=)
+COMPARISON_KERNEL(Less, <)
+COMPARISON_KERNEL(LessEqual, <=)
+COMPARISON_KERNEL(Greater, >)
+COMPARISON_KERNEL(GreaterEqual, >=)
+
+COMPARISON_KERNEL_LAUNCH(Equal)
+COMPARISON_KERNEL_LAUNCH(NotEqual)
+COMPARISON_KERNEL_LAUNCH(Less)
+COMPARISON_KERNEL_LAUNCH(LessEqual)
+COMPARISON_KERNEL_LAUNCH(Greater)
+COMPARISON_KERNEL_LAUNCH(GreaterEqual)
+
+TEST(float16, arithmetic_on_gpu) {
+  TestAdd(1, 2, 3);
+  TestSub(2, 1, 1);
+  TestMul(2, 3, 6);
+  TestDiv(6, 2, 3);
+  TestNeg(1, -1);
+}
+
+TEST(float16, compound_on_gpu) {
+  TestAddAssign(1, 2, 3);
+  TestSubAssign(2, 1, 1);
+  TestMulAssign(2, 3, 6);
+  TestDivAssign(6, 2, 3);
+}
+
+TEST(float16, comparison_on_gpu) {
+  TestEqual(1, 1, true);
+  TestEqual(1, 2, false);
+  TestNotEqual(2, 3, true);
+  TestNotEqual(2, 2, false);
+  TestLess(3, 4, true);
+  TestLess(3, 3, false);
+  TestLessEqual(3, 3, true);
+  TestLessEqual(3, 2, false);
+  TestGreater(4, 3, true);
+  TestGreater(4, 4, false);
+  TestGreaterEqual(4, 4, true);
+  TestGreaterEqual(4, 5, false);
+}
+#endif  // CUDA_VERSION
+
+TEST(float16, conversion_on_gpu) {
+  // Explicit conversion to and from cuda half
+  EXPECT_EQ(float16(float16(1.0f).to_half()).x, 0x3c00);
+  EXPECT_EQ(float16(float16(0.5f).to_half()).x, 0x3800);
+  EXPECT_EQ(float16(float16(0.33333f).to_half()).x, 0x3555);
+  EXPECT_EQ(float16(float16(0.0f).to_half()).x, 0x0000);
+  EXPECT_EQ(float16(float16(-0.0f).to_half()).x, 0x8000);
+  EXPECT_EQ(float16(float16(65504.0f).to_half()).x, 0x7bff);
+  EXPECT_EQ(float16(float16(65536.0f).to_half()).x, 0x7c00);
+
+  // Assignment operator
+  float16 v_assign;
+  v_assign = float16(1.0f).to_half();
+  EXPECT_EQ(v_assign.x, 0x3c00);
+}
+
+TEST(float16, dense_tensor_on_gpu) {
+  phi::DenseTensor src_tensor;
+  phi::DenseTensor gpu_tensor;
+  phi::DenseTensor dst_tensor;
+
+  float16 *src_ptr =
+      src_tensor.mutable_data<float16>(common::make_ddim({2, 2}), CPUPlace());
+
+  float16 arr[4] = {
+      float16(1.0f), float16(0.5f), float16(0.33333f), float16(0.0f)};
+  memcpy(src_ptr, arr, 4 * sizeof(float16));
+
+  // CPU DenseTensor to GPU DenseTensor
+  phi::GPUPlace gpu_place(0);
+  phi::GPUContext gpu_ctx(gpu_place);
+  gpu_ctx.SetAllocator(paddle::memory::allocation::AllocatorFacade::Instance()
+                           .GetAllocator(gpu_place, gpu_ctx.stream())
+                           .get());
+  gpu_ctx.PartialInitWithAllocator();
+  framework::TensorCopy(src_tensor, gpu_place, gpu_ctx, &gpu_tensor);
+
+  // GPU DenseTensor to CPU DenseTensor
+  framework::TensorCopy(gpu_tensor, CPUPlace(), gpu_ctx, &dst_tensor);
+
+  // Sync before comparing DenseTensors
+  gpu_ctx.Wait();
+  const float16 *dst_ptr = dst_tensor.data<float16>();
+  ASSERT_NE(src_ptr, dst_ptr);
+  for (size_t i = 0; i < 4; ++i) {
+    EXPECT_EQ(src_ptr[i].x, dst_ptr[i].x);
+  }
+}
+
+template <typename T>
+struct Functor {
+  bool operator()(const T &val) {
+    return std::type_index(typeid(T)) ==
+           std::type_index(typeid(phi::dtype::float16));
+  }
+};
+
+TEST(float16, typeid) {
+  // the framework heavily used typeid hash
+  Functor<float16> functor;
+  float16 a = float16(.0f);
+  Functor<int> functor2;
+  int b(0);
+
+  // compile time assert
+  PADDLE_ENFORCE_EQ(
+      functor(a),
+      true,
+      common::errors::Unavailable("The float16 support in GPU failed."));
+  PADDLE_ENFORCE_EQ(
+      functor2(b),
+      false,
+      common::errors::Unavailable("The float16 support in GPU failed."));
+}
+
+// GPU test
+TEST(float16, isinf) {
+  float16 a;
+  a.x = 0x7c00;
+  float16 b = float16(INFINITY);
+  // underflow to 0
+  float16 native_a(5e-40f);
+  EXPECT_EQ(std::isinf(a), true);
+  EXPECT_EQ(std::isinf(b), true);
+#ifndef _WIN32
+  // overflow to inf
+  float16 native_b(5e40f);
+  EXPECT_EQ(std::isinf(native_b), true);
+#endif
+  EXPECT_EQ(native_a, float16(0));
+}
+
+TEST(float16, isnan) {
+  float16 a;
+  a.x = 0x7fff;
+  float16 b = float16(NAN);
+  float16 c = float16(5e40);
+  // inf * +-0 will get a nan
+  float16 d = c * float16(0);
+  EXPECT_EQ(std::isnan(a), true);
+  EXPECT_EQ(std::isnan(b), true);
+  EXPECT_EQ(std::isnan(d), true);
+}
+
+TEST(float16, cast) {
+  float16 a;
+  a.x = 0x0070;
+  auto b = a;
+  {
+    // change semantic, keep the same value
+    float16 c = reinterpret_cast<float16 &>(reinterpret_cast<unsigned &>(b));
+    EXPECT_EQ(b, c);
+  }
+
+  {
+    // use uint32 low 16 bit store float16
+    uint32_t c = reinterpret_cast<uint32_t &>(b);
+    float16 d;
+    d.x = c;
+    EXPECT_EQ(b, d);
+  }
+}
+
+}  // namespace platform
+}  // namespace paddle

test/cpp/fluid/platform/init_phi_test.cc

@@ -0,0 +1,29 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+#include <gtest/gtest.h>
+
+#include "paddle/extension.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/common/memory_utils.h"
+
+TEST(InitPhi, InitPhi) {
+  phi::MemoryUtils::Instance().CheckMemoryMethod();
+  phi::MemoryUtils::Instance().InitDevices();
+  ASSERT_EQ(phi::DeviceContextPool::IsInitialized(), true);
+}
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

test/cpp/fluid/platform/init_test.cc

@@ -0,0 +1,60 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+#include "paddle/fluid/platform/init.h"
+
+#include "gtest/gtest.h"
+#include "paddle/phi/core/platform/device_context.h"
+
+TEST(InitDevices, CPU) {
+  using paddle::framework::InitDevices;
+  using phi::DeviceContextPool;
+
+#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_XPU) && \
+    !defined(PADDLE_WITH_HIP)
+  InitDevices();
+  DeviceContextPool& pool = DeviceContextPool::Instance();
+  ASSERT_EQ(pool.Size(), 1U);
+#endif
+}
+
+TEST(InitDevices, CUDA) {
+  using paddle::framework::InitDevices;
+  using phi::DeviceContextPool;
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  int count = paddle::platform::GetGPUDeviceCount();
+  InitDevices();
+  DeviceContextPool& pool = DeviceContextPool::Instance();
+  ASSERT_EQ(pool.Size(), 2U + static_cast<unsigned>(count));
+#endif
+}
+
+TEST(InitDevices, XPU) {
+  using paddle::framework::InitDevices;
+  using phi::DeviceContextPool;
+
+#ifdef PADDLE_WITH_XPU
+  int count = paddle::platform::GetXPUDeviceCount();
+  InitDevices();
+  DeviceContextPool& pool = DeviceContextPool::Instance();
+  ASSERT_EQ(pool.Size(), 2U + static_cast<unsigned>(count));
+#endif
+}
+
+#ifndef _WIN32
+TEST(SignalHandle, SignalHandle) {
+  std::string msg = "Signal raises";
+  paddle::framework::SignalHandle(msg.c_str(), static_cast<int>(msg.size()));
+}
+#endif

test/cpp/fluid/platform/lodtensor_printer_test.cc

@@ -0,0 +1,23 @@
+//  Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "paddle/fluid/platform/densetensor_printer.h"
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/framework/scope.h"
+
+TEST(DenseTensorPrinter, PrintVar) {
+  paddle::framework::Scope scope;
+  std::stringstream ss;
+  paddle::platform::PrintVar(&scope, "NotAVar", "We don't have var", &ss);
+}

test/cpp/fluid/platform/os_info_test.cc

@@ -0,0 +1,41 @@
+//  Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "paddle/phi/core/os_info.h"
+
+#include <thread>
+
+#include "gtest/gtest.h"
+
+TEST(ThreadInfo, TestThreadIdUtils) {
+  using phi::GetAllThreadIds;
+  using phi::GetCurrentThreadId;
+  using phi::GetCurrentThreadStdId;
+  EXPECT_EQ(std::hash<std::thread::id>()(std::this_thread::get_id()),
+            GetCurrentThreadId().std_tid);
+  auto ids = GetAllThreadIds();
+  EXPECT_TRUE(ids.find(GetCurrentThreadStdId()) != ids.end());
+}
+
+TEST(ThreadInfo, TestThreadNameUtils) {
+  using phi::GetAllThreadNames;
+  using phi::GetCurrentThreadName;
+  using phi::GetCurrentThreadStdId;
+  using phi::SetCurrentThreadName;
+  SetCurrentThreadName("MainThread");
+  EXPECT_FALSE(SetCurrentThreadName("MainThread"));
+  auto names = GetAllThreadNames();
+  EXPECT_TRUE(names.find(GetCurrentThreadStdId()) != names.end());
+  EXPECT_EQ("MainThread", names[GetCurrentThreadStdId()]);
+  EXPECT_EQ("MainThread", GetCurrentThreadName());
+}

test/cpp/fluid/platform/place_test.cc

@@ -0,0 +1,57 @@
+//  Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "paddle/phi/common/place.h"
+
+#include "gtest/gtest.h"
+
+TEST(Place, Equality) {
+  phi::CPUPlace cpu;
+  phi::GPUPlace g0(0), g1(1), gg0(0);
+  phi::XPUPlace x0(0), x1(1), xx0(0);
+
+  EXPECT_EQ(cpu, cpu);
+  EXPECT_EQ(g0, g0);
+  EXPECT_EQ(g1, g1);
+  EXPECT_EQ(g0, gg0);
+  EXPECT_EQ(x0, x0);
+  EXPECT_EQ(x1, x1);
+  EXPECT_EQ(x0, xx0);
+
+  EXPECT_NE(g0, g1);
+  EXPECT_NE(x0, x1);
+
+  EXPECT_TRUE(phi::places_are_same_class(g0, gg0));
+  EXPECT_TRUE(phi::places_are_same_class(x0, xx0));
+  EXPECT_FALSE(phi::places_are_same_class(g0, cpu));
+  EXPECT_FALSE(phi::places_are_same_class(x0, cpu));
+  EXPECT_FALSE(phi::places_are_same_class(g0, x0));
+}
+
+TEST(Place, Print) {
+  {
+    std::stringstream ss;
+    ss << phi::XPUPlace(1);
+    EXPECT_EQ("Place(xpu:1)", ss.str());
+  }
+  {
+    std::stringstream ss;
+    ss << phi::GPUPlace(1);
+    EXPECT_EQ("Place(gpu:1)", ss.str());
+  }
+  {
+    std::stringstream ss;
+    ss << phi::CPUPlace();
+    EXPECT_EQ("Place(cpu)", ss.str());
+  }
+}

test/cpp/fluid/platform/profiler/CMakeLists.txt

@@ -0,0 +1,16 @@
+cc_test(
+  test_event_node
+  SRCS test_event_node.cc
+  DEPS event_node profiler_logger)
+cc_test(
+  test_extra_info
+  SRCS test_extra_info.cc
+  DEPS phi glog common)
+cc_test(
+  test_serialization_logger
+  SRCS dump/test_serialization_logger.cc
+  DEPS event_bind)
+cc_test(
+  new_profiler_test
+  SRCS profiler_test.cc
+  DEPS new_profiler)

test/cpp/fluid/platform/profiler/dump/test_serialization_logger.cc

@@ -0,0 +1,315 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/framework/type_defs.h"
+#include "paddle/fluid/platform/profiler/dump/deserialization_reader.h"
+#include "paddle/fluid/platform/profiler/dump/serialization_logger.h"
+#include "paddle/fluid/platform/profiler/event_node.h"
+#include "paddle/fluid/platform/profiler/event_python.h"
+
+using paddle::framework::AttributeMap;
+using paddle::platform::DeserializationReader;
+using paddle::platform::DeviceTraceEvent;
+using paddle::platform::HostTraceEvent;
+using paddle::platform::HostTraceEventNode;
+using paddle::platform::KernelEventInfo;
+using paddle::platform::MemcpyEventInfo;
+using paddle::platform::MemsetEventInfo;
+using paddle::platform::MemTraceEvent;
+using paddle::platform::NodeTrees;
+using paddle::platform::OperatorSupplementEvent;
+using paddle::platform::RuntimeTraceEvent;
+using paddle::platform::SerializationLogger;
+using phi::TracerEventType;
+using phi::TracerMemEventType;
+
+TEST(SerializationLoggerTest, dump_case0) {
+  std::list<HostTraceEvent> host_events;
+  std::list<RuntimeTraceEvent> runtime_events;
+  std::list<DeviceTraceEvent> device_events;
+  std::list<MemTraceEvent> mem_events;
+  std::list<OperatorSupplementEvent> op_supplement_events;
+  host_events.emplace_back(std::string("dataloader#1"),
+                           TracerEventType::Dataloader,
+                           1000,
+                           10000,
+                           10,
+                           10);
+  host_events.emplace_back(
+      std::string("op1"), TracerEventType::Operator, 11000, 20000, 10, 10);
+  host_events.emplace_back(
+      std::string("op2"), TracerEventType::Operator, 21000, 30000, 10, 10);
+  host_events.emplace_back(
+      std::string("op3"), TracerEventType::Operator, 31000, 40000, 10, 11);
+  mem_events.emplace_back(11500,
+                          0x1000,
+                          TracerMemEventType::Allocate,
+                          10,
+                          10,
+                          50,
+                          "GPU:0",
+                          50,
+                          50,
+                          100,
+                          100);
+  mem_events.emplace_back(11900,
+                          0x1000,
+                          TracerMemEventType::Free,
+                          10,
+                          10,
+                          -50,
+                          "GPU:0",
+                          0,
+                          50,
+                          100,
+                          100);
+  std::map<std::string, std::vector<std::vector<int64_t>>> input_shapes;
+  std::map<std::string, std::vector<std::string>> dtypes;
+  input_shapes[std::string("X")].push_back(std::vector<int64_t>{1, 2, 3});
+  input_shapes[std::string("X")].push_back(std::vector<int64_t>{4, 5, 6, 7});
+  dtypes[std::string("X")].emplace_back("int8");
+  dtypes[std::string("X")].emplace_back("float32");
+  AttributeMap attrs;
+  op_supplement_events.emplace_back(
+      11600, "op1", input_shapes, dtypes, "op1()", attrs, 0, 10, 10);
+  runtime_events.emplace_back(
+      std::string("cudalaunch1"), 15000, 17000, 10, 10, 1, 0);
+  runtime_events.emplace_back(
+      std::string("cudalaunch2"), 25000, 35000, 10, 10, 2, 0);
+  runtime_events.emplace_back(
+      std::string("cudalaunch3"), 33000, 37000, 10, 11, 3, 0);
+  runtime_events.emplace_back(
+      std::string("cudaMemcpy1"), 18000, 19000, 10, 10, 4, 0);
+  runtime_events.emplace_back(
+      std::string("cudaMemset1"), 38000, 39000, 10, 11, 5, 0);
+  device_events.emplace_back(std::string("kernel1"),
+                             TracerEventType::Kernel,
+                             40000,
+                             55000,
+                             0,
+                             10,
+                             10,
+                             1,
+                             KernelEventInfo());
+  device_events.emplace_back(std::string("kernel2"),
+                             TracerEventType::Kernel,
+                             70000,
+                             95000,
+                             0,
+                             10,
+                             10,
+                             2,
+                             KernelEventInfo());
+  device_events.emplace_back(std::string("kernel3"),
+                             TracerEventType::Kernel,
+                             60000,
+                             65000,
+                             0,
+                             10,
+                             11,
+                             3,
+                             KernelEventInfo());
+  device_events.emplace_back(std::string("memcpy1"),
+                             TracerEventType::Memcpy,
+                             56000,
+                             59000,
+                             0,
+                             10,
+                             10,
+                             4,
+                             MemcpyEventInfo());
+  device_events.emplace_back(std::string("memset1"),
+                             TracerEventType::Memset,
+                             66000,
+                             69000,
+                             0,
+                             10,
+                             11,
+                             5,
+                             MemsetEventInfo());
+  SerializationLogger logger("test_serialization_logger_case0.pb");
+  logger.LogMetaInfo(std::string("1.0.2"), 0);
+  NodeTrees tree(host_events,
+                 runtime_events,
+                 device_events,
+                 mem_events,
+                 op_supplement_events);
+  std::map<uint64_t, std::vector<HostTraceEventNode*>> nodes =
+      tree.Traverse(true);
+  EXPECT_EQ(nodes[10].size(), 4u);
+  EXPECT_EQ(nodes[11].size(), 2u);
+  std::vector<HostTraceEventNode*> thread1_nodes = nodes[10];
+  std::vector<HostTraceEventNode*> thread2_nodes = nodes[11];
+  for (auto& thread1_node : thread1_nodes) {
+    if (thread1_node->Name() == "root node") {
+      EXPECT_EQ(thread1_node->GetChildren().size(), 3u);
+    }
+    if (thread1_node->Name() == "op1") {
+      EXPECT_EQ(thread1_node->GetChildren().size(), 0u);
+      EXPECT_EQ(thread1_node->GetRuntimeTraceEventNodes().size(), 2u);
+      EXPECT_EQ(thread1_node->GetMemTraceEventNodes().size(), 2u);
+      EXPECT_NE(thread1_node->GetOperatorSupplementEventNode(), nullptr);
+    }
+  }
+  for (auto& thread2_node : thread2_nodes) {
+    if (thread2_node->Name() == "op3") {
+      EXPECT_EQ(thread2_node->GetChildren().size(), 0u);
+      EXPECT_EQ(thread2_node->GetRuntimeTraceEventNodes().size(), 2u);
+    }
+  }
+  tree.LogMe(&logger);
+  logger.LogExtraInfo(std::unordered_map<std::string, std::string>());
+}
+
+TEST(SerializationLoggerTest, dump_case1) {
+  std::list<HostTraceEvent> host_events;
+  std::list<RuntimeTraceEvent> runtime_events;
+  std::list<DeviceTraceEvent> device_events;
+  std::list<MemTraceEvent> mem_events;
+  std::list<OperatorSupplementEvent> op_supplement_events;
+  runtime_events.emplace_back(
+      std::string("cudalaunch1"), 15000, 17000, 10, 10, 1, 0);
+  runtime_events.emplace_back(
+      std::string("cudalaunch2"), 25000, 35000, 10, 10, 2, 0);
+  runtime_events.emplace_back(
+      std::string("cudalaunch3"), 33000, 37000, 10, 11, 3, 0);
+  runtime_events.emplace_back(
+      std::string("cudaMemcpy1"), 18000, 19000, 10, 10, 4, 0);
+  runtime_events.emplace_back(
+      std::string("cudaMemset1"), 38000, 39000, 10, 11, 5, 0);
+  device_events.emplace_back(std::string("kernel1"),
+                             TracerEventType::Kernel,
+                             40000,
+                             55000,
+                             0,
+                             10,
+                             10,
+                             1,
+                             KernelEventInfo());
+  device_events.emplace_back(std::string("kernel2"),
+                             TracerEventType::Kernel,
+                             70000,
+                             95000,
+                             0,
+                             10,
+                             10,
+                             2,
+                             KernelEventInfo());
+  device_events.emplace_back(std::string("kernel3"),
+                             TracerEventType::Kernel,
+                             60000,
+                             65000,
+                             0,
+                             10,
+                             11,
+                             3,
+                             KernelEventInfo());
+  device_events.emplace_back(std::string("memcpy1"),
+                             TracerEventType::Memcpy,
+                             56000,
+                             59000,
+                             0,
+                             10,
+                             10,
+                             4,
+                             MemcpyEventInfo());
+  device_events.emplace_back(std::string("memset1"),
+                             TracerEventType::Memset,
+                             66000,
+                             69000,
+                             0,
+                             10,
+                             11,
+                             5,
+                             MemsetEventInfo());
+  SerializationLogger logger("test_serialization_logger_case1.pb");
+  logger.LogMetaInfo(std::string("1.0.2"), 0);
+  NodeTrees tree(host_events,
+                 runtime_events,
+                 device_events,
+                 mem_events,
+                 op_supplement_events);
+  std::map<uint64_t, std::vector<HostTraceEventNode*>> nodes =
+      tree.Traverse(true);
+  EXPECT_EQ(nodes[10].size(), 1u);
+  EXPECT_EQ(nodes[11].size(), 1u);
+  std::vector<HostTraceEventNode*> thread1_nodes = nodes[10];
+  std::vector<HostTraceEventNode*> thread2_nodes = nodes[11];
+  for (auto& thread1_node : thread1_nodes) {
+    if (thread1_node->Name() == "root node") {
+      EXPECT_EQ(thread1_node->GetRuntimeTraceEventNodes().size(), 3u);
+    }
+  }
+  for (auto& thread2_node : thread2_nodes) {
+    if (thread2_node->Name() == "root node") {
+      EXPECT_EQ(thread2_node->GetChildren().size(), 0u);
+      EXPECT_EQ(thread2_node->GetRuntimeTraceEventNodes().size(), 2u);
+    }
+  }
+  tree.LogMe(&logger);
+  logger.LogExtraInfo(std::unordered_map<std::string, std::string>());
+}
+
+TEST(DeserializationReaderTest, restore_case0) {
+  DeserializationReader reader("test_serialization_logger_case0.pb");
+  auto profiler_result = reader.Parse();
+  auto tree = profiler_result->GetNodeTrees();
+  std::map<uint64_t, std::vector<HostTraceEventNode*>> nodes =
+      tree->Traverse(true);
+  EXPECT_EQ(nodes[10].size(), 4u);
+  EXPECT_EQ(nodes[11].size(), 2u);
+  std::vector<HostTraceEventNode*> thread1_nodes = nodes[10];
+  std::vector<HostTraceEventNode*> thread2_nodes = nodes[11];
+  for (auto& thread1_node : thread1_nodes) {
+    if (thread1_node->Name() == "root node") {
+      EXPECT_EQ(thread1_node->GetChildren().size(), 3u);
+    }
+    if (thread1_node->Name() == "op1") {
+      EXPECT_EQ(thread1_node->GetChildren().size(), 0u);
+      EXPECT_EQ(thread1_node->GetRuntimeTraceEventNodes().size(), 2u);
+      EXPECT_EQ(thread1_node->GetMemTraceEventNodes().size(), 2u);
+      EXPECT_NE(thread1_node->GetOperatorSupplementEventNode(), nullptr);
+    }
+  }
+  for (auto& thread2_node : thread2_nodes) {
+    if (thread2_node->Name() == "op3") {
+      EXPECT_EQ(thread2_node->GetChildren().size(), 0u);
+      EXPECT_EQ(thread2_node->GetRuntimeTraceEventNodes().size(), 2u);
+    }
+  }
+}
+
+TEST(DeserializationReaderTest, restore_case1) {
+  DeserializationReader reader("test_serialization_logger_case1.pb");
+  auto profiler_result = reader.Parse();
+  auto tree = profiler_result->GetNodeTrees();
+  std::map<uint64_t, std::vector<HostTraceEventNode*>> nodes =
+      tree->Traverse(true);
+  EXPECT_EQ(nodes[10].size(), 1u);
+  EXPECT_EQ(nodes[11].size(), 1u);
+  std::vector<HostTraceEventNode*> thread1_nodes = nodes[10];
+  std::vector<HostTraceEventNode*> thread2_nodes = nodes[11];
+  for (auto& thread1_node : thread1_nodes) {
+    if (thread1_node->Name() == "root node") {
+      EXPECT_EQ(thread1_node->GetRuntimeTraceEventNodes().size(), 3u);
+    }
+  }
+  for (auto& thread2_node : thread2_nodes) {
+    if (thread2_node->Name() == "root node") {
+      EXPECT_EQ(thread2_node->GetChildren().size(), 0u);
+      EXPECT_EQ(thread2_node->GetRuntimeTraceEventNodes().size(), 2u);
+    }
+  }
+}

test/cpp/fluid/platform/profiler/profiler_test.cc

@@ -0,0 +1,145 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <set>
+#include <string>
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#ifdef PADDLE_WITH_CUDA
+#include <cuda.h>
+#endif
+#ifdef PADDLE_WITH_HIP
+#include <hip/hip_runtime.h>
+#endif
+#include "paddle/fluid/platform/profiler/event_python.h"
+#include "paddle/fluid/platform/profiler/profiler.h"
+#include "paddle/phi/common/place.h"
+#include "paddle/phi/core/platform/profiler.h"
+#include "paddle/phi/core/platform/profiler/event_tracing.h"
+
+TEST(ProfilerTest, TestHostTracer) {
+  using paddle::platform::Profiler;
+  using paddle::platform::ProfilerOptions;
+  using paddle::platform::ProfilerResult;
+  using paddle::platform::RecordInstantEvent;
+  using phi::TracerEventType;
+  ProfilerOptions options;
+  options.trace_level = 2;
+  options.trace_switch = 3;
+  auto profiler = Profiler::Create(options);
+  EXPECT_TRUE(profiler);
+  profiler->Prepare();
+  profiler->Start();
+  {
+    RecordInstantEvent(
+        "TestTraceLevel_record1", TracerEventType::UserDefined, 2);
+    RecordInstantEvent(
+        "TestTraceLevel_record2", TracerEventType::UserDefined, 3);
+  }
+  auto profiler_result = profiler->Stop();
+  auto nodetree = profiler_result->GetNodeTrees();
+  std::set<std::string> host_events;
+  for (const auto& pair : nodetree->Traverse(true)) {
+    for (const auto evt : pair.second) {
+      host_events.insert(evt->Name());
+    }
+  }
+  EXPECT_EQ(host_events.count("TestTraceLevel_record1"), 1u);
+  EXPECT_EQ(host_events.count("TestTraceLevel_record2"), 0u);
+}
+
+TEST(ProfilerTest, TestCudaTracer) {
+  using paddle::platform::Profiler;
+  using paddle::platform::ProfilerOptions;
+  using paddle::platform::ProfilerResult;
+  ProfilerOptions options;
+  options.trace_level = 0;
+  options.trace_switch = 3;
+  auto profiler = Profiler::Create(options);
+  EXPECT_TRUE(profiler);
+  profiler->Prepare();
+  profiler->Start();
+#ifdef PADDLE_WITH_CUDA
+  cudaStream_t stream;
+  cudaStreamCreate(&stream);
+  cudaStreamSynchronize(stream);
+#endif
+#ifdef PADDLE_WITH_HIP
+  hipStream_t stream;
+  hipStreamCreate(&stream);
+  hipStreamSynchronize(stream);
+#endif
+  auto profiler_result = profiler->Stop();
+  auto nodetree = profiler_result->GetNodeTrees();
+  std::vector<std::string> runtime_events;
+  for (const auto& pair : nodetree->Traverse(true)) {
+    for (const auto host_node : pair.second) {
+      for (auto runtime_node : host_node->GetRuntimeTraceEventNodes()) {
+        runtime_events.push_back(runtime_node->Name());
+      }
+    }
+  }
+#ifdef PADDLE_WITH_CUPTI
+#ifndef PADDLE_WITH_XPU
+  EXPECT_GT(runtime_events.size(), 0u);
+#endif
+#endif
+}
+
+TEST(ProfilerTest, TestHostTracerForMem) {
+  using paddle::platform::EnableHostEventRecorder;
+  using paddle::platform::MemTraceEventNode;
+  using paddle::platform::Profiler;
+  using paddle::platform::ProfilerOptions;
+  using paddle::platform::ProfilerResult;
+  using paddle::platform::RecordInstantEvent;
+  using paddle::platform::RecordMemEvent;
+  using phi::CPUPlace;
+  using phi::RecordEvent;
+  using phi::TracerEventType;
+  using phi::TracerMemEventType;
+  ProfilerOptions options;
+  options.trace_level = 1;
+  options.trace_switch = 3;
+  auto profiler = Profiler::Create(options);
+  EXPECT_TRUE(profiler);
+  EnableHostEventRecorder();
+  profiler->Prepare();
+  profiler->Start();
+  {
+    RecordEvent event1(
+        "TestTracerForMem_phase1", TracerEventType::UserDefined, 1);
+    RecordMemEvent(reinterpret_cast<void*>(0),
+                   CPUPlace(),
+                   1024,
+                   TracerMemEventType::Allocate);
+    RecordMemEvent(
+        reinterpret_cast<void*>(0), CPUPlace(), 1024, TracerMemEventType::Free);
+  }
+  {
+    RecordEvent event2(
+        "TestTracerForMem_phase2", TracerEventType::UserDefined, 1);
+    RecordMemEvent(reinterpret_cast<void*>(1024),
+                   CPUPlace(),
+                   1024,
+                   TracerMemEventType::Allocate);
+    RecordMemEvent(reinterpret_cast<void*>(1024),
+                   CPUPlace(),
+                   1024,
+                   TracerMemEventType::Free);
+  }
+  auto profiler_result = profiler->Stop();
+  auto nodetree = profiler_result->GetNodeTrees();
+}

test/cpp/fluid/platform/profiler/test_event_node.cc

@@ -0,0 +1,390 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/framework/type_defs.h"
+#include "paddle/fluid/platform/profiler/chrometracing_logger.h"
+#include "paddle/fluid/platform/profiler/event_node.h"
+
+using paddle::framework::AttributeMap;
+using paddle::platform::ChromeTracingLogger;
+using paddle::platform::CudaRuntimeTraceEventNode;
+using paddle::platform::DeviceTraceEvent;
+using paddle::platform::DeviceTraceEventNode;
+using paddle::platform::HostTraceEvent;
+using paddle::platform::HostTraceEventNode;
+using paddle::platform::KernelEventInfo;
+using paddle::platform::MemcpyEventInfo;
+using paddle::platform::MemsetEventInfo;
+using paddle::platform::MemTraceEvent;
+using paddle::platform::MemTraceEventNode;
+using paddle::platform::NodeTrees;
+using paddle::platform::OperatorSupplementEvent;
+using paddle::platform::OperatorSupplementEventNode;
+using paddle::platform::RuntimeTraceEvent;
+using phi::TracerEventType;
+using phi::TracerMemEventType;
+
+TEST(NodeTreesTest, LogMe_case0) {
+  std::list<HostTraceEvent> host_events;
+  std::list<RuntimeTraceEvent> runtime_events;
+  std::list<DeviceTraceEvent> device_events;
+  std::list<MemTraceEvent> mem_events;
+  std::list<OperatorSupplementEvent> op_supplement_events;
+  host_events.emplace_back(std::string("dataloader#1"),
+                           TracerEventType::Dataloader,
+                           1000,
+                           10000,
+                           10,
+                           10);
+  host_events.emplace_back(
+      std::string("op1"), TracerEventType::Operator, 11000, 20000, 10, 10);
+  host_events.emplace_back(
+      std::string("op2"), TracerEventType::Operator, 21000, 30000, 10, 10);
+  host_events.emplace_back(
+      std::string("op3"), TracerEventType::Operator, 31000, 40000, 10, 11);
+  mem_events.emplace_back(11500,
+                          0x1000,
+                          TracerMemEventType::Allocate,
+                          10,
+                          10,
+                          50,
+                          "GPU:0",
+                          50,
+                          50,
+                          100,
+                          100);
+  mem_events.emplace_back(11900,
+                          0x1000,
+                          TracerMemEventType::Free,
+                          10,
+                          10,
+                          -50,
+                          "GPU:0",
+                          0,
+                          50,
+                          100,
+                          100);
+  std::map<std::string, std::vector<std::vector<int64_t>>> input_shapes;
+  std::map<std::string, std::vector<std::string>> dtypes;
+  input_shapes[std::string("X")].push_back(std::vector<int64_t>{1, 2, 3});
+  input_shapes[std::string("X")].push_back(std::vector<int64_t>{4, 5, 6, 7});
+  dtypes[std::string("X")].emplace_back("int8");
+  dtypes[std::string("X")].emplace_back("float32");
+  AttributeMap attrs;
+  op_supplement_events.emplace_back(
+      11600, "op1", input_shapes, dtypes, "op1()", attrs, 0, 10, 10);
+  runtime_events.emplace_back(
+      std::string("cudalaunch1"), 15000, 17000, 10, 10, 1, 0);
+  runtime_events.emplace_back(
+      std::string("cudalaunch2"), 25000, 35000, 10, 10, 2, 0);
+  runtime_events.emplace_back(
+      std::string("cudalaunch3"), 33000, 37000, 10, 11, 3, 0);
+  runtime_events.emplace_back(
+      std::string("cudaMemcpy1"), 18000, 19000, 10, 10, 4, 0);
+  runtime_events.emplace_back(
+      std::string("cudaMemset1"), 38000, 39000, 10, 11, 5, 0);
+  device_events.emplace_back(std::string("kernel1"),
+                             TracerEventType::Kernel,
+                             40000,
+                             55000,
+                             0,
+                             10,
+                             10,
+                             1,
+                             KernelEventInfo());
+  device_events.emplace_back(std::string("kernel2"),
+                             TracerEventType::Kernel,
+                             70000,
+                             95000,
+                             0,
+                             10,
+                             10,
+                             2,
+                             KernelEventInfo());
+  device_events.emplace_back(std::string("kernel3"),
+                             TracerEventType::Kernel,
+                             60000,
+                             65000,
+                             0,
+                             10,
+                             11,
+                             3,
+                             KernelEventInfo());
+  device_events.emplace_back(std::string("memcpy1"),
+                             TracerEventType::Memcpy,
+                             56000,
+                             59000,
+                             0,
+                             10,
+                             10,
+                             4,
+                             MemcpyEventInfo());
+  device_events.emplace_back(std::string("memset1"),
+                             TracerEventType::Memset,
+                             66000,
+                             69000,
+                             0,
+                             10,
+                             11,
+                             5,
+                             MemsetEventInfo());
+  ChromeTracingLogger logger("test_nodetrees_logme_case0.json");
+  logger.LogMetaInfo(std::string("1.0.2"), 0);
+  NodeTrees tree(host_events,
+                 runtime_events,
+                 device_events,
+                 mem_events,
+                 op_supplement_events);
+  std::map<uint64_t, std::vector<HostTraceEventNode*>> nodes =
+      tree.Traverse(true);
+  EXPECT_EQ(nodes[10].size(), 4u);
+  EXPECT_EQ(nodes[11].size(), 2u);
+  std::vector<HostTraceEventNode*> thread1_nodes = nodes[10];
+  std::vector<HostTraceEventNode*> thread2_nodes = nodes[11];
+  for (auto& thread1_node : thread1_nodes) {
+    if (thread1_node->Name() == "root node") {
+      EXPECT_EQ(thread1_node->GetChildren().size(), 3u);
+    }
+    if (thread1_node->Name() == "op1") {
+      EXPECT_EQ(thread1_node->GetChildren().size(), 0u);
+      EXPECT_EQ(thread1_node->GetRuntimeTraceEventNodes().size(), 2u);
+      EXPECT_EQ(thread1_node->GetMemTraceEventNodes().size(), 2u);
+      EXPECT_NE(thread1_node->GetOperatorSupplementEventNode(), nullptr);
+    }
+  }
+  for (auto& thread2_node : thread2_nodes) {
+    if (thread2_node->Name() == "op3") {
+      EXPECT_EQ(thread2_node->GetChildren().size(), 0u);
+      EXPECT_EQ(thread2_node->GetRuntimeTraceEventNodes().size(), 2u);
+    }
+  }
+  tree.LogMe(&logger);
+  logger.LogExtraInfo(std::unordered_map<std::string, std::string>());
+}
+
+TEST(NodeTreesTest, LogMe_case1) {
+  std::list<HostTraceEvent> host_events;
+  std::list<RuntimeTraceEvent> runtime_events;
+  std::list<DeviceTraceEvent> device_events;
+  std::list<MemTraceEvent> mem_events;
+  std::list<OperatorSupplementEvent> op_supplement_events;
+  runtime_events.emplace_back(
+      std::string("cudalaunch1"), 15000, 17000, 10, 10, 1, 0);
+  runtime_events.emplace_back(
+      std::string("cudalaunch2"), 25000, 35000, 10, 10, 2, 0);
+  runtime_events.emplace_back(
+      std::string("cudalaunch3"), 33000, 37000, 10, 11, 3, 0);
+  runtime_events.emplace_back(
+      std::string("cudaMemcpy1"), 18000, 19000, 10, 10, 4, 0);
+  runtime_events.emplace_back(
+      std::string("cudaMemset1"), 38000, 39000, 10, 11, 5, 0);
+  device_events.emplace_back(std::string("kernel1"),
+                             TracerEventType::Kernel,
+                             40000,
+                             55000,
+                             0,
+                             10,
+                             10,
+                             1,
+                             KernelEventInfo());
+  device_events.emplace_back(std::string("kernel2"),
+                             TracerEventType::Kernel,
+                             70000,
+                             95000,
+                             0,
+                             10,
+                             10,
+                             2,
+                             KernelEventInfo());
+  device_events.emplace_back(std::string("kernel3"),
+                             TracerEventType::Kernel,
+                             60000,
+                             65000,
+                             0,
+                             10,
+                             11,
+                             3,
+                             KernelEventInfo());
+  device_events.emplace_back(std::string("memcpy1"),
+                             TracerEventType::Memcpy,
+                             56000,
+                             59000,
+                             0,
+                             10,
+                             10,
+                             4,
+                             MemcpyEventInfo());
+  device_events.emplace_back(std::string("memset1"),
+                             TracerEventType::Memset,
+                             66000,
+                             69000,
+                             0,
+                             10,
+                             11,
+                             5,
+                             MemsetEventInfo());
+  ChromeTracingLogger logger("test_nodetrees_logme_case1.json");
+  logger.LogMetaInfo(std::string("1.0.2"), 0);
+  NodeTrees tree(host_events,
+                 runtime_events,
+                 device_events,
+                 mem_events,
+                 op_supplement_events);
+  std::map<uint64_t, std::vector<HostTraceEventNode*>> nodes =
+      tree.Traverse(true);
+  EXPECT_EQ(nodes[10].size(), 1u);
+  EXPECT_EQ(nodes[11].size(), 1u);
+  std::vector<HostTraceEventNode*> thread1_nodes = nodes[10];
+  std::vector<HostTraceEventNode*> thread2_nodes = nodes[11];
+  for (auto& thread1_node : thread1_nodes) {
+    if (thread1_node->Name() == "root node") {
+      EXPECT_EQ(thread1_node->GetRuntimeTraceEventNodes().size(), 3u);
+    }
+  }
+  for (auto& thread2_node : thread2_nodes) {
+    if (thread2_node->Name() == "root node") {
+      EXPECT_EQ(thread2_node->GetChildren().size(), 0u);
+      EXPECT_EQ(thread2_node->GetRuntimeTraceEventNodes().size(), 2u);
+    }
+  }
+  tree.LogMe(&logger);
+  logger.LogExtraInfo(std::unordered_map<std::string, std::string>());
+}
+
+TEST(NodeTreesTest, HandleTrees_case0) {
+  std::list<HostTraceEvent> host_events;
+  std::list<RuntimeTraceEvent> runtime_events;
+  std::list<DeviceTraceEvent> device_events;
+  std::list<MemTraceEvent> mem_events;
+  std::list<OperatorSupplementEvent> op_supplement_events;
+  host_events.emplace_back(
+      std::string("op1"), TracerEventType::Operator, 10000, 100000, 10, 10);
+  host_events.emplace_back(
+      std::string("op2"), TracerEventType::Operator, 30000, 70000, 10, 10);
+  host_events.emplace_back(
+      std::string("op3"), TracerEventType::Operator, 2000, 120000, 10, 11);
+  mem_events.emplace_back(11500,
+                          0x1000,
+                          TracerMemEventType::Allocate,
+                          10,
+                          10,
+                          50,
+                          "GPU:0",
+                          50,
+                          50,
+                          100,
+                          100);
+  mem_events.emplace_back(11900,
+                          0x1000,
+                          TracerMemEventType::Free,
+                          10,
+                          10,
+                          -50,
+                          "GPU:0",
+                          0,
+                          50,
+                          100,
+                          100);
+  AttributeMap attrs;
+  op_supplement_events.emplace_back(
+      11600,
+      "op1",
+      std::map<std::string, std::vector<std::vector<int64_t>>>(),
+      std::map<std::string, std::vector<std::string>>(),
+      "op1()",
+      attrs,
+      0,
+      10,
+      10);
+  runtime_events.emplace_back(
+      std::string("cudalaunch1"), 15000, 25000, 10, 10, 1, 0);
+  runtime_events.emplace_back(
+      std::string("cudalaunch2"), 35000, 45000, 10, 10, 2, 0);
+  runtime_events.emplace_back(
+      std::string("cudalaunch3"), 10000, 55000, 10, 11, 3, 0);
+  device_events.emplace_back(std::string("kernel1"),
+                             TracerEventType::Kernel,
+                             40000,
+                             55000,
+                             0,
+                             10,
+                             10,
+                             1,
+                             KernelEventInfo());
+  device_events.emplace_back(std::string("kernel2"),
+                             TracerEventType::Kernel,
+                             70000,
+                             95000,
+                             0,
+                             10,
+                             10,
+                             2,
+                             KernelEventInfo());
+  device_events.emplace_back(std::string("kernel3"),
+                             TracerEventType::Kernel,
+                             60000,
+                             75000,
+                             0,
+                             10,
+                             11,
+                             3,
+                             KernelEventInfo());
+  ChromeTracingLogger logger("test_nodetrees_handletrees_case0.json");
+  logger.LogMetaInfo(std::string("1.0.2"), 0);
+  NodeTrees tree(host_events,
+                 runtime_events,
+                 device_events,
+                 mem_events,
+                 op_supplement_events);
+  std::map<uint64_t, std::vector<HostTraceEventNode*>> nodes =
+      tree.Traverse(true);
+  EXPECT_EQ(nodes[10].size(), 3u);
+  EXPECT_EQ(nodes[11].size(), 2u);
+  std::vector<HostTraceEventNode*> thread1_nodes = nodes[10];
+  std::vector<HostTraceEventNode*> thread2_nodes = nodes[11];
+  for (auto& thread1_node : thread1_nodes) {
+    if (thread1_node->Name() == "root node") {
+      EXPECT_EQ(thread1_node->GetChildren().size(), 1u);
+    }
+    if (thread1_node->Name() == "op1") {
+      EXPECT_EQ(thread1_node->GetChildren().size(), 1u);
+      EXPECT_EQ(thread1_node->GetRuntimeTraceEventNodes().size(), 1u);
+    }
+  }
+  for (auto& thread2_node : thread2_nodes) {
+    if (thread2_node->Name() == "op3") {
+      EXPECT_EQ(thread2_node->GetChildren().size(), 0u);
+      EXPECT_EQ(thread2_node->GetRuntimeTraceEventNodes().size(), 1u);
+    }
+  }
+  std::function<void(HostTraceEventNode*)> host_event_node_handle(
+      [&](HostTraceEventNode* a) { logger.LogHostTraceEventNode(*a); });
+  std::function<void(CudaRuntimeTraceEventNode*)> runtime_event_node_handle(
+      [&](CudaRuntimeTraceEventNode* a) {
+        logger.LogRuntimeTraceEventNode(*a);
+      });
+  std::function<void(DeviceTraceEventNode*)> device_event_node_handle(
+      [&](DeviceTraceEventNode* a) { logger.LogDeviceTraceEventNode(*a); });
+  std::function<void(MemTraceEventNode*)> mem_event_node_handle(
+      [&](MemTraceEventNode* a) { logger.LogMemTraceEventNode(*a); });
+  std::function<void(OperatorSupplementEventNode*)>
+      op_supplement_event_node_handle([&](OperatorSupplementEventNode* a) {});
+  tree.HandleTrees(host_event_node_handle,
+                   runtime_event_node_handle,
+                   device_event_node_handle,
+                   mem_event_node_handle,
+                   op_supplement_event_node_handle);
+  logger.LogExtraInfo(std::unordered_map<std::string, std::string>());
+}

test/cpp/fluid/platform/profiler/test_extra_info.cc

@@ -0,0 +1,31 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "gtest/gtest.h"
+#include "paddle/phi/core/platform/profiler/extra_info.h"
+
+using paddle::platform::ExtraInfo;
+
+TEST(ExtraInfoTest, case0) {
+  ExtraInfo instance;
+  instance.AddExtraInfo(std::string("info1"), std::string("%d"), 20);
+  instance.AddExtraInfo(std::string("info2"), std::string("%s"), "helloworld");
+  std::unordered_map<std::string, std::string> map = instance.GetExtraInfo();
+  EXPECT_EQ(map["info1"], "20");
+  EXPECT_EQ(map["info2"], "helloworld");
+  EXPECT_EQ(map.size(), 2u);
+  instance.Clear();
+  map = instance.GetExtraInfo();
+  EXPECT_EQ(map.size(), 0u);
+}

test/cpp/fluid/platform/profiler_test.cc

@@ -0,0 +1,159 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/core/platform/profiler.h"
+
+#include <string>
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+
+TEST(Event, CpuElapsedTime) {
+  using phi::Event;
+  using phi::EventType;
+
+  Event start_event(EventType::kPushRange, "test", 0);
+  int counter = 0;
+  while (counter != 1000) {
+    counter++;
+  }
+#ifdef _WIN32
+  Sleep(1);
+#endif
+  Event stop_event(EventType::kPopRange, "test", 0);
+  EXPECT_GT(start_event.CpuElapsedMs(stop_event), 0);
+}
+
+TEST(RecordEvent, RecordEvent) {
+  using paddle::platform::EventSortingKey;
+  using phi::Event;
+  using phi::EventRole;
+  using phi::EventType;
+  using phi::PopEvent;
+  using phi::ProfilerState;
+  using phi::PushEvent;
+  using phi::RecordEvent;
+
+  ProfilerState state = ProfilerState::kCPU;
+  paddle::platform::EnableProfiler(state);
+
+  /* Usage 1:
+   *  PushEvent(evt_name);
+   *  ...
+   *  code to be analyzed
+   *  ...
+   * PopEvent(evt_name);
+   */
+  LOG(INFO) << "Usage 1: PushEvent & PopEvent";
+  for (int loop = 0; loop < 3; ++loop) {
+    for (int i = 1; i < 5; ++i) {
+      std::string name = "op_" + std::to_string(i);
+      PushEvent(name, EventRole::kOrdinary);
+      int counter = 1;
+      while (counter != i * 1000) counter++;
+      PopEvent(name, EventRole::kOrdinary);
+    }
+  }
+
+  /* Usage 2:
+   * {
+   *   RecordEvent record_event(name);
+   *   ...
+   *   code to be analyzed
+   *   ...
+   * }
+   */
+  LOG(INFO) << "Usage 2: RecordEvent";
+  for (int i = 1; i < 5; ++i) {
+    std::string name = "evs_op_" + std::to_string(i);
+    RecordEvent record_event(name);
+    int counter = 1;
+    while (counter != i * 1000) counter++;
+  }
+
+  /* Usage 3
+   * {
+   *   RecordEvent record_event(name1, dev_ctx);
+   *   ...
+   *   code to be analyzed
+   *   ...
+   *   {
+   *     RecordEvent nested_record_event(name2, dev_ctx);
+   *     ...
+   *     code to be analyzed
+   *     ...
+   *   }
+   * }
+   */
+  LOG(INFO) << "Usage 3: nested RecordEvent";
+  for (int i = 1; i < 5; ++i) {
+    std::string name = "ano_evs_op_" + std::to_string(i);
+    RecordEvent record_event(name);
+    int counter = 1;
+    while (counter != i * 100) counter++;
+    {
+      std::string nested_name = "nested_ano_evs_op_" + std::to_string(i);
+      RecordEvent nested_record_event(nested_name);
+      int nested_counter = 1;
+      while (nested_counter != i * 100) nested_counter++;
+    }
+  }
+
+  // Bad Usage:
+  PushEvent("event_without_pop", EventRole::kOrdinary);
+  PopEvent("event_without_push", EventRole::kOrdinary);
+  std::vector<std::vector<Event>> events = paddle::platform::GetAllEvents();
+
+  int cuda_startup_count = 0;
+  int start_profiler_count = 0;
+  for (auto& item : events) {
+    for (size_t j = 0; j < item.size(); ++j) {
+      if (item[j].name() == "_cuda_startup_") ++cuda_startup_count;
+      if (item[j].name() == "_start_profiler_") ++start_profiler_count;
+      if (item[j].name() == "push") {
+        EXPECT_EQ(item[j + 1].name(), "pop");
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+        EXPECT_GT(item[j].CudaElapsedMs(item[j + 1]), 0);
+#else
+        EXPECT_GT(item[j].CpuElapsedMs(item[j + 1]), 0);
+#endif
+      }
+    }
+  }
+  EXPECT_EQ(cuda_startup_count % 5, 0);
+  EXPECT_EQ(start_profiler_count, 1);
+
+  // Will remove parsing-related code from test later
+  DisableProfiler(EventSortingKey::kTotal, "/tmp/profiler");
+}
+
+#ifdef PADDLE_WITH_CUDA
+TEST(TMP, stream_wait) {
+  cudaStream_t stream;
+  cudaStreamCreate(&stream);
+  cudaStreamSynchronize(stream);
+  cudaStreamSynchronize(stream);
+  cudaStreamSynchronize(stream);
+}
+#endif
+
+#ifdef PADDLE_WITH_HIP
+TEST(TMP, stream_wait) {
+  hipStream_t stream;
+  hipStreamCreate(&stream);
+  hipStreamSynchronize(stream);
+  hipStreamSynchronize(stream);
+  hipStreamSynchronize(stream);
+}
+#endif

test/cpp/fluid/platform/test_limit_gpu_memory.cu

@@ -0,0 +1,96 @@
+// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "gtest/gtest.h"
+#include "paddle/common/flags.h"
+#include "paddle/phi/core/platform/cuda_device_guard.h"
+#include "paddle/phi/core/platform/device/gpu/gpu_info.h"
+COMMON_DECLARE_uint64(gpu_memory_limit_mb);
+
+namespace paddle {
+namespace platform {
+
+static constexpr uint64_t GPU_MEMORY_LIMIT_MB = 500;
+static constexpr int DEVICE_ID = 0;
+
+TEST(test_record_malloc, test_limit_gpu_memory) {
+  FLAGS_gpu_memory_limit_mb = GPU_MEMORY_LIMIT_MB;
+  size_t limit = FLAGS_gpu_memory_limit_mb << 20;
+
+  {
+    ASSERT_TRUE(IsGpuMallocRecorded(DEVICE_ID));
+    ASSERT_EQ(RecordedGpuMallocSize(DEVICE_ID), 0UL);
+  }
+
+  size_t avail, total;
+  {
+    size_t actual_avail, actual_total;
+    RecordedGpuMemGetInfo(
+        &avail, &total, &actual_avail, &actual_total, DEVICE_ID);
+    ASSERT_EQ(total, limit);
+    ASSERT_EQ(paddle::platform::GpuGetLastError(), gpuSuccess);
+  }
+
+  {
+    CUDADeviceGuard guard(DEVICE_ID);
+    GpuMemoryUsage(&avail, &total);
+    ASSERT_EQ(total, limit);
+    ASSERT_EQ(paddle::platform::GpuGetLastError(), gpuSuccess);
+  }
+
+  gpuError_t err = gpuSuccess;
+
+  void *p1 = nullptr;
+  size_t size1 = limit / 4 * 3;
+  {
+    err = platform::RecordedGpuMalloc(&p1, size1, DEVICE_ID);
+    ASSERT_EQ(err, gpuSuccess);
+    ASSERT_EQ(paddle::platform::GpuGetLastError(), gpuSuccess);
+    ASSERT_NE(p1, nullptr);
+
+    ASSERT_EQ(RecordedGpuMallocSize(DEVICE_ID), size1);
+  }
+
+  void *p2 = nullptr;
+  size_t size2 = limit / 2;
+  {
+    err = platform::RecordedGpuMalloc(&p2, size2, DEVICE_ID);
+    ASSERT_EQ(err, gpuErrorOutOfMemory);
+    ASSERT_EQ(paddle::platform::GpuGetLastError(), gpuSuccess);
+    ASSERT_EQ(p2, nullptr);
+
+    ASSERT_EQ(RecordedGpuMallocSize(DEVICE_ID), size1);
+  }
+
+  {
+    platform::RecordedGpuFree(p1, size1, DEVICE_ID);
+    ASSERT_EQ(RecordedGpuMallocSize(DEVICE_ID), 0UL);
+  }
+
+  {
+    err = platform::RecordedGpuMalloc(&p2, size2, DEVICE_ID);
+    ASSERT_EQ(err, gpuSuccess);
+    ASSERT_EQ(paddle::platform::GpuGetLastError(), gpuSuccess);
+    ASSERT_NE(p2, nullptr);
+    ASSERT_EQ(RecordedGpuMallocSize(DEVICE_ID), size2);
+  }
+
+  {
+    platform::RecordedGpuFree(p2, size2, DEVICE_ID);
+    ASSERT_EQ(RecordedGpuMallocSize(DEVICE_ID), 0UL);
+  }
+}
+
+}  // namespace platform
+}  // namespace paddle

test/cpp/fluid/platform/timer_test.cc

@@ -0,0 +1,46 @@
+//  Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "paddle/phi/core/platform/timer.h"
+
+#include "gtest/gtest.h"
+
+TEST(Timer, Reset) {
+  paddle::platform::Timer timeline;
+  timeline.Start();
+  sleep(3);
+  timeline.Pause();
+  timeline.Reset();
+}
+
+TEST(Timer, Start) {
+  paddle::platform::Timer timeline;
+  timeline.Start();
+  sleep(3);
+  timeline.Pause();
+}
+
+TEST(Timer, Pause) {
+  paddle::platform::Timer timeline;
+  timeline.Start();
+  sleep(3);
+  timeline.Pause();
+}
+
+TEST(Timer, Resume) {
+  paddle::platform::Timer timeline;
+  timeline.Start();
+  sleep(3);
+  timeline.Pause();
+  timeline.Resume();
+}