chore: import upstream snapshot with attribution

test/cpp/phi/CMakeLists.txt

@@ -0,0 +1,7 @@
+add_definitions(-DPADDLE_DLL_EXPORT)
+add_subdirectory(api)
+add_subdirectory(common)
+add_subdirectory(core)
+add_subdirectory(kernels)
+add_subdirectory(ops)
+add_subdirectory(memory)

test/cpp/phi/api/CMakeLists.txt

@@ -0,0 +1,80 @@
+if(WIN32)
+  set(COMMON_API_TEST_DEPS type_info common)
+else()
+  set(COMMON_API_TEST_DEPS phi common)
+endif()
+
+if(WITH_GPU)
+  nv_test(
+    test_phi_tensor
+    SRCS test_phi_tensor.cc
+    DEPS glog ${COMMON_API_TEST_DEPS})
+  nv_test(
+    test_allocator
+    SRCS test_allocator.cu
+    DEPS phi common)
+  nv_test(
+    test_cuda_stream
+    SRCS test_cuda_stream.cu
+    DEPS phi common)
+  nv_test(
+    test_from_blob
+    SRCS test_from_blob.cc
+    DEPS ${COMMON_API_TEST_DEPS})
+elseif(WITH_ROCM)
+  hip_test(
+    test_phi_tensor
+    SRCS test_phi_tensor.cc
+    DEPS glog ${COMMON_API_TEST_DEPS})
+  hip_test(
+    test_allocator
+    SRCS test_allocator.cu
+    DEPS phi common)
+  hip_test(
+    test_cuda_stream
+    SRCS test_cuda_stream.cu
+    DEPS phi common)
+  hip_test(
+    test_from_blob
+    SRCS test_from_blob.cc
+    DEPS ${COMMON_API_TEST_DEPS})
+else()
+  cc_test(
+    test_phi_tensor
+    SRCS test_phi_tensor.cc
+    DEPS glog ${COMMON_API_TEST_DEPS})
+  cc_test(
+    test_from_blob
+    SRCS test_from_blob.cc
+    DEPS ${COMMON_API_TEST_DEPS})
+endif()
+
+cc_test(
+  test_phi_exception
+  SRCS test_phi_exception.cc
+  DEPS gtest)
+
+cc_test(
+  test_to_api
+  SRCS test_to_api.cc
+  DEPS ${COMMON_API_TEST_DEPS})
+cc_test(
+  test_slice_api
+  SRCS test_slice_api.cc
+  DEPS ${COMMON_API_TEST_DEPS})
+cc_test(
+  test_scale_benchmark
+  SRCS test_scale_benchmark.cc
+  DEPS ${COMMON_API_TEST_DEPS})
+cc_test(
+  test_data_transform
+  SRCS test_data_transform.cc
+  DEPS ${COMMON_API_TEST_DEPS})
+cc_test(
+  test_strings_empty_api
+  SRCS test_strings_empty_api.cc
+  DEPS ${COMMON_API_TEST_DEPS})
+cc_test(
+  test_strings_lower_upper_api
+  SRCS test_strings_lower_upper_api.cc
+  DEPS ${COMMON_API_TEST_DEPS})

test/cpp/phi/api/scale_api.h

@@ -0,0 +1,286 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "glog/logging.h"
+#include "paddle/common/flags.h"
+#include "paddle/phi/api/include/tensor.h"
+#include "paddle/phi/api/lib/kernel_dispatch.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/common/int_array.h"
+#include "paddle/phi/common/scalar.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/meta_tensor.h"
+#include "paddle/phi/infermeta/unary.h"
+#include "paddle/phi/kernels/scale_kernel.h"
+
+COMMON_DECLARE_int32(low_precision_op_list);
+namespace paddle {
+namespace experimental {
+
+Tensor scale_kernel_context(const Tensor& x,
+                            const Scalar& scale,
+                            const Scalar& bias,
+                            bool bias_after_scale) {
+  Backend kernel_backend = Backend::UNDEFINED;
+  DataLayout kernel_layout = DataLayout::UNDEFINED;
+  DataType kernel_data_type = DataType::UNDEFINED;
+
+  if (kernel_backend == Backend::UNDEFINED ||
+      kernel_layout == DataLayout::UNDEFINED ||
+      kernel_data_type == DataType::UNDEFINED) {
+    auto kernel_key_set = ParseKernelKeyByInputArgs(x);
+    auto kernel_key = kernel_key_set.GetHighestPriorityKernelKey();
+    if (kernel_backend == Backend::UNDEFINED) {
+      kernel_backend = kernel_key.backend();
+    }
+    if (kernel_layout == DataLayout::UNDEFINED) {
+      kernel_layout = kernel_key.layout();
+    }
+    if (kernel_data_type == DataType::UNDEFINED) {
+      kernel_data_type = kernel_key.dtype();
+    }
+  }
+  auto kernel_result = phi::KernelFactory::Instance().SelectKernelOrThrowError(
+      "scale", {kernel_backend, kernel_layout, kernel_data_type});
+  const auto& kernel = kernel_result.kernel;
+  if (FLAGS_low_precision_op_list) {
+    phi::KernelFactory::Instance().AddToLowPrecisionKernelList(
+        "scale", kernel_data_type);
+  }
+  VLOG(6) << "scale API kernel key: [" << kernel_backend << ", "
+          << kernel_layout << ", " << kernel_data_type << "]";
+  VLOG(6) << "scale API kernel: " << kernel;
+
+  auto* dev_ctx = GetDeviceContextByBackend(kernel_backend);
+  auto kernel_context = phi::KernelContext(dev_ctx);
+
+  auto dense_x = std::dynamic_pointer_cast<phi::DenseTensor>(x.impl());
+  kernel_context.EmplaceBackInput(dense_x.get());
+
+  kernel_context.EmplaceBackAttr(scale);
+  kernel_context.EmplaceBackAttr(bias);
+  kernel_context.EmplaceBackAttr(bias_after_scale);
+
+  auto dense_out = std::make_shared<phi::DenseTensor>();
+  phi::MetaTensor meta_out(dense_out.get());
+  phi::UnchangedInferMeta(*dense_x, &meta_out);
+  kernel_context.EmplaceBackOutput(dense_out.get());
+
+  Tensor out;
+  out.set_impl(dense_out);
+
+  kernel(&kernel_context);
+  return out;
+}
+
+static void ScaleCPU(DataType kernel_dtype,
+                     const phi::CPUContext& dev_ctx,
+                     const phi::DenseTensor& x,
+                     const Scalar& scale,
+                     const Scalar& bias,
+                     bool bias_after_scale,
+                     phi::DenseTensor* dense_out) {
+  switch (kernel_dtype) {
+    case phi::DataType::FLOAT64: {
+      phi::ScaleKernel<double>(
+          dev_ctx, x, scale, bias, bias_after_scale, dense_out);
+      break;
+    }
+    case phi::DataType::FLOAT32: {
+      phi::ScaleKernel<float>(
+          dev_ctx, x, scale, bias, bias_after_scale, dense_out);
+      break;
+    }
+    case phi::DataType::BFLOAT16: {
+      phi::ScaleKernel<phi::dtype::bfloat16>(
+          dev_ctx, x, scale, bias, bias_after_scale, dense_out);
+      break;
+    }
+    case phi::DataType::INT64: {
+      phi::ScaleKernel<int64_t>(
+          dev_ctx, x, scale, bias, bias_after_scale, dense_out);
+      break;
+    }
+    case phi::DataType::INT32: {
+      phi::ScaleKernel<int32_t>(
+          dev_ctx, x, scale, bias, bias_after_scale, dense_out);
+      break;
+    }
+    case phi::DataType::INT16: {
+      phi::ScaleKernel<int16_t>(
+          dev_ctx, x, scale, bias, bias_after_scale, dense_out);
+      break;
+    }
+    case phi::DataType::INT8: {
+      phi::ScaleKernel<int8_t>(
+          dev_ctx, x, scale, bias, bias_after_scale, dense_out);
+      break;
+    }
+    case phi::DataType::UINT8: {
+      phi::ScaleKernel<uint8_t>(
+          dev_ctx, x, scale, bias, bias_after_scale, dense_out);
+      break;
+    }
+    default: {
+      PADDLE_THROW(common::errors::Fatal(
+          "Detected unsupported data type."
+          "Only Float64, Float32, BFloat16, Int64, Int32, Int16, Int8, UInt8 "
+          "are supported for now."));
+      break;
+    }
+  }
+}
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+static void ScaleGPU(DataType kernel_dtype,
+                     const phi::GPUContext& dev_ctx,
+                     const phi::DenseTensor& x,
+                     const Scalar& scale,
+                     const Scalar& bias,
+                     bool bias_after_scale,
+                     phi::DenseTensor* dense_out) {
+  switch (kernel_dtype) {
+    case phi::DataType::FLOAT64: {
+      phi::ScaleKernel<double>(
+          dev_ctx, x, scale, bias, bias_after_scale, dense_out);
+      break;
+    }
+    case phi::DataType::FLOAT32: {
+      phi::ScaleKernel<float>(
+          dev_ctx, x, scale, bias, bias_after_scale, dense_out);
+      break;
+    }
+    case phi::DataType::FLOAT16: {
+      phi::ScaleKernel<phi::dtype::float16>(
+          dev_ctx, x, scale, bias, bias_after_scale, dense_out);
+      break;
+    }
+    case phi::DataType::INT64: {
+      phi::ScaleKernel<int64_t>(
+          dev_ctx, x, scale, bias, bias_after_scale, dense_out);
+      break;
+    }
+    case phi::DataType::INT32: {
+      phi::ScaleKernel<int32_t>(
+          dev_ctx, x, scale, bias, bias_after_scale, dense_out);
+      break;
+    }
+    case phi::DataType::INT16: {
+      phi::ScaleKernel<int16_t>(
+          dev_ctx, x, scale, bias, bias_after_scale, dense_out);
+      break;
+    }
+    case phi::DataType::INT8: {
+      phi::ScaleKernel<int8_t>(
+          dev_ctx, x, scale, bias, bias_after_scale, dense_out);
+      break;
+    }
+    case phi::DataType::UINT8: {
+      phi::ScaleKernel<uint8_t>(
+          dev_ctx, x, scale, bias, bias_after_scale, dense_out);
+      break;
+    }
+    default: {
+      PADDLE_THROW(common::errors::Fatal(
+          "Detected unsupported data type."
+          "Only Float64, Float32, Float16, Int64, Int32, Int16, Int8, UInt8 "
+          "are "
+          "supported for now."));
+      break;
+    }
+  }
+}
+#endif
+
+Tensor scale_switch_case(const Tensor& x,
+                         const Scalar& scale,
+                         const Scalar& bias,
+                         bool bias_after_scale) {
+  Backend kernel_backend = Backend::UNDEFINED;
+  DataLayout kernel_layout = DataLayout::UNDEFINED;
+  DataType kernel_data_type = DataType::UNDEFINED;
+
+  if (kernel_backend == Backend::UNDEFINED ||
+      kernel_layout == DataLayout::UNDEFINED ||
+      kernel_data_type == DataType::UNDEFINED) {
+    auto kernel_key_set = ParseKernelKeyByInputArgs(x);
+    auto kernel_key = kernel_key_set.GetHighestPriorityKernelKey();
+    if (kernel_backend == Backend::UNDEFINED) {
+      kernel_backend = kernel_key.backend();
+    }
+    if (kernel_layout == DataLayout::UNDEFINED) {
+      kernel_layout = kernel_key.layout();
+    }
+    if (kernel_data_type == DataType::UNDEFINED) {
+      kernel_data_type = kernel_key.dtype();
+    }
+  }
+  auto kernel_result = phi::KernelFactory::Instance().SelectKernelOrThrowError(
+      "scale", {kernel_backend, kernel_layout, kernel_data_type});
+  const auto& kernel = kernel_result.kernel;
+  if (FLAGS_low_precision_op_list) {
+    phi::KernelFactory::Instance().AddToLowPrecisionKernelList(
+        "scale", kernel_data_type);
+  }
+  VLOG(6) << "scale API kernel key: [" << kernel_backend << ", "
+          << kernel_layout << ", " << kernel_data_type << "]";
+  VLOG(6) << "scale API kernel: " << kernel;
+
+  auto* dev_ctx = GetDeviceContextByBackend(kernel_backend);
+
+  auto dense_x = std::dynamic_pointer_cast<phi::DenseTensor>(x.impl());
+
+  auto dense_out = std::make_shared<phi::DenseTensor>();
+  phi::MetaTensor meta_out(dense_out.get());
+  phi::UnchangedInferMeta(*dense_x, &meta_out);
+
+  Tensor out;
+  out.set_impl(dense_out);
+
+  switch (kernel_backend) {
+    case Backend::CPU:
+      ScaleCPU(kernel_data_type,
+               static_cast<const phi::CPUContext&>(*dev_ctx),
+               *dense_x,
+               scale,
+               bias,
+               bias_after_scale,
+               dense_out.get());
+      break;
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+    case Backend::GPU:
+      ScaleGPU(kernel_data_type,
+               static_cast<const phi::GPUContext&>(*dev_ctx),
+               *dense_x,
+               scale,
+               bias,
+               bias_after_scale,
+               dense_out.get());
+      break;
+#endif
+    default:
+      PADDLE_THROW(common::errors::Fatal(
+          "Detected unsupported backend."
+          "Only CPU and CUDA Backend are supported for now."
+          "Please double check if your backend falls into the above two "
+          "categories."));
+  }
+
+  return out;
+}
+
+}  // namespace experimental
+}  // namespace paddle

test/cpp/phi/api/test_allocator.cu

@@ -0,0 +1,72 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+
+#include "paddle/phi/api/include/context_pool.h"
+
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/common/memory_utils.h"
+#include "paddle/phi/common/place.h"
+#include "paddle/phi/common/transform.h"
+#include "paddle/phi/core/allocator.h"
+#include "paddle/phi/core/device_context.h"
+
+using phi::memory_utils::Copy;
+
+template <typename T>
+class Scale {
+ public:
+  explicit Scale(const T& scale) : scale_(scale) {}
+  HOSTDEVICE T operator()(const T& a) const { return a * scale_; }
+
+ private:
+  T scale_;
+};
+
+TEST(Allocator, CPU) {
+  phi::Allocator* allocator = paddle::GetAllocator(phi::CPUPlace());
+  auto cpu_allocation = allocator->Allocate(sizeof(float) * 4);
+  float* cpu_buf = static_cast<float*>(cpu_allocation->ptr());
+  ASSERT_NE(cpu_buf, nullptr);
+  cpu_buf[0] = 1.0f;
+  cpu_buf[1] = 2.0f;
+  cpu_buf[2] = 3.0f;
+  cpu_buf[3] = 4.0f;
+  for (size_t i = 0; i < 4; ++i) {
+    cpu_buf[i] = cpu_buf[i] + 1;
+  }
+  for (size_t i = 0; i < 4; ++i) {
+    ASSERT_NEAR(cpu_buf[i], static_cast<float>(2.0 + i), 1e-5);
+  }
+}
+
+TEST(Allocator, GPU) {
+  phi::GPUPlace gpu0(0);
+  float cpu_buf[4] = {0.1, 0.2, 0.3, 0.4};
+  phi::Allocator* allocator = paddle::GetAllocator(gpu0);
+  auto gpu_allocation = allocator->Allocate(sizeof(cpu_buf));
+  float* gpu_buf = static_cast<float*>(gpu_allocation->ptr());
+
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* ctx = reinterpret_cast<phi::GPUContext*>(pool.Get(gpu0));
+  Copy(gpu0, gpu_buf, phi::CPUPlace(), cpu_buf, sizeof(cpu_buf), ctx->stream());
+  phi::Transform<phi::GPUContext> trans;
+  trans(*ctx, gpu_buf, gpu_buf + 4, gpu_buf, Scale<float>(10));
+  ctx->Wait();
+  Copy(phi::CPUPlace(), cpu_buf, gpu0, gpu_buf, sizeof(cpu_buf), ctx->stream());
+  for (int i = 0; i < 4; ++i) {
+    ASSERT_NEAR(cpu_buf[i], static_cast<float>(i + 1), 1e-5);
+  }
+}

test/cpp/phi/api/test_cuda_stream.cu

@@ -0,0 +1,26 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+
+#include "paddle/phi/api/include/context_pool.h"
+#include "paddle/phi/core/cuda_stream.h"
+
+TEST(CUDAStream, GPU) {
+  phi::GPUPlace gpu0(0);
+  phi::CUDAStream* stream = paddle::GetCurrentCUDAStream(gpu0);
+  EXPECT_TRUE(stream != nullptr);
+  gpuStream_t raw_stream = stream->raw_stream();
+  EXPECT_TRUE(raw_stream != nullptr);
+}

test/cpp/phi/api/test_data_transform.cc

@@ -0,0 +1,107 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+
+#include <memory>
+
+#include "paddle/phi/api/include/api.h"
+#include "paddle/phi/common/complex.h"
+#include "paddle/phi/common/place.h"
+#include "paddle/phi/core/compat/convert_utils.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(matmul, CPU, ALL_LAYOUT);
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+PD_DECLARE_KERNEL(full, GPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(matmul, GPU, ALL_LAYOUT);
+#endif
+
+namespace paddle {
+namespace tests {
+
+// TODO(chenweihang): Remove this test after the API is used in the dygraph
+TEST(API, data_transform_same_place) {
+  // 1. create tensor
+  auto x =
+      paddle::experimental::full({3, 3}, 1.0, DataType::COMPLEX128, CPUPlace());
+
+  auto y =
+      paddle::experimental::full({3, 3}, 2.0, DataType::FLOAT32, CPUPlace());
+
+  std::vector<phi::dtype::complex<double>> sum(9, 6.0);
+
+  // 2. test API
+  auto out = paddle::experimental::matmul(x, y, false, false);
+
+  // 3. check result
+  ASSERT_EQ(out.dims().size(), 2);
+  ASSERT_EQ(out.dims()[0], 3);
+  ASSERT_EQ(out.dims()[1], 3);
+  ASSERT_EQ(out.numel(), 9);
+  ASSERT_EQ(out.type(), phi::DataType::COMPLEX128);
+  ASSERT_EQ(out.layout(), phi::DataLayout::NCHW);
+  ASSERT_EQ(out.initialized(), true);
+
+  auto dense_out = std::dynamic_pointer_cast<phi::DenseTensor>(out.impl());
+
+  for (size_t i = 0; i < 9; i++) {
+    ASSERT_NEAR(sum[i].real,
+                dense_out->data<phi::dtype::complex<double>>()[i].real,
+                1e-6f);
+    ASSERT_NEAR(sum[i].imag,
+                dense_out->data<phi::dtype::complex<double>>()[i].imag,
+                1e-6f);
+  }
+}
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+TEST(Tensor, data_transform_diff_place) {
+  // 1. create tensor
+  auto x = paddle::experimental::full(
+      {3, 3}, 1.0, phi::DataType::FLOAT64, CPUPlace());
+
+  auto y = paddle::experimental::full(
+      {3, 3}, 2.0, phi::DataType::FLOAT64, GPUPlace());
+
+  std::vector<float> sum(9, 6.0);
+
+  // 2. test API
+  auto out = paddle::experimental::matmul(x, y, false, false);
+
+  // 3. check result
+  ASSERT_EQ(out.dims().size(), 2);
+  ASSERT_EQ(out.dims()[0], 3);
+  ASSERT_EQ(out.dims()[1], 3);
+  ASSERT_EQ(out.numel(), 9);
+  ASSERT_EQ(out.dtype(), phi::DataType::FLOAT64);
+  ASSERT_EQ(out.layout(), phi::DataLayout::NCHW);
+  ASSERT_EQ(out.initialized(), true);
+  ASSERT_EQ(out.impl()->place(), phi::TransToPhiPlace(phi::Backend::GPU));
+
+  auto ref_out = experimental::copy_to(out, CPUPlace(), true);
+
+  auto dense_out = std::dynamic_pointer_cast<phi::DenseTensor>(ref_out.impl());
+  for (size_t i = 0; i < 9; i++) {
+    ASSERT_NEAR(sum[i], dense_out->data<double>()[i], 1e-6f);
+  }
+}
+
+#endif
+
+}  // namespace tests
+}  // namespace paddle

test/cpp/phi/api/test_from_blob.cc

@@ -0,0 +1,233 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include "paddle/phi/api/include/api.h"
+#include "paddle/phi/api/include/tensor_utils.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+#include "paddle/phi/api/include/context_pool.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/backends/gpu/gpu_info.h"
+#include "paddle/phi/common/memory_utils.h"
+#endif
+
+PD_DECLARE_KERNEL(pow, CPU, ALL_LAYOUT);
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+PD_DECLARE_KERNEL(pow, GPU, ALL_LAYOUT);
+#endif
+
+using paddle::from_blob;
+using phi::DataType;
+
+namespace paddle {
+phi::Place GetPlaceFromPtr(void* data);
+}  // namespace paddle
+
+TEST(from_blob, CPU) {
+  // 1. create data
+  int64_t data[] = {4, 3, 2, 1};  // NOLINT
+
+  ASSERT_EQ(paddle::GetPlaceFromPtr(data), phi::CPUPlace());
+
+  // 2. test API
+  auto test_tensor = from_blob(data, {1, 2, 2}, DataType::INT64);
+
+  // 3. check result
+  // 3.1 check tensor attributes
+  ASSERT_EQ(test_tensor.dims().size(), 3);
+  ASSERT_EQ(test_tensor.dims()[0], 1);
+  ASSERT_EQ(test_tensor.dims()[1], 2);
+  ASSERT_EQ(test_tensor.dims()[2], 2);
+  ASSERT_EQ(test_tensor.numel(), 4);
+  ASSERT_EQ(test_tensor.is_cpu(), true);
+  ASSERT_EQ(test_tensor.dtype(), DataType::INT64);
+  ASSERT_EQ(test_tensor.layout(), phi::DataLayout::NCHW);
+  ASSERT_EQ(test_tensor.is_dense_tensor(), true);
+
+  // 3.2 check tensor values
+  auto* test_tensor_data = test_tensor.template data<int64_t>();
+  for (int64_t i = 0; i < 4; i++) {
+    ASSERT_EQ(test_tensor_data[i], 4 - i);
+  }
+
+  // 3.3 check whether memory is shared
+  ASSERT_EQ(data, test_tensor_data);
+
+  // 3.4 test other API
+  auto test_tensor_pow = paddle::experimental::pow(test_tensor, 2);
+  auto* test_tensor_pow_data = test_tensor_pow.template data<int64_t>();
+  for (int64_t i = 0; i < 4; i++) {
+    ASSERT_EQ(test_tensor_pow_data[i],
+              static_cast<int64_t>(std::pow(4 - i, 2)));
+  }
+}
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+
+using phi::memory_utils::Copy;
+
+TEST(GetPlaceFromPtr, GPU) {
+  using paddle::GetPlaceFromPtr;
+
+  std::array<float, 6> cpu_data = {};
+  auto cpu_data_place = GetPlaceFromPtr(cpu_data.data());
+  ASSERT_EQ(cpu_data_place, phi::CPUPlace());
+  std::cout << "cpu_data_place: " << cpu_data_place << std::endl;
+
+  auto alloc_ptr =
+      paddle::GetAllocator(phi::GPUPlace(0))->Allocate(sizeof(cpu_data));
+  float* gpu0_data = static_cast<float*>(alloc_ptr->ptr());
+  auto gpu0_data_place = GetPlaceFromPtr(gpu0_data);
+  ASSERT_EQ(gpu0_data_place, phi::GPUPlace(0));
+  std::cout << "gpu0_data_place: " << gpu0_data_place << std::endl;
+  alloc_ptr.release();
+
+  if (phi::backends::gpu::GetGPUDeviceCount() > 1) {
+    float* gpu1_data =
+        static_cast<float*>(paddle::GetAllocator(phi::GPUPlace(1))
+                                ->Allocate(sizeof(cpu_data))
+                                ->ptr());
+    auto gpu1_data_place = GetPlaceFromPtr(gpu1_data);
+    ASSERT_EQ(gpu1_data_place, phi::GPUPlace(1));
+    std::cout << "gpu1_data_place: " << gpu1_data_place << std::endl;
+  }
+
+  // Test GPUPinnedPlace (cudaMemoryTypeHost)
+  auto pinned_alloc_ptr =
+      paddle::GetAllocator(phi::GPUPinnedPlace())->Allocate(sizeof(cpu_data));
+  float* pinned_data = static_cast<float*>(pinned_alloc_ptr->ptr());
+  auto pinned_data_place = GetPlaceFromPtr(pinned_data);
+  ASSERT_EQ(pinned_data_place, phi::GPUPinnedPlace());
+  std::cout << "pinned_data_place: " << pinned_data_place << std::endl;
+  pinned_alloc_ptr.release();
+}
+
+TEST(from_blob, GPU) {
+  // 1. create data
+  std::array<float, 6> cpu_data = {0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f};
+  phi::GPUPlace gpu0(0);
+  phi::Allocator* allocator = paddle::GetAllocator(gpu0);
+  auto gpu_allocation = allocator->Allocate(sizeof(cpu_data));
+  float* gpu_data = static_cast<float*>(gpu_allocation->ptr());
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* ctx = reinterpret_cast<phi::GPUContext*>(pool.Get(gpu0));
+  Copy(gpu0,
+       gpu_data,
+       phi::CPUPlace(),
+       cpu_data.data(),
+       sizeof(cpu_data),
+       ctx->stream());
+
+  // 2. test API
+  auto gpu_tensor = from_blob(gpu_data, {2, 3}, DataType::FLOAT32);
+
+  // 3. check result
+  // 3.1 check tensor attributes
+  ASSERT_EQ(gpu_tensor.dims().size(), 2);
+  ASSERT_EQ(gpu_tensor.dims()[0], 2);
+  ASSERT_EQ(gpu_tensor.dims()[1], 3);
+  ASSERT_EQ(gpu_tensor.numel(), 6);
+  // ASSERT_EQ(gpu_tensor.is_gpu(), true);
+  ASSERT_EQ(gpu_tensor.dtype(), DataType::FLOAT32);
+
+  // 3.2 check tensor values
+  auto* gpu_tensor_data = gpu_tensor.template data<float>();
+  std::array<float, 6> gpu_tensor_data_cpu = {};
+  Copy(phi::CPUPlace(),
+       gpu_tensor_data_cpu.data(),
+       gpu0,
+       gpu_tensor_data,
+       sizeof(cpu_data),
+       ctx->stream());
+  for (int64_t i = 0; i < 6; i++) {
+    ASSERT_NEAR(
+        gpu_tensor_data_cpu[i], static_cast<float>((i + 1) * 0.1f), 1e-5);
+  }
+
+  // 3.3 check whether memory is shared
+  ASSERT_EQ(gpu_data, gpu_tensor_data);
+
+  // 3.4 test other API
+  auto gpu_tensor_pow = paddle::experimental::pow(gpu_tensor, 2);
+  auto* gpu_tensor_pow_data = gpu_tensor_pow.template data<float>();
+  std::array<float, 6> gpu_tensor_pow_data_cpu = {};
+  Copy(phi::CPUPlace(),
+       gpu_tensor_pow_data_cpu.data(),
+       gpu0,
+       gpu_tensor_pow_data,
+       sizeof(cpu_data),
+       ctx->stream());
+  for (int64_t i = 0; i < 6; i++) {
+    ASSERT_NEAR(gpu_tensor_pow_data_cpu[i],
+                static_cast<float>(std::pow(i + 1, 2) * 0.01f),
+                1e-5);
+  }
+}
+#endif
+
+TEST(from_blob, Option) {
+  int delete_count = 0, f_delete_count = 0;
+  auto deleter = [&delete_count](void* data) {
+    delete[] static_cast<int64_t*>(data);
+    delete_count++;
+  };
+  auto f_deleter = [&f_delete_count](void* ptr) {
+    delete[] static_cast<float*>(ptr);
+    f_delete_count++;
+  };
+  {
+    auto data = new int64_t[8];
+    for (int64_t i = 0; i < 8; i++) {
+      data[i] = i;
+    }
+    auto test_tensor = from_blob(data,
+                                 {1, 2, 2, 2},
+                                 DataType::INT64,
+                                 phi::DataLayout::NHWC,
+                                 phi::CPUPlace(),
+                                 deleter);
+    ASSERT_EQ(test_tensor.layout(), phi::DataLayout::NHWC);
+    ASSERT_EQ(delete_count, 0);
+
+    auto f_data = new float[8];
+    for (int i = 0; i < 8; i++) {
+      f_data[i] = static_cast<float>(i);
+    }
+    auto test_tensor_f = from_blob(f_data,
+                                   {1, 2, 2, 2},
+                                   DataType::FLOAT32,
+                                   common::DataLayout::NHWC,
+                                   phi::CPUPlace(),
+                                   f_deleter);
+    ASSERT_EQ(test_tensor_f.layout(), phi::DataLayout::NHWC);
+    ASSERT_EQ(f_delete_count, 0);
+  }
+  ASSERT_EQ(delete_count, 1);
+  ASSERT_EQ(f_delete_count, 1);
+}
+
+TEST(from_blob, Strides) {
+  int64_t data[8] = {0, 1, 2, 3, 4, 5, 6, 7};
+  auto test_tensor =
+      from_blob(data, {1, 2, 2, 1}, {0, 4, 2, 0}, DataType::INT64);
+  ASSERT_EQ(test_tensor.shape()[1], 2);
+  ASSERT_EQ(test_tensor.shape()[2], 2);
+  ASSERT_EQ(test_tensor.strides()[1], 4);
+  ASSERT_EQ(test_tensor.strides()[2], 2);
+}

test/cpp/phi/api/test_phi_exception.cc

@@ -0,0 +1,164 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <iostream>
+#include <string>
+
+#include "gtest/gtest.h"
+#include "paddle/common/exception.h"
+
+namespace paddle {
+namespace tests {
+
+TEST(PD_THROW, empty) {
+  bool caught_exception = false;
+  try {
+    PD_THROW();
+  } catch (const std::exception& e) {
+    caught_exception = true;
+    std::string err_msg = e.what();
+    EXPECT_TRUE(err_msg.find("An error occurred.") != std::string::npos);
+#if _WIN32
+    EXPECT_TRUE(err_msg.find("test\\cpp\\phi\\api\\test_phi_exception.cc") !=
+                std::string::npos);
+#else
+    EXPECT_TRUE(err_msg.find("test/cpp/phi/api/test_phi_exception.cc") !=
+                std::string::npos);
+#endif
+  }
+  EXPECT_TRUE(caught_exception);
+}
+
+TEST(PD_THROW, non_empty) {
+  bool caught_exception = false;
+  try {
+    PD_THROW("PD_THROW returns ",
+             false,
+             ". DataType of ",
+             1,
+             " is INT. ",
+             "DataType of ",
+             0.23,
+             " is FLOAT. ");
+  } catch (const std::exception& e) {
+    caught_exception = true;
+    std::string err_msg = e.what();
+    EXPECT_TRUE(err_msg.find("PD_THROW returns 0. DataType of 1 is INT. ") !=
+                std::string::npos);
+#if _WIN32
+    EXPECT_TRUE(err_msg.find("test\\cpp\\phi\\api\\test_phi_exception.cc") !=
+                std::string::npos);
+#else
+    EXPECT_TRUE(err_msg.find("test/cpp/phi/api/test_phi_exception.cc") !=
+                std::string::npos);
+#endif
+  }
+  EXPECT_TRUE(caught_exception);
+}
+
+TEST(PD_CHECK, OK) {
+  PD_CHECK(true);
+  PD_CHECK(true, "PD_CHECK returns ", true, "now");
+
+  const size_t a = 1;
+  const size_t b = 10;
+  PD_CHECK(a < b);
+  PD_CHECK(a < b, "PD_CHECK returns ", true, a, "should < ", b);
+}
+
+TEST(PD_CHECK, FAILED) {
+  bool caught_exception = false;
+  try {
+    PD_CHECK(false);
+  } catch (const std::exception& e) {
+    caught_exception = true;
+    std::string err_msg = e.what();
+    EXPECT_TRUE(err_msg.find("Expected false, but it's not satisfied.") !=
+                std::string::npos);
+#if _WIN32
+    EXPECT_TRUE(err_msg.find("test\\cpp\\phi\\api\\test_phi_exception.cc") !=
+                std::string::npos);
+#else
+    EXPECT_TRUE(err_msg.find("test/cpp/phi/api/test_phi_exception.cc") !=
+                std::string::npos);
+#endif
+  }
+  EXPECT_TRUE(caught_exception);
+
+  caught_exception = false;
+  try {
+    PD_CHECK(false,
+             "PD_CHECK returns ",
+             false,
+             ". DataType of ",
+             1,
+             " is INT. ",
+             "DataType of ",
+             0.23,
+             " is FLOAT. ");
+  } catch (const std::exception& e) {
+    caught_exception = true;
+    std::string err_msg = e.what();
+    EXPECT_TRUE(err_msg.find("PD_CHECK returns 0. DataType of 1 is INT. ") !=
+                std::string::npos);
+#if _WIN32
+    EXPECT_TRUE(err_msg.find("test\\cpp\\phi\\api\\test_phi_exception.cc") !=
+                std::string::npos);
+#else
+    EXPECT_TRUE(err_msg.find("test/cpp/phi/api/test_phi_exception.cc") !=
+                std::string::npos);
+#endif
+  }
+  EXPECT_TRUE(caught_exception);
+
+  const size_t a = 1;
+  const size_t b = 10;
+  caught_exception = false;
+  try {
+    PD_CHECK(a > b);
+  } catch (const std::exception& e) {
+    caught_exception = true;
+    std::string err_msg = e.what();
+    EXPECT_TRUE(err_msg.find("Expected a > b, but it's not satisfied.") !=
+                std::string::npos);
+#if _WIN32
+    EXPECT_TRUE(err_msg.find("test\\cpp\\phi\\api\\test_phi_exception.cc") !=
+                std::string::npos);
+#else
+    EXPECT_TRUE(err_msg.find("test/cpp/phi/api/test_phi_exception.cc") !=
+                std::string::npos);
+#endif
+  }
+  EXPECT_TRUE(caught_exception);
+
+  const size_t c = 123;
+  const float d = 0.345;
+  caught_exception = false;
+  try {
+    PD_CHECK(c < d, "PD_CHECK returns ", false, ", because ", c, " > ", d);
+  } catch (const std::exception& e) {
+    caught_exception = true;
+    std::string err_msg = e.what();
+    EXPECT_TRUE(err_msg.find("PD_CHECK returns 0, because 123 > 0.345") !=
+                std::string::npos);
+#if _WIN32
+    EXPECT_TRUE(err_msg.find("test\\cpp\\phi\\api\\test_phi_exception.cc") !=
+                std::string::npos);
+#else
+    EXPECT_TRUE(err_msg.find("test/cpp/phi/api/test_phi_exception.cc") !=
+                std::string::npos);
+#endif
+  }
+  EXPECT_TRUE(caught_exception);
+}
+
+}  // namespace tests
+}  // namespace paddle

test/cpp/phi/api/test_phi_tensor.cc

@@ -0,0 +1,431 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/phi/api/include/api.h"
+#include "paddle/phi/api/include/tensor.h"
+#include "paddle/phi/common/data_type.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/selected_rows.h"
+
+PD_DECLARE_KERNEL(empty, CPU, ALL_LAYOUT);
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+PD_DECLARE_KERNEL(empty, GPU, ALL_LAYOUT);
+#endif
+
+namespace paddle {
+namespace tests {
+
+using Tensor = paddle::Tensor;
+using DataType = phi::DataType;
+
+template <typename T>
+Tensor InitCPUTensorForTest() {
+  std::vector<int64_t> tensor_shape{5, 5};
+  DataType dtype = phi::CppTypeToDataType<T>::Type();
+  Tensor t1 = paddle::experimental::empty(tensor_shape, dtype, phi::CPUPlace());
+  auto* p_data_ptr = t1.data<T>();
+  for (int64_t i = 0; i < t1.size(); i++) {
+    p_data_ptr[i] = T(5);
+  }
+  return t1;
+}
+
+template <typename T>
+void TestCopyTensor() {
+  auto t1 = InitCPUTensorForTest<T>();
+  auto t1_cpu_cp = t1.copy_to(phi::CPUPlace(), /*blocking=*/false);
+  PADDLE_ENFORCE_EQ(t1_cpu_cp.place(),
+                    phi::CPUPlace(),
+                    common::errors::InvalidArgument("t1_cpu_cp should copy to "
+                                                    "CPUPlace, but got %s",
+                                                    t1_cpu_cp.place()));
+  for (int64_t i = 0; i < t1.size(); i++) {
+    PADDLE_ENFORCE_EQ(
+        t1_cpu_cp.template data<T>()[i],
+        T(5),
+        common::errors::InvalidArgument(
+            "t1_cpu_cp.template data<T>()[%d] should be equal to T(5) ", i));
+  }
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  VLOG(2) << "Do GPU copy test";
+  auto t1_gpu_cp = t1_cpu_cp.copy_to(phi::GPUPlace(), /*blocking=*/false);
+  PADDLE_ENFORCE_EQ(t1_gpu_cp.place(),
+                    phi::GPUPlace(),
+                    common::errors::InvalidArgument("t1_gpu_cp should copy to "
+                                                    "GPUPlace, but got %s",
+                                                    t1_gpu_cp.place()));
+  auto t1_gpu_cp_cp = t1_gpu_cp.copy_to(phi::GPUPlace(), /*blocking=*/false);
+  PADDLE_ENFORCE_EQ(
+      t1_gpu_cp_cp.place(),
+      phi::GPUPlace(),
+      common::errors::InvalidArgument("t1_gpu_cp_cp should copy to "
+                                      "GPUPlace, but got %s",
+                                      t1_gpu_cp_cp.place()));
+  auto t1_gpu_cp_cp_cpu =
+      t1_gpu_cp_cp.copy_to(phi::CPUPlace(), /*blocking=*/false);
+  PADDLE_ENFORCE_EQ(
+      t1_gpu_cp_cp_cpu.place(),
+      phi::CPUPlace(),
+      common::errors::InvalidArgument("t1_gpu_cp_cp_cpu should copy to "
+                                      "CPUPlace, but got %s",
+                                      t1_gpu_cp_cp_cpu.place()));
+  for (int64_t i = 0; i < t1.size(); i++) {
+    PADDLE_ENFORCE_EQ(
+        t1_gpu_cp_cp_cpu.template data<T>()[i],
+        T(5),
+        common::errors::InvalidArgument(
+            "t1_gpu_cp_cp_cpu.template data<T>()[%d] should be equal to T(5) ",
+            i));
+  }
+#endif
+}
+
+void TestAPIPlace() {
+  std::vector<int64_t> tensor_shape = {5, 5};
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  auto t1 = paddle::experimental::empty(
+      tensor_shape, DataType::FLOAT32, phi::GPUPlace());
+  PADDLE_ENFORCE_EQ(t1.place(),
+                    phi::GPUPlace(),
+                    common::errors::InvalidArgument(
+                        "t1 should copy to GPUPlace, but got %s", t1.place()));
+#endif
+  auto t2 = paddle::experimental::empty(
+      tensor_shape, DataType::FLOAT32, phi::CPUPlace());
+  PADDLE_ENFORCE_EQ(t2.place(),
+                    phi::CPUPlace(),
+                    common::errors::InvalidArgument(
+                        "t2 should copy to CPUPlace, but got %s", t2.place()));
+}
+
+void TestAPISizeAndShape() {
+  std::vector<int64_t> tensor_shape = {5, 5};
+  auto t1 = paddle::experimental::empty(tensor_shape);
+  PADDLE_ENFORCE_EQ(
+      t1.size(),
+      25,
+      common::errors::InvalidArgument("t1.size should be equal to 25, "
+                                      "but got %d",
+                                      t1.size()));
+  PADDLE_ENFORCE_EQ(t1.shape(),
+                    tensor_shape,
+                    common::errors::InvalidArgument(
+                        "t1.shape should be equal to tensor_shape, "));
+}
+
+void TestAPISlice() {
+  std::vector<int64_t> tensor_shape_origin1 = {5, 5};
+  std::vector<int64_t> tensor_shape_sub1 = {3, 5};
+  std::vector<int64_t> tensor_shape_origin2 = {5, 5, 5};
+  std::vector<int64_t> tensor_shape_sub2 = {1, 5, 5};
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  auto t1 = paddle::experimental::empty(
+      tensor_shape_origin1, DataType::FLOAT32, phi::GPUPlace());
+  PADDLE_ENFORCE_EQ(
+      t1.slice(0, 5).shape(),
+      tensor_shape_origin1,
+      common::errors::InvalidArgument("t1.slice(0, 5).shape should be equal to "
+                                      "{5, 5}"));
+  PADDLE_ENFORCE_EQ(
+      t1.slice(0, 3).shape(),
+      tensor_shape_sub1,
+      common::errors::InvalidArgument("t1.slice(0, 3).shape should be equal to "
+                                      "{3, 5}"));
+  auto t2 = paddle::experimental::empty(
+      tensor_shape_origin2, DataType::FLOAT32, phi::GPUPlace());
+  PADDLE_ENFORCE_EQ(
+      t2.slice(4, 5).shape(),
+      tensor_shape_sub2,
+      common::errors::InvalidArgument("t2.slice(4, 5).shape should be equal to "
+                                      "{1, 5, 5}"));
+#endif
+  auto t3 = paddle::experimental::empty(
+      tensor_shape_origin1, DataType::FLOAT32, phi::CPUPlace());
+  PADDLE_ENFORCE_EQ(
+      t3.slice(0, 5).shape(),
+      tensor_shape_origin1,
+      common::errors::InvalidArgument("t3.slice(0, 5).shape should be equal to "
+                                      "{5, 5}"));
+  PADDLE_ENFORCE_EQ(
+      t3.slice(0, 3).shape(),
+      tensor_shape_sub1,
+      common::errors::InvalidArgument("t3.slice(0, 3).shape should be equal to "
+                                      "{3, 5}"));
+  auto t4 = paddle::experimental::empty(
+      tensor_shape_origin2, DataType::FLOAT32, phi::CPUPlace());
+  PADDLE_ENFORCE_EQ(
+      t4.slice(4, 5).shape(),
+      tensor_shape_sub2,
+      common::errors::InvalidArgument("t4.slice(4, 5).shape should be equal to "
+                                      "{1, 5, 5}"));
+
+  // Test writing function for sliced tensor
+  auto t = InitCPUTensorForTest<float>();
+  auto t_sliced = t.slice(0, 1);
+  auto* t_sliced_data_ptr = t_sliced.data<float>();
+  for (int64_t i = 0; i < t_sliced.size(); i++) {
+    t_sliced_data_ptr[i] += static_cast<float>(5);
+  }
+  auto* t_data_ptr = t.data<float>();
+  for (int64_t i = 0; i < t_sliced.size(); i++) {
+    PADDLE_ENFORCE_EQ(t_data_ptr[i],
+                      static_cast<float>(10),
+                      common::errors::InvalidArgument(
+                          "Required t_data_ptr[%d] should be equal "
+                          "to static_cast<float>(10) ",
+                          i));
+  }
+}
+
+template <typename T>
+paddle::DataType TestDtype() {
+  std::vector<int64_t> tensor_shape = {5, 5};
+  DataType dtype = phi::CppTypeToDataType<T>::Type();
+  auto t1 = paddle::experimental::empty(tensor_shape, dtype, phi::CPUPlace());
+  return t1.type();
+}
+
+template <typename T>
+void TestCast(paddle::DataType data_type) {
+  std::vector<int64_t> tensor_shape = {5, 5};
+  DataType dtype = phi::CppTypeToDataType<T>::Type();
+  auto t1 = paddle::experimental::empty(tensor_shape, dtype, phi::CPUPlace());
+  auto t2 = t1.cast(data_type);
+  PADDLE_ENFORCE_EQ(
+      t2.type(),
+      data_type,
+      common::errors::InvalidArgument("t2.type() should be equal to data_type, "
+                                      "but got %s",
+                                      t2.type()));
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  auto tg1 = paddle::experimental::empty(tensor_shape, dtype, phi::GPUPlace());
+  auto tg2 = tg1.cast(data_type);
+  PADDLE_ENFORCE_EQ(tg2.type(),
+                    data_type,
+                    common::errors::InvalidArgument(
+                        "tg2.type() should be equal to data_type, "
+                        "but got %s",
+                        tg2.type()));
+#endif
+}
+
+void GroupTestCopy() {
+  VLOG(2) << "Float cpu-cpu-gpu-gpu-cpu";
+  TestCopyTensor<float>();
+  VLOG(2) << "Double cpu-cpu-gpu-gpu-cpu";
+  TestCopyTensor<double>();
+  VLOG(2) << "int cpu-cpu-gpu-gpu-cpu";
+  TestCopyTensor<int32_t>();
+  VLOG(2) << "int64 cpu-cpu-gpu-gpu-cpu";
+  TestCopyTensor<int64_t>();
+  VLOG(2) << "int16 cpu-cpu-gpu-gpu-cpu";
+  TestCopyTensor<int16_t>();
+  VLOG(2) << "int8 cpu-cpu-gpu-gpu-cpu";
+  TestCopyTensor<int8_t>();
+  VLOG(2) << "uint8 cpu-cpu-gpu-gpu-cpu";
+  TestCopyTensor<uint8_t>();
+  VLOG(2) << "complex<float> cpu-cpu-gpu-gpu-cpu";
+  TestCopyTensor<paddle::complex64>();
+  VLOG(2) << "complex<double> cpu-cpu-gpu-gpu-cpu";
+  TestCopyTensor<paddle::complex128>();
+  VLOG(2) << "Fp16 cpu-cpu-gpu-gpu-cpu";
+  TestCopyTensor<paddle::float16>();
+}
+
+void GroupTestCast() {
+  VLOG(2) << "int16_t cast";
+  TestCast<int16_t>(paddle::DataType::FLOAT32);
+  VLOG(2) << "int32 cast";
+  TestCast<int32_t>(paddle::DataType::FLOAT32);
+  VLOG(2) << "int64 cast";
+  TestCast<int64_t>(paddle::DataType::FLOAT32);
+  VLOG(2) << "double cast";
+  TestCast<double>(paddle::DataType::FLOAT32);
+  VLOG(2) << "bool cast";
+  TestCast<bool>(paddle::DataType::FLOAT32);
+  VLOG(2) << "uint8 cast";
+  TestCast<uint8_t>(paddle::DataType::FLOAT32);
+  VLOG(2) << "float cast";
+  TestCast<float>(paddle::DataType::FLOAT32);
+  VLOG(2) << "complex<float> cast";
+  TestCast<paddle::complex64>(paddle::DataType::FLOAT32);
+  VLOG(2) << "complex<double> cast";
+  TestCast<paddle::complex128>(paddle::DataType::FLOAT32);
+  VLOG(2) << "float16 cast";
+  TestCast<paddle::float16>(paddle::DataType::FLOAT16);
+}
+
+void GroupTestDtype() {
+  PADDLE_ENFORCE_EQ(
+      TestDtype<bool>(),
+      paddle::DataType::BOOL,
+      common::errors::InvalidArgument("TestDtype<bool>() should be equal to "
+                                      "paddle::DataType::BOOL, but got %s",
+                                      TestDtype<bool>()));
+  PADDLE_ENFORCE_EQ(
+      TestDtype<int8_t>(),
+      paddle::DataType::INT8,
+      common::errors::InvalidArgument("TestDtype<int8_t>() should be equal to "
+                                      "paddle::DataType::INT8, but got %s",
+                                      TestDtype<int8_t>()));
+  PADDLE_ENFORCE_EQ(
+      TestDtype<uint8_t>(),
+      paddle::DataType::UINT8,
+      common::errors::InvalidArgument("TestDtype<uint8_t>() should be equal to "
+                                      "paddle::DataType::UINT8, but got %s",
+                                      TestDtype<uint8_t>()));
+  PADDLE_ENFORCE_EQ(
+      TestDtype<int16_t>(),
+      paddle::DataType::INT16,
+      common::errors::InvalidArgument("TestDtype<int16_t>() should be equal to "
+                                      "paddle::DataType::INT16, but got %s",
+                                      TestDtype<int16_t>()));
+  PADDLE_ENFORCE_EQ(
+      TestDtype<int32_t>(),
+      paddle::DataType::INT32,
+      common::errors::InvalidArgument("TestDtype<int32_t>() should be equal to "
+                                      "paddle::DataType::INT32, but got %s",
+                                      TestDtype<int32_t>()));
+  PADDLE_ENFORCE_EQ(
+      TestDtype<int64_t>(),
+      paddle::DataType::INT64,
+      common::errors::InvalidArgument("TestDtype<int64_t>() should be equal to "
+                                      "paddle::DataType::INT64, but got %s",
+                                      TestDtype<int64_t>()));
+  PADDLE_ENFORCE_EQ(TestDtype<paddle::float16>(),
+                    paddle::DataType::FLOAT16,
+                    common::errors::InvalidArgument(
+                        "TestDtype<paddle::float16>() should be equal to "
+                        "paddle::DataType::FLOAT16, but got %s",
+                        TestDtype<paddle::float16>()));
+  PADDLE_ENFORCE_EQ(
+      TestDtype<float>(),
+      paddle::DataType::FLOAT32,
+      common::errors::InvalidArgument("TestDtype<float>() should be equal to "
+                                      "paddle::DataType::FLOAT32, but got %s",
+                                      TestDtype<float>()));
+  PADDLE_ENFORCE_EQ(
+      TestDtype<double>(),
+      paddle::DataType::FLOAT64,
+      common::errors::InvalidArgument("TestDtype<double>() should be equal to "
+                                      "paddle::DataType::FLOAT64, but got %s",
+                                      TestDtype<double>()));
+  PADDLE_ENFORCE_EQ(TestDtype<paddle::complex64>(),
+                    paddle::DataType::COMPLEX64,
+                    common::errors::InvalidArgument(
+                        "TestDtype<paddle::complex64>() should be equal to "
+                        "paddle::DataType::COMPLEX64, but got %s",
+                        TestDtype<paddle::complex64>()));
+  PADDLE_ENFORCE_EQ(TestDtype<paddle::complex128>(),
+                    paddle::DataType::COMPLEX128,
+                    common::errors::InvalidArgument(
+                        "TestDtype<paddle::complex128>() should be equal to "
+                        "paddle::DataType::COMPLEX128, but got %s",
+                        TestDtype<paddle::complex128>()));
+}
+
+void TestInitialized() {
+  auto test_tensor = paddle::experimental::empty({1, 1});
+  PADDLE_ENFORCE_EQ(test_tensor.initialized(),
+                    true,
+                    common::errors::InvalidArgument(
+                        "test_tensor should be initialized, but got %s",
+                        test_tensor.initialized()));
+  float* tensor_data = test_tensor.data<float>();
+  for (int i = 0; i < test_tensor.size(); i++) {
+    tensor_data[i] = 0.5;
+  }
+  for (int i = 0; i < test_tensor.size(); i++) {
+    PADDLE_ENFORCE_EQ(tensor_data[i],
+                      0.5,
+                      common::errors::InvalidArgument(
+                          "tensor_data[%d] should be equal to 0.5, "
+                          "but got %f",
+                          i,
+                          tensor_data[i]));
+  }
+}
+
+void TestDataInterface() {
+  // Test DenseTensor
+  auto test_tensor = paddle::experimental::empty({1, 1});
+  PADDLE_ENFORCE_EQ(test_tensor.initialized(),
+                    true,
+                    common::errors::InvalidArgument(
+                        "test_tensor should be initialized, but got %s",
+                        test_tensor.initialized()));
+  void* tensor_ptr = test_tensor.data();
+  PADDLE_ENFORCE_NE(
+      tensor_ptr,
+      nullptr,
+      common::errors::InvalidArgument(
+          "test_tensor should not be NULL, but got %p", tensor_ptr));
+  const void* const_tensor_ptr = test_tensor.data();
+  PADDLE_ENFORCE_NE(
+      const_tensor_ptr,
+      nullptr,
+      common::errors::InvalidArgument("const_tensor should not be NULL, "
+                                      "but got %p",
+                                      const_tensor_ptr));
+  // Test SelectedRows
+  std::vector<int64_t> rows = {0};
+  std::shared_ptr<phi::SelectedRows> selected_rows =
+      std::make_shared<phi::SelectedRows>(rows, 1);
+  selected_rows->mutable_value()->Resize(common::make_ddim({1, 1}));
+  selected_rows->mutable_value()->mutable_data<float>(phi::CPUPlace())[0] =
+      static_cast<float>(10.0f);
+  paddle::Tensor sr_tensor = paddle::Tensor(selected_rows);
+  PADDLE_ENFORCE_EQ(sr_tensor.initialized(),
+                    true,
+                    common::errors::InvalidArgument(
+                        "sr_tensor should be initialized, but got %s",
+                        sr_tensor.initialized()));
+  tensor_ptr = sr_tensor.data();
+  PADDLE_ENFORCE_NE(tensor_ptr,
+                    nullptr,
+                    common::errors::InvalidArgument(
+                        "tensor should not be NULL, but got %p", tensor_ptr));
+  const_tensor_ptr = sr_tensor.data();
+  PADDLE_ENFORCE_NE(
+      const_tensor_ptr,
+      nullptr,
+      common::errors::InvalidArgument("const_tensor should not be NULL, "
+                                      "but got %p",
+                                      const_tensor_ptr));
+}
+
+TEST(PhiTensor, All) {
+  VLOG(2) << "TestCopy";
+  GroupTestCopy();
+  VLOG(2) << "TestDtype";
+  GroupTestDtype();
+  VLOG(2) << "TestShape";
+  TestAPISizeAndShape();
+  VLOG(2) << "TestPlace";
+  TestAPIPlace();
+  VLOG(2) << "TestSlice";
+  TestAPISlice();
+  VLOG(2) << "TestCast";
+  GroupTestCast();
+  VLOG(2) << "TestInitialized";
+  TestInitialized();
+  VLOG(2) << "TestDataInterface";
+  TestDataInterface();
+}
+
+}  // namespace tests
+}  // namespace paddle

test/cpp/phi/api/test_scale_benchmark.cc

@@ -0,0 +1,64 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+
+#include <memory>
+
+#include "paddle/phi/api/include/api.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "test/cpp/phi/api/scale_api.h"
+#include "test/cpp/phi/core/timer.h"
+
+PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
+
+namespace paddle {
+namespace tests {
+
+TEST(API, scale) {
+  auto x = experimental::full({3, 4}, 1.0, phi::DataType::FLOAT32, CPUPlace());
+
+  const size_t cycles = 300;
+  phi::tests::Timer timer;
+  double t1{}, t2{}, t3{};
+
+  for (size_t i = 0; i < cycles; ++i) {
+    timer.tic();
+    for (size_t i = 0; i < cycles; ++i) {
+      auto out = experimental::scale_kernel_context(x, 2.0, 1.0, true);
+    }
+    t1 += timer.toc();
+
+    timer.tic();
+    for (size_t i = 0; i < cycles; ++i) {
+      auto out = experimental::scale(x, 2.0, 1.0, true);
+    }
+    t2 += timer.toc();
+
+    timer.tic();
+    for (size_t i = 0; i < cycles; ++i) {
+      auto out = experimental::scale_switch_case(x, 2.0, 1.0, true);
+    }
+    t3 += timer.toc();
+  }
+
+  LOG(INFO) << "The cost of kernel_context is " << t1 << "ms.";
+  LOG(INFO) << "The cost of variadic_args_kernel_fn is " << t2 << "ms.";
+  LOG(INFO) << "The cost of switch_case is " << t3 << "ms.";
+}
+
+}  // namespace tests
+}  // namespace paddle

test/cpp/phi/api/test_slice_api.cc

@@ -0,0 +1,47 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+
+#include <memory>
+
+#include "paddle/phi/api/include/api.h"
+#include "paddle/phi/api/include/tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
+
+namespace paddle {
+namespace tests {
+
+TEST(Tensor, slice) {
+  auto x = paddle::experimental::full({4, 3}, 1, phi::DataType::INT64);
+  auto slice_x = x.slice(1, 2);
+
+  // check slice result
+  ASSERT_EQ(slice_x.dims().size(), 2);
+  ASSERT_EQ(slice_x.dims()[0], 1);
+  ASSERT_EQ(slice_x.dims()[1], 3);
+  ASSERT_EQ(slice_x.numel(), 3);
+  ASSERT_EQ(slice_x.is_cpu(), true);
+  ASSERT_EQ(slice_x.type(), phi::DataType::INT64);
+  ASSERT_EQ(slice_x.layout(), phi::DataLayout::NCHW);
+  ASSERT_EQ(slice_x.initialized(), true);
+  for (int64_t i = 0; i < slice_x.numel(); ++i) {
+    ASSERT_EQ(slice_x.mutable_data<int64_t>()[i], 1);
+  }
+}
+
+}  // namespace tests
+}  // namespace paddle

test/cpp/phi/api/test_strings_empty_api.cc

@@ -0,0 +1,87 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See
+the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+
+#include <memory>
+
+#include "paddle/phi/api/include/strings_api.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/common/backend.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/string_tensor.h"
+
+PD_DECLARE_KERNEL(strings_empty, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(strings_empty_like, CPU, ALL_LAYOUT);
+
+namespace paddle {
+namespace tests {
+
+using phi::CPUPlace;
+using phi::StringTensor;
+using phi::StringTensorMeta;
+
+TEST(API, strings_empty) {
+  // 1. create tensor
+  auto cpu = CPUPlace();
+  const auto alloc =
+      std::make_shared<paddle::experimental::DefaultAllocator>(cpu);
+
+  auto dense_shape = std::make_shared<phi::DenseTensor>(
+      alloc.get(),
+      phi::DenseTensorMeta(
+          phi::DataType::INT64, common::make_ddim({2}), phi::DataLayout::NCHW));
+  auto* dev_ctx =
+      phi::DeviceContextPool::Instance().GetByPlace(phi::CPUPlace());
+  auto* shape_data = dev_ctx->template Alloc<int64_t>(dense_shape.get());
+
+  shape_data[0] = 2;
+  shape_data[1] = 3;
+
+  paddle::Tensor tensor_shape(dense_shape);
+
+  // 2. test API
+  auto empty_out = paddle::experimental::strings::empty(tensor_shape);
+
+  // 3. check result
+  ASSERT_EQ(empty_out.dims().size(), 2);
+  ASSERT_EQ(empty_out.dims()[0], 2);
+  ASSERT_EQ(empty_out.dims()[1], 3);
+  ASSERT_EQ(empty_out.numel(), 6);
+}
+
+TEST(API, strings_empty_like) {
+  auto cpu = CPUPlace();
+  const auto alloc =
+      std::make_shared<paddle::experimental::DefaultAllocator>(cpu);
+  // 1. create tensor
+  const phi::DDim dims({1, 2});
+  StringTensorMeta meta(dims);
+  auto cpu_strings_x = std::make_shared<phi::StringTensor>(
+      alloc.get(), phi::StringTensorMeta(meta));
+
+  // 2. test API
+  paddle::Tensor x(cpu_strings_x);
+  auto empty_like_out = paddle::experimental::strings::empty_like(x);
+
+  // 3. check result
+  ASSERT_EQ(empty_like_out.dims().size(), 2);
+  ASSERT_EQ(empty_like_out.dims()[0], 1);
+  ASSERT_EQ(empty_like_out.numel(), 2);
+}
+
+}  // namespace tests
+}  // namespace paddle

test/cpp/phi/api/test_strings_lower_upper_api.cc

@@ -0,0 +1,147 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See
+the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+
+#include <memory>
+
+#include "paddle/phi/api/include/strings_api.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/common/pstring.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/string_tensor.h"
+
+PD_DECLARE_KERNEL(strings_lower, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(strings_upper, CPU, ALL_LAYOUT);
+
+namespace paddle {
+namespace tests {
+
+using phi::CPUPlace;
+using phi::StringTensor;
+using phi::StringTensorMeta;
+
+TEST(API, case_convert) {
+  auto cpu = CPUPlace();
+  const auto alloc =
+      std::make_shared<paddle::experimental::DefaultAllocator>(cpu);
+  // 1. create tensor
+  const phi::DDim dims({1, 2});
+  StringTensorMeta meta(dims);
+  auto cpu_strings_x = std::make_shared<phi::StringTensor>(
+      alloc.get(), phi::StringTensorMeta(meta));
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* dev_ctx = pool.Get(phi::CPUPlace());
+
+  pstring* cpu_strings_x_data =
+      dev_ctx->template Alloc<pstring>(cpu_strings_x.get());
+  std::string strs[] = {"A Short Pstring.",  // NOLINT
+                        "A Large Pstring Whose Length Is Longer Than 22."};
+  for (int i = 0; i < 2; ++i) {
+    cpu_strings_x_data[i] = strs[i];
+  }
+  // 2. get expected results
+  std::string expected_results[] = {// NOLINT
+                                    strs[0],
+                                    strs[0],
+                                    strs[1],
+                                    strs[1]};
+  std::transform(
+      strs[0].begin(), strs[0].end(), expected_results[0].begin(), ::tolower);
+  std::transform(
+      strs[0].begin(), strs[0].end(), expected_results[1].begin(), ::toupper);
+  std::transform(
+      strs[1].begin(), strs[1].end(), expected_results[2].begin(), ::tolower);
+  std::transform(
+      strs[1].begin(), strs[1].end(), expected_results[3].begin(), ::toupper);
+  // 3. test API, ascii encoding
+  paddle::Tensor x(cpu_strings_x);
+  auto lower_out = paddle::experimental::strings::lower(x, false);
+  auto upper_out = paddle::experimental::strings::upper(x, false);
+
+  auto lower_tensor =
+      std::dynamic_pointer_cast<phi::StringTensor>(lower_out.impl());
+  auto upper_tensor =
+      std::dynamic_pointer_cast<phi::StringTensor>(upper_out.impl());
+  ASSERT_EQ(lower_tensor->dims(), dims);
+  ASSERT_EQ(upper_tensor->dims(), dims);
+
+  auto lower_tensor_ptr = lower_tensor->data();
+  auto upper_tensor_ptr = upper_tensor->data();
+
+  const std::string cpu_results[] = {// NOLINT
+                                     lower_tensor_ptr[0].data(),
+                                     upper_tensor_ptr[0].data(),
+                                     lower_tensor_ptr[1].data(),
+                                     upper_tensor_ptr[1].data()};
+
+  for (int i = 0; i < 4; ++i) {
+    ASSERT_EQ(cpu_results[i], expected_results[i]);
+  }
+}
+
+TEST(API, case_convert_utf8) {
+  auto cpu = CPUPlace();
+  const auto alloc =
+      std::make_shared<paddle::experimental::DefaultAllocator>(cpu);
+  // 1. create tensor
+  const phi::DDim dims({1, 2});
+  StringTensorMeta meta(dims);
+  auto cpu_strings_x = std::make_shared<phi::StringTensor>(
+      alloc.get(), phi::StringTensorMeta(meta));
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* dev_ctx = pool.Get(phi::CPUPlace());
+
+  pstring* cpu_strings_x_data =
+      dev_ctx->template Alloc<pstring>(cpu_strings_x.get());
+  std::string strs[] = {"óÓsscHloëË",  // NOLINT
+                        "óÓsscHloëËóÓsscHloëËóÓsscHloëË"};
+  for (int i = 0; i < 2; ++i) {
+    cpu_strings_x_data[i] = strs[i];
+  }
+  // 2. get expected results
+  std::string expected_results[] = {// NOLINT
+                                    "óósschloëë",
+                                    "ÓÓSSCHLOËË",
+                                    "óósschloëëóósschloëëóósschloëë",
+                                    "ÓÓSSCHLOËËÓÓSSCHLOËËÓÓSSCHLOËË"};
+  // 3. test API, ascii encoding
+  paddle::Tensor x(cpu_strings_x);
+  auto lower_out = paddle::experimental::strings::lower(x, true);
+  auto upper_out = paddle::experimental::strings::upper(x, true);
+
+  auto lower_tensor =
+      std::dynamic_pointer_cast<phi::StringTensor>(lower_out.impl());
+  auto upper_tensor =
+      std::dynamic_pointer_cast<phi::StringTensor>(upper_out.impl());
+  ASSERT_EQ(lower_tensor->dims(), dims);
+  ASSERT_EQ(upper_tensor->dims(), dims);
+
+  auto lower_tensor_ptr = lower_tensor->data();
+  auto upper_tensor_ptr = upper_tensor->data();
+
+  const char* cpu_results[] = {// NOLINT
+                               lower_tensor_ptr[0].data(),
+                               upper_tensor_ptr[0].data(),
+                               lower_tensor_ptr[1].data(),
+                               upper_tensor_ptr[1].data()};
+
+  for (int i = 0; i < 4; ++i) {
+    ASSERT_EQ(std::string(cpu_results[i]), expected_results[i]);
+  }
+}
+
+}  // namespace tests
+}  // namespace paddle

test/cpp/phi/api/test_to_api.cc

@@ -0,0 +1,96 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+
+#include <memory>
+
+#include "paddle/phi/api/include/api.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+namespace paddle {
+namespace tests {
+
+using DDim = phi::DDim;
+
+paddle::Tensor CreateInputTensor() {
+  const auto alloc =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace());
+  auto dense_x = std::make_shared<phi::DenseTensor>(
+      alloc.get(),
+      phi::DenseTensorMeta(phi::DataType::INT64,
+                           common::make_ddim({3, 4}),
+                           phi::DataLayout::NCHW));
+  auto* dev_ctx =
+      phi::DeviceContextPool::Instance().GetByPlace(phi::CPUPlace());
+  auto* dense_x_data = dev_ctx->template Alloc<int64_t>(dense_x.get());
+
+  for (int64_t i = 0; i < 12; ++i) {
+    dense_x_data[i] = i;
+  }
+
+  return paddle::Tensor(dense_x);
+}
+
+void CheckOutputResult(const paddle::Tensor& out) {
+  ASSERT_EQ(out.dims().size(), 2);
+  ASSERT_EQ(out.dims()[0], 3);
+  ASSERT_EQ(out.dims()[1], 4);
+  ASSERT_EQ(out.is_cpu(), true);
+  ASSERT_EQ(out.type(), phi::DataType::INT64);
+  ASSERT_EQ(out.layout(), phi::DataLayout::NCHW);
+  ASSERT_EQ(out.initialized(), true);
+
+  for (int64_t i = 0; i < 12; ++i) {
+    ASSERT_EQ(out.data<int64_t>()[i], i);
+  }
+}
+
+TEST(API, copy_to) {
+  // 1. create tensor
+  auto x = CreateInputTensor();
+
+// 2. test API
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  auto tmp = paddle::experimental::copy_to(x, phi::GPUPlace(), false);
+  auto out = paddle::experimental::copy_to(tmp, phi::CPUPlace(), true);
+#else
+  auto out = paddle::experimental::copy_to(x, phi::CPUPlace(), false);
+#endif
+
+  // 3. check result
+  CheckOutputResult(out);
+}
+
+TEST(Tensor, copy_to) {
+  // 1. create tensor
+  auto x = CreateInputTensor();
+
+// 2. test API
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  auto tmp = x.copy_to(phi::GPUPlace(), false);
+  auto out = tmp.copy_to(phi::CPUPlace(), true);
+#else
+  auto out = x.copy_to(phi::CPUPlace(), false);
+#endif
+
+  // 3. check result
+  CheckOutputResult(out);
+}
+
+}  // namespace tests
+}  // namespace paddle

test/cpp/phi/backends/CMakeLists.txt

@@ -0,0 +1,3 @@
+if(WITH_CUSTOM_DEVICE)
+  paddle_test(capi_test SRCS custom/capi_test.cc DEPS phi common)
+endif()

test/cpp/phi/backends/custom/capi_test.cc

@@ -0,0 +1,78 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include <cstring>
+#include <string>
+
+#include "paddle/phi/capi/all.h"
+
+#ifndef UNUSED
+#define UNUSED __attribute__((unused))
+#endif
+
+#include "paddle/phi/capi/capi.h"
+
+TEST(CustomKernel, CAPI) {
+  std::string str = "capi";
+  EXPECT_EQ(str.data(), PD_StringAttr(&str));
+
+  std::vector<int32_t> int32_vec({1, 2, 3});
+  auto int32_list = PD_ListInt32Attr(&int32_vec);
+  EXPECT_EQ(int32_list.data, int32_vec.data());
+  EXPECT_EQ(int32_list.size, int32_vec.size());
+
+  std::vector<int64_t> int64_vec({1, 2, 3});
+  auto int64_list = PD_ListInt64Attr(&int64_vec);
+  EXPECT_EQ(int64_list.data, int64_vec.data());
+  EXPECT_EQ(int64_list.size, int64_vec.size());
+
+  std::vector<float> float_vec({1, 2, 3});
+  auto float_list = PD_ListFloatAttr(&float_vec);
+  EXPECT_EQ(float_list.data, float_vec.data());
+  EXPECT_EQ(float_list.size, float_vec.size());
+
+  std::vector<double> double_vec({1, 2, 3});
+  auto double_list = PD_ListDoubleAttr(&double_vec);
+  EXPECT_EQ(double_list.data, double_vec.data());
+  EXPECT_EQ(double_list.size, double_vec.size());
+
+  std::vector<std::string> string_vec{"capi", "api"};
+  auto string_list = PD_ListStringAttr(&string_vec);
+  auto string_data = reinterpret_cast<void**>(string_list.data);
+  for (size_t i = 0; i < string_vec.size(); ++i) {
+    EXPECT_EQ(string_data[i], string_vec[i].data());
+  }
+
+  std::vector<bool> bool_vec{true, false, true};
+  auto bool_list = PD_ListBoolAttr(&bool_vec);
+  auto bool_data = reinterpret_cast<uint8_t*>(bool_list.data);
+  for (size_t i = 0; i < bool_vec.size(); ++i) {
+    EXPECT_EQ(bool_data[i], static_cast<uint8_t>(bool_vec[i]));
+  }
+
+  std::vector<float*> ptr_vec;
+  for (size_t i = 0; i < float_vec.size(); ++i) {
+    ptr_vec.push_back(&float_vec[i]);
+  }
+  auto ptr_list = PD_TensorVectorToList(reinterpret_cast<PD_Tensor*>(&ptr_vec));
+  EXPECT_EQ(ptr_list.data, ptr_vec.data());
+  EXPECT_EQ(ptr_list.size, ptr_vec.size());
+}
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

test/cpp/phi/common/CMakeLists.txt

@@ -0,0 +1,44 @@
+cc_test(
+  phi_test_backend
+  SRCS test_backend.cc
+  DEPS gtest)
+cc_test(
+  phi_test_data_layout
+  SRCS test_data_layout.cc
+  DEPS gtest)
+cc_test(
+  phi_test_data_type
+  SRCS test_data_type.cc
+  DEPS gtest)
+cc_test(
+  phi_test_place
+  SRCS test_place.cc
+  DEPS phi common)
+cc_test(
+  phi_test_int_array
+  SRCS test_int_array.cc
+  DEPS phi common)
+cc_test(
+  phi_test_scalar_cpu
+  SRCS test_scalar.cc
+  DEPS phi common)
+if(WITH_GPU)
+  nv_test(
+    phi_test_scalar
+    SRCS test_scalar.cu
+    DEPS phi common)
+  nv_test(
+    transform_test
+    SRCS transform_test.cu
+    DEPS phi common)
+endif()
+if(WITH_ROCM)
+  hip_test(
+    phi_test_scalar
+    SRCS test_scalar.cu
+    DEPS phi common)
+  hip_test(
+    transform_test
+    SRCS transform_test.cu
+    DEPS phi common)
+endif()

test/cpp/phi/common/test_backend.cc

@@ -0,0 +1,77 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+
+#include <iostream>
+
+#include "paddle/common/exception.h"
+#include "paddle/phi/common/backend.h"
+
+namespace phi {
+namespace tests {
+
+TEST(Backend, OStream) {
+  std::ostringstream oss;
+  oss << Backend::UNDEFINED;
+  EXPECT_EQ(oss.str(), "Undefined");
+  oss.str("");
+  oss << Backend::CPU;
+  EXPECT_EQ(oss.str(), "CPU");
+  oss.str("");
+  oss << Backend::GPU;
+  EXPECT_EQ(oss.str(), "GPU");
+  oss.str("");
+  oss << Backend::XPU;
+  EXPECT_EQ(oss.str(), "XPU");
+  oss.str("");
+  oss << Backend::ONEDNN;
+  EXPECT_EQ(oss.str(), "ONEDNN");
+  oss.str("");
+  oss << Backend::GPUDNN;
+  EXPECT_EQ(oss.str(), "GPUDNN");
+  oss.str("");
+  oss << Backend::KPS;
+  EXPECT_EQ(oss.str(), "KPS");
+  oss.str("");
+  try {
+    oss << Backend::NUM_BACKENDS;
+  } catch (const std::exception& exception) {
+    std::string ex_msg = exception.what();
+    EXPECT_TRUE(ex_msg.find("Invalid enum backend type") != std::string::npos);
+  }
+}
+
+TEST(Backend, StringToBackend) {
+  using paddle::experimental::StringToBackend;
+  EXPECT_EQ(Backend::UNDEFINED, StringToBackend("Undefined"));
+  EXPECT_EQ(Backend::CPU, StringToBackend("CPU"));
+  EXPECT_EQ(Backend::GPU, StringToBackend("GPU"));
+  EXPECT_EQ(Backend::XPU, StringToBackend("XPU"));
+  EXPECT_EQ(Backend::ONEDNN, StringToBackend("OneDNN"));
+  EXPECT_EQ(Backend::GPUDNN, StringToBackend("GPUDNN"));
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  EXPECT_EQ(Backend::GPU, StringToBackend("KPS"));
+#else
+  EXPECT_EQ(Backend::KPS, StringToBackend("KPS"));
+#endif
+  EXPECT_EQ(static_cast<Backend>(
+                static_cast<size_t>(Backend::NUM_BACKENDS) +
+                phi::CustomRegisteredDeviceMap::Instance()
+                    .GetOrRegisterGlobalDeviceTypeId("CustomBackend")),
+            StringToBackend("CustomBackend"));
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/common/test_data_layout.cc

@@ -0,0 +1,52 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+
+#include <iostream>
+#include <sstream>
+
+#include "paddle/common/exception.h"
+#include "paddle/common/layout.h"
+
+namespace phi {
+namespace tests {
+
+TEST(DataLayout, OStream) {
+  std::ostringstream oss;
+  oss << DataLayout::UNDEFINED;
+  EXPECT_EQ(oss.str(), "Undefined(AnyLayout)");
+  oss.str("");
+  oss << DataLayout::ANY;
+  EXPECT_EQ(oss.str(), "Undefined(AnyLayout)");
+  oss.str("");
+  oss << DataLayout::NHWC;
+  EXPECT_EQ(oss.str(), "NHWC");
+  oss.str("");
+  oss << DataLayout::NCHW;
+  EXPECT_EQ(oss.str(), "NCHW");
+  oss.str("");
+  oss << DataLayout::ONEDNN;
+  EXPECT_EQ(oss.str(), "ONEDNN");
+  oss.str("");
+  try {
+    oss << DataLayout::NUM_DATA_LAYOUTS;
+  } catch (const std::exception& exception) {
+    std::string ex_msg = exception.what();
+    EXPECT_TRUE(ex_msg.find("Unknown Data Layout type") != std::string::npos);
+  }
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/common/test_data_type.cc

@@ -0,0 +1,90 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+
+#include <iostream>
+#include <sstream>
+
+#include "paddle/common/exception.h"
+#include "paddle/phi/common/data_type.h"
+#include "paddle/phi/common/type_traits.h"
+
+namespace phi {
+namespace tests {
+
+TEST(DataType, OStream) {
+  std::ostringstream oss;
+  oss << DataType::UNDEFINED;
+  EXPECT_EQ(oss.str(), "Undefined");
+  oss.str("");
+  oss << DataType::BOOL;
+  EXPECT_EQ(oss.str(), "bool");
+  oss.str("");
+  oss << DataType::INT8;
+  EXPECT_EQ(oss.str(), "int8");
+  oss.str("");
+  oss << DataType::UINT8;
+  EXPECT_EQ(oss.str(), "uint8");
+  oss.str("");
+  oss << DataType::INT16;
+  EXPECT_EQ(oss.str(), "int16");
+  oss.str("");
+  oss << DataType::INT32;
+  EXPECT_EQ(oss.str(), "int32");
+  oss.str("");
+  oss << DataType::INT64;
+  EXPECT_EQ(oss.str(), "int64");
+  oss.str("");
+  oss << DataType::BFLOAT16;
+  EXPECT_EQ(oss.str(), "bfloat16");
+  oss.str("");
+  oss << DataType::FLOAT16;
+  EXPECT_EQ(oss.str(), "float16");
+  oss.str("");
+  oss << DataType::FLOAT32;
+  EXPECT_EQ(oss.str(), "float32");
+  oss.str("");
+  oss << DataType::FLOAT64;
+  EXPECT_EQ(oss.str(), "float64");
+  oss.str("");
+  oss << DataType::COMPLEX64;
+  EXPECT_EQ(oss.str(), "complex64");
+  oss.str("");
+  oss << DataType::COMPLEX128;
+  EXPECT_EQ(oss.str(), "complex128");
+  oss.str("");
+  oss << DataType::PSTRING;
+  EXPECT_EQ(oss.str(), "pstring");
+  oss.str("");
+  try {
+    oss << DataType::NUM_DATA_TYPES;
+  } catch (const std::exception& exception) {
+    std::string ex_msg = exception.what();
+    EXPECT_TRUE(ex_msg.find("Invalid enum data type") != std::string::npos);
+  }
+}
+
+TEST(TypeTraits, Complex) {
+  EXPECT_EQ(dtype::ToReal(DataType::COMPLEX64), DataType::FLOAT32);
+  EXPECT_EQ(dtype::ToReal(DataType::COMPLEX128), DataType::FLOAT64);
+  EXPECT_EQ(dtype::ToReal(DataType::FLOAT32), DataType::FLOAT32);
+
+  EXPECT_EQ(dtype::ToComplex(DataType::FLOAT32), DataType::COMPLEX64);
+  EXPECT_EQ(dtype::ToComplex(DataType::FLOAT64), DataType::COMPLEX128);
+  EXPECT_EQ(dtype::ToComplex(DataType::COMPLEX64), DataType::COMPLEX64);
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/common/test_int_array.cc

@@ -0,0 +1,154 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "gtest/gtest.h"
+#include "paddle/phi/api/include/api.h"
+#include "paddle/phi/api/include/context_pool.h"
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/common/int_array.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/full_kernel.h"
+
+PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+PD_DECLARE_KERNEL(full, GPU, ALL_LAYOUT);
+#endif
+
+namespace phi {
+namespace tests {
+
+TEST(IntArray, ConstructFromCPUDenseTensor) {
+  auto& pool = paddle::experimental::DeviceContextPool::Instance();
+  const auto* dev_ctx = static_cast<const CPUContext*>(pool.Get(CPUPlace()));
+  DenseTensor shape = Full<int>(*dev_ctx, {2}, 3);
+  DenseTensor out = Full<int>(*dev_ctx, shape, 1);
+  ASSERT_EQ(out.dims().size(), 2);
+  ASSERT_EQ(out.dims()[0], 3);
+  ASSERT_EQ(out.dims()[1], 3);
+  ASSERT_EQ(out.numel(), 9);
+}
+
+TEST(IntArray, ConstructFromCPUDenseTensorVector) {
+  auto& pool = paddle::experimental::DeviceContextPool::Instance();
+  const auto* dev_ctx = static_cast<const CPUContext*>(pool.Get(CPUPlace()));
+  DenseTensor shape0 = Full<int>(*dev_ctx, {1}, 3);
+  DenseTensor shape1 = Full<int64_t>(*dev_ctx, {1}, 3);
+  std::vector<DenseTensor> shape{shape0, shape1};
+  DenseTensor out = Full<int>(*dev_ctx, shape, 1);
+  ASSERT_EQ(out.dims().size(), 2);
+  ASSERT_EQ(out.dims()[0], 3);
+  ASSERT_EQ(out.dims()[1], 3);
+  ASSERT_EQ(out.numel(), 9);
+}
+
+TEST(IntArray, ConstructFromCPUTensor) {
+  auto shape = paddle::experimental::full({2}, 3, DataType::INT64);
+  auto out = paddle::experimental::full(shape, 1);
+  ASSERT_EQ(out.dims().size(), 2);
+  ASSERT_EQ(out.dims()[0], 3);
+  ASSERT_EQ(out.dims()[1], 3);
+  ASSERT_EQ(out.numel(), 9);
+}
+
+TEST(IntArray, ConstructFromCPUTensorVector) {
+  auto shape0 = paddle::experimental::full({2}, 3, DataType::INT64);
+  auto shape1 = paddle::experimental::full({2}, 3, DataType::INT32);
+
+  std::vector<paddle::Tensor> shape{shape0, shape0};
+  auto out = paddle::experimental::full(shape, 1);
+
+  std::vector<paddle::Tensor> shape_new{shape0, shape1};
+  auto out1 = paddle::experimental::full(shape_new, 1);
+
+  ASSERT_EQ(out.dims().size(), 2);
+  ASSERT_EQ(out.dims()[0], 3);
+  ASSERT_EQ(out.dims()[1], 3);
+  ASSERT_EQ(out.numel(), 9);
+
+  ASSERT_EQ(out1.dims().size(), 2);
+  ASSERT_EQ(out1.dims()[0], 3);
+  ASSERT_EQ(out1.dims()[1], 3);
+  ASSERT_EQ(out1.numel(), 9);
+}
+
+TEST(IntArray, ThrowException) {
+  auto shape = paddle::experimental::full({2}, 3, DataType::FLOAT32);
+  auto create_int_array = [&shape]() -> paddle::experimental::IntArray {
+    paddle::experimental::IntArray int_array{shape};
+    return int_array;
+  };
+  ASSERT_ANY_THROW(create_int_array());
+}
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+TEST(IntArray, ConstructFromGPUDenseTensor) {
+  auto& pool = paddle::experimental::DeviceContextPool::Instance();
+  const auto* dev_ctx =
+      static_cast<const phi::GPUContext*>(pool.Get(GPUPlace()));
+  DenseTensor shape = Full<int>(*dev_ctx, {2}, 3);
+  DenseTensor out = Full<int>(*dev_ctx, shape, 1);
+  ASSERT_EQ(out.dims().size(), 2);
+  ASSERT_EQ(out.dims()[0], 3);
+  ASSERT_EQ(out.dims()[1], 3);
+  ASSERT_EQ(out.numel(), 9);
+}
+
+TEST(IntArray, ConstructFromGPUDenseTensorVector) {
+  auto& pool = paddle::experimental::DeviceContextPool::Instance();
+  const auto* dev_ctx =
+      static_cast<const phi::GPUContext*>(pool.Get(GPUPlace()));
+  DenseTensor shape0 = Full<int>(*dev_ctx, {1}, 3);
+  DenseTensor shape1 = Full<int64_t>(*dev_ctx, {1}, 3);
+  std::vector<DenseTensor> shape{shape0, shape1};
+  DenseTensor out = Full<int>(*dev_ctx, shape, 1);
+  ASSERT_EQ(out.dims().size(), 2);
+  ASSERT_EQ(out.dims()[0], 3);
+  ASSERT_EQ(out.dims()[1], 3);
+  ASSERT_EQ(out.numel(), 9);
+}
+
+TEST(IntArray, ConstructFromGPUTensor) {
+  auto shape = paddle::experimental::full({2}, 3, DataType::INT64, GPUPlace());
+  auto out = paddle::experimental::full(shape, 1);
+  ASSERT_EQ(out.dims().size(), 2);
+  ASSERT_EQ(out.dims()[0], 3);
+  ASSERT_EQ(out.dims()[1], 3);
+  ASSERT_EQ(out.numel(), 9);
+}
+
+TEST(IntArray, ConstructFromGPUTensorVector) {
+  auto shape0 = paddle::experimental::full({2}, 3, DataType::INT64, GPUPlace());
+  auto shape1 = paddle::experimental::full({2}, 3, DataType::INT32, GPUPlace());
+
+  std::vector<paddle::Tensor> shape{shape0, shape0};
+  auto out = paddle::experimental::full(shape, 1);
+
+  std::vector<paddle::Tensor> shape_new{shape0, shape1};
+  auto out1 = paddle::experimental::full(shape_new, 1);
+
+  ASSERT_EQ(out.dims().size(), 2);
+  ASSERT_EQ(out.dims()[0], 3);
+  ASSERT_EQ(out.dims()[1], 3);
+  ASSERT_EQ(out.numel(), 9);
+
+  ASSERT_EQ(out1.dims().size(), 2);
+  ASSERT_EQ(out1.dims()[0], 3);
+  ASSERT_EQ(out1.dims()[1], 3);
+  ASSERT_EQ(out1.numel(), 9);
+}
+#endif
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/common/test_place.cc

@@ -0,0 +1,82 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <map>  // NOLINT
+
+#include "gtest/gtest.h"
+#include "paddle/phi/common/place.h"
+
+namespace phi {
+namespace tests {
+
+TEST(PhiPlace, place) {
+  Place place;
+  EXPECT_EQ(place.GetType(), AllocationType::UNDEFINED);
+
+  place.Reset(AllocationType::GPU, 1);
+  EXPECT_EQ(place.GetType(), AllocationType::GPU);
+  EXPECT_EQ(place.GetDeviceId(), 1);
+}
+
+TEST(Place, cpu_place) {
+  CPUPlace place;
+  EXPECT_EQ(place.GetType(), AllocationType::CPU);
+  std::cout << "cpu place repr: " << place << std::endl;
+}
+
+TEST(Place, gpu_place) {
+  GPUPlace place;
+  EXPECT_EQ(place.GetType(), AllocationType::GPU);
+  EXPECT_EQ(place.GetDeviceId(), 0);
+
+  GPUPlace place1(2);
+  EXPECT_EQ(place1.GetType(), AllocationType::GPU);
+  EXPECT_EQ(place1.GetDeviceId(), 2);
+  std::cout << "gpu place repr: " << place1 << std::endl;
+
+  GPUPinnedPlace place2;
+  EXPECT_EQ(place2.GetType(), AllocationType::GPUPINNED);
+  std::cout << "gpu pinned place repr: " << place2 << std::endl;
+
+  EXPECT_NE(place2, CPUPlace());
+}
+
+TEST(Place, convert_place) {
+  Place base_place(AllocationType::CPU);
+  CPUPlace cpu_place = base_place;
+  EXPECT_EQ(cpu_place.GetType(), base_place.GetType());
+  base_place.Reset(AllocationType::GPU, 2);
+  GPUPlace gpu_place = base_place;
+  EXPECT_EQ(gpu_place.GetType(), base_place.GetType());
+  EXPECT_EQ(gpu_place.GetDeviceId(), base_place.GetDeviceId());
+  Place place = gpu_place;
+  EXPECT_EQ(gpu_place.GetType(), place.GetType());
+  EXPECT_EQ(gpu_place.GetDeviceId(), place.GetDeviceId());
+  place = cpu_place;
+  EXPECT_EQ(cpu_place.GetType(), place.GetType());
+
+  std::map<Place, int> maps;
+  maps[CPUPlace()] = 1;
+  maps[GPUPlace(0)] = 2;
+  maps[GPUPlace(1)] = 3;
+  maps[GPUPlace(2)] = 4;
+  maps[GPUPlace(3)] = 5;
+  maps[GPUPinnedPlace()] = 6;
+  for (auto& map_item : maps) {
+    std::cout << map_item.first << ":" << map_item.second << std::endl;
+  }
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/common/test_scalar.cc

@@ -0,0 +1,139 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <complex>
+#include <sstream>
+#include <string>
+
+#include "gtest/gtest.h"
+#include "paddle/phi/common/scalar.h"
+
+namespace phi {
+namespace tests {
+
+bool StartsWith(const std::string& s, const std::string& prefix) {
+  return s.rfind(prefix, 0) == 0;
+}
+
+TEST(Scalar, Formatting) {
+  paddle::experimental::Scalar s;
+
+  s = paddle::experimental::Scalar(static_cast<float>(42.1));
+  ASSERT_PRED2(StartsWith, s.ToString(), "Scalar(float32(");
+
+  s = paddle::experimental::Scalar(static_cast<double>(42.1));
+  ASSERT_PRED2(StartsWith, s.ToString(), "Scalar(float64(");
+
+  s = paddle::experimental::Scalar(static_cast<int>(42.1));
+  ASSERT_PRED2(StartsWith, s.ToString(), "Scalar(int32(");
+
+  s = paddle::experimental::Scalar(static_cast<int64_t>(42.1));
+  ASSERT_PRED2(StartsWith, s.ToString(), "Scalar(int64(");
+
+  s = paddle::experimental::Scalar(static_cast<bool>(true));
+  ASSERT_PRED2(StartsWith, s.ToString(), "Scalar(bool(");
+
+  s = paddle::experimental::Scalar(std::complex<float>(42.1, 42.1));
+  ASSERT_PRED2(StartsWith, s.ToString(), "Scalar(complex64(");
+
+  s = paddle::experimental::Scalar(std::complex<double>(42.1, 42.1));
+  ASSERT_PRED2(StartsWith, s.ToString(), "Scalar(complex128(");
+
+  s = paddle::experimental::Scalar(static_cast<phi::float16>(42.1));
+  ASSERT_PRED2(StartsWith, s.ToString(), "Scalar(float16(");
+
+  s = paddle::experimental::Scalar(static_cast<phi::bfloat16>(42.1));
+  ASSERT_PRED2(StartsWith, s.ToString(), "Scalar(bfloat16(");
+
+  s = paddle::experimental::Scalar(static_cast<int8_t>(42.1));
+  ASSERT_PRED2(StartsWith, s.ToString(), "Scalar(int8(");
+
+  s = paddle::experimental::Scalar(static_cast<int16_t>(42.1));
+  ASSERT_PRED2(StartsWith, s.ToString(), "Scalar(int16(");
+
+  s = paddle::experimental::Scalar(static_cast<uint8_t>(42.1));
+  ASSERT_PRED2(StartsWith, s.ToString(), "Scalar(uint8(");
+
+  s = paddle::experimental::Scalar(static_cast<uint16_t>(42.1));
+  ASSERT_PRED2(StartsWith, s.ToString(), "Scalar(uint16(");
+
+  s = paddle::experimental::Scalar(static_cast<uint32_t>(42.1));
+  ASSERT_PRED2(StartsWith, s.ToString(), "Scalar(uint32(");
+
+  s = paddle::experimental::Scalar(static_cast<uint64_t>(42.1));
+  ASSERT_PRED2(StartsWith, s.ToString(), "Scalar(uint64(");
+
+  std::stringstream ss;
+  s = paddle::experimental::Scalar(static_cast<uint64_t>(42.1));
+  ss << s;
+  ASSERT_PRED2(StartsWith, s.ToString(), "Scalar(uint64(");
+}
+
+TEST(Scalar, Equality) {
+  auto s_bool = paddle::experimental::Scalar(static_cast<bool>(true));
+
+  auto s_int8 = paddle::experimental::Scalar(static_cast<int8_t>(42.1));
+  auto s_int16 = paddle::experimental::Scalar(static_cast<int16_t>(42.1));
+  auto s_int32 = paddle::experimental::Scalar(static_cast<int32_t>(42.1));
+  auto s_int64 = paddle::experimental::Scalar(static_cast<int64_t>(42.1));
+
+  auto s_uint8 = paddle::experimental::Scalar(static_cast<uint8_t>(42.1));
+  auto s_uint16 = paddle::experimental::Scalar(static_cast<uint16_t>(42.1));
+  auto s_uint32 = paddle::experimental::Scalar(static_cast<uint32_t>(42.1));
+  auto s_uint64 = paddle::experimental::Scalar(static_cast<uint64_t>(42.1));
+
+  auto s_float16 =
+      paddle::experimental::Scalar(static_cast<phi::float16>(42.1));
+  auto s_bfloat16 =
+      paddle::experimental::Scalar(static_cast<phi::bfloat16>(42.1));
+  auto s_float = paddle::experimental::Scalar(static_cast<float>(42.1));
+  auto s_double = paddle::experimental::Scalar(static_cast<double>(42.1));
+
+  auto s_cfloat = paddle::experimental::Scalar(std::complex<float>(42.1, 42.1));
+  auto s_cdouble =
+      paddle::experimental::Scalar(std::complex<double>(42.1, 42.1));
+
+  ASSERT_EQ(s_bool, s_bool);
+
+  ASSERT_EQ(s_int8, s_int8);
+  ASSERT_EQ(s_int16, s_int16);
+  ASSERT_EQ(s_int32, s_int32);
+  ASSERT_EQ(s_int64, s_int64);
+
+  ASSERT_EQ(s_uint8, s_uint8);
+  ASSERT_EQ(s_uint16, s_uint16);
+  ASSERT_EQ(s_uint32, s_uint32);
+  ASSERT_EQ(s_uint64, s_uint64);
+
+  ASSERT_EQ(s_float16, s_float16);
+  ASSERT_EQ(s_bfloat16, s_bfloat16);
+  ASSERT_EQ(s_float, s_float);
+  ASSERT_EQ(s_double, s_double);
+
+  ASSERT_EQ(s_cfloat, s_cfloat);
+  ASSERT_EQ(s_cdouble, s_cdouble);
+
+  ASSERT_NE(s_float, s_double);
+}
+
+TEST(Scalar, WrapAsScalars) {
+  std::vector<int32_t> v{1, 2, 3};
+  auto out = paddle::experimental::WrapAsScalars(v);
+  ASSERT_EQ(out[0].dtype(), DataType::INT32);
+  ASSERT_EQ(out[0].to<int32_t>(), 1);
+  ASSERT_EQ(out[1].to<int32_t>(), 2);
+  ASSERT_EQ(out[2].to<int32_t>(), 3);
+}
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/common/test_scalar.cu

@@ -0,0 +1,178 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <map>  // NOLINT
+
+#include "gtest/gtest.h"
+#include "paddle/phi/api/include/tensor.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/common/complex.h"
+#include "paddle/phi/common/float16.h"
+#include "paddle/phi/common/place.h"
+#include "paddle/phi/common/scalar.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+namespace phi {
+namespace tests {
+
+__global__ void FillTensor(float* data) { data[0] = 1; }
+
+TEST(Scalar, ConstructFromDenseTensor1) {
+  // 1. create tensor
+  const auto alloc =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace());
+  phi::DenseTensor dense_x(alloc.get(),
+                           phi::DenseTensorMeta(phi::DataType::FLOAT16,
+                                                common::make_ddim({1}),
+                                                phi::DataLayout::NCHW));
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* dev_ctx = reinterpret_cast<phi::CPUContext*>(pool.Get(phi::CPUPlace()));
+
+  auto* dense_x_data = dev_ctx->Alloc<float16>(&dense_x);
+  dense_x_data[0] = 1;
+  phi::Scalar scalar_test(dense_x);
+  ASSERT_NEAR(1, scalar_test.to<float16>(), 1e-6);
+}
+
+TEST(Scalar, ConstructFromDenseTensor2) {
+  // 1. create tensor
+  const auto alloc =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace());
+  phi::DenseTensor dense_x(
+      alloc.get(),
+      phi::DenseTensorMeta(
+          phi::DataType::INT16, common::make_ddim({1}), phi::DataLayout::NCHW));
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* dev_ctx = reinterpret_cast<phi::CPUContext*>(pool.Get(phi::CPUPlace()));
+
+  auto* dense_x_data = dev_ctx->Alloc<int16_t>(&dense_x);
+  dense_x_data[0] = 1;
+  phi::Scalar scalar_test(dense_x);
+  ASSERT_EQ(1, scalar_test.to<int16_t>());
+}
+
+TEST(Scalar, ConstructFromDenseTensor3) {
+  // 1. create tensor
+  const auto alloc =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace());
+  phi::DenseTensor dense_x(
+      alloc.get(),
+      phi::DenseTensorMeta(
+          phi::DataType::INT8, common::make_ddim({1}), phi::DataLayout::NCHW));
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* dev_ctx = reinterpret_cast<phi::CPUContext*>(pool.Get(phi::CPUPlace()));
+
+  auto* dense_x_data = dev_ctx->Alloc<int8_t>(&dense_x);
+  dense_x_data[0] = 1;
+  phi::Scalar scalar_test(dense_x);
+  ASSERT_EQ(1, scalar_test.to<int8_t>());
+}
+
+TEST(Scalar, ConstructFromDenseTensor4) {
+  // 1. create tensor
+  const auto alloc =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace());
+  phi::DenseTensor dense_x(
+      alloc.get(),
+      phi::DenseTensorMeta(
+          phi::DataType::BOOL, common::make_ddim({1}), phi::DataLayout::NCHW));
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* dev_ctx = reinterpret_cast<phi::CPUContext*>(pool.Get(phi::CPUPlace()));
+
+  auto* dense_x_data = dev_ctx->Alloc<bool>(&dense_x);
+  dense_x_data[0] = true;
+  phi::Scalar scalar_test(dense_x);
+  ASSERT_EQ(true, scalar_test.to<bool>());
+}
+
+TEST(Scalar, ConstructFromDenseTensor5) {
+  // 1. create tensor
+  const auto alloc =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace());
+  phi::DenseTensor dense_x(alloc.get(),
+                           phi::DenseTensorMeta(phi::DataType::COMPLEX64,
+                                                common::make_ddim({1}),
+                                                phi::DataLayout::NCHW));
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* dev_ctx = reinterpret_cast<phi::CPUContext*>(pool.Get(phi::CPUPlace()));
+
+  auto* dense_x_data = dev_ctx->Alloc<complex64>(&dense_x);
+  dense_x_data[0] = 1;
+  phi::Scalar scalar_test(dense_x);
+  complex64 expected_value(1, 0);
+  EXPECT_TRUE(expected_value == scalar_test.to<complex64>());
+}
+
+TEST(Scalar, ConstructFromDenseTensor6) {
+  // 1. create tensor
+  const auto alloc =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace());
+  phi::DenseTensor dense_x(alloc.get(),
+                           phi::DenseTensorMeta(phi::DataType::COMPLEX128,
+                                                common::make_ddim({1}),
+                                                phi::DataLayout::NCHW));
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* dev_ctx = reinterpret_cast<phi::CPUContext*>(pool.Get(phi::CPUPlace()));
+
+  auto* dense_x_data = dev_ctx->Alloc<complex128>(&dense_x);
+  dense_x_data[0] = 1;
+  phi::Scalar scalar_test(dense_x);
+  complex128 expected_value(1, 0);
+  EXPECT_TRUE(expected_value == scalar_test.to<complex128>());
+}
+
+TEST(Scalar, ConstructFromDenseTensor7) {
+  // 1. create tensor
+  const auto alloc =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::GPUPlace());
+  phi::DenseTensor dense_x(alloc.get(),
+                           phi::DenseTensorMeta(phi::DataType::FLOAT32,
+                                                common::make_ddim({1}),
+                                                phi::DataLayout::NCHW));
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* dev_ctx = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  auto* dense_x_data = dev_ctx->Alloc<float>(&dense_x);
+  FillTensor<<<1, 1, 0, dev_ctx->stream()>>>(dense_x_data);
+  dev_ctx->Wait();
+  phi::Scalar scalar_test(dense_x);
+  ASSERT_NEAR(1, scalar_test.to<float>(), 1e-6);
+}
+
+TEST(Scalar, ConstructFromTensor) {
+  // 1. create tensor
+  const auto alloc =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::GPUPlace());
+  auto dense_x = std::make_shared<phi::DenseTensor>(
+      alloc.get(),
+      phi::DenseTensorMeta(phi::DataType::FLOAT32,
+                           common::make_ddim({1}),
+                           phi::DataLayout::NCHW));
+
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* dev_ctx = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  auto* dense_x_data = dev_ctx->Alloc<float>(dense_x.get());
+  FillTensor<<<1, 1, 0, dev_ctx->stream()>>>(dense_x_data);
+  dev_ctx->Wait();
+  paddle::Tensor x(dense_x);
+  paddle::experimental::Scalar scalar_test(x);
+  ASSERT_NEAR(1, scalar_test.to<float>(), 1e-6);
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/common/transform_test.cu

@@ -0,0 +1,99 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+
+#include "paddle/phi/common/transform.h"
+
+#include "paddle/common/hostdevice.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/common/memory_utils.h"
+
+namespace phi {
+template <typename T>
+class Scale {
+ public:
+  explicit Scale(const T& scale) : scale_(scale) {}
+  HOSTDEVICE T operator()(const T& a) const { return a * scale_; }
+
+ private:
+  T scale_;
+};
+
+template <typename T>
+class Multiply {
+ public:
+  HOSTDEVICE T operator()(const T& a, const T& b) const { return a * b; }
+};
+
+TEST(Transform, CPUUnary) {
+  CPUContext ctx;
+  float buf[4] = {0.1, 0.2, 0.3, 0.4};
+  Transform<CPUContext> trans;
+  trans(ctx, buf, buf + 4, buf, Scale<float>(10));
+  for (int i = 0; i < 4; ++i) {
+    ASSERT_NEAR(buf[i], static_cast<float>(i + 1), 1e-5);
+  }
+}
+
+TEST(Transform, GPUUnary) {
+  GPUPlace gpu0(0);
+  DeviceContextPool& pool = DeviceContextPool::Instance();
+  auto* ctx = reinterpret_cast<GPUContext*>(pool.Get(GPUPlace()));
+
+  float cpu_buf[4] = {0.1, 0.2, 0.3, 0.4};
+  auto gpu_allocation = memory_utils::Alloc(gpu0, sizeof(float) * 4);
+  float* gpu_buf = static_cast<float*>(gpu_allocation->ptr());
+  memory_utils::Copy(
+      gpu0, gpu_buf, CPUPlace(), cpu_buf, sizeof(cpu_buf), ctx->stream());
+  Transform<GPUContext> trans;
+  trans(*ctx, gpu_buf, gpu_buf + 4, gpu_buf, Scale<float>(10));
+  ctx->Wait();
+  memory_utils::Copy(
+      CPUPlace(), cpu_buf, gpu0, gpu_buf, sizeof(cpu_buf), ctx->stream());
+  for (int i = 0; i < 4; ++i) {
+    ASSERT_NEAR(cpu_buf[i], static_cast<float>(i + 1), 1e-5);
+  }
+}
+
+TEST(Transform, CPUBinary) {
+  int buf[4] = {1, 2, 3, 4};
+  Transform<CPUContext> trans;
+  CPUContext ctx;
+  trans(ctx, buf, buf + 4, buf, buf, Multiply<int>());
+  for (int i = 0; i < 4; ++i) {
+    ASSERT_EQ((i + 1) * (i + 1), buf[i]);
+  }
+}
+
+TEST(Transform, GPUBinary) {
+  int buf[4] = {1, 2, 3, 4};
+  GPUPlace gpu0(0);
+  DeviceContextPool& pool = DeviceContextPool::Instance();
+  auto* ctx = reinterpret_cast<GPUContext*>(pool.Get(GPUPlace()));
+
+  auto gpu_allocation = memory_utils::Alloc(gpu0, sizeof(buf));
+  int* gpu_buf = static_cast<int*>(gpu_allocation->ptr());
+  memory_utils::Copy(
+      gpu0, gpu_buf, CPUPlace(), buf, sizeof(buf), ctx->stream());
+  Transform<GPUContext> trans;
+  trans(*ctx, gpu_buf, gpu_buf + 4, gpu_buf, gpu_buf, Multiply<int>());
+  ctx->Wait();
+  memory_utils::Copy(
+      CPUPlace(), buf, gpu0, gpu_buf, sizeof(buf), ctx->stream());
+  for (int i = 0; i < 4; ++i) {
+    ASSERT_EQ((i + 1) * (i + 1), buf[i]);
+  }
+}
+}  // namespace phi

test/cpp/phi/core/CMakeLists.txt

@@ -0,0 +1,117 @@
+cc_test(
+  test_custom_kernel
+  SRCS test_custom_kernel.cc
+  DEPS phi common)
+if(WIN32)
+  cc_test(
+    test_dense_tensor
+    SRCS test_dense_tensor.cc
+    DEPS type_info common)
+else()
+  cc_test(
+    test_dense_tensor
+    SRCS test_dense_tensor.cc
+    DEPS phi common)
+endif()
+cc_test(test_intrusive_ptr SRCS test_intrusive_ptr.cc)
+cc_test(test_type_info SRCS test_type_info.cc)
+if(WIN32)
+  paddle_test(test_kernel_factory SRCS test_kernel_factory.cc DEPS phi common)
+else()
+  cc_test(
+    test_kernel_factory
+    SRCS test_kernel_factory.cc
+    DEPS phi common)
+endif()
+cc_test(
+  test_sparse_coo_tensor
+  SRCS test_sparse_coo_tensor.cc
+  DEPS phi common)
+cc_test(
+  test_sparse_csr_tensor
+  SRCS test_sparse_csr_tensor.cc
+  DEPS phi common)
+cc_test(
+  test_op_utils
+  SRCS test_op_utils.cc
+  DEPS op_compat_infos)
+cc_test(
+  test_meta_fn_utils
+  SRCS test_meta_fn_utils.cc
+  DEPS phi common)
+
+if(WIN32)
+  cc_test(
+    test_ddim
+    SRCS test_ddim.cc
+    DEPS type_info common)
+else()
+  cc_test(
+    test_ddim
+    SRCS test_ddim.cc
+    DEPS phi common)
+endif()
+if(WITH_GPU)
+  nv_test(
+    test_dim
+    SRCS test_dim.cu
+    DEPS phi common)
+elseif(WITH_ROCM)
+  hip_test(
+    test_dim
+    SRCS test_dim.cu
+    DEPS phi common)
+endif()
+
+cc_test(
+  selected_rows_test
+  SRCS test_selected_rows.cc
+  DEPS phi common)
+if(WITH_TESTING AND TEST selected_rows_test)
+  set_tests_properties(selected_rows_test PROPERTIES TIMEOUT 120)
+endif()
+if(NOT WIN32)
+  cc_test(test_rw_lock SRCS test_rw_lock.cc)
+endif()
+cc_test(
+  test_string_tensor
+  SRCS test_string_tensor.cc
+  DEPS phi common)
+cc_test(unroll_array_ops_test SRCS unroll_array_ops_test.cc)
+
+cc_test(
+  test_tensor_array
+  SRCS test_tensor_array.cc
+  DEPS phi common)
+
+if(WITH_GPU)
+  if(WIN32)
+    nv_test(
+      test_mixed_vector
+      SRCS test_mixed_vector.cc test_mixed_vector.cu
+      DEPS type_info common tensor)
+  else()
+    nv_test(
+      test_mixed_vector
+      SRCS test_mixed_vector.cc test_mixed_vector.cu
+      DEPS phi common tensor)
+  endif()
+elseif(WITH_ROCM)
+  hip_test(
+    test_mixed_vector
+    SRCS test_mixed_vector.cc test_mixed_vector.cu
+    DEPS phi common tensor)
+else()
+  cc_test(
+    test_mixed_vector
+    SRCS test_mixed_vector.cc
+    DEPS phi common tensor)
+endif()
+
+if(NOT WIN32)
+  paddle_test(test_c_tcp_store SRCS test_tcp_store.cc DEPS phi common)
+endif()
+
+if(WITH_XPU)
+  paddle_test(data_type_transform_test_xpu SRCS data_type_transform_test_xpu.cc)
+endif()

test/cpp/phi/core/allocator.h

@@ -0,0 +1,38 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <memory>
+
+#include "paddle/phi/core/allocator.h"
+
+namespace phi {
+namespace tests {
+
+class FancyAllocator : public phi::Allocator {
+ public:
+  static void Delete(Allocation* allocation) {
+    ::operator delete(allocation->ptr());
+  }
+
+  AllocationPtr Allocate(size_t bytes_size) override {
+    void* data = ::operator new(bytes_size);
+    auto* allocation = new phi::Allocation(data, bytes_size, phi::CPUPlace());
+    return AllocationPtr(allocation, Delete);
+  }
+};
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/core/data_type_transform_test_xpu.cc

@@ -0,0 +1,219 @@
+// Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/framework/tensor_util.h"
+#include "paddle/phi/core/framework/data_type_transform.h"
+#include "paddle/phi/core/kernel_factory.h"
+
+template <typename InT, typename OutT>
+void TransformTest(const phi::KernelKey& kernel_type_for_var,
+                   const phi::KernelKey& expected_kernel_type,
+                   const phi::CPUPlace& cpu_place,
+                   const phi::XPUPlace& xpu_place,
+                   const InT* cpu_data,
+                   const int data_number) {
+  phi::XPUContext context(xpu_place);
+  phi::DenseTensor in;
+  phi::DenseTensor in_xpu;
+  phi::DenseTensor out;
+  phi::DenseTensor out_xpu;
+
+  // copy from cpu_data to cpu tensor
+  InT* in_ptr =
+      in.mutable_data<InT>(common::make_ddim({data_number}), cpu_place);
+  memcpy(in_ptr, cpu_data, sizeof(InT) * data_number);
+
+  // test case 1: on xpu
+  {
+    // copy from cpu tensor to xpu tensor
+    paddle::framework::TensorCopy(in, xpu_place, context, &in_xpu);
+    context.Wait();
+
+    // call trans data
+    phi::TransDataType(
+        kernel_type_for_var, expected_kernel_type, in_xpu, &out_xpu);
+
+    // copy from xpu tensor to cpu tensor
+    paddle::framework::TensorCopy(out_xpu, cpu_place, context, &out);
+    context.Wait();
+
+    // check result
+    OutT* out_ptr = out.data<OutT>();
+    for (int i = 0; i < data_number; ++i) {
+      EXPECT_EQ(out_ptr[i], static_cast<OutT>(cpu_data[i]));
+    }
+  }
+
+  // test case 2: on cpu
+  {
+    // call trans data
+    phi::TransDataType(kernel_type_for_var, expected_kernel_type, in, &out);
+
+    // check result
+    OutT* out_ptr = out.data<OutT>();
+    for (int i = 0; i < data_number; ++i) {
+      EXPECT_EQ(out_ptr[i], static_cast<OutT>(cpu_data[i]));
+    }
+  }
+}
+
+TEST(DataTypeTransform, XPUTransform) {
+  auto cpu_place = phi::CPUPlace();
+  auto xpu_place = phi::XPUPlace(0);
+  phi::XPUContext context(xpu_place);
+
+  auto kernel_fp16 = phi::KernelKey(
+      xpu_place, phi::DataLayout::ALL_LAYOUT, phi::DataType::FLOAT16);
+  auto kernel_fp32 = phi::KernelKey(
+      xpu_place, phi::DataLayout::ALL_LAYOUT, phi::DataType::FLOAT32);
+  auto kernel_fp64 = phi::KernelKey(
+      xpu_place, phi::DataLayout::ALL_LAYOUT, phi::DataType::FLOAT64);
+  auto kernel_int16 = phi::KernelKey(
+      xpu_place, phi::DataLayout::ALL_LAYOUT, phi::DataType::INT16);
+  auto kernel_int32 = phi::KernelKey(
+      xpu_place, phi::DataLayout::ALL_LAYOUT, phi::DataType::INT32);
+  auto kernel_int64 = phi::KernelKey(
+      xpu_place, phi::DataLayout::ALL_LAYOUT, phi::DataType::INT64);
+  auto kernel_bool = phi::KernelKey(
+      xpu_place, phi::DataLayout::ALL_LAYOUT, phi::DataType::BOOL);
+
+  {
+    // float16 -> any
+    phi::dtype::float16 cpu_data[6] = {phi::dtype::float16(0),
+                                       phi::dtype::float16(1),
+                                       phi::dtype::float16(2),
+                                       phi::dtype::float16(3),
+                                       phi::dtype::float16(4),
+                                       phi::dtype::float16(5)};
+    TransformTest<phi::dtype::float16, float>(
+        kernel_fp16, kernel_fp32, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<phi::dtype::float16, double>(
+        kernel_fp16, kernel_fp64, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<phi::dtype::float16, int32_t>(
+        kernel_fp16, kernel_int32, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<phi::dtype::float16, int64_t>(
+        kernel_fp16, kernel_int64, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<phi::dtype::float16, bool>(
+        kernel_fp16, kernel_bool, cpu_place, xpu_place, cpu_data, 6);
+  }
+  {
+    // float -> any
+    float cpu_data[6] = {0, 1, 2, 3, 4, 5};
+    TransformTest<float, phi::dtype::float16>(
+        kernel_fp32, kernel_fp16, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<float, float>(
+        kernel_fp32, kernel_fp32, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<float, double>(
+        kernel_fp32, kernel_fp64, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<float, int16_t>(
+        kernel_fp32, kernel_int16, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<float, int32_t>(
+        kernel_fp32, kernel_int32, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<float, int64_t>(
+        kernel_fp32, kernel_int64, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<float, bool>(
+        kernel_fp32, kernel_bool, cpu_place, xpu_place, cpu_data, 6);
+  }
+  {
+    // double -> any
+    double cpu_data[6] = {0, 1, 2, 3, 4, 5};
+    TransformTest<double, phi::dtype::float16>(
+        kernel_fp64, kernel_fp16, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<double, float>(
+        kernel_fp64, kernel_fp32, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<double, double>(
+        kernel_fp64, kernel_fp64, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<double, int16_t>(
+        kernel_fp64, kernel_int16, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<double, int32_t>(
+        kernel_fp64, kernel_int32, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<double, int64_t>(
+        kernel_fp64, kernel_int64, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<double, bool>(
+        kernel_fp64, kernel_bool, cpu_place, xpu_place, cpu_data, 6);
+  }
+  {
+    // int16 -> any
+    int16_t cpu_data[6] = {0, 1, 2, 3, 4, 5};
+    TransformTest<int16_t, phi::dtype::float16>(
+        kernel_int16, kernel_fp16, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<int16_t, float>(
+        kernel_int16, kernel_fp32, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<int16_t, double>(
+        kernel_int16, kernel_fp64, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<int16_t, int16_t>(
+        kernel_int16, kernel_int16, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<int16_t, int32_t>(
+        kernel_int16, kernel_int32, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<int16_t, int64_t>(
+        kernel_int16, kernel_int64, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<int16_t, bool>(
+        kernel_int16, kernel_bool, cpu_place, xpu_place, cpu_data, 6);
+  }
+  {
+    // int32 -> any
+    int32_t cpu_data[6] = {0, 1, 2, 3, 4, 5};
+    TransformTest<int32_t, phi::dtype::float16>(
+        kernel_int32, kernel_fp16, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<int32_t, float>(
+        kernel_int32, kernel_fp32, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<int32_t, double>(
+        kernel_int32, kernel_fp64, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<int32_t, int16_t>(
+        kernel_int32, kernel_int16, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<int32_t, int32_t>(
+        kernel_int32, kernel_int32, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<int32_t, int64_t>(
+        kernel_int32, kernel_int64, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<int32_t, bool>(
+        kernel_int32, kernel_bool, cpu_place, xpu_place, cpu_data, 6);
+  }
+  {
+    // int64 -> any
+    int64_t cpu_data[6] = {0, 1, 2, 3, 4, 5};
+    TransformTest<int64_t, phi::dtype::float16>(
+        kernel_int64, kernel_fp16, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<int64_t, float>(
+        kernel_int64, kernel_fp32, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<int64_t, double>(
+        kernel_int64, kernel_fp64, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<int64_t, int16_t>(
+        kernel_int64, kernel_int16, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<int64_t, int32_t>(
+        kernel_int64, kernel_int32, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<int64_t, int64_t>(
+        kernel_int64, kernel_int64, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<int64_t, bool>(
+        kernel_int64, kernel_bool, cpu_place, xpu_place, cpu_data, 6);
+  }
+  {
+    // bool -> any
+    bool cpu_data[6] = {0, 1, 0, 1, 1, 0};
+    TransformTest<bool, phi::dtype::float16>(
+        kernel_bool, kernel_fp16, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<bool, float>(
+        kernel_bool, kernel_fp32, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<bool, double>(
+        kernel_bool, kernel_fp64, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<bool, int16_t>(
+        kernel_bool, kernel_int16, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<bool, int32_t>(
+        kernel_bool, kernel_int32, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<bool, int64_t>(
+        kernel_bool, kernel_int64, cpu_place, xpu_place, cpu_data, 6);
+    TransformTest<bool, bool>(
+        kernel_bool, kernel_bool, cpu_place, xpu_place, cpu_data, 6);
+  }
+}

test/cpp/phi/core/random.h

@@ -0,0 +1,47 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <random>
+#include <type_traits>
+
+namespace phi {
+namespace tests {
+
+template <typename T,
+          typename =
+              typename std::enable_if<std::is_arithmetic<T>::value>::type>
+class RandomGenerator {
+  using distribution_type =
+      typename std::conditional<std::is_integral<T>::value,
+                                std::uniform_int_distribution<T>,
+                                std::uniform_real_distribution<T>>::type;
+
+  std::default_random_engine engine;
+  distribution_type distribution;
+
+ public:
+  auto operator()() -> decltype(distribution(engine)) {
+    return distribution(engine);
+  }
+};
+
+template <typename Container, typename T = typename Container::value_type>
+auto make_generator(Container const&) -> decltype(RandomGenerator<T>()) {
+  return RandomGenerator<T>();
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/core/test_custom_kernel.cc

@@ -0,0 +1,303 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#if defined _WIN32 || defined __APPLE__
+#else
+#define _LINUX
+#endif
+
+#include <gtest/gtest.h>
+
+#ifdef _LINUX
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/common/int_array.h"
+#include "paddle/phi/common/scalar.h"
+#include "paddle/phi/core/kernel_context.h"
+#include "paddle/phi/core/kernel_factory.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/meta_tensor.h"
+#include "paddle/phi/infermeta/binary.h"
+
+// user kernel function
+namespace custom_kernel {
+
+// Here we use fake_dot for test
+// input 3: two Tensors and one std::vector<Tensor>
+// attribute 11: fake_attributes
+// output 2: one Tensor* and one std::vector<Tensor*>
+template <typename T, typename Context>
+void FakeDot(const Context& dev_ctx,
+             const phi::DenseTensor& x,
+             const phi::DenseTensor& y,
+             const std::vector<const phi::DenseTensor*>& fake_input_vec,
+             bool fake_attr_bool,
+             int fake_attr_int,
+             float fake_attr_float,
+             double fake_attr_double,
+             int64_t fake_attr_int64,
+             phi::DataType fake_attr_dtype,
+             const phi::Scalar& fake_attr_scalar,
+             const phi::IntArray& fake_attr_int_array,
+             const std::vector<int64_t>& fake_attr_int64_vec,
+             const std::vector<int>& fake_attr_int_vec,
+             phi::DenseTensor* out,
+             std::vector<phi::DenseTensor*> fake_out_vec) {
+  // print param info
+  std::cout << "fake_input_vec.size: " << fake_input_vec.size() << std::endl;
+  std::cout << "fake_attr_bool: " << fake_attr_bool << std::endl;
+  std::cout << "fake_attr_int: " << fake_attr_int << std::endl;
+  std::cout << "fake_attr_float: " << fake_attr_float << std::endl;
+  std::cout << "fake_attr_double: " << fake_attr_double << std::endl;
+  std::cout << "fake_attr_int64: " << fake_attr_int64 << std::endl;
+  std::cout << "fake_attr_dtype: " << fake_attr_dtype << std::endl;
+  std::cout << "fake_attr_int64_vec: " << fake_attr_int64_vec.size()
+            << std::endl;
+  std::cout << "fake_attr_int_vec: " << fake_attr_int_vec.size() << std::endl;
+  std::cout << "fake_out_vec: " << fake_out_vec.size() << std::endl;
+
+  // assert check
+  assert(fake_input_vec.size() == 2);
+  assert(fake_attr_bool == false);
+  assert(fake_attr_int == 1);
+  assert(fake_attr_float == 2);
+  assert(fake_attr_double == 3);
+  assert(fake_attr_int64 == 4);
+  assert(fake_attr_dtype == phi::DataType::UINT32);
+  assert(fake_attr_int64_vec.empty());
+  assert(fake_attr_int_vec.empty());
+  assert(fake_out_vec.size() == 2);
+
+  auto const *x_ptr = x.data<T>(), *x_ptr_ = &x_ptr[0];
+  auto const *y_ptr = y.data<T>(), *y_ptr_ = &y_ptr[0];
+  T* z = dev_ctx.template Alloc<T>(out);
+  auto&& d = x.dims();
+  auto const N = x.numel();
+  auto const B = d[d.size() - 1];
+  for (int j = 0; j < N / B; j++) {
+    T ss = 0;
+    for (int i = 0; i < B; i++) ss += (*x_ptr_++) * (*y_ptr_++);
+    z[j] = ss;
+  }
+}
+}  // namespace custom_kernel
+
+PD_REGISTER_BUILTIN_KERNEL(fake_dot,
+                           CPU,
+                           ALL_LAYOUT,
+                           custom_kernel::FakeDot,
+                           float,
+                           double,
+                           int,
+                           int64_t,
+                           int8_t,
+                           uint8_t) {}
+
+namespace phi {
+namespace tests {
+
+// Upper code will store dot kernels info into OpKernelInfoMap
+TEST(CustomKernel, custom_kernel_dot) {
+  std::string op_name = "fake_dot";
+  phi::Backend backend = phi::Backend::CPU;
+  phi::DataLayout layout = phi::DataLayout::ALL_LAYOUT;
+
+  // 1.custom kernel info parsed and store
+  EXPECT_TRUE(phi::CustomKernelMap::Instance().GetMap().find(op_name) !=
+              phi::CustomKernelMap::Instance().GetMap().end());
+
+  auto& custom_kernels = phi::CustomKernelMap::Instance().Kernels();
+  // 2.info check
+  EXPECT_EQ(6, static_cast<int>(custom_kernels[op_name].size()));
+  auto& custom_fake_dot_kernels = custom_kernels[op_name];
+  EXPECT_TRUE(custom_fake_dot_kernels.find(
+                  phi::KernelKey(backend, layout, phi::DataType::FLOAT32)) !=
+              custom_fake_dot_kernels.end());
+  EXPECT_TRUE(custom_fake_dot_kernels.find(
+                  phi::KernelKey(backend, layout, phi::DataType::FLOAT64)) !=
+              custom_fake_dot_kernels.end());
+  EXPECT_TRUE(custom_fake_dot_kernels.find(
+                  phi::KernelKey(backend, layout, phi::DataType::INT32)) !=
+              custom_fake_dot_kernels.end());
+  EXPECT_TRUE(custom_fake_dot_kernels.find(
+                  phi::KernelKey(backend, layout, phi::DataType::INT64)) !=
+              custom_fake_dot_kernels.end());
+  EXPECT_TRUE(custom_fake_dot_kernels.find(
+                  phi::KernelKey(backend, layout, phi::DataType::INT8)) !=
+              custom_fake_dot_kernels.end());
+  EXPECT_TRUE(custom_fake_dot_kernels.find(
+                  phi::KernelKey(backend, layout, phi::DataType::UINT8)) !=
+              custom_fake_dot_kernels.end());
+
+  // 3.before register
+  auto& kernels = phi::KernelFactory::Instance().kernels();
+  EXPECT_TRUE(kernels.find(op_name) == kernels.end());
+
+  // mock fake_dot is supported by phi for check while registering
+  auto& fake_dot_kernels = kernels[op_name];
+
+  EXPECT_TRUE(fake_dot_kernels.find(
+                  phi::KernelKey(backend, layout, phi::DataType::FLOAT32)) ==
+              fake_dot_kernels.end());
+  EXPECT_TRUE(fake_dot_kernels.find(
+                  phi::KernelKey(backend, layout, phi::DataType::FLOAT64)) ==
+              fake_dot_kernels.end());
+  EXPECT_TRUE(fake_dot_kernels.find(
+                  phi::KernelKey(backend, layout, phi::DataType::INT32)) ==
+              fake_dot_kernels.end());
+  EXPECT_TRUE(fake_dot_kernels.find(
+                  phi::KernelKey(backend, layout, phi::DataType::INT64)) ==
+              fake_dot_kernels.end());
+  EXPECT_TRUE(fake_dot_kernels.find(
+                  phi::KernelKey(backend, layout, phi::DataType::INT8)) ==
+              fake_dot_kernels.end());
+  EXPECT_TRUE(fake_dot_kernels.find(
+                  phi::KernelKey(backend, layout, phi::DataType::UINT8)) ==
+              fake_dot_kernels.end());
+
+  // register
+  phi::CustomKernelMap::Instance().RegisterCustomKernels();
+
+  EXPECT_EQ(0, static_cast<int>(custom_fake_dot_kernels.size()));
+
+  EXPECT_TRUE(fake_dot_kernels.find(
+                  phi::KernelKey(backend, layout, phi::DataType::FLOAT32)) !=
+              fake_dot_kernels.end());
+  EXPECT_TRUE(fake_dot_kernels.find(
+                  phi::KernelKey(backend, layout, phi::DataType::FLOAT64)) !=
+              fake_dot_kernels.end());
+  EXPECT_TRUE(fake_dot_kernels.find(
+                  phi::KernelKey(backend, layout, phi::DataType::INT32)) !=
+              fake_dot_kernels.end());
+  EXPECT_TRUE(fake_dot_kernels.find(
+                  phi::KernelKey(backend, layout, phi::DataType::INT64)) !=
+              fake_dot_kernels.end());
+  EXPECT_TRUE(fake_dot_kernels.find(
+                  phi::KernelKey(backend, layout, phi::DataType::INT8)) !=
+              fake_dot_kernels.end());
+  EXPECT_TRUE(fake_dot_kernels.find(
+                  phi::KernelKey(backend, layout, phi::DataType::UINT8)) !=
+              fake_dot_kernels.end());
+
+  // 4.kernel select
+  auto kernel_result = phi::KernelFactory::Instance().SelectKernelOrThrowError(
+      op_name, phi::KernelKey(backend, layout, phi::DataType::UINT8));
+  const auto& kernel = kernel_result.kernel;
+
+  // 5.prepare parameters for kernel
+  const auto alloc =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace());
+  auto dense_x = std::make_shared<phi::DenseTensor>(
+      alloc.get(),
+      phi::DenseTensorMeta(phi::DataType::UINT8,
+                           common::make_ddim({2, 3}),
+                           phi::DataLayout::NCHW));
+  auto* dev_ctx = phi::DeviceContextPool::Instance().Get(phi::CPUPlace());
+  auto* dense_x_data = dev_ctx->template Alloc<uint8_t>(dense_x.get());
+
+  auto dense_y = std::make_shared<phi::DenseTensor>(
+      alloc.get(),
+      phi::DenseTensorMeta(phi::DataType::UINT8,
+                           common::make_ddim({2, 3}),
+                           phi::DataLayout::NCHW));
+  auto* dense_y_data = dev_ctx->template Alloc<uint8_t>(dense_y.get());
+
+  // dot x,y and result
+  std::array<uint8_t, 2> sum = {0, 0};
+  for (size_t i = 0; i < 2; ++i) {
+    for (size_t j = 0; j < 3; ++j) {
+      dense_x_data[i * 3 + j] = (i * 3 + j);
+      dense_y_data[i * 3 + j] = (i * 3 + j);
+      sum[i] += (i * 3 + j) * (i * 3 + j);
+    }
+  }
+
+  // 6.prepare kernel_context
+  auto kernel_context = phi::KernelContext(dev_ctx);
+  kernel_context.EmplaceBackInput(dense_x.get());  // idx:0, index:[0,1)
+  kernel_context.EmplaceBackInput(dense_y.get());  // idx:1, index:[1,2)
+
+  // fake_input_vec: idx:2, index:[2,4)
+  size_t fake_input_vec_idx = 2;
+  size_t fake_input_vec_index_start = 2;
+  size_t fake_input_vec_index_end = 4;
+  kernel_context.EmplaceBackInputWithoutSetRange(dense_x.get());
+  kernel_context.EmplaceBackInputWithoutSetRange(dense_y.get());
+  kernel_context.AssignInputRange(
+      std::make_pair(fake_input_vec_index_start, fake_input_vec_index_end),
+      fake_input_vec_idx);
+
+  bool fake_attr_bool = false;
+  int fake_attr_int = 1;
+  float fake_attr_float = 2.0;
+  double fake_attr_double = 3.0;
+  int64_t fake_attr_int64 = 4;
+  phi::DataType fake_attr_dtype = phi::DataType::UINT32;
+  phi::DenseTensor tmp_tensor;
+  tmp_tensor.Resize({1});
+  dev_ctx->template Alloc<uint8_t>(&tmp_tensor);
+  phi::Scalar fake_attr_scalar{tmp_tensor};
+  phi::IntArray fake_attr_int_array;
+  std::vector<int64_t> fake_attr_int64_vec;
+  std::vector<int> fake_attr_int_vec;
+
+  kernel_context.EmplaceBackAttr(fake_attr_bool);
+  kernel_context.EmplaceBackAttr(fake_attr_int);
+  kernel_context.EmplaceBackAttr(fake_attr_float);
+  kernel_context.EmplaceBackAttr(fake_attr_double);
+  kernel_context.EmplaceBackAttr(fake_attr_int64);
+  kernel_context.EmplaceBackAttr(fake_attr_dtype);
+  kernel_context.EmplaceBackAttr(fake_attr_scalar);
+  kernel_context.EmplaceBackAttr(fake_attr_int_array);
+  kernel_context.EmplaceBackAttr(fake_attr_int64_vec);
+  kernel_context.EmplaceBackAttr(fake_attr_int_vec);
+
+  auto dense_out = std::make_shared<phi::DenseTensor>();
+
+  phi::MetaTensor meta_out(dense_out.get());
+  phi::DotInferMeta(*dense_x, *dense_y, &meta_out);
+  kernel_context.EmplaceBackOutput(dense_out.get());  // idx:0 index:[0,1)
+
+  // fake_input_vec: idx:1, index:[1,3)
+  size_t fake_out_vec_idx = 1;
+  size_t fake_out_vec_index_start = 1;
+  size_t fake_out_vec_index_end = 3;
+  kernel_context.EmplaceBackOutputWithoutSetRange(dense_out.get());
+  kernel_context.EmplaceBackOutputWithoutSetRange(dense_out.get());
+  kernel_context.AssignOutputRange(
+      std::make_pair(fake_out_vec_index_start, fake_out_vec_index_end),
+      fake_out_vec_idx);
+
+  // 7.kernel call
+  kernel(&kernel_context);
+
+  // 8.check result
+  ASSERT_EQ(dense_out->dims().size(), 1);
+  ASSERT_EQ(dense_out->dims()[0], 2);
+  ASSERT_EQ(dense_out->numel(), 2);
+  ASSERT_EQ(dense_out->dtype(), phi::DataType::UINT8);
+  ASSERT_EQ(dense_out->layout(), phi::DataLayout::NCHW);
+  ASSERT_EQ(dense_out->initialized(), true);
+
+  auto expect_result = sum;
+  auto actual_result0 = dense_out->data<uint8_t>()[0];
+  auto actual_result1 = dense_out->data<uint8_t>()[1];
+  ASSERT_EQ(expect_result[0], actual_result0);
+  ASSERT_EQ(expect_result[1], actual_result1);
+}
+
+}  // namespace tests
+}  // namespace phi
+
+#endif

test/cpp/phi/core/test_ddim.cc

@@ -0,0 +1,136 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <sstream>
+
+#include "gtest/gtest.h"
+#include "paddle/phi/core/ddim.h"
+
+namespace phi {
+namespace tests {
+
+TEST(DDim, Equality) {
+  // default construct ddim
+  phi::DDim default_ddim;
+  EXPECT_EQ(arity(default_ddim), -1);
+  EXPECT_EQ(default_ddim[0], 0);
+
+  // construct a zero-DDim
+  phi::DDim zero_ddim = common::make_ddim({});
+  EXPECT_EQ(arity(zero_ddim), 0);
+  EXPECT_EQ(zero_ddim.size(), 0);
+  EXPECT_EQ(common::product(zero_ddim), 1);
+
+  std::vector<int64_t> zero_vec;
+  phi::DDim zero_ddim1 = common::make_ddim(zero_vec);
+  EXPECT_EQ(arity(zero_ddim1), 0);
+  EXPECT_EQ(zero_ddim1.size(), 0);
+  EXPECT_EQ(common::product(zero_ddim1), 1);
+
+  // zero-DDim to vector
+  std::vector<int64_t> zero_ddim_vec = common::vectorize(zero_ddim);
+  EXPECT_EQ(zero_ddim_vec.size(), size_t(0));
+
+  // reshape zero-DDim
+  std::vector<int> reshape_vec = {1};
+  phi::DDim reshape_ddim = zero_ddim.reshape(reshape_vec);
+  EXPECT_EQ(arity(reshape_ddim), 1);
+  EXPECT_EQ(reshape_ddim.size(), 1);
+  EXPECT_EQ(common::product(reshape_ddim), 1);
+
+  // construct a DDim from an initialization list
+  phi::DDim ddim = common::make_ddim({9, 1, 5});
+  EXPECT_EQ(ddim[0], 9);
+  EXPECT_EQ(ddim[1], 1);
+  EXPECT_EQ(ddim[2], 5);
+
+  // arity of a DDim
+  EXPECT_EQ(common::arity(ddim), 3);
+  EXPECT_EQ(ddim.size(), 3);
+
+  // mutate a DDim
+  ddim[1] = 2;
+  EXPECT_EQ(ddim[1], 2);
+  ddim[0] = 6;
+  EXPECT_EQ(ddim[0], 6);
+
+  // construct a DDim from a vector
+  std::vector<int64_t> vec({9, 1, 5});
+  phi::DDim vddim = common::make_ddim(vec);
+  EXPECT_EQ(vddim[0], 9);
+  EXPECT_EQ(vddim[1], 1);
+  EXPECT_EQ(vddim[2], 5);
+
+  // vectorize a DDim
+  std::vector<int64_t> res_vec = common::vectorize(vddim);
+  EXPECT_EQ(res_vec[0], 9);
+  EXPECT_EQ(res_vec[1], 1);
+  EXPECT_EQ(res_vec[2], 5);
+  phi::Dim<3> d(3, 2, 1);
+  res_vec = common::vectorize(phi::DDim(d));
+  EXPECT_EQ(res_vec[0], 3);
+  EXPECT_EQ(res_vec[1], 2);
+  EXPECT_EQ(res_vec[2], 1);
+
+  // product of a DDim
+  EXPECT_EQ(common::product(vddim), 45);
+  EXPECT_EQ(common::product(common::make_ddim({3, 2, 5, 3})), 90);
+
+  // slice a DDim
+  phi::DDim ddim2 = common::make_ddim({1, 2, 3, 4, 5, 6});
+  phi::DDim slice_dim1 = common::slice_ddim(ddim2, 2, 5);
+  EXPECT_EQ(arity(slice_dim1), 3);
+  EXPECT_EQ(slice_dim1[0], 3);
+  EXPECT_EQ(slice_dim1[1], 4);
+  EXPECT_EQ(slice_dim1[2], 5);
+
+  phi::DDim slice_dim2 = common::slice_ddim(ddim2, 0, 6);
+  EXPECT_EQ(arity(slice_dim2), 6);
+  EXPECT_EQ(slice_dim2[0], 1);
+  EXPECT_EQ(slice_dim2[1], 2);
+  EXPECT_EQ(slice_dim2[2], 3);
+  EXPECT_EQ(slice_dim2[3], 4);
+  EXPECT_EQ(slice_dim2[4], 5);
+  EXPECT_EQ(slice_dim2[5], 6);
+
+  phi::DDim slice_dim3 = common::slice_ddim(ddim2, 1, 1);
+  EXPECT_EQ(arity(slice_dim3), 0);
+  EXPECT_EQ(slice_dim3.size(), 0);
+  EXPECT_EQ(common::product(slice_dim3), 1);
+}
+
+TEST(DDim, Print) {
+  // print a DDim
+  std::stringstream ss1;
+  phi::DDim ddim = common::make_ddim({2, 3, 4});
+  ss1 << ddim;
+  EXPECT_EQ("2, 3, 4", ss1.str());
+
+  // print a zero-DDim
+  std::stringstream ss2;
+  phi::DDim zero_ddim = common::make_ddim({});
+  ss2 << zero_ddim;
+  EXPECT_EQ("", ss2.str());
+}
+
+TEST(DDim, Hash) {
+  // hash a DDim
+  std::size_t h = 0;
+  phi::DDim ddim = common::make_ddim({2, 3, 4});
+  h = std::hash<phi::DDim>()(ddim);
+  EXPECT_EQ(h, 0xa16fb2b2967ul);
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/core/test_dense_tensor.cc

@@ -0,0 +1,376 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "test/cpp/phi/core/allocator.h"
+
+namespace phi {
+namespace tests {
+
+TEST(dense_tensor, meta) {
+  const DDim dims({1, 2});
+  const DataType dtype{DataType::INT8};
+  const DataLayout layout{DataLayout::NHWC};
+  // TODO(Shixiaowei02): need to check the lod is valid.
+  const LegacyLoD lod{};
+
+  DenseTensorMeta meta_0;
+  PADDLE_ENFORCE_EQ(meta_0.valid(),
+                    false,
+                    common::errors::InvalidArgument(
+                        "Fail in default DenseTensorMeta. Expected "
+                        "meta_0 to be invalid, but got: %s",
+                        meta_0.valid()));
+
+  DenseTensorMeta meta_1(dtype, dims);
+  PADDLE_ENFORCE_EQ(
+      meta_1.dtype,
+      dtype,
+      common::errors::InvalidArgument("Fail in DenseTensorMeta with dtype and "
+                                      "dims. Expected dtype: %s, but got: %s",
+                                      dtype,
+                                      meta_1.dtype));
+  PADDLE_ENFORCE_EQ(
+      meta_1.dims,
+      dims,
+      common::errors::InvalidArgument("Fail in DenseTensorMeta with dtype and "
+                                      "dims. Expected dims: %s, but got: %s",
+                                      dims,
+                                      meta_1.dims));
+  PADDLE_ENFORCE_EQ(meta_1.valid(),
+                    true,
+                    common::errors::InvalidArgument(
+                        "Fail in DenseTensorMeta with dtype and dims. Expected "
+                        "meta_1 to be valid, but got: %s",
+                        meta_1.valid()));
+
+  DenseTensorMeta meta_2(dtype, dims, layout);
+  PADDLE_ENFORCE_EQ(meta_2.dtype,
+                    dtype,
+                    common::errors::InvalidArgument(
+                        "Fail in DenseTensorMeta with dtype, dims and layout. "
+                        "Expected dtype: %s, but got: %s",
+                        dtype,
+                        meta_2.dtype));
+  PADDLE_ENFORCE_EQ(meta_2.dims,
+                    dims,
+                    common::errors::InvalidArgument(
+                        "Fail in DenseTensorMeta with dtype, dims "
+                        "and layout. Expected dims: %s, but got: %s",
+                        dims,
+                        meta_2.dims));
+  PADDLE_ENFORCE_EQ(meta_2.layout,
+                    layout,
+                    common::errors::InvalidArgument(
+                        "Fail in DenseTensorMeta with dtype, dims and layout. "
+                        "Expected layout: %s, but got: %s",
+                        layout,
+                        meta_2.layout));
+  PADDLE_ENFORCE_EQ(meta_2.valid(),
+                    true,
+                    common::errors::InvalidArgument(
+                        "Fail in DenseTensorMeta with dtype, dims and layout. "
+                        "Expected meta_2 to be valid, but got: %s",
+                        meta_2.valid()));
+
+  DenseTensorMeta meta_3(dtype, dims, layout, lod);
+  PADDLE_ENFORCE_EQ(meta_3.dtype,
+                    dtype,
+                    common::errors::InvalidArgument(
+                        "Fail in DenseTensorMeta with dtype, dims, layout and "
+                        "lod. Expected dtype: %s, but got: %s",
+                        dtype,
+                        meta_3.dtype));
+  PADDLE_ENFORCE_EQ(meta_3.dims,
+                    dims,
+                    common::errors::InvalidArgument(
+                        "Fail in DenseTensorMeta with dtype, dims, layout and "
+                        "lod. Expected dims: %s, but got: %s",
+                        dims,
+                        meta_3.dims));
+  PADDLE_ENFORCE_EQ(meta_3.layout,
+                    layout,
+                    common::errors::InvalidArgument(
+                        "Fail in DenseTensorMeta with dtype, dims, layout and "
+                        "lod. Expected layout: %s, but got: %s",
+                        layout,
+                        meta_3.layout));
+  PADDLE_ENFORCE_EQ(meta_3.legacy_lod,
+                    lod,
+                    common::errors::InvalidArgument(
+                        "Fail in DenseTensorMeta with dtype, dims, layout and "
+                        "lod. Expected lod: %s, but got: %s",
+                        lod,
+                        meta_3.legacy_lod));
+  PADDLE_ENFORCE_EQ(meta_3.valid(),
+                    true,
+                    common::errors::InvalidArgument(
+                        "Fail in DenseTensorMeta with dtype, dims, layout and "
+                        "lod. Expected meta_3 to be valid, but got: %s",
+                        meta_3.valid()));
+
+  DenseTensorMeta meta_4(meta_3);
+  PADDLE_ENFORCE_EQ(
+      meta_4.dtype,
+      dtype,
+      common::errors::InvalidArgument(
+          "Fail in copy DenseTensorMeta. Expected dtype: %s, but got: %s",
+          dtype,
+          meta_4.dtype));
+  PADDLE_ENFORCE_EQ(
+      meta_4.dims,
+      dims,
+      common::errors::InvalidArgument(
+          "Fail in copy DenseTensorMeta. Expected dims: %s, but got: %s",
+          dims,
+          meta_4.dims));
+  PADDLE_ENFORCE_EQ(
+      meta_4.layout,
+      layout,
+      common::errors::InvalidArgument(
+          "Fail in copy DenseTensorMeta. Expected layout: %s, but got: %s",
+          layout,
+          meta_4.layout));
+  PADDLE_ENFORCE_EQ(
+      meta_4.legacy_lod,
+      lod,
+      common::errors::InvalidArgument(
+          "Fail in copy DenseTensorMeta. Expected lod: %s, but got: %s",
+          lod,
+          meta_4.legacy_lod));
+  PADDLE_ENFORCE_EQ(
+      meta_4.valid(),
+      true,
+      common::errors::InvalidArgument("Fail in copy DenseTensorMeta. Expected "
+                                      "meta_4 to be valid, but got: %s",
+                                      meta_4.valid()));
+
+  DenseTensorMeta meta_5(meta_4);
+  PADDLE_ENFORCE_EQ(
+      meta_5.dtype,
+      dtype,
+      common::errors::InvalidArgument(
+          "Fail in copy DenseTensorMeta. Expected dtype: %s, but got: %s",
+          dtype,
+          meta_5.dtype));
+  PADDLE_ENFORCE_EQ(
+      meta_5.dims,
+      dims,
+      common::errors::InvalidArgument(
+          "Fail in copy DenseTensorMeta. Expected dims: %s, but got: %s",
+          dims,
+          meta_5.dims));
+  PADDLE_ENFORCE_EQ(
+      meta_5.layout,
+      layout,
+      common::errors::InvalidArgument(
+          "Fail in copy DenseTensorMeta. Expected layout: %s, but got: %s",
+          layout,
+          meta_5.layout));
+  PADDLE_ENFORCE_EQ(
+      meta_5.legacy_lod,
+      lod,
+      common::errors::InvalidArgument(
+          "Fail in copy DenseTensorMeta. Expected lod: %s, but got: %s",
+          lod,
+          meta_5.legacy_lod));
+  PADDLE_ENFORCE_EQ(
+      meta_5.valid(),
+      true,
+      common::errors::InvalidArgument("Fail in copy DenseTensorMeta. Expected "
+                                      "meta_5 to be valid, but got: %s",
+                                      meta_5.valid()));
+}
+
+TEST(dense_tensor, zero_size_strides) {
+  const auto zero_size_dims = common::make_ddim({0, 2048});
+  const auto zero_size_strides = DenseTensorMeta::calc_strides(zero_size_dims);
+  const auto expected_zero_size_strides = common::make_ddim({2048, 1});
+  EXPECT_EQ(zero_size_strides, expected_zero_size_strides);
+
+  DenseTensorMeta zero_size_meta(DataType::FLOAT32, zero_size_dims);
+  EXPECT_EQ(zero_size_meta.strides, expected_zero_size_strides);
+  EXPECT_TRUE(zero_size_meta.is_contiguous());
+
+  const auto reshaped_zero_size_dims = common::make_ddim({0, 512, 4});
+  EXPECT_EQ(DenseTensorMeta::calc_strides(reshaped_zero_size_dims),
+            common::make_ddim({2048, 4, 1}));
+
+  const auto unknown_dims = common::make_ddim({-1, 2048});
+  EXPECT_EQ(DenseTensorMeta::calc_strides(unknown_dims), unknown_dims);
+}
+
+TEST(dense_tensor, def_ctor) {
+  DenseTensor tensor_0;
+  PADDLE_ENFORCE_EQ(
+      tensor_0.valid(),
+      true,
+      common::errors::InvalidArgument("Fail in default DenseTensor. Expected "
+                                      "tensor_0 to be valid, but got: %s",
+                                      tensor_0.valid()));
+}
+
+TEST(dense_tensor, ctor) {
+  const DDim dims({1, 2});
+  const DataType dtype{DataType::INT8};
+  const DataLayout layout{DataLayout::NHWC};
+  const LegacyLoD lod{};
+  DenseTensorMeta meta(dtype, dims, layout, lod);
+
+  auto fancy_allocator = std::unique_ptr<Allocator>(new FancyAllocator);
+  auto* alloc = fancy_allocator.get();
+
+  auto check_dense_tensor = [](const DenseTensor& t,
+                               const DenseTensorMeta& m) -> bool {
+    bool r{true};
+    r = r && (t.numel() == product(m.dims));
+    r = r && (t.dims() == m.dims);
+    r = r && (t.dtype() == m.dtype);
+    r = r && (t.layout() == m.layout);
+    r = r && (t.place() == phi::CPUPlace());
+    r = r && t.initialized();
+    r = r && t.IsSharedWith(t);
+    return r;
+  };
+
+  DenseTensor tensor_0(alloc, meta);
+  check_dense_tensor(tensor_0, meta);
+
+  DenseTensor tensor_1(alloc, DenseTensorMeta(meta));
+  check_dense_tensor(tensor_0, meta);
+}
+
+TEST(dense_tensor, resize) {
+  const DDim dims({1, 2});
+  const DataType dtype{DataType::INT8};
+  const DataLayout layout{DataLayout::NHWC};
+  const LegacyLoD lod{};
+  DenseTensorMeta meta(dtype, dims, layout, lod);
+
+  auto fancy_allocator = std::unique_ptr<Allocator>(new FancyAllocator);
+  auto* alloc = fancy_allocator.get();
+  DenseTensor tensor_0(alloc, meta);
+
+  PADDLE_ENFORCE_EQ(
+      tensor_0.capacity(),
+      2u,
+      common::errors::InvalidArgument(
+          "Fail to initialize DenseTensor. Expected capacity: 2, but got: %s",
+          tensor_0.capacity()));
+  tensor_0.ResizeAndAllocate({1, 2, 3});
+  PADDLE_ENFORCE_EQ(
+      tensor_0.capacity(),
+      6u,
+      common::errors::InvalidArgument(
+          "Fail to resize DenseTensor. Expected capacity: 6, but got: %s",
+          tensor_0.capacity()));
+}
+
+TEST(dense_tensor, shallow_copy) {
+  const DDim dims({1, 2});
+  const DataType dtype{DataType::INT8};
+  const DataLayout layout{DataLayout::NHWC};
+  const LegacyLoD lod{};
+  DenseTensorMeta meta(dtype, dims, layout, lod);
+
+  auto fancy_allocator = std::unique_ptr<Allocator>(new FancyAllocator);
+  auto* alloc = fancy_allocator.get();
+  DenseTensor tensor_0(alloc, meta);
+
+  DenseTensor tensor_1(tensor_0);
+  PADDLE_ENFORCE_EQ(tensor_0.meta(),
+                    tensor_1.meta(),
+                    common::errors::InvalidArgument(
+                        "Fail to copy DenseTensor. Expected tensor_0 and "
+                        "tensor_1 to have the same meta"));
+}
+
+TEST(dense_tensor, dim_indexing) {
+  const DDim dims({4, 3, 2, 0});
+  const DataType dtype{DataType::INT8};
+  const DataLayout layout{DataLayout::NHWC};
+  const LegacyLoD lod{};
+  DenseTensorMeta meta(dtype, dims, layout, lod);
+
+  auto fancy_allocator = std::unique_ptr<Allocator>(new FancyAllocator);
+  auto* alloc = fancy_allocator.get();
+  DenseTensor tensor_0(alloc, meta);
+  int ndim = tensor_0.dims().size();
+  auto tensor_0_dims = tensor_0.dims();
+  for (int i = -ndim; i < ndim; ++i) {
+    PADDLE_ENFORCE_EQ(
+        tensor_0_dims[(i + ndim) % ndim],
+        tensor_0.dims(i),
+        common::errors::InvalidArgument(
+            "Dimension mismatch at index %d. Expected: %d, but got: %d",
+            i,
+            tensor_0_dims[i],
+            tensor_0.dims(i)));
+  }
+
+  // throw exception for index >= ndim
+  bool caught_exception = false;
+  try {
+    tensor_0.dims(ndim);
+  } catch (const common::enforce::EnforceNotMet& error) {
+    caught_exception = true;
+  }
+  PADDLE_ENFORCE_EQ(
+      caught_exception,
+      true,
+      common::errors::InvalidArgument(
+          "Expected an exception to be thrown for index >= ndim"));
+
+  // throw exception for index < -ndim
+  caught_exception = false;
+  try {
+    tensor_0.dims(-ndim - 1);
+  } catch (const common::enforce::EnforceNotMet& error) {
+    caught_exception = true;
+  }
+  PADDLE_ENFORCE_EQ(
+      caught_exception,
+      true,
+      common::errors::InvalidArgument(
+          "Expected an exception to be thrown for index < -ndim"));
+}
+
+TEST(dense_tensor, storage_properties) {
+  const DataType dtype{DataType::FLOAT32};
+  const DDim dims({1, 2});
+  DenseTensorMeta meta(dtype, dims);
+
+  auto fancy_allocator = std::unique_ptr<Allocator>(new FancyAllocator);
+  DenseTensor tensor(fancy_allocator.get(), meta);
+
+  // test error type storage properties
+#ifdef PADDLE_WITH_DNNL
+  bool caught_exception = false;
+  try {
+    tensor.storage_properties<OneDNNStorageProperties>();
+  } catch (common::enforce::EnforceNotMet& error) {
+    caught_exception = true;
+  }
+  PADDLE_ENFORCE_EQ(caught_exception,
+                    true,
+                    common::errors::InvalidArgument(
+                        "Fail to get storage properties. Expected an exception "
+                        "to be thrown for OneDNNStorageProperties"));
+#endif
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/core/test_dim.cu

@@ -0,0 +1,102 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <thrust/device_vector.h>
+
+#include <sstream>
+
+#include "gtest/gtest.h"
+#include "paddle/common/dim.h"
+
+namespace phi {
+namespace tests {
+
+__global__ void test(phi::Dim<2>* o) { o[0] = common::make_dim(5, 6); }
+
+__global__ void dyn_idx_gpu(int64_t* o) {
+  auto d = common::make_dim(5, 6);
+  o[0] = d[1];
+}
+
+TEST(Dim, Equality) {
+  // construct a Dim on the CPU
+  auto a = common::make_dim(3, 4);
+  EXPECT_EQ(a[0], 3);
+  EXPECT_EQ(a[1], 4);
+
+  // construct a Dim on the GPU
+  thrust::device_vector<phi::Dim<2>> t(2);
+#ifdef PADDLE_WITH_HIP
+  hipLaunchKernelGGL(
+      test, dim3(1), dim3(1), 0, 0, thrust::raw_pointer_cast(t.data()));
+#else
+  test<<<1, 1>>>(thrust::raw_pointer_cast(t.data()));
+#endif
+  a = t[0];
+  EXPECT_EQ(a[0], 5);
+  EXPECT_EQ(a[1], 6);
+
+  // product
+  EXPECT_EQ(common::product(a), 30);
+
+  // mutate a Dim
+  auto b = common::make_dim(7, 8);
+  b[1] = 10;
+  EXPECT_EQ(b[0], 7);
+  EXPECT_EQ(b[1], 10);
+
+  b[0] = 8;
+  b[1] = 11;
+  EXPECT_EQ(b[0], 8);
+  EXPECT_EQ(b[1], 11);
+
+  // dynamic access on GPU
+  thrust::device_vector<int64_t> r(1);
+#ifdef PADDLE_WITH_HIP
+  hipLaunchKernelGGL(
+      dyn_idx_gpu, dim3(1), dim3(1), 0, 0, thrust::raw_pointer_cast(r.data()));
+#else
+  dyn_idx_gpu<<<1, 1>>>(thrust::raw_pointer_cast(r.data()));
+#endif
+  int64_t res = r[0];
+  EXPECT_EQ(res, 6);
+}
+
+TEST(Dim, Bool) {
+  auto a = common::make_dim(3, 4);
+  auto b = common::make_dim(5, 6);
+  auto c = common::make_dim(3, 4);
+
+  // comparison
+  EXPECT_TRUE(a == a);
+  EXPECT_FALSE(a == b);
+  EXPECT_TRUE(a == c);
+}
+
+TEST(Dim, Print) {
+  {
+    std::stringstream ss;
+    auto a = common::make_dim(2, 3);
+    ss << a;
+    EXPECT_EQ(ss.str(), "2, 3");
+  }
+  {
+    std::stringstream ss;
+    ss << common::make_dim(8);
+    EXPECT_EQ(ss.str(), "8");
+  }
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/core/test_intrusive_ptr.cc

@@ -0,0 +1,157 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <future>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "paddle/phi/core/utils/intrusive_ptr.h"
+#include "paddle/phi/core/utils/intrusive_ref_counter.h"
+
+namespace phi {
+namespace tests {
+
+struct SharedObject : public intrusive_ref_counter<SharedObject> {
+  int i{0};
+};
+
+TEST(intrusive_ref_counter, async) {
+  SharedObject obj;
+  const size_t num{100};
+  std::vector<std::future<void>> results;
+  auto add_ref_and_release = [](const SharedObject* p) {
+    intrusive_ptr_add_ref<SharedObject>(p);
+    intrusive_ptr_release<SharedObject>(p);
+  };
+  for (size_t i = 0; i < num; ++i) {
+    results.emplace_back(std::async(add_ref_and_release, &obj));
+  }
+  for (auto& result : results) {
+    result.get();
+  }
+  PADDLE_ENFORCE_EQ(obj.use_count(),
+                    1U,
+                    common::errors::InvalidArgument(
+                        "Required obj.use_count() should be equal to 1, "
+                        "But received obj.use_count() = %d.",
+                        obj.use_count()));
+}
+
+TEST(intrusive_ptr, default_ctor) {
+  intrusive_ptr<SharedObject> p;
+  PADDLE_ENFORCE_EQ(p == nullptr,
+                    true,
+                    common::errors::Fatal("Input pointer is not a nullptr"));
+}
+TEST(intrusive_ptr, private_ctor) {
+  auto p = make_intrusive<SharedObject>();
+  const auto* ptr0 = p.get();
+  auto p1 = std::move(p);
+  intrusive_ptr<intrusive_ref_counter<SharedObject>> p2(std::move(p1));
+  const auto* ptr1 = p2.get();
+  PADDLE_ENFORCE_EQ(ptr0,
+                    ptr1,
+                    common::errors::InvalidArgument(
+                        "Required ptr0 should be equal to ptr1. "));
+}
+
+TEST(intrusive_ptr, reset_with_obj) {
+  SharedObject obj;
+  obj.i = 1;
+  intrusive_ptr<SharedObject> p;
+  p.reset(&obj, true);
+  PADDLE_ENFORCE_EQ(p->i,
+                    obj.i,
+                    common::errors::InvalidArgument(
+                        "Required p->i should be equal to obj.i. "));
+}
+
+TEST(intrusive_ptr, reset_with_ptr) {
+  auto* ptr = new SharedObject;
+  ptr->i = 1;
+  intrusive_ptr<SharedObject> p;
+  p.reset(ptr, false);
+  PADDLE_ENFORCE_EQ((*p).i,
+                    ptr->i,
+                    common::errors::InvalidArgument(
+                        "Required (*p).i should be equal to ptr->i. "));
+  p.reset();
+  PADDLE_ENFORCE_EQ(
+      p == nullptr,
+      true,
+      common::errors::Fatal(
+          "p is not a nullptr, something wrong with intrusive_ptr<T>.reset"));
+}
+TEST(intrusive_ptr, op_comp) {
+  auto p = make_intrusive<SharedObject>();
+  auto copy = copy_intrusive<SharedObject>(p);
+  auto null = intrusive_ptr<SharedObject>();
+  auto p1 = make_intrusive<SharedObject>();
+  PADDLE_ENFORCE_EQ(p == copy,
+                    true,
+                    common::errors::Fatal(
+                        "intrusive_ptr p is not equal to its copy, something "
+                        "wrong with copy constructor "));
+  PADDLE_ENFORCE_EQ(
+      p != p1,
+      true,
+      common::errors::Fatal("intrusive_ptr p is equal to another pointer, "
+                            "something wrong with constructor"));
+  PADDLE_ENFORCE_EQ(
+      p == copy.get(),
+      true,
+      common::errors::Fatal(
+          "blank intrusive_ptr p's content is not equal to its copy, something "
+          "wrong with constructor or get function"));
+  PADDLE_ENFORCE_EQ(
+      p != p1.get(),
+      true,
+      common::errors::Fatal(
+          "intrusive_ptr p's content is equal to another blank pointer, "
+          "something wrong with constructor or get function"));
+  PADDLE_ENFORCE_EQ(
+      p.get() == copy,
+      true,
+      common::errors::Fatal(
+          "blank intrusive_ptr p's content is not equal to its copy, something "
+          "wrong with constructor or get function"));
+  PADDLE_ENFORCE_EQ(
+      p.get() != p1,
+      true,
+      common::errors::Fatal(
+          "intrusive_ptr p's content is equal to another blank pointer, "
+          "something wrong with constructor or get function"));
+  PADDLE_ENFORCE_EQ(
+      null == nullptr,
+      true,
+      common::errors::Fatal("variable or constant whose name is null is not a "
+                            "nullptr, something wrong with operator=="));
+  PADDLE_ENFORCE_EQ(
+      nullptr == null,
+      true,
+      common::errors::Fatal("variable or constant whose name is null is not a "
+                            "nullptr, something wrong with operator=="));
+  PADDLE_ENFORCE_EQ(p != nullptr,
+                    true,
+                    common::errors::Fatal(
+                        "intrusive_ptr p is not not_equal to null, something "
+                        "wrong with constructor or operator!= "));
+  PADDLE_ENFORCE_EQ(nullptr != p,
+                    true,
+                    common::errors::Fatal(
+                        "intrusive_ptr p is not not_equal to null, something "
+                        "wrong with constructor or operator!= "));
+}
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/core/test_kernel_factory.cc

@@ -0,0 +1,149 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <iostream>
+#include <sstream>
+
+#include "gtest/gtest.h"
+#include "paddle/phi/common/float16.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_factory.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+PD_DECLARE_KERNEL(scale, CPU, ALL_LAYOUT);
+
+namespace phi {
+namespace tests {
+
+// TODO(chenweihang): add more unittests later
+
+TEST(KernelKey, ConstructAndOStream) {
+  phi::KernelKey key(
+      phi::Backend::CPU, phi::DataLayout::NCHW, phi::DataType::FLOAT32);
+  EXPECT_EQ(key.backend(), phi::Backend::CPU);
+  EXPECT_EQ(key.layout(), phi::DataLayout::NCHW);
+  EXPECT_EQ(key.dtype(), phi::DataType::FLOAT32);
+  std::ostringstream oss;
+  oss << key;
+  std::cout << oss.str();
+  oss.flush();
+}
+
+TEST(KernelFactory, SelectedKernelMap) {
+  auto kernel_map = phi::KernelFactory::Instance().SelectKernelMap("scale");
+  EXPECT_GT(kernel_map.size(), 1UL);
+  for (auto& iter : kernel_map) {
+    std::cout << iter.first << ": " << iter.second;
+  }
+}
+
+template <typename T, typename Context>
+void TestKernel(const Context& dev_ctx,
+                const DenseTensor& x,
+                const DenseTensor& param,
+                DenseTensor* out) {}
+
+TEST(KernelRegistry, SetFP32Input) {
+  phi::KernelKey kernel_key(
+      phi::Backend::CPU, phi::DataLayout::ALL_LAYOUT, phi::DataType::FLOAT16);
+  auto test_kernel =
+      phi::KernelFactory::Instance().SelectKernel("test", kernel_key);
+  EXPECT_TRUE(test_kernel.IsValid());
+  auto& arg_defs = test_kernel.args_def();
+  auto& input_defs = arg_defs.input_defs();
+  auto& attr_defs = arg_defs.attribute_defs();
+  auto& output_defs = arg_defs.output_defs();
+  EXPECT_EQ(input_defs.size(), 2UL);
+  EXPECT_EQ(attr_defs.size(), 0UL);
+  EXPECT_EQ(output_defs.size(), 1UL);
+  EXPECT_EQ(input_defs.at(0).dtype, phi::DataType::FLOAT16);
+  EXPECT_EQ(input_defs.at(1).dtype, phi::DataType::FLOAT32);
+  EXPECT_EQ(output_defs.at(0).dtype, phi::DataType::FLOAT16);
+}
+
+TEST(AttributeType, OStream) {
+  std::ostringstream oss;
+  oss << phi::AttributeType::UNDEFINED;
+  EXPECT_EQ(oss.str(), "Undefined");
+  oss.str("");
+  oss << phi::AttributeType::BOOL;
+  EXPECT_EQ(oss.str(), "bool");
+  oss.str("");
+  oss << phi::AttributeType::INT32;
+  EXPECT_EQ(oss.str(), "int");
+  oss.str("");
+  oss << phi::AttributeType::INT64;
+  EXPECT_EQ(oss.str(), "int64_t");
+  oss.str("");
+  oss << phi::AttributeType::FLOAT32;
+  EXPECT_EQ(oss.str(), "float");
+  oss.str("");
+  oss << phi::AttributeType::FLOAT64;
+  EXPECT_EQ(oss.str(), "double");
+  oss.str("");
+  oss << phi::AttributeType::STRING;
+  EXPECT_EQ(oss.str(), "string");
+  oss.str("");
+  oss << phi::AttributeType::BOOLS;
+  EXPECT_EQ(oss.str(), "vector<bool>");
+  oss.str("");
+  oss << phi::AttributeType::INT32S;
+  EXPECT_EQ(oss.str(), "vector<int>");
+  oss.str("");
+  oss << phi::AttributeType::INT64S;
+  EXPECT_EQ(oss.str(), "vector<int64_t>");
+  oss.str("");
+  oss << phi::AttributeType::FLOAT32S;
+  EXPECT_EQ(oss.str(), "vector<float>");
+  oss.str("");
+  oss << phi::AttributeType::FLOAT64S;
+  EXPECT_EQ(oss.str(), "vector<double>");
+  oss.str("");
+  oss << phi::AttributeType::STRINGS;
+  EXPECT_EQ(oss.str(), "vector<string>");
+  oss.str("");
+  oss << phi::AttributeType::SCALAR;
+  EXPECT_EQ(oss.str(), "Scalar");
+  oss.str("");
+  oss << phi::AttributeType::SCALARS;
+  EXPECT_EQ(oss.str(), "vector<Scalar>");
+  oss.str("");
+  oss << phi::AttributeType::INT_ARRAY;
+  EXPECT_EQ(oss.str(), "IntArray");
+  oss.str("");
+  oss << phi::AttributeType::DATA_TYPE;
+  EXPECT_EQ(oss.str(), "DataType");
+  oss.str("");
+  oss << phi::AttributeType::DATA_LAYOUT;
+  EXPECT_EQ(oss.str(), "DataLayout");
+  oss.str("");
+  oss << phi::AttributeType::PLACE;
+  EXPECT_EQ(oss.str(), "Place");
+  oss.str("");
+}
+
+}  // namespace tests
+}  // namespace phi
+
+PD_REGISTER_KERNEL(test,
+                   CPU,
+                   ALL_LAYOUT,
+                   phi::tests::TestKernel,
+                   float,
+                   double,
+                   phi::dtype::float16) {
+  if (kernel_key.dtype() == phi::DataType::FLOAT16) {
+    kernel->InputAt(1).SetDataType(phi::DataType::FLOAT32);
+  }
+}

test/cpp/phi/core/test_meta_fn_utils.cc

@@ -0,0 +1,50 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <iostream>
+
+#include "gtest/gtest.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/infermeta_utils.h"
+#include "paddle/phi/infermeta/unary.h"
+
+namespace phi {
+namespace tests {
+
+TEST(MetaFnFactory, InferMetaFnExists) {
+  phi::DenseTensor dense_x;
+  dense_x.Resize(common::make_ddim({3, 4}));
+
+  phi::MetaTensor meta_x(&dense_x);
+  phi::DenseTensor dense_out1;
+  phi::MetaTensor meta_out(&dense_out1);
+  phi::UnchangedInferMeta(meta_x, &meta_out);
+}
+
+void TestEmptyVectorInputInferMeta(const std::vector<const MetaTensor*>& inputs,
+                                   std::vector<MetaTensor*> outputs) {
+  ASSERT_EQ(inputs.size(), 0UL);
+  ASSERT_EQ(outputs.size(), 0UL);
+}
+
+TEST(MetaFnFactory, EmptyVectorInputInferMetaFn) {
+  phi::InferMetaContext ctx;
+  ctx.EmplaceBackInput(MetaTensor());
+  ctx.EmplaceBackOutput(MetaTensor());
+
+  PD_INFER_META(TestEmptyVectorInputInferMeta)(&ctx);
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/core/test_mixed_vector.cc

@@ -0,0 +1,74 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#include "paddle/phi/core/mixed_vector.h"
+
+#include "glog/logging.h"
+#include "gtest/gtest-message.h"
+#include "gtest/gtest-test-part.h"
+#include "gtest/gtest.h"
+#include "gtest/gtest_pred_impl.h"
+
+template <typename T>
+using vec = phi::Vector<T>;
+
+TEST(mixed_vector, CPU_VECTOR) {
+  vec<int> tmp;
+  for (int i = 0; i < 10; ++i) {
+    tmp.push_back(i);
+  }
+  ASSERT_EQ(tmp.size(), 10UL);
+  vec<int> tmp2;
+  tmp2 = tmp;
+  ASSERT_EQ(tmp2.size(), 10UL);
+  for (int i = 0; i < 10; ++i) {
+    ASSERT_EQ(tmp2[i], i);
+    ASSERT_EQ(tmp2[i], tmp[i]);
+  }
+  int cnt = 0;
+  for (auto& t : tmp2) {
+    ASSERT_EQ(t, cnt);
+    ++cnt;
+  }
+}
+
+TEST(mixed_vector, InitWithCount) {
+  phi::Vector<int> vec(10, 10);
+  for (int i = 0; i < 10; ++i) {
+    ASSERT_EQ(vec[i], 10);
+  }
+}
+
+TEST(mixed_vector, ForEach) {
+  vec<int> tmp;
+  for (auto& v : tmp) {
+    VLOG(3) << v;
+  }
+}
+
+TEST(mixed_vector, Reserve) {
+  phi::Vector<int> vec;
+  vec.reserve(1);
+  vec.push_back(0);
+  vec.push_back(0);
+  vec.push_back(0);
+}
+
+TEST(mixed_vector, Resize) {
+  phi::Vector<int> vec;
+  vec.resize(1);
+  vec.push_back(0);
+  vec.push_back(0);
+  vec.push_back(0);
+}

test/cpp/phi/core/test_mixed_vector.cu

@@ -0,0 +1,111 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#ifdef PADDLE_WITH_CUDA
+#include <cuda_runtime.h>
+#endif
+#ifdef PADDLE_WITH_HIP
+#include <hip/hip_runtime.h>
+#endif
+
+#include <memory>
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/backends/gpu/gpu_info.h"
+#include "paddle/phi/common/place.h"
+#include "paddle/phi/core/mixed_vector.h"
+
+template <typename T>
+using vec = phi::MixVector<T>;
+using gpuStream_t = phi::gpuStream_t;
+
+static __global__ void multiply_10(int* ptr) {
+  for (int i = 0; i < 10; ++i) {
+    ptr[i] *= 10;
+  }
+}
+
+gpuStream_t GetCUDAStream(phi::GPUPlace place) {
+  return reinterpret_cast<const phi::GPUContext*>(
+             phi::DeviceContextPool::Instance().Get(place))
+      ->stream();
+}
+
+TEST(mixed_vector, GPU_VECTOR) {
+  std::vector<int> x;
+  for (int i = 0; i < 10; ++i) {
+    x.push_back(i);
+  }
+  vec<int> tmp(&x);
+  ASSERT_EQ(tmp.size(), 10UL);
+  phi::GPUPlace gpu(0);
+
+#ifdef PADDLE_WITH_HIP
+  hipLaunchKernelGGL(multiply_10,
+                     dim3(1),
+                     dim3(1),
+                     0,
+                     GetCUDAStream(gpu),
+                     tmp.MutableData(gpu));
+#else
+  multiply_10<<<1, 1, 0, GetCUDAStream(gpu)>>>(tmp.MutableData(gpu));
+#endif
+
+  for (int i = 0; i < 10; ++i) {
+    ASSERT_EQ(tmp[i], i * 10);
+  }
+}
+
+TEST(mixed_vector, MultiGPU) {
+  if (phi::backends::gpu::GetGPUDeviceCount() < 2) {
+    LOG(WARNING) << "Skip mixed_vector.MultiGPU since there are not multiple "
+                    "GPUs in your machine.";
+    return;
+  }
+
+  std::vector<int> x;
+  for (int i = 0; i < 10; ++i) {
+    x.push_back(i);
+  }
+  vec<int> tmp(&x);
+  ASSERT_EQ(tmp.size(), 10UL);
+  phi::GPUPlace gpu0(0);
+  phi::backends::gpu::SetDeviceId(0);
+
+#ifdef PADDLE_WITH_HIP
+  hipLaunchKernelGGL(multiply_10,
+                     dim3(1),
+                     dim3(1),
+                     0,
+                     GetCUDAStream(gpu0),
+                     tmp.MutableData(gpu0));
+#else
+  multiply_10<<<1, 1, 0, GetCUDAStream(gpu0)>>>(tmp.MutableData(gpu0));
+#endif
+  phi::GPUPlace gpu1(1);
+  auto* gpu1_ptr = tmp.MutableData(gpu1);
+  phi::backends::gpu::SetDeviceId(1);
+
+#ifdef PADDLE_WITH_HIP
+  hipLaunchKernelGGL(
+      multiply_10, dim3(1), dim3(1), 0, GetCUDAStream(gpu1), gpu1_ptr);
+#else
+  multiply_10<<<1, 1, 0, GetCUDAStream(gpu1)>>>(gpu1_ptr);
+#endif
+  for (int i = 0; i < 10; ++i) {
+    ASSERT_EQ(tmp[i], i * 100);
+  }
+}

test/cpp/phi/core/test_op_utils.cc

@@ -0,0 +1,32 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <iostream>
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/operators/ops_signature/signatures.h"
+#include "paddle/phi/core/compat/op_utils.h"
+
+namespace phi {
+namespace tests {
+
+TEST(OpUtilsMap, ArgMappingFnExists) {
+  std::cout << "enter ArgMappingFnExists";
+  auto scale_arg_mapping_fn =
+      phi::OpUtilsMap::Instance().GetArgumentMappingFn("scale");
+  EXPECT_NE(scale_arg_mapping_fn, nullptr);
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/core/test_rw_lock.cc

@@ -0,0 +1,84 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>  // NOLINT
+
+#include <thread>  // NOLINT
+
+#include "paddle/phi/core/utils/rw_lock.h"
+
+namespace phi {
+namespace tests {
+
+void f1(phi::RWLock *lock) {
+  lock->RDLock();
+  lock->UNLock();
+}
+
+TEST(RWLOCK, read_read) {
+  phi::RWLock lock;
+  lock.RDLock();
+  std::thread t1(f1, &lock);
+  std::thread t2(f1, &lock);
+  t1.join();
+  t2.join();
+  lock.UNLock();
+}
+
+void f2(phi::RWLock *lock, std::vector<int> *result) {
+  lock->RDLock();
+  ASSERT_EQ(result->size(), 0UL);
+  lock->UNLock();
+}
+
+void f3(phi::RWLock *lock, std::vector<int> *result) {
+  lock->WRLock();
+  result->push_back(1);
+  lock->UNLock();
+}
+
+TEST(RWLOCK, read_write) {
+  phi::RWLock lock;
+  std::vector<int> result;
+
+  lock.RDLock();
+  std::thread t1(f2, &lock, &result);
+  t1.join();
+  std::thread t2(f3, &lock, &result);
+  std::this_thread::sleep_for(std::chrono::seconds(1));
+  ASSERT_EQ(result.size(), 0UL);
+  lock.UNLock();
+  t2.join();
+  ASSERT_EQ(result.size(), 1UL);
+}
+
+void f4(phi::RWLock *lock, std::vector<int> *result) {
+  lock->RDLock();
+  ASSERT_EQ(result->size(), 1UL);
+  lock->UNLock();
+}
+
+TEST(RWLOCK, write_read) {
+  phi::RWLock lock;
+  std::vector<int> result;
+
+  lock.WRLock();
+  std::thread t1(f4, &lock, &result);
+  std::this_thread::sleep_for(std::chrono::seconds(1));
+  result.push_back(1);
+  lock.UNLock();
+  t1.join();
+}
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/core/test_selected_rows.cc

@@ -0,0 +1,185 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <ctime>
+
+#include <thread>  // NOLINT
+
+#include "gtest/gtest.h"
+#include "paddle/phi/core/selected_rows.h"
+
+namespace phi {
+namespace tests {
+class SelectedRowsTester : public ::testing::Test {
+ public:
+  void SetUp() override {
+    std::vector<int64_t> rows{0, 4, 7};
+    int64_t height = 10;
+    int64_t row_numel = 100;
+    selected_rows_ = std::make_unique<SelectedRows>(rows, height);
+
+    phi::DenseTensor* value = selected_rows_->mutable_value();
+    auto* data = value->mutable_data<float>(
+        common::make_ddim({static_cast<int64_t>(rows.size()), row_numel}),
+        place_);
+    for (int64_t i = 0; i < value->numel(); ++i) {
+      data[i] = static_cast<float>(i);
+    }
+  }
+
+ protected:
+  phi::CPUPlace place_;
+  std::unique_ptr<phi::SelectedRows> selected_rows_{nullptr};
+};
+
+TEST_F(SelectedRowsTester, height) { ASSERT_EQ(selected_rows_->height(), 10); }
+
+TEST_F(SelectedRowsTester, dims) {
+  ASSERT_EQ(selected_rows_->value().dims(), common::make_ddim({3, 100}));
+}
+
+TEST_F(SelectedRowsTester, complete_dims) {
+  ASSERT_EQ(selected_rows_->GetCompleteDims(), common::make_ddim({10, 100}));
+}
+
+TEST(SelectedRows, SparseTable) {
+  phi::CPUPlace cpu;
+  SelectedRows table;
+
+  int64_t table_size = 100;
+  int64_t embedding_width = 8;
+  // initialize a sparse table
+  table.mutable_value()->Resize(
+      common::make_ddim({table_size, embedding_width}));
+  auto* data = table.mutable_value()->mutable_data<float>(cpu);
+  for (int64_t i = 0; i < table_size; ++i) {
+    for (int64_t j = 0; j < embedding_width; ++j) {
+      data[i * embedding_width + j] = static_cast<float>(i);
+    }
+  }
+  ASSERT_EQ(table.AutoGrownIndex(10, true, false), 0);
+  ASSERT_EQ(table.AutoGrownIndex(8, true, false), 1);
+  ASSERT_EQ(table.AutoGrownIndex(8, true, false), 1);
+  ASSERT_EQ(table.AutoGrownIndex(6, true, false), 2);
+  for (int64_t i = 11; i < 20; i++) {
+    ASSERT_EQ(table.AutoGrownIndex(i, true, true), -1);
+    ASSERT_TRUE(!table.HasKey(i));
+  }
+  ASSERT_TRUE(table.HasKey(10));
+  ASSERT_TRUE(table.HasKey(8));
+  ASSERT_TRUE(table.HasKey(6));
+  ASSERT_EQ(table.rows().size(), 3UL);
+
+  phi::DenseTensor ids;
+  ids.Resize(common::make_ddim({4}));
+  auto* ids_data = ids.mutable_data<int64_t>(cpu);
+  ids_data[0] = static_cast<int64_t>(6);
+  ids_data[1] = static_cast<int64_t>(6);
+  ids_data[2] = static_cast<int64_t>(8);
+  ids_data[3] = static_cast<int64_t>(10);
+
+  phi::DenseTensor get_value;
+  auto* value_data = get_value.mutable_data<float>(
+      common::make_ddim({4, embedding_width}), cpu);
+  table.Get(ids, &get_value);
+
+  for (int j = 0; j < embedding_width; ++j) {
+    ASSERT_EQ(value_data[0 * embedding_width + j], 2);
+  }
+  for (int j = 0; j < embedding_width; ++j) {
+    ASSERT_EQ(value_data[1 * embedding_width + j], 2);
+  }
+  for (int j = 0; j < embedding_width; ++j) {
+    ASSERT_EQ(value_data[2 * embedding_width + j], 1);
+  }
+  for (int j = 0; j < embedding_width; ++j) {
+    ASSERT_EQ(value_data[3 * embedding_width + j], 0);
+  }
+}
+
+void f1(SelectedRows* table, int table_size) {
+  for (int i = 1000000; i > 0; --i) {
+    auto id = i % table_size;
+    int64_t index1 = table->AutoGrownIndex(id, true);
+    int64_t index2 = table->AutoGrownIndex(id, false);
+    int64_t index3 = table->AutoGrownIndex(id, true);
+    ASSERT_EQ(index1, index2);
+    ASSERT_EQ(index2, index3);
+  }
+}
+
+void f2(SelectedRows* table, int table_size) {
+  for (int i = 0; i < 1000000; ++i) {
+    auto id = i % table_size;
+    int64_t index1 = table->AutoGrownIndex(id, true);
+    int64_t index2 = table->AutoGrownIndex(id, false);
+    int64_t index3 = table->AutoGrownIndex(id, true);
+    ASSERT_EQ(index1, index2);
+    ASSERT_EQ(index2, index3);
+  }
+}
+
+void f3(SelectedRows* table, int table_size) {
+  clock_t t1 = clock();
+  for (int i = 100000; i > 0; --i) {
+    auto id1 = table->AutoGrownIndex(i % table_size, true);
+    auto id2 = table->Index(i % table_size);
+    ASSERT_EQ(id1, id2);
+  }
+  clock_t t2 = clock();
+  std::cout << "f3 run time:" << t2 - t1 << std::endl;
+}
+
+void f4(SelectedRows* table, int table_size) {
+  clock_t t1 = clock();
+  for (int i = 0; i < 100000; ++i) {
+    auto id1 = table->AutoGrownIndex(i % table_size, true);
+    auto id2 = table->Index(i % table_size);
+    ASSERT_EQ(id1, id2);
+  }
+  clock_t t2 = clock();
+  std::cout << "f4 run time:" << t2 - t1 << std::endl;
+}
+
+TEST(SelectedRows, MultiThreadAutoIndex) {
+  phi::CPUPlace cpu;
+  SelectedRows table;
+
+  int64_t table_size = 100000;
+  int64_t embedding_width = 8;
+  // initialize a sparse table
+  table.mutable_value()->Resize(
+      common::make_ddim({table_size, embedding_width}));
+  auto* data = table.mutable_value()->mutable_data<float>(cpu);
+  for (int64_t i = 0; i < table_size; ++i) {
+    for (int64_t j = 0; j < embedding_width; ++j) {
+      data[i * embedding_width + j] = static_cast<float>(i);
+    }
+  }
+
+  std::thread t1(f1, &table, table_size);
+  std::thread t11(f1, &table, table_size);
+  std::thread t2(f2, &table, table_size);
+  std::thread t22(f2, &table, table_size);
+  t1.join();
+  t11.join();
+  t2.join();
+  t22.join();
+  std::thread t3(f3, &table, table_size);
+  std::thread t4(f4, &table, table_size);
+  t3.join();
+  t4.join();
+}
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/core/test_sparse_coo_tensor.cc

@@ -0,0 +1,114 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/sparse_coo_tensor.h"
+#include "test/cpp/phi/core/allocator.h"
+
+namespace phi {
+namespace tests {
+
+TEST(sparse_coo_tensor, construct) {
+  phi::CPUPlace cpu;
+  auto dense_dims = common::make_ddim({3, 3});
+  std::vector<float> non_zero_data = {1.0, 2.0, 3.0};
+  std::vector<int64_t> indices_data = {0, 1, 2, 0, 2, 1};
+  auto fancy_allocator = std::unique_ptr<Allocator>(new FancyAllocator);
+  auto* alloc = fancy_allocator.get();
+  auto indices_dims =
+      common::make_ddim({2, static_cast<int>(non_zero_data.size())});
+  DenseTensorMeta indices_meta(DataType::INT64, indices_dims, DataLayout::NCHW);
+  DenseTensor indices(alloc, indices_meta);
+  memcpy(indices.mutable_data<int64_t>(cpu),
+         &indices_data[0],
+         indices_data.size() * sizeof(int64_t));
+
+  auto elements_dims =
+      common::make_ddim({static_cast<int>(non_zero_data.size())});
+  DenseTensorMeta elements_meta(
+      DataType::FLOAT32, elements_dims, DataLayout::NCHW);
+  DenseTensor elements(alloc, elements_meta);
+
+  memcpy(elements.mutable_data<float>(cpu),
+         &non_zero_data[0],
+         non_zero_data.size() * sizeof(float));
+
+  SparseCooTensor sparse(indices, elements, dense_dims);
+
+  CHECK(sparse.initialized() == true);
+  PADDLE_ENFORCE_EQ(
+      sparse.nnz(),
+      non_zero_data.size(),
+      common::errors::InvalidArgument(
+          "Required sparse.nnz() should be equal to non_zero_data.size(). "));
+  PADDLE_ENFORCE_EQ(sparse.numel(),
+                    9,
+                    common::errors::InvalidArgument(
+                        "Required sparse.numel() should be equal to 9. "));
+  CHECK(sparse.dims() == dense_dims);
+  CHECK(sparse.dtype() == DataType::FLOAT32);
+  CHECK(sparse.place() == phi::CPUPlace());
+}
+
+TEST(sparse_coo_tensor, other_function) {
+  auto fancy_allocator = std::unique_ptr<Allocator>(new FancyAllocator);
+  auto* alloc = fancy_allocator.get();
+  auto dense_dims = common::make_ddim({4, 4});
+  const int non_zero_num = 2;
+  auto indices_dims = common::make_ddim({2, non_zero_num});
+  DenseTensorMeta indices_meta(DataType::INT64, indices_dims, DataLayout::NCHW);
+  DenseTensor indices(alloc, indices_meta);
+
+  auto elements_dims = common::make_ddim({non_zero_num});
+  DenseTensorMeta elements_meta(
+      DataType::FLOAT32, elements_dims, DataLayout::NCHW);
+  DenseTensor elements(alloc, elements_meta);
+
+  SparseCooTensor coo(indices, elements, dense_dims);
+  CHECK(coo.initialized());
+  PADDLE_ENFORCE_EQ(coo.dims(),
+                    dense_dims,
+                    common::errors::InvalidArgument(
+                        "Required coo.dims() should be equal to dense_dims. "));
+
+  // Test Resize
+  auto dense_dims_3d = common::make_ddim({2, 4, 4});
+  coo.Resize(dense_dims_3d, 1, 3);
+  PADDLE_ENFORCE_EQ(coo.nnz(),
+                    3,
+                    common::errors::InvalidArgument(
+                        "Required coo.nnz() should be equal to 3. "));
+
+  // Test shallow_copy
+  SparseCooTensor coo2(coo);
+  PADDLE_ENFORCE_EQ(
+      coo.dims(),
+      coo2.dims(),
+      common::errors::Fatal("`coo.dims()` is not equal to `coo2.dims()`, "
+                            "something wrong with shallow copy assignment"));
+
+  // Test shallow_copy_assignment
+  SparseCooTensor coo3 = coo2;
+  CHECK(coo3.dims() == coo2.dims());
+  PADDLE_ENFORCE_EQ(
+      coo3.dims(),
+      coo2.dims(),
+      common::errors::Fatal("`coo3.dims()` is not equal to `coo2.dims()`, "
+                            "something wrong with shallow copy assignment"));
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/core/test_sparse_csr_tensor.cc

@@ -0,0 +1,125 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/sparse_csr_tensor.h"
+#include "test/cpp/phi/core/allocator.h"
+
+namespace phi {
+namespace tests {
+
+TEST(sparse_csr_tensor, construct) {
+  phi::CPUPlace cpu;
+  auto dense_dims = common::make_ddim({3, 3});
+  std::vector<float> non_zero_data = {1.0, 2.0, 3.0};
+  std::vector<int64_t> crows_data = {0, 1, 1, 3};
+  std::vector<int64_t> cols_data = {1, 0, 2};
+
+  auto fancy_allocator = std::unique_ptr<Allocator>(new FancyAllocator);
+  auto alloc = fancy_allocator.get();
+  // create non_zero_crows
+  auto crows_dims = common::make_ddim({static_cast<int>(crows_data.size())});
+  DenseTensorMeta crows_meta(DataType::INT64, crows_dims, DataLayout::NCHW);
+  DenseTensor crows(alloc, crows_meta);
+  memcpy(crows.mutable_data<int64_t>(cpu),
+         &crows_data[0],
+         crows_data.size() * sizeof(int64_t));
+
+  // create non_zero_cols
+  auto cols_dims = common::make_ddim({static_cast<int>(cols_data.size())});
+  DenseTensorMeta cols_meta(DataType::INT64, cols_dims, DataLayout::NCHW);
+  DenseTensor cols(alloc, cols_meta);
+  memcpy(cols.mutable_data<int64_t>(cpu),
+         &cols_data[0],
+         cols_data.size() * sizeof(int64_t));
+
+  // create non_zero_elements
+  auto elements_dims =
+      common::make_ddim({static_cast<int>(non_zero_data.size())});
+  DenseTensorMeta elements_meta(
+      DataType::FLOAT32, elements_dims, DataLayout::NCHW);
+  DenseTensor elements(alloc, elements_meta);
+  memcpy(elements.mutable_data<float>(cpu),
+         &non_zero_data[0],
+         non_zero_data.size() * sizeof(float));
+
+  SparseCsrTensor sparse(crows, cols, elements, dense_dims);
+
+  PADDLE_ENFORCE_EQ(sparse.non_zero_cols().numel(),
+                    non_zero_data.size(),
+                    common::errors::InvalidArgument(
+                        "Required sparse.non_zero_cols().numel() should be "
+                        "equal to non_zero_data.size(). "));
+  PADDLE_ENFORCE_EQ(sparse.numel(),
+                    9,
+                    common::errors::InvalidArgument(
+                        "Required sparse.numel() should be equal to 9. "));
+  CHECK(sparse.dims() == dense_dims);
+  CHECK(sparse.dtype() == DataType::FLOAT32);
+  CHECK(sparse.place() == phi::CPUPlace());
+  CHECK(sparse.initialized() == true);
+}
+TEST(sparse_csr_tensor, other_function) {
+  auto fancy_allocator = std::unique_ptr<Allocator>(new FancyAllocator);
+  auto alloc = fancy_allocator.get();
+  auto dense_dims = common::make_ddim({4, 4});
+  auto crows_dims = common::make_ddim({dense_dims[0] + 1});
+  DenseTensorMeta crows_meta(DataType::INT64, crows_dims, DataLayout::NCHW);
+  DenseTensor crows(alloc, crows_meta);
+
+  const int64_t non_zero_num = 5;
+  auto cols_dims = common::make_ddim({non_zero_num});
+  DenseTensorMeta cols_meta(DataType::INT64, cols_dims, DataLayout::NCHW);
+  DenseTensor cols(alloc, cols_meta);
+  DenseTensorMeta values_meta(DataType::FLOAT32, cols_dims, DataLayout::NCHW);
+  DenseTensor values(alloc, values_meta);
+
+  SparseCsrTensor csr(crows, cols, values, dense_dims);
+  CHECK(csr.initialized());
+  PADDLE_ENFORCE_EQ(csr.dims(),
+                    dense_dims,
+                    common::errors::InvalidArgument(
+                        "Required csr.dims() should be equal to dense_dims. "));
+
+  // Test Resize
+  auto dense_dims_3d = common::make_ddim({2, 4, 4});
+  csr.Resize(dense_dims_3d, 2);
+  PADDLE_ENFORCE_EQ(
+      csr.non_zero_cols().numel(),
+      2,
+      common::errors::InvalidArgument(
+          "Required csr.non_zero_cols().numel() should be equal to 2. "));
+
+  // Test shallow_copy
+  SparseCsrTensor csr2(csr);
+  PADDLE_ENFORCE_EQ(
+      csr.dims(),
+      csr2.dims(),
+      common::errors::Fatal("`csr.dims()` should be equal to `csr2.dims()`, "
+                            "something wrong with shallow copy"));
+
+  // Test shallow_copy_assignment
+  SparseCsrTensor csr3 = csr2;
+  PADDLE_ENFORCE_EQ(
+      csr3.dims(),
+      csr2.dims(),
+      common::errors::Fatal("``csr3.dims()` should be equal to `csr2.dims()`, "
+                            "something wrong with shallow copy assignment"));
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/core/test_string_tensor.cc

@@ -0,0 +1,356 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <sstream>
+#include <string>
+#include <utility>
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/common/pstring.h"
+#include "paddle/phi/core/string_tensor.h"
+#include "test/cpp/phi/core/allocator.h"
+
+namespace phi {
+namespace tests {
+
+using pstring = ::phi::dtype::pstring;
+
+TEST(string_tensor, ctor) {
+  const DDim dims({1, 2});
+  StringTensorMeta meta(dims);
+  const auto string_allocator =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace());
+  const auto alloc = string_allocator.get();
+  auto check_string_tensor = [](const StringTensor& t,
+                                const StringTensorMeta& m) -> bool {
+    bool r{true};
+    r = r && (t.numel() == product(m.dims));
+    r = r && (t.dims() == m.dims);
+    r = r && (t.place() == phi::CPUPlace());
+    r = r && t.initialized();
+    r = r && t.IsSharedWith(t);
+    r = r && (t.meta() == m);
+    return r;
+  };
+  auto cpu = CPUPlace();
+
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  CPUContext* cpu_ctx = reinterpret_cast<CPUContext*>(pool.Get(cpu));
+
+  StringTensor tensor_0(alloc, meta);
+  check_string_tensor(tensor_0, meta);
+
+  pstring pshort_str = pstring("A short pstring.");
+  pstring plong_str =
+      pstring("A large pstring whose length is longer than 22.");
+
+  pstring* data = cpu_ctx->template Alloc<pstring>(&tensor_0);
+  data[0] = plong_str;
+  data[1] = pshort_str;
+  PADDLE_ENFORCE_EQ(tensor_0.data()[0],
+                    plong_str,
+                    common::errors::InvalidArgument(
+                        "The tensor_0 should be equal to '%s', but got '%s'.",
+                        plong_str,
+                        tensor_0.data()[0]));
+  PADDLE_ENFORCE_EQ(tensor_0.data()[1],
+                    pshort_str,
+                    common::errors::InvalidArgument(
+                        "The tensor_0 should be equal to '%s', but got '%s'.",
+                        pshort_str,
+                        tensor_0.data()[1]));
+
+  // Test Copy Constructor
+  StringTensor tensor_1(tensor_0);
+  PADDLE_ENFORCE_EQ(tensor_1.data()[0],
+                    plong_str,
+                    common::errors::InvalidArgument(
+                        "The tensor_1 should be equal to '%s', but got '%s'.",
+                        plong_str,
+                        tensor_1.data()[0]));
+  PADDLE_ENFORCE_EQ(tensor_1.data()[1],
+                    pshort_str,
+                    common::errors::InvalidArgument(
+                        "The tensor_1 should be equal to '%s', but got '%s'.",
+                        pshort_str,
+                        tensor_1.data()[1]));
+
+  // Test Copy Assignment
+  StringTensor tensor_2(alloc, meta);
+  tensor_2 = tensor_1;
+  PADDLE_ENFORCE_EQ(tensor_2.data()[0],
+                    plong_str,
+                    common::errors::InvalidArgument(
+                        "The tensor_2 should be equal to '%s', but got '%s'.",
+                        plong_str,
+                        tensor_2.data()[0]));
+  PADDLE_ENFORCE_EQ(tensor_2.data()[1],
+                    pshort_str,
+                    common::errors::InvalidArgument(
+                        "The tensor_2 should be equal to '%s', but got '%s'.",
+                        pshort_str,
+                        tensor_2.data()[1]));
+
+  // Test Move Assignment
+  StringTensor tensor_3(alloc, meta);
+  tensor_3 = std::move(tensor_1);
+  PADDLE_ENFORCE_EQ(tensor_3.data()[0],
+                    plong_str,
+                    common::errors::InvalidArgument(
+                        "The tensor_3 should be equal to '%s', but got '%s'.",
+                        plong_str,
+                        tensor_3.data()[0]));
+  PADDLE_ENFORCE_EQ(tensor_3.data()[1],
+                    pshort_str,
+                    common::errors::InvalidArgument(
+                        "The tensor_3 should be equal to '%s', but got '%s'.",
+                        pshort_str,
+                        tensor_3.data()[1]));
+
+  tensor_3.set_meta(meta);
+}
+
+TEST(pstring, func) {
+  // Test Ctor
+  pstring empty_str;
+  pstring nchar_str(5, 'A');
+  pstring copy_nchar_str(nchar_str);
+  PADDLE_ENFORCE_EQ(
+      empty_str,
+      "",
+      common::errors::InvalidArgument(
+          "The empty_str should be empty, but got '%s'.", empty_str));
+  PADDLE_ENFORCE_EQ(
+      nchar_str,
+      "AAAAA",
+      common::errors::InvalidArgument(
+          "The nchar_str should be 'AAAAA', but got '%s'.", nchar_str));
+  PADDLE_ENFORCE_EQ(copy_nchar_str,
+                    "AAAAA",
+                    common::errors::InvalidArgument(
+                        "The copy_nchar_str should be 'AAAAA', but got '%s'.",
+                        copy_nchar_str));
+
+  // Test Move Ctor
+  pstring move_nchar_str(nchar_str);
+  PADDLE_ENFORCE_EQ(move_nchar_str,
+                    "AAAAA",
+                    common::errors::InvalidArgument(
+                        "The move_nchar_str should be 'AAAAA', but got '%s'.",
+                        move_nchar_str));
+  pstring std_str(std::string("BBBB"));
+  PADDLE_ENFORCE_EQ(
+      std_str,
+      "BBBB",
+      common::errors::InvalidArgument(
+          "The std_str should be 'BBBB', but got '%s'.", std_str));
+
+  pstring long_str = "A large pstring whose length is longer than 22.";
+  pstring short_str = "A short pstring.";
+
+  // Test operator+
+  pstring plus_str = move_nchar_str + std_str;
+  PADDLE_ENFORCE_EQ(
+      plus_str,
+      "AAAAABBBB",
+      common::errors::InvalidArgument(
+          "The plus_str should be 'AAAAABBBB', but got '%s'.", plus_str));
+
+  // Test insert
+  plus_str.insert(5, 1, 'C');
+  PADDLE_ENFORCE_EQ(
+      plus_str,
+      "AAAAACBBBB",
+      common::errors::InvalidArgument(
+          "The plus_str should be 'AAAAABBBB', but got '%s'.", plus_str));
+  plus_str.insert(5, "DDD", 0, 2);
+  PADDLE_ENFORCE_EQ(
+      plus_str,
+      "AAAAADDCBBBB",
+      common::errors::InvalidArgument(
+          "The plus_str should be 'AAAAABBBB', but got '%s'.", plus_str));
+
+  // Test pushback
+  plus_str.push_back('E');
+  PADDLE_ENFORCE_EQ(
+      plus_str,
+      "AAAAADDCBBBBE",
+      common::errors::InvalidArgument(
+          "The plus_str should be 'AAAAADDCBBBBE', but got '%s'.", plus_str));
+
+  // Test append
+  plus_str.append("FF");
+  PADDLE_ENFORCE_EQ(
+      plus_str,
+      "AAAAADDCBBBBEFF",
+      common::errors::InvalidArgument(
+          "The plus_str should be 'AAAAADDCBBBBEFF', but got '%s'.", plus_str));
+  plus_str.append(2, 'G');
+  PADDLE_ENFORCE_EQ(
+      plus_str,
+      "AAAAADDCBBBBEFFGG",
+      common::errors::InvalidArgument(
+          "The plus_str should be 'AAAAADDCBBBBEFFGG', but got '%s'.",
+          plus_str));
+
+  // Test operator[]
+  PADDLE_ENFORCE_EQ(
+      long_str[0],
+      'A',
+      common::errors::InvalidArgument(
+          "The long_str[0] should be 'A', but got '%s'.", long_str[0]));
+  PADDLE_ENFORCE_EQ(
+      short_str[0],
+      'A',
+      common::errors::InvalidArgument(
+          "The short_str[0] should be 'A', but got '%s'.", short_str[0]));
+
+  // Test capacity
+  PADDLE_ENFORCE_EQ(short_str.capacity(),
+                    22UL,
+                    common::errors::InvalidArgument(
+                        "The short_str's capacity should be 22, but got %d.",
+                        short_str.capacity()));
+
+  // Test reserve
+  pstring reserve_str;
+  PADDLE_ENFORCE_EQ(reserve_str.capacity(),
+                    22UL,
+                    common::errors::InvalidArgument(
+                        "The reserve_str's capacity should be 22, but got %d.",
+                        reserve_str.capacity()));
+  // small -> large
+  reserve_str.reserve(100);
+  PADDLE_ENFORCE_EQ(reserve_str.capacity(),
+                    111UL,
+                    common::errors::InvalidArgument(
+                        "The reserve_str's capacity should be 111, but got %d.",
+                        reserve_str.capacity()));  // align(100) - 1 = 111
+  // reserve more memory
+  reserve_str.reserve(200);
+  PADDLE_ENFORCE_EQ(reserve_str.capacity(),
+                    207UL,
+                    common::errors::InvalidArgument(
+                        "The reserve_str's capacity should be 207, but got %d.",
+                        reserve_str.capacity()));  // align(200) - 1 = 207
+
+  // Test operator<<
+  std::ostringstream oss1, oss2;
+  oss1 << long_str;
+  PADDLE_ENFORCE_EQ(
+      oss1.str(),
+      long_str,
+      common::errors::InvalidArgument(
+          "The oss1 should be '%s', but got '%s'.", long_str, oss1.str()));
+
+  // Test iterator
+  for (auto str_item : long_str) {
+    oss2 << str_item;
+  }
+  PADDLE_ENFORCE_EQ(
+      oss2.str(),
+      long_str,
+      common::errors::InvalidArgument(
+          "The oss2 should be '%s', but got '%s'.", long_str, oss2.str()));
+
+  // Test comparison operators
+  PADDLE_ENFORCE_EQ((long_str < short_str),
+                    true,
+                    common::errors::InvalidArgument(
+                        "The long_str should be less than short_str."));
+
+  PADDLE_ENFORCE_EQ((long_str > short_str),
+                    false,
+                    common::errors::InvalidArgument(
+                        "The long_str should not be greater than short_str."));
+  PADDLE_ENFORCE_EQ((long_str == short_str),
+                    false,
+                    common::errors::InvalidArgument(
+                        "The long_str should not be equal to short_str."));
+  PADDLE_ENFORCE_EQ((long_str != short_str),
+                    true,
+                    common::errors::InvalidArgument(
+                        "The long_str should not be equal to short_str."));
+  PADDLE_ENFORCE_EQ((short_str < long_str),
+                    false,
+                    common::errors::InvalidArgument(
+                        "The short_str should not be less than long_str."));
+  PADDLE_ENFORCE_EQ((short_str > long_str),
+                    true,
+                    common::errors::InvalidArgument(
+                        "The short_str should be greater than long_str."));
+  PADDLE_ENFORCE_EQ((move_nchar_str < plus_str),
+                    true,
+                    common::errors::InvalidArgument(
+                        "The move_nchar_str should be less than plus_str."));
+  PADDLE_ENFORCE_EQ((plus_str > move_nchar_str),
+                    true,
+                    common::errors::InvalidArgument(
+                        "The plus_str should be greater than move_nchar_str."));
+
+  // Test empty
+  PADDLE_ENFORCE_EQ(
+      empty_str.empty(),
+      true,
+      common::errors::InvalidArgument("The empty_str should be empty."));
+  PADDLE_ENFORCE_EQ(
+      nchar_str.empty(),
+      false,
+      common::errors::InvalidArgument("The nchar_str should not be empty."));
+  PADDLE_ENFORCE_EQ(empty_str.length(),
+                    0UL,
+                    common::errors::InvalidArgument(
+                        "The empty_str's length should be 0, but got %d.",
+                        empty_str.length()));
+
+  // Test Resize
+  nchar_str.resize(6, 'B');
+  PADDLE_ENFORCE_EQ(
+      nchar_str,
+      "AAAAAB",
+      common::errors::InvalidArgument(
+          "The nchar_str should be 'AAAAAB', but got '%s'.", nchar_str));
+
+  // Test operator =
+  long_str = std::move(nchar_str);
+  PADDLE_ENFORCE_EQ(
+      long_str,
+      "AAAAAB",
+      common::errors::InvalidArgument(
+          "The long_str should be 'AAAAAB', but got '%s'.", long_str));
+  long_str = short_str;
+  PADDLE_ENFORCE_EQ(
+      short_str,
+      long_str,
+      common::errors::InvalidArgument(
+          "The short_str should be '%s', but got '%s'.", long_str, short_str));
+  short_str = 'A';
+  PADDLE_ENFORCE_EQ(
+      short_str,
+      "A",
+      common::errors::InvalidArgument(
+          "The short_str should be 'A', but got '%s'.", short_str));
+  short_str = std::move(copy_nchar_str);
+  PADDLE_ENFORCE_EQ(
+      short_str,
+      "AAAAA",
+      common::errors::InvalidArgument(
+          "The short_str should be 'AAAAA', but got '%s'.", short_str));
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/core/test_tcp_store.cc

@@ -0,0 +1,54 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "gtest/gtest.h"
+#include "paddle/phi/core/distributed/store/tcp_store.h"
+#include "paddle/phi/core/distributed/store/tcp_utils.h"
+
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+namespace phi {
+namespace distributed {
+
+TEST(MasterDaemon, init) {
+  int socket = tcputils::tcp_listen("", std::to_string(0), AF_INET);
+  std::unique_ptr<detail::MasterDaemon> d =
+      detail::MasterDaemon::createDaemon(socket, 1, 100);
+  d->start();
+  printf("started to sleep 2s\n");
+#ifdef _WIN32
+  Sleep(2 * 1000);
+#else
+  usleep(2 * 1000 * 1000);
+#endif
+  printf("end to reset\n");
+
+  d.reset();
+}
+
+/* now for only c compile test
+TEST(TCPStore, init) {
+  TCPStore store("127.0.0.1", 6170, true, 1);
+  store.add("my", 3);
+  auto ret1 = store.get("my");
+  store.add("my", 3);
+  auto ret2 = store.get("my");
+  PADDLE_ENFORCE_EQ(ret1[0] + 3, ret2[0],
+    paddle::errors::Fatal("result of add is not right"));
+}
+*/
+}  // namespace distributed
+}  // namespace phi

test/cpp/phi/core/test_tensor_array.cc

@@ -0,0 +1,122 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <sstream>
+#include <string>
+#include <utility>
+
+#include "gtest/gtest.h"
+#include "paddle/common/errors.h"
+#include "paddle/phi/backends/all_context.h"
+#include "paddle/phi/core/enforce.h"
+#include "paddle/phi/core/tensor_array.h"
+#include "test/cpp/phi/core/allocator.h"
+
+namespace phi {
+namespace tests {
+
+using pstring = ::phi::dtype::pstring;
+
+TEST(tensor_array, tensor_array_not_init) {
+  const DDim dims({1, 2});
+  const DataType dtype{DataType::INT8};
+  const DataLayout layout{DataLayout::NHWC};
+  const LegacyLoD lod{};
+  DenseTensorMeta meta(dtype, dims, layout, lod);
+  DenseTensor tensor_0;
+  tensor_0.set_meta(meta);
+
+  std::vector<DenseTensor> tensors;
+  tensors.push_back(tensor_0);
+  tensors.push_back(tensor_0);
+  tensors.push_back(tensor_0);
+
+  TensorArray tensor_array(tensors);
+
+  try {
+    tensor_array.dims();
+  } catch (const common::enforce::EnforceNotMet& error) {
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("dims") != std::string::npos);
+  }
+
+  try {
+    tensor_array.place();
+  } catch (const common::enforce::EnforceNotMet& error) {
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("place") != std::string::npos);
+  }
+
+  try {
+    tensor_array.dtype();
+  } catch (const common::enforce::EnforceNotMet& error) {
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("dtype") != std::string::npos);
+  }
+
+  try {
+    tensor_array.layout();
+  } catch (const common::enforce::EnforceNotMet& error) {
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("layout") != std::string::npos);
+  }
+
+  try {
+    tensor_array.numel();
+  } catch (const common::enforce::EnforceNotMet& error) {
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("numel") != std::string::npos);
+  }
+
+  try {
+    tensor_array.valid();
+  } catch (const common::enforce::EnforceNotMet& error) {
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("valid") != std::string::npos);
+  }
+
+  EXPECT_TRUE(!tensor_array.initialized());
+}
+
+TEST(tensor_array, tensor_array_init) {
+  const DDim dims1({1, 2});
+  const DDim dims2({1, 2, 3});
+  const DataType dtype{DataType::INT8};
+  const DataLayout layout{DataLayout::NHWC};
+  const LegacyLoD lod{};
+
+  DenseTensorMeta meta1(dtype, dims1, layout, lod);
+  DenseTensorMeta meta2(dtype, dims2, layout, lod);
+
+  auto fancy_allocator = std::unique_ptr<Allocator>(new FancyAllocator);
+  auto* alloc = fancy_allocator.get();
+  DenseTensor tensor_0;
+  tensor_0.set_meta(meta1);
+
+  DenseTensor tensor_1;
+  tensor_1.set_meta(meta2);
+
+  std::vector<DenseTensor> tensors;
+  tensors.push_back(tensor_0);
+  tensors.push_back(tensor_1);
+  tensors.push_back(tensor_0);
+
+  TensorArray tensor_array(tensors);
+  tensor_array.AllocateFrom(alloc, DataType::INT8);
+
+  EXPECT_TRUE(tensor_array.initialized());
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/core/test_type_info.cc

@@ -0,0 +1,86 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <memory>
+#include "gtest/gtest.h"
+#include "paddle/phi/core/utils/type_registry.h"
+
+namespace phi {
+
+template <typename BaseT, typename DerivedT>
+const TypeInfo<BaseT> TypeInfoTraits<BaseT, DerivedT>::kType =
+    RegisterStaticType<BaseT>(DerivedT::name());
+
+template <typename BaseT, typename DerivedT>
+bool TypeInfoTraits<BaseT, DerivedT>::classof(const BaseT* obj) {
+  return obj->type_info() == kType;
+}
+
+template <typename BaseT, typename DerivedT>
+TypeInfoTraits<BaseT, DerivedT>::TypeInfoTraits() {
+  static_cast<BaseT*>(static_cast<DerivedT*>(this))->type_info_ = kType;
+}
+
+namespace tests {
+
+template <typename T>
+class Base {
+ public:
+  TypeInfo<Base<T>> type_info() const { return type_info_; }
+
+ private:
+  template <typename T1, typename T2>
+  friend class phi::TypeInfoTraits;
+  TypeInfo<Base<T>> type_info_{TypeInfo<Base<T>>::kUnknownType};
+};
+
+template <typename T>
+class DerivedA : public Base<T>, public TypeInfoTraits<Base<T>, DerivedA<T>> {
+ public:
+  static const char* name() { return "DerivedA"; }
+};
+
+template <typename T>
+class DerivedB : public Base<T>, public TypeInfoTraits<Base<T>, DerivedB<T>> {
+ public:
+  static const char* name() { return "DerivedB"; }
+};
+
+template <typename T>
+void check_type_info() {
+  std::unique_ptr<Base<T>> base(new Base<T>);
+  std::unique_ptr<Base<T>> derived_a(new DerivedA<T>);
+  std::unique_ptr<Base<T>> derived_b(new DerivedB<T>);
+
+  EXPECT_EQ(DerivedA<T>::classof(derived_a.get()), true);
+  EXPECT_EQ(DerivedB<T>::classof(derived_b.get()), true);
+  EXPECT_EQ(DerivedB<T>::classof(derived_a.get()), false);
+  EXPECT_EQ(DerivedA<T>::classof(derived_b.get()), false);
+
+  EXPECT_EQ(base->type_info().id(), 0);
+  EXPECT_EQ(derived_a->type_info().id(), 1);
+  EXPECT_EQ(derived_b->type_info().id(), 2);
+
+  EXPECT_EQ(base->type_info().name(), "Unknown");
+  EXPECT_EQ(derived_a->type_info().name(), "DerivedA");
+  EXPECT_EQ(derived_b->type_info().name(), "DerivedB");
+}
+
+TEST(type_info, base) {
+  check_type_info<int>();
+  check_type_info<float>();
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/core/timer.h

@@ -0,0 +1,39 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <chrono>  // NOLINT
+
+namespace phi {
+namespace tests {
+
+class Timer {
+ public:
+  std::chrono::high_resolution_clock::time_point start;
+  std::chrono::high_resolution_clock::time_point startu;
+
+  void tic() { start = std::chrono::high_resolution_clock::now(); }
+  double toc() {
+    startu = std::chrono::high_resolution_clock::now();
+    std::chrono::duration<double> time_span =
+        std::chrono::duration_cast<std::chrono::duration<double>>(startu -
+                                                                  start);
+    double used_time_ms = static_cast<double>(time_span.count()) * 1000.0;
+    return used_time_ms;
+  }
+};
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/core/unroll_array_ops_test.cc

@@ -0,0 +1,83 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/common/unroll_array_ops.h"
+
+#include <gtest/gtest.h>
+
+#include <array>
+
+namespace phi {
+namespace framework {
+
+template <typename T>
+bool CheckEquality(const T* p, size_t n, T val) {
+  return std::all_of(p, p + n, [val](const T& v) { return v == val; });
+}
+
+template <int D1, int D2>
+bool FillConstantTestMain() {
+  static_assert(D1 >= D2);
+  std::array<int, D1> arr = {};
+  arr.fill(0);
+
+  common::UnrollFillConstant<D2>::Run(arr.data(), 1);
+  return CheckEquality(arr.data(), D2, 1) &&
+         CheckEquality(arr.data() + D2, arr.size() - D2, 0);
+}
+
+TEST(unroll_ops, fill_constant) {
+  EXPECT_TRUE((FillConstantTestMain<9, 0>()));
+  EXPECT_TRUE((FillConstantTestMain<9, 1>()));
+  EXPECT_TRUE((FillConstantTestMain<9, 4>()));
+  EXPECT_TRUE((FillConstantTestMain<9, 9>()));
+}
+
+TEST(unroll_ops, assign) {
+  const int a[] = {1, 2, 3, 4, 5};  // NOLINT
+  int b[] = {0, 0, 0, 0, 0};        // NOLINT
+  common::UnrollAssign<3>::Run(a, b);
+  EXPECT_EQ(b[0], 1);
+  EXPECT_EQ(b[1], 2);
+  EXPECT_EQ(b[2], 3);
+  EXPECT_EQ(b[3], 0);
+  EXPECT_EQ(b[4], 0);
+}
+
+TEST(unroll_ops, var_args_assign) {
+  int a[] = {0, 0, 0};  // NOLINT
+  common::UnrollVarArgsAssign<int>::Run(a, 1, 2);
+  EXPECT_EQ(a[0], 1);
+  EXPECT_EQ(a[1], 2);
+  EXPECT_EQ(a[2], 0);
+}
+
+TEST(unroll_ops, compare) {
+  int a[] = {1, 2, 3};  // NOLINT
+  int b[] = {1, 2, 4};  // NOLINT
+  EXPECT_TRUE(common::UnrollCompare<2>::Run(a, b));
+  EXPECT_FALSE(common::UnrollCompare<3>::Run(a, b));
+
+  b[0] = -1;
+  EXPECT_TRUE(common::UnrollCompare<0>::Run(a, b));
+  EXPECT_FALSE(common::UnrollCompare<1>::Run(a, b));
+}
+
+TEST(unroll_ops, product) {
+  int a[] = {2, 3, 4};  // NOLINT
+  EXPECT_EQ(common::UnrollProduct<3>::Run(a), a[0] * a[1] * a[2]);
+}
+
+}  // namespace framework
+}  // namespace phi

test/cpp/phi/kernels/CMakeLists.txt

@@ -0,0 +1,136 @@
+cc_test(
+  test_math_function
+  SRCS test_math_function.cc
+  DEPS phi common)
+if(WITH_GPU)
+  nv_test(
+    test_math_function_gpu
+    SRCS test_math_function.cu
+    DEPS phi common)
+  nv_test(
+    test_broadcast_gpu
+    SRCS test_ternary_broadcast.cu
+    DEPS gtest)
+endif()
+if(WITH_ROCM)
+  hip_test(
+    test_math_function_gpu
+    SRCS test_math_function.cu
+    DEPS phi common)
+endif()
+
+cc_test(
+  test_cpu_vec
+  SRCS test_cpu_vec.cc
+  DEPS phi common)
+
+# For String Kernels
+if(WIN32)
+  cc_test(
+    test_strings_lower_upper_dev_api
+    SRCS test_strings_lower_upper_dev_api.cc
+    DEPS type_info common)
+else()
+  cc_test(
+    test_strings_lower_upper_dev_api
+    SRCS test_strings_lower_upper_dev_api.cc
+    DEPS phi common)
+endif()
+if(WITH_GPU)
+  nv_test(
+    test_strings_lower_upper_dev_gpu_api
+    SRCS test_strings_lower_upper_dev_api.cu
+    DEPS phi common)
+elseif(WITH_ROCM)
+  hip_test(
+    test_strings_lower_upper_dev_gpu_api
+    SRCS test_strings_lower_upper_dev_api.cu
+    DEPS phi common)
+endif()
+
+cc_test(
+  test_strings_copy_dev_api
+  SRCS test_strings_copy_dev_api.cc
+  DEPS phi common)
+if(WITH_GPU)
+  nv_test(
+    test_strings_copy_dev_gpu_api
+    SRCS test_strings_copy_dev_api.cu
+    DEPS phi common)
+elseif(WITH_ROCM)
+  hip_test(
+    test_strings_copy_dev_gpu_api
+    SRCS test_strings_copy_dev_api.cu
+    DEPS phi common)
+endif()
+
+if(WIN32)
+  cc_test(
+    test_memcpy_dev_api
+    SRCS test_memcpy_dev_api.cc
+    DEPS type_info common)
+  cc_test(
+    test_transfer_layout_dev_api
+    SRCS test_memcpy_dev_api.cc
+    DEPS type_info common)
+else()
+  cc_test(
+    test_memcpy_dev_api
+    SRCS test_memcpy_dev_api.cc
+    DEPS phi common)
+  cc_test(
+    test_transfer_layout_dev_api
+    SRCS test_transfer_layout_dev_api.cc
+    DEPS phi common)
+endif()
+
+if(WITH_GPU)
+  nv_test(
+    test_gpu_timer
+    SRCS test_gpu_timer.cu
+    DEPS gtest)
+  nv_test(
+    test_auto_tune
+    SRCS test_auto_tune.cu
+    DEPS gtest)
+  cc_test(
+    test_fused_adam_kernel
+    SRCS test_fused_adam_kernel.cc
+    DEPS gtest phi common)
+elseif(WITH_ROCM)
+  hip_test(
+    test_gpu_timer
+    SRCS test_gpu_timer.cu
+    DEPS gtest)
+  hip_test(
+    test_auto_tune
+    SRCS test_auto_tune.cu
+    DEPS gtest)
+endif()
+
+cc_test(
+  test_cache
+  SRCS test_cache.cc
+  DEPS gtest phi common)
+
+cc_test(
+  strided_memcpy_test
+  SRCS strided_memcpy_test.cc
+  DEPS phi common)
+
+if(WIN32)
+  cc_test(
+    sequence_padding_test
+    SRCS sequence_padding_test.cc
+    DEPS type_info common)
+else()
+  cc_test(
+    sequence_padding_test
+    SRCS sequence_padding_test.cc
+    DEPS phi common)
+endif()
+
+cc_test(
+  sequence_pooling_test
+  SRCS sequence_pooling_test.cc
+  DEPS phi common)

test/cpp/phi/kernels/sequence_padding_test.cc

@@ -0,0 +1,133 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+#include <gtest/gtest.h>
+
+#include "paddle/phi/kernels/funcs/sequence_padding.h"
+
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/core/tensor_utils.h"
+
+template <typename DeviceContext, typename T>
+void TestSequencePadding(const DeviceContext &context,
+                         const phi::LegacyLoD &lod,
+                         const size_t sequence_width) {
+  phi::DenseTensor cpu_seq;
+  phi::DenseTensor cpu_seq_back;
+  phi::DenseTensor seq;
+  phi::DenseTensor seq_back;
+  phi::DenseTensor padding;
+  phi::DenseTensor cpu_pad_value;
+  phi::DenseTensor pad_value;
+
+  const size_t level = lod.size() - 1;
+  auto seq_dims = common::make_ddim({static_cast<int64_t>(lod[level].back()),
+                                     static_cast<int64_t>(sequence_width)});
+
+  cpu_seq.set_lod(lod);
+  auto *dev_ctx = static_cast<phi::CPUContext *>(
+      phi::DeviceContextPool::Instance().Get(phi::CPUPlace()));
+  cpu_seq.Resize(seq_dims);
+  dev_ctx->template Alloc<T>(&cpu_seq);
+
+  for (int64_t i = 0; i < cpu_seq.numel(); ++i) {
+    cpu_seq.data<T>()[i] = static_cast<T>(i);
+  }
+
+  auto place = context.GetPlace();
+  if (place.GetType() == phi::AllocationType::CPU) {
+    seq = cpu_seq;
+  } else {
+    phi::Copy(context, cpu_seq, place, true, &seq);
+    seq.set_lod(lod);
+  }
+
+  const size_t max_sequence_length =
+      phi::funcs::MaximumSequenceLength(lod[level]);
+  const size_t num_sequences = lod[level].size() - 1;
+  auto padding_dims =
+      common::make_ddim({static_cast<int64_t>(max_sequence_length),
+                         static_cast<int64_t>(num_sequences),
+                         static_cast<int64_t>(sequence_width)});
+
+  padding.Resize(padding_dims);
+  context.template Alloc<T>(&padding);
+
+  cpu_pad_value.Resize({1});
+  T *pad_value_data = dev_ctx->template Alloc<T>(&cpu_pad_value);
+  *pad_value_data = static_cast<T>(0);
+  if (place.GetType() == phi::AllocationType::CPU) {
+    pad_value = cpu_pad_value;
+  } else {
+    phi::Copy(context, cpu_pad_value, place, true, &pad_value);
+  }
+
+  phi::funcs::PaddingDenseTensorFunctor<DeviceContext, T>()(
+      context,
+      seq,
+      &padding,
+      pad_value,
+      -1,
+      0,
+      false,
+      phi::funcs::kLengthBatchWidth);
+
+  seq_back.set_lod(lod);
+  seq_back.Resize(seq_dims);
+  context.template Alloc<T>(&seq_back);
+  phi::funcs::UnpaddingDenseTensorFunctor<DeviceContext, T>()(
+      context, padding, &seq_back, -1, 0, false, phi::funcs::kLengthBatchWidth);
+
+  if (place.GetType() == phi::AllocationType::CPU) {
+    cpu_seq_back = seq_back;
+  } else {
+    phi::Copy(context, seq_back, phi::CPUPlace(), true, &cpu_seq_back);
+    cpu_seq_back.set_lod(lod);
+  }
+
+  EXPECT_EQ(cpu_seq.numel(), cpu_seq_back.numel());
+  EXPECT_EQ(cpu_seq.dims(), cpu_seq_back.dims());
+  for (int64_t i = 0; i < cpu_seq.numel(); ++i) {
+    EXPECT_EQ(cpu_seq.data<T>()[i], cpu_seq_back.data<T>()[i]);
+  }
+}
+
+TEST(Seq2BatchPadding, CPU) {
+  auto place = phi::CPUPlace();
+  auto *context = static_cast<phi::CPUContext *>(
+      phi::DeviceContextPool::Instance().Get(place));
+
+  phi::LegacyLoD lod1;
+  lod1.push_back(std::vector<size_t>{0, 10});
+  TestSequencePadding<phi::CPUContext, float>(*context, lod1, 16);
+
+  phi::LegacyLoD lod2;
+  lod2.push_back(std::vector<size_t>{0, 2, 7, 10});
+  TestSequencePadding<phi::CPUContext, float>(*context, lod2, 128);
+}
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+TEST(SequencePadding, CUDA) {
+  auto place = phi::GPUPlace(0);
+  auto *context = static_cast<phi::GPUContext *>(
+      phi::DeviceContextPool::Instance().Get(place));
+
+  phi::LegacyLoD lod1;
+  lod1.push_back(std::vector<size_t>{0, 10});
+  TestSequencePadding<phi::GPUContext, float>(*context, lod1, 16);
+
+  phi::LegacyLoD lod2;
+  lod2.push_back(std::vector<size_t>{0, 2, 7, 10});
+  TestSequencePadding<phi::GPUContext, float>(*context, lod2, 128);
+}
+#endif

test/cpp/phi/kernels/sequence_pooling_test.cc

@@ -0,0 +1,148 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/common/place.h"
+#include "paddle/phi/core/tensor_utils.h"
+#include "paddle/phi/kernels/funcs/sequence_pooling.h"
+
+template <typename DeviceContext, typename T>
+void TestSequencePoolingSum(const DeviceContext &context,
+                            const phi::LegacyLoD &lod,
+                            const int64_t second_dim) {
+  phi::DenseTensor cpu_out_grad;
+  phi::DenseTensor cpu_in_grad;
+  phi::DenseTensor out_grad;
+  phi::DenseTensor in_grad;
+
+  // construct out_grad's tensor in cpu
+  const size_t out_first_dim = lod[0].size() - 1;
+  auto out_dims =
+      common::make_ddim({static_cast<int64_t>(out_first_dim), second_dim});
+
+  cpu_out_grad.mutable_data<T>(out_dims, phi::CPUPlace());
+  for (int64_t i = 0; i < cpu_out_grad.numel(); ++i) {
+    cpu_out_grad.data<T>()[i] = static_cast<T>(i);
+  }
+
+  // copy to dst out_grad
+  auto place = context.GetPlace();
+  if (place == phi::CPUPlace()) {
+    out_grad = cpu_out_grad;
+  } else {
+    phi::Copy(context, cpu_out_grad, place, true, &out_grad);
+  }
+
+  // construct in_grad
+  in_grad.set_lod(lod);
+  auto in_dims =
+      common::make_ddim({static_cast<int64_t>(lod[0].back()), second_dim});
+  in_grad.mutable_data<T>(in_dims, place);
+
+  // check tensor construction result
+  PADDLE_ENFORCE_EQ(
+      in_grad.dims().size(),
+      out_grad.dims().size(),
+      common::errors::InvalidArgument(
+          "The dimension of input and output shall be same. Expected %ld == "
+          "%ld, but got %ld != %ld. Please check the input value.",
+          in_grad.dims().size(),
+          out_grad.dims().size(),
+          in_grad.dims().size(),
+          out_grad.dims().size()));
+  for (int64_t i = 1; i < out_grad.dims().size(); ++i) {
+    PADDLE_ENFORCE_EQ(
+        in_grad.dims()[i],
+        out_grad.dims()[i],
+        common::errors::InvalidArgument(
+            "The dimension of input and output shall be same. Expected %ld == "
+            "%ld, but got %ld != %ld. Please check the input value.",
+            in_grad.dims()[i],
+            out_grad.dims()[i],
+            in_grad.dims()[i],
+            out_grad.dims()[i]));
+  }
+
+  // call functor
+  phi::funcs::SequencePoolGradFunctor<DeviceContext, T>()(
+      context, "SUM", out_grad, &in_grad);
+
+  if (place == phi::CPUPlace()) {
+    cpu_in_grad = in_grad;
+  } else {
+    phi::Copy(context, in_grad, phi::CPUPlace(), true, &cpu_in_grad);
+    cpu_in_grad.set_lod(in_grad.lod());
+  }
+
+  EXPECT_EQ(in_grad.numel(), static_cast<int64_t>(lod[0].back() * second_dim));
+  EXPECT_EQ(in_grad.lod(), lod);
+
+  if (place == phi::CPUPlace()) {
+    for (size_t i = 0; i < in_grad.lod()[0].size() - 1; ++i) {
+      int64_t begin = static_cast<int64_t>(in_grad.lod()[0][i]);
+      int64_t end = static_cast<int64_t>(in_grad.lod()[0][i + 1]);
+      phi::DenseTensor tmp = in_grad.Slice(begin, end);
+      for (int64_t j = 0; j != tmp.numel() / second_dim; ++j) {
+        for (int64_t m = 0; m != second_dim; ++m) {
+          EXPECT_EQ(tmp.data<T>()[m + j * second_dim],
+                    out_grad.data<T>()[m + i * second_dim]);
+        }
+      }
+    }
+  } else {
+    for (size_t i = 0; i < cpu_in_grad.lod()[0].size() - 1; ++i) {
+      int64_t begin = static_cast<int64_t>(cpu_in_grad.lod()[0][i]);
+      int64_t end = static_cast<int64_t>(cpu_in_grad.lod()[0][i + 1]);
+      phi::DenseTensor tmp = cpu_in_grad.Slice(begin, end);
+      for (int64_t j = 0; j != tmp.numel() / second_dim; ++j) {
+        for (int64_t m = 0; m != second_dim; ++m) {
+          EXPECT_EQ(tmp.data<T>()[m + j * second_dim],
+                    cpu_out_grad.data<T>()[m + i * second_dim]);
+        }
+      }
+    }
+  }
+}
+
+TEST(SequencePoolingGrad, CPU_SUM) {
+  auto place = phi::CPUPlace();
+  auto *context = static_cast<phi::CPUContext *>(
+      phi::DeviceContextPool::Instance().Get(place));
+
+  phi::LegacyLoD lod1;
+  lod1.push_back(std::vector<size_t>{0, 10});
+  TestSequencePoolingSum<phi::CPUContext, float>(*context, lod1, 128);
+
+  phi::LegacyLoD lod2;
+  lod2.push_back(std::vector<size_t>{0, 2, 7, 10});
+  TestSequencePoolingSum<phi::CPUContext, float>(*context, lod2, 128);
+}
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+TEST(SequencePoolingGrad, CUDA_SUM) {
+  auto place = phi::GPUPlace(0);
+  auto *context = static_cast<phi::GPUContext *>(
+      phi::DeviceContextPool::Instance().Get(place));
+
+  phi::LegacyLoD lod1;
+  lod1.push_back(std::vector<size_t>{0, 10});
+  TestSequencePoolingSum<phi::GPUContext, float>(*context, lod1, 128);
+
+  phi::LegacyLoD lod2;
+  lod2.push_back(std::vector<size_t>{0, 2, 7, 10});
+  TestSequencePoolingSum<phi::GPUContext, float>(*context, lod2, 128);
+}
+#endif

test/cpp/phi/kernels/strided_memcpy_test.cc

@@ -0,0 +1,172 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/kernels/funcs/strided_memcpy.h"
+#include <array>
+
+#include "gtest/gtest.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/common/memory_utils.h"
+namespace phi {
+namespace tests {
+
+TEST(StridedMemcpy, CPUCrop) {
+  // clang-format off
+  int src[] = {// NOLINT
+      0, 1, 2, 0, 0,
+      0, 3, 4, 0, 0,
+      0, 0, 0, 0, 0,
+  };
+  // clang-format on
+
+  phi::DDim src_stride({5, 1});
+
+  std::array<int, 4> dst = {};
+  phi::DDim dst_dim({2, 2});
+  phi::DDim dst_stride({2, 1});
+
+  phi::CPUContext ctx;
+  phi::funcs::StridedMemcpy<int>(
+      ctx, src + 1, src_stride, dst_dim, dst_stride, dst.data());
+
+  ASSERT_EQ(1, dst[0]);
+  ASSERT_EQ(2, dst[1]);
+  ASSERT_EQ(3, dst[2]);
+  ASSERT_EQ(4, dst[3]);
+}
+
+TEST(StridedMemcpy, CPUConcat) {
+  // clang-format off
+  int src[] = { // NOLINT
+      1, 2,
+      3, 4
+  };
+  // clang-format on
+
+  std::array<int, 8> dst = {};
+  phi::DDim src_stride({2, 1});
+  phi::DDim dst_dim({2, 2});
+  phi::DDim dst_stride({4, 1});
+  phi::CPUContext ctx;
+
+  phi::funcs::StridedMemcpy<int>(
+      ctx, src, src_stride, dst_dim, dst_stride, dst.data());
+  phi::funcs::StridedMemcpy<int>(
+      ctx, src, src_stride, dst_dim, dst_stride, dst.data() + 2);
+
+  // clang-format off
+  int expect_dst[] = { // NOLINT
+      1, 2, 1, 2,
+      3, 4, 3, 4
+  };
+  // clang-format on
+  for (size_t i = 0; i < sizeof(expect_dst) / sizeof(int); ++i) {
+    ASSERT_EQ(expect_dst[i], dst[i]);
+  }
+}
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+TEST(StridedMemcpy, GPUCrop) {
+  // clang-format off
+  std::array<int, 15> src = {
+      0, 1, 2, 0, 0,
+      0, 3, 4, 0, 0,
+      0, 0, 0, 0, 0,
+  };
+  // clang-format on
+
+  phi::GPUPlace gpu0(0);
+  phi::CPUPlace cpu;
+
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* ctx = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  auto src_allocation = phi::memory_utils::Alloc(gpu0, sizeof(src));
+
+  int* gpu_src = reinterpret_cast<int*>(src_allocation->ptr());
+  memory_utils::Copy(
+      gpu0, gpu_src, cpu, src.data(), sizeof(src), ctx->stream());
+
+  phi::DDim src_stride({5, 1});
+
+  std::array<int, 4> dst = {};
+  auto dst_allocation = phi::memory_utils::Alloc(gpu0, sizeof(dst));
+  int* gpu_dst = reinterpret_cast<int*>(dst_allocation->ptr());
+
+  phi::DDim dst_dim({2, 2});
+  phi::DDim dst_stride({2, 1});
+
+  phi::funcs::StridedMemcpy<int>(
+      *ctx, gpu_src + 1, src_stride, dst_dim, dst_stride, gpu_dst);
+
+  memory_utils::Copy(
+      cpu, dst.data(), gpu0, gpu_dst, sizeof(dst), ctx->stream());
+  ctx->Wait();
+
+  ASSERT_EQ(1, dst[0]);
+  ASSERT_EQ(2, dst[1]);
+  ASSERT_EQ(3, dst[2]);
+  ASSERT_EQ(4, dst[3]);
+}
+
+TEST(StridedMemcpy, GPUConcat) {
+  // clang-format off
+  std::array<int, 4> src = {
+      1, 2,
+      3, 4
+  };
+  // clang-format on
+
+  phi::GPUPlace gpu0(0);
+  phi::CPUPlace cpu;
+
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* ctx = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  auto gpu_src_allocation = phi::memory_utils::Alloc(gpu0, sizeof(src));
+  int* gpu_src = reinterpret_cast<int*>(gpu_src_allocation->ptr());
+  memory_utils::Copy(
+      gpu0, gpu_src, cpu, src.data(), sizeof(src), ctx->stream());
+
+  std::array<int, 8> dst = {};
+  auto gpu_dst_allocation = phi::memory_utils::Alloc(gpu0, sizeof(dst));
+  int* gpu_dst = reinterpret_cast<int*>(gpu_dst_allocation->ptr());
+
+  phi::DDim src_stride({2, 1});
+  phi::DDim dst_dim({2, 2});
+  phi::DDim dst_stride({4, 1});
+
+  phi::funcs::StridedMemcpy<int>(
+      *ctx, gpu_src, src_stride, dst_dim, dst_stride, gpu_dst);
+  phi::funcs::StridedMemcpy<int>(
+      *ctx, gpu_src, src_stride, dst_dim, dst_stride, gpu_dst + 2);
+
+  memory_utils::Copy(
+      cpu, dst.data(), gpu0, gpu_dst, sizeof(dst), ctx->stream());
+  ctx->Wait();
+
+  // clang-format off
+  std::array<int, 8> expect_dst = {
+      1, 2, 1, 2,
+      3, 4, 3, 4
+  };
+  // clang-format on
+  for (size_t i = 0; i < sizeof(expect_dst) / sizeof(int); ++i) {
+    ASSERT_EQ(expect_dst[i], dst[i]);
+  }
+}
+
+#endif
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/kernels/test_auto_tune.cu

@@ -0,0 +1,137 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include "glog/logging.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/backends/gpu/gpu_launch_config.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/tensor_meta.h"
+#include "paddle/phi/core/tensor_utils.h"
+#include "paddle/phi/kernels/autotune/auto_tune_base.h"
+#include "paddle/phi/kernels/funcs/aligned_vector.h"
+
+namespace tune = phi::autotune;
+
+template <typename T, int VecSize>
+__global__ void VecSumTest(const T* x, T* y, int N) {
+#ifdef __HIPCC__
+  int idx = hipBlockDim_x * hipBlockIdx_x + hipThreadIdx_x;
+#else
+  int idx = blockDim.x * blockIdx.x + threadIdx.x;
+#endif
+  using LoadT = phi::AlignedVector<T, VecSize>;
+  for (int i = idx * VecSize; i < N; i += blockDim.x * gridDim.x * VecSize) {
+    LoadT x_vec;
+    LoadT y_vec;
+    phi::Load<T, VecSize>(&x[i], &x_vec);
+    phi::Load<T, VecSize>(&y[i], &y_vec);
+#pragma unroll
+    for (int j = 0; j < VecSize; j++) {
+      y_vec[j] = x_vec[j] + y_vec[j];
+    }
+    phi::Store<T, VecSize>(y_vec, &y[i]);
+  }
+}
+
+template <int Vecsize>
+float Algo(const phi::GPUContext& ctx,
+           const phi::DenseTensor& d_in,
+           phi::DenseTensor* d_out,
+           size_t N,
+           size_t threads,
+           size_t blocks) {
+  const float* d_in_data = d_in.data<float>();
+  float* d_out_data = d_out->data<float>();
+#ifdef __HIPCC__
+  hipLaunchKernelGGL(HIP_KERNEL_NAME(VecSumTest<float, Vecsize>),
+                     dim3(blocks),
+                     dim3(threads),
+                     0,
+                     0,
+                     d_in_data,
+                     d_out_data,
+                     N);
+#else
+  VLOG(3) << "Vecsize is " << Vecsize;
+  VecSumTest<float, Vecsize>
+      <<<blocks, threads, 0, ctx.stream()>>>(d_in_data, d_out_data, N);
+#endif
+  return Vecsize;
+}
+
+TEST(AutoTune, sum) {
+  int64_t N = 1 << 20;
+  size_t blocks = 512;
+  size_t threads = 256;
+  size_t size = sizeof(float) * N;
+
+  const auto alloc_cpu =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace());
+  auto in1 = std::make_shared<phi::DenseTensor>(
+      alloc_cpu.get(),
+      phi::DenseTensorMeta(phi::DataType::FLOAT32,
+                           common::make_ddim({N}),
+                           phi::DataLayout::NCHW));
+  auto in2 = std::make_shared<phi::DenseTensor>(
+      alloc_cpu.get(),
+      phi::DenseTensorMeta(phi::DataType::FLOAT32,
+                           common::make_ddim({N}),
+                           phi::DataLayout::NCHW));
+
+  float* in1_data = in1->data<float>();
+  float* in2_data = in2->data<float>();
+  for (size_t i = 0; i < N; i++) {
+    in1_data[i] = 1.0f;
+    in2_data[i] = 2.0f;
+  }
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  const auto alloc_cuda =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::GPUPlace());
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto place = phi::GPUPlace();
+  auto* dev_ctx = static_cast<const phi::GPUContext*>(pool.GetByPlace(place));
+  auto stream = dev_ctx->stream();
+
+  auto d_in1 = std::make_shared<phi::DenseTensor>(
+      alloc_cuda.get(),
+      phi::DenseTensorMeta(phi::DataType::FLOAT32,
+                           common::make_ddim({N}),
+                           phi::DataLayout::NCHW));
+  auto d_in2 = std::make_shared<phi::DenseTensor>(
+      alloc_cuda.get(),
+      phi::DenseTensorMeta(phi::DataType::FLOAT32,
+                           common::make_ddim({N}),
+                           phi::DataLayout::NCHW));
+  phi::Copy(*dev_ctx, *in1.get(), phi::GPUPlace(), false, d_in1.get());
+  phi::Copy(*dev_ctx, *in2.get(), phi::GPUPlace(), false, d_in2.get());
+
+  // 1. Test call_back.
+  VLOG(3) << ">>> [CallBack]: Test case.";
+  auto callback1 = tune::MakeCallback<float>(Algo<4>);
+  auto callback2 = tune::MakeCallback<float>(Algo<2>);
+  auto callback3 = tune::MakeCallback<float>(Algo<1>);
+  std::vector<decltype(callback1)> callbacks{callback1, callback2, callback3};
+  for (int i = 0; i < callbacks.size(); ++i) {
+    dev_ctx->Wait();
+    phi::GpuTimer timer;
+    timer.Start(0);
+    callbacks[i].Run(*dev_ctx, *d_in1.get(), d_in2.get(), N, threads, blocks);
+    timer.Stop(0);
+    VLOG(3) << "kernel[" << i << "]: time cost is " << timer.ElapsedTime();
+  }
+#endif
+}

test/cpp/phi/kernels/test_cache.cc

@@ -0,0 +1,68 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include <cmath>
+#include <functional>
+
+#include "paddle/phi/kernels/autotune/cache.h"
+
+#include "glog/logging.h"
+
+enum ConvAlgos { GEMMKernel = 0, CuDNNKernel_1 = 1, CuDNNKernel_2 = 2 };
+
+TEST(AlgosCache, AlgosCache) {
+  auto autotune_cache = phi::autotune::AutoTuneCache::Instance();
+  auto& cache =
+      autotune_cache.GetConv(phi::autotune::AlgorithmType::kConvForward);
+
+  std::vector<int64_t> x_shape = {4, 224, 224, 3};
+  std::vector<int64_t> w_shape = {32, 3, 3, 3};
+  std::vector<int> paddings = {0, 0};
+  std::vector<int> strides = {2, 2};
+  std::vector<int> dilations = {1, 1};
+  phi::DataType dtype = phi::CppTypeToDataType<float>::Type();
+
+  phi::autotune::ConvCacheKey key(
+      x_shape, w_shape, paddings, strides, dilations, dtype, 0, 0);
+  EXPECT_EQ(cache.Find(key), false);
+  phi::autotune::ConvAutoTuneResult node(
+      static_cast<int64_t>(ConvAlgos::GEMMKernel), 0, false);
+  cache.Set(key, node);
+  EXPECT_EQ(cache.Size(), 1);
+  EXPECT_EQ(cache.Find(key), true);
+  auto algo = cache.Get(key);
+  EXPECT_EQ(algo.algo, ConvAlgos::GEMMKernel);
+
+  x_shape = {4, 128, 128, 3};
+  phi::autotune::ConvCacheKey key1(
+      x_shape, w_shape, paddings, strides, dilations, dtype, 0, 1);
+  EXPECT_EQ(cache.Find(key1), false);
+  phi::autotune::ConvAutoTuneResult node1(
+      static_cast<int64_t>(ConvAlgos::CuDNNKernel_1), 0, false);
+  cache.Set(key1, node1);
+  EXPECT_EQ(cache.Size(), 2);
+  EXPECT_EQ(cache.CacheHits(), 1);
+  EXPECT_EQ(cache.CacheMisses(), 2);
+
+  float cache_hit_rate = static_cast<float>(1) / static_cast<float>(3);
+  EXPECT_LT(std::abs(cache_hit_rate - cache.CacheHitRate()), 1e-5);
+
+  autotune_cache.UpdateStatus();
+  EXPECT_EQ(autotune_cache.Size(), 2);
+  EXPECT_EQ(autotune_cache.CacheHits(), 1);
+  EXPECT_EQ(autotune_cache.CacheMisses(), 2);
+  EXPECT_LT(std::abs(cache_hit_rate - autotune_cache.CacheHitRate()), 1e-5);
+}

test/cpp/phi/kernels/test_cpu_vec.cc

@@ -0,0 +1,346 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <cmath>
+#include <cstring>
+#include <random>
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/phi/common/port.h"
+#include "paddle/phi/kernels/funcs/cpu_vec.h"
+
+namespace phi {
+namespace tests {
+
+inline double GetCurrentUS() {
+  struct timeval time = {};
+  gettimeofday(&time, nullptr);
+  return 1e+6 * time.tv_sec + time.tv_usec;  // NOLINT
+}
+constexpr int repeat = 1000;
+
+template <typename T>
+inline T _sigmoid(T x) {
+  const T min = SIGMOID_THRESHOLD_MIN;
+  const T max = SIGMOID_THRESHOLD_MAX;
+  T tmp = (x < min) ? min : ((x > max) ? max : x);
+  return static_cast<T>(1) / (static_cast<T>(1) + std::exp(-tmp));
+}
+
+template <typename T>
+inline T _tanh(T x) {
+  return static_cast<T>(2) * _sigmoid<T>(static_cast<T>(2) * x) -
+         static_cast<T>(1);
+}
+
+template <typename T>
+void ref_sigmoid(const int n, const T* x, T* y) {
+  for (int i = 0; i < n; ++i) {
+    y[i] = _sigmoid(x[i]);
+  }
+}
+
+template <typename T>
+void ref_tanh(const int n, const T* x, T* y) {
+  for (int i = 0; i < n; ++i) {
+    y[i] = _tanh(x[i]);
+  }
+}
+template <typename T>
+void ref_relu(const int n, const T* x, T* y) {
+  for (int i = 0; i < n; ++i) {
+    y[i] = x[i] > 0 ? x[i] : 0;
+  }
+}
+
+template <typename T>
+void RandomVec(const int n,
+               T* a,
+               const T lower = static_cast<T>(-20.f),
+               const T upper = static_cast<T>(20.f)) {
+  static unsigned int seed = 100;
+  std::mt19937 rng(seed++);
+  std::uniform_real_distribution<double> uniform_dist(0, 1);
+  for (int i = 0; i < n; ++i) {
+    a[i] = static_cast<T>(uniform_dist(rng) * (upper - lower) + lower);
+  }
+}
+
+template <typename T>
+void TestAndBench(const int n,
+                  std::function<void(const int, const T*, T*)> tgt,
+                  std::function<void(const int, const T*, T*)> ref) {
+  std::vector<T> x(n);
+  std::vector<T> ytgt(n), yref(n);
+  RandomVec<T>(n, x.data());
+
+  const T* x_data = x.data();
+  T* ytgt_data = ytgt.data();
+  T* yref_data = yref.data();
+  auto st = GetCurrentUS();
+  for (int i = 0; i < repeat; ++i) {
+    tgt(n, x_data, ytgt_data);
+  }
+  auto mt = GetCurrentUS();
+  for (int i = 0; i < repeat; ++i) {
+    ref(n, x_data, yref_data);
+  }
+  auto et = GetCurrentUS();
+
+  VLOG(3) << "Vec size " << n << ": refer takes: " << (et - mt) / repeat
+          << " us, tgt takes: " << (mt - st) / repeat;
+  for (int i = 0; i < n; ++i) {
+    EXPECT_NEAR(ytgt_data[i], yref_data[i], 1e-3);
+  }
+}
+
+TEST(CpuVecTest, sigmoid) {
+  using namespace phi::funcs;  // NOLINT
+  for (auto sz : {1, 2, 15, 16, 30, 32, 128, 200, 512}) {
+    TestAndBench<float>(sz, vec_sigmoid<float>, ref_sigmoid<float>);
+    TestAndBench<float>(
+        sz, vec_sigmoid<float, backends::cpu::avx>, ref_sigmoid<float>);
+    TestAndBench<float>(
+        sz, vec_sigmoid<float, backends::cpu::avx2>, ref_sigmoid<float>);
+    TestAndBench<float>(
+        sz, vec_sigmoid<float, backends::cpu::avx512f>, ref_sigmoid<float>);
+  }
+  TestAndBench<double>(30, vec_sigmoid<double>, ref_sigmoid<double>);
+}
+
+TEST(CpuVecTest, tanh) {
+  using namespace phi::funcs;  // NOLINT
+  for (auto sz : {1, 2, 15, 16, 30, 32, 128, 200, 512}) {
+    TestAndBench<float>(sz, vec_tanh<float>, ref_tanh<float>);
+    TestAndBench<float>(
+        sz, vec_tanh<float, backends::cpu::avx>, ref_tanh<float>);
+    TestAndBench<float>(
+        sz, vec_tanh<float, backends::cpu::avx2>, ref_tanh<float>);
+    TestAndBench<float>(
+        sz, vec_tanh<float, backends::cpu::avx512f>, ref_tanh<float>);
+  }
+  TestAndBench<double>(30, vec_tanh<double>, ref_tanh<double>);
+}
+
+TEST(CpuVecTest, relu) {
+  using namespace phi::funcs;  // NOLINT
+  for (auto sz : {1, 2, 15, 16, 30, 32, 128, 200, 512}) {
+    TestAndBench<float>(sz, vec_relu<float>, ref_relu<float>);
+    TestAndBench<float>(
+        sz, vec_relu<float, backends::cpu::avx>, ref_relu<float>);
+    TestAndBench<float>(
+        sz, vec_relu<float, backends::cpu::avx2>, ref_relu<float>);
+    TestAndBench<float>(
+        sz, vec_relu<float, backends::cpu::avx512f>, ref_relu<float>);
+  }
+  TestAndBench<double>(30, vec_relu<double>, ref_relu<double>);
+}
+
+template <typename T>
+void compare_sum(size_t n,
+                 std::function<void(const size_t, const T*, T*)> tgt,
+                 std::function<void(const size_t, const T*, T*)> ref) {
+  std::vector<T> x(n);
+  T ytgt_data, yref_data;
+  RandomVec<T>(n, x.data(), static_cast<T>(-2), static_cast<T>(2));
+
+  const T* x_data = x.data();
+  tgt(n, x_data, &ytgt_data);
+  ref(n, x_data, &yref_data);
+  EXPECT_NEAR(ytgt_data, yref_data, 1e-3);
+}
+
+TEST(CpuVecTest, vec_sum) {
+  using namespace phi::funcs;  // NOLINT
+  for (size_t sz : {1, 2, 15, 16, 30, 32, 128, 200, 512}) {
+    compare_sum<float>(
+        sz, vec_sum<float>, vec_sum<float, backends::cpu::isa_any>);
+    compare_sum<float>(sz,
+                       vec_sum<float, backends::cpu::avx>,
+                       vec_sum<float, backends::cpu::isa_any>);
+  }
+  compare_sum<double>(
+      30U, vec_sum<double>, vec_sum<double, backends::cpu::isa_any>);
+}
+
+template <typename T>
+void compare_clip(
+    size_t n,
+    T threshold,
+    std::function<void(const size_t, const T, const T*, T*)> tgt,
+    std::function<void(const size_t, const T, const T*, T*)> ref) {
+  std::vector<T> x(n);
+  std::vector<T> ytgt(n), yref(n);
+  RandomVec<T>(n, x.data(), static_cast<T>(-2), static_cast<T>(2));
+
+  const T* x_data = x.data();
+  T* yref_data = yref.data();
+  T* ytgt_data = ytgt.data();
+  tgt(n, threshold, x_data, ytgt_data);
+  ref(n, threshold, x_data, yref_data);
+  for (size_t i = 0; i < n; ++i) {
+    EXPECT_NEAR(ytgt_data[i], yref_data[i], 1e-3);
+  }
+}
+
+TEST(CpuVecTest, vec_clip) {
+  using namespace phi::funcs;  // NOLINT
+  for (size_t sz : {1, 2, 15, 16, 30, 32, 128, 200, 512}) {
+    compare_clip<float>(
+        sz, -4.f, vec_clip<float>, vec_clip<float, backends::cpu::isa_any>);
+    compare_clip<float>(sz,
+                        -1.1f,
+                        vec_clip<float, backends::cpu::avx>,
+                        vec_clip<float, backends::cpu::isa_any>);
+  }
+  compare_clip<double>(
+      30U, 1.0, vec_clip<double>, vec_clip<double, backends::cpu::isa_any>);
+}
+
+template <typename T>
+void compare_mul(
+    size_t n,
+    std::function<void(const size_t, const T*, const T*, T*)> tgt,
+    std::function<void(const size_t, const T*, const T*, T*)> ref) {
+  std::vector<T> x(n), y(n);
+  std::vector<T> ztgt(n), zref(n);
+
+  RandomVec<T>(n, x.data(), static_cast<T>(-2), static_cast<T>(2));
+  RandomVec<T>(n, y.data(), static_cast<T>(-2), static_cast<T>(2));
+
+  const T* x_data = x.data();
+  const T* y_data = y.data();
+  T* ztgt_data = ztgt.data();
+  T* zref_data = zref.data();
+
+  tgt(n, x_data, y_data, ztgt_data);
+  ref(n, x_data, y_data, zref_data);
+  for (size_t i = 0; i < n; ++i) {
+    EXPECT_NEAR(ztgt_data[i], zref_data[i], 1e-3);
+  }
+}
+
+TEST(CpuVecTest, vec_mul) {
+  using namespace phi::funcs;  // NOLINT
+  for (size_t sz : {1, 2, 15, 16, 30, 32, 128, 200, 512}) {
+    compare_mul<float>(
+        sz, vec_mul<float>, vec_mul<float, backends::cpu::isa_any>);
+    compare_mul<float>(sz,
+                       vec_mul<float, backends::cpu::avx>,
+                       vec_mul<float, backends::cpu::isa_any>);
+  }
+  compare_mul<double>(
+      30U, vec_mul<double>, vec_mul<double, backends::cpu::isa_any>);
+}
+
+template <typename T>
+void compare_mul_reduce(
+    size_t n,
+    std::function<void(const size_t, const T*, const T*, T*)> tgt,
+    std::function<void(const size_t, const T*, const T*, T*)> ref) {
+  std::vector<T> x(n), y(n);
+  T ztgt_data, zref_data;
+
+  RandomVec<T>(n, x.data(), static_cast<T>(-2), static_cast<T>(2));
+  RandomVec<T>(n, y.data(), static_cast<T>(-2), static_cast<T>(2));
+
+  const T* x_data = x.data();
+  const T* y_data = y.data();
+
+  tgt(n, x_data, y_data, &ztgt_data);
+  ref(n, x_data, y_data, &zref_data);
+  EXPECT_NEAR(ztgt_data, zref_data, 1e-3);
+}
+
+TEST(CpuVecTest, vec_mul_reduce) {
+  using namespace phi::funcs;  // NOLINT
+  for (size_t sz : {1, 2, 15, 16, 30, 32, 128, 200, 512}) {
+    compare_mul_reduce<float>(sz,
+                              vec_mul_reduce<float>,
+                              vec_mul_reduce<float, backends::cpu::isa_any>);
+    compare_mul_reduce<float>(sz,
+                              vec_mul_reduce<float, backends::cpu::avx>,
+                              vec_mul_reduce<float, backends::cpu::isa_any>);
+  }
+  compare_mul_reduce<double>(30U,
+                             vec_mul_reduce<double>,
+                             vec_mul_reduce<double, backends::cpu::isa_any>);
+}
+
+template <typename T>
+void TestInplace(const int n,
+                 std::function<void(const int, const T*, T*)> tgt,
+                 std::function<void(const int, const T*, T*)> ref) {
+  std::vector<T> x(n);
+  std::vector<T> ytgt(n), yref(n);
+  RandomVec<T>(n, x.data());
+
+  const T* x_data = x.data();
+  T* yref_data = yref.data();
+  T* ytgt_data = ytgt.data();
+  std::memcpy(yref_data, x_data, sizeof(T) * n);
+  std::memcpy(ytgt_data, x_data, sizeof(T) * n);
+
+  ref(n, yref_data, yref_data);
+  tgt(n, ytgt_data, ytgt_data);
+
+  for (int i = 0; i < n; ++i) {
+    EXPECT_NEAR(ytgt_data[i], yref_data[i], 1e-3);
+  }
+}
+
+TEST(CpuVecTest, inplace_sigmoid) {
+  using namespace phi::funcs;  // NOLINT
+  for (auto sz : {1, 2, 15, 16, 30, 32, 128, 200, 512}) {
+    TestInplace<float>(sz, vec_sigmoid<float>, ref_sigmoid<float>);
+    TestInplace<float>(
+        sz, vec_sigmoid<float, backends::cpu::avx>, ref_sigmoid<float>);
+    TestInplace<float>(
+        sz, vec_sigmoid<float, backends::cpu::avx2>, ref_sigmoid<float>);
+    TestInplace<float>(
+        sz, vec_sigmoid<float, backends::cpu::avx512f>, ref_sigmoid<float>);
+  }
+  TestInplace<double>(30, vec_sigmoid<double>, ref_sigmoid<double>);
+}
+
+TEST(CpuVecTest, inplace_tanh) {
+  using namespace phi::funcs;  // NOLINT
+  for (auto sz : {1, 2, 15, 16, 30, 32, 128, 200, 512}) {
+    TestInplace<float>(sz, vec_tanh<float>, ref_tanh<float>);
+    TestInplace<float>(
+        sz, vec_tanh<float, backends::cpu::avx>, ref_tanh<float>);
+    TestInplace<float>(
+        sz, vec_tanh<float, backends::cpu::avx2>, ref_tanh<float>);
+    TestInplace<float>(
+        sz, vec_tanh<float, backends::cpu::avx512f>, ref_tanh<float>);
+  }
+  TestInplace<double>(30, vec_tanh<double>, ref_tanh<double>);
+}
+
+TEST(CpuVecTest, inplace_relu) {
+  using namespace phi::funcs;  // NOLINT
+  for (auto sz : {1, 2, 15, 16, 30, 32, 128, 200, 512}) {
+    TestInplace<float>(sz, vec_relu<float>, ref_relu<float>);
+    TestInplace<float>(
+        sz, vec_relu<float, backends::cpu::avx>, ref_relu<float>);
+    TestInplace<float>(
+        sz, vec_relu<float, backends::cpu::avx2>, ref_relu<float>);
+    TestInplace<float>(
+        sz, vec_relu<float, backends::cpu::avx512f>, ref_relu<float>);
+  }
+  TestInplace<double>(30, vec_relu<double>, ref_relu<double>);
+}
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/kernels/test_fused_adam_kernel.cc

@@ -0,0 +1,524 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <vector>
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/generator.h"
+
+#ifdef PADDLE_WITH_CUDA
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#endif
+
+#include "gtest/gtest.h"
+
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/common/amp_type_traits.h"
+#include "paddle/phi/core/tensor_utils.h"
+#include "paddle/phi/infermeta/multiary.h"
+#include "paddle/phi/kernels/abs_kernel.h"
+#include "paddle/phi/kernels/adam_kernel.h"
+#include "paddle/phi/kernels/adamw_kernel.h"
+#include "paddle/phi/kernels/cast_kernel.h"
+#include "paddle/phi/kernels/elementwise_subtract_kernel.h"
+#include "paddle/phi/kernels/full_kernel.h"
+#include "paddle/phi/kernels/fused_adam_kernel.h"
+#include "paddle/phi/kernels/gaussian_kernel.h"
+#include "paddle/phi/kernels/legacy/reduce_max_kernel.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+auto GenerateRandomTensorVectors(
+    const Context &ctx, const std::vector<std::vector<int64_t>> &shapes) {
+  size_t n = shapes.size();
+  std::vector<DenseTensor> tensors(n);
+  for (size_t i = 0; i < n; ++i) {
+    GaussianKernel<T, Context>(ctx,
+                               shapes[i],
+                               0.0f,
+                               1.0f,
+                               0,
+                               phi::CppTypeToDataType<T>::Type(),
+                               &tensors[i]);
+  }
+  return tensors;
+}
+
+template <typename T, typename Context>
+auto GenerateConstantTensorVectors(
+    const Context &ctx,
+    const std::vector<std::vector<int64_t>> &shapes,
+    T value) {
+  size_t n = shapes.size();
+  std::vector<DenseTensor> tensors(n);
+  for (size_t i = 0; i < n; ++i) {
+    FullKernel<T, Context>(
+        ctx, shapes[i], value, phi::CppTypeToDataType<T>::Type(), &tensors[i]);
+  }
+  return tensors;
+}
+
+static auto ToConstTensorPtrVector(const std::vector<DenseTensor> &tensors) {
+  std::vector<const DenseTensor *> results;
+  results.reserve(tensors.size());
+  for (const auto &t : tensors) {
+    results.push_back(&t);
+  }
+  return results;
+}
+
+static auto ToMutableTensorPtrVector(
+    std::vector<DenseTensor> &tensors) {  // NOLINT
+  std::vector<DenseTensor *> results;
+  results.reserve(tensors.size());
+  for (auto &t : tensors) {
+    results.push_back(&t);
+  }
+  return results;
+}
+
+static auto ToMetaTensorVector(const std::vector<DenseTensor> &tensors) {
+  std::vector<MetaTensor> results;
+  results.reserve(tensors.size());
+  for (auto &t : tensors) {
+    results.emplace_back(t);
+  }
+  return results;
+}
+
+static auto ToConstMetaTensorPtrVector(
+    const std::vector<MetaTensor> &meta_tensors) {
+  std::vector<const MetaTensor *> results;
+  results.reserve(meta_tensors.size());
+  for (auto &t : meta_tensors) {
+    results.push_back(&t);
+  }
+  return results;
+}
+
+static auto ToMutableMetaTensorPtrVector(
+    std::vector<MetaTensor> &meta_tensors) {  // NOLINT
+  std::vector<MetaTensor *> results;
+  results.reserve(meta_tensors.size());
+  for (auto &t : meta_tensors) {
+    results.push_back(&t);
+  }
+  return results;
+}
+
+template <typename T, typename Context>
+struct AdamInfo {
+  using AdamWScalarT = double;
+
+  const Context *ctx;
+  std::vector<std::vector<int64_t>> shapes;
+
+  std::vector<DenseTensor> params;
+  std::vector<DenseTensor> master_params;
+  std::vector<DenseTensor> moment1s;
+  std::vector<DenseTensor> moment2s;
+  std::vector<DenseTensor> moment2s_max;
+  std::vector<DenseTensor> beta1_pows;
+  std::vector<DenseTensor> beta2_pows;
+  DenseTensor learning_rate;
+  DenseTensor adamw_learning_rate;
+  float beta1;
+
+  float beta2;
+  float weight_decay;
+  float epsilon = 1e-6;
+  bool multi_precision;
+  bool use_adamw;
+  int chunk_size = 4096;
+  bool amsgrad;
+
+  using MT = typename phi::dtype::MPTypeTrait<T>::Type;
+
+  AdamInfo(const Context &ctx_ref,
+           const std::vector<std::vector<int64_t>> &shapes,
+           float beta1,
+           float beta2,
+           float weight_decay,
+           bool multi_precision,
+           bool use_adamw,
+           bool amsgrad)
+      : ctx(&ctx_ref),
+        shapes(shapes),
+        beta1(beta1),
+        beta2(beta2),
+        weight_decay(weight_decay),
+        multi_precision(multi_precision),
+        use_adamw(use_adamw),
+        amsgrad(amsgrad) {
+    std::vector<std::vector<int64_t>> one_shapes(shapes.size(),
+                                                 std::vector<int64_t>(1, 1));
+    std::vector<std::vector<int64_t>> learning_rate_shapes(
+        one_shapes.begin(), one_shapes.begin() + 1);
+
+    params = GenerateRandomTensorVectors<T, Context>(*ctx, shapes);
+    learning_rate = GenerateConstantTensorVectors<double, Context>(
+        *ctx, learning_rate_shapes, 1e-3)[0];
+    adamw_learning_rate = GenerateConstantTensorVectors<AdamWScalarT, Context>(
+        *ctx, learning_rate_shapes, 1e-3)[0];
+    moment1s = GenerateConstantTensorVectors<MT, Context>(*ctx, shapes, 0);
+    moment2s = GenerateConstantTensorVectors<MT, Context>(*ctx, shapes, 0);
+    moment2s_max = GenerateConstantTensorVectors<MT, Context>(*ctx, shapes, 0);
+
+    if (multi_precision) {
+      master_params.resize(shapes.size());
+      for (size_t i = 0; i < shapes.size(); ++i) {
+        master_params[i] = Cast<T, Context>(
+            *ctx, params[i], phi::CppTypeToDataType<MT>::Type());
+      }
+    }
+
+    beta1_pows =
+        GenerateConstantTensorVectors<MT, Context>(*ctx, one_shapes, beta1);
+    beta2_pows =
+        GenerateConstantTensorVectors<MT, Context>(*ctx, one_shapes, beta2);
+  }
+
+  void Update(bool use_fused, const std::vector<DenseTensor> &grads) {
+    if (use_fused) {
+      UpdateWithFusedAdam(grads);
+    } else {
+      for (size_t j = 0; j < params.size(); ++j) {
+        if (use_adamw) {
+          UpdateWithAdamWBaseline(grads, j);
+        } else {
+          UpdateWithAdamBaseline(grads, j);
+        }
+      }
+    }
+  }
+
+  static AdamInfo<T, Context> DeepCopy(const AdamInfo &other) {
+    AdamInfo copied(*other.ctx,
+                    other.shapes,
+                    other.beta1,
+                    other.beta2,
+                    other.weight_decay,
+                    other.multi_precision,
+                    other.use_adamw,
+                    other.amsgrad);
+    auto copy_tensor = [&other](const DenseTensor &x, DenseTensor *y) {
+      Copy<Context>(*other.ctx, x, x.place(), false, y);
+    };
+
+    auto copy_tensors = [&other](const std::vector<DenseTensor> &xs,
+                                 std::vector<DenseTensor> *ys) {
+      for (size_t i = 0; i < xs.size(); ++i) {
+        Copy<Context>(*other.ctx, xs[i], xs[i].place(), false, &((*ys)[i]));
+      }
+    };
+
+    copy_tensors(other.params, &copied.params);
+    copy_tensors(other.master_params, &copied.master_params);
+    copy_tensors(other.moment1s, &copied.moment1s);
+    copy_tensors(other.moment2s, &copied.moment2s);
+    copy_tensors(other.moment2s_max, &copied.moment2s_max);
+    copy_tensors(other.beta1_pows, &copied.beta1_pows);
+    copy_tensors(other.beta2_pows, &copied.beta2_pows);
+    copy_tensor(other.learning_rate, &copied.learning_rate);
+    copy_tensor(other.adamw_learning_rate, &copied.adamw_learning_rate);
+    copied.epsilon = other.epsilon;
+    copied.chunk_size = other.chunk_size;
+    other.ctx->Wait();
+    return copied;
+  }
+
+ private:
+  void UpdateWithFusedAdam(const std::vector<DenseTensor> &grads) {
+    auto param_metas = ToMetaTensorVector(params);
+    auto grad_metas = ToMetaTensorVector(grads);
+    auto master_param_metas = ToMetaTensorVector(master_params);
+    auto moment1_metas = ToMetaTensorVector(moment1s);
+    auto moment2_metas = ToMetaTensorVector(moment2s);
+    auto moment2_max_metas = ToMetaTensorVector(moment2s_max);
+    auto beta1_pow_metas = ToMetaTensorVector(beta1_pows);
+    auto beta2_pow_metas = ToMetaTensorVector(beta2_pows);
+
+    FusedAdamInferMeta(ToConstMetaTensorPtrVector(param_metas),
+                       ToConstMetaTensorPtrVector(grad_metas),
+                       adamw_learning_rate,
+                       ToConstMetaTensorPtrVector(moment1_metas),
+                       ToConstMetaTensorPtrVector(moment2_metas),
+                       ToConstMetaTensorPtrVector(moment2_max_metas),
+                       ToConstMetaTensorPtrVector(beta1_pow_metas),
+                       ToConstMetaTensorPtrVector(beta2_pow_metas),
+                       multi_precision
+                           ? paddle::make_optional(
+                                 ToConstMetaTensorPtrVector(master_param_metas))
+                           : paddle::none,
+                       MetaTensor(),
+                       beta1,
+                       beta2,
+                       epsilon,
+                       chunk_size,
+                       weight_decay,
+                       use_adamw,
+                       multi_precision,
+                       false,
+                       amsgrad,
+                       ToMutableMetaTensorPtrVector(param_metas),
+                       ToMutableMetaTensorPtrVector(moment1_metas),
+                       ToMutableMetaTensorPtrVector(moment2_metas),
+                       ToMutableMetaTensorPtrVector(moment2_max_metas),
+                       ToMutableMetaTensorPtrVector(beta1_pow_metas),
+                       ToMutableMetaTensorPtrVector(beta2_pow_metas),
+                       ToMutableMetaTensorPtrVector(master_param_metas));
+
+    FusedAdamKernel<T, Context>(
+        *ctx,
+        ToConstTensorPtrVector(params),
+        ToConstTensorPtrVector(grads),
+        adamw_learning_rate,
+        ToConstTensorPtrVector(moment1s),
+        ToConstTensorPtrVector(moment2s),
+        ToConstTensorPtrVector(moment2s_max),
+        ToConstTensorPtrVector(beta1_pows),
+        ToConstTensorPtrVector(beta2_pows),
+        multi_precision
+            ? paddle::make_optional(ToConstTensorPtrVector(master_params))
+            : paddle::none,
+        paddle::none,
+        beta1,
+        beta2,
+        epsilon,
+        chunk_size,
+        weight_decay,
+        use_adamw,
+        multi_precision,
+        false,
+        amsgrad,
+        ToMutableTensorPtrVector(params),
+        ToMutableTensorPtrVector(moment1s),
+        ToMutableTensorPtrVector(moment2s),
+        ToMutableTensorPtrVector(moment2s_max),
+        ToMutableTensorPtrVector(beta1_pows),
+        ToMutableTensorPtrVector(beta2_pows),
+        ToMutableTensorPtrVector(master_params));
+  }
+
+  void UpdateWithAdamWBaseline(const std::vector<DenseTensor> &grads,
+                               size_t idx) {
+    AdamwDenseKernel<T, Context>(
+        *ctx,
+        params[idx],
+        grads[idx],
+        adamw_learning_rate,
+        moment1s[idx],
+        moment2s[idx],
+        moment2s_max[idx],
+        beta1_pows[idx],
+        beta2_pows[idx],
+        multi_precision ? paddle::make_optional(master_params[idx])
+                        : paddle::none,
+        paddle::none,
+        beta1,
+        beta2,
+        epsilon,
+        1.0,
+        weight_decay,
+        true,
+        false,
+        1000,
+        multi_precision,
+        false,
+        amsgrad,
+        &params[idx],
+        &moment1s[idx],
+        &moment2s[idx],
+        &moment2s_max[idx],
+        &beta1_pows[idx],
+        &beta2_pows[idx],
+        multi_precision ? &master_params[idx] : nullptr);
+  }
+
+  void UpdateWithAdamBaseline(const std::vector<DenseTensor> &grads,
+                              size_t idx) {
+    AdamDenseKernel<T, Context>(
+        *ctx,
+        params[idx],
+        grads[idx],
+        learning_rate,
+        moment1s[idx],
+        moment2s[idx],
+        moment2s_max[idx],
+        beta1_pows[idx],
+        beta2_pows[idx],
+        multi_precision ? paddle::make_optional(master_params[idx])
+                        : paddle::none,
+        paddle::none,
+        beta1,
+        beta2,
+        epsilon,
+        false,
+        1000,
+        multi_precision,
+        false,
+        amsgrad,
+        &params[idx],
+        &moment1s[idx],
+        &moment2s[idx],
+        &moment2s_max[idx],
+        &beta1_pows[idx],
+        &beta2_pows[idx],
+        multi_precision ? &master_params[idx] : nullptr);
+  }
+};
+
+template <typename T, typename Context>
+auto MaxDiff(const Context &ctx,
+             const DenseTensor &x_t,
+             const DenseTensor &y_t) {
+  using MT = typename AdamInfo<T, Context>::MT;
+  auto mp_dtype = phi::CppTypeToDataType<MT>::Type();
+  auto x = Cast<T, Context>(ctx, x_t, mp_dtype);
+  auto y = Cast<T, Context>(ctx, y_t, mp_dtype);
+
+  EXPECT_EQ(x.dims(), y.dims());
+  DenseTensor diff, diff_reduced, diff_reduced_cpu;
+
+  diff.Resize(x.dims());
+  ctx.template Alloc<MT>(&diff);
+  SubtractKernel<MT, Context>(ctx, x, y, &diff);
+  AbsKernel<MT, Context>(ctx, diff, &diff);
+
+  diff_reduced.Resize({1});
+  ctx.template Alloc<MT>(&diff_reduced);
+  MaxRawKernel<MT, Context>(ctx,
+                            diff,
+                            common::vectorize<int64_t>(x.dims()),
+                            false,
+                            true,
+                            &diff_reduced);
+
+  diff_reduced_cpu.Resize(diff_reduced.dims());
+  ctx.template HostAlloc<MT>(&diff_reduced_cpu);
+  Copy<Context>(ctx, diff_reduced, CPUPlace(), true, &diff_reduced_cpu);
+  EXPECT_EQ(diff_reduced_cpu.place(), CPUPlace());
+  return diff_reduced_cpu.data<MT>()[0];
+}
+
+template <typename T, typename Context>
+auto MaxDiff(const Context &ctx,
+             const std::vector<DenseTensor> &xs,
+             const std::vector<DenseTensor> &ys) {
+  using MT = typename AdamInfo<T, Context>::MT;
+  MT diff = 0;
+  for (size_t i = 0; i < xs.size(); ++i) {
+    diff = std::max<MT>(diff, MaxDiff<T, Context>(ctx, xs[i], ys[i]));
+  }
+  return diff;
+}
+
+template <typename T, typename PlaceType>
+void TestFusedAdamBase(const std::vector<std::vector<int64_t>> &shapes,
+                       float atol,
+                       bool use_adamw,
+                       bool amsgrad,
+                       bool multi_precision = false,
+                       float beta1 = 0.9,
+                       float beta2 = 0.99,
+                       float weight_decay = 0.1,
+                       size_t steps = 5,
+                       uint64_t seed = 10) {
+  const auto &ctx = *phi::DeviceContextPool::Instance().GetByPlace(PlaceType());
+  using Context = typename std::remove_const<
+      typename std::remove_pointer<decltype(&ctx)>::type>::type;
+  ctx.GetGenerator()->SetCurrentSeed(seed);
+  AdamInfo<T, Context> info1(ctx,
+                             shapes,
+                             beta1,
+                             beta2,
+                             weight_decay,
+                             multi_precision,
+                             use_adamw,
+                             amsgrad);
+  auto info2 = AdamInfo<T, Context>::DeepCopy(info1);
+
+  for (size_t i = 0; i < steps; ++i) {
+    auto grads = GenerateRandomTensorVectors<T>(ctx, shapes);
+    info1.Update(false, grads);
+    info2.Update(true, grads);
+  }
+
+  using MT = typename decltype(info1)::MT;
+
+#define PD_ADAM_TEST_COMP(__field, __dtype)                          \
+  do {                                                               \
+    MT __diff = MaxDiff<__dtype>(ctx, info1.__field, info2.__field); \
+    EXPECT_LE(__diff, static_cast<MT>(atol))                         \
+        << #__field << " has diff when use_adamw = " << use_adamw    \
+        << " , multi_precision = " << multi_precision;               \
+  } while (0)
+
+  PD_ADAM_TEST_COMP(beta1_pows, MT);
+  PD_ADAM_TEST_COMP(beta2_pows, MT);
+  PD_ADAM_TEST_COMP(params, T);
+  PD_ADAM_TEST_COMP(master_params, MT);
+  PD_ADAM_TEST_COMP(moment1s, MT);
+  PD_ADAM_TEST_COMP(moment2s, MT);
+  PD_ADAM_TEST_COMP(moment2s_max, MT);
+}
+
+static auto GenerateRandomShapes(size_t n, uint64_t low, uint64_t high) {
+  std::random_device device;
+  std::default_random_engine engine(device());
+  std::uniform_int_distribution<uint64_t> dist(low, high);
+  std::vector<std::vector<int64_t>> shapes(n);
+  for (size_t i = 0; i < n; ++i) {
+    shapes[i].push_back(static_cast<int64_t>(dist(engine)));
+  }
+  return shapes;
+}
+
+TEST(fused_adam, test_fp32_cpu) {
+  auto shapes = GenerateRandomShapes(30, 10, 20);
+  float atol = 0.0f;
+  for (auto use_adamw : {false, true}) {
+    for (auto amsgrad : {false, true}) {
+      TestFusedAdamBase<float, CPUPlace>(shapes, atol, use_adamw, amsgrad);
+    }
+  }
+}
+
+#ifdef PADDLE_WITH_CUDA
+TEST(fused_adam, test_fp32_gpu) {
+  auto shapes = GenerateRandomShapes(40, 0, 2 << 18);
+  for (auto use_adamw : {false, true}) {
+    // AdamwDenseKernel uses torch-compatible math (double-precision
+    // intermediates, FMA intrinsics) while FusedAdamKernel uses the
+    // original float-precision math, so allow a small tolerance for adamw.
+    float atol = use_adamw ? 1e-5f : 0.0f;
+    for (auto amsgrad : {false, true}) {
+      TestFusedAdamBase<float, GPUPlace>(shapes, atol, use_adamw, amsgrad);
+    }
+  }
+}
+
+TEST(fused_adam, test_fp16_gpu) {
+  auto shapes = GenerateRandomShapes(40, 0, 2 << 18);
+  float atol = 5e-3f;
+  for (auto use_adamw : {false, true}) {
+    for (auto amsgrad : {false, true}) {
+      TestFusedAdamBase<dtype::float16, GPUPlace>(
+          shapes, atol, use_adamw, amsgrad, true);
+    }
+  }
+}
+#endif
+
+}  // namespace phi

test/cpp/phi/kernels/test_gpu_timer.cu

@@ -0,0 +1,119 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include <functional>
+
+#include "glog/logging.h"
+#include "paddle/phi/kernels/autotune/gpu_timer.h"
+#include "paddle/phi/kernels/funcs/aligned_vector.h"
+
+template <typename T, int VecSize>
+__global__ void VecSum(T *x, T *y, int N) {
+#ifdef __HIPCC__
+  int idx = hipBlockDim_x * hipBlockIdx_x + hipThreadIdx_x;
+#else
+  int idx = blockDim.x * blockIdx.x + threadIdx.x;
+#endif
+  using LoadT = phi::AlignedVector<T, VecSize>;
+  for (int i = idx * VecSize; i < N; i += blockDim.x * gridDim.x * VecSize) {
+    LoadT x_vec;
+    LoadT y_vec;
+    phi::Load<T, VecSize>(&x[i], &x_vec);
+    phi::Load<T, VecSize>(&y[i], &y_vec);
+#pragma unroll
+    for (int j = 0; j < VecSize; j++) {
+      y_vec[j] = x_vec[j] + y_vec[j];
+    }
+    phi::Store<T, VecSize>(y_vec, &y[i]);
+  }
+}
+
+template <int Vecsize, int Threads, size_t Blocks>
+void Algo(float *d_in, float *d_out, size_t N) {
+#ifdef __HIPCC__
+  hipLaunchKernelGGL(HIP_KERNEL_NAME(VecSum<float, Vecsize>),
+                     dim3(Blocks),
+                     dim3(Threads),
+                     0,
+                     0,
+                     d_in,
+                     d_out,
+                     N);
+#else
+  VecSum<float, Vecsize><<<Blocks, Threads>>>(d_in, d_out, N);
+#endif
+}
+
+TEST(GpuTimer, Sum) {
+  float *in1, *in2, *out;
+  float *d_in1, *d_in2;
+  size_t N = 1 << 20;
+  size_t size = sizeof(float) * N;
+#ifdef __HIPCC__
+  hipMalloc(reinterpret_cast<void **>(&d_in1), size);
+  hipMalloc(reinterpret_cast<void **>(&d_in2), size);
+#else
+  cudaMalloc(reinterpret_cast<void **>(&d_in1), size);
+  cudaMalloc(reinterpret_cast<void **>(&d_in2), size);
+#endif
+  in1 = reinterpret_cast<float *>(malloc(size));
+  in2 = reinterpret_cast<float *>(malloc(size));
+  out = reinterpret_cast<float *>(malloc(size));
+  for (size_t i = 0; i < N; i++) {
+    in1[i] = 1.0f;
+    in2[i] = 2.0f;
+  }
+
+#ifdef __HIPCC__
+  hipMemcpy(d_in1, in1, size, hipMemcpyHostToDevice);
+  hipMemcpy(d_in2, in2, size, hipMemcpyHostToDevice);
+#else
+  cudaMemcpy(d_in1, in1, size, cudaMemcpyHostToDevice);
+  cudaMemcpy(d_in2, in2, size, cudaMemcpyHostToDevice);
+#endif
+
+  using Functor = std::function<void(float *, float *, size_t)>;
+  Functor algo0 = Algo<4, 256, 1024>;
+  Functor algo1 = Algo<1, 256, 1024>;
+  Functor algo2 = Algo<1, 256, 8>;
+
+  std::vector<Functor> algos = {algo0, algo1, algo2};
+
+  for (int j = 0; j < algos.size(); ++j) {
+    auto algo = algos[j];
+    phi::GpuTimer timer;
+    timer.Start(0);
+    algo(d_in1, d_in2, N);
+    timer.Stop(0);
+    VLOG(3) << "algo: " << j << " cost: " << timer.ElapsedTime() << "ms";
+  }
+
+#ifdef __HIPCC__
+  hipMemcpy(out, d_in2, size, hipMemcpyDeviceToHost);
+#else
+  cudaMemcpy(out, d_in2, size, cudaMemcpyDeviceToHost);
+#endif
+  free(in1);
+  free(in2);
+  free(out);
+#ifdef __HIPCC__
+  hipFree(d_in1);
+  hipFree(d_in2);
+#else
+  cudaFree(d_in1);
+  cudaFree(d_in2);
+#endif
+}

test/cpp/phi/kernels/test_math_function.cc

@@ -0,0 +1,380 @@
+//  Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <array>
+#include <set>
+
+#include "gtest/gtest.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/kernels/funcs/blas/blas.h"
+#include "paddle/phi/kernels/funcs/math_function.h"
+
+namespace phi {
+namespace tests {
+
+template <typename T>
+inline phi::funcs::BlasT<phi::CPUContext, T> GetBlas(
+    const phi::CPUContext& context) {
+  return phi::funcs::GetBlas<phi::CPUContext, T>(context);
+}
+
+TEST(math_function, gemm_notrans_cblas) {
+  phi::DenseTensor input1;
+  phi::DenseTensor input2;
+  phi::DenseTensor input3;
+
+  int m = 2;
+  int n = 3;
+  int k = 3;
+  auto* dev_ctx =
+      phi::DeviceContextPool::Instance().GetByPlace(phi::CPUPlace());
+
+  input1.Resize({2, 3});
+  float* input1_ptr = dev_ctx->template Alloc<float>(&input1);
+  std::array<float, 6> arr1 = {0, 1, 2, 3, 4, 5};
+  memcpy(input1_ptr, arr1.data(), 6 * sizeof(float));
+  input2.Resize({3, 4});
+  float* input2_ptr = dev_ctx->template Alloc<float>(&input2);
+  std::array<float, 12> arr2 = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
+  memcpy(input2_ptr, arr2.data(), 12 * sizeof(float));
+  input3.Resize({2, 4});
+  float* input3_ptr = dev_ctx->template Alloc<float>(&input3);
+  std::array<float, 8> arr3 = {0, 1, 2, 3, 4, 5, 6, 7};
+  memcpy(input3_ptr, arr3.data(), 8 * sizeof(float));
+
+  GetBlas<float>(*dev_ctx).GEMM(false,
+                                false,
+                                m,
+                                n,
+                                k,
+                                1,
+                                input1_ptr,
+                                3,
+                                input2_ptr + 1,
+                                4,
+                                1,
+                                input3_ptr + 1,
+                                4);
+
+  EXPECT_EQ(input3_ptr[0], 0);
+  EXPECT_EQ(input3_ptr[1], 24);
+  EXPECT_EQ(input3_ptr[2], 28);
+  EXPECT_EQ(input3_ptr[3], 32);
+  EXPECT_EQ(input3_ptr[4], 4);
+  EXPECT_EQ(input3_ptr[5], 73);
+  EXPECT_EQ(input3_ptr[6], 86);
+  EXPECT_EQ(input3_ptr[7], 99);
+}
+#ifdef PADDLE_WITH_LIBXSMM
+template <typename T>
+void MklSmmCompare(int m, int n, int k) {
+  phi::DenseTensor mat_a;
+  phi::DenseTensor mat_b;
+  phi::DenseTensor mat_c_smm;
+  phi::DenseTensor mat_c_mkl;
+
+  auto* dev_ctx =
+      phi::DeviceContextPool::Instance().GetByPlace(phi::CPUPlace());
+
+  mat_a.Resize({m, k});
+  T* A = dev_ctx->template Alloc<T>(&mat_a);
+  mat_b.Resize({k, n});
+  T* B = dev_ctx->template Alloc<T>(&mat_b);
+  mat_c_smm.Resize({m, n});
+  T* CSMM = dev_ctx->template Alloc<T>(&mat_c_smm);
+  mat_c_mkl.Resize({m, n});
+  T* CMKL = dev_ctx->template Alloc<T>(&mat_c_mkl);
+  T alpha = static_cast<T>(1);
+  T beta = static_cast<T>(0);
+  for (int i = 0; i < mat_a.numel(); ++i) {
+    A[i] = static_cast<T>(i);
+  }
+  for (int i = 0; i < mat_b.numel(); ++i) {
+    B[i] = static_cast<T>(i);
+  }
+  // lda,ldb,ldc follow RowMajor
+  int lda = k;
+  int ldb = n;
+  int ldc = n;
+
+  auto smm = [&, m, n, k, lda, ldb, ldc, alpha, beta]() {
+    const char transa = 'N';
+    const char transb = 'N';
+    phi::funcs::CBlas<T>::SMM_GEMM(&transa,
+                                   &transb,
+                                   &n,
+                                   &m,
+                                   &k,
+                                   &alpha,
+                                   B,
+                                   &ldb,
+                                   A,
+                                   &lda,
+                                   &beta,
+                                   CSMM,
+                                   &ldc);
+  };
+
+  auto mkl = [&, m, n, k, lda, ldb, ldc, alpha, beta]() {
+    phi::funcs::CBlas<T>::GEMM(CblasRowMajor,
+                               CblasNoTrans,
+                               CblasNoTrans,
+                               m,
+                               n,
+                               k,
+                               alpha,
+                               A,
+                               lda,
+                               B,
+                               ldb,
+                               beta,
+                               CMKL,
+                               ldc);
+  };
+
+  smm();
+  mkl();
+  ASSERT_EQ(mat_c_mkl.numel(), mat_c_smm.numel());
+  for (int i = 0; i < mat_c_mkl.numel(); ++i) {
+    EXPECT_FLOAT_EQ(CSMM[i], CMKL[i]);
+  }
+}
+TEST(math_function, gemm_mkl_vs_smm) {
+  MklSmmCompare<float>(1, 2, 3);
+  MklSmmCompare<double>(1, 2, 3);
+  MklSmmCompare<float>(3, 2, 1);
+  MklSmmCompare<double>(3, 2, 1);
+  MklSmmCompare<float>(3, 8, 5);
+  MklSmmCompare<double>(3, 8, 5);
+}
+#endif
+
+TEST(math_function, gemm_trans_cblas) {
+  phi::DenseTensor input1;
+  phi::DenseTensor input2;
+  phi::DenseTensor input3;
+
+  int m = 2;
+  int n = 3;
+  int k = 3;
+  auto* dev_ctx =
+      phi::DeviceContextPool::Instance().GetByPlace(phi::CPUPlace());
+
+  input1.Resize({2, 3});
+  float* input1_ptr = dev_ctx->template Alloc<float>(&input1);
+  std::array<float, 6> arr1 = {0, 1, 2, 3, 4, 5};
+  memcpy(input1_ptr, arr1.data(), 6 * sizeof(float));
+  input2.Resize({4, 3});
+  float* input2_ptr = dev_ctx->template Alloc<float>(&input2);
+  std::array<float, 12> arr2 = {0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11};
+  memcpy(input2_ptr, arr2.data(), 12 * sizeof(float));
+  input3.Resize({2, 4});
+  float* input3_ptr = dev_ctx->template Alloc<float>(&input3);
+  std::array<float, 8> arr3 = {0, 1, 2, 3, 4, 5, 6, 7};
+  memcpy(input3_ptr, arr3.data(), 8 * sizeof(float));
+
+  GetBlas<float>(*dev_ctx).GEMM(false,
+                                true,
+                                m,
+                                n,
+                                k,
+                                1,
+                                input1_ptr,
+                                3,
+                                input2_ptr + 3,
+                                3,
+                                1,
+                                input3_ptr + 1,
+                                4);
+
+  EXPECT_EQ(input3_ptr[0], 0);
+  EXPECT_EQ(input3_ptr[1], 24);
+  EXPECT_EQ(input3_ptr[2], 28);
+  EXPECT_EQ(input3_ptr[3], 32);
+  EXPECT_EQ(input3_ptr[4], 4);
+  EXPECT_EQ(input3_ptr[5], 73);
+  EXPECT_EQ(input3_ptr[6], 86);
+  EXPECT_EQ(input3_ptr[7], 99);
+}
+
+TEST(math_function, zero) {
+  phi::DenseTensor tensor;
+  auto* dev_ctx =
+      phi::DeviceContextPool::Instance().GetByPlace(phi::CPUPlace());
+
+  tensor.Resize({2, 2});
+  float* t = dev_ctx->template Alloc<float>(&tensor);
+  phi::funcs::SetConstant<phi::CPUContext, float> functor;
+  functor(*dev_ctx, &tensor, 0);
+  EXPECT_EQ(t[0], 0);
+  EXPECT_EQ(t[1], 0);
+  EXPECT_EQ(t[2], 0);
+  EXPECT_EQ(t[3], 0);
+
+  functor(*dev_ctx, &tensor, 1);
+
+  EXPECT_EQ(t[0], 1);
+  EXPECT_EQ(t[1], 1);
+  EXPECT_EQ(t[2], 1);
+  EXPECT_EQ(t[3], 1);
+}
+
+template <typename T>
+void GemvTest(int m, int n, bool trans) {
+  phi::DenseTensor mat_a;
+  phi::DenseTensor vec_b;
+  phi::DenseTensor vec_c;
+  int b_num = trans ? m : n;
+  int c_num = trans ? n : m;
+
+  auto* dev_ctx =
+      phi::DeviceContextPool::Instance().GetByPlace(phi::CPUPlace());
+
+  mat_a.Resize({m, n});
+  T* data_a = dev_ctx->template Alloc<T>(&mat_a);
+  vec_b.Resize({b_num});
+  T* data_b = dev_ctx->template Alloc<T>(&vec_b);
+  vec_c.Resize({c_num});
+  T* data_c = dev_ctx->template Alloc<T>(&vec_c);
+  for (int i = 0; i < mat_a.numel(); ++i) {
+    data_a[i] = static_cast<T>(i);
+  }
+  for (int i = 0; i < vec_b.numel(); ++i) {
+    data_b[i] = static_cast<T>(i);
+  }
+
+  GetBlas<T>(*dev_ctx).GEMV(trans,
+                            static_cast<int>(m),
+                            static_cast<int>(n),
+                            1.,
+                            data_a,
+                            data_b,
+                            0.,
+                            data_c);
+
+  if (!trans) {
+    for (int i = 0; i < m; ++i) {
+      T sum = 0.0;
+      for (int j = 0; j < n; ++j) {
+        sum += data_a[i * n + j] * data_b[j];
+      }
+      ASSERT_FLOAT_EQ(data_c[i], sum);
+    }
+  } else {
+    for (int i = 0; i < n; ++i) {
+      T sum = 0.0;
+      for (int j = 0; j < m; ++j) {
+        sum += data_a[j * n + i] * data_b[j];
+      }
+      ASSERT_FLOAT_EQ(data_c[i], sum);
+    }
+  }
+}
+
+TEST(math_function, gemv) {
+  GemvTest<float>(3, 13, false);
+  GemvTest<double>(4, 5, false);
+  GemvTest<float>(12, 7, true);
+  GemvTest<double>(7, 9, true);
+}
+
+TEST(math_function, set_constant) {
+  phi::DenseTensor t;
+  auto* dev_ctx =
+      phi::DeviceContextPool::Instance().GetByPlace(phi::CPUPlace());
+  t.Resize({10, 10});
+  dev_ctx->template Alloc<int>(&t);
+  phi::funcs::set_constant(*dev_ctx, &t, static_cast<int>(10));
+  for (int64_t i = 0; i < t.numel(); ++i) {
+    PADDLE_ENFORCE_EQ(10,
+                      t.data<int>()[i],
+                      common::errors::InvalidArgument(
+                          "Each value of input tensor should be 10, "
+                          "but received %d.",
+                          t.data<int>()[i]));
+  }
+}
+
+template <typename T>
+void GemmWarpTest(int m, int n, int k, T alpha, T beta) {
+  phi::DenseTensor mat_a;
+  phi::DenseTensor mat_b;
+  phi::DenseTensor mat_c_ref;
+  phi::DenseTensor mat_c_mkl;
+
+  auto* dev_ctx =
+      phi::DeviceContextPool::Instance().GetByPlace(phi::CPUPlace());
+
+  mat_a.Resize({m, k});
+  T* A = dev_ctx->template Alloc<T>(&mat_a);
+  mat_b.Resize({k, n});
+  T* B = dev_ctx->template Alloc<T>(&mat_b);
+  mat_c_ref.Resize({m, n});
+  T* CREF = dev_ctx->template Alloc<T>(&mat_c_ref);
+  mat_c_mkl.Resize({m, n});
+  T* CMKL = dev_ctx->template Alloc<T>(&mat_c_mkl);
+
+  ASSERT_EQ(mat_c_mkl.numel(), mat_c_ref.numel());
+  for (int i = 0; i < mat_a.numel(); ++i) {
+    A[i] = static_cast<T>(i);
+  }
+  for (int i = 0; i < mat_b.numel(); ++i) {
+    B[i] = static_cast<T>(i + 1);
+  }
+  for (int i = 0; i < mat_c_ref.numel(); ++i) {
+    CREF[i] = static_cast<T>(i + 2);
+    CMKL[i] = CREF[i];
+  }
+
+  // this would call gemm_warp
+  GetBlas<T>(*dev_ctx).GEMM(
+      CblasNoTrans, CblasNoTrans, m, n, k, alpha, A, B, beta, CREF);
+
+  // lda,ldb,ldc follow RowMajor
+  int lda = k;
+  int ldb = n;
+  int ldc = n;
+  phi::funcs::CBlas<T>::GEMM(CblasRowMajor,
+                             CblasNoTrans,
+                             CblasNoTrans,
+                             m,
+                             n,
+                             k,
+                             alpha,
+                             A,
+                             lda,
+                             B,
+                             ldb,
+                             beta,
+                             CMKL,
+                             ldc);
+
+  for (int i = 0; i < mat_c_mkl.numel(); ++i) {
+    EXPECT_FLOAT_EQ(CREF[i], CMKL[i]);
+  }
+}
+
+TEST(math_function, gemm_warp) {
+  GemmWarpTest<float>(3, 2, 5, 1.f, 0.f);
+  GemmWarpTest<float>(3, 2, 5, 2.f, 1.f);
+  GemmWarpTest<float>(8, 5, 6, 1.f, 0.f);
+  GemmWarpTest<float>(8, 5, 6, 2.f, 1.f);
+  GemmWarpTest<double>(3, 2, 5, 1.0, 0.0);
+  GemmWarpTest<double>(3, 2, 5, 2.0, 1.0);
+  GemmWarpTest<double>(8, 5, 6, 1.0, 0.0);
+  GemmWarpTest<double>(8, 5, 6, 2.0, 1.0);
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/kernels/test_math_function.cu

@@ -0,0 +1,527 @@
+//  Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "gtest/gtest.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/core/tensor_utils.h"
+#include "paddle/phi/kernels/funcs/blas/blas.h"
+#include "paddle/phi/kernels/funcs/math_function.h"
+
+namespace phi {
+namespace tests {
+
+void fill_fp16_data(phi::dtype::float16* in_ptr,
+                    size_t size,
+                    const std::vector<float>& data) {
+  PADDLE_ENFORCE_EQ(
+      size,
+      data.size(),
+      common::errors::InvalidArgument(
+          "The size of argument data should"
+          " be equal to the argument size. Expected %d, but received %d.",
+          size,
+          data.size()));
+  for (size_t i = 0; i < data.size(); ++i) {
+    in_ptr[i] = phi::dtype::float16(data[i]);
+  }
+}
+
+template <typename T>
+inline phi::funcs::BlasT<phi::GPUContext, T> GetBlas(
+    const phi::GPUContext& context) {
+  return phi::funcs::GetBlas<phi::GPUContext, T>(context);
+}
+
+TEST(math_function, notrans_mul_trans_fp32) {
+  phi::DenseTensor input1;
+  phi::DenseTensor input1_gpu;
+  phi::DenseTensor input2_gpu;
+  phi::DenseTensor out_gpu;
+  phi::DenseTensor out;
+
+  phi::CPUPlace cpu_place;
+  phi::GPUPlace gpu_place(0);
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* context = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  float* input1_ptr = input1.mutable_data<float>({2, 3}, cpu_place);
+  float arr[6] = {0, 1, 2, 3, 4, 5};
+  memcpy(input1_ptr, arr, 6 * sizeof(float));
+
+  phi::Copy(*context, input1, gpu_place, true, &input1_gpu);
+  phi::Copy(*context, input1, gpu_place, true, &input2_gpu);
+
+  out_gpu.mutable_data<float>({2, 2}, gpu_place);
+  GetBlas<float>(*context).MatMul(
+      input1_gpu, false, input2_gpu, true, 1, &out_gpu, 0);
+
+  phi::Copy(*context, out_gpu, cpu_place, true, &out);
+
+  float* out_ptr = out.data<float>();
+  context->Wait();
+  EXPECT_EQ(out_ptr[0], 5);
+  EXPECT_EQ(out_ptr[1], 14);
+  EXPECT_EQ(out_ptr[2], 14);
+  EXPECT_EQ(out_ptr[3], 50);
+}
+
+TEST(math_function, notrans_mul_trans_fp16) {
+  phi::DenseTensor input1;
+  phi::DenseTensor input1_gpu;
+  phi::DenseTensor input2_gpu;
+  phi::DenseTensor out_gpu;
+  phi::DenseTensor out;
+
+  phi::CPUPlace cpu_place;
+  phi::GPUPlace gpu_place(0);
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* context = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  // fp16 GEMM in cublas requires GPU compute capability >= 53
+  if (context->GetComputeCapability() < 53) {
+    return;
+  }
+
+  phi::dtype::float16* input1_ptr =
+      input1.mutable_data<phi::dtype::float16>({2, 3}, cpu_place);
+  fill_fp16_data(input1_ptr, input1.numel(), {0, 1, 2, 3, 4, 5});
+
+  phi::Copy(*context, input1, gpu_place, true, &input1_gpu);
+  phi::Copy(*context, input1, gpu_place, true, &input2_gpu);
+
+  out_gpu.mutable_data<phi::dtype::float16>({2, 2}, gpu_place);
+
+  GetBlas<phi::dtype::float16>(*context).MatMul(input1_gpu,
+                                                false,
+                                                input2_gpu,
+                                                true,
+                                                phi::dtype::float16(1),
+                                                &out_gpu,
+                                                phi::dtype::float16(0));
+
+  phi::Copy(*context, out_gpu, cpu_place, true, &out);
+
+  phi::dtype::float16* out_ptr = out.data<phi::dtype::float16>();
+  context->Wait();
+  EXPECT_EQ(static_cast<float>(out_ptr[0]), 5);
+  EXPECT_EQ(static_cast<float>(out_ptr[1]), 14);
+  EXPECT_EQ(static_cast<float>(out_ptr[2]), 14);
+  EXPECT_EQ(static_cast<float>(out_ptr[3]), 50);
+}
+
+TEST(math_function, trans_mul_notrans_fp32) {
+  phi::DenseTensor input1;
+  phi::DenseTensor input1_gpu;
+  phi::DenseTensor input2_gpu;
+  phi::DenseTensor out_gpu;
+  phi::DenseTensor out;
+
+  phi::CPUPlace cpu_place;
+  phi::GPUPlace gpu_place(0);
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* context = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  float* input1_ptr = input1.mutable_data<float>({2, 3}, cpu_place);
+  float arr[6] = {0, 1, 2, 3, 4, 5};
+  memcpy(input1_ptr, arr, 6 * sizeof(float));
+
+  phi::Copy(*context, input1, gpu_place, true, &input1_gpu);
+  phi::Copy(*context, input1, gpu_place, true, &input2_gpu);
+
+  out_gpu.mutable_data<float>({3, 3}, gpu_place);
+
+  GetBlas<float>(*context).MatMul(
+      input1_gpu, true, input2_gpu, false, 1, &out_gpu, 0);
+
+  phi::Copy(*context, out_gpu, cpu_place, true, &out);
+
+  float* out_ptr = out.data<float>();
+  context->Wait();
+  EXPECT_EQ(out_ptr[0], 9);
+  EXPECT_EQ(out_ptr[1], 12);
+  EXPECT_EQ(out_ptr[2], 15);
+  EXPECT_EQ(out_ptr[3], 12);
+  EXPECT_EQ(out_ptr[4], 17);
+  EXPECT_EQ(out_ptr[5], 22);
+  EXPECT_EQ(out_ptr[6], 15);
+  EXPECT_EQ(out_ptr[7], 22);
+  EXPECT_EQ(out_ptr[8], 29);
+}
+
+TEST(math_function, trans_mul_notrans_fp16) {
+  phi::DenseTensor input1;
+  phi::DenseTensor input1_gpu;
+  phi::DenseTensor input2_gpu;
+  phi::DenseTensor out_gpu;
+  phi::DenseTensor out;
+
+  phi::CPUPlace cpu_place;
+  phi::GPUPlace gpu_place(0);
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* context = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  // fp16 GEMM in cublas requires GPU compute capability >= 53
+  if (context->GetComputeCapability() < 53) {
+    return;
+  }
+
+  phi::dtype::float16* input1_ptr =
+      input1.mutable_data<phi::dtype::float16>({2, 3}, cpu_place);
+  fill_fp16_data(input1_ptr, input1.numel(), {0, 1, 2, 3, 4, 5});
+
+  phi::Copy(*context, input1, gpu_place, true, &input1_gpu);
+  phi::Copy(*context, input1, gpu_place, true, &input2_gpu);
+
+  out_gpu.mutable_data<phi::dtype::float16>({3, 3}, gpu_place);
+
+  GetBlas<phi::dtype::float16>(*context).MatMul(input1_gpu,
+                                                true,
+                                                input2_gpu,
+                                                false,
+                                                phi::dtype::float16(1),
+                                                &out_gpu,
+                                                phi::dtype::float16(0));
+
+  phi::Copy(*context, out_gpu, cpu_place, true, &out);
+
+  phi::dtype::float16* out_ptr = out.data<phi::dtype::float16>();
+  context->Wait();
+  EXPECT_EQ(static_cast<float>(out_ptr[0]), 9);
+  EXPECT_EQ(static_cast<float>(out_ptr[1]), 12);
+  EXPECT_EQ(static_cast<float>(out_ptr[2]), 15);
+  EXPECT_EQ(static_cast<float>(out_ptr[3]), 12);
+  EXPECT_EQ(static_cast<float>(out_ptr[4]), 17);
+  EXPECT_EQ(static_cast<float>(out_ptr[5]), 22);
+  EXPECT_EQ(static_cast<float>(out_ptr[6]), 15);
+  EXPECT_EQ(static_cast<float>(out_ptr[7]), 22);
+  EXPECT_EQ(static_cast<float>(out_ptr[8]), 29);
+}
+
+TEST(math_function, gemm_notrans_cublas_fp32) {
+  phi::DenseTensor input1;
+  phi::DenseTensor input2;
+  phi::DenseTensor input3;
+  phi::DenseTensor input1_gpu;
+  phi::DenseTensor input2_gpu;
+  phi::DenseTensor input3_gpu;
+
+  phi::CPUPlace cpu_place;
+  phi::GPUPlace gpu_place(0);
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* context = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  int m = 2;
+  int n = 3;
+  int k = 3;
+  float* input1_ptr = input1.mutable_data<float>({2, 3}, cpu_place);
+  float arr1[6] = {0, 1, 2, 3, 4, 5};
+  memcpy(input1_ptr, arr1, 6 * sizeof(float));
+  float* input2_ptr = input2.mutable_data<float>({3, 4}, cpu_place);
+  float arr2[12] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
+  memcpy(input2_ptr, arr2, 12 * sizeof(float));
+  float* input3_ptr = input3.mutable_data<float>({2, 4}, cpu_place);
+  float arr3[8] = {0, 1, 2, 3, 4, 5, 6, 7};
+  memcpy(input3_ptr, arr3, 8 * sizeof(float));
+
+  phi::Copy(*context, input1, gpu_place, true, &input1_gpu);
+  phi::Copy(*context, input2, gpu_place, true, &input2_gpu);
+  phi::Copy(*context, input3, gpu_place, true, &input3_gpu);
+  float* a = input1_gpu.data<float>();
+  float* b = input2_gpu.data<float>();
+  float* c = input3_gpu.mutable_data<float>(gpu_place);
+
+  GetBlas<float>(*context).GEMM(
+      false, false, m, n, k, 1, a, 3, b + 1, 4, 1, c + 1, 4);
+
+  phi::Copy(*context, input3_gpu, cpu_place, true, &input3);
+
+  // numpy code:
+  // a = np.arange(6).reshape(2, 3)
+  // b = np.arange(12).reshape(3, 4)[:, 1:]
+  // c = np.arange(8).reshape(2, 4)[:, 1:]
+  // out = np.arange(8).reshape(2, 4)
+  // out[:, 1:] = np.dot(a, b) + c
+  context->Wait();
+  EXPECT_EQ(input3_ptr[0], 0);
+  EXPECT_EQ(input3_ptr[1], 24);
+  EXPECT_EQ(input3_ptr[2], 28);
+  EXPECT_EQ(input3_ptr[3], 32);
+  EXPECT_EQ(input3_ptr[4], 4);
+  EXPECT_EQ(input3_ptr[5], 73);
+  EXPECT_EQ(input3_ptr[6], 86);
+  EXPECT_EQ(input3_ptr[7], 99);
+}
+
+TEST(math_function, gemm_notrans_cublas_fp16) {
+  phi::DenseTensor input1;
+  phi::DenseTensor input2;
+  phi::DenseTensor input3;
+  phi::DenseTensor input1_gpu;
+  phi::DenseTensor input2_gpu;
+  phi::DenseTensor input3_gpu;
+
+  phi::CPUPlace cpu_place;
+  phi::GPUPlace gpu_place(0);
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* context = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  // fp16 GEMM in cublas requires GPU compute capability >= 53
+  if (context->GetComputeCapability() < 53) {
+    return;
+  }
+
+  int m = 2;
+  int n = 3;
+  int k = 3;
+  phi::dtype::float16* input1_ptr =
+      input1.mutable_data<phi::dtype::float16>({2, 3}, cpu_place);
+  fill_fp16_data(input1_ptr, input1.numel(), {0, 1, 2, 3, 4, 5});
+  phi::dtype::float16* input2_ptr =
+      input2.mutable_data<phi::dtype::float16>({3, 4}, cpu_place);
+  fill_fp16_data(
+      input2_ptr, input2.numel(), {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11});
+  phi::dtype::float16* input3_ptr =
+      input3.mutable_data<phi::dtype::float16>({2, 4}, cpu_place);
+  fill_fp16_data(input3_ptr, input3.numel(), {0, 1, 2, 3, 4, 5, 6, 7});
+
+  phi::Copy(*context, input1, gpu_place, true, &input1_gpu);
+  phi::Copy(*context, input2, gpu_place, true, &input2_gpu);
+  phi::Copy(*context, input3, gpu_place, true, &input3_gpu);
+  phi::dtype::float16* a = input1_gpu.data<phi::dtype::float16>();
+  phi::dtype::float16* b = input2_gpu.data<phi::dtype::float16>();
+  phi::dtype::float16* c =
+      input3_gpu.mutable_data<phi::dtype::float16>(gpu_place);
+
+  GetBlas<phi::dtype::float16>(*context).GEMM(
+      false,
+      false,
+      m,
+      n,
+      k,
+      static_cast<phi::dtype::float16>(1),
+      a,
+      3,
+      b + 1,
+      4,
+      static_cast<phi::dtype::float16>(1),
+      c + 1,
+      4);
+
+  phi::Copy(*context, input3_gpu, cpu_place, true, &input3);
+
+  // numpy code:
+  // a = np.arange(6).reshape(2, 3)
+  // b = np.arange(12).reshape(3, 4)[:, 1:]
+  // c = np.arange(8).reshape(2, 4)[:, 1:]
+  // out = np.arange(8).reshape(2, 4)
+  // out[:, 1:] = np.dot(a, b) + c
+  context->Wait();
+  EXPECT_EQ(static_cast<float>(input3_ptr[0]), 0);
+  EXPECT_EQ(static_cast<float>(input3_ptr[1]), 24);
+  EXPECT_EQ(static_cast<float>(input3_ptr[2]), 28);
+  EXPECT_EQ(static_cast<float>(input3_ptr[3]), 32);
+  EXPECT_EQ(static_cast<float>(input3_ptr[4]), 4);
+  EXPECT_EQ(static_cast<float>(input3_ptr[5]), 73);
+  EXPECT_EQ(static_cast<float>(input3_ptr[6]), 86);
+  EXPECT_EQ(static_cast<float>(input3_ptr[7]), 99);
+}
+
+TEST(math_function, gemm_trans_cublas_fp32) {
+  phi::DenseTensor input1;
+  phi::DenseTensor input2;
+  phi::DenseTensor input3;
+  phi::DenseTensor input1_gpu;
+  phi::DenseTensor input2_gpu;
+  phi::DenseTensor input3_gpu;
+
+  phi::CPUPlace cpu_place;
+  phi::GPUPlace gpu_place(0);
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* context = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  int m = 2;
+  int n = 3;
+  int k = 3;
+  float* input1_ptr = input1.mutable_data<float>({2, 3}, cpu_place);
+  float arr1[6] = {0, 1, 2, 3, 4, 5};
+  memcpy(input1_ptr, arr1, 6 * sizeof(float));
+  float* input2_ptr = input2.mutable_data<float>({4, 3}, cpu_place);
+  float arr2[12] = {0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11};
+  memcpy(input2_ptr, arr2, 12 * sizeof(float));
+  float* input3_ptr = input3.mutable_data<float>({2, 4}, cpu_place);
+  float arr3[8] = {0, 1, 2, 3, 4, 5, 6, 7};
+  memcpy(input3_ptr, arr3, 8 * sizeof(float));
+
+  phi::Copy(*context, input1, gpu_place, true, &input1_gpu);
+  phi::Copy(*context, input2, gpu_place, true, &input2_gpu);
+  phi::Copy(*context, input3, gpu_place, true, &input3_gpu);
+  float* a = input1_gpu.data<float>();
+  float* b = input2_gpu.data<float>();
+  float* c = input3_gpu.mutable_data<float>(gpu_place);
+
+  GetBlas<float>(*context).GEMM(
+      false, true, m, n, k, 1, a, 3, b + 3, 3, 1, c + 1, 4);
+
+  phi::Copy(*context, input3_gpu, cpu_place, true, &input3);
+
+  context->Wait();
+  EXPECT_EQ(input3_ptr[0], 0);
+  EXPECT_EQ(input3_ptr[1], 24);
+  EXPECT_EQ(input3_ptr[2], 28);
+  EXPECT_EQ(input3_ptr[3], 32);
+  EXPECT_EQ(input3_ptr[4], 4);
+  EXPECT_EQ(input3_ptr[5], 73);
+  EXPECT_EQ(input3_ptr[6], 86);
+  EXPECT_EQ(input3_ptr[7], 99);
+}
+
+TEST(math_function, gemm_trans_cublas_fp16) {
+  phi::DenseTensor input1;
+  phi::DenseTensor input2;
+  phi::DenseTensor input3;
+  phi::DenseTensor input1_gpu;
+  phi::DenseTensor input2_gpu;
+  phi::DenseTensor input3_gpu;
+
+  phi::CPUPlace cpu_place;
+  phi::GPUPlace gpu_place(0);
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* context = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  // fp16 GEMM in cublas requires GPU compute capability >= 53
+  if (context->GetComputeCapability() < 53) {
+    return;
+  }
+
+  int m = 2;
+  int n = 3;
+  int k = 3;
+  phi::dtype::float16* input1_ptr =
+      input1.mutable_data<phi::dtype::float16>({2, 3}, cpu_place);
+  fill_fp16_data(input1_ptr, input1.numel(), {0, 1, 2, 3, 4, 5});
+  phi::dtype::float16* input2_ptr =
+      input2.mutable_data<phi::dtype::float16>({4, 3}, cpu_place);
+  fill_fp16_data(
+      input2_ptr, input2.numel(), {0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11});
+  phi::dtype::float16* input3_ptr =
+      input3.mutable_data<phi::dtype::float16>({2, 4}, cpu_place);
+  fill_fp16_data(input3_ptr, input3.numel(), {0, 1, 2, 3, 4, 5, 6, 7});
+
+  phi::Copy(*context, input1, gpu_place, true, &input1_gpu);
+  phi::Copy(*context, input2, gpu_place, true, &input2_gpu);
+  phi::Copy(*context, input3, gpu_place, true, &input3_gpu);
+  phi::dtype::float16* a = input1_gpu.data<phi::dtype::float16>();
+  phi::dtype::float16* b = input2_gpu.data<phi::dtype::float16>();
+  phi::dtype::float16* c =
+      input3_gpu.mutable_data<phi::dtype::float16>(gpu_place);
+
+  GetBlas<phi::dtype::float16>(*context).GEMM(
+      false,
+      true,
+      m,
+      n,
+      k,
+      static_cast<phi::dtype::float16>(1),
+      a,
+      3,
+      b + 3,
+      3,
+      static_cast<phi::dtype::float16>(1),
+      c + 1,
+      4);
+
+  phi::Copy(*context, input3_gpu, cpu_place, true, &input3);
+
+  context->Wait();
+  EXPECT_EQ(static_cast<float>(input3_ptr[0]), 0);
+  EXPECT_EQ(static_cast<float>(input3_ptr[1]), 24);
+  EXPECT_EQ(static_cast<float>(input3_ptr[2]), 28);
+  EXPECT_EQ(static_cast<float>(input3_ptr[3]), 32);
+  EXPECT_EQ(static_cast<float>(input3_ptr[4]), 4);
+  EXPECT_EQ(static_cast<float>(input3_ptr[5]), 73);
+  EXPECT_EQ(static_cast<float>(input3_ptr[6]), 86);
+  EXPECT_EQ(static_cast<float>(input3_ptr[7]), 99);
+}
+
+template <typename T>
+void GemvTest(int m, int n, bool trans) {
+  phi::DenseTensor mat_a;
+  phi::DenseTensor vec_b;
+  phi::DenseTensor vec_c;
+
+  phi::CPUPlace cpu_place;
+  phi::GPUPlace gpu_place(0);
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* context = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  T* data_a = mat_a.mutable_data<T>({m, n}, cpu_place);
+  T* data_b = vec_b.mutable_data<T>({trans ? m : n}, cpu_place);
+  T* data_c = vec_c.mutable_data<T>({trans ? n : m}, cpu_place);
+
+  phi::DenseTensor g_mat_a;
+  phi::DenseTensor g_vec_b;
+  phi::DenseTensor g_vec_c;
+  T* g_data_a = g_mat_a.mutable_data<T>(mat_a.dims(), gpu_place);
+  T* g_data_b = g_vec_b.mutable_data<T>(vec_b.dims(), gpu_place);
+  T* g_data_c = g_vec_c.mutable_data<T>(vec_c.dims(), gpu_place);
+
+  for (int i = 0; i < mat_a.numel(); ++i) {
+    data_a[i] = static_cast<T>(i);
+  }
+  for (int i = 0; i < vec_b.numel(); ++i) {
+    data_b[i] = static_cast<T>(i);
+  }
+
+  phi::Copy(*context, mat_a, gpu_place, true, &g_mat_a);
+  phi::Copy(*context, vec_b, gpu_place, true, &g_vec_b);
+
+  GetBlas<T>(*context).GEMV(trans,
+                            static_cast<int>(m),
+                            static_cast<int>(n),
+                            1.,
+                            g_data_a,
+                            g_data_b,
+                            0.,
+                            g_data_c);
+
+  phi::Copy(*context, g_vec_c, cpu_place, true, &vec_c);
+
+  if (!trans) {
+    for (int i = 0; i < m; ++i) {
+      T sum = 0.0;
+      for (int j = 0; j < n; ++j) {
+        sum += data_a[i * n + j] * data_b[j];
+      }
+      ASSERT_FLOAT_EQ(data_c[i], sum);
+    }
+  } else {
+    for (int i = 0; i < n; ++i) {
+      T sum = 0.0;
+      for (int j = 0; j < m; ++j) {
+        sum += data_a[j * n + i] * data_b[j];
+      }
+      ASSERT_FLOAT_EQ(data_c[i], sum);
+    }
+  }
+}
+
+TEST(math_function, gemv) {
+  GemvTest<float>(3, 13, false);
+  GemvTest<double>(3, 13, false);
+  GemvTest<float>(3, 13, true);
+  GemvTest<double>(3, 13, true);
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/kernels/test_memcpy_dev_api.cc

@@ -0,0 +1,79 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include <memory>
+
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/common/place.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/device_context.h"
+#include "paddle/phi/kernels/memcpy_kernel.h"
+
+namespace phi {
+namespace tests {
+
+using DDim = phi::DDim;
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+TEST(DEV_API, memcpy_d2h) {
+  // 1. create tensor
+  const auto cpu_alloc =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace());
+  phi::DenseTensor x_cpu(cpu_alloc.get(),
+                         phi::DenseTensorMeta(phi::DataType::FLOAT32,
+                                              common::make_ddim({3, 2, 2, 3}),
+                                              phi::DataLayout::NCHW));
+  auto& pool = phi::DeviceContextPool::Instance();
+  auto* cpu_ctx = pool.GetByPlace(phi::CPUPlace());
+  auto* x_cpu_data = cpu_ctx->template Alloc<float>(&x_cpu);
+
+  for (int i = 0; i < x_cpu.numel(); i++) {
+    x_cpu_data[i] = static_cast<float>(i);
+  }
+
+  const auto alloc =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::GPUPlace());
+  phi::DenseTensor x;
+
+  // 2. test API
+  auto* dev_ctx = pool.GetByPlace(phi::GPUPlace());
+
+  phi::MemcpyH2DKernel<phi::GPUContext>(*dev_ctx, x_cpu, 1, &x);
+  phi::DenseTensor out;
+  phi::MemcpyD2HKernel<phi::GPUContext>(*dev_ctx, x, 1, &out);
+
+  // 3. check result
+  std::vector<int64_t> expect_shape = {12, 3};
+  ASSERT_EQ(out.dims(), x.dims());
+  ASSERT_EQ(out.meta().dtype, phi::DataType::FLOAT32);
+  ASSERT_EQ(out.meta().layout, phi::DataLayout::NCHW);
+
+  bool value_equal = true;
+  auto* dense_out_data = out.data<float>();
+  for (int i = 0; i < x_cpu.numel(); i++) {
+    if (x_cpu_data[i] != dense_out_data[i]) {
+      value_equal = false;
+      break;
+    }
+  }
+  ASSERT_EQ(value_equal, true);
+}
+
+#endif
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/kernels/test_strings_copy_dev_api.cc

@@ -0,0 +1,67 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+
+#include <algorithm>
+#include <memory>
+#include <string>
+
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/common/pstring.h"
+#include "paddle/phi/core/string_tensor.h"
+#include "paddle/phi/kernels/strings/strings_copy_kernel.h"
+
+namespace phi {
+namespace tests {
+
+using DDim = phi::DDim;
+using pstring = phi::dtype::pstring;
+
+TEST(DEV_API, strings_copy) {
+  // 1. create tensor
+  const DDim dims({2, 3});
+  StringTensorMeta meta(dims);
+
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+
+  auto* dev_ctx = reinterpret_cast<phi::CPUContext*>(pool.Get(phi::CPUPlace()));
+
+  const auto string_allocator =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace());
+  const auto alloc = string_allocator.get();
+  StringTensor string_src(alloc, meta);
+  StringTensor string_dst(alloc, meta);
+
+  // 2. Assign input text
+  const char* input[] = {"A Short Pstring.",  // NOLINT
+                         "A Large Pstring Whose Length Is Longer Than 22.",
+                         "abc",
+                         "defg",
+                         "hijklmn",
+                         "opqrst"};
+  pstring* string_src_data = dev_ctx->template Alloc<pstring>(&string_src);
+
+  for (int i = 0; i < string_src.numel(); ++i) {
+    string_src_data[i] = input[i];
+  }
+  phi::strings::Copy(*dev_ctx, string_src, false, &string_dst);
+  for (int64_t i = 0; i < string_src.numel(); i++) {
+    ASSERT_EQ(string_src.data()[i], string_dst.data()[i]);
+  }
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/kernels/test_strings_copy_dev_api.cu

@@ -0,0 +1,73 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+
+#include <algorithm>
+#include <memory>
+#include <string>
+
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/common/pstring.h"
+#include "paddle/phi/core/string_tensor.h"
+#include "paddle/phi/kernels/strings/strings_copy_kernel.h"
+#include "paddle/phi/kernels/strings/strings_empty_kernel.h"
+
+namespace phi {
+namespace tests {
+
+using DDim = phi::DDim;
+using pstring = phi::dtype::pstring;
+
+TEST(DEV_API, strings_copy) {
+  // 1. create tensor
+  const DDim dims({2, 3});
+  StringTensorMeta meta(dims);
+
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+
+  auto* dev_ctx = reinterpret_cast<phi::CPUContext*>(pool.Get(phi::CPUPlace()));
+  auto* gpu_dev_ctx =
+      reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  StringTensor string_src = phi::strings::Empty(*dev_ctx, std::move(meta));
+  StringTensor string_dst = phi::strings::Empty(*dev_ctx, std::move(meta));
+  // 2. Assign input text
+  const char* input[] = {"A Short Pstring.",
+                         "A Large Pstring Whose Length Is Longer Than 22.",
+                         "abc",
+                         "defg",
+                         "hijklmn",
+                         "opqrst"};
+  pstring* string_src_data = dev_ctx->template Alloc<pstring>(&string_src);
+  for (int i = 0; i < string_src.numel(); ++i) {
+    string_src_data[i] = input[i];
+  }
+  StringTensor string_gpu1 = phi::strings::Empty(*gpu_dev_ctx, std::move(meta));
+  StringTensor string_gpu2 = phi::strings::Empty(*gpu_dev_ctx, std::move(meta));
+
+  // cpu->gpu
+  phi::strings::Copy(*gpu_dev_ctx, string_src, false, &string_gpu1);
+  // gpu->gpu
+  phi::strings::Copy(*gpu_dev_ctx, string_gpu1, false, &string_gpu2);
+  // gpu->cpu
+  phi::strings::Copy(*gpu_dev_ctx, string_gpu2, false, &string_dst);
+  for (int64_t i = 0; i < string_src.numel(); i++) {
+    ASSERT_EQ(string_src.data()[i], string_dst.data()[i]);
+  }
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/kernels/test_strings_lower_upper_dev_api.cc

@@ -0,0 +1,136 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+
+#include <algorithm>
+#include <memory>
+#include <string>
+
+#include "glog/logging.h"
+
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/common/data_type.h"
+#include "paddle/phi/common/pstring.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/string_tensor.h"
+#include "paddle/phi/kernels/strings/strings_empty_kernel.h"
+#include "paddle/phi/kernels/strings/strings_lower_upper_kernel.h"
+namespace phi {
+namespace tests {
+
+using DDim = phi::DDim;
+using pstring = ::phi::dtype::pstring;
+
+TEST(DEV_API, strings_cast_convert) {
+  // 1. create tensor
+  const DDim dims({1, 2});
+  StringTensorMeta meta(dims);
+  const auto string_allocator =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace());
+  const auto alloc = string_allocator.get();
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+
+  auto* dev_ctx = pool.Get(phi::CPUPlace());
+
+  StringTensor dense_x(alloc, meta);
+
+  std::string short_str = "A Short Pstring.";
+  std::string long_str = "A Large Pstring Whose Length Is Longer Than 22.";
+
+  pstring* dense_x_data = dev_ctx->template Alloc<pstring>(&dense_x);
+  dense_x_data[0] = short_str;
+  dense_x_data[1] = long_str;
+
+  // 2. get expected results
+  std::string expected_results[] = {// NOLINT
+                                    short_str,
+                                    short_str,
+                                    long_str,
+                                    long_str};
+  std::transform(short_str.begin(),
+                 short_str.end(),
+                 expected_results[0].begin(),
+                 ::tolower);
+  std::transform(short_str.begin(),
+                 short_str.end(),
+                 expected_results[1].begin(),
+                 ::toupper);
+  std::transform(
+      long_str.begin(), long_str.end(), expected_results[2].begin(), ::tolower);
+  std::transform(
+      long_str.begin(), long_str.end(), expected_results[3].begin(), ::toupper);
+
+  // 3. test API, ascii encoding
+  auto dense_lower_out = phi::strings::StringLower(
+      *(static_cast<phi::CPUContext*>(dev_ctx)), dense_x, false);
+  auto dense_upper_out = phi::strings::StringUpper(
+      *(static_cast<phi::CPUContext*>(dev_ctx)), dense_x, false);
+
+  // 4. check results
+  ASSERT_EQ(dense_lower_out.numel(), 2);
+  ASSERT_EQ(dense_upper_out.numel(), 2);
+
+  // lower case
+  ASSERT_EQ(dense_lower_out.data()[0].data(), expected_results[0]);
+  ASSERT_EQ(dense_lower_out.data()[1].data(), expected_results[2]);
+
+  // upper case
+  ASSERT_EQ(dense_upper_out.data()[0].data(), expected_results[1]);
+  ASSERT_EQ(dense_upper_out.data()[1].data(), expected_results[3]);
+}
+
+TEST(DEV_API, strings_cast_convert_utf8) {
+  // 1. create tensor
+  const DDim dims({1, 1});
+  StringTensorMeta meta(dims);
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* dev_ctx = pool.Get(phi::CPUPlace());
+
+  const auto string_allocator =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace());
+  const auto alloc = string_allocator.get();
+  StringTensor dense_x(alloc, meta);
+
+  std::string utf8_str = "óÓsscHloëËóÓsscHloëËóÓsscHloëË";
+
+  pstring* dense_x_data = dev_ctx->template Alloc<pstring>(&dense_x);
+  dense_x_data[0] = utf8_str;
+
+  // 2. get expected results
+  std::string expected_results[] = {// NOLINT
+                                    "óósschloëëóósschloëëóósschloëë",
+                                    "ÓÓSSCHLOËËÓÓSSCHLOËËÓÓSSCHLOËË"};
+
+  // 3. test API, ascii encoding
+  auto dense_lower_out = phi::strings::StringLower(
+      *(static_cast<phi::CPUContext*>(dev_ctx)), dense_x, true);
+  auto dense_upper_out = phi::strings::StringUpper(
+      *(static_cast<phi::CPUContext*>(dev_ctx)), dense_x, true);
+
+  // 4. check results
+  ASSERT_EQ(dense_lower_out.numel(), 1);
+  ASSERT_EQ(dense_upper_out.numel(), 1);
+
+  // lower case
+  VLOG(0) << dense_lower_out.data()[0].data();
+  ASSERT_EQ(dense_lower_out.data()[0].data(), expected_results[0]);
+
+  // upper case
+  ASSERT_EQ(dense_upper_out.data()[0].data(), expected_results[1]);
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/kernels/test_strings_lower_upper_dev_api.cu

@@ -0,0 +1,161 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+#include <stdio.h>
+
+#include <algorithm>
+#include <memory>
+#include <string>
+#include <vector>
+
+#if (defined(__NVCC__) || defined(__HIPCC__))
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#endif
+
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/backends/gpu/gpu_helper.h"
+#include "paddle/phi/common/data_type.h"
+#include "paddle/phi/common/pstring.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/string_tensor.h"
+#include "paddle/phi/kernels/strings/strings_copy_kernel.h"
+#include "paddle/phi/kernels/strings/strings_empty_kernel.h"
+#include "paddle/phi/kernels/strings/strings_lower_upper_kernel.h"
+namespace phi {
+namespace tests {
+
+namespace framework = paddle::framework;
+using DDim = phi::DDim;
+using pstring = ::phi::dtype::pstring;
+using phi::CPUPlace;
+using phi::GPUPlace;
+
+TEST(DEV_API, strings_cast_convert) {
+  auto gpu0 = GPUPlace();
+  auto cpu = CPUPlace();
+
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  GPUContext* dev_ctx = reinterpret_cast<GPUContext*>(pool.Get(gpu0));
+  CPUContext* cpu_ctx = reinterpret_cast<CPUContext*>(pool.Get(cpu));
+
+  // 1. create tensor
+  const DDim dims({1, 2});
+  StringTensorMeta meta(dims);
+  StringTensor gpu_strings_x = phi::strings::Empty(*dev_ctx, std::move(meta));
+  StringTensor cpu_strings_x = phi::strings::Empty(*cpu_ctx, std::move(meta));
+  StringTensor cpu_strings_lower_out =
+      phi::strings::Empty(*cpu_ctx, std::move(meta));
+  StringTensor cpu_strings_upper_out =
+      phi::strings::Empty(*cpu_ctx, std::move(meta));
+
+  std::string short_str = "A Short Pstring.";
+  std::string long_str = "A Large Pstring Whose Length Is Longer Than 22.";
+
+  pstring* cpu_strings_x_data =
+      cpu_ctx->template Alloc<pstring>(&cpu_strings_x);
+  cpu_strings_x_data[0] = short_str;
+  cpu_strings_x_data[1] = long_str;
+
+  phi::strings::Copy(*dev_ctx, cpu_strings_x, false, &gpu_strings_x);
+
+  // 2. get expected results
+  std::string expected_results[] = {short_str, short_str, long_str, long_str};
+  std::transform(short_str.begin(),
+                 short_str.end(),
+                 expected_results[0].begin(),
+                 ::tolower);
+  std::transform(short_str.begin(),
+                 short_str.end(),
+                 expected_results[1].begin(),
+                 ::toupper);
+  std::transform(
+      long_str.begin(), long_str.end(), expected_results[2].begin(), ::tolower);
+  std::transform(
+      long_str.begin(), long_str.end(), expected_results[3].begin(), ::toupper);
+
+  // 3. test API, ascii encoding
+  auto gpu_strings_lower_out =
+      phi::strings::StringLower(*dev_ctx, gpu_strings_x, false);
+  auto gpu_strings_upper_out =
+      phi::strings::StringUpper(*dev_ctx, gpu_strings_x, false);
+
+  phi::strings::Copy(
+      *dev_ctx, gpu_strings_lower_out, false, &cpu_strings_lower_out);
+  phi::strings::Copy(
+      *dev_ctx, gpu_strings_upper_out, false, &cpu_strings_upper_out);
+
+  // 4. check results
+  ASSERT_EQ(gpu_strings_lower_out.numel(), 2);
+  ASSERT_EQ(gpu_strings_upper_out.numel(), 2);
+  const char* cpu_results[] = {cpu_strings_lower_out.data()[0].data(),
+                               cpu_strings_upper_out.data()[0].data(),
+                               cpu_strings_lower_out.data()[1].data(),
+                               cpu_strings_upper_out.data()[1].data()};
+  for (int i = 0; i < 4; ++i) {
+    ASSERT_EQ(cpu_results[i], expected_results[i]);
+  }
+}
+
+TEST(DEV_API, strings_cast_convert_utf8) {
+  auto gpu0 = GPUPlace();
+  auto cpu = CPUPlace();
+
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  GPUContext* dev_ctx = reinterpret_cast<GPUContext*>(pool.Get(gpu0));
+  CPUContext* cpu_ctx = reinterpret_cast<CPUContext*>(pool.Get(cpu));
+
+  // 1. create tensor
+  const DDim dims({1, 1});
+  StringTensorMeta meta(dims);
+  StringTensor gpu_strings_x = phi::strings::Empty(*dev_ctx, std::move(meta));
+  StringTensor cpu_strings_x = phi::strings::Empty(*cpu_ctx, std::move(meta));
+  StringTensor cpu_strings_lower_out =
+      phi::strings::Empty(*cpu_ctx, std::move(meta));
+  StringTensor cpu_strings_upper_out =
+      phi::strings::Empty(*cpu_ctx, std::move(meta));
+  std::string utf8_str = "óÓsscHloëË";
+  pstring* cpu_strings_x_data =
+      cpu_ctx->template Alloc<pstring>(&cpu_strings_x);
+
+  cpu_strings_x_data[0] = utf8_str;
+  phi::strings::Copy(*dev_ctx, cpu_strings_x, false, &gpu_strings_x);
+
+  // 2. get expected results
+  std::string expected_results[] = {"óósschloëë", "ÓÓSSCHLOËË"};
+
+  // 3. test API, ascii encoding
+  auto gpu_strings_lower_out =
+      phi::strings::StringLower(*dev_ctx, gpu_strings_x, true);
+  auto gpu_strings_upper_out =
+      phi::strings::StringUpper(*dev_ctx, gpu_strings_x, true);
+  phi::strings::Copy(
+      *dev_ctx, gpu_strings_lower_out, false, &cpu_strings_lower_out);
+  phi::strings::Copy(
+      *dev_ctx, gpu_strings_upper_out, false, &cpu_strings_upper_out);
+
+  // 4. check results
+  const char* cpu_results[] = {cpu_strings_lower_out.data()[0].data(),
+                               cpu_strings_upper_out.data()[0].data()};
+  ASSERT_EQ(cpu_strings_lower_out.numel(), 1);
+  ASSERT_EQ(cpu_strings_upper_out.numel(), 1);
+  for (int i = 0; i < 2; ++i) {
+    ASSERT_EQ(cpu_results[i], expected_results[i]);
+  }
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/kernels/test_ternary_broadcast.cu

@@ -0,0 +1,221 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+#include <vector>
+#include "glog/logging.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/core/tensor_utils.h"
+#include "paddle/phi/kernels/funcs/broadcast_function.h"
+
+template <typename T>
+struct AddTernary_1 {
+  inline HOSTDEVICE T operator()(T a, T b, T c) const { return a + b + c; }
+};
+
+template <typename T>
+struct AddTernary_2 {
+  inline HOSTDEVICE T operator()(T a, T b, T c) const { return a + b + c; }
+};
+
+template <typename T>
+struct AddTernary_3 {
+  inline HOSTDEVICE T operator()(T a, T b, T c) const { return a + b + c; }
+};
+
+template <typename T>
+void InitValue(T* data, size_t numel, const int val) {
+  for (auto i = 0; i < numel; ++i) {
+    data[i] = static_cast<T>(val);
+  }
+}
+
+template <typename T, typename Func>
+void TestCase(const phi::GPUContext& dev_ctx,
+              const phi::DDim& dim1,
+              const phi::DDim& dim2,
+              const phi::DDim& dim3,
+              const phi::DDim& dim_out,
+              const size_t times,
+              Func compute) {
+  phi::DataType dtype = phi::CppTypeToDataType<T>::Type();
+  const auto alloc_cpu =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace());
+  const auto alloc_gpu =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::GPUPlace());
+
+  auto in1 = std::make_shared<phi::DenseTensor>(
+      alloc_cpu.get(),
+      phi::DenseTensorMeta(dtype, dim1, phi::DataLayout::NCHW));
+  auto in2 = std::make_shared<phi::DenseTensor>(
+      alloc_cpu.get(),
+      phi::DenseTensorMeta(dtype, dim2, phi::DataLayout::NCHW));
+  auto in3 = std::make_shared<phi::DenseTensor>(
+      alloc_cpu.get(),
+      phi::DenseTensorMeta(dtype, dim3, phi::DataLayout::NCHW));
+  InitValue(in1->data<T>(), in1->numel(), 1);
+  InitValue(in2->data<T>(), in2->numel(), 1);
+  InitValue(in3->data<T>(), in3->numel(), 1);
+
+  auto d_in1 = std::make_shared<phi::DenseTensor>(
+      alloc_gpu.get(),
+      phi::DenseTensorMeta(dtype, dim1, phi::DataLayout::NCHW));
+  auto d_in2 = std::make_shared<phi::DenseTensor>(
+      alloc_gpu.get(),
+      phi::DenseTensorMeta(dtype, dim2, phi::DataLayout::NCHW));
+  auto d_in3 = std::make_shared<phi::DenseTensor>(
+      alloc_gpu.get(),
+      phi::DenseTensorMeta(dtype, dim3, phi::DataLayout::NCHW));
+  auto d_out = std::make_shared<phi::DenseTensor>(
+      alloc_gpu.get(),
+      phi::DenseTensorMeta(dtype, dim_out, phi::DataLayout::NCHW));
+  phi::Copy(dev_ctx, *in1.get(), phi::GPUPlace(), false, d_in1.get());
+  phi::Copy(dev_ctx, *in2.get(), phi::GPUPlace(), false, d_in2.get());
+  phi::Copy(dev_ctx, *in3.get(), phi::GPUPlace(), false, d_in3.get());
+
+  std::vector<const phi::DenseTensor*> inputs{
+      d_in1.get(), d_in2.get(), d_in3.get()};
+  std::vector<phi::DenseTensor*> outputs{d_out.get()};
+  for (int i = 0; i < times; ++i) {
+    phi::funcs::BroadcastKernel<T>(dev_ctx, inputs, &outputs, compute);
+  }
+  dev_ctx.Wait();
+}
+
+TEST(Broadcast, add) {
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  auto place = phi::GPUPlace();
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* dev_ctx = static_cast<const phi::GPUContext*>(pool.GetByPlace(place));
+  size_t times = 10;
+
+  do {
+    auto dim1 = common::make_ddim({1, 2048, 3584});
+    auto dim2 = common::make_ddim({1, 2048, 1});
+    auto dim3 = common::make_ddim({1, 1, 3584});
+    auto dim_out = common::make_ddim({1, 2048, 3584});
+    TestCase<float>(
+        *dev_ctx, dim1, dim2, dim3, dim_out, times, AddTernary_1<float>());
+    TestCase<phi::dtype::float16>(*dev_ctx,
+                                  dim1,
+                                  dim2,
+                                  dim3,
+                                  dim_out,
+                                  times,
+                                  AddTernary_1<phi::dtype::float16>());
+    TestCase<phi::dtype::bfloat16>(*dev_ctx,
+                                   dim1,
+                                   dim2,
+                                   dim3,
+                                   dim_out,
+                                   times,
+                                   AddTernary_1<phi::dtype::bfloat16>());
+    TestCase<phi::dtype::complex<float>>(
+        *dev_ctx,
+        dim1,
+        dim2,
+        dim3,
+        dim_out,
+        times,
+        AddTernary_1<phi::dtype::complex<float>>());
+    TestCase<phi::dtype::complex<double>>(
+        *dev_ctx,
+        dim1,
+        dim2,
+        dim3,
+        dim_out,
+        times,
+        AddTernary_1<phi::dtype::complex<double>>());
+  } while (0);
+
+  do {
+    auto dim1 = common::make_ddim({1, 256, 4, 256, 256});
+    auto dim2 = common::make_ddim({1, 256, 1, 1, 256});
+    auto dim3 = common::make_ddim({1, 1, 4, 256, 256});
+    auto dim_out = common::make_ddim({1, 256, 4, 256, 256});
+    TestCase<float>(
+        *dev_ctx, dim1, dim2, dim3, dim_out, times, AddTernary_2<float>());
+    TestCase<phi::dtype::float16>(*dev_ctx,
+                                  dim1,
+                                  dim2,
+                                  dim3,
+                                  dim_out,
+                                  times,
+                                  AddTernary_2<phi::dtype::float16>());
+    TestCase<phi::dtype::bfloat16>(*dev_ctx,
+                                   dim1,
+                                   dim2,
+                                   dim3,
+                                   dim_out,
+                                   times,
+                                   AddTernary_2<phi::dtype::bfloat16>());
+    TestCase<phi::dtype::complex<float>>(
+        *dev_ctx,
+        dim1,
+        dim2,
+        dim3,
+        dim_out,
+        times,
+        AddTernary_2<phi::dtype::complex<float>>());
+    TestCase<phi::dtype::complex<double>>(
+        *dev_ctx,
+        dim1,
+        dim2,
+        dim3,
+        dim_out,
+        times,
+        AddTernary_2<phi::dtype::complex<double>>());
+  } while (0);
+
+  do {
+    auto dim1 = common::make_ddim({1, 256, 256});
+    auto dim2 = common::make_ddim({1, 1, 256});
+    auto dim3 = common::make_ddim({1, 256, 1});
+    auto dim_out = common::make_ddim({1, 256, 256});
+    TestCase<float>(
+        *dev_ctx, dim1, dim2, dim3, dim_out, times, AddTernary_3<float>());
+    TestCase<phi::dtype::float16>(*dev_ctx,
+                                  dim1,
+                                  dim2,
+                                  dim3,
+                                  dim_out,
+                                  times,
+                                  AddTernary_3<phi::dtype::float16>());
+    TestCase<phi::dtype::bfloat16>(*dev_ctx,
+                                   dim1,
+                                   dim2,
+                                   dim3,
+                                   dim_out,
+                                   times,
+                                   AddTernary_3<phi::dtype::bfloat16>());
+    TestCase<phi::dtype::complex<float>>(
+        *dev_ctx,
+        dim1,
+        dim2,
+        dim3,
+        dim_out,
+        times,
+        AddTernary_3<phi::dtype::complex<float>>());
+    TestCase<phi::dtype::complex<double>>(
+        *dev_ctx,
+        dim1,
+        dim2,
+        dim3,
+        dim_out,
+        times,
+        AddTernary_3<phi::dtype::complex<double>>());
+  } while (0);
+#endif
+}

test/cpp/phi/kernels/test_transfer_layout_dev_api.cc

@@ -0,0 +1,74 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include <memory>
+
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/common/place.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/device_context.h"
+#include "paddle/phi/infermeta/unary.h"
+#include "paddle/phi/kernels/transfer_layout_kernel.h"
+
+namespace phi {
+namespace tests {
+
+#ifdef PADDLE_WITH_DNNL
+TEST(DEV_API, transfer_layout) {
+  // 1. create tensor
+
+  const int n = 2;
+  const int c = 3;
+  const int h = 4;
+  const int w = 5;
+
+  DenseTensor x;
+  MetaTensor meta_x(&x);
+  meta_x.set_dtype(DataType::FLOAT32);
+  meta_x.set_layout(DataLayout::ONEDNN);
+  meta_x.set_dims(common::make_ddim({n, c, h, w}));
+
+  DenseTensor out;
+
+  // 2. test API
+  auto& pool = phi::DeviceContextPool::Instance();
+  auto place = phi::CPUPlace();
+  auto* dev_ctx = static_cast<const phi::CPUContext*>(pool.GetByPlace(place));
+
+  MetaTensor meta_out(&out);
+  TransferLayoutInferMeta(x,
+                          static_cast<int>(x.layout()),
+                          static_cast<int>(DataLayout::NHWC),
+                          &meta_out);
+  TransferLayoutKernel<CPUContext>(*dev_ctx,
+                                   x,
+                                   static_cast<int>(x.layout()),
+                                   static_cast<int>(DataLayout::NHWC),
+                                   &out);
+
+  // 3. check result
+  std::vector<int64_t> expect_shape = {12, 3};
+  ASSERT_EQ(out.dims(), common::make_ddim({n, h, w, c}));
+  ASSERT_EQ(out.dims().size(), 4);
+  ASSERT_EQ(out.meta().dtype, DataType::FLOAT32);
+  ASSERT_EQ(out.meta().layout, DataLayout::NHWC);
+}
+
+#endif
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/memory/CMakeLists.txt

@@ -0,0 +1,21 @@
+if(WITH_GPU)
+  if(WIN32)
+    message(STATUS "Skip compact_allocator_test on Windows")
+  else()
+    nv_test(
+      compact_allocator_test
+      SRCS compact_allocator_test.cc
+      DEPS phi common)
+  endif()
+endif()
+
+if(WITH_GPU)
+  if(WIN32)
+    message(STATUS "Skip gen_compact_test on Windows")
+  else()
+    nv_test(
+      gen_compact_test
+      SRCS gen_compact_test.cc
+      DEPS phi common)
+  endif()
+endif()

test/cpp/phi/memory/compact_allocator_test.cc

@@ -0,0 +1,43 @@
+// Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/core/memory/allocation/allocator.h"
+#include "paddle/phi/core/memory/allocation/cuda_virtual_mem_allocator.h"
+#include "paddle/phi/core/memory/allocation/retry_allocator.h"
+#include "paddle/phi/core/memory/allocation/virtual_memory_auto_growth_best_fit_allocator.h"
+#include "paddle/phi/core/platform/device/gpu/gpu_info.h"
+#ifdef PADDLE_WITH_CUDA
+#include <cuda.h>
+#include <cuda_runtime.h>
+#endif
+#include "gtest/gtest.h"
+namespace paddle {
+namespace memory {
+namespace allocation {
+
+TEST(VirtualMemoryAutoGrowthBestFitAllocator, TestCompact) {
+  auto vmm_cuda_allocator =
+      std::make_shared<CUDAVirtualMemAllocator>(phi::GPUPlace());
+  auto vma_allocator =
+      std::make_shared<VirtualMemoryAutoGrowthBestFitAllocator>(
+          vmm_cuda_allocator, platform::GpuMinChunkSize(), phi::GPUPlace());
+  size_t mb = (1 << 20);
+  vma_allocator->Allocate(1 * mb);
+  vma_allocator->Allocate(2 * mb);
+  vma_allocator->Compact(phi::GPUPlace());
+}
+
+}  // namespace allocation
+}  // namespace memory
+}  // namespace paddle

test/cpp/phi/memory/gen_compact_test.cc

@@ -0,0 +1,265 @@
+/* Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/common/flags.h"
+#include "paddle/phi/api/include/api.h"
+#include "paddle/phi/api/include/tensor.h"
+#include "paddle/phi/api/lib/api_gen_utils.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/meta_tensor.h"
+
+PD_DECLARE_bool(enable_compact_mem);
+PD_DECLARE_int64(max_reserved_threshold_in_gb);
+PD_DECLARE_int64(cur_allocated_threshold_in_gb);
+PD_DECLARE_bool(try_allocate);
+PD_DECLARE_bool(use_virtual_memory_auto_growth);
+PD_DECLARE_uint64(vmm_small_pool_size_in_mb);
+
+namespace paddle {
+namespace memory {
+namespace allocation {
+using paddle::experimental::CheckAndDoCompact;
+class CheckAndDoCompactTest : public ::testing::Test {
+ protected:
+  void SetUp() override {
+    // Set default flags
+    FLAGS_enable_compact_mem = true;
+    FLAGS_try_allocate = true;
+    FLAGS_use_virtual_memory_auto_growth = true;
+    FLAGS_vmm_small_pool_size_in_mb = 2;
+    FLAGS_v = 10;
+  }
+
+  void TearDown() override { meta_tensors_.clear(); }
+
+  std::vector<phi::MetaTensor*> meta_tensors_;
+};
+
+TEST_F(CheckAndDoCompactTest, DisabledByFlag) {
+  FLAGS_enable_compact_mem = false;
+  CheckAndDoCompact(meta_tensors_, "test_api");
+}
+
+TEST_F(CheckAndDoCompactTest, NoCompactWhenBelowMaxReservedThreshold) {
+  FLAGS_enable_compact_mem = true;
+  FLAGS_max_reserved_threshold_in_gb = 80;
+  FLAGS_cur_allocated_threshold_in_gb = 0;
+  CheckAndDoCompact(meta_tensors_, "test_api");
+}
+
+TEST_F(CheckAndDoCompactTest, NoCompactWhenBelowCurAllocatedThreshold) {
+  FLAGS_enable_compact_mem = true;
+  FLAGS_max_reserved_threshold_in_gb = 0;
+  FLAGS_cur_allocated_threshold_in_gb = 80;
+  CheckAndDoCompact(meta_tensors_, "test_api");
+}
+
+TEST_F(CheckAndDoCompactTest, CompactWhenNeeded) {
+  FLAGS_cur_allocated_threshold_in_gb = 0;
+  FLAGS_max_reserved_threshold_in_gb = 0;
+}
+
+TEST_F(CheckAndDoCompactTest, SkipZeroNumelTensors) {
+  phi::DenseTensor zero_tensor;
+  phi::DenseTensorMeta zero_meta(phi::DataType::FLOAT32, phi::DDim({0}));
+  zero_tensor.set_meta(zero_meta);
+  phi::MetaTensor meta_tensor(zero_tensor);
+  meta_tensors_.push_back(&meta_tensor);
+
+  FLAGS_cur_allocated_threshold_in_gb = 0;
+  FLAGS_max_reserved_threshold_in_gb = 0;
+
+  CheckAndDoCompact(meta_tensors_, "test_api");
+}
+
+TEST_F(CheckAndDoCompactTest, SkipNagetiveNumelTensors) {
+  phi::DenseTensor negative_tensor;
+  phi::DenseTensorMeta negative_meta(phi::DataType::FLOAT32, phi::DDim({-1}));
+  negative_meta.is_scalar = true;
+  negative_tensor.set_meta(negative_meta);
+  phi::MetaTensor meta_tensor(negative_tensor);
+  meta_tensors_.push_back(&meta_tensor);
+
+  FLAGS_cur_allocated_threshold_in_gb = 0;
+  FLAGS_max_reserved_threshold_in_gb = 0;
+
+  CheckAndDoCompact(meta_tensors_, "test_api");
+}
+
+TEST_F(CheckAndDoCompactTest, ReqLessThenMaxFree) {
+  FLAGS_cur_allocated_threshold_in_gb = 0;
+  FLAGS_max_reserved_threshold_in_gb = 0;
+  auto var1 = paddle::experimental::full(
+      {10, 1024, 1024}, 1, paddle::DataType::FLOAT32, paddle::GPUPlace());
+
+  var1.reset();
+
+  phi::DenseTensor tensor;
+  phi::DenseTensorMeta meta(phi::DataType::FLOAT32, phi::DDim({2, 1024, 1024}));
+  tensor.set_meta(meta);
+  phi::MetaTensor meta_tensor(tensor);
+
+  meta_tensors_.push_back(&meta_tensor);
+  CheckAndDoCompact(meta_tensors_, "test_api");
+}
+
+TEST_F(CheckAndDoCompactTest, ReqMoreThenLargestNFree) {
+  FLAGS_cur_allocated_threshold_in_gb = 0;
+  FLAGS_max_reserved_threshold_in_gb = 0;
+  auto var1 = paddle::experimental::full(
+      {10, 1024, 1024}, 1, paddle::DataType::FLOAT32, paddle::GPUPlace());
+  var1.reset();
+
+  phi::DenseTensor tensor;
+  phi::DenseTensorMeta meta(phi::DataType::FLOAT32,
+                            phi::DDim({20, 1024, 1024}));
+  tensor.set_meta(meta);
+  phi::MetaTensor meta_tensor(tensor);
+
+  meta_tensors_.push_back(&meta_tensor);
+  CheckAndDoCompact(meta_tensors_, "test_api");
+}
+
+TEST_F(CheckAndDoCompactTest, TryAllocDisable) {
+  FLAGS_try_allocate = false;
+  FLAGS_cur_allocated_threshold_in_gb = 0;
+  FLAGS_max_reserved_threshold_in_gb = 0;
+  auto var1 = paddle::experimental::full(
+      {10, 1024, 1024}, 1, paddle::DataType::FLOAT32, paddle::GPUPlace());
+  auto var2 = paddle::experimental::full(
+      {2, 1024, 1024}, 1, paddle::DataType::FLOAT32, paddle::GPUPlace());
+  auto var3 = paddle::experimental::full(
+      {5, 1024, 1024}, 1, paddle::DataType::FLOAT32, paddle::GPUPlace());
+  var1.reset();
+  var3.reset();
+
+  phi::DenseTensor tensor1;
+  phi::DenseTensorMeta meta1(phi::DataType::FLOAT32,
+                             phi::DDim({8, 1024, 1024}));
+  tensor1.set_meta(meta1);
+  phi::MetaTensor meta_tensor1(tensor1);
+
+  phi::DenseTensor tensor2;
+  phi::DenseTensorMeta meta2(phi::DataType::FLOAT32,
+                             phi::DDim({4, 1024, 1024}));
+  tensor2.set_meta(meta2);
+  phi::MetaTensor meta_tensor2(tensor2);
+
+  meta_tensors_.push_back(&meta_tensor1);
+  meta_tensors_.push_back(&meta_tensor2);
+  CheckAndDoCompact(meta_tensors_, "test_api");
+}
+
+TEST_F(CheckAndDoCompactTest, TryAllocSucc) {
+  FLAGS_try_allocate = true;
+  FLAGS_cur_allocated_threshold_in_gb = 0;
+  FLAGS_max_reserved_threshold_in_gb = 0;
+  auto var1 = paddle::experimental::full(
+      {15, 1024, 1024}, 1, paddle::DataType::FLOAT32, paddle::GPUPlace());
+  auto var2 = paddle::experimental::full(
+      {2, 1024, 1024}, 1, paddle::DataType::FLOAT32, paddle::GPUPlace());
+  auto var3 = paddle::experimental::full(
+      {10, 1024, 1024}, 1, paddle::DataType::FLOAT32, paddle::GPUPlace());
+  var1.reset();
+  var3.reset();
+
+  phi::DenseTensor tensor1;
+  phi::DenseTensorMeta meta1(phi::DataType::FLOAT32,
+                             phi::DDim({10, 1024, 1024}));
+  tensor1.set_meta(meta1);
+  phi::MetaTensor meta_tensor1(tensor1);
+
+  phi::DenseTensor tensor2;
+  phi::DenseTensorMeta meta2(phi::DataType::FLOAT32,
+                             phi::DDim({9, 1024, 1024}));
+  tensor2.set_meta(meta2);
+  phi::MetaTensor meta_tensor2(tensor2);
+
+  meta_tensors_.push_back(&meta_tensor1);
+  meta_tensors_.push_back(&meta_tensor2);
+  CheckAndDoCompact(meta_tensors_, "test_api");
+}
+
+TEST_F(CheckAndDoCompactTest, TryAllocSuccNoSplit) {
+  FLAGS_try_allocate = true;
+  FLAGS_cur_allocated_threshold_in_gb = 0;
+  FLAGS_max_reserved_threshold_in_gb = 0;
+  auto var1 = paddle::experimental::full(
+      {10, 1024, 1024}, 1, paddle::DataType::FLOAT32, paddle::GPUPlace());
+  auto var2 = paddle::experimental::full(
+      {2, 1024, 1024}, 1, paddle::DataType::FLOAT32, paddle::GPUPlace());
+  auto var3 = paddle::experimental::full(
+      {10, 1024, 1024}, 1, paddle::DataType::FLOAT32, paddle::GPUPlace());
+  var1.reset();
+  var3.reset();
+
+  phi::DenseTensor tensor1;
+  phi::DenseTensorMeta meta1(phi::DataType::FLOAT32,
+                             phi::DDim({10, 1024, 1024}));
+  tensor1.set_meta(meta1);
+  phi::MetaTensor meta_tensor1(tensor1);
+
+  phi::DenseTensor tensor2;
+  phi::DenseTensorMeta meta2(phi::DataType::FLOAT32,
+                             phi::DDim({10, 1024, 1024}));
+  tensor2.set_meta(meta2);
+  phi::MetaTensor meta_tensor2(tensor2);
+
+  meta_tensors_.push_back(&meta_tensor1);
+  meta_tensors_.push_back(&meta_tensor2);
+  CheckAndDoCompact(meta_tensors_, "test_api");
+}
+
+TEST_F(CheckAndDoCompactTest, TryAllocFail) {
+  FLAGS_try_allocate = true;
+  FLAGS_cur_allocated_threshold_in_gb = 0;
+  FLAGS_max_reserved_threshold_in_gb = 0;
+  auto var1 = paddle::experimental::full(
+      {10, 1024, 1024}, 1, paddle::DataType::FLOAT32, paddle::GPUPlace());
+  auto var2 = paddle::experimental::full(
+      {2, 1024, 1024}, 1, paddle::DataType::FLOAT32, paddle::GPUPlace());
+  auto var3 = paddle::experimental::full(
+      {5, 1024, 1024}, 1, paddle::DataType::FLOAT32, paddle::GPUPlace());
+  var1.reset();
+  var3.reset();
+
+  phi::DenseTensor tensor1;
+  phi::DenseTensorMeta meta1(phi::DataType::FLOAT32,
+                             phi::DDim({11, 1024, 1024}));
+  tensor1.set_meta(meta1);
+  phi::MetaTensor meta_tensor1(tensor1);
+
+  phi::DenseTensor tensor2;
+  phi::DenseTensorMeta meta2(phi::DataType::FLOAT32,
+                             phi::DDim({2, 1024, 1024}));
+  tensor2.set_meta(meta2);
+  phi::MetaTensor meta_tensor2(tensor2);
+
+  meta_tensors_.push_back(&meta_tensor1);
+  meta_tensors_.push_back(&meta_tensor2);
+  CheckAndDoCompact(meta_tensors_, "test_api");
+}
+
+TEST_F(CheckAndDoCompactTest, MetaNullptr) {
+  FLAGS_cur_allocated_threshold_in_gb = 0;
+  FLAGS_max_reserved_threshold_in_gb = 0;
+  meta_tensors_.push_back(nullptr);
+  CheckAndDoCompact(meta_tensors_, "test_api");
+}
+
+}  // namespace allocation
+}  // namespace memory
+}  // namespace paddle

test/cpp/phi/ops/CMakeLists.txt

@@ -0,0 +1,11 @@
+if(WIN32)
+  cc_test(
+    test_op_signature
+    SRCS test_op_signature.cc
+    DEPS type_info common)
+else()
+  cc_test(
+    test_op_signature
+    SRCS test_op_signature.cc
+    DEPS phi common)
+endif()

test/cpp/phi/ops/test_op_signature.cc

@@ -0,0 +1,636 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "test/cpp/phi/ops/test_op_signature.h"
+
+#include <gtest/gtest.h>
+
+#include <memory>
+#include <unordered_set>
+
+#include "paddle/fluid/operators/ops_signature/signatures.h"
+#include "paddle/phi/core/compat/op_utils.h"
+
+namespace phi {
+namespace tests {
+
+// The unittests in this file are just order to pass the CI-Coverage，
+// so it isn't necessary to check the all cases.
+
+TEST(ARG_MAP, fill_constant) {
+  TestArgumentMappingContext arg_case1(
+      {"ShapeTensor", "ValueTensor"}, {}, {}, {}, {"Out"});
+  auto signature1 = (*OpUtilsMap::Instance().GetArgumentMappingFn(
+      "fill_constant"))(arg_case1);
+  EXPECT_STREQ(signature1.name, "full_sr");
+
+  TestArgumentMappingContext arg_case2(
+      {"ShapeTensor"},
+      {},
+      {{"str_value", paddle::any{std::string{"10"}}}},
+      {},
+      {"Out"});
+  auto signature2 = (*OpUtilsMap::Instance().GetArgumentMappingFn(
+      "fill_constant"))(arg_case2);
+  EXPECT_STREQ(signature2.name, "full_sr");
+
+  TestArgumentMappingContext arg_case3(
+      {"ShapeTensor"},
+      {},
+      {{"value", paddle::any{0}}, {"str_value", paddle::any{std::string{""}}}},
+      {},
+      {"Out"});
+  auto signature3 = (*OpUtilsMap::Instance().GetArgumentMappingFn(
+      "fill_constant"))(arg_case3);
+  EXPECT_STREQ(signature3.name, "full_sr");
+
+  TestArgumentMappingContext arg_case4(
+      {"ShapeTensorList", "ValueTensor"}, {}, {}, {}, {"Out"});
+  auto signature4 = (*OpUtilsMap::Instance().GetArgumentMappingFn(
+      "fill_constant"))(arg_case4);
+  EXPECT_STREQ(signature4.name, "full_sr");
+
+  TestArgumentMappingContext arg_case5(
+      {"ShapeTensorList"},
+      {},
+      {{"str_value", paddle::any{std::string{"10"}}}},
+      {},
+      {"Out"});
+  auto signature5 = (*OpUtilsMap::Instance().GetArgumentMappingFn(
+      "fill_constant"))(arg_case5);
+  EXPECT_STREQ(signature5.name, "full_sr");
+
+  TestArgumentMappingContext arg_case6(
+      {"ShapeTensorList"},
+      {},
+      {{"value", paddle::any{0}}, {"str_value", paddle::any{std::string{""}}}},
+      {},
+      {"Out"});
+  auto signature6 = (*OpUtilsMap::Instance().GetArgumentMappingFn(
+      "fill_constant"))(arg_case6);
+  EXPECT_STREQ(signature6.name, "full_sr");
+
+  TestArgumentMappingContext arg_case7(
+      {"ValueTensor"},
+      {},
+      {{"shape", paddle::any{std::vector<int64_t>{2, 3}}}},
+      {},
+      {"Out"});
+  auto signature7 = (*OpUtilsMap::Instance().GetArgumentMappingFn(
+      "fill_constant"))(arg_case7);
+  EXPECT_STREQ(signature7.name, "full_sr");
+
+  TestArgumentMappingContext arg_case8(
+      {},
+      {},
+      {{"shape", paddle::any{std::vector<int64_t>{2, 3}}},
+       {"value", paddle::any{0}},
+       {"str_value", paddle::any{std::string{""}}}},
+      {},
+      {"Out"});
+  auto signature8 = (*OpUtilsMap::Instance().GetArgumentMappingFn(
+      "fill_constant"))(arg_case8);
+  EXPECT_STREQ(signature8.name, "full_sr");
+
+  TestArgumentMappingContext arg_case9(
+      {},
+      {},
+      {{"shape", paddle::any{std::vector<int64_t>{2, 3}}},
+       {"str_value", paddle::any{std::string{"10"}}}},
+      {},
+      {"Out"});
+  auto signature9 = (*OpUtilsMap::Instance().GetArgumentMappingFn(
+      "fill_constant"))(arg_case9);
+  EXPECT_STREQ(signature9.name, "full_sr");
+}
+
+TEST(ARG_MAP, set_value) {
+  TestArgumentMappingContext arg_case(
+      {"Input", "StartsTensorList", "EndsTensorList", "StepsTensorList"},
+      {},
+      {{"fp32_values", paddle::any{std::vector<float>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case1(
+      {"Input", "StartsTensorList", "EndsTensorList", "StepsTensorList"},
+      {},
+      {{"fp64_values", paddle::any{std::vector<double>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case1)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case2(
+      {"Input", "StartsTensorList", "EndsTensorList", "StepsTensorList"},
+      {},
+      {{"int32_values", paddle::any{std::vector<int>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case2)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case3(
+      {"Input", "StartsTensorList", "EndsTensorList", "StepsTensorList"},
+      {},
+      {{"int64_values", paddle::any{std::vector<int64_t>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case3)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case4(
+      {"Input", "StartsTensorList", "EndsTensorList", "StepsTensorList"},
+      {},
+      {{"bool_values", paddle::any{std::vector<int>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case4)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case5(
+      {"Input", "StartsTensorList", "EndsTensorList", "ValueTensor"},
+      {},
+      {},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case5)
+          .name,
+      "set_value_with_tensor");
+
+  TestArgumentMappingContext arg_case6(
+      {"Input", "StartsTensorList", "EndsTensorList"},
+      {},
+      {{"fp64_values", paddle::any{std::vector<double>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case6)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case7(
+      {"Input", "StartsTensorList", "EndsTensorList"},
+      {},
+      {{"int32_values", paddle::any{std::vector<int>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case7)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case8(
+      {"Input", "StartsTensorList", "EndsTensorList"},
+      {},
+      {{"int64_values", paddle::any{std::vector<int64_t>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case8)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case9(
+      {"Input", "StartsTensorList", "EndsTensorList"},
+      {},
+      {{"bool_values", paddle::any{std::vector<int>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case9)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case10(
+      {"Input", "StartsTensorList", "StepsTensorList", "ValueTensor"},
+      {},
+      {},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case10)
+          .name,
+      "set_value_with_tensor");
+
+  TestArgumentMappingContext arg_case11(
+      {"Input", "StartsTensorList", "StepsTensorList"},
+      {},
+      {{"fp64_values", paddle::any{std::vector<double>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case11)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case12(
+      {"Input", "StartsTensorList", "StepsTensorList"},
+      {},
+      {{"int32_values", paddle::any{std::vector<int>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case12)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case13(
+      {"Input", "StartsTensorList", "StepsTensorList"},
+      {},
+      {{"int64_values", paddle::any{std::vector<int64_t>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case13)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case14(
+      {"Input", "StartsTensorList", "StepsTensorList"},
+      {},
+      {{"bool_values", paddle::any{std::vector<int>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case14)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case15(
+      {"Input", "StartsTensorList", "ValueTensor"}, {}, {}, {"Out"}, {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case15)
+          .name,
+      "set_value_with_tensor");
+
+  TestArgumentMappingContext arg_case16(
+      {"Input", "StartsTensorList", "StepsTensorList"},
+      {},
+      {{"fp32_values", paddle::any{std::vector<float>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case16)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case17(
+      {"Input", "StartsTensorList", "StepsTensorList"},
+      {},
+      {{"fp64_values", paddle::any{std::vector<double>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case17)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case18(
+      {"Input", "StartsTensorList", "StepsTensorList"},
+      {},
+      {{"int32_values", paddle::any{std::vector<int>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case18)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case19(
+      {"Input", "StartsTensorList", "StepsTensorList"},
+      {},
+      {{"int64_values", paddle::any{std::vector<int64_t>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case19)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case20(
+      {"Input", "StartsTensorList", "StepsTensorList"},
+      {},
+      {{"bool_values", paddle::any{std::vector<int>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case20)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case21(
+      {"Input", "EndsTensorList", "StepsTensorList", "ValueTensor"},
+      {},
+      {},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case21)
+          .name,
+      "set_value_with_tensor");
+
+  TestArgumentMappingContext arg_case22(
+      {"Input", "EndsTensorList", "StepsTensorList"},
+      {},
+      {{"fp64_values", paddle::any{std::vector<double>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case22)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case23(
+      {"Input", "EndsTensorList", "StepsTensorList"},
+      {},
+      {{"int32_values", paddle::any{std::vector<int>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case23)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case24(
+      {"Input", "EndsTensorList", "StepsTensorList"},
+      {},
+      {{"int64_values", paddle::any{std::vector<int64_t>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case24)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case25(
+      {"Input", "EndsTensorList", "StepsTensorList"},
+      {},
+      {{"bool_values", paddle::any{std::vector<int>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case25)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case26(
+      {"Input", "EndsTensorList", "ValueTensor"}, {}, {}, {"Out"}, {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case26)
+          .name,
+      "set_value_with_tensor");
+
+  TestArgumentMappingContext arg_case27(
+      {"Input", "EndsTensorList"},
+      {},
+      {{"fp32_values", paddle::any{std::vector<float>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case27)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case28(
+      {"Input", "EndsTensorList"},
+      {},
+      {{"fp64_values", paddle::any{std::vector<double>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case28)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case29(
+      {"Input", "EndsTensorList"},
+      {},
+      {{"int32_values", paddle::any{std::vector<int>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case29)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case30(
+      {"Input", "EndsTensorList"},
+      {},
+      {{"int64_values", paddle::any{std::vector<int64_t>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case30)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case31(
+      {"Input", "EndsTensorList"},
+      {},
+      {{"bool_values", paddle::any{std::vector<int>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case31)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case32(
+      {"Input", "StepsTensorList", "ValueTensor"}, {}, {}, {"Out"}, {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case32)
+          .name,
+      "set_value_with_tensor");
+
+  TestArgumentMappingContext arg_case33(
+      {"Input", "StepsTensorList"},
+      {},
+      {{"fp32_values", paddle::any{std::vector<float>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case33)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case34(
+      {"Input", "StepsTensorList"},
+      {},
+      {{"fp64_values", paddle::any{std::vector<double>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case34)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case35(
+      {"Input", "StepsTensorList"},
+      {},
+      {{"int32_values", paddle::any{std::vector<int>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case35)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case36(
+      {"Input", "StepsTensorList"},
+      {},
+      {{"int64_values", paddle::any{std::vector<int64_t>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case36)
+          .name,
+      "set_value");
+
+  TestArgumentMappingContext arg_case37(
+      {"Input", "StepsTensorList"},
+      {},
+      {{"bool_values", paddle::any{std::vector<int>{1}}}},
+      {"Out"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case37)
+          .name,
+      "set_value");
+}
+
+TEST(ARG_MAP, set_value_grad) {
+  TestArgumentMappingContext arg_case(
+      {"Out@GRAD", "StartsTensorList", "EndsTensorList"},
+      {},
+      {},
+      {"Input@GRAD", "ValueTensor@GRAD"},
+      {});
+  EXPECT_STREQ(
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value_grad"))(arg_case)
+          .name,
+      "set_value_grad");
+
+  TestArgumentMappingContext arg_case1(
+      {"Out@GRAD", "StartsTensorList", "StepsTensorList"},
+      {},
+      {},
+      {"Input@GRAD", "ValueTensor@GRAD"},
+      {});
+  EXPECT_STREQ((*OpUtilsMap::Instance().GetArgumentMappingFn("set_value_grad"))(
+                   arg_case1)
+                   .name,
+               "set_value_grad");
+
+  TestArgumentMappingContext arg_case2({"Out@GRAD", "StartsTensorList"},
+                                       {},
+                                       {},
+                                       {"Input@GRAD", "ValueTensor@GRAD"},
+                                       {});
+  EXPECT_STREQ((*OpUtilsMap::Instance().GetArgumentMappingFn("set_value_grad"))(
+                   arg_case2)
+                   .name,
+               "set_value_grad");
+
+  TestArgumentMappingContext arg_case3(
+      {"Out@GRAD", "EndsTensorList", "StepsTensorList"},
+      {},
+      {},
+      {"Input@GRAD", "ValueTensor@GRAD"},
+      {});
+  EXPECT_STREQ((*OpUtilsMap::Instance().GetArgumentMappingFn("set_value_grad"))(
+                   arg_case3)
+                   .name,
+               "set_value_grad");
+
+  TestArgumentMappingContext arg_case4({"Out@GRAD", "EndsTensorList"},
+                                       {},
+                                       {},
+                                       {"Input@GRAD", "ValueTensor@GRAD"},
+                                       {});
+  EXPECT_STREQ((*OpUtilsMap::Instance().GetArgumentMappingFn("set_value_grad"))(
+                   arg_case4)
+                   .name,
+               "set_value_grad");
+
+  TestArgumentMappingContext arg_case5({"Out@GRAD", "StepsTensorList"},
+                                       {},
+                                       {},
+                                       {"Input@GRAD", "ValueTensor@GRAD"},
+                                       {});
+  EXPECT_STREQ((*OpUtilsMap::Instance().GetArgumentMappingFn("set_value_grad"))(
+                   arg_case5)
+                   .name,
+               "set_value_grad");
+}
+
+TEST(ARG_MAP, allclose) {
+  TestArgumentMappingContext arg_case1(
+      {"Input", "Other", "Rtol"},
+      {},
+      {{"atol", paddle::any(std::string{"1e-8"})},
+       {"equal_nan", paddle::any(false)}},
+      {"Out"},
+      {});
+  auto signature1 =
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("allclose"))(arg_case1);
+  EXPECT_STREQ(signature1.name, "allclose");
+  EXPECT_STREQ(signature1.attr_names[0], "Rtol");
+
+  TestArgumentMappingContext arg_case2(
+      {"Input", "Other", "Atol"},
+      {},
+      {{"rtol", paddle::any(std::string{"1e-5"})},
+       {"equal_nan", paddle::any(false)}},
+      {"Out"},
+      {});
+  auto signature2 =
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("allclose"))(arg_case2);
+  EXPECT_STREQ(signature2.name, "allclose");
+  EXPECT_STREQ(signature2.attr_names[1], "Atol");
+}
+
+TEST(ARG_MAP, reshape) {
+  TestArgumentMappingContext arg_case1({"X", "ShapeTensor"}, {}, {}, {"Out"});
+  auto signature1 =
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("reshape2"))(arg_case1);
+  EXPECT_STREQ(signature1.name, "reshape");
+
+  TestArgumentMappingContext arg_case2({"X", "Shape"}, {}, {}, {"Out"});
+  auto signature2 =
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("reshape2"))(arg_case2);
+  EXPECT_STREQ(signature2.name, "reshape");
+
+  TestArgumentMappingContext arg_case3(
+      {"X"}, {}, {{"shape", paddle::any(std::vector<int>({1, 2}))}}, {"Out"});
+  auto signature3 =
+      (*OpUtilsMap::Instance().GetArgumentMappingFn("reshape2"))(arg_case3);
+  EXPECT_STREQ(signature3.name, "reshape");
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/ops/test_op_signature.h

@@ -0,0 +1,120 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <gtest/gtest.h>
+
+#include <memory>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "paddle/phi/core/compat/op_utils.h"
+
+namespace phi {
+namespace tests {
+
+class TestArgumentMappingContext : public phi::ArgumentMappingContext {
+ public:
+  TestArgumentMappingContext(
+      std::unordered_set<std::string> dense_tensor_ins,
+      std::unordered_set<std::string> sr_ins,
+      std::unordered_map<std::string, paddle::any> op_attrs,
+      std::unordered_set<std::string> dense_tensor_outs,
+      std::unordered_set<std::string> sr_outs = {})
+      : dense_tensor_inputs(dense_tensor_ins),
+        selected_rows_inputs(sr_ins),
+        attrs(op_attrs),
+        dense_tensor_outputs(dense_tensor_outs),
+        selected_rows_outputs(sr_outs) {}
+
+  bool HasInput(const std::string& name) const override {
+    return dense_tensor_inputs.count(name) > 0 ||
+           selected_rows_inputs.count(name) > 0;
+  }
+
+  bool HasOutput(const std::string& name) const override {
+    return dense_tensor_outputs.count(name) > 0 ||
+           selected_rows_outputs.count(name) > 0;
+  }
+
+  bool HasAttr(const std::string& name) const override {
+    return attrs.count(name) > 0;
+  }
+
+  paddle::any Attr(const std::string& name) const override {
+    return attrs.at(name);
+  }
+
+  size_t InputSize(const std::string& name) const override {
+    return dense_tensor_inputs.count(name) + selected_rows_inputs.count(name);
+  }
+
+  size_t OutputSize(const std::string& name) const override {
+    return dense_tensor_outputs.size() + selected_rows_outputs.size();
+  }
+
+  bool IsDenseTensorInput(const std::string& name) const override {
+    return dense_tensor_inputs.count(name) > 0;
+  }
+
+  bool IsDenseTensorInputs(const std::string& name) const override {
+    return dense_tensor_inputs.count(name) > 0;
+  }
+
+  bool IsSelectedRowsInput(const std::string& name) const override {
+    return selected_rows_inputs.count(name) > 0;
+  }
+
+  bool IsSelectedRowsInputs(const std::string& name) const override {
+    return selected_rows_inputs.count(name) > 0;
+  }
+
+  // add member if needed
+  bool IsDenseTensorVectorInput(const std::string& name) const override {
+    return false;
+  }
+
+  bool IsSparseCooTensorInput(const std::string& name) const override {
+    return false;
+  }
+
+  bool IsSparseCsrTensorInput(const std::string& name) const override {
+    return false;
+  }
+
+  bool IsSparseCooTensorOutput(const std::string& name) const override {
+    return false;
+  }
+
+  bool IsDenseTensorOutput(const std::string& name) const override {
+    return dense_tensor_outputs.count(name) > 0;
+  }
+
+  bool IsSelectedRowsOutput(const std::string& name) const override {
+    return selected_rows_outputs.count(name) > 0;
+  }
+
+  bool IsForInferShape() const override { return false; }
+
+ private:
+  const std::unordered_set<std::string> dense_tensor_inputs;
+  const std::unordered_set<std::string> selected_rows_inputs;
+  const std::unordered_map<std::string, paddle::any> attrs;
+  const std::unordered_set<std::string> dense_tensor_outputs;
+  const std::unordered_set<std::string> selected_rows_outputs;
+};
+
+}  // namespace tests
+}  // namespace phi