chore: import upstream snapshot with attribution

test/cpp/phi/kernels/CMakeLists.txt

@@ -0,0 +1,136 @@
+cc_test(
+  test_math_function
+  SRCS test_math_function.cc
+  DEPS phi common)
+if(WITH_GPU)
+  nv_test(
+    test_math_function_gpu
+    SRCS test_math_function.cu
+    DEPS phi common)
+  nv_test(
+    test_broadcast_gpu
+    SRCS test_ternary_broadcast.cu
+    DEPS gtest)
+endif()
+if(WITH_ROCM)
+  hip_test(
+    test_math_function_gpu
+    SRCS test_math_function.cu
+    DEPS phi common)
+endif()
+
+cc_test(
+  test_cpu_vec
+  SRCS test_cpu_vec.cc
+  DEPS phi common)
+
+# For String Kernels
+if(WIN32)
+  cc_test(
+    test_strings_lower_upper_dev_api
+    SRCS test_strings_lower_upper_dev_api.cc
+    DEPS type_info common)
+else()
+  cc_test(
+    test_strings_lower_upper_dev_api
+    SRCS test_strings_lower_upper_dev_api.cc
+    DEPS phi common)
+endif()
+if(WITH_GPU)
+  nv_test(
+    test_strings_lower_upper_dev_gpu_api
+    SRCS test_strings_lower_upper_dev_api.cu
+    DEPS phi common)
+elseif(WITH_ROCM)
+  hip_test(
+    test_strings_lower_upper_dev_gpu_api
+    SRCS test_strings_lower_upper_dev_api.cu
+    DEPS phi common)
+endif()
+
+cc_test(
+  test_strings_copy_dev_api
+  SRCS test_strings_copy_dev_api.cc
+  DEPS phi common)
+if(WITH_GPU)
+  nv_test(
+    test_strings_copy_dev_gpu_api
+    SRCS test_strings_copy_dev_api.cu
+    DEPS phi common)
+elseif(WITH_ROCM)
+  hip_test(
+    test_strings_copy_dev_gpu_api
+    SRCS test_strings_copy_dev_api.cu
+    DEPS phi common)
+endif()
+
+if(WIN32)
+  cc_test(
+    test_memcpy_dev_api
+    SRCS test_memcpy_dev_api.cc
+    DEPS type_info common)
+  cc_test(
+    test_transfer_layout_dev_api
+    SRCS test_memcpy_dev_api.cc
+    DEPS type_info common)
+else()
+  cc_test(
+    test_memcpy_dev_api
+    SRCS test_memcpy_dev_api.cc
+    DEPS phi common)
+  cc_test(
+    test_transfer_layout_dev_api
+    SRCS test_transfer_layout_dev_api.cc
+    DEPS phi common)
+endif()
+
+if(WITH_GPU)
+  nv_test(
+    test_gpu_timer
+    SRCS test_gpu_timer.cu
+    DEPS gtest)
+  nv_test(
+    test_auto_tune
+    SRCS test_auto_tune.cu
+    DEPS gtest)
+  cc_test(
+    test_fused_adam_kernel
+    SRCS test_fused_adam_kernel.cc
+    DEPS gtest phi common)
+elseif(WITH_ROCM)
+  hip_test(
+    test_gpu_timer
+    SRCS test_gpu_timer.cu
+    DEPS gtest)
+  hip_test(
+    test_auto_tune
+    SRCS test_auto_tune.cu
+    DEPS gtest)
+endif()
+
+cc_test(
+  test_cache
+  SRCS test_cache.cc
+  DEPS gtest phi common)
+
+cc_test(
+  strided_memcpy_test
+  SRCS strided_memcpy_test.cc
+  DEPS phi common)
+
+if(WIN32)
+  cc_test(
+    sequence_padding_test
+    SRCS sequence_padding_test.cc
+    DEPS type_info common)
+else()
+  cc_test(
+    sequence_padding_test
+    SRCS sequence_padding_test.cc
+    DEPS phi common)
+endif()
+
+cc_test(
+  sequence_pooling_test
+  SRCS sequence_pooling_test.cc
+  DEPS phi common)

test/cpp/phi/kernels/sequence_padding_test.cc

@@ -0,0 +1,133 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+#include <gtest/gtest.h>
+
+#include "paddle/phi/kernels/funcs/sequence_padding.h"
+
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/core/tensor_utils.h"
+
+template <typename DeviceContext, typename T>
+void TestSequencePadding(const DeviceContext &context,
+                         const phi::LegacyLoD &lod,
+                         const size_t sequence_width) {
+  phi::DenseTensor cpu_seq;
+  phi::DenseTensor cpu_seq_back;
+  phi::DenseTensor seq;
+  phi::DenseTensor seq_back;
+  phi::DenseTensor padding;
+  phi::DenseTensor cpu_pad_value;
+  phi::DenseTensor pad_value;
+
+  const size_t level = lod.size() - 1;
+  auto seq_dims = common::make_ddim({static_cast<int64_t>(lod[level].back()),
+                                     static_cast<int64_t>(sequence_width)});
+
+  cpu_seq.set_lod(lod);
+  auto *dev_ctx = static_cast<phi::CPUContext *>(
+      phi::DeviceContextPool::Instance().Get(phi::CPUPlace()));
+  cpu_seq.Resize(seq_dims);
+  dev_ctx->template Alloc<T>(&cpu_seq);
+
+  for (int64_t i = 0; i < cpu_seq.numel(); ++i) {
+    cpu_seq.data<T>()[i] = static_cast<T>(i);
+  }
+
+  auto place = context.GetPlace();
+  if (place.GetType() == phi::AllocationType::CPU) {
+    seq = cpu_seq;
+  } else {
+    phi::Copy(context, cpu_seq, place, true, &seq);
+    seq.set_lod(lod);
+  }
+
+  const size_t max_sequence_length =
+      phi::funcs::MaximumSequenceLength(lod[level]);
+  const size_t num_sequences = lod[level].size() - 1;
+  auto padding_dims =
+      common::make_ddim({static_cast<int64_t>(max_sequence_length),
+                         static_cast<int64_t>(num_sequences),
+                         static_cast<int64_t>(sequence_width)});
+
+  padding.Resize(padding_dims);
+  context.template Alloc<T>(&padding);
+
+  cpu_pad_value.Resize({1});
+  T *pad_value_data = dev_ctx->template Alloc<T>(&cpu_pad_value);
+  *pad_value_data = static_cast<T>(0);
+  if (place.GetType() == phi::AllocationType::CPU) {
+    pad_value = cpu_pad_value;
+  } else {
+    phi::Copy(context, cpu_pad_value, place, true, &pad_value);
+  }
+
+  phi::funcs::PaddingDenseTensorFunctor<DeviceContext, T>()(
+      context,
+      seq,
+      &padding,
+      pad_value,
+      -1,
+      0,
+      false,
+      phi::funcs::kLengthBatchWidth);
+
+  seq_back.set_lod(lod);
+  seq_back.Resize(seq_dims);
+  context.template Alloc<T>(&seq_back);
+  phi::funcs::UnpaddingDenseTensorFunctor<DeviceContext, T>()(
+      context, padding, &seq_back, -1, 0, false, phi::funcs::kLengthBatchWidth);
+
+  if (place.GetType() == phi::AllocationType::CPU) {
+    cpu_seq_back = seq_back;
+  } else {
+    phi::Copy(context, seq_back, phi::CPUPlace(), true, &cpu_seq_back);
+    cpu_seq_back.set_lod(lod);
+  }
+
+  EXPECT_EQ(cpu_seq.numel(), cpu_seq_back.numel());
+  EXPECT_EQ(cpu_seq.dims(), cpu_seq_back.dims());
+  for (int64_t i = 0; i < cpu_seq.numel(); ++i) {
+    EXPECT_EQ(cpu_seq.data<T>()[i], cpu_seq_back.data<T>()[i]);
+  }
+}
+
+TEST(Seq2BatchPadding, CPU) {
+  auto place = phi::CPUPlace();
+  auto *context = static_cast<phi::CPUContext *>(
+      phi::DeviceContextPool::Instance().Get(place));
+
+  phi::LegacyLoD lod1;
+  lod1.push_back(std::vector<size_t>{0, 10});
+  TestSequencePadding<phi::CPUContext, float>(*context, lod1, 16);
+
+  phi::LegacyLoD lod2;
+  lod2.push_back(std::vector<size_t>{0, 2, 7, 10});
+  TestSequencePadding<phi::CPUContext, float>(*context, lod2, 128);
+}
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+TEST(SequencePadding, CUDA) {
+  auto place = phi::GPUPlace(0);
+  auto *context = static_cast<phi::GPUContext *>(
+      phi::DeviceContextPool::Instance().Get(place));
+
+  phi::LegacyLoD lod1;
+  lod1.push_back(std::vector<size_t>{0, 10});
+  TestSequencePadding<phi::GPUContext, float>(*context, lod1, 16);
+
+  phi::LegacyLoD lod2;
+  lod2.push_back(std::vector<size_t>{0, 2, 7, 10});
+  TestSequencePadding<phi::GPUContext, float>(*context, lod2, 128);
+}
+#endif

test/cpp/phi/kernels/sequence_pooling_test.cc

@@ -0,0 +1,148 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/common/place.h"
+#include "paddle/phi/core/tensor_utils.h"
+#include "paddle/phi/kernels/funcs/sequence_pooling.h"
+
+template <typename DeviceContext, typename T>
+void TestSequencePoolingSum(const DeviceContext &context,
+                            const phi::LegacyLoD &lod,
+                            const int64_t second_dim) {
+  phi::DenseTensor cpu_out_grad;
+  phi::DenseTensor cpu_in_grad;
+  phi::DenseTensor out_grad;
+  phi::DenseTensor in_grad;
+
+  // construct out_grad's tensor in cpu
+  const size_t out_first_dim = lod[0].size() - 1;
+  auto out_dims =
+      common::make_ddim({static_cast<int64_t>(out_first_dim), second_dim});
+
+  cpu_out_grad.mutable_data<T>(out_dims, phi::CPUPlace());
+  for (int64_t i = 0; i < cpu_out_grad.numel(); ++i) {
+    cpu_out_grad.data<T>()[i] = static_cast<T>(i);
+  }
+
+  // copy to dst out_grad
+  auto place = context.GetPlace();
+  if (place == phi::CPUPlace()) {
+    out_grad = cpu_out_grad;
+  } else {
+    phi::Copy(context, cpu_out_grad, place, true, &out_grad);
+  }
+
+  // construct in_grad
+  in_grad.set_lod(lod);
+  auto in_dims =
+      common::make_ddim({static_cast<int64_t>(lod[0].back()), second_dim});
+  in_grad.mutable_data<T>(in_dims, place);
+
+  // check tensor construction result
+  PADDLE_ENFORCE_EQ(
+      in_grad.dims().size(),
+      out_grad.dims().size(),
+      common::errors::InvalidArgument(
+          "The dimension of input and output shall be same. Expected %ld == "
+          "%ld, but got %ld != %ld. Please check the input value.",
+          in_grad.dims().size(),
+          out_grad.dims().size(),
+          in_grad.dims().size(),
+          out_grad.dims().size()));
+  for (int64_t i = 1; i < out_grad.dims().size(); ++i) {
+    PADDLE_ENFORCE_EQ(
+        in_grad.dims()[i],
+        out_grad.dims()[i],
+        common::errors::InvalidArgument(
+            "The dimension of input and output shall be same. Expected %ld == "
+            "%ld, but got %ld != %ld. Please check the input value.",
+            in_grad.dims()[i],
+            out_grad.dims()[i],
+            in_grad.dims()[i],
+            out_grad.dims()[i]));
+  }
+
+  // call functor
+  phi::funcs::SequencePoolGradFunctor<DeviceContext, T>()(
+      context, "SUM", out_grad, &in_grad);
+
+  if (place == phi::CPUPlace()) {
+    cpu_in_grad = in_grad;
+  } else {
+    phi::Copy(context, in_grad, phi::CPUPlace(), true, &cpu_in_grad);
+    cpu_in_grad.set_lod(in_grad.lod());
+  }
+
+  EXPECT_EQ(in_grad.numel(), static_cast<int64_t>(lod[0].back() * second_dim));
+  EXPECT_EQ(in_grad.lod(), lod);
+
+  if (place == phi::CPUPlace()) {
+    for (size_t i = 0; i < in_grad.lod()[0].size() - 1; ++i) {
+      int64_t begin = static_cast<int64_t>(in_grad.lod()[0][i]);
+      int64_t end = static_cast<int64_t>(in_grad.lod()[0][i + 1]);
+      phi::DenseTensor tmp = in_grad.Slice(begin, end);
+      for (int64_t j = 0; j != tmp.numel() / second_dim; ++j) {
+        for (int64_t m = 0; m != second_dim; ++m) {
+          EXPECT_EQ(tmp.data<T>()[m + j * second_dim],
+                    out_grad.data<T>()[m + i * second_dim]);
+        }
+      }
+    }
+  } else {
+    for (size_t i = 0; i < cpu_in_grad.lod()[0].size() - 1; ++i) {
+      int64_t begin = static_cast<int64_t>(cpu_in_grad.lod()[0][i]);
+      int64_t end = static_cast<int64_t>(cpu_in_grad.lod()[0][i + 1]);
+      phi::DenseTensor tmp = cpu_in_grad.Slice(begin, end);
+      for (int64_t j = 0; j != tmp.numel() / second_dim; ++j) {
+        for (int64_t m = 0; m != second_dim; ++m) {
+          EXPECT_EQ(tmp.data<T>()[m + j * second_dim],
+                    cpu_out_grad.data<T>()[m + i * second_dim]);
+        }
+      }
+    }
+  }
+}
+
+TEST(SequencePoolingGrad, CPU_SUM) {
+  auto place = phi::CPUPlace();
+  auto *context = static_cast<phi::CPUContext *>(
+      phi::DeviceContextPool::Instance().Get(place));
+
+  phi::LegacyLoD lod1;
+  lod1.push_back(std::vector<size_t>{0, 10});
+  TestSequencePoolingSum<phi::CPUContext, float>(*context, lod1, 128);
+
+  phi::LegacyLoD lod2;
+  lod2.push_back(std::vector<size_t>{0, 2, 7, 10});
+  TestSequencePoolingSum<phi::CPUContext, float>(*context, lod2, 128);
+}
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+TEST(SequencePoolingGrad, CUDA_SUM) {
+  auto place = phi::GPUPlace(0);
+  auto *context = static_cast<phi::GPUContext *>(
+      phi::DeviceContextPool::Instance().Get(place));
+
+  phi::LegacyLoD lod1;
+  lod1.push_back(std::vector<size_t>{0, 10});
+  TestSequencePoolingSum<phi::GPUContext, float>(*context, lod1, 128);
+
+  phi::LegacyLoD lod2;
+  lod2.push_back(std::vector<size_t>{0, 2, 7, 10});
+  TestSequencePoolingSum<phi::GPUContext, float>(*context, lod2, 128);
+}
+#endif

test/cpp/phi/kernels/strided_memcpy_test.cc

@@ -0,0 +1,172 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/kernels/funcs/strided_memcpy.h"
+#include <array>
+
+#include "gtest/gtest.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/common/memory_utils.h"
+namespace phi {
+namespace tests {
+
+TEST(StridedMemcpy, CPUCrop) {
+  // clang-format off
+  int src[] = {// NOLINT
+      0, 1, 2, 0, 0,
+      0, 3, 4, 0, 0,
+      0, 0, 0, 0, 0,
+  };
+  // clang-format on
+
+  phi::DDim src_stride({5, 1});
+
+  std::array<int, 4> dst = {};
+  phi::DDim dst_dim({2, 2});
+  phi::DDim dst_stride({2, 1});
+
+  phi::CPUContext ctx;
+  phi::funcs::StridedMemcpy<int>(
+      ctx, src + 1, src_stride, dst_dim, dst_stride, dst.data());
+
+  ASSERT_EQ(1, dst[0]);
+  ASSERT_EQ(2, dst[1]);
+  ASSERT_EQ(3, dst[2]);
+  ASSERT_EQ(4, dst[3]);
+}
+
+TEST(StridedMemcpy, CPUConcat) {
+  // clang-format off
+  int src[] = { // NOLINT
+      1, 2,
+      3, 4
+  };
+  // clang-format on
+
+  std::array<int, 8> dst = {};
+  phi::DDim src_stride({2, 1});
+  phi::DDim dst_dim({2, 2});
+  phi::DDim dst_stride({4, 1});
+  phi::CPUContext ctx;
+
+  phi::funcs::StridedMemcpy<int>(
+      ctx, src, src_stride, dst_dim, dst_stride, dst.data());
+  phi::funcs::StridedMemcpy<int>(
+      ctx, src, src_stride, dst_dim, dst_stride, dst.data() + 2);
+
+  // clang-format off
+  int expect_dst[] = { // NOLINT
+      1, 2, 1, 2,
+      3, 4, 3, 4
+  };
+  // clang-format on
+  for (size_t i = 0; i < sizeof(expect_dst) / sizeof(int); ++i) {
+    ASSERT_EQ(expect_dst[i], dst[i]);
+  }
+}
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+TEST(StridedMemcpy, GPUCrop) {
+  // clang-format off
+  std::array<int, 15> src = {
+      0, 1, 2, 0, 0,
+      0, 3, 4, 0, 0,
+      0, 0, 0, 0, 0,
+  };
+  // clang-format on
+
+  phi::GPUPlace gpu0(0);
+  phi::CPUPlace cpu;
+
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* ctx = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  auto src_allocation = phi::memory_utils::Alloc(gpu0, sizeof(src));
+
+  int* gpu_src = reinterpret_cast<int*>(src_allocation->ptr());
+  memory_utils::Copy(
+      gpu0, gpu_src, cpu, src.data(), sizeof(src), ctx->stream());
+
+  phi::DDim src_stride({5, 1});
+
+  std::array<int, 4> dst = {};
+  auto dst_allocation = phi::memory_utils::Alloc(gpu0, sizeof(dst));
+  int* gpu_dst = reinterpret_cast<int*>(dst_allocation->ptr());
+
+  phi::DDim dst_dim({2, 2});
+  phi::DDim dst_stride({2, 1});
+
+  phi::funcs::StridedMemcpy<int>(
+      *ctx, gpu_src + 1, src_stride, dst_dim, dst_stride, gpu_dst);
+
+  memory_utils::Copy(
+      cpu, dst.data(), gpu0, gpu_dst, sizeof(dst), ctx->stream());
+  ctx->Wait();
+
+  ASSERT_EQ(1, dst[0]);
+  ASSERT_EQ(2, dst[1]);
+  ASSERT_EQ(3, dst[2]);
+  ASSERT_EQ(4, dst[3]);
+}
+
+TEST(StridedMemcpy, GPUConcat) {
+  // clang-format off
+  std::array<int, 4> src = {
+      1, 2,
+      3, 4
+  };
+  // clang-format on
+
+  phi::GPUPlace gpu0(0);
+  phi::CPUPlace cpu;
+
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* ctx = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  auto gpu_src_allocation = phi::memory_utils::Alloc(gpu0, sizeof(src));
+  int* gpu_src = reinterpret_cast<int*>(gpu_src_allocation->ptr());
+  memory_utils::Copy(
+      gpu0, gpu_src, cpu, src.data(), sizeof(src), ctx->stream());
+
+  std::array<int, 8> dst = {};
+  auto gpu_dst_allocation = phi::memory_utils::Alloc(gpu0, sizeof(dst));
+  int* gpu_dst = reinterpret_cast<int*>(gpu_dst_allocation->ptr());
+
+  phi::DDim src_stride({2, 1});
+  phi::DDim dst_dim({2, 2});
+  phi::DDim dst_stride({4, 1});
+
+  phi::funcs::StridedMemcpy<int>(
+      *ctx, gpu_src, src_stride, dst_dim, dst_stride, gpu_dst);
+  phi::funcs::StridedMemcpy<int>(
+      *ctx, gpu_src, src_stride, dst_dim, dst_stride, gpu_dst + 2);
+
+  memory_utils::Copy(
+      cpu, dst.data(), gpu0, gpu_dst, sizeof(dst), ctx->stream());
+  ctx->Wait();
+
+  // clang-format off
+  std::array<int, 8> expect_dst = {
+      1, 2, 1, 2,
+      3, 4, 3, 4
+  };
+  // clang-format on
+  for (size_t i = 0; i < sizeof(expect_dst) / sizeof(int); ++i) {
+    ASSERT_EQ(expect_dst[i], dst[i]);
+  }
+}
+
+#endif
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/kernels/test_auto_tune.cu

@@ -0,0 +1,137 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include "glog/logging.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/backends/gpu/gpu_launch_config.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/tensor_meta.h"
+#include "paddle/phi/core/tensor_utils.h"
+#include "paddle/phi/kernels/autotune/auto_tune_base.h"
+#include "paddle/phi/kernels/funcs/aligned_vector.h"
+
+namespace tune = phi::autotune;
+
+template <typename T, int VecSize>
+__global__ void VecSumTest(const T* x, T* y, int N) {
+#ifdef __HIPCC__
+  int idx = hipBlockDim_x * hipBlockIdx_x + hipThreadIdx_x;
+#else
+  int idx = blockDim.x * blockIdx.x + threadIdx.x;
+#endif
+  using LoadT = phi::AlignedVector<T, VecSize>;
+  for (int i = idx * VecSize; i < N; i += blockDim.x * gridDim.x * VecSize) {
+    LoadT x_vec;
+    LoadT y_vec;
+    phi::Load<T, VecSize>(&x[i], &x_vec);
+    phi::Load<T, VecSize>(&y[i], &y_vec);
+#pragma unroll
+    for (int j = 0; j < VecSize; j++) {
+      y_vec[j] = x_vec[j] + y_vec[j];
+    }
+    phi::Store<T, VecSize>(y_vec, &y[i]);
+  }
+}
+
+template <int Vecsize>
+float Algo(const phi::GPUContext& ctx,
+           const phi::DenseTensor& d_in,
+           phi::DenseTensor* d_out,
+           size_t N,
+           size_t threads,
+           size_t blocks) {
+  const float* d_in_data = d_in.data<float>();
+  float* d_out_data = d_out->data<float>();
+#ifdef __HIPCC__
+  hipLaunchKernelGGL(HIP_KERNEL_NAME(VecSumTest<float, Vecsize>),
+                     dim3(blocks),
+                     dim3(threads),
+                     0,
+                     0,
+                     d_in_data,
+                     d_out_data,
+                     N);
+#else
+  VLOG(3) << "Vecsize is " << Vecsize;
+  VecSumTest<float, Vecsize>
+      <<<blocks, threads, 0, ctx.stream()>>>(d_in_data, d_out_data, N);
+#endif
+  return Vecsize;
+}
+
+TEST(AutoTune, sum) {
+  int64_t N = 1 << 20;
+  size_t blocks = 512;
+  size_t threads = 256;
+  size_t size = sizeof(float) * N;
+
+  const auto alloc_cpu =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace());
+  auto in1 = std::make_shared<phi::DenseTensor>(
+      alloc_cpu.get(),
+      phi::DenseTensorMeta(phi::DataType::FLOAT32,
+                           common::make_ddim({N}),
+                           phi::DataLayout::NCHW));
+  auto in2 = std::make_shared<phi::DenseTensor>(
+      alloc_cpu.get(),
+      phi::DenseTensorMeta(phi::DataType::FLOAT32,
+                           common::make_ddim({N}),
+                           phi::DataLayout::NCHW));
+
+  float* in1_data = in1->data<float>();
+  float* in2_data = in2->data<float>();
+  for (size_t i = 0; i < N; i++) {
+    in1_data[i] = 1.0f;
+    in2_data[i] = 2.0f;
+  }
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  const auto alloc_cuda =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::GPUPlace());
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto place = phi::GPUPlace();
+  auto* dev_ctx = static_cast<const phi::GPUContext*>(pool.GetByPlace(place));
+  auto stream = dev_ctx->stream();
+
+  auto d_in1 = std::make_shared<phi::DenseTensor>(
+      alloc_cuda.get(),
+      phi::DenseTensorMeta(phi::DataType::FLOAT32,
+                           common::make_ddim({N}),
+                           phi::DataLayout::NCHW));
+  auto d_in2 = std::make_shared<phi::DenseTensor>(
+      alloc_cuda.get(),
+      phi::DenseTensorMeta(phi::DataType::FLOAT32,
+                           common::make_ddim({N}),
+                           phi::DataLayout::NCHW));
+  phi::Copy(*dev_ctx, *in1.get(), phi::GPUPlace(), false, d_in1.get());
+  phi::Copy(*dev_ctx, *in2.get(), phi::GPUPlace(), false, d_in2.get());
+
+  // 1. Test call_back.
+  VLOG(3) << ">>> [CallBack]: Test case.";
+  auto callback1 = tune::MakeCallback<float>(Algo<4>);
+  auto callback2 = tune::MakeCallback<float>(Algo<2>);
+  auto callback3 = tune::MakeCallback<float>(Algo<1>);
+  std::vector<decltype(callback1)> callbacks{callback1, callback2, callback3};
+  for (int i = 0; i < callbacks.size(); ++i) {
+    dev_ctx->Wait();
+    phi::GpuTimer timer;
+    timer.Start(0);
+    callbacks[i].Run(*dev_ctx, *d_in1.get(), d_in2.get(), N, threads, blocks);
+    timer.Stop(0);
+    VLOG(3) << "kernel[" << i << "]: time cost is " << timer.ElapsedTime();
+  }
+#endif
+}

test/cpp/phi/kernels/test_cache.cc

@@ -0,0 +1,68 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include <cmath>
+#include <functional>
+
+#include "paddle/phi/kernels/autotune/cache.h"
+
+#include "glog/logging.h"
+
+enum ConvAlgos { GEMMKernel = 0, CuDNNKernel_1 = 1, CuDNNKernel_2 = 2 };
+
+TEST(AlgosCache, AlgosCache) {
+  auto autotune_cache = phi::autotune::AutoTuneCache::Instance();
+  auto& cache =
+      autotune_cache.GetConv(phi::autotune::AlgorithmType::kConvForward);
+
+  std::vector<int64_t> x_shape = {4, 224, 224, 3};
+  std::vector<int64_t> w_shape = {32, 3, 3, 3};
+  std::vector<int> paddings = {0, 0};
+  std::vector<int> strides = {2, 2};
+  std::vector<int> dilations = {1, 1};
+  phi::DataType dtype = phi::CppTypeToDataType<float>::Type();
+
+  phi::autotune::ConvCacheKey key(
+      x_shape, w_shape, paddings, strides, dilations, dtype, 0, 0);
+  EXPECT_EQ(cache.Find(key), false);
+  phi::autotune::ConvAutoTuneResult node(
+      static_cast<int64_t>(ConvAlgos::GEMMKernel), 0, false);
+  cache.Set(key, node);
+  EXPECT_EQ(cache.Size(), 1);
+  EXPECT_EQ(cache.Find(key), true);
+  auto algo = cache.Get(key);
+  EXPECT_EQ(algo.algo, ConvAlgos::GEMMKernel);
+
+  x_shape = {4, 128, 128, 3};
+  phi::autotune::ConvCacheKey key1(
+      x_shape, w_shape, paddings, strides, dilations, dtype, 0, 1);
+  EXPECT_EQ(cache.Find(key1), false);
+  phi::autotune::ConvAutoTuneResult node1(
+      static_cast<int64_t>(ConvAlgos::CuDNNKernel_1), 0, false);
+  cache.Set(key1, node1);
+  EXPECT_EQ(cache.Size(), 2);
+  EXPECT_EQ(cache.CacheHits(), 1);
+  EXPECT_EQ(cache.CacheMisses(), 2);
+
+  float cache_hit_rate = static_cast<float>(1) / static_cast<float>(3);
+  EXPECT_LT(std::abs(cache_hit_rate - cache.CacheHitRate()), 1e-5);
+
+  autotune_cache.UpdateStatus();
+  EXPECT_EQ(autotune_cache.Size(), 2);
+  EXPECT_EQ(autotune_cache.CacheHits(), 1);
+  EXPECT_EQ(autotune_cache.CacheMisses(), 2);
+  EXPECT_LT(std::abs(cache_hit_rate - autotune_cache.CacheHitRate()), 1e-5);
+}

test/cpp/phi/kernels/test_cpu_vec.cc

@@ -0,0 +1,346 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <cmath>
+#include <cstring>
+#include <random>
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/phi/common/port.h"
+#include "paddle/phi/kernels/funcs/cpu_vec.h"
+
+namespace phi {
+namespace tests {
+
+inline double GetCurrentUS() {
+  struct timeval time = {};
+  gettimeofday(&time, nullptr);
+  return 1e+6 * time.tv_sec + time.tv_usec;  // NOLINT
+}
+constexpr int repeat = 1000;
+
+template <typename T>
+inline T _sigmoid(T x) {
+  const T min = SIGMOID_THRESHOLD_MIN;
+  const T max = SIGMOID_THRESHOLD_MAX;
+  T tmp = (x < min) ? min : ((x > max) ? max : x);
+  return static_cast<T>(1) / (static_cast<T>(1) + std::exp(-tmp));
+}
+
+template <typename T>
+inline T _tanh(T x) {
+  return static_cast<T>(2) * _sigmoid<T>(static_cast<T>(2) * x) -
+         static_cast<T>(1);
+}
+
+template <typename T>
+void ref_sigmoid(const int n, const T* x, T* y) {
+  for (int i = 0; i < n; ++i) {
+    y[i] = _sigmoid(x[i]);
+  }
+}
+
+template <typename T>
+void ref_tanh(const int n, const T* x, T* y) {
+  for (int i = 0; i < n; ++i) {
+    y[i] = _tanh(x[i]);
+  }
+}
+template <typename T>
+void ref_relu(const int n, const T* x, T* y) {
+  for (int i = 0; i < n; ++i) {
+    y[i] = x[i] > 0 ? x[i] : 0;
+  }
+}
+
+template <typename T>
+void RandomVec(const int n,
+               T* a,
+               const T lower = static_cast<T>(-20.f),
+               const T upper = static_cast<T>(20.f)) {
+  static unsigned int seed = 100;
+  std::mt19937 rng(seed++);
+  std::uniform_real_distribution<double> uniform_dist(0, 1);
+  for (int i = 0; i < n; ++i) {
+    a[i] = static_cast<T>(uniform_dist(rng) * (upper - lower) + lower);
+  }
+}
+
+template <typename T>
+void TestAndBench(const int n,
+                  std::function<void(const int, const T*, T*)> tgt,
+                  std::function<void(const int, const T*, T*)> ref) {
+  std::vector<T> x(n);
+  std::vector<T> ytgt(n), yref(n);
+  RandomVec<T>(n, x.data());
+
+  const T* x_data = x.data();
+  T* ytgt_data = ytgt.data();
+  T* yref_data = yref.data();
+  auto st = GetCurrentUS();
+  for (int i = 0; i < repeat; ++i) {
+    tgt(n, x_data, ytgt_data);
+  }
+  auto mt = GetCurrentUS();
+  for (int i = 0; i < repeat; ++i) {
+    ref(n, x_data, yref_data);
+  }
+  auto et = GetCurrentUS();
+
+  VLOG(3) << "Vec size " << n << ": refer takes: " << (et - mt) / repeat
+          << " us, tgt takes: " << (mt - st) / repeat;
+  for (int i = 0; i < n; ++i) {
+    EXPECT_NEAR(ytgt_data[i], yref_data[i], 1e-3);
+  }
+}
+
+TEST(CpuVecTest, sigmoid) {
+  using namespace phi::funcs;  // NOLINT
+  for (auto sz : {1, 2, 15, 16, 30, 32, 128, 200, 512}) {
+    TestAndBench<float>(sz, vec_sigmoid<float>, ref_sigmoid<float>);
+    TestAndBench<float>(
+        sz, vec_sigmoid<float, backends::cpu::avx>, ref_sigmoid<float>);
+    TestAndBench<float>(
+        sz, vec_sigmoid<float, backends::cpu::avx2>, ref_sigmoid<float>);
+    TestAndBench<float>(
+        sz, vec_sigmoid<float, backends::cpu::avx512f>, ref_sigmoid<float>);
+  }
+  TestAndBench<double>(30, vec_sigmoid<double>, ref_sigmoid<double>);
+}
+
+TEST(CpuVecTest, tanh) {
+  using namespace phi::funcs;  // NOLINT
+  for (auto sz : {1, 2, 15, 16, 30, 32, 128, 200, 512}) {
+    TestAndBench<float>(sz, vec_tanh<float>, ref_tanh<float>);
+    TestAndBench<float>(
+        sz, vec_tanh<float, backends::cpu::avx>, ref_tanh<float>);
+    TestAndBench<float>(
+        sz, vec_tanh<float, backends::cpu::avx2>, ref_tanh<float>);
+    TestAndBench<float>(
+        sz, vec_tanh<float, backends::cpu::avx512f>, ref_tanh<float>);
+  }
+  TestAndBench<double>(30, vec_tanh<double>, ref_tanh<double>);
+}
+
+TEST(CpuVecTest, relu) {
+  using namespace phi::funcs;  // NOLINT
+  for (auto sz : {1, 2, 15, 16, 30, 32, 128, 200, 512}) {
+    TestAndBench<float>(sz, vec_relu<float>, ref_relu<float>);
+    TestAndBench<float>(
+        sz, vec_relu<float, backends::cpu::avx>, ref_relu<float>);
+    TestAndBench<float>(
+        sz, vec_relu<float, backends::cpu::avx2>, ref_relu<float>);
+    TestAndBench<float>(
+        sz, vec_relu<float, backends::cpu::avx512f>, ref_relu<float>);
+  }
+  TestAndBench<double>(30, vec_relu<double>, ref_relu<double>);
+}
+
+template <typename T>
+void compare_sum(size_t n,
+                 std::function<void(const size_t, const T*, T*)> tgt,
+                 std::function<void(const size_t, const T*, T*)> ref) {
+  std::vector<T> x(n);
+  T ytgt_data, yref_data;
+  RandomVec<T>(n, x.data(), static_cast<T>(-2), static_cast<T>(2));
+
+  const T* x_data = x.data();
+  tgt(n, x_data, &ytgt_data);
+  ref(n, x_data, &yref_data);
+  EXPECT_NEAR(ytgt_data, yref_data, 1e-3);
+}
+
+TEST(CpuVecTest, vec_sum) {
+  using namespace phi::funcs;  // NOLINT
+  for (size_t sz : {1, 2, 15, 16, 30, 32, 128, 200, 512}) {
+    compare_sum<float>(
+        sz, vec_sum<float>, vec_sum<float, backends::cpu::isa_any>);
+    compare_sum<float>(sz,
+                       vec_sum<float, backends::cpu::avx>,
+                       vec_sum<float, backends::cpu::isa_any>);
+  }
+  compare_sum<double>(
+      30U, vec_sum<double>, vec_sum<double, backends::cpu::isa_any>);
+}
+
+template <typename T>
+void compare_clip(
+    size_t n,
+    T threshold,
+    std::function<void(const size_t, const T, const T*, T*)> tgt,
+    std::function<void(const size_t, const T, const T*, T*)> ref) {
+  std::vector<T> x(n);
+  std::vector<T> ytgt(n), yref(n);
+  RandomVec<T>(n, x.data(), static_cast<T>(-2), static_cast<T>(2));
+
+  const T* x_data = x.data();
+  T* yref_data = yref.data();
+  T* ytgt_data = ytgt.data();
+  tgt(n, threshold, x_data, ytgt_data);
+  ref(n, threshold, x_data, yref_data);
+  for (size_t i = 0; i < n; ++i) {
+    EXPECT_NEAR(ytgt_data[i], yref_data[i], 1e-3);
+  }
+}
+
+TEST(CpuVecTest, vec_clip) {
+  using namespace phi::funcs;  // NOLINT
+  for (size_t sz : {1, 2, 15, 16, 30, 32, 128, 200, 512}) {
+    compare_clip<float>(
+        sz, -4.f, vec_clip<float>, vec_clip<float, backends::cpu::isa_any>);
+    compare_clip<float>(sz,
+                        -1.1f,
+                        vec_clip<float, backends::cpu::avx>,
+                        vec_clip<float, backends::cpu::isa_any>);
+  }
+  compare_clip<double>(
+      30U, 1.0, vec_clip<double>, vec_clip<double, backends::cpu::isa_any>);
+}
+
+template <typename T>
+void compare_mul(
+    size_t n,
+    std::function<void(const size_t, const T*, const T*, T*)> tgt,
+    std::function<void(const size_t, const T*, const T*, T*)> ref) {
+  std::vector<T> x(n), y(n);
+  std::vector<T> ztgt(n), zref(n);
+
+  RandomVec<T>(n, x.data(), static_cast<T>(-2), static_cast<T>(2));
+  RandomVec<T>(n, y.data(), static_cast<T>(-2), static_cast<T>(2));
+
+  const T* x_data = x.data();
+  const T* y_data = y.data();
+  T* ztgt_data = ztgt.data();
+  T* zref_data = zref.data();
+
+  tgt(n, x_data, y_data, ztgt_data);
+  ref(n, x_data, y_data, zref_data);
+  for (size_t i = 0; i < n; ++i) {
+    EXPECT_NEAR(ztgt_data[i], zref_data[i], 1e-3);
+  }
+}
+
+TEST(CpuVecTest, vec_mul) {
+  using namespace phi::funcs;  // NOLINT
+  for (size_t sz : {1, 2, 15, 16, 30, 32, 128, 200, 512}) {
+    compare_mul<float>(
+        sz, vec_mul<float>, vec_mul<float, backends::cpu::isa_any>);
+    compare_mul<float>(sz,
+                       vec_mul<float, backends::cpu::avx>,
+                       vec_mul<float, backends::cpu::isa_any>);
+  }
+  compare_mul<double>(
+      30U, vec_mul<double>, vec_mul<double, backends::cpu::isa_any>);
+}
+
+template <typename T>
+void compare_mul_reduce(
+    size_t n,
+    std::function<void(const size_t, const T*, const T*, T*)> tgt,
+    std::function<void(const size_t, const T*, const T*, T*)> ref) {
+  std::vector<T> x(n), y(n);
+  T ztgt_data, zref_data;
+
+  RandomVec<T>(n, x.data(), static_cast<T>(-2), static_cast<T>(2));
+  RandomVec<T>(n, y.data(), static_cast<T>(-2), static_cast<T>(2));
+
+  const T* x_data = x.data();
+  const T* y_data = y.data();
+
+  tgt(n, x_data, y_data, &ztgt_data);
+  ref(n, x_data, y_data, &zref_data);
+  EXPECT_NEAR(ztgt_data, zref_data, 1e-3);
+}
+
+TEST(CpuVecTest, vec_mul_reduce) {
+  using namespace phi::funcs;  // NOLINT
+  for (size_t sz : {1, 2, 15, 16, 30, 32, 128, 200, 512}) {
+    compare_mul_reduce<float>(sz,
+                              vec_mul_reduce<float>,
+                              vec_mul_reduce<float, backends::cpu::isa_any>);
+    compare_mul_reduce<float>(sz,
+                              vec_mul_reduce<float, backends::cpu::avx>,
+                              vec_mul_reduce<float, backends::cpu::isa_any>);
+  }
+  compare_mul_reduce<double>(30U,
+                             vec_mul_reduce<double>,
+                             vec_mul_reduce<double, backends::cpu::isa_any>);
+}
+
+template <typename T>
+void TestInplace(const int n,
+                 std::function<void(const int, const T*, T*)> tgt,
+                 std::function<void(const int, const T*, T*)> ref) {
+  std::vector<T> x(n);
+  std::vector<T> ytgt(n), yref(n);
+  RandomVec<T>(n, x.data());
+
+  const T* x_data = x.data();
+  T* yref_data = yref.data();
+  T* ytgt_data = ytgt.data();
+  std::memcpy(yref_data, x_data, sizeof(T) * n);
+  std::memcpy(ytgt_data, x_data, sizeof(T) * n);
+
+  ref(n, yref_data, yref_data);
+  tgt(n, ytgt_data, ytgt_data);
+
+  for (int i = 0; i < n; ++i) {
+    EXPECT_NEAR(ytgt_data[i], yref_data[i], 1e-3);
+  }
+}
+
+TEST(CpuVecTest, inplace_sigmoid) {
+  using namespace phi::funcs;  // NOLINT
+  for (auto sz : {1, 2, 15, 16, 30, 32, 128, 200, 512}) {
+    TestInplace<float>(sz, vec_sigmoid<float>, ref_sigmoid<float>);
+    TestInplace<float>(
+        sz, vec_sigmoid<float, backends::cpu::avx>, ref_sigmoid<float>);
+    TestInplace<float>(
+        sz, vec_sigmoid<float, backends::cpu::avx2>, ref_sigmoid<float>);
+    TestInplace<float>(
+        sz, vec_sigmoid<float, backends::cpu::avx512f>, ref_sigmoid<float>);
+  }
+  TestInplace<double>(30, vec_sigmoid<double>, ref_sigmoid<double>);
+}
+
+TEST(CpuVecTest, inplace_tanh) {
+  using namespace phi::funcs;  // NOLINT
+  for (auto sz : {1, 2, 15, 16, 30, 32, 128, 200, 512}) {
+    TestInplace<float>(sz, vec_tanh<float>, ref_tanh<float>);
+    TestInplace<float>(
+        sz, vec_tanh<float, backends::cpu::avx>, ref_tanh<float>);
+    TestInplace<float>(
+        sz, vec_tanh<float, backends::cpu::avx2>, ref_tanh<float>);
+    TestInplace<float>(
+        sz, vec_tanh<float, backends::cpu::avx512f>, ref_tanh<float>);
+  }
+  TestInplace<double>(30, vec_tanh<double>, ref_tanh<double>);
+}
+
+TEST(CpuVecTest, inplace_relu) {
+  using namespace phi::funcs;  // NOLINT
+  for (auto sz : {1, 2, 15, 16, 30, 32, 128, 200, 512}) {
+    TestInplace<float>(sz, vec_relu<float>, ref_relu<float>);
+    TestInplace<float>(
+        sz, vec_relu<float, backends::cpu::avx>, ref_relu<float>);
+    TestInplace<float>(
+        sz, vec_relu<float, backends::cpu::avx2>, ref_relu<float>);
+    TestInplace<float>(
+        sz, vec_relu<float, backends::cpu::avx512f>, ref_relu<float>);
+  }
+  TestInplace<double>(30, vec_relu<double>, ref_relu<double>);
+}
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/kernels/test_fused_adam_kernel.cc

@@ -0,0 +1,524 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <vector>
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/generator.h"
+
+#ifdef PADDLE_WITH_CUDA
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#endif
+
+#include "gtest/gtest.h"
+
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/common/amp_type_traits.h"
+#include "paddle/phi/core/tensor_utils.h"
+#include "paddle/phi/infermeta/multiary.h"
+#include "paddle/phi/kernels/abs_kernel.h"
+#include "paddle/phi/kernels/adam_kernel.h"
+#include "paddle/phi/kernels/adamw_kernel.h"
+#include "paddle/phi/kernels/cast_kernel.h"
+#include "paddle/phi/kernels/elementwise_subtract_kernel.h"
+#include "paddle/phi/kernels/full_kernel.h"
+#include "paddle/phi/kernels/fused_adam_kernel.h"
+#include "paddle/phi/kernels/gaussian_kernel.h"
+#include "paddle/phi/kernels/legacy/reduce_max_kernel.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+auto GenerateRandomTensorVectors(
+    const Context &ctx, const std::vector<std::vector<int64_t>> &shapes) {
+  size_t n = shapes.size();
+  std::vector<DenseTensor> tensors(n);
+  for (size_t i = 0; i < n; ++i) {
+    GaussianKernel<T, Context>(ctx,
+                               shapes[i],
+                               0.0f,
+                               1.0f,
+                               0,
+                               phi::CppTypeToDataType<T>::Type(),
+                               &tensors[i]);
+  }
+  return tensors;
+}
+
+template <typename T, typename Context>
+auto GenerateConstantTensorVectors(
+    const Context &ctx,
+    const std::vector<std::vector<int64_t>> &shapes,
+    T value) {
+  size_t n = shapes.size();
+  std::vector<DenseTensor> tensors(n);
+  for (size_t i = 0; i < n; ++i) {
+    FullKernel<T, Context>(
+        ctx, shapes[i], value, phi::CppTypeToDataType<T>::Type(), &tensors[i]);
+  }
+  return tensors;
+}
+
+static auto ToConstTensorPtrVector(const std::vector<DenseTensor> &tensors) {
+  std::vector<const DenseTensor *> results;
+  results.reserve(tensors.size());
+  for (const auto &t : tensors) {
+    results.push_back(&t);
+  }
+  return results;
+}
+
+static auto ToMutableTensorPtrVector(
+    std::vector<DenseTensor> &tensors) {  // NOLINT
+  std::vector<DenseTensor *> results;
+  results.reserve(tensors.size());
+  for (auto &t : tensors) {
+    results.push_back(&t);
+  }
+  return results;
+}
+
+static auto ToMetaTensorVector(const std::vector<DenseTensor> &tensors) {
+  std::vector<MetaTensor> results;
+  results.reserve(tensors.size());
+  for (auto &t : tensors) {
+    results.emplace_back(t);
+  }
+  return results;
+}
+
+static auto ToConstMetaTensorPtrVector(
+    const std::vector<MetaTensor> &meta_tensors) {
+  std::vector<const MetaTensor *> results;
+  results.reserve(meta_tensors.size());
+  for (auto &t : meta_tensors) {
+    results.push_back(&t);
+  }
+  return results;
+}
+
+static auto ToMutableMetaTensorPtrVector(
+    std::vector<MetaTensor> &meta_tensors) {  // NOLINT
+  std::vector<MetaTensor *> results;
+  results.reserve(meta_tensors.size());
+  for (auto &t : meta_tensors) {
+    results.push_back(&t);
+  }
+  return results;
+}
+
+template <typename T, typename Context>
+struct AdamInfo {
+  using AdamWScalarT = double;
+
+  const Context *ctx;
+  std::vector<std::vector<int64_t>> shapes;
+
+  std::vector<DenseTensor> params;
+  std::vector<DenseTensor> master_params;
+  std::vector<DenseTensor> moment1s;
+  std::vector<DenseTensor> moment2s;
+  std::vector<DenseTensor> moment2s_max;
+  std::vector<DenseTensor> beta1_pows;
+  std::vector<DenseTensor> beta2_pows;
+  DenseTensor learning_rate;
+  DenseTensor adamw_learning_rate;
+  float beta1;
+
+  float beta2;
+  float weight_decay;
+  float epsilon = 1e-6;
+  bool multi_precision;
+  bool use_adamw;
+  int chunk_size = 4096;
+  bool amsgrad;
+
+  using MT = typename phi::dtype::MPTypeTrait<T>::Type;
+
+  AdamInfo(const Context &ctx_ref,
+           const std::vector<std::vector<int64_t>> &shapes,
+           float beta1,
+           float beta2,
+           float weight_decay,
+           bool multi_precision,
+           bool use_adamw,
+           bool amsgrad)
+      : ctx(&ctx_ref),
+        shapes(shapes),
+        beta1(beta1),
+        beta2(beta2),
+        weight_decay(weight_decay),
+        multi_precision(multi_precision),
+        use_adamw(use_adamw),
+        amsgrad(amsgrad) {
+    std::vector<std::vector<int64_t>> one_shapes(shapes.size(),
+                                                 std::vector<int64_t>(1, 1));
+    std::vector<std::vector<int64_t>> learning_rate_shapes(
+        one_shapes.begin(), one_shapes.begin() + 1);
+
+    params = GenerateRandomTensorVectors<T, Context>(*ctx, shapes);
+    learning_rate = GenerateConstantTensorVectors<double, Context>(
+        *ctx, learning_rate_shapes, 1e-3)[0];
+    adamw_learning_rate = GenerateConstantTensorVectors<AdamWScalarT, Context>(
+        *ctx, learning_rate_shapes, 1e-3)[0];
+    moment1s = GenerateConstantTensorVectors<MT, Context>(*ctx, shapes, 0);
+    moment2s = GenerateConstantTensorVectors<MT, Context>(*ctx, shapes, 0);
+    moment2s_max = GenerateConstantTensorVectors<MT, Context>(*ctx, shapes, 0);
+
+    if (multi_precision) {
+      master_params.resize(shapes.size());
+      for (size_t i = 0; i < shapes.size(); ++i) {
+        master_params[i] = Cast<T, Context>(
+            *ctx, params[i], phi::CppTypeToDataType<MT>::Type());
+      }
+    }
+
+    beta1_pows =
+        GenerateConstantTensorVectors<MT, Context>(*ctx, one_shapes, beta1);
+    beta2_pows =
+        GenerateConstantTensorVectors<MT, Context>(*ctx, one_shapes, beta2);
+  }
+
+  void Update(bool use_fused, const std::vector<DenseTensor> &grads) {
+    if (use_fused) {
+      UpdateWithFusedAdam(grads);
+    } else {
+      for (size_t j = 0; j < params.size(); ++j) {
+        if (use_adamw) {
+          UpdateWithAdamWBaseline(grads, j);
+        } else {
+          UpdateWithAdamBaseline(grads, j);
+        }
+      }
+    }
+  }
+
+  static AdamInfo<T, Context> DeepCopy(const AdamInfo &other) {
+    AdamInfo copied(*other.ctx,
+                    other.shapes,
+                    other.beta1,
+                    other.beta2,
+                    other.weight_decay,
+                    other.multi_precision,
+                    other.use_adamw,
+                    other.amsgrad);
+    auto copy_tensor = [&other](const DenseTensor &x, DenseTensor *y) {
+      Copy<Context>(*other.ctx, x, x.place(), false, y);
+    };
+
+    auto copy_tensors = [&other](const std::vector<DenseTensor> &xs,
+                                 std::vector<DenseTensor> *ys) {
+      for (size_t i = 0; i < xs.size(); ++i) {
+        Copy<Context>(*other.ctx, xs[i], xs[i].place(), false, &((*ys)[i]));
+      }
+    };
+
+    copy_tensors(other.params, &copied.params);
+    copy_tensors(other.master_params, &copied.master_params);
+    copy_tensors(other.moment1s, &copied.moment1s);
+    copy_tensors(other.moment2s, &copied.moment2s);
+    copy_tensors(other.moment2s_max, &copied.moment2s_max);
+    copy_tensors(other.beta1_pows, &copied.beta1_pows);
+    copy_tensors(other.beta2_pows, &copied.beta2_pows);
+    copy_tensor(other.learning_rate, &copied.learning_rate);
+    copy_tensor(other.adamw_learning_rate, &copied.adamw_learning_rate);
+    copied.epsilon = other.epsilon;
+    copied.chunk_size = other.chunk_size;
+    other.ctx->Wait();
+    return copied;
+  }
+
+ private:
+  void UpdateWithFusedAdam(const std::vector<DenseTensor> &grads) {
+    auto param_metas = ToMetaTensorVector(params);
+    auto grad_metas = ToMetaTensorVector(grads);
+    auto master_param_metas = ToMetaTensorVector(master_params);
+    auto moment1_metas = ToMetaTensorVector(moment1s);
+    auto moment2_metas = ToMetaTensorVector(moment2s);
+    auto moment2_max_metas = ToMetaTensorVector(moment2s_max);
+    auto beta1_pow_metas = ToMetaTensorVector(beta1_pows);
+    auto beta2_pow_metas = ToMetaTensorVector(beta2_pows);
+
+    FusedAdamInferMeta(ToConstMetaTensorPtrVector(param_metas),
+                       ToConstMetaTensorPtrVector(grad_metas),
+                       adamw_learning_rate,
+                       ToConstMetaTensorPtrVector(moment1_metas),
+                       ToConstMetaTensorPtrVector(moment2_metas),
+                       ToConstMetaTensorPtrVector(moment2_max_metas),
+                       ToConstMetaTensorPtrVector(beta1_pow_metas),
+                       ToConstMetaTensorPtrVector(beta2_pow_metas),
+                       multi_precision
+                           ? paddle::make_optional(
+                                 ToConstMetaTensorPtrVector(master_param_metas))
+                           : paddle::none,
+                       MetaTensor(),
+                       beta1,
+                       beta2,
+                       epsilon,
+                       chunk_size,
+                       weight_decay,
+                       use_adamw,
+                       multi_precision,
+                       false,
+                       amsgrad,
+                       ToMutableMetaTensorPtrVector(param_metas),
+                       ToMutableMetaTensorPtrVector(moment1_metas),
+                       ToMutableMetaTensorPtrVector(moment2_metas),
+                       ToMutableMetaTensorPtrVector(moment2_max_metas),
+                       ToMutableMetaTensorPtrVector(beta1_pow_metas),
+                       ToMutableMetaTensorPtrVector(beta2_pow_metas),
+                       ToMutableMetaTensorPtrVector(master_param_metas));
+
+    FusedAdamKernel<T, Context>(
+        *ctx,
+        ToConstTensorPtrVector(params),
+        ToConstTensorPtrVector(grads),
+        adamw_learning_rate,
+        ToConstTensorPtrVector(moment1s),
+        ToConstTensorPtrVector(moment2s),
+        ToConstTensorPtrVector(moment2s_max),
+        ToConstTensorPtrVector(beta1_pows),
+        ToConstTensorPtrVector(beta2_pows),
+        multi_precision
+            ? paddle::make_optional(ToConstTensorPtrVector(master_params))
+            : paddle::none,
+        paddle::none,
+        beta1,
+        beta2,
+        epsilon,
+        chunk_size,
+        weight_decay,
+        use_adamw,
+        multi_precision,
+        false,
+        amsgrad,
+        ToMutableTensorPtrVector(params),
+        ToMutableTensorPtrVector(moment1s),
+        ToMutableTensorPtrVector(moment2s),
+        ToMutableTensorPtrVector(moment2s_max),
+        ToMutableTensorPtrVector(beta1_pows),
+        ToMutableTensorPtrVector(beta2_pows),
+        ToMutableTensorPtrVector(master_params));
+  }
+
+  void UpdateWithAdamWBaseline(const std::vector<DenseTensor> &grads,
+                               size_t idx) {
+    AdamwDenseKernel<T, Context>(
+        *ctx,
+        params[idx],
+        grads[idx],
+        adamw_learning_rate,
+        moment1s[idx],
+        moment2s[idx],
+        moment2s_max[idx],
+        beta1_pows[idx],
+        beta2_pows[idx],
+        multi_precision ? paddle::make_optional(master_params[idx])
+                        : paddle::none,
+        paddle::none,
+        beta1,
+        beta2,
+        epsilon,
+        1.0,
+        weight_decay,
+        true,
+        false,
+        1000,
+        multi_precision,
+        false,
+        amsgrad,
+        &params[idx],
+        &moment1s[idx],
+        &moment2s[idx],
+        &moment2s_max[idx],
+        &beta1_pows[idx],
+        &beta2_pows[idx],
+        multi_precision ? &master_params[idx] : nullptr);
+  }
+
+  void UpdateWithAdamBaseline(const std::vector<DenseTensor> &grads,
+                              size_t idx) {
+    AdamDenseKernel<T, Context>(
+        *ctx,
+        params[idx],
+        grads[idx],
+        learning_rate,
+        moment1s[idx],
+        moment2s[idx],
+        moment2s_max[idx],
+        beta1_pows[idx],
+        beta2_pows[idx],
+        multi_precision ? paddle::make_optional(master_params[idx])
+                        : paddle::none,
+        paddle::none,
+        beta1,
+        beta2,
+        epsilon,
+        false,
+        1000,
+        multi_precision,
+        false,
+        amsgrad,
+        &params[idx],
+        &moment1s[idx],
+        &moment2s[idx],
+        &moment2s_max[idx],
+        &beta1_pows[idx],
+        &beta2_pows[idx],
+        multi_precision ? &master_params[idx] : nullptr);
+  }
+};
+
+template <typename T, typename Context>
+auto MaxDiff(const Context &ctx,
+             const DenseTensor &x_t,
+             const DenseTensor &y_t) {
+  using MT = typename AdamInfo<T, Context>::MT;
+  auto mp_dtype = phi::CppTypeToDataType<MT>::Type();
+  auto x = Cast<T, Context>(ctx, x_t, mp_dtype);
+  auto y = Cast<T, Context>(ctx, y_t, mp_dtype);
+
+  EXPECT_EQ(x.dims(), y.dims());
+  DenseTensor diff, diff_reduced, diff_reduced_cpu;
+
+  diff.Resize(x.dims());
+  ctx.template Alloc<MT>(&diff);
+  SubtractKernel<MT, Context>(ctx, x, y, &diff);
+  AbsKernel<MT, Context>(ctx, diff, &diff);
+
+  diff_reduced.Resize({1});
+  ctx.template Alloc<MT>(&diff_reduced);
+  MaxRawKernel<MT, Context>(ctx,
+                            diff,
+                            common::vectorize<int64_t>(x.dims()),
+                            false,
+                            true,
+                            &diff_reduced);
+
+  diff_reduced_cpu.Resize(diff_reduced.dims());
+  ctx.template HostAlloc<MT>(&diff_reduced_cpu);
+  Copy<Context>(ctx, diff_reduced, CPUPlace(), true, &diff_reduced_cpu);
+  EXPECT_EQ(diff_reduced_cpu.place(), CPUPlace());
+  return diff_reduced_cpu.data<MT>()[0];
+}
+
+template <typename T, typename Context>
+auto MaxDiff(const Context &ctx,
+             const std::vector<DenseTensor> &xs,
+             const std::vector<DenseTensor> &ys) {
+  using MT = typename AdamInfo<T, Context>::MT;
+  MT diff = 0;
+  for (size_t i = 0; i < xs.size(); ++i) {
+    diff = std::max<MT>(diff, MaxDiff<T, Context>(ctx, xs[i], ys[i]));
+  }
+  return diff;
+}
+
+template <typename T, typename PlaceType>
+void TestFusedAdamBase(const std::vector<std::vector<int64_t>> &shapes,
+                       float atol,
+                       bool use_adamw,
+                       bool amsgrad,
+                       bool multi_precision = false,
+                       float beta1 = 0.9,
+                       float beta2 = 0.99,
+                       float weight_decay = 0.1,
+                       size_t steps = 5,
+                       uint64_t seed = 10) {
+  const auto &ctx = *phi::DeviceContextPool::Instance().GetByPlace(PlaceType());
+  using Context = typename std::remove_const<
+      typename std::remove_pointer<decltype(&ctx)>::type>::type;
+  ctx.GetGenerator()->SetCurrentSeed(seed);
+  AdamInfo<T, Context> info1(ctx,
+                             shapes,
+                             beta1,
+                             beta2,
+                             weight_decay,
+                             multi_precision,
+                             use_adamw,
+                             amsgrad);
+  auto info2 = AdamInfo<T, Context>::DeepCopy(info1);
+
+  for (size_t i = 0; i < steps; ++i) {
+    auto grads = GenerateRandomTensorVectors<T>(ctx, shapes);
+    info1.Update(false, grads);
+    info2.Update(true, grads);
+  }
+
+  using MT = typename decltype(info1)::MT;
+
+#define PD_ADAM_TEST_COMP(__field, __dtype)                          \
+  do {                                                               \
+    MT __diff = MaxDiff<__dtype>(ctx, info1.__field, info2.__field); \
+    EXPECT_LE(__diff, static_cast<MT>(atol))                         \
+        << #__field << " has diff when use_adamw = " << use_adamw    \
+        << " , multi_precision = " << multi_precision;               \
+  } while (0)
+
+  PD_ADAM_TEST_COMP(beta1_pows, MT);
+  PD_ADAM_TEST_COMP(beta2_pows, MT);
+  PD_ADAM_TEST_COMP(params, T);
+  PD_ADAM_TEST_COMP(master_params, MT);
+  PD_ADAM_TEST_COMP(moment1s, MT);
+  PD_ADAM_TEST_COMP(moment2s, MT);
+  PD_ADAM_TEST_COMP(moment2s_max, MT);
+}
+
+static auto GenerateRandomShapes(size_t n, uint64_t low, uint64_t high) {
+  std::random_device device;
+  std::default_random_engine engine(device());
+  std::uniform_int_distribution<uint64_t> dist(low, high);
+  std::vector<std::vector<int64_t>> shapes(n);
+  for (size_t i = 0; i < n; ++i) {
+    shapes[i].push_back(static_cast<int64_t>(dist(engine)));
+  }
+  return shapes;
+}
+
+TEST(fused_adam, test_fp32_cpu) {
+  auto shapes = GenerateRandomShapes(30, 10, 20);
+  float atol = 0.0f;
+  for (auto use_adamw : {false, true}) {
+    for (auto amsgrad : {false, true}) {
+      TestFusedAdamBase<float, CPUPlace>(shapes, atol, use_adamw, amsgrad);
+    }
+  }
+}
+
+#ifdef PADDLE_WITH_CUDA
+TEST(fused_adam, test_fp32_gpu) {
+  auto shapes = GenerateRandomShapes(40, 0, 2 << 18);
+  for (auto use_adamw : {false, true}) {
+    // AdamwDenseKernel uses torch-compatible math (double-precision
+    // intermediates, FMA intrinsics) while FusedAdamKernel uses the
+    // original float-precision math, so allow a small tolerance for adamw.
+    float atol = use_adamw ? 1e-5f : 0.0f;
+    for (auto amsgrad : {false, true}) {
+      TestFusedAdamBase<float, GPUPlace>(shapes, atol, use_adamw, amsgrad);
+    }
+  }
+}
+
+TEST(fused_adam, test_fp16_gpu) {
+  auto shapes = GenerateRandomShapes(40, 0, 2 << 18);
+  float atol = 5e-3f;
+  for (auto use_adamw : {false, true}) {
+    for (auto amsgrad : {false, true}) {
+      TestFusedAdamBase<dtype::float16, GPUPlace>(
+          shapes, atol, use_adamw, amsgrad, true);
+    }
+  }
+}
+#endif
+
+}  // namespace phi

test/cpp/phi/kernels/test_gpu_timer.cu

@@ -0,0 +1,119 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include <functional>
+
+#include "glog/logging.h"
+#include "paddle/phi/kernels/autotune/gpu_timer.h"
+#include "paddle/phi/kernels/funcs/aligned_vector.h"
+
+template <typename T, int VecSize>
+__global__ void VecSum(T *x, T *y, int N) {
+#ifdef __HIPCC__
+  int idx = hipBlockDim_x * hipBlockIdx_x + hipThreadIdx_x;
+#else
+  int idx = blockDim.x * blockIdx.x + threadIdx.x;
+#endif
+  using LoadT = phi::AlignedVector<T, VecSize>;
+  for (int i = idx * VecSize; i < N; i += blockDim.x * gridDim.x * VecSize) {
+    LoadT x_vec;
+    LoadT y_vec;
+    phi::Load<T, VecSize>(&x[i], &x_vec);
+    phi::Load<T, VecSize>(&y[i], &y_vec);
+#pragma unroll
+    for (int j = 0; j < VecSize; j++) {
+      y_vec[j] = x_vec[j] + y_vec[j];
+    }
+    phi::Store<T, VecSize>(y_vec, &y[i]);
+  }
+}
+
+template <int Vecsize, int Threads, size_t Blocks>
+void Algo(float *d_in, float *d_out, size_t N) {
+#ifdef __HIPCC__
+  hipLaunchKernelGGL(HIP_KERNEL_NAME(VecSum<float, Vecsize>),
+                     dim3(Blocks),
+                     dim3(Threads),
+                     0,
+                     0,
+                     d_in,
+                     d_out,
+                     N);
+#else
+  VecSum<float, Vecsize><<<Blocks, Threads>>>(d_in, d_out, N);
+#endif
+}
+
+TEST(GpuTimer, Sum) {
+  float *in1, *in2, *out;
+  float *d_in1, *d_in2;
+  size_t N = 1 << 20;
+  size_t size = sizeof(float) * N;
+#ifdef __HIPCC__
+  hipMalloc(reinterpret_cast<void **>(&d_in1), size);
+  hipMalloc(reinterpret_cast<void **>(&d_in2), size);
+#else
+  cudaMalloc(reinterpret_cast<void **>(&d_in1), size);
+  cudaMalloc(reinterpret_cast<void **>(&d_in2), size);
+#endif
+  in1 = reinterpret_cast<float *>(malloc(size));
+  in2 = reinterpret_cast<float *>(malloc(size));
+  out = reinterpret_cast<float *>(malloc(size));
+  for (size_t i = 0; i < N; i++) {
+    in1[i] = 1.0f;
+    in2[i] = 2.0f;
+  }
+
+#ifdef __HIPCC__
+  hipMemcpy(d_in1, in1, size, hipMemcpyHostToDevice);
+  hipMemcpy(d_in2, in2, size, hipMemcpyHostToDevice);
+#else
+  cudaMemcpy(d_in1, in1, size, cudaMemcpyHostToDevice);
+  cudaMemcpy(d_in2, in2, size, cudaMemcpyHostToDevice);
+#endif
+
+  using Functor = std::function<void(float *, float *, size_t)>;
+  Functor algo0 = Algo<4, 256, 1024>;
+  Functor algo1 = Algo<1, 256, 1024>;
+  Functor algo2 = Algo<1, 256, 8>;
+
+  std::vector<Functor> algos = {algo0, algo1, algo2};
+
+  for (int j = 0; j < algos.size(); ++j) {
+    auto algo = algos[j];
+    phi::GpuTimer timer;
+    timer.Start(0);
+    algo(d_in1, d_in2, N);
+    timer.Stop(0);
+    VLOG(3) << "algo: " << j << " cost: " << timer.ElapsedTime() << "ms";
+  }
+
+#ifdef __HIPCC__
+  hipMemcpy(out, d_in2, size, hipMemcpyDeviceToHost);
+#else
+  cudaMemcpy(out, d_in2, size, cudaMemcpyDeviceToHost);
+#endif
+  free(in1);
+  free(in2);
+  free(out);
+#ifdef __HIPCC__
+  hipFree(d_in1);
+  hipFree(d_in2);
+#else
+  cudaFree(d_in1);
+  cudaFree(d_in2);
+#endif
+}

test/cpp/phi/kernels/test_math_function.cc

@@ -0,0 +1,380 @@
+//  Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <array>
+#include <set>
+
+#include "gtest/gtest.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/kernels/funcs/blas/blas.h"
+#include "paddle/phi/kernels/funcs/math_function.h"
+
+namespace phi {
+namespace tests {
+
+template <typename T>
+inline phi::funcs::BlasT<phi::CPUContext, T> GetBlas(
+    const phi::CPUContext& context) {
+  return phi::funcs::GetBlas<phi::CPUContext, T>(context);
+}
+
+TEST(math_function, gemm_notrans_cblas) {
+  phi::DenseTensor input1;
+  phi::DenseTensor input2;
+  phi::DenseTensor input3;
+
+  int m = 2;
+  int n = 3;
+  int k = 3;
+  auto* dev_ctx =
+      phi::DeviceContextPool::Instance().GetByPlace(phi::CPUPlace());
+
+  input1.Resize({2, 3});
+  float* input1_ptr = dev_ctx->template Alloc<float>(&input1);
+  std::array<float, 6> arr1 = {0, 1, 2, 3, 4, 5};
+  memcpy(input1_ptr, arr1.data(), 6 * sizeof(float));
+  input2.Resize({3, 4});
+  float* input2_ptr = dev_ctx->template Alloc<float>(&input2);
+  std::array<float, 12> arr2 = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
+  memcpy(input2_ptr, arr2.data(), 12 * sizeof(float));
+  input3.Resize({2, 4});
+  float* input3_ptr = dev_ctx->template Alloc<float>(&input3);
+  std::array<float, 8> arr3 = {0, 1, 2, 3, 4, 5, 6, 7};
+  memcpy(input3_ptr, arr3.data(), 8 * sizeof(float));
+
+  GetBlas<float>(*dev_ctx).GEMM(false,
+                                false,
+                                m,
+                                n,
+                                k,
+                                1,
+                                input1_ptr,
+                                3,
+                                input2_ptr + 1,
+                                4,
+                                1,
+                                input3_ptr + 1,
+                                4);
+
+  EXPECT_EQ(input3_ptr[0], 0);
+  EXPECT_EQ(input3_ptr[1], 24);
+  EXPECT_EQ(input3_ptr[2], 28);
+  EXPECT_EQ(input3_ptr[3], 32);
+  EXPECT_EQ(input3_ptr[4], 4);
+  EXPECT_EQ(input3_ptr[5], 73);
+  EXPECT_EQ(input3_ptr[6], 86);
+  EXPECT_EQ(input3_ptr[7], 99);
+}
+#ifdef PADDLE_WITH_LIBXSMM
+template <typename T>
+void MklSmmCompare(int m, int n, int k) {
+  phi::DenseTensor mat_a;
+  phi::DenseTensor mat_b;
+  phi::DenseTensor mat_c_smm;
+  phi::DenseTensor mat_c_mkl;
+
+  auto* dev_ctx =
+      phi::DeviceContextPool::Instance().GetByPlace(phi::CPUPlace());
+
+  mat_a.Resize({m, k});
+  T* A = dev_ctx->template Alloc<T>(&mat_a);
+  mat_b.Resize({k, n});
+  T* B = dev_ctx->template Alloc<T>(&mat_b);
+  mat_c_smm.Resize({m, n});
+  T* CSMM = dev_ctx->template Alloc<T>(&mat_c_smm);
+  mat_c_mkl.Resize({m, n});
+  T* CMKL = dev_ctx->template Alloc<T>(&mat_c_mkl);
+  T alpha = static_cast<T>(1);
+  T beta = static_cast<T>(0);
+  for (int i = 0; i < mat_a.numel(); ++i) {
+    A[i] = static_cast<T>(i);
+  }
+  for (int i = 0; i < mat_b.numel(); ++i) {
+    B[i] = static_cast<T>(i);
+  }
+  // lda,ldb,ldc follow RowMajor
+  int lda = k;
+  int ldb = n;
+  int ldc = n;
+
+  auto smm = [&, m, n, k, lda, ldb, ldc, alpha, beta]() {
+    const char transa = 'N';
+    const char transb = 'N';
+    phi::funcs::CBlas<T>::SMM_GEMM(&transa,
+                                   &transb,
+                                   &n,
+                                   &m,
+                                   &k,
+                                   &alpha,
+                                   B,
+                                   &ldb,
+                                   A,
+                                   &lda,
+                                   &beta,
+                                   CSMM,
+                                   &ldc);
+  };
+
+  auto mkl = [&, m, n, k, lda, ldb, ldc, alpha, beta]() {
+    phi::funcs::CBlas<T>::GEMM(CblasRowMajor,
+                               CblasNoTrans,
+                               CblasNoTrans,
+                               m,
+                               n,
+                               k,
+                               alpha,
+                               A,
+                               lda,
+                               B,
+                               ldb,
+                               beta,
+                               CMKL,
+                               ldc);
+  };
+
+  smm();
+  mkl();
+  ASSERT_EQ(mat_c_mkl.numel(), mat_c_smm.numel());
+  for (int i = 0; i < mat_c_mkl.numel(); ++i) {
+    EXPECT_FLOAT_EQ(CSMM[i], CMKL[i]);
+  }
+}
+TEST(math_function, gemm_mkl_vs_smm) {
+  MklSmmCompare<float>(1, 2, 3);
+  MklSmmCompare<double>(1, 2, 3);
+  MklSmmCompare<float>(3, 2, 1);
+  MklSmmCompare<double>(3, 2, 1);
+  MklSmmCompare<float>(3, 8, 5);
+  MklSmmCompare<double>(3, 8, 5);
+}
+#endif
+
+TEST(math_function, gemm_trans_cblas) {
+  phi::DenseTensor input1;
+  phi::DenseTensor input2;
+  phi::DenseTensor input3;
+
+  int m = 2;
+  int n = 3;
+  int k = 3;
+  auto* dev_ctx =
+      phi::DeviceContextPool::Instance().GetByPlace(phi::CPUPlace());
+
+  input1.Resize({2, 3});
+  float* input1_ptr = dev_ctx->template Alloc<float>(&input1);
+  std::array<float, 6> arr1 = {0, 1, 2, 3, 4, 5};
+  memcpy(input1_ptr, arr1.data(), 6 * sizeof(float));
+  input2.Resize({4, 3});
+  float* input2_ptr = dev_ctx->template Alloc<float>(&input2);
+  std::array<float, 12> arr2 = {0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11};
+  memcpy(input2_ptr, arr2.data(), 12 * sizeof(float));
+  input3.Resize({2, 4});
+  float* input3_ptr = dev_ctx->template Alloc<float>(&input3);
+  std::array<float, 8> arr3 = {0, 1, 2, 3, 4, 5, 6, 7};
+  memcpy(input3_ptr, arr3.data(), 8 * sizeof(float));
+
+  GetBlas<float>(*dev_ctx).GEMM(false,
+                                true,
+                                m,
+                                n,
+                                k,
+                                1,
+                                input1_ptr,
+                                3,
+                                input2_ptr + 3,
+                                3,
+                                1,
+                                input3_ptr + 1,
+                                4);
+
+  EXPECT_EQ(input3_ptr[0], 0);
+  EXPECT_EQ(input3_ptr[1], 24);
+  EXPECT_EQ(input3_ptr[2], 28);
+  EXPECT_EQ(input3_ptr[3], 32);
+  EXPECT_EQ(input3_ptr[4], 4);
+  EXPECT_EQ(input3_ptr[5], 73);
+  EXPECT_EQ(input3_ptr[6], 86);
+  EXPECT_EQ(input3_ptr[7], 99);
+}
+
+TEST(math_function, zero) {
+  phi::DenseTensor tensor;
+  auto* dev_ctx =
+      phi::DeviceContextPool::Instance().GetByPlace(phi::CPUPlace());
+
+  tensor.Resize({2, 2});
+  float* t = dev_ctx->template Alloc<float>(&tensor);
+  phi::funcs::SetConstant<phi::CPUContext, float> functor;
+  functor(*dev_ctx, &tensor, 0);
+  EXPECT_EQ(t[0], 0);
+  EXPECT_EQ(t[1], 0);
+  EXPECT_EQ(t[2], 0);
+  EXPECT_EQ(t[3], 0);
+
+  functor(*dev_ctx, &tensor, 1);
+
+  EXPECT_EQ(t[0], 1);
+  EXPECT_EQ(t[1], 1);
+  EXPECT_EQ(t[2], 1);
+  EXPECT_EQ(t[3], 1);
+}
+
+template <typename T>
+void GemvTest(int m, int n, bool trans) {
+  phi::DenseTensor mat_a;
+  phi::DenseTensor vec_b;
+  phi::DenseTensor vec_c;
+  int b_num = trans ? m : n;
+  int c_num = trans ? n : m;
+
+  auto* dev_ctx =
+      phi::DeviceContextPool::Instance().GetByPlace(phi::CPUPlace());
+
+  mat_a.Resize({m, n});
+  T* data_a = dev_ctx->template Alloc<T>(&mat_a);
+  vec_b.Resize({b_num});
+  T* data_b = dev_ctx->template Alloc<T>(&vec_b);
+  vec_c.Resize({c_num});
+  T* data_c = dev_ctx->template Alloc<T>(&vec_c);
+  for (int i = 0; i < mat_a.numel(); ++i) {
+    data_a[i] = static_cast<T>(i);
+  }
+  for (int i = 0; i < vec_b.numel(); ++i) {
+    data_b[i] = static_cast<T>(i);
+  }
+
+  GetBlas<T>(*dev_ctx).GEMV(trans,
+                            static_cast<int>(m),
+                            static_cast<int>(n),
+                            1.,
+                            data_a,
+                            data_b,
+                            0.,
+                            data_c);
+
+  if (!trans) {
+    for (int i = 0; i < m; ++i) {
+      T sum = 0.0;
+      for (int j = 0; j < n; ++j) {
+        sum += data_a[i * n + j] * data_b[j];
+      }
+      ASSERT_FLOAT_EQ(data_c[i], sum);
+    }
+  } else {
+    for (int i = 0; i < n; ++i) {
+      T sum = 0.0;
+      for (int j = 0; j < m; ++j) {
+        sum += data_a[j * n + i] * data_b[j];
+      }
+      ASSERT_FLOAT_EQ(data_c[i], sum);
+    }
+  }
+}
+
+TEST(math_function, gemv) {
+  GemvTest<float>(3, 13, false);
+  GemvTest<double>(4, 5, false);
+  GemvTest<float>(12, 7, true);
+  GemvTest<double>(7, 9, true);
+}
+
+TEST(math_function, set_constant) {
+  phi::DenseTensor t;
+  auto* dev_ctx =
+      phi::DeviceContextPool::Instance().GetByPlace(phi::CPUPlace());
+  t.Resize({10, 10});
+  dev_ctx->template Alloc<int>(&t);
+  phi::funcs::set_constant(*dev_ctx, &t, static_cast<int>(10));
+  for (int64_t i = 0; i < t.numel(); ++i) {
+    PADDLE_ENFORCE_EQ(10,
+                      t.data<int>()[i],
+                      common::errors::InvalidArgument(
+                          "Each value of input tensor should be 10, "
+                          "but received %d.",
+                          t.data<int>()[i]));
+  }
+}
+
+template <typename T>
+void GemmWarpTest(int m, int n, int k, T alpha, T beta) {
+  phi::DenseTensor mat_a;
+  phi::DenseTensor mat_b;
+  phi::DenseTensor mat_c_ref;
+  phi::DenseTensor mat_c_mkl;
+
+  auto* dev_ctx =
+      phi::DeviceContextPool::Instance().GetByPlace(phi::CPUPlace());
+
+  mat_a.Resize({m, k});
+  T* A = dev_ctx->template Alloc<T>(&mat_a);
+  mat_b.Resize({k, n});
+  T* B = dev_ctx->template Alloc<T>(&mat_b);
+  mat_c_ref.Resize({m, n});
+  T* CREF = dev_ctx->template Alloc<T>(&mat_c_ref);
+  mat_c_mkl.Resize({m, n});
+  T* CMKL = dev_ctx->template Alloc<T>(&mat_c_mkl);
+
+  ASSERT_EQ(mat_c_mkl.numel(), mat_c_ref.numel());
+  for (int i = 0; i < mat_a.numel(); ++i) {
+    A[i] = static_cast<T>(i);
+  }
+  for (int i = 0; i < mat_b.numel(); ++i) {
+    B[i] = static_cast<T>(i + 1);
+  }
+  for (int i = 0; i < mat_c_ref.numel(); ++i) {
+    CREF[i] = static_cast<T>(i + 2);
+    CMKL[i] = CREF[i];
+  }
+
+  // this would call gemm_warp
+  GetBlas<T>(*dev_ctx).GEMM(
+      CblasNoTrans, CblasNoTrans, m, n, k, alpha, A, B, beta, CREF);
+
+  // lda,ldb,ldc follow RowMajor
+  int lda = k;
+  int ldb = n;
+  int ldc = n;
+  phi::funcs::CBlas<T>::GEMM(CblasRowMajor,
+                             CblasNoTrans,
+                             CblasNoTrans,
+                             m,
+                             n,
+                             k,
+                             alpha,
+                             A,
+                             lda,
+                             B,
+                             ldb,
+                             beta,
+                             CMKL,
+                             ldc);
+
+  for (int i = 0; i < mat_c_mkl.numel(); ++i) {
+    EXPECT_FLOAT_EQ(CREF[i], CMKL[i]);
+  }
+}
+
+TEST(math_function, gemm_warp) {
+  GemmWarpTest<float>(3, 2, 5, 1.f, 0.f);
+  GemmWarpTest<float>(3, 2, 5, 2.f, 1.f);
+  GemmWarpTest<float>(8, 5, 6, 1.f, 0.f);
+  GemmWarpTest<float>(8, 5, 6, 2.f, 1.f);
+  GemmWarpTest<double>(3, 2, 5, 1.0, 0.0);
+  GemmWarpTest<double>(3, 2, 5, 2.0, 1.0);
+  GemmWarpTest<double>(8, 5, 6, 1.0, 0.0);
+  GemmWarpTest<double>(8, 5, 6, 2.0, 1.0);
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/kernels/test_math_function.cu

@@ -0,0 +1,527 @@
+//  Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "gtest/gtest.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/core/tensor_utils.h"
+#include "paddle/phi/kernels/funcs/blas/blas.h"
+#include "paddle/phi/kernels/funcs/math_function.h"
+
+namespace phi {
+namespace tests {
+
+void fill_fp16_data(phi::dtype::float16* in_ptr,
+                    size_t size,
+                    const std::vector<float>& data) {
+  PADDLE_ENFORCE_EQ(
+      size,
+      data.size(),
+      common::errors::InvalidArgument(
+          "The size of argument data should"
+          " be equal to the argument size. Expected %d, but received %d.",
+          size,
+          data.size()));
+  for (size_t i = 0; i < data.size(); ++i) {
+    in_ptr[i] = phi::dtype::float16(data[i]);
+  }
+}
+
+template <typename T>
+inline phi::funcs::BlasT<phi::GPUContext, T> GetBlas(
+    const phi::GPUContext& context) {
+  return phi::funcs::GetBlas<phi::GPUContext, T>(context);
+}
+
+TEST(math_function, notrans_mul_trans_fp32) {
+  phi::DenseTensor input1;
+  phi::DenseTensor input1_gpu;
+  phi::DenseTensor input2_gpu;
+  phi::DenseTensor out_gpu;
+  phi::DenseTensor out;
+
+  phi::CPUPlace cpu_place;
+  phi::GPUPlace gpu_place(0);
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* context = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  float* input1_ptr = input1.mutable_data<float>({2, 3}, cpu_place);
+  float arr[6] = {0, 1, 2, 3, 4, 5};
+  memcpy(input1_ptr, arr, 6 * sizeof(float));
+
+  phi::Copy(*context, input1, gpu_place, true, &input1_gpu);
+  phi::Copy(*context, input1, gpu_place, true, &input2_gpu);
+
+  out_gpu.mutable_data<float>({2, 2}, gpu_place);
+  GetBlas<float>(*context).MatMul(
+      input1_gpu, false, input2_gpu, true, 1, &out_gpu, 0);
+
+  phi::Copy(*context, out_gpu, cpu_place, true, &out);
+
+  float* out_ptr = out.data<float>();
+  context->Wait();
+  EXPECT_EQ(out_ptr[0], 5);
+  EXPECT_EQ(out_ptr[1], 14);
+  EXPECT_EQ(out_ptr[2], 14);
+  EXPECT_EQ(out_ptr[3], 50);
+}
+
+TEST(math_function, notrans_mul_trans_fp16) {
+  phi::DenseTensor input1;
+  phi::DenseTensor input1_gpu;
+  phi::DenseTensor input2_gpu;
+  phi::DenseTensor out_gpu;
+  phi::DenseTensor out;
+
+  phi::CPUPlace cpu_place;
+  phi::GPUPlace gpu_place(0);
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* context = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  // fp16 GEMM in cublas requires GPU compute capability >= 53
+  if (context->GetComputeCapability() < 53) {
+    return;
+  }
+
+  phi::dtype::float16* input1_ptr =
+      input1.mutable_data<phi::dtype::float16>({2, 3}, cpu_place);
+  fill_fp16_data(input1_ptr, input1.numel(), {0, 1, 2, 3, 4, 5});
+
+  phi::Copy(*context, input1, gpu_place, true, &input1_gpu);
+  phi::Copy(*context, input1, gpu_place, true, &input2_gpu);
+
+  out_gpu.mutable_data<phi::dtype::float16>({2, 2}, gpu_place);
+
+  GetBlas<phi::dtype::float16>(*context).MatMul(input1_gpu,
+                                                false,
+                                                input2_gpu,
+                                                true,
+                                                phi::dtype::float16(1),
+                                                &out_gpu,
+                                                phi::dtype::float16(0));
+
+  phi::Copy(*context, out_gpu, cpu_place, true, &out);
+
+  phi::dtype::float16* out_ptr = out.data<phi::dtype::float16>();
+  context->Wait();
+  EXPECT_EQ(static_cast<float>(out_ptr[0]), 5);
+  EXPECT_EQ(static_cast<float>(out_ptr[1]), 14);
+  EXPECT_EQ(static_cast<float>(out_ptr[2]), 14);
+  EXPECT_EQ(static_cast<float>(out_ptr[3]), 50);
+}
+
+TEST(math_function, trans_mul_notrans_fp32) {
+  phi::DenseTensor input1;
+  phi::DenseTensor input1_gpu;
+  phi::DenseTensor input2_gpu;
+  phi::DenseTensor out_gpu;
+  phi::DenseTensor out;
+
+  phi::CPUPlace cpu_place;
+  phi::GPUPlace gpu_place(0);
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* context = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  float* input1_ptr = input1.mutable_data<float>({2, 3}, cpu_place);
+  float arr[6] = {0, 1, 2, 3, 4, 5};
+  memcpy(input1_ptr, arr, 6 * sizeof(float));
+
+  phi::Copy(*context, input1, gpu_place, true, &input1_gpu);
+  phi::Copy(*context, input1, gpu_place, true, &input2_gpu);
+
+  out_gpu.mutable_data<float>({3, 3}, gpu_place);
+
+  GetBlas<float>(*context).MatMul(
+      input1_gpu, true, input2_gpu, false, 1, &out_gpu, 0);
+
+  phi::Copy(*context, out_gpu, cpu_place, true, &out);
+
+  float* out_ptr = out.data<float>();
+  context->Wait();
+  EXPECT_EQ(out_ptr[0], 9);
+  EXPECT_EQ(out_ptr[1], 12);
+  EXPECT_EQ(out_ptr[2], 15);
+  EXPECT_EQ(out_ptr[3], 12);
+  EXPECT_EQ(out_ptr[4], 17);
+  EXPECT_EQ(out_ptr[5], 22);
+  EXPECT_EQ(out_ptr[6], 15);
+  EXPECT_EQ(out_ptr[7], 22);
+  EXPECT_EQ(out_ptr[8], 29);
+}
+
+TEST(math_function, trans_mul_notrans_fp16) {
+  phi::DenseTensor input1;
+  phi::DenseTensor input1_gpu;
+  phi::DenseTensor input2_gpu;
+  phi::DenseTensor out_gpu;
+  phi::DenseTensor out;
+
+  phi::CPUPlace cpu_place;
+  phi::GPUPlace gpu_place(0);
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* context = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  // fp16 GEMM in cublas requires GPU compute capability >= 53
+  if (context->GetComputeCapability() < 53) {
+    return;
+  }
+
+  phi::dtype::float16* input1_ptr =
+      input1.mutable_data<phi::dtype::float16>({2, 3}, cpu_place);
+  fill_fp16_data(input1_ptr, input1.numel(), {0, 1, 2, 3, 4, 5});
+
+  phi::Copy(*context, input1, gpu_place, true, &input1_gpu);
+  phi::Copy(*context, input1, gpu_place, true, &input2_gpu);
+
+  out_gpu.mutable_data<phi::dtype::float16>({3, 3}, gpu_place);
+
+  GetBlas<phi::dtype::float16>(*context).MatMul(input1_gpu,
+                                                true,
+                                                input2_gpu,
+                                                false,
+                                                phi::dtype::float16(1),
+                                                &out_gpu,
+                                                phi::dtype::float16(0));
+
+  phi::Copy(*context, out_gpu, cpu_place, true, &out);
+
+  phi::dtype::float16* out_ptr = out.data<phi::dtype::float16>();
+  context->Wait();
+  EXPECT_EQ(static_cast<float>(out_ptr[0]), 9);
+  EXPECT_EQ(static_cast<float>(out_ptr[1]), 12);
+  EXPECT_EQ(static_cast<float>(out_ptr[2]), 15);
+  EXPECT_EQ(static_cast<float>(out_ptr[3]), 12);
+  EXPECT_EQ(static_cast<float>(out_ptr[4]), 17);
+  EXPECT_EQ(static_cast<float>(out_ptr[5]), 22);
+  EXPECT_EQ(static_cast<float>(out_ptr[6]), 15);
+  EXPECT_EQ(static_cast<float>(out_ptr[7]), 22);
+  EXPECT_EQ(static_cast<float>(out_ptr[8]), 29);
+}
+
+TEST(math_function, gemm_notrans_cublas_fp32) {
+  phi::DenseTensor input1;
+  phi::DenseTensor input2;
+  phi::DenseTensor input3;
+  phi::DenseTensor input1_gpu;
+  phi::DenseTensor input2_gpu;
+  phi::DenseTensor input3_gpu;
+
+  phi::CPUPlace cpu_place;
+  phi::GPUPlace gpu_place(0);
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* context = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  int m = 2;
+  int n = 3;
+  int k = 3;
+  float* input1_ptr = input1.mutable_data<float>({2, 3}, cpu_place);
+  float arr1[6] = {0, 1, 2, 3, 4, 5};
+  memcpy(input1_ptr, arr1, 6 * sizeof(float));
+  float* input2_ptr = input2.mutable_data<float>({3, 4}, cpu_place);
+  float arr2[12] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
+  memcpy(input2_ptr, arr2, 12 * sizeof(float));
+  float* input3_ptr = input3.mutable_data<float>({2, 4}, cpu_place);
+  float arr3[8] = {0, 1, 2, 3, 4, 5, 6, 7};
+  memcpy(input3_ptr, arr3, 8 * sizeof(float));
+
+  phi::Copy(*context, input1, gpu_place, true, &input1_gpu);
+  phi::Copy(*context, input2, gpu_place, true, &input2_gpu);
+  phi::Copy(*context, input3, gpu_place, true, &input3_gpu);
+  float* a = input1_gpu.data<float>();
+  float* b = input2_gpu.data<float>();
+  float* c = input3_gpu.mutable_data<float>(gpu_place);
+
+  GetBlas<float>(*context).GEMM(
+      false, false, m, n, k, 1, a, 3, b + 1, 4, 1, c + 1, 4);
+
+  phi::Copy(*context, input3_gpu, cpu_place, true, &input3);
+
+  // numpy code:
+  // a = np.arange(6).reshape(2, 3)
+  // b = np.arange(12).reshape(3, 4)[:, 1:]
+  // c = np.arange(8).reshape(2, 4)[:, 1:]
+  // out = np.arange(8).reshape(2, 4)
+  // out[:, 1:] = np.dot(a, b) + c
+  context->Wait();
+  EXPECT_EQ(input3_ptr[0], 0);
+  EXPECT_EQ(input3_ptr[1], 24);
+  EXPECT_EQ(input3_ptr[2], 28);
+  EXPECT_EQ(input3_ptr[3], 32);
+  EXPECT_EQ(input3_ptr[4], 4);
+  EXPECT_EQ(input3_ptr[5], 73);
+  EXPECT_EQ(input3_ptr[6], 86);
+  EXPECT_EQ(input3_ptr[7], 99);
+}
+
+TEST(math_function, gemm_notrans_cublas_fp16) {
+  phi::DenseTensor input1;
+  phi::DenseTensor input2;
+  phi::DenseTensor input3;
+  phi::DenseTensor input1_gpu;
+  phi::DenseTensor input2_gpu;
+  phi::DenseTensor input3_gpu;
+
+  phi::CPUPlace cpu_place;
+  phi::GPUPlace gpu_place(0);
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* context = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  // fp16 GEMM in cublas requires GPU compute capability >= 53
+  if (context->GetComputeCapability() < 53) {
+    return;
+  }
+
+  int m = 2;
+  int n = 3;
+  int k = 3;
+  phi::dtype::float16* input1_ptr =
+      input1.mutable_data<phi::dtype::float16>({2, 3}, cpu_place);
+  fill_fp16_data(input1_ptr, input1.numel(), {0, 1, 2, 3, 4, 5});
+  phi::dtype::float16* input2_ptr =
+      input2.mutable_data<phi::dtype::float16>({3, 4}, cpu_place);
+  fill_fp16_data(
+      input2_ptr, input2.numel(), {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11});
+  phi::dtype::float16* input3_ptr =
+      input3.mutable_data<phi::dtype::float16>({2, 4}, cpu_place);
+  fill_fp16_data(input3_ptr, input3.numel(), {0, 1, 2, 3, 4, 5, 6, 7});
+
+  phi::Copy(*context, input1, gpu_place, true, &input1_gpu);
+  phi::Copy(*context, input2, gpu_place, true, &input2_gpu);
+  phi::Copy(*context, input3, gpu_place, true, &input3_gpu);
+  phi::dtype::float16* a = input1_gpu.data<phi::dtype::float16>();
+  phi::dtype::float16* b = input2_gpu.data<phi::dtype::float16>();
+  phi::dtype::float16* c =
+      input3_gpu.mutable_data<phi::dtype::float16>(gpu_place);
+
+  GetBlas<phi::dtype::float16>(*context).GEMM(
+      false,
+      false,
+      m,
+      n,
+      k,
+      static_cast<phi::dtype::float16>(1),
+      a,
+      3,
+      b + 1,
+      4,
+      static_cast<phi::dtype::float16>(1),
+      c + 1,
+      4);
+
+  phi::Copy(*context, input3_gpu, cpu_place, true, &input3);
+
+  // numpy code:
+  // a = np.arange(6).reshape(2, 3)
+  // b = np.arange(12).reshape(3, 4)[:, 1:]
+  // c = np.arange(8).reshape(2, 4)[:, 1:]
+  // out = np.arange(8).reshape(2, 4)
+  // out[:, 1:] = np.dot(a, b) + c
+  context->Wait();
+  EXPECT_EQ(static_cast<float>(input3_ptr[0]), 0);
+  EXPECT_EQ(static_cast<float>(input3_ptr[1]), 24);
+  EXPECT_EQ(static_cast<float>(input3_ptr[2]), 28);
+  EXPECT_EQ(static_cast<float>(input3_ptr[3]), 32);
+  EXPECT_EQ(static_cast<float>(input3_ptr[4]), 4);
+  EXPECT_EQ(static_cast<float>(input3_ptr[5]), 73);
+  EXPECT_EQ(static_cast<float>(input3_ptr[6]), 86);
+  EXPECT_EQ(static_cast<float>(input3_ptr[7]), 99);
+}
+
+TEST(math_function, gemm_trans_cublas_fp32) {
+  phi::DenseTensor input1;
+  phi::DenseTensor input2;
+  phi::DenseTensor input3;
+  phi::DenseTensor input1_gpu;
+  phi::DenseTensor input2_gpu;
+  phi::DenseTensor input3_gpu;
+
+  phi::CPUPlace cpu_place;
+  phi::GPUPlace gpu_place(0);
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* context = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  int m = 2;
+  int n = 3;
+  int k = 3;
+  float* input1_ptr = input1.mutable_data<float>({2, 3}, cpu_place);
+  float arr1[6] = {0, 1, 2, 3, 4, 5};
+  memcpy(input1_ptr, arr1, 6 * sizeof(float));
+  float* input2_ptr = input2.mutable_data<float>({4, 3}, cpu_place);
+  float arr2[12] = {0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11};
+  memcpy(input2_ptr, arr2, 12 * sizeof(float));
+  float* input3_ptr = input3.mutable_data<float>({2, 4}, cpu_place);
+  float arr3[8] = {0, 1, 2, 3, 4, 5, 6, 7};
+  memcpy(input3_ptr, arr3, 8 * sizeof(float));
+
+  phi::Copy(*context, input1, gpu_place, true, &input1_gpu);
+  phi::Copy(*context, input2, gpu_place, true, &input2_gpu);
+  phi::Copy(*context, input3, gpu_place, true, &input3_gpu);
+  float* a = input1_gpu.data<float>();
+  float* b = input2_gpu.data<float>();
+  float* c = input3_gpu.mutable_data<float>(gpu_place);
+
+  GetBlas<float>(*context).GEMM(
+      false, true, m, n, k, 1, a, 3, b + 3, 3, 1, c + 1, 4);
+
+  phi::Copy(*context, input3_gpu, cpu_place, true, &input3);
+
+  context->Wait();
+  EXPECT_EQ(input3_ptr[0], 0);
+  EXPECT_EQ(input3_ptr[1], 24);
+  EXPECT_EQ(input3_ptr[2], 28);
+  EXPECT_EQ(input3_ptr[3], 32);
+  EXPECT_EQ(input3_ptr[4], 4);
+  EXPECT_EQ(input3_ptr[5], 73);
+  EXPECT_EQ(input3_ptr[6], 86);
+  EXPECT_EQ(input3_ptr[7], 99);
+}
+
+TEST(math_function, gemm_trans_cublas_fp16) {
+  phi::DenseTensor input1;
+  phi::DenseTensor input2;
+  phi::DenseTensor input3;
+  phi::DenseTensor input1_gpu;
+  phi::DenseTensor input2_gpu;
+  phi::DenseTensor input3_gpu;
+
+  phi::CPUPlace cpu_place;
+  phi::GPUPlace gpu_place(0);
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* context = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  // fp16 GEMM in cublas requires GPU compute capability >= 53
+  if (context->GetComputeCapability() < 53) {
+    return;
+  }
+
+  int m = 2;
+  int n = 3;
+  int k = 3;
+  phi::dtype::float16* input1_ptr =
+      input1.mutable_data<phi::dtype::float16>({2, 3}, cpu_place);
+  fill_fp16_data(input1_ptr, input1.numel(), {0, 1, 2, 3, 4, 5});
+  phi::dtype::float16* input2_ptr =
+      input2.mutable_data<phi::dtype::float16>({4, 3}, cpu_place);
+  fill_fp16_data(
+      input2_ptr, input2.numel(), {0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11});
+  phi::dtype::float16* input3_ptr =
+      input3.mutable_data<phi::dtype::float16>({2, 4}, cpu_place);
+  fill_fp16_data(input3_ptr, input3.numel(), {0, 1, 2, 3, 4, 5, 6, 7});
+
+  phi::Copy(*context, input1, gpu_place, true, &input1_gpu);
+  phi::Copy(*context, input2, gpu_place, true, &input2_gpu);
+  phi::Copy(*context, input3, gpu_place, true, &input3_gpu);
+  phi::dtype::float16* a = input1_gpu.data<phi::dtype::float16>();
+  phi::dtype::float16* b = input2_gpu.data<phi::dtype::float16>();
+  phi::dtype::float16* c =
+      input3_gpu.mutable_data<phi::dtype::float16>(gpu_place);
+
+  GetBlas<phi::dtype::float16>(*context).GEMM(
+      false,
+      true,
+      m,
+      n,
+      k,
+      static_cast<phi::dtype::float16>(1),
+      a,
+      3,
+      b + 3,
+      3,
+      static_cast<phi::dtype::float16>(1),
+      c + 1,
+      4);
+
+  phi::Copy(*context, input3_gpu, cpu_place, true, &input3);
+
+  context->Wait();
+  EXPECT_EQ(static_cast<float>(input3_ptr[0]), 0);
+  EXPECT_EQ(static_cast<float>(input3_ptr[1]), 24);
+  EXPECT_EQ(static_cast<float>(input3_ptr[2]), 28);
+  EXPECT_EQ(static_cast<float>(input3_ptr[3]), 32);
+  EXPECT_EQ(static_cast<float>(input3_ptr[4]), 4);
+  EXPECT_EQ(static_cast<float>(input3_ptr[5]), 73);
+  EXPECT_EQ(static_cast<float>(input3_ptr[6]), 86);
+  EXPECT_EQ(static_cast<float>(input3_ptr[7]), 99);
+}
+
+template <typename T>
+void GemvTest(int m, int n, bool trans) {
+  phi::DenseTensor mat_a;
+  phi::DenseTensor vec_b;
+  phi::DenseTensor vec_c;
+
+  phi::CPUPlace cpu_place;
+  phi::GPUPlace gpu_place(0);
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* context = reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  T* data_a = mat_a.mutable_data<T>({m, n}, cpu_place);
+  T* data_b = vec_b.mutable_data<T>({trans ? m : n}, cpu_place);
+  T* data_c = vec_c.mutable_data<T>({trans ? n : m}, cpu_place);
+
+  phi::DenseTensor g_mat_a;
+  phi::DenseTensor g_vec_b;
+  phi::DenseTensor g_vec_c;
+  T* g_data_a = g_mat_a.mutable_data<T>(mat_a.dims(), gpu_place);
+  T* g_data_b = g_vec_b.mutable_data<T>(vec_b.dims(), gpu_place);
+  T* g_data_c = g_vec_c.mutable_data<T>(vec_c.dims(), gpu_place);
+
+  for (int i = 0; i < mat_a.numel(); ++i) {
+    data_a[i] = static_cast<T>(i);
+  }
+  for (int i = 0; i < vec_b.numel(); ++i) {
+    data_b[i] = static_cast<T>(i);
+  }
+
+  phi::Copy(*context, mat_a, gpu_place, true, &g_mat_a);
+  phi::Copy(*context, vec_b, gpu_place, true, &g_vec_b);
+
+  GetBlas<T>(*context).GEMV(trans,
+                            static_cast<int>(m),
+                            static_cast<int>(n),
+                            1.,
+                            g_data_a,
+                            g_data_b,
+                            0.,
+                            g_data_c);
+
+  phi::Copy(*context, g_vec_c, cpu_place, true, &vec_c);
+
+  if (!trans) {
+    for (int i = 0; i < m; ++i) {
+      T sum = 0.0;
+      for (int j = 0; j < n; ++j) {
+        sum += data_a[i * n + j] * data_b[j];
+      }
+      ASSERT_FLOAT_EQ(data_c[i], sum);
+    }
+  } else {
+    for (int i = 0; i < n; ++i) {
+      T sum = 0.0;
+      for (int j = 0; j < m; ++j) {
+        sum += data_a[j * n + i] * data_b[j];
+      }
+      ASSERT_FLOAT_EQ(data_c[i], sum);
+    }
+  }
+}
+
+TEST(math_function, gemv) {
+  GemvTest<float>(3, 13, false);
+  GemvTest<double>(3, 13, false);
+  GemvTest<float>(3, 13, true);
+  GemvTest<double>(3, 13, true);
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/kernels/test_memcpy_dev_api.cc

@@ -0,0 +1,79 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include <memory>
+
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/common/place.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/device_context.h"
+#include "paddle/phi/kernels/memcpy_kernel.h"
+
+namespace phi {
+namespace tests {
+
+using DDim = phi::DDim;
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+TEST(DEV_API, memcpy_d2h) {
+  // 1. create tensor
+  const auto cpu_alloc =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace());
+  phi::DenseTensor x_cpu(cpu_alloc.get(),
+                         phi::DenseTensorMeta(phi::DataType::FLOAT32,
+                                              common::make_ddim({3, 2, 2, 3}),
+                                              phi::DataLayout::NCHW));
+  auto& pool = phi::DeviceContextPool::Instance();
+  auto* cpu_ctx = pool.GetByPlace(phi::CPUPlace());
+  auto* x_cpu_data = cpu_ctx->template Alloc<float>(&x_cpu);
+
+  for (int i = 0; i < x_cpu.numel(); i++) {
+    x_cpu_data[i] = static_cast<float>(i);
+  }
+
+  const auto alloc =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::GPUPlace());
+  phi::DenseTensor x;
+
+  // 2. test API
+  auto* dev_ctx = pool.GetByPlace(phi::GPUPlace());
+
+  phi::MemcpyH2DKernel<phi::GPUContext>(*dev_ctx, x_cpu, 1, &x);
+  phi::DenseTensor out;
+  phi::MemcpyD2HKernel<phi::GPUContext>(*dev_ctx, x, 1, &out);
+
+  // 3. check result
+  std::vector<int64_t> expect_shape = {12, 3};
+  ASSERT_EQ(out.dims(), x.dims());
+  ASSERT_EQ(out.meta().dtype, phi::DataType::FLOAT32);
+  ASSERT_EQ(out.meta().layout, phi::DataLayout::NCHW);
+
+  bool value_equal = true;
+  auto* dense_out_data = out.data<float>();
+  for (int i = 0; i < x_cpu.numel(); i++) {
+    if (x_cpu_data[i] != dense_out_data[i]) {
+      value_equal = false;
+      break;
+    }
+  }
+  ASSERT_EQ(value_equal, true);
+}
+
+#endif
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/kernels/test_strings_copy_dev_api.cc

@@ -0,0 +1,67 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+
+#include <algorithm>
+#include <memory>
+#include <string>
+
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/common/pstring.h"
+#include "paddle/phi/core/string_tensor.h"
+#include "paddle/phi/kernels/strings/strings_copy_kernel.h"
+
+namespace phi {
+namespace tests {
+
+using DDim = phi::DDim;
+using pstring = phi::dtype::pstring;
+
+TEST(DEV_API, strings_copy) {
+  // 1. create tensor
+  const DDim dims({2, 3});
+  StringTensorMeta meta(dims);
+
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+
+  auto* dev_ctx = reinterpret_cast<phi::CPUContext*>(pool.Get(phi::CPUPlace()));
+
+  const auto string_allocator =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace());
+  const auto alloc = string_allocator.get();
+  StringTensor string_src(alloc, meta);
+  StringTensor string_dst(alloc, meta);
+
+  // 2. Assign input text
+  const char* input[] = {"A Short Pstring.",  // NOLINT
+                         "A Large Pstring Whose Length Is Longer Than 22.",
+                         "abc",
+                         "defg",
+                         "hijklmn",
+                         "opqrst"};
+  pstring* string_src_data = dev_ctx->template Alloc<pstring>(&string_src);
+
+  for (int i = 0; i < string_src.numel(); ++i) {
+    string_src_data[i] = input[i];
+  }
+  phi::strings::Copy(*dev_ctx, string_src, false, &string_dst);
+  for (int64_t i = 0; i < string_src.numel(); i++) {
+    ASSERT_EQ(string_src.data()[i], string_dst.data()[i]);
+  }
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/kernels/test_strings_copy_dev_api.cu

@@ -0,0 +1,73 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+
+#include <algorithm>
+#include <memory>
+#include <string>
+
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/common/pstring.h"
+#include "paddle/phi/core/string_tensor.h"
+#include "paddle/phi/kernels/strings/strings_copy_kernel.h"
+#include "paddle/phi/kernels/strings/strings_empty_kernel.h"
+
+namespace phi {
+namespace tests {
+
+using DDim = phi::DDim;
+using pstring = phi::dtype::pstring;
+
+TEST(DEV_API, strings_copy) {
+  // 1. create tensor
+  const DDim dims({2, 3});
+  StringTensorMeta meta(dims);
+
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+
+  auto* dev_ctx = reinterpret_cast<phi::CPUContext*>(pool.Get(phi::CPUPlace()));
+  auto* gpu_dev_ctx =
+      reinterpret_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+
+  StringTensor string_src = phi::strings::Empty(*dev_ctx, std::move(meta));
+  StringTensor string_dst = phi::strings::Empty(*dev_ctx, std::move(meta));
+  // 2. Assign input text
+  const char* input[] = {"A Short Pstring.",
+                         "A Large Pstring Whose Length Is Longer Than 22.",
+                         "abc",
+                         "defg",
+                         "hijklmn",
+                         "opqrst"};
+  pstring* string_src_data = dev_ctx->template Alloc<pstring>(&string_src);
+  for (int i = 0; i < string_src.numel(); ++i) {
+    string_src_data[i] = input[i];
+  }
+  StringTensor string_gpu1 = phi::strings::Empty(*gpu_dev_ctx, std::move(meta));
+  StringTensor string_gpu2 = phi::strings::Empty(*gpu_dev_ctx, std::move(meta));
+
+  // cpu->gpu
+  phi::strings::Copy(*gpu_dev_ctx, string_src, false, &string_gpu1);
+  // gpu->gpu
+  phi::strings::Copy(*gpu_dev_ctx, string_gpu1, false, &string_gpu2);
+  // gpu->cpu
+  phi::strings::Copy(*gpu_dev_ctx, string_gpu2, false, &string_dst);
+  for (int64_t i = 0; i < string_src.numel(); i++) {
+    ASSERT_EQ(string_src.data()[i], string_dst.data()[i]);
+  }
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/kernels/test_strings_lower_upper_dev_api.cc

@@ -0,0 +1,136 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+
+#include <algorithm>
+#include <memory>
+#include <string>
+
+#include "glog/logging.h"
+
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/common/data_type.h"
+#include "paddle/phi/common/pstring.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/string_tensor.h"
+#include "paddle/phi/kernels/strings/strings_empty_kernel.h"
+#include "paddle/phi/kernels/strings/strings_lower_upper_kernel.h"
+namespace phi {
+namespace tests {
+
+using DDim = phi::DDim;
+using pstring = ::phi::dtype::pstring;
+
+TEST(DEV_API, strings_cast_convert) {
+  // 1. create tensor
+  const DDim dims({1, 2});
+  StringTensorMeta meta(dims);
+  const auto string_allocator =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace());
+  const auto alloc = string_allocator.get();
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+
+  auto* dev_ctx = pool.Get(phi::CPUPlace());
+
+  StringTensor dense_x(alloc, meta);
+
+  std::string short_str = "A Short Pstring.";
+  std::string long_str = "A Large Pstring Whose Length Is Longer Than 22.";
+
+  pstring* dense_x_data = dev_ctx->template Alloc<pstring>(&dense_x);
+  dense_x_data[0] = short_str;
+  dense_x_data[1] = long_str;
+
+  // 2. get expected results
+  std::string expected_results[] = {// NOLINT
+                                    short_str,
+                                    short_str,
+                                    long_str,
+                                    long_str};
+  std::transform(short_str.begin(),
+                 short_str.end(),
+                 expected_results[0].begin(),
+                 ::tolower);
+  std::transform(short_str.begin(),
+                 short_str.end(),
+                 expected_results[1].begin(),
+                 ::toupper);
+  std::transform(
+      long_str.begin(), long_str.end(), expected_results[2].begin(), ::tolower);
+  std::transform(
+      long_str.begin(), long_str.end(), expected_results[3].begin(), ::toupper);
+
+  // 3. test API, ascii encoding
+  auto dense_lower_out = phi::strings::StringLower(
+      *(static_cast<phi::CPUContext*>(dev_ctx)), dense_x, false);
+  auto dense_upper_out = phi::strings::StringUpper(
+      *(static_cast<phi::CPUContext*>(dev_ctx)), dense_x, false);
+
+  // 4. check results
+  ASSERT_EQ(dense_lower_out.numel(), 2);
+  ASSERT_EQ(dense_upper_out.numel(), 2);
+
+  // lower case
+  ASSERT_EQ(dense_lower_out.data()[0].data(), expected_results[0]);
+  ASSERT_EQ(dense_lower_out.data()[1].data(), expected_results[2]);
+
+  // upper case
+  ASSERT_EQ(dense_upper_out.data()[0].data(), expected_results[1]);
+  ASSERT_EQ(dense_upper_out.data()[1].data(), expected_results[3]);
+}
+
+TEST(DEV_API, strings_cast_convert_utf8) {
+  // 1. create tensor
+  const DDim dims({1, 1});
+  StringTensorMeta meta(dims);
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* dev_ctx = pool.Get(phi::CPUPlace());
+
+  const auto string_allocator =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace());
+  const auto alloc = string_allocator.get();
+  StringTensor dense_x(alloc, meta);
+
+  std::string utf8_str = "óÓsscHloëËóÓsscHloëËóÓsscHloëË";
+
+  pstring* dense_x_data = dev_ctx->template Alloc<pstring>(&dense_x);
+  dense_x_data[0] = utf8_str;
+
+  // 2. get expected results
+  std::string expected_results[] = {// NOLINT
+                                    "óósschloëëóósschloëëóósschloëë",
+                                    "ÓÓSSCHLOËËÓÓSSCHLOËËÓÓSSCHLOËË"};
+
+  // 3. test API, ascii encoding
+  auto dense_lower_out = phi::strings::StringLower(
+      *(static_cast<phi::CPUContext*>(dev_ctx)), dense_x, true);
+  auto dense_upper_out = phi::strings::StringUpper(
+      *(static_cast<phi::CPUContext*>(dev_ctx)), dense_x, true);
+
+  // 4. check results
+  ASSERT_EQ(dense_lower_out.numel(), 1);
+  ASSERT_EQ(dense_upper_out.numel(), 1);
+
+  // lower case
+  VLOG(0) << dense_lower_out.data()[0].data();
+  ASSERT_EQ(dense_lower_out.data()[0].data(), expected_results[0]);
+
+  // upper case
+  ASSERT_EQ(dense_upper_out.data()[0].data(), expected_results[1]);
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/kernels/test_strings_lower_upper_dev_api.cu

@@ -0,0 +1,161 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+#include <stdio.h>
+
+#include <algorithm>
+#include <memory>
+#include <string>
+#include <vector>
+
+#if (defined(__NVCC__) || defined(__HIPCC__))
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#endif
+
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/backends/gpu/gpu_helper.h"
+#include "paddle/phi/common/data_type.h"
+#include "paddle/phi/common/pstring.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/string_tensor.h"
+#include "paddle/phi/kernels/strings/strings_copy_kernel.h"
+#include "paddle/phi/kernels/strings/strings_empty_kernel.h"
+#include "paddle/phi/kernels/strings/strings_lower_upper_kernel.h"
+namespace phi {
+namespace tests {
+
+namespace framework = paddle::framework;
+using DDim = phi::DDim;
+using pstring = ::phi::dtype::pstring;
+using phi::CPUPlace;
+using phi::GPUPlace;
+
+TEST(DEV_API, strings_cast_convert) {
+  auto gpu0 = GPUPlace();
+  auto cpu = CPUPlace();
+
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  GPUContext* dev_ctx = reinterpret_cast<GPUContext*>(pool.Get(gpu0));
+  CPUContext* cpu_ctx = reinterpret_cast<CPUContext*>(pool.Get(cpu));
+
+  // 1. create tensor
+  const DDim dims({1, 2});
+  StringTensorMeta meta(dims);
+  StringTensor gpu_strings_x = phi::strings::Empty(*dev_ctx, std::move(meta));
+  StringTensor cpu_strings_x = phi::strings::Empty(*cpu_ctx, std::move(meta));
+  StringTensor cpu_strings_lower_out =
+      phi::strings::Empty(*cpu_ctx, std::move(meta));
+  StringTensor cpu_strings_upper_out =
+      phi::strings::Empty(*cpu_ctx, std::move(meta));
+
+  std::string short_str = "A Short Pstring.";
+  std::string long_str = "A Large Pstring Whose Length Is Longer Than 22.";
+
+  pstring* cpu_strings_x_data =
+      cpu_ctx->template Alloc<pstring>(&cpu_strings_x);
+  cpu_strings_x_data[0] = short_str;
+  cpu_strings_x_data[1] = long_str;
+
+  phi::strings::Copy(*dev_ctx, cpu_strings_x, false, &gpu_strings_x);
+
+  // 2. get expected results
+  std::string expected_results[] = {short_str, short_str, long_str, long_str};
+  std::transform(short_str.begin(),
+                 short_str.end(),
+                 expected_results[0].begin(),
+                 ::tolower);
+  std::transform(short_str.begin(),
+                 short_str.end(),
+                 expected_results[1].begin(),
+                 ::toupper);
+  std::transform(
+      long_str.begin(), long_str.end(), expected_results[2].begin(), ::tolower);
+  std::transform(
+      long_str.begin(), long_str.end(), expected_results[3].begin(), ::toupper);
+
+  // 3. test API, ascii encoding
+  auto gpu_strings_lower_out =
+      phi::strings::StringLower(*dev_ctx, gpu_strings_x, false);
+  auto gpu_strings_upper_out =
+      phi::strings::StringUpper(*dev_ctx, gpu_strings_x, false);
+
+  phi::strings::Copy(
+      *dev_ctx, gpu_strings_lower_out, false, &cpu_strings_lower_out);
+  phi::strings::Copy(
+      *dev_ctx, gpu_strings_upper_out, false, &cpu_strings_upper_out);
+
+  // 4. check results
+  ASSERT_EQ(gpu_strings_lower_out.numel(), 2);
+  ASSERT_EQ(gpu_strings_upper_out.numel(), 2);
+  const char* cpu_results[] = {cpu_strings_lower_out.data()[0].data(),
+                               cpu_strings_upper_out.data()[0].data(),
+                               cpu_strings_lower_out.data()[1].data(),
+                               cpu_strings_upper_out.data()[1].data()};
+  for (int i = 0; i < 4; ++i) {
+    ASSERT_EQ(cpu_results[i], expected_results[i]);
+  }
+}
+
+TEST(DEV_API, strings_cast_convert_utf8) {
+  auto gpu0 = GPUPlace();
+  auto cpu = CPUPlace();
+
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  GPUContext* dev_ctx = reinterpret_cast<GPUContext*>(pool.Get(gpu0));
+  CPUContext* cpu_ctx = reinterpret_cast<CPUContext*>(pool.Get(cpu));
+
+  // 1. create tensor
+  const DDim dims({1, 1});
+  StringTensorMeta meta(dims);
+  StringTensor gpu_strings_x = phi::strings::Empty(*dev_ctx, std::move(meta));
+  StringTensor cpu_strings_x = phi::strings::Empty(*cpu_ctx, std::move(meta));
+  StringTensor cpu_strings_lower_out =
+      phi::strings::Empty(*cpu_ctx, std::move(meta));
+  StringTensor cpu_strings_upper_out =
+      phi::strings::Empty(*cpu_ctx, std::move(meta));
+  std::string utf8_str = "óÓsscHloëË";
+  pstring* cpu_strings_x_data =
+      cpu_ctx->template Alloc<pstring>(&cpu_strings_x);
+
+  cpu_strings_x_data[0] = utf8_str;
+  phi::strings::Copy(*dev_ctx, cpu_strings_x, false, &gpu_strings_x);
+
+  // 2. get expected results
+  std::string expected_results[] = {"óósschloëë", "ÓÓSSCHLOËË"};
+
+  // 3. test API, ascii encoding
+  auto gpu_strings_lower_out =
+      phi::strings::StringLower(*dev_ctx, gpu_strings_x, true);
+  auto gpu_strings_upper_out =
+      phi::strings::StringUpper(*dev_ctx, gpu_strings_x, true);
+  phi::strings::Copy(
+      *dev_ctx, gpu_strings_lower_out, false, &cpu_strings_lower_out);
+  phi::strings::Copy(
+      *dev_ctx, gpu_strings_upper_out, false, &cpu_strings_upper_out);
+
+  // 4. check results
+  const char* cpu_results[] = {cpu_strings_lower_out.data()[0].data(),
+                               cpu_strings_upper_out.data()[0].data()};
+  ASSERT_EQ(cpu_strings_lower_out.numel(), 1);
+  ASSERT_EQ(cpu_strings_upper_out.numel(), 1);
+  for (int i = 0; i < 2; ++i) {
+    ASSERT_EQ(cpu_results[i], expected_results[i]);
+  }
+}
+
+}  // namespace tests
+}  // namespace phi

test/cpp/phi/kernels/test_ternary_broadcast.cu

@@ -0,0 +1,221 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+#include <vector>
+#include "glog/logging.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/core/tensor_utils.h"
+#include "paddle/phi/kernels/funcs/broadcast_function.h"
+
+template <typename T>
+struct AddTernary_1 {
+  inline HOSTDEVICE T operator()(T a, T b, T c) const { return a + b + c; }
+};
+
+template <typename T>
+struct AddTernary_2 {
+  inline HOSTDEVICE T operator()(T a, T b, T c) const { return a + b + c; }
+};
+
+template <typename T>
+struct AddTernary_3 {
+  inline HOSTDEVICE T operator()(T a, T b, T c) const { return a + b + c; }
+};
+
+template <typename T>
+void InitValue(T* data, size_t numel, const int val) {
+  for (auto i = 0; i < numel; ++i) {
+    data[i] = static_cast<T>(val);
+  }
+}
+
+template <typename T, typename Func>
+void TestCase(const phi::GPUContext& dev_ctx,
+              const phi::DDim& dim1,
+              const phi::DDim& dim2,
+              const phi::DDim& dim3,
+              const phi::DDim& dim_out,
+              const size_t times,
+              Func compute) {
+  phi::DataType dtype = phi::CppTypeToDataType<T>::Type();
+  const auto alloc_cpu =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace());
+  const auto alloc_gpu =
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::GPUPlace());
+
+  auto in1 = std::make_shared<phi::DenseTensor>(
+      alloc_cpu.get(),
+      phi::DenseTensorMeta(dtype, dim1, phi::DataLayout::NCHW));
+  auto in2 = std::make_shared<phi::DenseTensor>(
+      alloc_cpu.get(),
+      phi::DenseTensorMeta(dtype, dim2, phi::DataLayout::NCHW));
+  auto in3 = std::make_shared<phi::DenseTensor>(
+      alloc_cpu.get(),
+      phi::DenseTensorMeta(dtype, dim3, phi::DataLayout::NCHW));
+  InitValue(in1->data<T>(), in1->numel(), 1);
+  InitValue(in2->data<T>(), in2->numel(), 1);
+  InitValue(in3->data<T>(), in3->numel(), 1);
+
+  auto d_in1 = std::make_shared<phi::DenseTensor>(
+      alloc_gpu.get(),
+      phi::DenseTensorMeta(dtype, dim1, phi::DataLayout::NCHW));
+  auto d_in2 = std::make_shared<phi::DenseTensor>(
+      alloc_gpu.get(),
+      phi::DenseTensorMeta(dtype, dim2, phi::DataLayout::NCHW));
+  auto d_in3 = std::make_shared<phi::DenseTensor>(
+      alloc_gpu.get(),
+      phi::DenseTensorMeta(dtype, dim3, phi::DataLayout::NCHW));
+  auto d_out = std::make_shared<phi::DenseTensor>(
+      alloc_gpu.get(),
+      phi::DenseTensorMeta(dtype, dim_out, phi::DataLayout::NCHW));
+  phi::Copy(dev_ctx, *in1.get(), phi::GPUPlace(), false, d_in1.get());
+  phi::Copy(dev_ctx, *in2.get(), phi::GPUPlace(), false, d_in2.get());
+  phi::Copy(dev_ctx, *in3.get(), phi::GPUPlace(), false, d_in3.get());
+
+  std::vector<const phi::DenseTensor*> inputs{
+      d_in1.get(), d_in2.get(), d_in3.get()};
+  std::vector<phi::DenseTensor*> outputs{d_out.get()};
+  for (int i = 0; i < times; ++i) {
+    phi::funcs::BroadcastKernel<T>(dev_ctx, inputs, &outputs, compute);
+  }
+  dev_ctx.Wait();
+}
+
+TEST(Broadcast, add) {
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  auto place = phi::GPUPlace();
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* dev_ctx = static_cast<const phi::GPUContext*>(pool.GetByPlace(place));
+  size_t times = 10;
+
+  do {
+    auto dim1 = common::make_ddim({1, 2048, 3584});
+    auto dim2 = common::make_ddim({1, 2048, 1});
+    auto dim3 = common::make_ddim({1, 1, 3584});
+    auto dim_out = common::make_ddim({1, 2048, 3584});
+    TestCase<float>(
+        *dev_ctx, dim1, dim2, dim3, dim_out, times, AddTernary_1<float>());
+    TestCase<phi::dtype::float16>(*dev_ctx,
+                                  dim1,
+                                  dim2,
+                                  dim3,
+                                  dim_out,
+                                  times,
+                                  AddTernary_1<phi::dtype::float16>());
+    TestCase<phi::dtype::bfloat16>(*dev_ctx,
+                                   dim1,
+                                   dim2,
+                                   dim3,
+                                   dim_out,
+                                   times,
+                                   AddTernary_1<phi::dtype::bfloat16>());
+    TestCase<phi::dtype::complex<float>>(
+        *dev_ctx,
+        dim1,
+        dim2,
+        dim3,
+        dim_out,
+        times,
+        AddTernary_1<phi::dtype::complex<float>>());
+    TestCase<phi::dtype::complex<double>>(
+        *dev_ctx,
+        dim1,
+        dim2,
+        dim3,
+        dim_out,
+        times,
+        AddTernary_1<phi::dtype::complex<double>>());
+  } while (0);
+
+  do {
+    auto dim1 = common::make_ddim({1, 256, 4, 256, 256});
+    auto dim2 = common::make_ddim({1, 256, 1, 1, 256});
+    auto dim3 = common::make_ddim({1, 1, 4, 256, 256});
+    auto dim_out = common::make_ddim({1, 256, 4, 256, 256});
+    TestCase<float>(
+        *dev_ctx, dim1, dim2, dim3, dim_out, times, AddTernary_2<float>());
+    TestCase<phi::dtype::float16>(*dev_ctx,
+                                  dim1,
+                                  dim2,
+                                  dim3,
+                                  dim_out,
+                                  times,
+                                  AddTernary_2<phi::dtype::float16>());
+    TestCase<phi::dtype::bfloat16>(*dev_ctx,
+                                   dim1,
+                                   dim2,
+                                   dim3,
+                                   dim_out,
+                                   times,
+                                   AddTernary_2<phi::dtype::bfloat16>());
+    TestCase<phi::dtype::complex<float>>(
+        *dev_ctx,
+        dim1,
+        dim2,
+        dim3,
+        dim_out,
+        times,
+        AddTernary_2<phi::dtype::complex<float>>());
+    TestCase<phi::dtype::complex<double>>(
+        *dev_ctx,
+        dim1,
+        dim2,
+        dim3,
+        dim_out,
+        times,
+        AddTernary_2<phi::dtype::complex<double>>());
+  } while (0);
+
+  do {
+    auto dim1 = common::make_ddim({1, 256, 256});
+    auto dim2 = common::make_ddim({1, 1, 256});
+    auto dim3 = common::make_ddim({1, 256, 1});
+    auto dim_out = common::make_ddim({1, 256, 256});
+    TestCase<float>(
+        *dev_ctx, dim1, dim2, dim3, dim_out, times, AddTernary_3<float>());
+    TestCase<phi::dtype::float16>(*dev_ctx,
+                                  dim1,
+                                  dim2,
+                                  dim3,
+                                  dim_out,
+                                  times,
+                                  AddTernary_3<phi::dtype::float16>());
+    TestCase<phi::dtype::bfloat16>(*dev_ctx,
+                                   dim1,
+                                   dim2,
+                                   dim3,
+                                   dim_out,
+                                   times,
+                                   AddTernary_3<phi::dtype::bfloat16>());
+    TestCase<phi::dtype::complex<float>>(
+        *dev_ctx,
+        dim1,
+        dim2,
+        dim3,
+        dim_out,
+        times,
+        AddTernary_3<phi::dtype::complex<float>>());
+    TestCase<phi::dtype::complex<double>>(
+        *dev_ctx,
+        dim1,
+        dim2,
+        dim3,
+        dim_out,
+        times,
+        AddTernary_3<phi::dtype::complex<double>>());
+  } while (0);
+#endif
+}

test/cpp/phi/kernels/test_transfer_layout_dev_api.cc

@@ -0,0 +1,74 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include <memory>
+
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/backends/context_pool.h"
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/common/place.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/device_context.h"
+#include "paddle/phi/infermeta/unary.h"
+#include "paddle/phi/kernels/transfer_layout_kernel.h"
+
+namespace phi {
+namespace tests {
+
+#ifdef PADDLE_WITH_DNNL
+TEST(DEV_API, transfer_layout) {
+  // 1. create tensor
+
+  const int n = 2;
+  const int c = 3;
+  const int h = 4;
+  const int w = 5;
+
+  DenseTensor x;
+  MetaTensor meta_x(&x);
+  meta_x.set_dtype(DataType::FLOAT32);
+  meta_x.set_layout(DataLayout::ONEDNN);
+  meta_x.set_dims(common::make_ddim({n, c, h, w}));
+
+  DenseTensor out;
+
+  // 2. test API
+  auto& pool = phi::DeviceContextPool::Instance();
+  auto place = phi::CPUPlace();
+  auto* dev_ctx = static_cast<const phi::CPUContext*>(pool.GetByPlace(place));
+
+  MetaTensor meta_out(&out);
+  TransferLayoutInferMeta(x,
+                          static_cast<int>(x.layout()),
+                          static_cast<int>(DataLayout::NHWC),
+                          &meta_out);
+  TransferLayoutKernel<CPUContext>(*dev_ctx,
+                                   x,
+                                   static_cast<int>(x.layout()),
+                                   static_cast<int>(DataLayout::NHWC),
+                                   &out);
+
+  // 3. check result
+  std::vector<int64_t> expect_shape = {12, 3};
+  ASSERT_EQ(out.dims(), common::make_ddim({n, h, w, c}));
+  ASSERT_EQ(out.dims().size(), 4);
+  ASSERT_EQ(out.meta().dtype, DataType::FLOAT32);
+  ASSERT_EQ(out.meta().layout, DataLayout::NHWC);
+}
+
+#endif
+}  // namespace tests
+}  // namespace phi