chore: import upstream snapshot with attribution

test/cpp/imperative/CMakeLists.txt

@@ -0,0 +1,85 @@
+if(WIN32)
+  cc_test(
+    nccl_context_test
+    SRCS nccl_context_test.cc
+    DEPS phi)
+else()
+  if(WITH_GLOO AND (WITH_NCCL OR WITH_RCCL))
+    cc_test(
+      nccl_context_test
+      SRCS nccl_context_test.cc
+      DEPS nccl_context)
+    cc_test(
+      heter_ccl_context_test
+      SRCS heter_ccl_context_test.cc
+      DEPS heter_ccl_context
+           nccl_context
+           imperative_gloo_context
+           gloo_context
+           gloo_wrapper
+           gloo
+           framework_io)
+    #set_tests_properties(heter_ccl_context_test PROPERTIES LABELS "RUN_TYPE=DIST")
+  endif()
+  if(WITH_XPU_BKCL)
+    cc_test(
+      bkcl_context_test
+      SRCS bkcl_context_test.cc
+      DEPS bkcl_context)
+  endif()
+endif()
+
+cc_test(
+  test_gradient_accmulator
+  SRCS test_gradient_accmulator.cc
+  DEPS selected_rows_utils gradient_accumulator phi common phi_utils)
+cc_test(
+  test_layer
+  SRCS test_layer.cc
+  DEPS layer proto_desc operator op_registry variable_helper generated_op)
+cc_test(
+  test_prepare_op
+  SRCS test_prepare_op.cc
+  DEPS prepared_operator
+       op_info
+       split_op
+       layer
+       activation_op
+       phi
+       common)
+cc_test(
+  test_tracer
+  SRCS test_tracer.cc
+  DEPS tracer
+       layer
+       proto_desc
+       operator
+       op_registry
+       variable_helper
+       generated_op
+       generated_static_op
+       elementwise_add_op)
+cc_test(
+  test_hooks
+  SRCS test_hooks.cc
+  DEPS tracer
+       basic_engine
+       layer
+       proto_desc
+       operator
+       op_registry
+       variable_helper
+       generated_op
+       elementwise_add_op)
+cc_test(
+  test_eager
+  SRCS test_eager.cc
+  DEPS tracer layer prepared_operator generated_op)
+if(WITH_NCCL
+   OR WITH_RCCL
+   OR WITH_XPU_BKCL)
+  cc_test(
+    test_group
+    SRCS test_group.cc
+    DEPS reducer phi common)
+endif()

test/cpp/imperative/bkcl_context_test.cc

@@ -0,0 +1,66 @@
+//   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/imperative/bkcl_context.h"
+
+#include <thread>  // NOLINT
+
+#include "gtest/gtest.h"
+
+namespace imperative = paddle::imperative;
+namespace platform = paddle::platform;
+
+int nrings = 2;
+imperative::ParallelStrategy GetStrategy(int local_rank) {
+  std::vector<std::string> eps = {"127.0.0.1:9866", "localhost:9867"};
+  imperative::ParallelStrategy strategy;
+  strategy.trainer_endpoints_ = eps;
+  strategy.current_endpoint_ = eps[local_rank];
+  strategy.nranks_ = 2;
+  strategy.local_rank_ = local_rank;
+  strategy.nrings_ = nrings;
+  return strategy;
+}
+
+#if defined(PADDLE_WITH_XPU_BKCL)
+void BcastBKCLId(int local_rank, std::vector<BKCLUniqueId>* bkcl_ids) {
+  auto strategy = GetStrategy(local_rank);
+  phi::XPUPlace xpu(local_rank);
+  imperative::BKCLParallelContext ctx(strategy, xpu);
+  ctx.BcastBKCLId(*bkcl_ids, 0);
+}
+
+TEST(BcastBKCLId, Run) {
+  std::vector<BKCLUniqueId> bkcl_ids;
+  bkcl_ids.resize(nrings);
+  for (int i = 0; i < nrings; ++i) {
+    bkcl_get_unique_id(&bkcl_ids[i]);
+  }
+
+  std::thread t(BcastBKCLId, 0, &bkcl_ids);
+
+  std::vector<BKCLUniqueId> recv_bkcl_ids;
+  recv_bkcl_ids.resize(nrings);
+  for (int i = 0; i < nrings; ++i) {
+    bkcl_get_unique_id(&recv_bkcl_ids[i]);
+  }
+  BcastBKCLId(1, &recv_bkcl_ids);
+
+  t.join();
+  for (int i = 0; i < nrings; ++i) {
+    EXPECT_EQ(
+        0, std::memcmp(&bkcl_ids[i], &recv_bkcl_ids[i], BKCL_UNIQUE_ID_BYTES));
+  }
+}
+#endif

test/cpp/imperative/heter_ccl_context_test.cc

@@ -0,0 +1,89 @@
+//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/imperative/heter_ccl_context.h"
+
+#include <chrono>
+#include <thread>  // NOLINT
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/framework/tensor_util.h"
+#include "paddle/fluid/framework/variable.h"
+
+namespace imperative = paddle::imperative;
+namespace platform = paddle::platform;
+namespace framework = paddle::framework;
+
+imperative::ParallelStrategy GetStrategy(int local_rank) {
+  std::vector<std::string> eps = {"127.0.0.1:37580", "127.0.0.1:37581"};
+  imperative::ParallelStrategy strategy;
+  strategy.trainer_endpoints_ = eps;
+  strategy.current_endpoint_ = eps[local_rank];
+  strategy.nranks_ = eps.size();
+  strategy.local_rank_ = local_rank;
+  return strategy;
+}
+
+#ifdef PADDLE_WITH_NCCL
+void AllReduceByStream(int local_rank, int device_id) {
+  int data_size = 32;
+  const auto& place = phi::GPUPlace(device_id);
+  phi::GPUContext ctx(place);
+
+  // heter_parallel_ctx
+  imperative::HeterParallelContext hpc(GetStrategy(local_rank), device_id);
+
+  // init
+  hpc.Init();
+
+  // input and output data
+  framework::Variable* src_dev_var(new framework::Variable());
+  auto* src_dev_tensor = src_dev_var->GetMutable<phi::DenseTensor>();
+  src_dev_tensor->mutable_data<float>(common::make_ddim({data_size}), place);
+
+  std::vector<float> src_vec;
+  for (int i = 0; i < data_size; i++) {
+    src_vec.push_back(1.0 + local_rank);
+  }
+  framework::TensorFromVector(src_vec, ctx, src_dev_tensor);
+  ctx.Wait();
+
+  framework::Variable* dst_dev_var(new framework::Variable());
+  auto* dst_dev_tensor = dst_dev_var->GetMutable<phi::DenseTensor>();
+  dst_dev_tensor->mutable_data<float>(common::make_ddim({data_size}), place);
+
+  // call allreduce
+  hpc.AllReduceByStream(*src_dev_var, dst_dev_var, 0, false);
+  std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+
+  // check result
+  std::vector<float> dst_vec;
+  framework::TensorToVector(*dst_dev_tensor, ctx, &dst_vec);
+  ctx.Wait();
+
+  EXPECT_EQ(dst_vec.size(), src_vec.size());
+  for (int i = 0; i < data_size; i++) {
+    EXPECT_EQ(dst_vec[i], 3.0);
+  }
+}
+
+TEST(AllReduceByStream, Run) {
+  if (platform::GetGPUDeviceCount() >= 2) {
+    std::thread t0(AllReduceByStream, 0, 0);
+    std::thread t1(AllReduceByStream, 1, 1);
+    t0.join();
+    t1.join();
+  }
+}
+#endif

test/cpp/imperative/nccl_context_test.cc

@@ -0,0 +1,123 @@
+//   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/imperative/nccl_context.h"
+
+#include <thread>  // NOLINT
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/framework/tensor_util.h"
+#include "paddle/fluid/framework/variable.h"
+#include "paddle/phi/core/platform/gen_comm_id_helper.h"
+
+namespace imperative = paddle::imperative;
+namespace platform = paddle::platform;
+namespace framework = paddle::framework;
+
+int nrings = 2;
+imperative::ParallelStrategy GetStrategy(int local_rank) {
+  std::vector<std::string> eps = {"127.0.0.1:9866", "localhost:9867"};
+  imperative::ParallelStrategy strategy;
+  strategy.trainer_endpoints_ = eps;
+  strategy.current_endpoint_ = eps[local_rank];
+  strategy.nranks_ = 2;
+  strategy.local_rank_ = local_rank;
+  strategy.nrings_ = nrings;
+  return strategy;
+}
+
+#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
+void BcastNCCLId(int local_rank, std::vector<ncclUniqueId>* nccl_ids) {
+  auto strategy = GetStrategy(local_rank);
+  int server_fd = platform::CreateListenSocket(strategy.current_endpoint_);
+
+  phi::GPUPlace gpu(local_rank);
+  imperative::NCCLParallelContext ctx(strategy, gpu);
+  ctx.BcastNCCLId(*nccl_ids, 0, server_fd);
+
+  platform::CloseSocket(server_fd);
+}
+
+TEST(BcastNCCLId, Run) {
+  std::vector<ncclUniqueId> nccl_ids;
+  nccl_ids.resize(nrings);
+  for (int i = 0; i < nrings; ++i) {
+    phi::dynload::ncclGetUniqueId(&nccl_ids[i]);
+  }
+
+  std::thread t(BcastNCCLId, 0, &nccl_ids);
+
+  std::vector<ncclUniqueId> recv_nccl_ids;
+  recv_nccl_ids.resize(nrings);
+  for (int i = 0; i < nrings; ++i) {
+    phi::dynload::ncclGetUniqueId(&recv_nccl_ids[i]);
+  }
+  BcastNCCLId(1, &recv_nccl_ids);
+
+  t.join();
+  for (int i = 0; i < nrings; ++i) {
+    EXPECT_EQ(0,
+              std::memcmp(nccl_ids[i].internal,
+                          recv_nccl_ids[i].internal,
+                          NCCL_UNIQUE_ID_BYTES));
+  }
+}
+
+void Broadcast(int local_rank, int device_id) {
+  int data_size = 4;
+  float test_data = 7;
+  const auto& place = phi::GPUPlace(device_id);
+  phi::GPUContext ctx(place);
+
+  imperative::NCCLParallelContext npc(GetStrategy(local_rank), place);
+
+  // init
+  npc.Init();
+
+  framework::Variable* src_dev_var(new framework::Variable());
+  auto* src_dev_tensor = src_dev_var->GetMutable<phi::DenseTensor>();
+  src_dev_tensor->mutable_data<float>(common::make_ddim({data_size}), place);
+
+  // fill data for rank 0 only
+  std::vector<float> src_vec;
+  if (local_rank == 0) {
+    for (int i = 0; i < data_size; i++) {
+      src_vec.push_back(test_data);
+    }
+    framework::TensorFromVector(src_vec, ctx, src_dev_tensor);
+  }
+  ctx.Wait();
+
+  npc.Broadcast(src_dev_var, 0);
+  std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+
+  // check result
+  std::vector<float> dst_vec;
+  framework::TensorToVector(*src_dev_tensor, ctx, &dst_vec);
+  ctx.Wait();
+
+  for (int i = 0; i < data_size; i++) {
+    EXPECT_EQ(dst_vec[i], test_data);
+  }
+}
+
+TEST(Broadcast, Run) {
+  if (platform::GetGPUDeviceCount() >= 2) {
+    std::thread t0(Broadcast, 0, 0);
+    std::thread t1(Broadcast, 1, 1);
+    t0.join();
+    t1.join();
+  }
+}
+#endif

test/cpp/imperative/test_eager.cc

@@ -0,0 +1,102 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/framework/op_registry.h"
+#include "paddle/fluid/imperative/basic_engine.h"
+#include "paddle/fluid/imperative/execution_context.h"
+#include "paddle/fluid/imperative/layer.h"
+#include "paddle/fluid/imperative/tracer.h"
+#include "paddle/fluid/imperative/type_defs.h"
+#include "paddle/fluid/imperative/var_helper.h"
+#include "paddle/phi/core/memory/memcpy.h"
+#include "paddle/phi/core/platform/device_context.h"
+
+namespace paddle {
+namespace imperative {
+extern std::string LayerDebugString(const std::string& op_type,
+                                    const NameVarMap<egr::EagerVariable>& ins,
+                                    const NameVarMap<egr::EagerVariable>& outs);
+
+extern std::shared_ptr<GradOpNode> CreateGradOpNode(
+    const framework::OperatorBase& op,
+    const NameTensorMap& ins,
+    const NameTensorMap& outs,
+    const framework::AttributeMap& attrs,
+    const framework::AttributeMap& default_attrs,
+    const phi::Place& place,
+    const std::map<std::string, std::string>& inplace_map);
+
+TEST(test_eager, eager_debug) {
+  std::shared_ptr<egr::EagerVariable> x_in(new egr::EagerVariable("x_in"));
+  std::shared_ptr<egr::EagerVariable> y_in(new egr::EagerVariable("y_in"));
+  std::shared_ptr<egr::EagerVariable> vout(new egr::EagerVariable("vout"));
+  imperative::NameVarMap<egr::EagerVariable> ins = {{"X", {x_in}},
+                                                    {"Y", {y_in}}};
+  imperative::NameVarMap<egr::EagerVariable> outs = {{"Out", {vout}}};
+  LayerDebugString("mul", ins, outs);
+}
+TEST(test_create_node, eager_node) {
+  auto op = framework::OpRegistry::CreateOp("mul", {}, {}, {}, false);
+  framework::Scope scope;
+  auto ctx = framework::RuntimeContext({}, {});
+  imperative::NameVarMap<egr::EagerVariable> ins = {{"X", {nullptr}},
+                                                    {"Y", {nullptr}}};
+  imperative::NameVarMap<egr::EagerVariable> outs = {{"Out", {nullptr}}};
+  CreateGradOpNode((*op.get()),
+                   ins,
+                   outs,
+                   framework::AttributeMap{},
+                   framework::AttributeMap{},
+                   phi::CPUPlace(),
+                   {});
+}
+TEST(test_var_helper, eager_var_helper) {
+  framework::Variable var0, var1, var3, var4, var5, var6, var7, var8;
+  InitializeVariable(&var0, paddle::framework::proto::VarType::FEED_MINIBATCH);
+  InitializeVariable(&var1, paddle::framework::proto::VarType::STEP_SCOPES);
+  InitializeVariable(&var3,
+                     paddle::framework::proto::VarType::DENSE_TENSOR_ARRAY);
+  InitializeVariable(&var4, paddle::framework::proto::VarType::STRINGS);
+  InitializeVariable(&var5, paddle::framework::proto::VarType::VOCAB);
+  InitializeVariable(&var6, paddle::framework::proto::VarType::READER);
+  InitializeVariable(&var7, paddle::framework::proto::VarType::RAW);
+  ASSERT_ANY_THROW(
+      InitializeVariable(&var8, paddle::framework::proto::VarType::FP64));
+
+  auto egr_tensor = std::make_shared<egr::EagerVariable>();
+  egr_tensor->MutableVar()
+      ->GetMutable<phi::SelectedRows>()
+      ->mutable_value()
+      ->mutable_data<float>(phi::CPUPlace());
+  VLOG(6) << "egr_tensor create with ";
+  ASSERT_TRUE(phi::is_cpu_place(GetPlace<egr::EagerVariable>(egr_tensor)));
+  ASSERT_TRUE(GetDataType<egr::EagerVariable>(egr_tensor) ==
+              framework::proto::VarType::FP32);
+  GetCachedValue<egr::EagerVariable>(egr_tensor,
+                                     phi::KernelKey(phi::Backend::CPU,
+                                                    phi::DataLayout::ALL_LAYOUT,
+                                                    phi::DataType::FLOAT32));
+  ASSERT_ANY_THROW(SetType<egr::EagerVariable>(
+      egr_tensor, paddle::framework::proto::VarType::DENSE_TENSOR_ARRAY));
+}
+}  // namespace imperative
+}  // namespace paddle
+
+USE_OP_ITSELF(mul);

test/cpp/imperative/test_gradient_accmulator.cc

@@ -0,0 +1,514 @@
+// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <type_traits>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/framework/convert_utils.h"
+#include "paddle/fluid/framework/variable.h"
+#include "paddle/fluid/imperative/gradient_accumulator.h"
+#include "paddle/phi/core/memory/memcpy.h"
+#include "paddle/phi/kernels/funcs/math_function.h"
+
+namespace paddle {
+namespace imperative {
+
+TEST(Test__SelectedRowsMerge_Test, SelectedRowsMerge) {
+  phi::CPUPlace cpu;
+
+  std::vector<int64_t> rows{0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+  int64_t table_size = 10;
+  int64_t embedding_width = 10;
+
+  auto sr1 = std::make_shared<phi::SelectedRows>(rows, table_size);
+  auto sr2 = std::make_shared<phi::SelectedRows>(rows, table_size);
+
+  // initialize a sparse table 1
+  sr1->mutable_value()->Resize(
+      common::make_ddim({table_size, embedding_width}));
+  auto* data_sr1 = sr1->mutable_value()->mutable_data<float>(cpu);
+  for (int64_t i = 0; i < table_size; ++i) {
+    for (int64_t j = 0; j < embedding_width; ++j) {
+      data_sr1[i * embedding_width + j] = static_cast<float>(i);
+    }
+  }
+
+  // initialize a sparse table 2
+  sr2->mutable_value()->Resize(
+      common::make_ddim({table_size, embedding_width}));
+  auto* data_sr2 = sr2->mutable_value()->mutable_data<float>(cpu);
+  for (int64_t i = 0; i < table_size; ++i) {
+    for (int64_t j = 0; j < embedding_width; ++j) {
+      data_sr2[i * embedding_width + j] = static_cast<float>(i);
+    }
+  }
+  // new 2 phi::Tensor
+  paddle::Tensor t1(sr1);
+  paddle::Tensor t2(sr2);
+
+  // call SelectedRowsMerge
+  auto new_buffer =
+      paddle::imperative::SelectedRowsMerge<paddle::Tensor>(t1, t2);
+  auto* new_buffer_tensor =
+      static_cast<phi::SelectedRows*>(new_buffer->impl().get());
+  auto* new_buffer_data_sr1 =
+      new_buffer_tensor->mutable_value()->mutable_data<float>(cpu);
+
+  // verify the MergeAdd result
+  for (int64_t i = 0; i < table_size; ++i) {
+    for (int64_t j = 0; j < embedding_width; ++j) {
+      EXPECT_EQ(new_buffer_data_sr1[i * embedding_width + j],
+                (static_cast<float>(i) + static_cast<float>(i)));
+    }
+  }
+}
+
+template <typename Place1, typename Place2, typename T>
+int TensorAddTest(Place1 place1, Place2 place2, T t1, T t2) {
+  framework::Variable var1;
+  framework::Variable var2;
+  std::vector<T> src_data(10, t1);
+  std::vector<T> dst_data(10, t2);
+  std::vector<T> result;
+  phi::CPUPlace src_place;
+  for (unsigned int i = 0; i < 10; i++) {
+    result.emplace_back(src_data[i] + dst_data[i]);
+  }
+
+  std::vector<int64_t> dims = {2, 5};
+  auto* src = var1.GetMutable<phi::DenseTensor>();
+  auto* dst = var2.GetMutable<phi::DenseTensor>();
+  src->Resize(common::make_ddim(dims));
+  dst->Resize(common::make_ddim(dims));
+  auto* src_mutable = src->mutable_data<T>(place1);
+  auto* dst_mutable = dst->mutable_data<T>(place2);
+
+  if (!std::is_same<Place1, phi::GPUPlace>::value) {
+    paddle::memory::Copy(place1,
+                         src_mutable,
+                         src_place,
+                         src_data.data(),
+                         sizeof(T) * src_data.size());
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  } else {
+    paddle::memory::Copy(place1,
+                         src_mutable,
+                         src_place,
+                         src_data.data(),
+                         sizeof(T) * src_data.size(),
+                         0);
+#endif
+  }
+
+  if (!std::is_same<Place2, phi::GPUPlace>::value) {
+    paddle::memory::Copy(place2,
+                         dst_mutable,
+                         src_place,
+                         dst_data.data(),
+                         sizeof(T) * dst_data.size());
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  } else {
+    paddle::memory::Copy(place2,
+                         dst_mutable,
+                         src_place,
+                         dst_data.data(),
+                         sizeof(T) * dst_data.size(),
+                         0);
+#endif
+  }
+  imperative::TensorAdd<framework::Variable>(var1, &var2);
+  phi::DenseTensor rlt;
+  phi::CPUPlace rlt_place;
+  framework::TensorCopySync(*dst, rlt_place, &rlt);
+
+  for (unsigned int i = 0; i < rlt.numel(); i++) {
+    if (rlt.data<T>()[i] != result[i]) return 1;
+  }
+
+  return 0;
+}
+
+TEST(test_add_functor, add_functor) {
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  phi::GPUPlace gpu_place(0);
+#endif
+  phi::CPUPlace cpu_place;
+
+  int cpu_res = 1;
+
+  // float32
+  cpu_res = TensorAddTest(
+      cpu_place, cpu_place, static_cast<float>(1.0), static_cast<float>(2.0));
+  EXPECT_EQ(cpu_res, 0);
+  // float16
+  cpu_res = TensorAddTest(cpu_place,
+                          cpu_place,
+                          static_cast<phi::dtype::float16>(1.0),
+                          static_cast<phi::dtype::float16>(2.0));
+  EXPECT_EQ(cpu_res, 0);
+  // double
+  cpu_res = TensorAddTest(
+      cpu_place, cpu_place, static_cast<double>(1.0), static_cast<double>(2.0));
+  EXPECT_EQ(cpu_res, 0);
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  int gpu_res = 1;
+  gpu_res = TensorAddTest(gpu_place, gpu_place, 1.0, 0.0);
+  EXPECT_EQ(gpu_res, 0);
+  gpu_res = TensorAddTest(
+      gpu_place, gpu_place, static_cast<double>(1.0), static_cast<double>(2.0));
+  EXPECT_EQ(gpu_res, 0);
+
+  // normal
+  gpu_res = TensorAddTest(
+      gpu_place, gpu_place, static_cast<float>(1.0), static_cast<float>(2.0));
+  EXPECT_EQ(gpu_res, 0);
+  gpu_res = TensorAddTest(gpu_place,
+                          gpu_place,
+                          static_cast<phi::dtype::float16>(1.0),
+                          static_cast<phi::dtype::float16>(2.0));
+  EXPECT_EQ(gpu_res, 0);
+  // different places
+  gpu_res = TensorAddTest(
+      cpu_place, gpu_place, static_cast<float>(1.0), static_cast<float>(2.0));
+  EXPECT_EQ(gpu_res, 0);
+  gpu_res = TensorAddTest(
+      gpu_place, cpu_place, static_cast<float>(1.0), static_cast<float>(2.0));
+  EXPECT_EQ(gpu_res, 0);
+  gpu_res = TensorAddTest(cpu_place,
+                          gpu_place,
+                          static_cast<phi::dtype::float16>(1.0),
+                          static_cast<phi::dtype::float16>(2.0));
+  EXPECT_EQ(gpu_res, 0);
+  gpu_res = TensorAddTest(gpu_place,
+                          cpu_place,
+                          static_cast<phi::dtype::float16>(1.0),
+                          static_cast<phi::dtype::float16>(2.0));
+  EXPECT_EQ(gpu_res, 0);
+#endif
+
+#ifdef PADDLE_WITH_XPU
+  phi::XPUPlace xpu_place(0);
+  int xpu_res = 1;
+  // normal
+  xpu_res = TensorAddTest(
+      xpu_place, xpu_place, static_cast<float>(1.0), static_cast<float>(2.0));
+  EXPECT_EQ(xpu_res, 0);
+  xpu_res = TensorAddTest(xpu_place,
+                          xpu_place,
+                          static_cast<phi::dtype::float16>(1.0),
+                          static_cast<phi::dtype::float16>(2.0));
+  EXPECT_EQ(xpu_res, 0);
+  xpu_res = TensorAddTest(
+      xpu_place, xpu_place, static_cast<double>(1.0), static_cast<double>(2.0));
+  EXPECT_EQ(xpu_res, 0);
+  // different places
+  xpu_res = TensorAddTest(
+      cpu_place, xpu_place, static_cast<float>(1.0), static_cast<float>(2.0));
+  EXPECT_EQ(xpu_res, 0);
+  xpu_res = TensorAddTest(
+      xpu_place, cpu_place, static_cast<float>(1.0), static_cast<float>(2.0));
+  EXPECT_EQ(xpu_res, 0);
+  xpu_res = TensorAddTest(cpu_place,
+                          xpu_place,
+                          static_cast<phi::dtype::float16>(1.0),
+                          static_cast<phi::dtype::float16>(2.0));
+  EXPECT_EQ(xpu_res, 0);
+  xpu_res = TensorAddTest(xpu_place,
+                          cpu_place,
+                          static_cast<phi::dtype::float16>(1.0),
+                          static_cast<phi::dtype::float16>(2.0));
+  EXPECT_EQ(xpu_res, 0);
+  xpu_res = TensorAddTest(
+      cpu_place, xpu_place, static_cast<double>(1.0), static_cast<double>(2.0));
+  EXPECT_EQ(xpu_res, 0);
+  xpu_res = TensorAddTest(
+      xpu_place, cpu_place, static_cast<double>(1.0), static_cast<double>(2.0));
+  EXPECT_EQ(xpu_res, 0);
+#endif
+}
+
+TEST(test_add_functor, exception) {
+  phi::GPUPinnedPlace cuda_pinned_place;
+  phi::GPUPlace cuda_place(0);
+  phi::CPUPlace cpu_place;
+
+  ASSERT_ANY_THROW(TensorAddTest(cpu_place, cpu_place, 1, 0));
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  ASSERT_ANY_THROW(
+      TensorAddTest(cuda_pinned_place, cuda_pinned_place, 1.0, 0.0));
+  ASSERT_ANY_THROW(TensorAddTest(cuda_pinned_place,
+                                 cuda_pinned_place,
+                                 static_cast<phi::dtype::float16>(1.0),
+                                 static_cast<phi::dtype::float16>(2.0)));
+#endif
+}
+
+static void CopyVar(const framework::Variable& var,
+                    framework::Variable* dst_ptr) {
+  auto& dst = *dst_ptr;
+  dst.Clear();
+  if (var.IsType<phi::DenseTensor>()) {
+    const auto& src_tensor = var.Get<phi::DenseTensor>();
+    auto* dst_tensor = dst.GetMutable<phi::DenseTensor>();
+    framework::TensorCopySync(src_tensor, src_tensor.place(), dst_tensor);
+  } else {
+    const auto& src_selected_rows = var.Get<phi::SelectedRows>();
+    auto* dst_selected_rows = dst.GetMutable<phi::SelectedRows>();
+    dst_selected_rows->set_rows(src_selected_rows.rows());
+    dst_selected_rows->set_height(src_selected_rows.height());
+    framework::TensorCopySync(src_selected_rows.value(),
+                              src_selected_rows.value().place(),
+                              dst_selected_rows->mutable_value());
+  }
+}
+
+static bool IsEqualVar(const framework::Variable& var1,
+                       const framework::Variable& var2) {
+  if (var1.Type() != var2.Type()) {
+    return false;
+  }
+
+  phi::DenseTensor t1, t2;
+
+  if (var1.IsType<phi::DenseTensor>()) {
+    framework::TensorCopySync(
+        var1.Get<phi::DenseTensor>(), phi::CPUPlace(), &t1);
+    framework::TensorCopySync(
+        var2.Get<phi::DenseTensor>(), phi::CPUPlace(), &t2);
+  } else {
+    auto& s1 = var1.Get<phi::SelectedRows>();
+    auto& s2 = var2.Get<phi::SelectedRows>();
+
+    if (s1.height() != s2.height()) {
+      return false;
+    }
+
+    if (s1.rows().size() != s2.rows().size()) {
+      return false;
+    }
+
+    auto row1_data = s1.rows().data();
+    auto row2_data = s2.rows().data();
+    if (std::memcmp(
+            row1_data, row2_data, s1.rows().size() * sizeof(*row1_data)) != 0) {
+      return false;
+    }
+
+    framework::TensorCopySync(
+        var1.Get<phi::SelectedRows>().value(), phi::CPUPlace(), &t1);
+    framework::TensorCopySync(
+        var2.Get<phi::SelectedRows>().value(), phi::CPUPlace(), &t2);
+  }
+
+  if (t1.type() != t2.type() || t1.dims() != t2.dims()) {
+    return false;
+  }
+
+  auto* t1_p = t1.data();
+  auto* t2_p = t2.data();
+  return std::memcmp(
+             t1_p,
+             t2_p,
+             t1.numel() * framework::SizeOfType(
+                              framework::TransToProtoVarType(t1.dtype()))) == 0;
+}
+
+template <typename T>
+static framework::Variable RandomTensor(const phi::DDim& dims,
+                                        const phi::Place& place,
+                                        int low = -10,
+                                        int high = 10) {
+  phi::DenseTensor cpu_tensor;
+  cpu_tensor.Resize(dims);
+  auto* ptr = cpu_tensor.mutable_data<T>(phi::CPUPlace());
+  std::uniform_int_distribution<int> dist(low, high);
+  std::random_device rd;
+  std::mt19937 engine(rd());
+  for (int64_t i = 0; i < cpu_tensor.numel(); ++i) {
+    ptr[i] = dist(engine);
+  }
+
+  framework::Variable ret;
+  framework::TensorCopySync(
+      cpu_tensor, place, ret.GetMutable<phi::DenseTensor>());
+  return ret;
+}
+
+template <typename T>
+static framework::Variable RandomSelectedRows(phi::DDim dims,
+                                              const phi::Place& place,
+                                              int64_t row_number,
+                                              int low = -10,
+                                              int high = 10) {
+  auto height = dims[0];
+  dims[0] = row_number;
+
+  framework::Variable ret;
+  auto* sr = ret.GetMutable<phi::SelectedRows>();
+  auto tensor_var = RandomTensor<T>(dims, place, low, high);
+  sr->mutable_value()->ShareDataWith(
+      tensor_var.template Get<phi::DenseTensor>());
+  sr->set_height(height);
+  sr->mutable_rows()->resize(row_number);
+  auto* row_data = sr->mutable_rows()->data();
+  std::uniform_int_distribution<int64_t> dist(0, height - 1);
+  std::random_device rd;
+  std::mt19937 engine(rd());
+  for (int64_t i = 0; i < dims[0]; ++i) {
+    row_data[i] = dist(engine);
+  }
+  return ret;
+}
+
+static std::unique_ptr<GradientAccumulator> CreateAccumulator(
+    const std::shared_ptr<VariableWrapper>& var, bool sort_gradient) {
+  if (sort_gradient) {  // NOLINT
+    return std::unique_ptr<GradientAccumulator>(
+        new SortedGradientAccumulator(var.get()));
+  } else {
+    return std::unique_ptr<GradientAccumulator>(
+        new EagerGradientAccumulator(var.get()));
+  }
+}
+
+static void TestGradientAccumulatorTestUnchangeInput(const phi::Place& place,
+                                                     bool sort_gradient) {
+  phi::DDim dim{10, 20};
+  int64_t maximum_row_number = 100;
+
+  std::uniform_int_distribution<int64_t> dist(1, maximum_row_number);
+  int seed = 0;
+  {
+    std::random_device rd;
+    seed = static_cast<int>(rd());
+  }
+
+  std::mt19937 engine(seed);
+
+  auto create_var = [&](bool use_tensor) {
+    if (use_tensor) {  // NOLINT
+      return RandomTensor<float>(dim, place);
+    } else {
+      return RandomSelectedRows<float>(dim, place, dist(engine));
+    }
+  };
+
+  std::vector<bool> use_tensors = {false, true};
+
+  for (auto use_tensor1 : use_tensors) {
+    for (auto use_tensor2 : use_tensors) {
+      /** g_accum1 && g_accum2: has not been initialized
+       *    test accumulate on this graph
+       */
+      auto g_var1 = std::make_shared<VariableWrapper>("g_var1");
+      g_var1->SetOverriddenStopGradient(false);
+      auto g_accum1 = CreateAccumulator(g_var1, sort_gradient);
+      g_accum1->IncreaseRefCnt();
+      g_accum1->IncreaseRefCnt();
+
+      auto g_var2 = std::make_shared<VariableWrapper>("g_var2");
+      g_var2->SetOverriddenStopGradient(false);
+      auto g_accum2 = CreateAccumulator(g_var2, sort_gradient);
+      g_accum2->IncreaseRefCnt();
+      g_accum2->IncreaseRefCnt();
+
+      auto var1 = create_var(use_tensor1);
+      auto var_wrapper1_1 = std::make_shared<VariableWrapper>("tmp1_1");
+      auto var_wrapper2_1 = std::make_shared<VariableWrapper>("tmp2_1");
+
+      ASSERT_EQ(var_wrapper1_1->IsEmpty(), true);
+      CopyVar(var1, var_wrapper1_1->MutableVar());
+      ASSERT_EQ(var_wrapper1_1->IsEmpty(), false);
+
+      ASSERT_EQ(var_wrapper2_1->IsEmpty(), true);
+      CopyVar(var1, var_wrapper2_1->MutableVar());
+      ASSERT_EQ(var_wrapper2_1->IsEmpty(), false);
+
+      auto var2 = create_var(use_tensor2);
+      auto var_wrapper1_2 = std::make_shared<VariableWrapper>("tmp1_2");
+      auto var_wrapper2_2 = std::make_shared<VariableWrapper>("tmp2_2");
+      CopyVar(var2, var_wrapper1_2->MutableVar());
+      CopyVar(var2, var_wrapper2_2->MutableVar());
+
+      // g_accum1: inner_var_ = var1 + var2
+      g_accum1->SumGrad(var_wrapper1_1, 0, false);
+      g_accum1->SumGrad(var_wrapper1_2, 1, false);
+      ASSERT_EQ(g_accum1->CurCnt(), g_accum1->RefCnt());
+      ASSERT_TRUE(g_accum1->SumGradCompleted());
+      // g_accum1: inner_var_ -> var_
+      g_accum1->AccumulateGrad();
+
+      // g_accum2: inner_var_ = var1 + var2
+      g_accum2->SumGrad(var_wrapper2_1, 0, true);
+      g_accum2->SumGrad(var_wrapper2_2, 1, true);
+      ASSERT_EQ(g_accum2->CurCnt(), g_accum2->RefCnt());
+      ASSERT_TRUE(g_accum2->SumGradCompleted());
+      // g_accum2: inner_var_ -> var_
+      g_accum2->AccumulateGrad();
+
+      ASSERT_TRUE(IsEqualVar(var_wrapper2_1->Var(), var1));
+      ASSERT_TRUE(IsEqualVar(var_wrapper2_2->Var(), var2));
+      ASSERT_TRUE(IsEqualVar(g_var1->Var(), g_var2->Var()));
+
+      /** g_accum3 && g_accum4: has been initialized
+       *    test accumulate on previous graph
+       */
+      auto var3 = create_var(use_tensor1);
+      auto var_wrapper3_3 = std::make_shared<VariableWrapper>("tmp1_3");
+      auto var_wrapper4_3 = std::make_shared<VariableWrapper>("tmp2_3");
+      var_wrapper3_3->SetOverriddenStopGradient(false);
+      var_wrapper4_3->SetOverriddenStopGradient(false);
+      CopyVar(var3, var_wrapper3_3->MutableVar());
+      CopyVar(var3, var_wrapper4_3->MutableVar());
+
+      auto g_accum3 = CreateAccumulator(var_wrapper3_3, sort_gradient);
+      g_accum3->IncreaseRefCnt();
+      auto g_accum4 = CreateAccumulator(var_wrapper4_3, sort_gradient);
+      g_accum4->IncreaseRefCnt();
+
+      auto var4 = create_var(use_tensor2);
+      auto var_wrapper3_4 = std::make_shared<VariableWrapper>("tmp1_4");
+      auto var_wrapper4_4 = std::make_shared<VariableWrapper>("tmp2_4");
+      CopyVar(var4, var_wrapper3_4->MutableVar());
+      CopyVar(var4, var_wrapper4_4->MutableVar());
+
+      g_accum3->SumGrad(var_wrapper3_4, 0, false);
+      ASSERT_TRUE(g_accum3->SumGradCompleted());
+      // g_accum4: var_(var_wrapper3_3) + inner_var_ -> var_
+      g_accum3->AccumulateGrad();
+
+      g_accum4->SumGrad(var_wrapper4_4, 0, false);
+      ASSERT_TRUE(g_accum4->SumGradCompleted());
+      // g_accum4: var_(var_wrapper4_3) + inner_var_ -> var_
+      g_accum4->AccumulateGrad();
+
+      ASSERT_TRUE(IsEqualVar(var_wrapper3_3->Var(), var_wrapper4_3->Var()));
+    }
+  }
+}
+
+TEST(test_gradient_accumulator, test_unchange_input) {
+  for (auto sort_gradient : {false, true}) {
+    TestGradientAccumulatorTestUnchangeInput(phi::CPUPlace(), sort_gradient);
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+    TestGradientAccumulatorTestUnchangeInput(phi::GPUPlace(0), sort_gradient);
+#endif
+  }
+}
+
+}  // namespace imperative
+}  // namespace paddle

test/cpp/imperative/test_group.cc

@@ -0,0 +1,186 @@
+// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <sstream>
+#include <string>
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/imperative/reducer.h"
+
+#include "paddle/fluid/framework/convert_utils.h"
+#include "paddle/fluid/framework/tensor_util.h"
+#include "paddle/phi/core/utils/data_type.h"
+
+namespace paddle {
+namespace imperative {
+
+TEST(TestGroup, TestPrintGroupMessage) {
+  Group group;
+  std::stringstream stream1, stream2;
+  stream1 << group;
+  ASSERT_STREQ(stream1.str().c_str(),
+               "numel: 0 ;is_sparse: 0 ;var number: 0\n[]\n");
+
+  std::vector<size_t> vars;
+  size_t vars_num = 102;
+  for (size_t i = 0; i < vars_num; ++i) {
+    vars.push_back(i);
+  }
+  group.variable_indices_ = vars;
+  group.all_length_ = 102;
+  group.is_sparse_ = false;
+
+  std::string head = "numel: 102 ;is_sparse: 0 ;var number: 102\n";
+  head = head + "[";
+  auto begin = vars.begin();
+  auto end = vars.end();
+  for (int i = 0; begin != end && i < 100; ++i, ++begin) {
+    if (i > 0) head += ' ';
+    head += std::to_string(*begin);
+  }
+  if (begin != end) {
+    head += " ...";
+  }
+  head += "]\n";
+  stream2 << group;
+  ASSERT_STREQ(stream2.str().c_str(), head.c_str());
+}
+
+template <typename T, typename Place>
+void GroupConcatSplit(Place place, size_t size) {
+  phi::CPUPlace cpu_place;
+  Group group;
+
+  // [[0.0], [0.0, 1.0], [0.0, 1.0, 2.0] .. ]
+  std::vector<framework::Variable> vars;
+  vars.resize(size);
+  for (size_t i = 0; i < size; ++i) {
+    auto len = i + 1;
+    auto* tensor = vars[i].GetMutable<phi::DenseTensor>();
+    tensor->Resize({static_cast<int64_t>(len)});
+    auto* data = tensor->mutable_data<T>(place);
+
+    std::vector<T> value;
+    for (size_t j = 0; j < len; ++j) {
+      value.push_back(static_cast<T>(1.0 * j));  // NOLINT
+    }
+
+    if (std::is_same<Place, phi::GPUPlace>::value) {
+#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
+      paddle::memory::Copy(
+          place, data, cpu_place, value.data(), sizeof(T) * value.size(), 0);
+#endif
+    } else {
+      paddle::memory::Copy(
+          place, data, cpu_place, value.data(), sizeof(T) * value.size());
+    }
+
+    phi::DenseTensor tmp;
+    tmp.ShareDataWith(*tensor).Resize({static_cast<int64_t>(len)});
+    group.dense_tensors_.push_back(std::move(tmp));
+    group.all_length_ += static_cast<int64_t>(len);
+    group.dtype_ = framework::TransToProtoVarType(tensor->dtype());
+  }
+
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* dev_ctx = pool.Get(place);
+
+  {  // concat
+    auto* tensor = group.dense_contents_.GetMutable<phi::DenseTensor>();
+    tensor->Resize(common::make_ddim({group.all_length_}))
+        .mutable_data(place, phi::TransToPhiDataType(group.dtype_));
+    group.ConcatTensors(*dev_ctx);
+
+    group.DivNRanks(*dev_ctx, 1);
+
+    phi::DenseTensor tmp;
+    framework::TensorCopySync(*tensor, cpu_place, &tmp);
+    auto* data = tmp.data<T>();
+    size_t offset = 0;
+    for (size_t i = 0; i < size; ++i) {
+      auto len = i + 1;
+      for (size_t j = 0; j < len; ++j) {
+        EXPECT_EQ(data[offset + j], static_cast<T>(1.0 * j));
+        // [[-0.0], [-0.0, -1.0], [-0.0, -1.0, -2.0] .. ]
+        data[offset + j] = -data[offset + j];
+      }
+      offset += len;
+    }
+    framework::TensorCopySync(tmp, place, tensor);
+  }
+
+  {  // split
+    group.SplitTensors(*dev_ctx);
+    for (size_t i = 0; i < size; ++i) {
+      auto len = i + 1;
+      auto& tensor = group.dense_tensors_[i];
+      phi::DenseTensor tmp;
+      framework::TensorCopySync(tensor, cpu_place, &tmp);
+      auto* data = tmp.data<T>();
+
+      for (size_t j = 0; j < len; ++j) {
+        EXPECT_EQ(data[j], static_cast<T>(-1.0 * j));
+      }
+    }
+  }
+}
+
+#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
+TEST(TestGroup, TestConcatSplit) {
+  phi::GPUPlace cuda_place(0);
+  phi::CPUPlace cpu_place;
+
+  int size = 3;
+  GroupConcatSplit<float>(cpu_place, size);
+  GroupConcatSplit<double>(cpu_place, size);
+  GroupConcatSplit<phi::dtype::float16>(cpu_place, size);
+
+  GroupConcatSplit<float>(cuda_place, size);
+  GroupConcatSplit<double>(cuda_place, size);
+  GroupConcatSplit<phi::dtype::float16>(cuda_place, size);
+
+  size = 15;
+  GroupConcatSplit<float>(cpu_place, size);
+  GroupConcatSplit<double>(cpu_place, size);
+  GroupConcatSplit<phi::dtype::float16>(cpu_place, size);
+
+  GroupConcatSplit<float>(cuda_place, size);
+  GroupConcatSplit<double>(cuda_place, size);
+  GroupConcatSplit<phi::dtype::float16>(cuda_place, size);
+}
+
+TEST(TestGroup, TestConcatSplitException) {
+  phi::GPUPinnedPlace place;
+
+  int size = 3;
+  ASSERT_ANY_THROW(GroupConcatSplit<float>(place, size));
+}
+#endif
+
+#if defined(PADDLE_WITH_XPU_BKCL)
+TEST(TestGroup, TestXPUConcatSplit) {
+  phi::XPUPlace xpu_place(0);
+  phi::CPUPlace cpu_place;
+
+  int size = 3;
+  GroupConcatSplit<float>(cpu_place, size);
+  GroupConcatSplit<float>(xpu_place, size);
+
+  size = 15;
+  GroupConcatSplit<float>(cpu_place, size);
+  GroupConcatSplit<float>(xpu_place, size);
+}
+#endif
+}  // namespace imperative
+}  // namespace paddle

test/cpp/imperative/test_hooks.cc

@@ -0,0 +1,287 @@
+// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/common/flags.h"
+#include "paddle/fluid/framework/op_registry.h"
+#include "paddle/fluid/imperative/basic_engine.h"
+#include "paddle/fluid/imperative/hooks.h"
+#include "paddle/fluid/imperative/tracer.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/memory/memcpy.h"
+
+PD_DECLARE_KERNEL(add, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(add_grad, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(matmul_with_flatten, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(matmul_with_flatten_grad, CPU, ALL_LAYOUT);
+
+COMMON_DECLARE_bool(sort_sum_gradient);
+
+namespace paddle {
+namespace imperative {
+
+using vb_vector = std::vector<std::shared_ptr<imperative::VarBase>>;
+using var_pair = std::pair<std::string, vb_vector>;
+
+std::shared_ptr<imperative::VariableWrapper> DoubleHook(
+    const std::shared_ptr<imperative::VariableWrapper>& var) {
+  // 1. create out var
+  auto out_var = std::make_shared<imperative::VariableWrapper>(var->Name());
+  out_var->SetType(var->Type());
+  out_var->SetDataType(var->DataType());
+  out_var->SetForwardDataType(var->ForwardDataType());
+  out_var->InnerSetOverriddenStopGradient(var->InnerOverriddenStopGradient());
+
+  // 2. get input and output var's tensor
+  auto* out_tensor = out_var->MutableVar()->GetMutable<phi::DenseTensor>();
+  auto& tensor = var->Var().Get<phi::DenseTensor>();
+  out_tensor->Resize(tensor.dims());
+
+  // 3. double calc
+  auto* data = tensor.data<float>();
+  auto* out_data = out_tensor->mutable_data<float>(phi::CPUPlace());
+  for (int64_t i = 0; i < out_tensor->numel(); ++i) {
+    out_data[i] = data[i] * 2.0;  // NOLINT
+  }
+
+  return out_var;
+}
+
+TEST(TestHooks, TestGradVarLeafBackwardHook) {
+  // 1. prepare
+  Tracer tracer;
+  std::shared_ptr<VarBase> x(new VarBase(true, "x"));
+  std::shared_ptr<VarBase> y(new VarBase(true, "y"));
+  std::shared_ptr<VarBase> out(new VarBase(true, "out"));
+  x->SetOverriddenStopGradient(false);
+  y->SetOverriddenStopGradient(false);
+
+  phi::CPUPlace place;
+  std::vector<float> src_data(10, 2.0);
+  std::vector<int64_t> x_dims = {2, 5};
+  std::vector<int64_t> y_dims = {5, 2};
+
+  auto* x_tensor = x->MutableVar()->GetMutable<phi::DenseTensor>();
+  auto* y_tensor = y->MutableVar()->GetMutable<phi::DenseTensor>();
+
+  x_tensor->Resize(common::make_ddim(x_dims));
+  auto* mutable_x = x_tensor->mutable_data<float>(place);
+  memory::Copy(place,
+               mutable_x,
+               place,
+               src_data.data(),
+               sizeof(float) * src_data.size());
+
+  y_tensor->Resize(common::make_ddim(y_dims));
+  auto* mutable_y = y_tensor->mutable_data<float>(place);
+  memory::Copy(place,
+               mutable_y,
+               place,
+               src_data.data(),
+               sizeof(float) * src_data.size());
+
+  var_pair x_pair = var_pair("X", vb_vector(1, x));
+  var_pair y_pair = var_pair("Y", vb_vector(1, y));
+  var_pair out_pair = var_pair("Out", vb_vector(1, out));
+
+  NameVarBaseMap ins = {x_pair, y_pair};
+  NameVarBaseMap outs = {out_pair};
+  framework::AttributeMap mul_attr_map;
+  mul_attr_map["use_onednn"] = false;
+
+  // add VariableWrapper hook
+  x->GradVarBase()->AddVariableWrapperHook(
+      std::make_shared<imperative::CppVariableWrapperHook>(DoubleHook));
+
+  // add Void hook
+  int64_t hook_value = 0;
+  x->GradVarBase()->AddVoidHook(
+      std::make_shared<std::function<void()>>([&]() { hook_value = 10; }));
+
+  // 2. forward
+  tracer.TraceOp<VarBase>("mul", ins, outs, mul_attr_map, place, true);
+
+  ASSERT_EQ(x->GradVarBase()->GradOpNum(), 0UL);
+  ASSERT_EQ(y->GradVarBase()->GradOpNum(), 0UL);
+  ASSERT_EQ(out->GradVarBase()->GradOpNum(), 1UL);
+
+  // 3. backward
+  std::vector<std::shared_ptr<imperative::VarBase>> tensors{out};
+  std::vector<std::shared_ptr<imperative::VarBase>> grad_tensors{nullptr};
+  BasicEngine engine;
+  engine.Init(tensors, grad_tensors);
+  engine.Execute();
+
+  // verify VariableWrapper hook result
+  phi::DenseTensor x_grad;
+  framework::TensorCopySync(
+      x->GradVar().Get<phi::DenseTensor>(), place, &x_grad);
+  for (int i = 0; i < x_grad.numel(); ++i) {
+    ASSERT_EQ(x_grad.data<float>()[i], 8.0);
+  }
+  // verify Void hook result
+  ASSERT_EQ(hook_value, 10);
+
+  phi::DenseTensor y_grad;
+  framework::TensorCopySync(
+      y->GradVar().Get<phi::DenseTensor>(), place, &y_grad);
+
+  for (int i = 0; i < y_grad.numel(); ++i) {
+    ASSERT_EQ(y_grad.data<float>()[i], 4.0);
+  }
+}
+
+void GradVarLeafBackwardHookWithGradAccumulatedTest() {
+  // 1. prepare
+  Tracer tracer;
+  std::shared_ptr<VarBase> x(new VarBase(true, "x"));
+  std::shared_ptr<VarBase> y(new VarBase(true, "y"));
+  std::shared_ptr<VarBase> z(new VarBase(true, "z"));
+  std::shared_ptr<VarBase> out_xy(new VarBase(true, "out_xy"));
+  std::shared_ptr<VarBase> out_xz(new VarBase(true, "out_xz"));
+  std::shared_ptr<VarBase> out(new VarBase(true, "out"));
+  x->SetOverriddenStopGradient(false);
+  y->SetOverriddenStopGradient(false);
+  z->SetOverriddenStopGradient(false);
+
+  phi::CPUPlace place;
+  std::vector<float> src_data(10, 2.0);
+  std::vector<int64_t> x_dims = {2, 5};
+  std::vector<int64_t> y_dims = {5, 2};
+  std::vector<int64_t> z_dims = {5, 2};
+
+  auto* x_tensor = x->MutableVar()->GetMutable<phi::DenseTensor>();
+  auto* y_tensor = y->MutableVar()->GetMutable<phi::DenseTensor>();
+  auto* z_tensor = z->MutableVar()->GetMutable<phi::DenseTensor>();
+
+  x_tensor->Resize(common::make_ddim(x_dims));
+  auto* mutable_x = x_tensor->mutable_data<float>(place);
+  memory::Copy(place,
+               mutable_x,
+               place,
+               src_data.data(),
+               sizeof(float) * src_data.size());
+
+  y_tensor->Resize(common::make_ddim(y_dims));
+  auto* mutable_y = y_tensor->mutable_data<float>(place);
+  memory::Copy(place,
+               mutable_y,
+               place,
+               src_data.data(),
+               sizeof(float) * src_data.size());
+
+  z_tensor->Resize(common::make_ddim(z_dims));
+  auto* mutable_z = z_tensor->mutable_data<float>(place);
+  memory::Copy(place,
+               mutable_z,
+               place,
+               src_data.data(),
+               sizeof(float) * src_data.size());
+
+  // add VariableWrapper hook
+  x->GradVarBase()->AddVariableWrapperHook(
+      std::make_shared<imperative::CppVariableWrapperHook>(DoubleHook));
+
+  // add Void hook
+  int64_t hook_value = 0;
+  x->GradVarBase()->AddVoidHook(
+      std::make_shared<std::function<void()>>([&]() { hook_value = 100; }));
+
+  // 2. forward
+  var_pair x_pair = var_pair("X", vb_vector(1, x));
+  var_pair y_pair = var_pair("Y", vb_vector(1, y));
+  var_pair out_xy_pair = var_pair("Out", vb_vector(1, out_xy));
+  NameVarBaseMap ins = {x_pair, y_pair};
+  NameVarBaseMap outs = {out_xy_pair};
+  framework::AttributeMap mul_attr_map;
+  mul_attr_map["use_onednn"] = false;
+  tracer.TraceOp<VarBase>("mul", ins, outs, mul_attr_map, place, true);
+
+  var_pair z_pair = var_pair("Y", vb_vector(1, z));
+  var_pair out_xz_pair = var_pair("Out", vb_vector(1, out_xz));
+  ins = {x_pair, z_pair};
+  outs = {out_xz_pair};
+  tracer.TraceOp<VarBase>("mul", ins, outs, mul_attr_map, place, true);
+
+  var_pair xy_pair = var_pair("X", vb_vector(1, out_xy));
+  var_pair xz_pair = var_pair("Y", vb_vector(1, out_xz));
+  var_pair out_pair = var_pair("Out", vb_vector(1, out));
+  ins = {xy_pair, xz_pair};
+  outs = {out_pair};
+  framework::AttributeMap add_attr_map;
+  tracer.TraceOp<VarBase>(
+      "elementwise_add", ins, outs, add_attr_map, place, true);
+
+  ASSERT_EQ(x->GradVarBase()->GradOpNum(), 0UL);
+  ASSERT_EQ(y->GradVarBase()->GradOpNum(), 0UL);
+  ASSERT_EQ(z->GradVarBase()->GradOpNum(), 0UL);
+  ASSERT_EQ(out->GradVarBase()->GradOpNum(), 1UL);
+
+  // 3. backward
+  std::vector<std::shared_ptr<imperative::VarBase>> tensors{out};
+  std::vector<std::shared_ptr<imperative::VarBase>> grad_tensors{nullptr};
+  BasicEngine engine;
+  engine.Init(tensors, grad_tensors);
+  engine.Execute();
+
+  // verify VariableWrapper hook result
+  phi::DenseTensor x_grad;
+  framework::TensorCopySync(
+      x->GradVar().Get<phi::DenseTensor>(), place, &x_grad);
+  for (int i = 0; i < x_grad.numel(); ++i) {
+    ASSERT_EQ(x_grad.data<float>()[i], 16.0);
+  }
+  // verify Void hook result
+  ASSERT_EQ(hook_value, 100);
+
+  phi::DenseTensor y_grad;
+  framework::TensorCopySync(
+      y->GradVar().Get<phi::DenseTensor>(), place, &y_grad);
+
+  for (int i = 0; i < y_grad.numel(); ++i) {
+    ASSERT_EQ(y_grad.data<float>()[i], 4.0);
+  }
+
+  phi::DenseTensor z_grad;
+  framework::TensorCopySync(
+      z->GradVar().Get<phi::DenseTensor>(), place, &z_grad);
+
+  for (int i = 0; i < z_grad.numel(); ++i) {
+    ASSERT_EQ(z_grad.data<float>()[i], 4.0);
+  }
+}
+
+TEST(TestHooks, TestGradVarLeafBackwardHookWithGradAccumulated) {
+  GradVarLeafBackwardHookWithGradAccumulatedTest();
+}
+
+TEST(TestHooks, TestGradVarLeafBackwardHookWithSortedGradAccumulated) {
+  FLAGS_sort_sum_gradient = true;
+  GradVarLeafBackwardHookWithGradAccumulatedTest();
+  FLAGS_sort_sum_gradient = false;
+}
+
+}  // namespace imperative
+}  // namespace paddle
+
+USE_OP_ITSELF(mul);
+USE_OP_ITSELF(mul_grad);
+USE_OP_ITSELF(elementwise_add);
+USE_OP_ITSELF(elementwise_add_grad);

test/cpp/imperative/test_layer.cc

@@ -0,0 +1,420 @@
+// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//
+// Created by Jiabin on 2019-08-16.
+//
+
+#include <paddle/fluid/framework/op_registry.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "paddle/common/macros.h"
+#include "paddle/fluid/imperative/execution_context.h"
+#include "paddle/fluid/imperative/infer_shape_context.h"
+#include "paddle/fluid/imperative/infer_var_type_context.h"
+#include "paddle/fluid/imperative/layer.h"
+
+namespace paddle {
+namespace imperative {
+
+using vb_vector = std::vector<std::shared_ptr<imperative::VarBase>>;
+
+using var_pair = std::pair<std::string, vb_vector>;
+
+extern void TestSetForwardDataTypeOfGradVarsEager(
+    const NameVarMap<egr::EagerVariable>& outs);
+template <typename VarType>
+class TestRuntimeInferVarTypeContext
+    : public RuntimeInferVarTypeContext<VarType> {
+ public:
+  TestRuntimeInferVarTypeContext(
+      const NameVarMap<VarType>& inputs,
+      const NameVarMap<VarType>& outputs,
+      const framework::AttributeMap& attrs_map,
+      const framework::AttributeMap& default_attrs_map)
+      : RuntimeInferVarTypeContext<VarType>(
+            inputs, outputs, attrs_map, default_attrs_map) {}
+
+  bool HasVar(const std::string& name) const {
+    return RuntimeInferVarTypeContext<VarType>::HasVar(name);
+  }
+
+  const std::vector<std::string>& InputVars(const std::string& name) const {
+    return RuntimeInferVarTypeContext<VarType>::InputVars(name);
+  }
+
+  const std::vector<std::string>& OutputVars(const std::string& name) const {
+    return RuntimeInferVarTypeContext<VarType>::OutputVars(name);
+  }
+
+  framework::proto::VarType::Type GetVarType(const std::string& name) const {
+    return RuntimeInferVarTypeContext<VarType>::GetVarType(name);
+  }
+
+  void SetVarType(const std::string& name,
+                  framework::proto::VarType::Type type) {
+    RuntimeInferVarTypeContext<VarType>::SetVarType(name, type);
+  }
+
+  framework::proto::VarType::Type GetVarDataType(
+      const std::string& name) const {
+    return RuntimeInferVarTypeContext<VarType>::GetVarDataType(name);
+  }
+
+  void SetVarDataType(const std::string& name,
+                      framework::proto::VarType::Type type) {
+    RuntimeInferVarTypeContext<VarType>::SetVarDataType(name, type);
+  }
+
+  std::vector<framework::proto::VarType::Type> GetVarDataTypes(
+      const std::string& name) const {
+    return RuntimeInferVarTypeContext<VarType>::GetVarDataTypes(name);
+  }
+
+  void SetVarDataTypes(
+      const std::string& name,
+      const std::vector<framework::proto::VarType::Type>& multiple_data_type) {
+    RuntimeInferVarTypeContext<VarType>::SetVarDataTypes(name,
+                                                         multiple_data_type);
+  }
+
+  std::vector<int64_t> GetVarShape(const std::string& name) const {
+    return RuntimeInferVarTypeContext<VarType>::GetVarShape(name);
+  }
+
+  void SetVarShape(const std::string& name, const std::vector<int64_t>& dims) {
+    RuntimeInferVarTypeContext<VarType>::SetVarShape(name, dims);
+  }
+
+  int32_t GetVarLoDLevel(const std::string& name) const {
+    return RuntimeInferVarTypeContext<VarType>::GetVarLoDLevel(name);
+  }
+
+  void SetVarLoDLevel(const std::string& name, int32_t lod_level) {
+    RuntimeInferVarTypeContext<VarType>::SetVarLoDLevel(name, lod_level);
+  }
+};
+
+TEST(test_layer, test_runtime_context) {
+  std::shared_ptr<imperative::VarBase> vin(
+      new imperative::VarBase(false, "vin"));
+  std::shared_ptr<imperative::VarBase> vin_b(
+      new imperative::VarBase(false, "vin_b"));
+  std::shared_ptr<imperative::VarBase> vout(
+      new imperative::VarBase(false, "vout"));
+  std::shared_ptr<imperative::VarBase> vout_b(
+      new imperative::VarBase(false, "vout_b"));
+  var_pair in_pair = var_pair("X", {vin, vin_b});
+  var_pair out_pair = var_pair("Out", {vout, vout_b});
+  imperative::NameVarBaseMap ins = {in_pair};
+  imperative::NameVarBaseMap outs = {out_pair};
+  framework::AttributeMap attrs;
+
+  auto* ctx =
+      new imperative::TestRuntimeInferVarTypeContext<imperative::VarBase>(
+          ins, outs, attrs, {});
+
+  ASSERT_TRUE(ctx->HasInput("X"));
+  ASSERT_TRUE(ctx->HasOutput("Out"));
+
+  ASSERT_EQ(2u, ctx->InputSize("X"));
+  ASSERT_EQ("vin", ctx->InputVarName("X", 0));
+
+  ASSERT_TRUE(
+      ctx->InputTypeAnyOf("X", framework::proto::VarType::DENSE_TENSOR));
+  ASSERT_TRUE(
+      ctx->InputTypeAllOf("X", framework::proto::VarType::DENSE_TENSOR));
+
+  ASSERT_EQ(framework::proto::VarType::DENSE_TENSOR, ctx->GetInputType("X"));
+  ASSERT_EQ(framework::proto::VarType::FP32, ctx->GetInputDataType("X"));
+
+  ctx->SyncTypeAndDataType("X", "Out");
+
+  // Remove DataType check, because it doesn't make sense of set dtype in
+  // dygraph
+
+  ASSERT_EQ(framework::proto::VarType::DENSE_TENSOR, ctx->GetOutputType("Out"));
+
+  ctx->SetOutputType(
+      "Out", framework::proto::VarType::SELECTED_ROWS, framework::ALL_ELEMENTS);
+  ctx->SetOutputType("Out", framework::proto::VarType::DENSE_TENSOR_ARRAY);
+  ASSERT_EQ(framework::proto::VarType::DENSE_TENSOR_ARRAY, vout->Type());
+  ASSERT_EQ(framework::proto::VarType::SELECTED_ROWS, vout_b->Type());
+
+  ctx->SetOutputDataType(
+      "Out", framework::proto::VarType::FP64, framework::ALL_ELEMENTS);
+  ctx->SetOutputDataType("Out", framework::proto::VarType::INT8);
+
+  // Remove DataType check, because it doesn't make sense of set dtype in
+  // dygraph
+
+  // no throw, but do nothing
+  ASSERT_NO_THROW(
+      ctx->InsertVar("vout", framework::proto::VarType::DENSE_TENSOR));
+  ASSERT_EQ(framework::proto::VarType::DENSE_TENSOR_ARRAY, vout->Type());
+
+  ASSERT_ANY_THROW(ctx->HasVar("vin"));
+  ASSERT_ANY_THROW(ctx->InputVars("X"));
+  ASSERT_ANY_THROW(ctx->OutputVars("Out"));
+  ASSERT_ANY_THROW(ctx->GetVarType("vin"));
+  ASSERT_ANY_THROW(
+      ctx->SetVarType("vin", framework::proto::VarType::DENSE_TENSOR));
+  ASSERT_ANY_THROW(ctx->GetVarDataType("vin"));
+  ASSERT_ANY_THROW(
+      ctx->SetVarDataType("vout", framework::proto::VarType::FP32));
+
+  ASSERT_ANY_THROW(ctx->GetVarDataTypes("vin"));
+  std::vector<framework::proto::VarType::Type> NullType;
+  ASSERT_ANY_THROW(ctx->SetVarDataTypes("vin", NullType));
+  ASSERT_ANY_THROW(ctx->GetVarShape("vin"));
+  ASSERT_ANY_THROW(ctx->SetVarShape("vin", {}));
+  ASSERT_ANY_THROW(ctx->GetVarLoDLevel("vin"));
+  ASSERT_ANY_THROW(ctx->SetVarLoDLevel("vin", 2));
+
+  ASSERT_TRUE(ctx->IsDygraph());
+}
+
+PADDLE_API std::string LayerDebugString(const std::string& op_type,
+                                        const NameVarBaseMap& ins,
+                                        const NameVarBaseMap& outs);
+
+TEST(test_layer, test_debug_string) {
+  phi::CPUPlace place;
+  std::shared_ptr<imperative::VarBase> vin(
+      new imperative::VarBase(false, "vin"));
+  var_pair in_pair = var_pair("X", vb_vector(1, vin));
+
+  auto test_func = [&](std::shared_ptr<imperative::VarBase>& vout) {
+    var_pair out_pair = var_pair("Out", vb_vector(1, vout));
+    imperative::NameVarBaseMap ins = {in_pair};
+    imperative::NameVarBaseMap outs = {out_pair};
+    return LayerDebugString("test_op", ins, outs);
+  };
+
+  // 1. test null
+  std::shared_ptr<imperative::VarBase> null_out(nullptr);
+  std::string res_null = test_func(null_out);
+  ASSERT_TRUE(res_null.find("NULL") != std::string::npos);
+
+  // 2. test uninit var
+  std::shared_ptr<imperative::VarBase> un_init_out(
+      new imperative::VarBase(false, "un_init_out"));
+  std::string res_un_init = test_func(un_init_out);
+  ASSERT_TRUE(res_un_init.find("NOT_INITED_VAR") != std::string::npos);
+
+  // 3. test unresolved type
+  std::shared_ptr<imperative::VarBase> ut_out(
+      new imperative::VarBase(false, "ut_out"));
+  ut_out->MutableVar()->GetMutable<phi::TensorArray>();
+  std::string res_ut = test_func(ut_out);
+  ASSERT_TRUE(res_ut.find("UNRESOLVED_TYPE") != std::string::npos);
+
+  // 4. test uninit lod tensor
+  std::shared_ptr<imperative::VarBase> dense_tensor(
+      new imperative::VarBase(false, "dense_tensor"));
+  auto tensor_l = dense_tensor->MutableVar()->GetMutable<phi::DenseTensor>();
+  std::string res_ui_dense_t = test_func(dense_tensor);
+  ASSERT_TRUE(res_ui_dense_t.find("NOT_INITED") != std::string::npos);
+
+  // 5. test init lod tensor
+  tensor_l->mutable_data<float>(place);
+  std::string res_lod_t = test_func(dense_tensor);
+  ASSERT_TRUE(res_lod_t.find("DenseTensor") != std::string::npos);
+
+  // 6. test uninit selected rows
+  std::shared_ptr<imperative::VarBase> selected_rows(
+      new imperative::VarBase(false, "selected_rows"));
+  auto tensor_sr = selected_rows->MutableVar()
+                       ->GetMutable<phi::SelectedRows>()
+                       ->mutable_value();
+  std::string res_ui_sr = test_func(selected_rows);
+  ASSERT_TRUE(res_ui_sr.find("NOT_INITED") != std::string::npos);
+
+  // 7. test init selected rows
+  tensor_sr->mutable_data<float>(place);
+  std::string res_sr = test_func(selected_rows);
+  ASSERT_TRUE(res_sr.find("SelectedRows") != std::string::npos);
+}
+
+static std::shared_ptr<imperative::GradOpNode> CreateGradNode(
+    size_t id,
+    const std::string& type,
+    const imperative::NameVarBaseMap& ins,
+    const imperative::NameVarBaseMap& outs,
+    const framework::AttributeMap& attrs,
+    const phi::Place& place) {
+  auto node = std::make_shared<imperative::GradOpNode>();
+  auto* op = &(node->emplace_back());
+  op->SetId(id);
+  op->SetPlace(place);
+  op->SetType(type);
+  op->SetAttrMap(attrs);
+  for (auto& pair : ins) {
+    std::vector<std::shared_ptr<VariableWrapper>> vars;
+    for (auto& var : pair.second) {
+      vars.emplace_back(var->SharedVar());
+    }
+    op->SetInput(pair.first, vars, false);
+  }
+
+  for (auto& pair : outs) {
+    std::vector<std::shared_ptr<VariableWrapper>> vars;
+    for (auto& var : pair.second) {
+      vars.emplace_back(var->SharedVar());
+    }
+    op->SetOutput(pair.first, vars, false);
+  }
+
+  return node;
+}
+
+TEST(test_layer, test_clear_backward_info) {
+  std::shared_ptr<imperative::VarBase> vin(
+      new imperative::VarBase(false, "vin"));
+  std::shared_ptr<imperative::VarBase> vout(
+      new imperative::VarBase(false, "vout"));
+  framework::OpDesc desc;
+  phi::CPUPlace place;
+  var_pair x_pair = var_pair("X", vb_vector(1, vin));
+  var_pair y_pair = var_pair("Y", vb_vector(1, vin));
+  var_pair out_pair = var_pair("Out", vb_vector(1, vout));
+  imperative::NameVarBaseMap ins = {x_pair, y_pair};
+  imperative::NameVarBaseMap outs = {out_pair};
+  framework::AttributeMap concat_att_map;
+  concat_att_map["axis"] = 1;
+
+  auto node = CreateGradNode(0, "mul", ins, outs, concat_att_map, place);
+  auto pending_node =
+      CreateGradNode(0, "mul", ins, outs, concat_att_map, place);
+  node->InsertGradPendingNode(pending_node);
+
+  ASSERT_EQ(node->size(), 1UL);
+  auto* op = &(node->back());
+
+  ASSERT_GT(op->GetInsMap().size(), 0UL);
+  ASSERT_GT(op->GetOutsMap().size(), 0UL);
+
+  op->ClearBackwardTrace();
+
+  ASSERT_EQ(op->GetInsMap().size(), 0UL);
+  ASSERT_EQ(op->GetOutsMap().size(), 0UL);
+}
+
+TEST(test_layer, test_varbase_basic) {
+  phi::CPUPlace place;
+  std::shared_ptr<imperative::VarBase> vin(
+      new imperative::VarBase(false, "vin"));
+  vin->MutableVar()->GetMutable<phi::DenseTensor>()->mutable_data<float>(place);
+  std::shared_ptr<imperative::VarBase> vout(vin->NewVarBase(place, false));
+  ASSERT_EQ(vout->Name(), "vin0");
+
+  std::shared_ptr<imperative::VarBase> vin_with_grad(
+      new imperative::VarBase(true, "vin"));
+  ASSERT_ANY_THROW(vin->MutableGradVar());
+  ASSERT_NO_THROW(ASSERT_TRUE(dynamic_cast<framework::Variable*>(
+                                  vin_with_grad->MutableGradVar()) != nullptr));
+  ASSERT_TRUE(dynamic_cast<framework::Variable*>(
+                  vin_with_grad->MutableGradVar()) != nullptr);
+  vin_with_grad->SetOverriddenStopGradient(false);
+  ASSERT_FALSE(vin_with_grad->OverriddenStopGradient());
+  ASSERT_NO_FATAL_FAILURE(vin_with_grad->SetPersistable(true));
+  ASSERT_FALSE(vin_with_grad->OverriddenStopGradient());
+  ASSERT_NO_FATAL_FAILURE(vin_with_grad->SetName("new_name"));
+  ASSERT_EQ(vin_with_grad->Name(), "new_name");
+}
+// TODO(jiabin): Add more ut here for layer
+
+TEST(test_layer, test_dygraph_execution_context) {
+  std::shared_ptr<imperative::VarBase> vin(
+      new imperative::VarBase(false, "vin"));
+  std::shared_ptr<imperative::VarBase> vout(
+      new imperative::VarBase(false, "vout"));
+  framework::OpDesc desc;
+  phi::CPUPlace place;
+  var_pair x_pair = var_pair("X", vb_vector(1, vin));
+  var_pair y_pair = var_pair("Y", vb_vector(1, vin));
+  var_pair out_pair = var_pair("Out", vb_vector(1, vout));
+  imperative::NameVarBaseMap ins = {x_pair, y_pair};
+  imperative::NameVarBaseMap outs = {out_pair};
+
+  framework::AttributeMap concat_att_map;
+  concat_att_map["axis"] = 1;
+
+  auto op = framework::OpRegistry::CreateOp("mul", {}, {}, {}, false);
+  phi::CPUPlace cpu_place;
+
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* dev_ctx = pool.Get(cpu_place);
+  paddle::framework::RuntimeContext ctx({}, {});
+  framework::Scope scope;
+
+  DygraphExecutionContext<imperative::VarBase> dy_exe_context(
+      *(op.get()), scope, *dev_ctx, ctx, ins, outs, concat_att_map, {});
+
+  ASSERT_EQ(dy_exe_context.InputSize("X"), 1u);
+  ASSERT_EQ(dy_exe_context.InputName("X"), "vin");
+  ASSERT_EQ(dy_exe_context.HasAttr("axis"), true);
+  auto attr_map = dy_exe_context.Attrs();
+  ASSERT_EQ(PADDLE_GET(int, attr_map["axis"]), 1);
+  ASSERT_EQ(dy_exe_context.OutputSize("Out"), 1u);
+  ASSERT_EQ(dy_exe_context.HasOutput("Out"), true);
+}
+
+TEST(test_layer, test_dygraph_infershape_context) {
+  std::shared_ptr<imperative::VarBase> vin(
+      new imperative::VarBase(false, "vin"));
+  std::shared_ptr<imperative::VarBase> vout(
+      new imperative::VarBase(false, "vout"));
+  framework::OpDesc desc;
+  phi::CPUPlace place;
+  var_pair x_pair = var_pair("X", vb_vector(1, vin));
+  var_pair y_pair = var_pair("Y", vb_vector(1, vin));
+  var_pair out_pair = var_pair("Out", vb_vector(1, vout));
+  imperative::NameVarBaseMap ins = {x_pair, y_pair};
+  imperative::NameVarBaseMap outs = {out_pair};
+
+  framework::AttributeMap concat_att_map;
+  concat_att_map["axis"] = 1;
+
+  DygraphInferShapeContext<imperative::VarBase> infer_shape_ctx(
+      &ins, &outs, &concat_att_map, {}, "dummy");
+
+  bool have_x = infer_shape_ctx.HasOutputs("Out");
+  ASSERT_EQ(have_x, true);
+  bool have_z = infer_shape_ctx.HasOutputs("Z");
+  ASSERT_EQ(have_z, false);
+}
+
+TEST(test_layer, test_inner_op_not_inited) {
+  OpBase op;
+  std::string kUnknown = "unknown";
+  ASSERT_EQ(op.Type(), kUnknown);
+  ASSERT_THROW(op.Info(), platform::EnforceNotMet);
+  ASSERT_THROW(op.InnerOp(), platform::EnforceNotMet);
+  ASSERT_THROW(op.CheckAttrs(), platform::EnforceNotMet);
+}
+
+TEST(test_layer, test_eager) {
+  imperative::NameTensorMap ins = {};
+  TestSetForwardDataTypeOfGradVarsEager(ins);
+}
+
+}  // namespace imperative
+}  // namespace paddle
+
+USE_OP_ITSELF(mul);

test/cpp/imperative/test_prepare_op.cc

@@ -0,0 +1,260 @@
+// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//
+// Created by Jiabin on 2019-08-19.
+//
+
+#include <paddle/fluid/framework/op_registry.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/framework/op_info.h"
+#include "paddle/fluid/imperative/prepared_operator.h"
+#include "paddle/fluid/imperative/type_defs.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+PD_DECLARE_KERNEL(split, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(relu, CPU, ALL_LAYOUT);
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+PD_DECLARE_KERNEL(relu, GPU, ALL_LAYOUT);
+#endif
+
+namespace paddle {
+namespace imperative {
+
+extern void TestHandleComplexGradToRealGradEager(
+    const NameVarMap<egr::EagerVariable>& outs);
+
+static framework::VariableNameMap CreateVarNameMap(
+    const framework::OpInfo& op_info,
+    const std::string& op_type,
+    const NameVarBaseMap& varbase_map,
+    bool is_input) {
+  if (op_info.proto_ == nullptr) {
+    return {};
+  }
+
+  framework::VariableNameMap result;
+
+  for (auto& var :
+       is_input ? op_info.Proto().inputs() : op_info.Proto().outputs()) {
+    auto it = varbase_map.find(var.name());
+    if (it == varbase_map.end()) {
+      PADDLE_ENFORCE_EQ(
+          var.dispensable(),
+          true,
+          common::errors::NotFound("Variable %s is not dispensable and "
+                                   "there are no such var in inputs",
+                                   var.name()));
+      result[var.name()] = {};
+    } else {
+      auto& var_vector = it->second;
+      std::vector<std::string> args;
+      args.reserve(var_vector.size());
+      for (auto& var_base : var_vector) {
+        args.emplace_back(var_base->Name());
+      }
+      result[var.name()] = std::move(args);
+    }
+  }
+  return result;
+}
+
+using vb_vector = std::vector<std::shared_ptr<imperative::VarBase>>;
+
+using var_pair = std::pair<std::string, vb_vector>;
+
+TEST(test_prepare_op, test_prepare_op) {
+  std::shared_ptr<imperative::VarBase> vin(
+      new imperative::VarBase(false, "vin"));
+  std::shared_ptr<imperative::VarBase> vout(
+      new imperative::VarBase(false, "vout"));
+  framework::OpDesc desc;
+  phi::CPUPlace place;
+  vin->MutableVar()->GetMutable<phi::DenseTensor>()->mutable_data<float>(place);
+  var_pair x_pair = var_pair("X", vb_vector(1, vin));
+  var_pair out_pair = var_pair("Out", vb_vector(1, vout));
+  imperative::NameVarBaseMap ins = {x_pair};
+  imperative::NameVarBaseMap outs = {out_pair};
+  framework::AttributeMap split_attr_map;
+  const auto& info = framework::OpInfoMap::Instance().Get("split");
+  if (info.Checker()) info.Checker()->Check(&split_attr_map);
+  framework::VariableNameMap var_in_map =
+      CreateVarNameMap(info, "split", ins, true);
+  framework::VariableNameMap var_out_map =
+      CreateVarNameMap(info, "split", outs, false);
+  auto op = framework::OpRegistry::CreateOp(
+      "split", var_in_map, var_out_map, split_attr_map);
+  ASSERT_NO_FATAL_FAILURE(PreparedOp preparedOp = PreparedOp::Prepare(
+                              ins,
+                              outs,
+                              dynamic_cast<framework::OperatorWithKernel&>(*op),
+                              place,
+                              split_attr_map,
+                              {}));
+}
+
+const phi::DenseTensor* GetTensorFromVar(const framework::Variable& var);
+
+TEST(test_prepare_op, test_get_tensor_from_var) {
+  std::shared_ptr<imperative::VarBase> vout_error(
+      new imperative::VarBase(false, "vout_error"));
+  vout_error->MutableVar()->GetMutable<phi::SelectedRows>();
+  auto* ts = GetTensorFromVar(*vout_error->MutableVar());
+  ASSERT_TRUE(ts != nullptr);
+}
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+TEST(test_prepare_op, test_prepare_data) {
+  std::shared_ptr<imperative::VarBase> vin(
+      new imperative::VarBase(false, "vin"));
+  std::shared_ptr<imperative::VarBase> vout(
+      new imperative::VarBase(false, "vout"));
+
+  framework::OpDesc desc;
+  phi::CPUPlace cpu_place;
+  phi::GPUPlace gpu_place(0);
+  std::vector<float> src_data(10, 2.0);
+  std::vector<int64_t> dims = {2, 5};
+
+  // prepare an cpu only input
+  auto* vin_tensor = vin->MutableVar()->GetMutable<phi::DenseTensor>();
+  vin_tensor->Resize(common::make_ddim(dims));
+  auto* vin_mutable_tensor = vin_tensor->mutable_data<float>(cpu_place);
+  paddle::memory::Copy(cpu_place,
+                       vin_mutable_tensor,
+                       cpu_place,
+                       src_data.data(),
+                       sizeof(float) * src_data.size());
+
+  var_pair x_pair = var_pair("X", vb_vector(1, vin));
+  var_pair out_pair = var_pair("Out", vb_vector(1, vout));
+  imperative::NameVarBaseMap ins = {x_pair};
+  imperative::NameVarBaseMap outs = {out_pair};
+  const std::string op_type = "relu";
+  framework::AttributeMap attr_map;
+  const auto& info = framework::OpInfoMap::Instance().Get(op_type);
+  if (info.Checker()) info.Checker()->Check(&attr_map);
+  framework::VariableNameMap var_in_map =
+      CreateVarNameMap(info, op_type, ins, true);
+  framework::VariableNameMap var_out_map =
+      CreateVarNameMap(info, op_type, outs, false);
+  auto op = framework::OpRegistry::CreateOp(
+      op_type, var_in_map, var_out_map, attr_map);
+
+  // test if it can be transformed to GPU place
+  auto prepared_op =
+      PreparedOp::Prepare(ins,
+                          outs,
+                          dynamic_cast<framework::OperatorWithKernel&>(*op),
+                          gpu_place,
+                          attr_map,
+                          {});
+  PrepareData<imperative::VarBase>(
+      dynamic_cast<framework::OperatorWithKernel&>(*op),
+      ins,
+      prepared_op.kernel_key(),
+      gpu_place);
+  for (const auto& name_pair : ins) {
+    for (const auto& vb : name_pair.second) {
+      ASSERT_TRUE(phi::is_same_place(vb->Var().Get<phi::DenseTensor>().place(),
+                                     gpu_place));
+    }
+  }
+}
+#endif
+
+void TestPrepareDataSamePlace(framework::AttributeMap attr_map) {
+  std::shared_ptr<imperative::VarBase> vin(
+      new imperative::VarBase(false, "vin"));
+  std::shared_ptr<imperative::VarBase> vout(
+      new imperative::VarBase(false, "vout"));
+
+  framework::OpDesc desc;
+  phi::CPUPlace cpu_place;
+  std::vector<float> src_data(10, 2.0);
+  std::vector<int64_t> dims = {2, 5};
+
+  // prepare an cpu only input
+  auto* vin_tensor = vin->MutableVar()->GetMutable<phi::DenseTensor>();
+  vin_tensor->Resize(common::make_ddim(dims));
+  auto* vin_mutable_tensor = vin_tensor->mutable_data<float>(cpu_place);
+  paddle::memory::Copy(cpu_place,
+                       vin_mutable_tensor,
+                       cpu_place,
+                       src_data.data(),
+                       sizeof(float) * src_data.size());
+
+  var_pair x_pair = var_pair("X", vb_vector(1, vin));
+  var_pair out_pair = var_pair("Out", vb_vector(1, vout));
+  imperative::NameVarBaseMap ins = {x_pair};
+  imperative::NameVarBaseMap outs = {out_pair};
+  const std::string op_type = "relu";
+  const auto& info = framework::OpInfoMap::Instance().Get(op_type);
+  if (info.Checker()) info.Checker()->Check(&attr_map);
+  framework::VariableNameMap var_in_map =
+      CreateVarNameMap(info, op_type, ins, true);
+  framework::VariableNameMap var_out_map =
+      CreateVarNameMap(info, op_type, outs, false);
+
+  auto op = framework::OpRegistry::CreateOp(
+      op_type, var_in_map, var_out_map, attr_map);
+
+  // test if it never transferred on GPU place
+  auto prepared_op =
+      PreparedOp::Prepare(ins,
+                          outs,
+                          dynamic_cast<framework::OperatorWithKernel&>(*op),
+                          cpu_place,
+                          attr_map,
+                          {});
+  PrepareData<imperative::VarBase>(
+      dynamic_cast<framework::OperatorWithKernel&>(*op),
+      ins,
+      prepared_op.kernel_key(),
+      cpu_place);
+  for (const auto& name_pair : ins) {
+    for (const auto& vb : name_pair.second) {
+      ASSERT_TRUE(phi::is_same_place(vb->Var().Get<phi::DenseTensor>().place(),
+                                     cpu_place));
+    }
+  }
+}
+
+TEST(test_prepare_op, test_prepare_data_same_place) {
+  TestPrepareDataSamePlace({});
+}
+
+TEST(test_prepare_op, test_complex_eager) {
+  NameVarMap<egr::EagerVariable> outs = {};
+  TestHandleComplexGradToRealGradEager(outs);
+}
+
+#ifdef PADDLE_WITH_DNNL
+TEST(test_prepare_op, test_prepare_data_cpu_onednn) {
+  TestPrepareDataSamePlace({{"use_onednn", true}});
+}
+#endif
+}  // namespace imperative
+}  // namespace paddle
+
+USE_OP_ITSELF(split);
+USE_OP_ITSELF(relu);
+#ifdef PADDLE_WITH_DNNL
+PD_DECLARE_KERNEL(relu, OneDNN, ONEDNN);
+#endif

test/cpp/imperative/test_tracer.cc

@@ -0,0 +1,656 @@
+// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//
+// Created by Jiabin on 2019-08-16.
+//
+
+#include <memory>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/framework/op_registry.h"
+#include "paddle/fluid/imperative/basic_engine.h"
+#include "paddle/fluid/imperative/execution_context.h"
+#include "paddle/fluid/imperative/tracer.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/memory/memcpy.h"
+#include "paddle/phi/core/platform/device_context.h"
+
+PD_DECLARE_KERNEL(add, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(add_grad, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(sum, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(sum_grad, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(matmul_with_flatten, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(matmul_with_flatten_grad, CPU, ALL_LAYOUT);
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+PD_DECLARE_KERNEL(add_grad, GPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(add, KPS, ALL_LAYOUT);
+PD_DECLARE_KERNEL(sum_grad, GPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(matmul_with_flatten, GPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(matmul_with_flatten_grad, GPU, ALL_LAYOUT);
+#endif
+
+namespace paddle {
+namespace imperative {
+
+using vb_vector = std::vector<std::shared_ptr<imperative::VarBase>>;
+using var_pair = std::pair<std::string, vb_vector>;
+
+using ev_vector = std::vector<std::shared_ptr<egr::EagerVariable>>;
+using ev_pair = std::pair<std::string, ev_vector>;
+TEST(test_tracer, test_trace_op) {
+  // Doing an mul
+  imperative::Tracer tracer;
+  std::shared_ptr<imperative::VarBase> x_in(
+      new imperative::VarBase(true, "x_in"));
+  std::shared_ptr<imperative::VarBase> y_in(
+      new imperative::VarBase(true, "y_in"));
+  std::shared_ptr<imperative::VarBase> vout(
+      new imperative::VarBase(true, "vout"));
+  phi::CPUPlace place;
+  std::vector<float> src_data(10, 2.0);
+  std::vector<int64_t> dims1 = {2, 5};
+  std::vector<int64_t> dims2 = {5, 2};
+
+  auto* x_in_tensor = x_in->MutableVar()->GetMutable<phi::DenseTensor>();
+  auto* y_in_tensor = y_in->MutableVar()->GetMutable<phi::DenseTensor>();
+  x_in_tensor->Resize(common::make_ddim(dims1));
+  auto* mutable_x = x_in_tensor->mutable_data<float>(place);
+  paddle::memory::Copy(place,
+                       mutable_x,
+                       place,
+                       src_data.data(),
+                       sizeof(float) * src_data.size());
+  y_in_tensor->Resize(common::make_ddim(dims2));
+  auto* mutable_y = y_in_tensor->mutable_data<float>(place);
+  paddle::memory::Copy(place,
+                       mutable_y,
+                       place,
+                       src_data.data(),
+                       sizeof(float) * src_data.size());
+
+  var_pair x_pair = var_pair("X", vb_vector(1, x_in));
+  var_pair y_pair = var_pair("Y", vb_vector(1, y_in));
+  var_pair out_pair = var_pair("Out", vb_vector(1, vout));
+  imperative::NameVarBaseMap ins = {x_pair, y_pair};
+  imperative::NameVarBaseMap outs = {out_pair};
+  framework::AttributeMap mul_attr_map;
+  mul_attr_map["use_onednn"] = false;
+  tracer.TraceOp<VarBase>("mul", ins, outs, mul_attr_map, place, true);
+
+#ifndef PADDLE_WITH_XPU
+  ASSERT_THROW(tracer.TraceOp<VarBase>(
+      "mul", ins, outs, mul_attr_map, phi::XPUPlace(0), true);
+               , platform::EnforceNotMet);
+#endif
+
+  const auto& out_tensor = vout->Var().Get<phi::DenseTensor>();
+  for (int i = 0; i < vout->Var().Get<phi::DenseTensor>().numel(); i++) {
+    ASSERT_EQ(out_tensor.data<float>()[i], 20.0);
+  }
+}
+
+TEST(test_tracer, test_trace_op_with_backward) {
+  // Doing an mul
+  imperative::Tracer tracer;
+  std::shared_ptr<imperative::VarBase> x_in(
+      new imperative::VarBase(true, "x_in"));
+  std::shared_ptr<imperative::VarBase> y_in(
+      new imperative::VarBase(true, "y_in"));
+  std::shared_ptr<imperative::VarBase> vout(
+      new imperative::VarBase(true, "vout"));
+  phi::CPUPlace place;
+  std::vector<float> src_data(10, 2.0);
+  std::vector<int64_t> dims1 = {2, 5};
+  std::vector<int64_t> dims2 = {5, 2};
+
+  auto* x_in_tensor = x_in->MutableVar()->GetMutable<phi::DenseTensor>();
+  auto* y_in_tensor = y_in->MutableVar()->GetMutable<phi::DenseTensor>();
+  x_in_tensor->Resize(common::make_ddim(dims1));
+  auto* mutable_x = x_in_tensor->mutable_data<float>(place);
+  paddle::memory::Copy(place,
+                       mutable_x,
+                       place,
+                       src_data.data(),
+                       sizeof(float) * src_data.size());
+  y_in_tensor->Resize(common::make_ddim(dims2));
+  auto* mutable_y = y_in_tensor->mutable_data<float>(place);
+  paddle::memory::Copy(place,
+                       mutable_y,
+                       place,
+                       src_data.data(),
+                       sizeof(float) * src_data.size());
+
+  var_pair x_pair = var_pair("X", vb_vector(1, x_in));
+  var_pair y_pair = var_pair("Y", vb_vector(1, y_in));
+  var_pair out_pair = var_pair("Out", vb_vector(1, vout));
+  imperative::NameVarBaseMap ins = {x_pair, y_pair};
+  imperative::NameVarBaseMap outs = {out_pair};
+  framework::AttributeMap mul_attr_map;
+  mul_attr_map["use_onednn"] = false;
+  tracer.TraceOp<VarBase>("mul", ins, outs, mul_attr_map, place, true);
+  const auto& out_tensor = vout->Var().Get<phi::DenseTensor>();
+  for (int i = 0; i < vout->Var().Get<phi::DenseTensor>().numel(); i++) {
+    ASSERT_EQ(out_tensor.data<float>()[i], 20.0);
+  }
+}
+
+TEST(test_tracer, test_track_backward_output) {
+  // Doing an mul
+  imperative::Tracer tracer;
+  std::shared_ptr<imperative::VarBase> x_in(
+      new imperative::VarBase(true, "x_in"));
+  std::shared_ptr<imperative::VarBase> y_in(
+      new imperative::VarBase(true, "y_in"));
+  x_in->SetOverriddenStopGradient(false);
+  std::shared_ptr<imperative::VarBase> vout(
+      new imperative::VarBase(true, "vout"));
+  phi::CPUPlace place;
+  std::vector<float> src_data(10, 2.0);
+  std::vector<int64_t> dims1 = {2, 5};
+  std::vector<int64_t> dims2 = {5, 2};
+
+  auto* x_in_tensor = x_in->MutableVar()->GetMutable<phi::DenseTensor>();
+  auto* y_in_tensor = y_in->MutableVar()->GetMutable<phi::DenseTensor>();
+  x_in_tensor->Resize(common::make_ddim(dims1));
+  auto* mutable_x = x_in_tensor->mutable_data<float>(place);
+  paddle::memory::Copy(place,
+                       mutable_x,
+                       place,
+                       src_data.data(),
+                       sizeof(float) * src_data.size());
+  y_in_tensor->Resize(common::make_ddim(dims2));
+  auto* mutable_y = y_in_tensor->mutable_data<float>(place);
+  paddle::memory::Copy(place,
+                       mutable_y,
+                       place,
+                       src_data.data(),
+                       sizeof(float) * src_data.size());
+
+  var_pair x_pair = var_pair("X", vb_vector(1, x_in));
+  var_pair y_pair = var_pair("Y", vb_vector(1, y_in));
+  var_pair out_pair = var_pair("Out", vb_vector(1, vout));
+  imperative::NameVarBaseMap ins = {x_pair, y_pair};
+  imperative::NameVarBaseMap outs = {out_pair};
+  framework::AttributeMap mul_attr_map;
+  mul_attr_map["use_onednn"] = false;
+  tracer.TraceOp<VarBase>("mul", ins, outs, mul_attr_map, place, true);
+  ASSERT_EQ(x_in->GradVarBase()->GradOpNum(), 0UL);
+  ASSERT_EQ(y_in->GradVarBase()->GradOpNum(), 0UL);
+  ASSERT_EQ(vout->GradVarBase()->GradOpNum(), 1UL);
+}
+
+TEST(test_tracer, test_track_backward_input) {
+  // Doing an mul
+  imperative::Tracer tracer;
+  std::shared_ptr<imperative::VarBase> x_in(
+      new imperative::VarBase(true, "x_in"));
+  std::shared_ptr<imperative::VarBase> y_in(
+      new imperative::VarBase(true, "y_in"));
+  std::shared_ptr<imperative::VarBase> vout(
+      new imperative::VarBase(true, "vout"));
+  phi::CPUPlace place;
+  x_in->SetOverriddenStopGradient(false);
+  std::vector<float> src_data(10, 2.0);
+  std::vector<int64_t> dims1 = {2, 5};
+  std::vector<int64_t> dims2 = {5, 2};
+
+  auto* x_in_tensor = x_in->MutableVar()->GetMutable<phi::DenseTensor>();
+  auto* y_in_tensor = y_in->MutableVar()->GetMutable<phi::DenseTensor>();
+  x_in_tensor->Resize(common::make_ddim(dims1));
+  auto* mutable_x = x_in_tensor->mutable_data<float>(place);
+  paddle::memory::Copy(place,
+                       mutable_x,
+                       place,
+                       src_data.data(),
+                       sizeof(float) * src_data.size());
+  y_in_tensor->Resize(common::make_ddim(dims2));
+  auto* mutable_y = y_in_tensor->mutable_data<float>(place);
+  paddle::memory::Copy(place,
+                       mutable_y,
+                       place,
+                       src_data.data(),
+                       sizeof(float) * src_data.size());
+
+  var_pair x_pair = var_pair("X", vb_vector(1, x_in));
+  var_pair y_pair = var_pair("Y", vb_vector(1, y_in));
+  var_pair out_pair = var_pair("Out", vb_vector(1, vout));
+  imperative::NameVarBaseMap ins = {x_pair, y_pair};
+  imperative::NameVarBaseMap outs = {out_pair};
+  framework::AttributeMap mul_attr_map;
+  mul_attr_map["use_onednn"] = false;
+  tracer.TraceOp<VarBase>("mul", ins, outs, mul_attr_map, place, true);
+
+  ASSERT_EQ(x_in->GradVarBase()->GradOpNum(), 0UL);
+  ASSERT_EQ(y_in->GradVarBase()->GradOpNum(), 0UL);
+  ASSERT_EQ(vout->GradVarBase()->GradOpNum(), 1UL);
+}
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+TEST(test_tracer, test_trace_op_with_multi_device_inputs) {
+  // Doing an mul
+  imperative::Tracer tracer;
+  std::shared_ptr<imperative::VarBase> x_in(
+      new imperative::VarBase(true, "x_in"));
+  x_in->SetOverriddenStopGradient(false);  // force to run backward
+  std::shared_ptr<imperative::VarBase> y_in(
+      new imperative::VarBase(true, "y_in"));
+  y_in->SetOverriddenStopGradient(false);
+  std::shared_ptr<imperative::VarBase> vout(
+      new imperative::VarBase(true, "vout"));
+  phi::CPUPlace place;
+  phi::GPUPlace gpu_place(0);
+  std::vector<float> src_data(10, 2.0);
+  std::vector<int64_t> dims1 = {2, 5};
+  std::vector<int64_t> dims2 = {2, 5};
+
+  auto* x_in_tensor = x_in->MutableVar()->GetMutable<phi::DenseTensor>();
+  auto* y_in_tensor = y_in->MutableVar()->GetMutable<phi::DenseTensor>();
+  x_in_tensor->Resize(common::make_ddim(dims1));
+  auto* mutable_x = x_in_tensor->mutable_data<float>(place);
+  paddle::memory::Copy(place,
+                       mutable_x,
+                       place,
+                       src_data.data(),
+                       sizeof(float) * src_data.size());
+  y_in_tensor->Resize(common::make_ddim(dims2));
+  auto* mutable_y = y_in_tensor->mutable_data<float>(gpu_place);
+  paddle::memory::Copy(gpu_place,
+                       mutable_y,
+                       place,
+                       src_data.data(),
+                       sizeof(float) * src_data.size(),
+                       0);
+  var_pair x_pair = var_pair("X", vb_vector(1, x_in));
+  var_pair y_pair = var_pair("Y", vb_vector(1, y_in));
+  var_pair out_pair = var_pair("Out", vb_vector(1, vout));
+  imperative::NameVarBaseMap ins = {x_pair, y_pair};
+  imperative::NameVarBaseMap outs = {out_pair};
+  framework::AttributeMap mul_attr_map;
+  mul_attr_map["use_onednn"] = false;
+  tracer.TraceOp<VarBase>(
+      "elementwise_add", ins, outs, mul_attr_map, gpu_place, true);
+
+  // run reduce sum
+  std::shared_ptr<imperative::VarBase> reduce_sum_out(
+      new imperative::VarBase(true, "reduce_sum_out"));
+  var_pair reduce_sum_in_pair = var_pair("X", vb_vector(1, vout));
+  var_pair reduce_sum_out_pair = var_pair("Out", vb_vector(1, reduce_sum_out));
+  imperative::NameVarBaseMap reduce_in = {reduce_sum_in_pair};
+  imperative::NameVarBaseMap reduce_out = {reduce_sum_out_pair};
+  framework::AttributeMap reduce_attr_map;
+  tracer.TraceOp<VarBase>(
+      "reduce_sum", reduce_in, reduce_out, reduce_attr_map, gpu_place, true);
+  imperative::BasicEngine engine;
+
+  std::vector<std::shared_ptr<imperative::VarBase>> tensors{reduce_sum_out};
+  std::vector<std::shared_ptr<imperative::VarBase>> grad_tensors{nullptr};
+  engine.Init(tensors, grad_tensors);
+  engine.Execute();
+
+  phi::DenseTensor rlt;
+  framework::TensorCopySync(vout->Var().Get<phi::DenseTensor>(), place, &rlt);
+  for (int i = 0; i < rlt.numel(); i++) {
+    ASSERT_EQ(rlt.data<float>()[i], 4.0);
+  }
+
+  phi::DenseTensor out_grad;
+  framework::TensorCopySync(
+      vout->GradVar().Get<phi::DenseTensor>(), place, &out_grad);
+  for (int i = 0; i < out_grad.numel(); ++i) {
+    ASSERT_EQ(out_grad.data<float>()[i], 1.0);
+  }
+
+  phi::DenseTensor x_grad;
+  framework::TensorCopySync(
+      x_in->GradVar().Get<phi::DenseTensor>(), place, &x_grad);
+
+  for (int i = 0; i < x_grad.numel(); ++i) {
+    ASSERT_EQ(x_grad.data<float>()[i], 1.0);
+  }
+
+  phi::DenseTensor y_grad;
+  framework::TensorCopySync(
+      y_in->GradVar().Get<phi::DenseTensor>(), place, &y_grad);
+
+  for (int i = 0; i < y_grad.numel(); ++i) {
+    ASSERT_EQ(y_grad.data<float>()[i], 1.0);
+  }
+}
+
+#endif
+
+TEST(test_tracer, test_unique_name_generator) {
+  // generate two unique names
+  imperative::Tracer tracer;
+  auto fc_1 = tracer.GenerateUniqueName("fc");
+  auto fc_2 = tracer.GenerateUniqueName("fc");
+  ASSERT_STREQ("fc_0", fc_1.c_str());
+  ASSERT_STREQ("fc_1", fc_2.c_str());
+  // use `eager_tmp` as key if not specify it.
+  auto tmp_var_2 = tracer.GenerateUniqueName();
+  ASSERT_STREQ("dygraph_tmp_2", tmp_var_2.c_str());
+  auto tmp_var_3 = tracer.GenerateUniqueName("dygraph_tmp");
+  ASSERT_STREQ("dygraph_tmp_3", tmp_var_3.c_str());
+}
+
+TEST(test_tracer, test_current_tracer) {
+  // use current_tracer
+  auto tracer = std::make_shared<imperative::Tracer>();
+  imperative::SetCurrentTracer(tracer);
+  auto current_tracer = imperative::GetCurrentTracer();
+  ASSERT_EQ(current_tracer, tracer);
+}
+
+TEST(test_tracer, test_expected_place) {
+  // default expected place is CPUPlace
+  imperative::Tracer tracer;
+  ASSERT_EQ(phi::is_cpu_place(tracer.ExpectedPlace()), true);
+  {
+#ifdef PADDLE_WITH_CUDA
+    // set to CUDAPlace
+    phi::GPUPlace gpu_place(0);
+    tracer.SetExpectedPlace(gpu_place);
+    ASSERT_EQ(phi::is_gpu_place(tracer.ExpectedPlace()), true);
+#endif
+  }
+  {
+#ifdef PADDLE_WITH_XPU
+    // set to XPUPlace
+    phi::XPUPlace xpu_place(0);
+    tracer.SetExpectedPlace(xpu_place);
+    ASSERT_EQ(phi::is_xpu_place(tracer.ExpectedPlace()), true);
+#endif
+  }
+}
+
+TEST(test_tracer, test_var_without_grad_var) {
+  // Doing an mul
+  imperative::Tracer tracer;
+  std::shared_ptr<imperative::VarBase> x_in(
+      new imperative::VarBase(true, "x_in"));
+  x_in->ClearGradVarBase();
+  std::shared_ptr<imperative::VarBase> y_in(
+      new imperative::VarBase(true, "y_in"));
+  std::shared_ptr<imperative::VarBase> vout(
+      new imperative::VarBase(true, "vout"));
+  x_in->SetOverriddenStopGradient(false);
+  y_in->SetOverriddenStopGradient(false);
+  phi::CPUPlace place;
+  std::vector<float> src_data(10, 2.0);
+  std::vector<int64_t> dims1 = {2, 5};
+  std::vector<int64_t> dims2 = {5, 2};
+
+  auto* x_in_tensor = x_in->MutableVar()->GetMutable<phi::DenseTensor>();
+  auto* y_in_tensor = y_in->MutableVar()->GetMutable<phi::DenseTensor>();
+  x_in_tensor->Resize(common::make_ddim(dims1));
+  auto* mutable_x = x_in_tensor->mutable_data<float>(place);
+  paddle::memory::Copy(place,
+                       mutable_x,
+                       place,
+                       src_data.data(),
+                       sizeof(float) * src_data.size());
+  y_in_tensor->Resize(common::make_ddim(dims2));
+  auto* mutable_y = y_in_tensor->mutable_data<float>(place);
+  paddle::memory::Copy(place,
+                       mutable_y,
+                       place,
+                       src_data.data(),
+                       sizeof(float) * src_data.size());
+
+  var_pair x_pair = var_pair("X", vb_vector(1, x_in));
+  var_pair y_pair = var_pair("Y", vb_vector(1, y_in));
+  var_pair out_pair = var_pair("Out", vb_vector(1, vout));
+  imperative::NameVarBaseMap ins = {x_pair, y_pair};
+  imperative::NameVarBaseMap outs = {out_pair};
+  framework::AttributeMap mul_attr_map;
+  mul_attr_map["use_onednn"] = false;
+  tracer.TraceOp<VarBase>("mul", ins, outs, mul_attr_map, place, true);
+
+  const auto& out_tensor = vout->Var().Get<phi::DenseTensor>();
+  for (int i = 0; i < vout->Var().Get<phi::DenseTensor>().numel(); i++) {
+    ASSERT_EQ(out_tensor.data<float>()[i], 20.0);
+  }
+
+  ASSERT_EQ(x_in->GradVarBase()->GradOpNum(), 0UL);
+  ASSERT_EQ(y_in->GradVarBase()->GradOpNum(), 0UL);
+  ASSERT_EQ(vout->GradVarBase()->GradOpNum(), 1UL);
+
+  std::vector<std::shared_ptr<imperative::VarBase>> tensors{vout};
+  std::vector<std::shared_ptr<imperative::VarBase>> grad_tensors{nullptr};
+  imperative::BasicEngine engine;
+  engine.Init(tensors, grad_tensors);
+  engine.Execute();
+
+  // check the grad
+  phi::DenseTensor x_grad;
+  framework::TensorCopySync(
+      x_in->GradVar().Get<phi::DenseTensor>(), place, &x_grad);
+
+  for (int i = 0; i < x_grad.numel(); ++i) {
+    ASSERT_EQ(x_grad.data<float>()[i], 4.0);
+  }
+
+  phi::DenseTensor y_grad;
+  framework::TensorCopySync(
+      y_in->GradVar().Get<phi::DenseTensor>(), place, &y_grad);
+
+  for (int i = 0; i < y_grad.numel(); ++i) {
+    ASSERT_EQ(y_grad.data<float>()[i], 4.0);
+  }
+}
+
+template <typename T>
+using WeakPtrSet =
+    std::set<std::weak_ptr<T>, std::owner_less<std::weak_ptr<T>>>;
+
+static void TestVarOpDestructionMain(const phi::Place& place,
+                                     int64_t tensor_size = 10,
+                                     size_t loop_num = 10) {
+  WeakPtrSet<VariableWrapper> var_wrappers;
+  WeakPtrSet<VarBase> var_bases;
+  WeakPtrSet<GradOpNode> op_bases;
+
+  Tracer tracer;
+
+  {
+    auto x = std::make_shared<VarBase>("x");
+    auto y = std::make_shared<VarBase>("y");
+
+    x->MutableVar()
+        ->GetMutable<phi::DenseTensor>()
+        ->Resize({tensor_size, tensor_size})
+        .mutable_data<float>(place);
+
+    y->MutableVar()
+        ->GetMutable<phi::DenseTensor>()
+        ->Resize({tensor_size, tensor_size})
+        .mutable_data<float>(place);
+
+    x->SetOverriddenStopGradient(false);
+    y->SetOverriddenStopGradient(true);
+
+    for (size_t i = 0; i < loop_num; ++i) {
+      size_t var_wrapper_num = var_wrappers.size();
+      size_t var_base_num = var_bases.size();
+      size_t op_base_num = op_bases.size();
+
+      auto z = std::make_shared<VarBase>("z_" + std::to_string(i));
+      tracer.TraceOp<VarBase>("mul",
+                              NameVarBaseMap{{"X", {x}}, {"Y", {y}}},
+                              NameVarBaseMap{{"Out", {z}}},
+                              framework::AttributeMap{},
+                              place,
+                              true);
+
+      ASSERT_EQ(z->GradOpNum(), 0UL);
+      ASSERT_EQ(z->GradVarBase()->GradOpNum(), 1UL);
+      auto new_op = z->GradVarBase()->GradNode();
+
+      ASSERT_EQ(x->GradOpNum(), 0UL);
+      ASSERT_EQ(y->GradOpNum(), 0UL);
+
+      std::unordered_set<std::shared_ptr<GradOpNode>> expected_pending_ops;
+      if (i == 0) {
+        ASSERT_EQ(x->GradVarBase()->GradOpNum(), 0UL);
+        ASSERT_EQ(y->GradVarBase()->GradOpNum(), 0UL);
+      } else {
+        ASSERT_EQ(x->GradVarBase()->GradOpNum(), 1UL);
+        ASSERT_EQ(y->GradVarBase()->GradOpNum(), 0UL);
+
+        if (x->GradVarBase()->GradNode()) {
+          expected_pending_ops.emplace(x->GradVarBase()->GradNode());
+        }
+
+        if (y->GradVarBase()->GradNode()) {
+          expected_pending_ops.emplace(y->GradVarBase()->GradNode());
+        }
+
+        std::unordered_set<std::shared_ptr<GradOpNode>> actual_pending_ops;
+        for (auto& op : new_op->GradPendingNodes()) {
+          actual_pending_ops.emplace(op);
+        }
+
+        ASSERT_TRUE(expected_pending_ops == actual_pending_ops);
+        ASSERT_EQ(expected_pending_ops.count(new_op), 0UL);
+      }
+
+      var_wrappers.emplace(x->SharedVar());
+      var_wrappers.emplace(x->GradVarBase()->SharedVar());
+      var_wrappers.emplace(y->SharedVar());
+      var_wrappers.emplace(y->GradVarBase()->SharedVar());
+      var_wrappers.emplace(z->SharedVar());
+      var_wrappers.emplace(z->GradVarBase()->SharedVar());
+
+      var_bases.emplace(x);
+      var_bases.emplace(x->GradVarBase());
+      var_bases.emplace(y);
+      var_bases.emplace(y->GradVarBase());
+      var_bases.emplace(z);
+      var_bases.emplace(z->GradVarBase());
+
+      for (auto& op : expected_pending_ops) {
+        op_bases.emplace(op);
+      }
+
+      if (i == 0) {
+        ASSERT_EQ(var_wrapper_num, 0UL);
+        ASSERT_EQ(var_base_num, 0UL);
+        ASSERT_EQ(op_base_num, 0UL);
+        ASSERT_EQ(var_wrappers.size(), 6UL);
+        ASSERT_EQ(var_bases.size(), 6UL);
+        ASSERT_EQ(op_bases.size(), 0UL);
+      } else {
+        ASSERT_EQ(var_wrappers.size(), var_wrapper_num + 2);
+        ASSERT_EQ(var_bases.size(), var_base_num + 2);
+        ASSERT_EQ(op_bases.size(), op_base_num + 1);
+      }
+
+      x = z;  // recurrent usage
+    }
+  }
+
+  for (auto& var : var_wrappers) {
+    ASSERT_TRUE(var.expired());
+  }
+
+  for (auto& var : var_bases) {
+    ASSERT_TRUE(var.expired());
+  }
+
+  for (auto& op : op_bases) {
+    ASSERT_TRUE(op.expired());
+  }
+}
+
+TEST(test_tracer, test_var_op_destruction) {
+  TestVarOpDestructionMain(phi::CPUPlace());
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  TestVarOpDestructionMain(phi::GPUPlace(0));
+#endif
+}
+
+TEST(test_tracer, test_execution_context) {
+  auto op = framework::OpRegistry::CreateOp("mul", {}, {}, {}, false);
+  framework::Scope scope;
+  auto ctx = framework::RuntimeContext({}, {});
+  NameVarBaseMap ins = {{"X", {nullptr}}, {"Y", {nullptr}}};
+  NameVarBaseMap outs = {{"Out", {nullptr}}};
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  auto* dev_ctx = pool.Get(phi::CPUPlace());
+  auto dy_ctx = DygraphExecutionContext<VarBase>((*op.get()),
+                                                 scope,
+                                                 *dev_ctx,
+                                                 ctx,
+                                                 ins,
+                                                 outs,
+                                                 framework::AttributeMap{},
+                                                 framework::AttributeMap{});
+  ASSERT_EQ(dy_ctx.OutputName("Out"), framework::kEmptyVarName);
+}
+
+TEST(test_tracer, eager_tracer) {
+  // Doing an mul
+  imperative::Tracer tracer;
+  std::shared_ptr<egr::EagerVariable> x_in(new egr::EagerVariable("x_in"));
+  std::shared_ptr<egr::EagerVariable> y_in(new egr::EagerVariable("y_in"));
+  std::shared_ptr<egr::EagerVariable> vout(new egr::EagerVariable("vout"));
+  phi::CPUPlace place;
+  std::vector<float> src_data(10, 2.0);
+  std::vector<int64_t> dims1 = {2, 5};
+  std::vector<int64_t> dims2 = {5, 2};
+
+  auto* x_in_tensor = x_in->MutableVar()->GetMutable<phi::DenseTensor>();
+  auto* y_in_tensor = y_in->MutableVar()->GetMutable<phi::DenseTensor>();
+  x_in_tensor->Resize(common::make_ddim(dims1));
+  auto* mutable_x = x_in_tensor->mutable_data<float>(place);
+  paddle::memory::Copy(place,
+                       mutable_x,
+                       place,
+                       src_data.data(),
+                       sizeof(float) * src_data.size());
+  y_in_tensor->Resize(common::make_ddim(dims2));
+  auto* mutable_y = y_in_tensor->mutable_data<float>(place);
+  paddle::memory::Copy(place,
+                       mutable_y,
+                       place,
+                       src_data.data(),
+                       sizeof(float) * src_data.size());
+
+  ev_pair x_pair = ev_pair("X", ev_vector(1, x_in));
+  ev_pair y_pair = ev_pair("Y", ev_vector(1, y_in));
+  ev_pair out_pair = ev_pair("Out", ev_vector(1, vout));
+  imperative::NameTensorMap ins = {x_pair, y_pair};
+  imperative::NameTensorMap outs = {out_pair};
+  framework::AttributeMap mul_attr_map;
+  mul_attr_map["use_onednn"] = false;
+  tracer.TraceOp<egr::EagerVariable>(
+      "mul", ins, outs, mul_attr_map, place, true);
+
+  const auto& out_tensor = vout->Var().Get<phi::DenseTensor>();
+  for (int i = 0; i < vout->Var().Get<phi::DenseTensor>().numel(); i++) {
+    ASSERT_EQ(out_tensor.data<float>()[i], 20.0);
+  }
+}
+
+}  // namespace imperative
+}  // namespace paddle
+
+USE_OP_ITSELF(mul);
+USE_OP_ITSELF(mul_grad);
+USE_OP_ITSELF(reduce_sum);
+USE_OP_ITSELF(reduce_sum_grad);
+USE_OP_ITSELF(elementwise_add);