chore: import upstream snapshot with attribution

test/cpp/eager/CMakeLists.txt

@@ -0,0 +1,26 @@
+set(eager_deps
+    phi
+    common
+    hook_utils
+    utils
+    global_utils
+    backward
+    tracer
+    layer
+    autograd_meta
+    eager_nan_inf_utils
+    grad_node_info
+    grad_tensor_holder
+    custom_operator_node)
+
+if(NOT (NOT WITH_PYTHON AND ON_INFER))
+  set(eager_deps ${eager_deps} accumulation_node prim_utils)
+endif()
+
+set(fluid_deps tracer layer proto_desc operator op_registry variable_helper)
+set(generated_deps final_dygraph_function final_dygraph_node dygraph_function
+                   dygraph_node)
+
+add_subdirectory(data_structure_tests)
+add_subdirectory(task_tests)
+add_subdirectory(performance_tests)

test/cpp/eager/data_structure_tests/CMakeLists.txt

@@ -0,0 +1,32 @@
+if(WITH_CINN)
+  set(eager_deps ${eager_deps} python)
+endif()
+if(NOT WIN32)
+  cc_test(
+    test_egr_ds_eager_tensor
+    SRCS eager_tensor_test.cc
+    DEPS final_dygraph_function ${eager_deps})
+endif()
+cc_test(
+  test_egr_ds_auotgrad_meta
+  SRCS autograd_meta_test.cc
+  DEPS final_dygraph_function ${eager_deps})
+
+if(NOT ((NOT WITH_PYTHON) AND ON_INFER))
+  cc_test(
+    test_egr_ds_grad_tensor_holder
+    SRCS grad_tensor_holder_test.cc
+    DEPS conditional_block_op ${eager_deps} ${generated_deps})
+  cc_test(
+    test_egr_ds_grad_node_info
+    SRCS grad_node_info_test.cc
+    DEPS conditional_block_op ${eager_deps} ${generated_deps})
+  cc_test(
+    test_egr_ds_accumulation_node
+    SRCS accumulation_node_test.cc
+    DEPS conditional_block_op ${eager_deps} ${generated_deps})
+  cc_test(
+    test_egr_ds_tensor_wrapper
+    SRCS tensor_wrapper_test.cc
+    DEPS conditional_block_op ${eager_deps} ${generated_deps})
+endif()

test/cpp/eager/data_structure_tests/accumulation_node_test.cc

@@ -0,0 +1,417 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/eager/accumulation/accumulation_node.h"
+
+#include <sstream>
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/eager/api/utils/hook_utils.h"
+#include "paddle/fluid/eager/eager_tensor.h"
+#include "paddle/fluid/eager/grad_node_info.h"
+#include "paddle/fluid/eager/grad_tensor_holder.h"
+#include "paddle/fluid/eager/hooks.h"
+#include "paddle/fluid/eager/utils.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/selected_rows.h"
+
+// TODO(jiabin): remove nolint here!!!
+using namespace egr;  // NOLINT
+TEST(AccumulationNode, SelectedRowsAddToTensor) {
+  // Construct Eager Tensor
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 1}));
+  std::vector<int64_t> rows = {0};
+  std::shared_ptr<phi::SelectedRows> sr0 =
+      std::make_shared<phi::SelectedRows>(rows, 1);
+  sr0->mutable_value()->Resize(common::make_ddim({1, 1}));
+  sr0->mutable_value()->mutable_data<float>(phi::CPUPlace())[0] =
+      static_cast<float>(10.0f);
+  paddle::Tensor et0 = paddle::Tensor(sr0);
+  std::shared_ptr<phi::DenseTensor> dt1 = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  dt1->mutable_data<float>(phi::CPUPlace())[0] = static_cast<float>(20.0f);
+  paddle::Tensor et1 = paddle::Tensor(dt1);
+  std::shared_ptr<phi::DenseTensor> input_dt =
+      std::make_shared<phi::DenseTensor>(
+          std::make_unique<paddle::experimental::DefaultAllocator>(
+              phi::CPUPlace())
+              .get(),
+          meta);
+  paddle::Tensor input_et = paddle::Tensor(input_dt);
+  auto grad_meta = EagerUtils::autograd_meta(&input_et);
+  // Initialize Grad Tensor
+  std::shared_ptr<phi::SelectedRows> grad_dt =
+      std::make_shared<phi::SelectedRows>(rows, 1);
+  grad_dt->mutable_value()->Resize(common::make_ddim({1, 1}));
+  grad_dt->mutable_value()->mutable_data<float>(phi::CPUPlace())[0] =
+      static_cast<float>(0.0f);
+  grad_meta->MutableGrad()->set_impl(grad_dt);
+  // AccumulationNode
+  auto node = std::make_shared<GradNodeAccumulation>(input_et);
+  grad_meta->SetGradNode(node);
+  grad_meta->SetStopGradient(false);
+  // operator()
+  paddle::small_vector<std::vector<paddle::Tensor>, kSlotSmallVectorSize>
+      et0_vec = {{et0}};
+  paddle::Tensor ret_et0 = node->operator()(et0_vec)[0][0];
+  auto* ret_et0_ptr =
+      std::dynamic_pointer_cast<phi::SelectedRows>(ret_et0.impl())
+          ->value()
+          .data<float>();
+  PADDLE_ENFORCE_EQ(ret_et0_ptr[0],
+                    static_cast<float>(10.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the selected rows "
+                        "should be 10.0f."));
+  paddle::small_vector<std::vector<paddle::Tensor>, kSlotSmallVectorSize>
+      et1_vec = {{et1}};
+  paddle::Tensor ret_et1 = node->operator()(et1_vec)[0][0];
+  auto* ret_et1_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(ret_et1.impl())
+          ->data<float>();
+  PADDLE_ENFORCE_EQ(ret_et1_ptr[0],
+                    static_cast<float>(20.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the dense tensor "
+                        "should be 20.0f"));
+  // Check Retain Grad
+  PADDLE_ENFORCE_EQ(
+      std::dynamic_pointer_cast<phi::SelectedRows>(et0.impl())
+          ->value()
+          .data<float>()[0],
+      static_cast<float>(10.0f),
+      common::errors::InvalidArgument(
+          "The value of the first element of the dense tensor should be "
+          "10.0f"));
+
+  paddle::Tensor* grad = EagerUtils::mutable_grad(input_et);
+  auto* grad_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(grad->impl())->data<float>();
+  PADDLE_ENFORCE_EQ(grad_ptr[0],
+                    static_cast<float>(30.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the dense tensor "
+                        "should be 30.0f"));
+}
+
+TEST(AccumulationNode, SelectedRowsMerge) {
+  // Construct Eager Tensor
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 1}));
+  std::vector<int64_t> rows = {0};
+  std::shared_ptr<phi::SelectedRows> sr0 =
+      std::make_shared<phi::SelectedRows>(rows, 1);
+  sr0->mutable_value()->Resize(common::make_ddim({1, 1}));
+  sr0->mutable_value()->mutable_data<float>(phi::CPUPlace())[0] =
+      static_cast<float>(10.0f);
+  paddle::Tensor et0 = paddle::Tensor(sr0);
+  std::shared_ptr<phi::SelectedRows> sr1 =
+      std::make_shared<phi::SelectedRows>(rows, 1);
+  sr1->mutable_value()->Resize(common::make_ddim({1, 1}));
+  sr1->mutable_value()->mutable_data<float>(phi::CPUPlace())[0] =
+      static_cast<float>(20.0f);
+  paddle::Tensor et1 = paddle::Tensor(sr1);
+  std::shared_ptr<phi::DenseTensor> input_dt =
+      std::make_shared<phi::DenseTensor>(
+          std::make_unique<paddle::experimental::DefaultAllocator>(
+              phi::CPUPlace())
+              .get(),
+          meta);
+  paddle::Tensor input_et = paddle::Tensor(input_dt);
+  auto grad_meta = EagerUtils::autograd_meta(&input_et);
+  // Initialize Grad Tensor
+  std::shared_ptr<phi::SelectedRows> grad_dt =
+      std::make_shared<phi::SelectedRows>(rows, 1);
+  grad_dt->mutable_value()->Resize(common::make_ddim({1, 1}));
+  grad_dt->mutable_value()->mutable_data<float>(phi::CPUPlace())[0] =
+      static_cast<float>(0.0f);
+  grad_meta->MutableGrad()->set_impl(grad_dt);
+  // AccumulationNode
+  auto node = std::make_shared<GradNodeAccumulation>(input_et);
+  grad_meta->SetGradNode(node);
+  grad_meta->SetStopGradient(false);
+  // operator()
+  paddle::small_vector<std::vector<paddle::Tensor>, kSlotSmallVectorSize>
+      et0_vec = {{et0}};
+  paddle::Tensor ret_et0 = node->operator()(et0_vec)[0][0];
+  auto* ret_et0_ptr =
+      std::dynamic_pointer_cast<phi::SelectedRows>(ret_et0.impl())
+          ->value()
+          .data<float>();
+  PADDLE_ENFORCE_EQ(ret_et0_ptr[0],
+                    static_cast<float>(10.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the selected rows "
+                        "should be 10.0f."));
+  paddle::small_vector<std::vector<paddle::Tensor>, kSlotSmallVectorSize>
+      et1_vec = {{et1}};
+  paddle::Tensor ret_et1 = node->operator()(et1_vec)[0][0];
+  auto* ret_et1_ptr =
+      std::dynamic_pointer_cast<phi::SelectedRows>(ret_et1.impl())
+          ->value()
+          .data<float>();
+  PADDLE_ENFORCE_EQ(ret_et1_ptr[0],
+                    static_cast<float>(20.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the selected rows "
+                        "should be 20.0f."));
+  // Check Retain Grad
+  PADDLE_ENFORCE_EQ(std::dynamic_pointer_cast<phi::SelectedRows>(et0.impl())
+                        ->value()
+                        .data<float>()[0],
+                    static_cast<float>(10.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the selected rows "
+                        "should be 10.0f."));
+  paddle::Tensor* grad = EagerUtils::mutable_grad(input_et);
+  auto* grad_ptr = std::dynamic_pointer_cast<phi::SelectedRows>(grad->impl())
+                       ->value()
+                       .data<float>();
+  PADDLE_ENFORCE_EQ(grad_ptr[0],
+                    static_cast<float>(30.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the selected rows "
+                        "should be 30.0f."));
+}
+
+TEST(AccumulationNode, SelectedRowsAddTensor) {
+  // Construct Eager Tensor
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 1}));
+  std::vector<int64_t> rows = {0};
+  std::shared_ptr<phi::SelectedRows> sr0 =
+      std::make_shared<phi::SelectedRows>(rows, 1);
+  sr0->mutable_value()->Resize(common::make_ddim({1, 1}));
+  sr0->mutable_value()->mutable_data<float>(phi::CPUPlace())[0] =
+      static_cast<float>(10.0f);
+  paddle::Tensor et0 = paddle::Tensor(sr0);
+  std::shared_ptr<phi::SelectedRows> sr1 =
+      std::make_shared<phi::SelectedRows>(rows, 1);
+  sr1->mutable_value()->Resize(common::make_ddim({1, 1}));
+  sr1->mutable_value()->mutable_data<float>(phi::CPUPlace())[0] =
+      static_cast<float>(20.0f);
+  paddle::Tensor et1 = paddle::Tensor(sr1);
+  std::shared_ptr<phi::DenseTensor> input_dt =
+      std::make_shared<phi::DenseTensor>(
+          std::make_unique<paddle::experimental::DefaultAllocator>(
+              phi::CPUPlace())
+              .get(),
+          meta);
+  paddle::Tensor input_et = paddle::Tensor(input_dt);
+  auto grad_meta = EagerUtils::autograd_meta(&input_et);
+  // Initialize Grad Tensor
+  std::shared_ptr<phi::DenseTensor> grad_dt =
+      std::make_shared<phi::DenseTensor>(
+          std::make_unique<paddle::experimental::DefaultAllocator>(
+              phi::CPUPlace())
+              .get(),
+          meta);
+  grad_dt->mutable_data<float>(phi::CPUPlace())[0] = static_cast<float>(0.0f);
+  grad_meta->MutableGrad()->set_impl(grad_dt);
+  // AccumulationNode
+  auto node = std::make_shared<GradNodeAccumulation>(input_et);
+  grad_meta->SetGradNode(node);
+  grad_meta->SetStopGradient(false);
+  // operator()
+  paddle::small_vector<std::vector<paddle::Tensor>, kSlotSmallVectorSize>
+      et0_vec = {{et0}};
+  paddle::Tensor ret_et0 = node->operator()(et0_vec)[0][0];
+  auto* ret_et0_ptr =
+      std::dynamic_pointer_cast<phi::SelectedRows>(ret_et0.impl())
+          ->value()
+          .data<float>();
+  PADDLE_ENFORCE_EQ(ret_et0_ptr[0],
+                    static_cast<float>(10.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the selected rows "
+                        "should be 10.0f."));
+  paddle::small_vector<std::vector<paddle::Tensor>, kSlotSmallVectorSize>
+      et1_vec = {{et1}};
+  paddle::Tensor ret_et1 = node->operator()(et1_vec)[0][0];
+  auto* ret_et1_ptr =
+      std::dynamic_pointer_cast<phi::SelectedRows>(ret_et1.impl())
+          ->value()
+          .data<float>();
+  PADDLE_ENFORCE_EQ(ret_et1_ptr[0],
+                    static_cast<float>(20.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the selected rows "
+                        "should be 20.0f"));
+  // Check Retain Grad
+  PADDLE_ENFORCE_EQ(std::dynamic_pointer_cast<phi::SelectedRows>(et0.impl())
+                        ->value()
+                        .data<float>()[0],
+                    static_cast<float>(10.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the selected rows "
+                        "should be 10.0f"));
+  paddle::Tensor* grad = EagerUtils::mutable_grad(input_et);
+  auto* grad_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(grad->impl())->data<float>();
+  PADDLE_ENFORCE_EQ(grad_ptr[0],
+                    static_cast<float>(30.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the dense tensor "
+                        "should be 30.0f"));
+}
+
+TEST(AccumulationNode, Tensor) {
+  // Construct Eager Tensor
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT16, common::make_ddim({1, 1}));
+  std::shared_ptr<phi::DenseTensor> dt0 = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  dt0->mutable_data<phi::dtype::float16>(phi::CPUPlace())[0] =
+      phi::dtype::float16(10.0f);
+  paddle::Tensor et0 = paddle::Tensor(dt0);
+
+  std::shared_ptr<phi::DenseTensor> dt1 = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+
+  dt1->mutable_data<phi::dtype::float16>(phi::CPUPlace())[0] =
+      phi::dtype::float16(20.0f);
+  paddle::Tensor et1 = paddle::Tensor(dt1);
+
+  std::shared_ptr<phi::DenseTensor> input_dt =
+      std::make_shared<phi::DenseTensor>(
+          std::make_unique<paddle::experimental::DefaultAllocator>(
+              phi::CPUPlace())
+              .get(),
+          meta);
+  paddle::Tensor input_et = paddle::Tensor(input_dt);
+  auto grad_meta = EagerUtils::autograd_meta(&input_et);
+
+  // Initialize Grad Tensor
+  std::shared_ptr<phi::DenseTensor> grad_dt =
+      std::make_shared<phi::DenseTensor>(
+          std::make_unique<paddle::experimental::DefaultAllocator>(
+              phi::CPUPlace())
+              .get(),
+          meta);
+  grad_dt->mutable_data<phi::dtype::float16>(phi::CPUPlace())[0] =
+      phi::dtype::float16(0.0f);
+  grad_meta->MutableGrad()->set_impl(grad_dt);
+
+  // AccumulationNode
+  auto node = std::make_shared<GradNodeAccumulation>(input_et);
+  grad_meta->SetGradNode(node);
+  grad_meta->SetStopGradient(false);
+
+  // operator()
+  paddle::small_vector<std::vector<paddle::Tensor>, kSlotSmallVectorSize>
+      et0_vec = {{et0}};
+  paddle::Tensor ret_et0 = node->operator()(et0_vec)[0][0];
+  auto* ret_et0_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(ret_et0.impl())
+          ->data<phi::dtype::float16>();
+  PADDLE_ENFORCE_EQ(ret_et0_ptr[0],
+                    phi::dtype::float16(10.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the dense tensor "
+                        "should be 10.0f."));
+  paddle::small_vector<std::vector<paddle::Tensor>, kSlotSmallVectorSize>
+      et1_vec = {{et1}};
+  paddle::Tensor ret_et1 = node->operator()(et1_vec)[0][0];
+
+  auto* ret_et1_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(ret_et1.impl())
+          ->data<phi::dtype::float16>();
+  PADDLE_ENFORCE_EQ(ret_et1_ptr[0],
+                    phi::dtype::float16(20.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the dense tensor "
+                        "should be 20.0f"));
+  // Check Retain Grad
+  PADDLE_ENFORCE_EQ(std::dynamic_pointer_cast<phi::DenseTensor>(et0.impl())
+                        ->data<phi::dtype::float16>()[0],
+                    phi::dtype::float16(10.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the dense tensor "
+                        "should be 10.0f"));
+  paddle::Tensor* grad = EagerUtils::mutable_grad(input_et);
+  auto* grad_ptr = std::dynamic_pointer_cast<phi::DenseTensor>(grad->impl())
+                       ->data<phi::dtype::float16>();
+  PADDLE_ENFORCE_EQ(grad_ptr[0],
+                    phi::dtype::float16(30.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the dense tensor "
+                        "should be 30.0f"));
+
+  // Reduce Hook case 1: Call RegisterReduceHook and run operator()
+  VLOG(6) << "Test Reduce Hook";
+  PADDLE_ENFORCE_EQ(std::dynamic_pointer_cast<phi::DenseTensor>(et0.impl())
+                        ->data<phi::dtype::float16>()[0],
+                    phi::dtype::float16(10.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the dense tensor "
+                        "should be 10.0f"));
+  auto reduce_hook_1 = [&]() -> void {
+    auto* input_et_ptr =
+        std::dynamic_pointer_cast<phi::DenseTensor>(input_et.impl())
+            ->mutable_data<phi::dtype::float16>(phi::CPUPlace());
+    input_et_ptr[0] = 36.0;
+    VLOG(6) << "Running Reduce Hook";
+  };
+
+  node->RegisterReduceHook(std::make_shared<egr::CppVoidHook>(reduce_hook_1));
+
+  // operator()
+  paddle::Tensor _ret = node->operator()(et0_vec)[0][0];
+
+  // Check operator() result, should be 36.0
+  auto* _ret_ptr = std::dynamic_pointer_cast<phi::DenseTensor>(_ret.impl())
+                       ->data<phi::dtype::float16>();
+  PADDLE_ENFORCE_EQ(_ret_ptr[0],
+                    phi::dtype::float16(10.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the dense tensor "
+                        "should be 10.0f"));
+
+  // Check Retain Grad, should be 36.0
+  auto* _ret_input_et_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(input_et.impl())
+          ->data<phi::dtype::float16>();
+  PADDLE_ENFORCE_EQ(_ret_input_et_ptr[0],
+                    phi::dtype::float16(36.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the dense tensor "
+                        "should be 36.0f"));
+  // Reduce Hook case 2: Call RegisterReduceHook and ApplyReduceHooks directly
+  VLOG(6) << "Test Reduce Hook";
+  auto reduce_hook_2 = [&]() -> void {
+    auto* ret_et0_ptr =
+        std::dynamic_pointer_cast<phi::DenseTensor>(et0.impl())
+            ->mutable_data<phi::dtype::float16>(phi::CPUPlace());
+    ret_et0_ptr[0] = 100.0;  // set to 100.0
+    VLOG(6) << "Running Reduce Hook";
+  };
+  node->RegisterReduceHook(std::make_shared<egr::CppVoidHook>(reduce_hook_2));
+  node->ApplyReduceHooks();
+
+  // Check ApplyReduceHooks result
+  PADDLE_ENFORCE_EQ(std::dynamic_pointer_cast<phi::DenseTensor>(et0.impl())
+                        ->data<phi::dtype::float16>()[0],
+                    phi::dtype::float16(100.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the dense tensor "
+                        "should be 100.0f"));
+}

test/cpp/eager/data_structure_tests/autograd_meta_test.cc

@@ -0,0 +1,151 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/eager/autograd_meta.h"
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/fluid/eager/eager_tensor.h"
+#include "paddle/fluid/eager/grad_node_info.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "test/cpp/eager/data_structure_tests/grad_node_test.h"
+
+TEST(AutogradMeta, Constructor) {
+  paddle::Tensor et1;
+  auto auto_grad = std::make_shared<egr::AutogradMeta>();
+  et1.set_autograd_meta(auto_grad);
+  auto* tmp_auto = static_cast<egr::AutogradMeta*>(et1.get_autograd_meta());
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->OutRankInfo().first,
+      size_t(0),
+      common::errors::InvalidArgument(
+          "The first element of OutRankInfo should be 0, but received %d.",
+          tmp_auto->OutRankInfo().first));
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->OutRankInfo().second,
+      size_t(0),
+      common::errors::InvalidArgument(
+          "The second element of OutRankInfo should be 0, but received %d.",
+          tmp_auto->OutRankInfo().second));
+  CHECK(tmp_auto->IsInitialized() == false);
+}
+
+TEST(AutogradMeta, MemberFunction) {
+  paddle::Tensor et1;
+  auto auto_grad = std::make_shared<egr::AutogradMeta>();
+  et1.set_autograd_meta(auto_grad);
+  auto* tmp_auto = static_cast<egr::AutogradMeta*>(et1.get_autograd_meta());
+  VLOG(6) << "Test Grad";
+  PADDLE_ENFORCE_EQ(tmp_auto->Grad().defined(),
+                    false,
+                    common::errors::Fatal("grad should not be defined now"));
+  auto* grad_t = tmp_auto->MutableGrad();
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 2}));
+  std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  auto* dt_ptr = dt->mutable_data<float>(phi::CPUPlace());
+  dt_ptr[0] = 5.0f;
+  dt_ptr[1] = 10.0f;
+  grad_t->set_impl(dt);
+  VLOG(6) << "Test Mutable Grad";
+  auto impl_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(tmp_auto->Grad().impl());
+  PADDLE_ENFORCE_EQ(
+      impl_ptr->data<float>()[0],
+      5.0f,
+      common::errors::InvalidArgument(
+          "The first element of grad tensor should be 5.0, but received %f.",
+          impl_ptr->data<float>()[0]));
+  PADDLE_ENFORCE_EQ(
+      impl_ptr->data<float>()[1],
+      10.0f,
+      common::errors::InvalidArgument(
+          "The second element of grad tensor should be 10.0, but received %f.",
+          impl_ptr->data<float>()[1]));
+  VLOG(6) << "Test IsInitialized";
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->IsInitialized(),
+      false,
+      common::errors::Fatal(
+          "egr::AutogradMeta variable tmp_auto should not be initialized now"));
+  VLOG(6) << "Test GradNodeSetter Getter";
+  auto grad_node = std::make_shared<eager_test::GradTestNode>();
+  tmp_auto->SetGradNode(grad_node);
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->IsInitialized(),
+      true,
+      common::errors::Fatal(
+          "egr::AutogradMeta variable tmp_auto should be initialized now"));
+  auto tmp_grad_node = tmp_auto->GetMutableGradNode();
+  std::dynamic_pointer_cast<eager_test::GradTestNode>(tmp_grad_node)->val_ =
+      5.0;
+  PADDLE_ENFORCE_EQ(
+      dynamic_cast<eager_test::GradTestNode*>(tmp_auto->GradNode())->val_,
+      5.0,
+      common::errors::InvalidArgument(
+          "The value of GradTestNode should be 5.0, but received %f.",
+          dynamic_cast<eager_test::GradTestNode*>(tmp_auto->GradNode())->val_));
+  VLOG(6) << "Test rank Setter Getter";
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->OutRankInfo().first,
+      size_t(0),
+      common::errors::InvalidArgument(
+          "The first element of OutRankInfo should be 0, but received %d.",
+          tmp_auto->OutRankInfo().first));
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->OutRankInfo().second,
+      size_t(0),
+      common::errors::InvalidArgument(
+          "The second element of OutRankInfo should be 0, but received %d.",
+          tmp_auto->OutRankInfo().second));
+  tmp_auto->SetSingleOutRankWithSlot(2, 3);
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->OutRankInfo().first,
+      size_t(2),
+      common::errors::InvalidArgument(
+          "The first element of OutRankInfo should be 2, but received %d.",
+          tmp_auto->OutRankInfo().first));
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->OutRankInfo().second,
+      size_t(3),
+      common::errors::InvalidArgument(
+          "The second element of OutRankInfo should be 3, but received %d.",
+          tmp_auto->OutRankInfo().second));
+  VLOG(6) << "Test stop gradient Setter Getter";
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->NumericStopGradient(),
+      -1,
+      common::errors::InvalidArgument(
+          "The NumericStopGradient value should be -1, but received %d.",
+          tmp_auto->NumericStopGradient()));
+  tmp_auto->SetStopGradient(true);
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->StopGradient(),
+      true,
+      common::errors::Fatal("tmp_auto->StopGradient() should be true now"));
+  VLOG(6) << "Test Persistable Setter Getter";
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->Persistable(),
+      false,
+      common::errors::Fatal("tmp_auto->Persistable() should be false now"));
+  tmp_auto->SetPersistable(true);
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->Persistable(),
+      true,
+      common::errors::Fatal(
+          "tmp_auto->Persistable() should be true now after SetPersistable()"));
+}

test/cpp/eager/data_structure_tests/eager_tensor_test.cc

@@ -0,0 +1,439 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/eager/eager_tensor.h"
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/common/layout.h"
+#include "paddle/fluid/imperative/var_helper.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+namespace eager_test {
+using AbstractAutogradMeta = paddle::AbstractAutogradMeta;
+class AutogradMetaTest : public AbstractAutogradMeta {
+ public:
+  explicit AutogradMetaTest(int val) : val_(val) {}
+  int val_ = 0;
+};
+}  // namespace eager_test
+
+TEST(Tensor, Constructor) {
+  paddle::Tensor et1 = paddle::Tensor();
+  paddle::Tensor et2 = paddle::Tensor("et2");
+
+  PADDLE_ENFORCE_EQ(et1.defined(),
+                    false,
+                    common::errors::InvalidArgument("Tensor et1 should be "
+                                                    "undefined."));
+  PADDLE_ENFORCE_EQ(et2.name(),
+                    "et2",
+                    common::errors::InvalidArgument("Tensor name should be "
+                                                    "'et2'."));
+
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 2}));
+  std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  auto* dt_ptr = dt->mutable_data<float>(phi::CPUPlace());
+  dt_ptr[0] = 5.0f;
+  dt_ptr[1] = 10.0f;
+  paddle::Tensor et3 = paddle::Tensor(dt);
+  auto* et3_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(et3.impl())->data<float>();
+  PADDLE_ENFORCE_EQ(et3_ptr[0],
+                    5.0f,
+                    common::errors::InvalidArgument("First element should be "
+                                                    "5.0f."));
+  PADDLE_ENFORCE_EQ(et3_ptr[1],
+                    10.0f,
+                    common::errors::InvalidArgument("Second element should be "
+                                                    "10.0f."));
+  // copy constructor
+  paddle::Tensor et4(et3);
+  auto* et4_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(et4.impl())->data<float>();
+  PADDLE_ENFORCE_EQ(et4_ptr[0],
+                    5.0f,
+                    common::errors::InvalidArgument("First element should be "
+                                                    "5.0f."));
+  PADDLE_ENFORCE_EQ(et4_ptr[1],
+                    10.0f,
+                    common::errors::InvalidArgument("Second element should be "
+                                                    "10.0f."));
+  paddle::Tensor et5(std::move(et4));
+  auto* et5_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(et5.impl())->data<float>();
+  PADDLE_ENFORCE_EQ(et5_ptr[0],
+                    5.0f,
+                    common::errors::InvalidArgument("First element should be "
+                                                    "5.0f."));
+  PADDLE_ENFORCE_EQ(et5_ptr[1],
+                    10.0f,
+                    common::errors::InvalidArgument("Second element should be "
+                                                    "10.0f."));
+}
+
+TEST(Tensor, MemberFunction) {
+  paddle::Tensor et3;
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 2}));
+  std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  auto* dt_ptr = dt->mutable_data<float>(phi::CPUPlace());
+  dt_ptr[0] = 5.0f;
+  dt_ptr[1] = 10.0f;
+  VLOG(6) << "Make Dense Tensor";
+  et3.set_name("et3");
+  VLOG(6) << "Set Name";
+  PADDLE_ENFORCE_EQ(et3.name(),
+                    "et3",
+                    common::errors::InvalidArgument("Tensor name should be "
+                                                    "'et3'."));
+  PADDLE_ENFORCE_EQ(et3.defined(),
+                    false,
+                    common::errors::InvalidArgument("Tensor et3 should be "
+                                                    "undefined."));
+  et3.set_impl(dt);
+  VLOG(6) << "Set impl";
+  PADDLE_ENFORCE_EQ(et3.initialized(),
+                    true,
+                    common::errors::InvalidArgument("Tensor et3 should be "
+                                                    "initialized."));
+  PADDLE_ENFORCE_EQ(et3.is_cpu(),
+                    true,
+                    common::errors::InvalidArgument("Tensor et3 should be "
+                                                    "on CPU."));
+  PADDLE_ENFORCE_EQ(et3.is_gpu(),
+                    false,
+                    common::errors::InvalidArgument("Tensor et3 should not be "
+                                                    "on GPU."));
+  PADDLE_ENFORCE_EQ(et3.numel(),
+                    2,
+                    common::errors::InvalidArgument("Tensor et3 should have "
+                                                    "2 elements."));
+  auto expected_dim = common::make_ddim({1, 2});
+  PADDLE_ENFORCE_EQ(
+      et3.dims(),
+      expected_dim,
+      common::errors::InvalidArgument("Tensor dimensions should be "
+                                      "{1, 2}."));
+  PADDLE_ENFORCE_EQ(et3.type(),
+                    phi::DataType::FLOAT32,
+                    common::errors::InvalidArgument("Tensor data type should "
+                                                    "be FLOAT32."));
+  PADDLE_ENFORCE_EQ(et3.layout(),
+                    phi::DataLayout::NCHW,
+                    common::errors::InvalidArgument("Tensor layout should be "
+                                                    "NCHW."));
+  CHECK(phi::is_cpu_place(et3.place()));
+  VLOG(6) << "Get impl";
+  auto* dt3_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(et3.impl())->data<float>();
+  PADDLE_ENFORCE_EQ(dt3_ptr[0],
+                    5.0f,
+                    common::errors::InvalidArgument("First element should be "
+                                                    "5.0f."));
+  PADDLE_ENFORCE_EQ(dt3_ptr[1],
+                    10.0f,
+                    common::errors::InvalidArgument("Second element should be "
+                                                    "10.0f."));
+  paddle::Tensor et4 = et3;
+  VLOG(6) << "copy =";
+  PADDLE_ENFORCE_EQ(et4.initialized(),
+                    true,
+                    common::errors::InvalidArgument("Tensor et4 should be "
+                                                    "initialized."));
+  auto* dt4_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(et4.impl())->data<float>();
+  PADDLE_ENFORCE_EQ(dt4_ptr[0],
+                    5.0f,
+                    common::errors::InvalidArgument("First element should be "
+                                                    "5.0f."));
+  PADDLE_ENFORCE_EQ(dt4_ptr[1],
+                    10.0f,
+                    common::errors::InvalidArgument("Second element should be "
+                                                    "10.0f."));
+  VLOG(6) << "move =";
+  paddle::Tensor et5 = std::move(et4);
+  auto* dt5_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(et5.impl())->data<float>();
+  PADDLE_ENFORCE_EQ(dt5_ptr[0],
+                    5.0f,
+                    common::errors::InvalidArgument("First element should be "
+                                                    "5.0f."));
+  PADDLE_ENFORCE_EQ(dt5_ptr[1],
+                    10.0f,
+                    common::errors::InvalidArgument("Second element should be "
+                                                    "10.0f."));
+  VLOG(6) << "AutogradMeta";
+  auto autograd_meta_test = std::make_shared<eager_test::AutogradMetaTest>(2);
+  et3.set_autograd_meta(autograd_meta_test);
+  auto* tmp_autograd_meta_test =
+      static_cast<eager_test::AutogradMetaTest*>(et3.get_autograd_meta());
+  PADDLE_ENFORCE_EQ(tmp_autograd_meta_test->val_,
+                    2,
+                    common::errors::InvalidArgument("AutogradMetaTest value "
+                                                    "should be 2."));
+}
+
+TEST(EagerVariable, Constructor) {
+  paddle::Tensor t3;
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 2}));
+  std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  auto* dt_ptr = dt->mutable_data<float>(phi::CPUPlace());
+  dt_ptr[0] = 5.0f;
+  dt_ptr[1] = 10.0f;
+  VLOG(6) << "Make Dense Tensor";
+  t3.set_name("t3");
+  VLOG(6) << "Set Name";
+  PADDLE_ENFORCE_EQ(t3.name(),
+                    "t3",
+                    common::errors::InvalidArgument("Tensor name should be "
+                                                    "'t3'."));
+  PADDLE_ENFORCE_EQ(
+      t3.defined(),
+      false,
+      common::errors::InvalidArgument(
+          "Tensor t3 should be undefined but got %d.", t3.defined()));
+  t3.set_impl(dt);
+
+  egr::EagerVariable et3 = egr::EagerVariable(t3);
+  VLOG(6) << "SyncToVar";
+  PADDLE_ENFORCE_EQ(et3.Var().Get<phi::DenseTensor>().data<float>()[0],
+                    5.0f,
+                    common::errors::InvalidArgument("First element should be "
+                                                    "5.0f."));
+  PADDLE_ENFORCE_EQ(et3.Var().Get<phi::DenseTensor>().data<float>()[1],
+                    10.0f,
+                    common::errors::InvalidArgument("Second element should be "
+                                                    "10.0f."));
+  VLOG(6) << "SyncToTensor";
+  paddle::Tensor t4;
+  t4.set_impl(et3.GetTensorBase());
+  PADDLE_ENFORCE_EQ(
+      t4.initialized(),
+      true,
+      common::errors::InvalidArgument(
+          "Tensor t4 should be initialized but got %d.", t4.initialized()));
+
+  VLOG(6) << "Check Tensor";
+  auto* dt3_tmp_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(t4.impl())->data<float>();
+  PADDLE_ENFORCE_EQ(dt3_tmp_ptr[0],
+                    5.0f,
+                    common::errors::InvalidArgument("First element should be "
+                                                    "5.0f."));
+  PADDLE_ENFORCE_EQ(dt3_tmp_ptr[1],
+                    10.0f,
+                    common::errors::InvalidArgument("Second element should be "
+                                                    "10.0f."));
+  t4.reset();
+  PADDLE_ENFORCE_EQ(
+      t4.defined(),
+      false,
+      common::errors::InvalidArgument(
+          "Tensor t4 should be undefined but got %d.", t4.defined()));
+
+  VLOG(6) << "Check Tensor Copy_";
+  std::vector<int64_t> rows = {1, 2};
+  std::vector<int64_t> dims = {2};
+  paddle::Tensor t7(std::make_shared<phi::SelectedRows>(rows, 2));
+  std::dynamic_pointer_cast<phi::SelectedRows>(t7.impl())
+      ->mutable_value()
+      ->Resize(common::make_ddim(dims));
+  auto* dt7_tmp_ptr = std::dynamic_pointer_cast<phi::SelectedRows>(t7.impl())
+                          ->mutable_value()
+                          ->mutable_data<float>(phi::CPUPlace());
+  dt7_tmp_ptr[0] = 6.0f;
+  dt7_tmp_ptr[1] = 11.0f;
+
+  paddle::Tensor t8;
+  paddle::Tensor t5;
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  paddle::Tensor t6;
+  paddle::Tensor t9;
+  VLOG(6) << "Check Tensor Copy_ Selected Rows";
+  t8.copy_(t7, phi::GPUPlace(0), false);
+  t9.copy_(t8, phi::CPUPlace(), false);
+  auto* dt9_tmp_ptr = std::dynamic_pointer_cast<phi::SelectedRows>(t9.impl())
+                          ->value()
+                          .data<float>();
+  PADDLE_ENFORCE_EQ(dt9_tmp_ptr[0],
+                    6.0f,
+                    common::errors::InvalidArgument("First element should be "
+                                                    "6.0f."));
+  PADDLE_ENFORCE_EQ(dt9_tmp_ptr[1],
+                    11.0f,
+                    common::errors::InvalidArgument("Second element should be "
+                                                    "11.0f."));
+  PADDLE_ENFORCE_EQ(
+      std::dynamic_pointer_cast<phi::SelectedRows>(t9.impl())->height(),
+      2,
+      common::errors::InvalidArgument("SelectedRows height should "
+                                      "be 2."));
+
+  VLOG(6) << "Check Tensor Copy_ Dense Tensor";
+  t5.copy_(t3, phi::GPUPlace(0), false);
+  t6.copy_(t5, phi::CPUPlace(), false);
+  auto* dt6_tmp_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(t6.impl())->data<float>();
+  PADDLE_ENFORCE_EQ(dt6_tmp_ptr[0],
+                    5.0f,
+                    common::errors::InvalidArgument("First element should be "
+                                                    "5.0f."));
+  PADDLE_ENFORCE_EQ(dt6_tmp_ptr[1],
+                    10.0f,
+                    common::errors::InvalidArgument("Second element should be "
+                                                    "10.0f."));
+#else
+  t5.copy_(t3, phi::CPUPlace(), false);
+  auto* dt5_tmp_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(t5.impl())->data<float>();
+  PADDLE_ENFORCE_EQ(dt5_tmp_ptr[0],
+                    5.0f,
+                    common::errors::InvalidArgument("First element should be "
+                                                    "5.0f."));
+  PADDLE_ENFORCE_EQ(dt5_tmp_ptr[1],
+                    10.0f,
+                    common::errors::InvalidArgument("Second element should be "
+                                                    "10.0f."));
+#endif
+
+  VLOG(6) << "Finish";
+}
+
+TEST(EagerVariable, DataLayout) {
+  paddle::Tensor tensor;
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32,
+                           common::make_ddim({1, 1, 1, 1}),
+                           phi::DataLayout::UNDEFINED);
+  std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  auto* dt_ptr = dt->mutable_data<float>(phi::CPUPlace());
+  dt_ptr[0] = 5.0f;
+  dt_ptr[1] = 5.0f;
+  dt_ptr[2] = 5.0f;
+  dt_ptr[3] = 5.0f;
+  tensor.set_impl(dt);
+  auto eager_var = std::make_shared<egr::EagerVariable>(tensor);
+  auto layout = paddle::imperative::GetDataLayout(eager_var);
+  PADDLE_ENFORCE_EQ(layout,
+                    phi::DataLayout::UNDEFINED,
+                    common::errors::InvalidArgument("Data layout should be "
+                                                    "UNDEFINED."));
+  paddle::imperative::SetDataLayout(eager_var, phi::DataLayout::NCHW);
+  layout = paddle::imperative::GetDataLayout(eager_var);
+  PADDLE_ENFORCE_EQ(layout,
+                    phi::DataLayout::NCHW,
+                    common::errors::InvalidArgument("Data layout should be "
+                                                    "NCHW."));
+}
+
+TEST(VariableCompatTensor, MemberFunction) {
+  egr::VariableCompatTensor var_tensor;
+  // test GetMutable and Get
+  var_tensor.GetMutable<phi::Vocab>();
+  auto& vocab = var_tensor.Get<phi::Vocab>();
+  EXPECT_EQ(vocab.size(), 0UL);
+  bool caught_exception = false;
+  try {
+    var_tensor.GetMutable<phi::Strings>();
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("The Variable type must be") != std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+  // test Type and IsType
+  EXPECT_TRUE(var_tensor.IsType<phi::Vocab>());
+  EXPECT_EQ(var_tensor.Type(),
+            static_cast<int>(paddle::framework::proto::VarType::VOCAB));
+  // test valid and initialized
+  EXPECT_TRUE(var_tensor.IsInitialized());
+  EXPECT_TRUE(var_tensor.valid());
+  EXPECT_TRUE(var_tensor.initialized());
+  // test name
+  EXPECT_EQ(var_tensor.name(), "VariableCompatTensor");
+  // test other throw error methods
+  caught_exception = false;
+  try {
+    var_tensor.numel();
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("numel") != std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+  caught_exception = false;
+  try {
+    var_tensor.dims();
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("dims") != std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+  caught_exception = false;
+  try {
+    var_tensor.dtype();
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("dtype") != std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+  caught_exception = false;
+  try {
+    var_tensor.layout();
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("layout") != std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+  caught_exception = false;
+  try {
+    var_tensor.place();
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("place") != std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+  caught_exception = false;
+  try {
+    var_tensor.AllocateFrom(nullptr, phi::DataType::UNDEFINED);
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("AllocateFrom") != std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+  // test Clear
+  var_tensor.Clear();
+  EXPECT_FALSE(var_tensor.IsInitialized());
+}

test/cpp/eager/data_structure_tests/grad_node_info_test.cc

@@ -0,0 +1,257 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/eager/grad_node_info.h"
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/fluid/eager/autograd_meta.h"
+#include "paddle/fluid/eager/eager_tensor.h"
+#include "paddle/fluid/eager/hooks.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/core/enforce.h"
+#include "test/cpp/eager/data_structure_tests/grad_node_test.h"
+
+TEST(GradNodeInfo, GradSlotMeta) {
+  auto grad_slot = egr::GradSlotMeta();
+  VLOG(6) << "Set SetStopGradient";
+  grad_slot.SetStopGradient();
+  PADDLE_ENFORCE_EQ(
+      grad_slot.IsStopGradient(),
+      true,
+      common::errors::Fatal("`grad_slot.IsStopGradient()` should be "
+                            "true, please check related function"));
+}
+
+void TestGradNodeBase(bool is_remove_gradient_hook) {
+  VLOG(6) << "Construct Grad Node";
+  auto grad_test_node0 = std::make_shared<eager_test::GradTestNode>(
+      /* val */ 5.0, /* in_num */ 2, /* out_num */ 2);
+  auto grad_test_node1 = std::make_shared<eager_test::GradTestNode>();
+  paddle::small_vector<std::vector<paddle::Tensor>, egr::kSlotSmallVectorSize>
+      grads;
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 1}));
+  std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  auto* dt_ptr = dt->mutable_data<float>(phi::CPUPlace());
+  dt_ptr[0] = 5.0f;
+  paddle::Tensor et1(dt);
+  grads = {{et1}};
+
+  std::vector<int64_t> mesh_shape = {2, 2};
+  std::vector<int64_t> process_ids = {0, 1, 2, 3};
+  std::vector<std::string> dim_names = {"dp", "mp"};
+  phi::distributed::ProcessMesh process_mesh(
+      mesh_shape, process_ids, dim_names);
+  std::vector<int64_t> dim_mapping = {-1, 1};
+  phi::distributed::TensorDistAttr dist_attr =
+      phi::distributed::TensorDistAttr();
+  dist_attr.set_process_mesh(process_mesh);
+  dist_attr.set_dims_mapping(dim_mapping);
+  dist_attr.set_dynamic_dims(std::vector<bool>(mesh_shape.size(), false));
+
+  std::shared_ptr<phi::distributed::DistTensor> dist_dt1 =
+      std::make_shared<phi::distributed::DistTensor>(
+          phi::distributed::DistTensor(dt, dist_attr));
+  paddle::Tensor dist_et1(dist_dt1);
+
+  VLOG(6) << "Test Grad Node Call";
+  auto res = (*grad_test_node0)(grads);
+  PADDLE_ENFORCE_EQ(
+      std::dynamic_pointer_cast<phi::DenseTensor>(res[0][0].impl())
+          ->data<float>()[0],
+      6.0f,
+      common::errors::InvalidArgument("Data of grads mismatch. Expected 6.0."));
+  egr::Edge tmp_edge1(grad_test_node1, 3, 4);
+  auto auto_grad1 = std::make_shared<egr::AutogradMeta>(tmp_edge1);
+  et1.set_autograd_meta(auto_grad1);
+
+  VLOG(6) << "Test Set Meta and Get Meta";
+  auto_grad1->SetStopGradient(true);
+  grad_test_node0->SetGradInMeta(et1, 0);
+  grad_test_node0->SetGradInMeta({et1}, 1);
+  grad_test_node0->SetGradOutMeta(et1, 0);
+  grad_test_node0->SetGradOutMeta({et1}, 1);
+  grad_test_node0->SetGradOutMeta(dist_et1, 0, dist_attr, dist_et1.dims());
+  PADDLE_ENFORCE_EQ(
+      grad_test_node0->InputMeta()[0].size(),
+      1UL,
+      common::errors::InvalidArgument("Size of input mismatch. Expected 1."));
+  PADDLE_ENFORCE_EQ(
+      grad_test_node0->InputMeta()[1].size(),
+      1UL,
+      common::errors::InvalidArgument("Size of input mismatch. Expected 1."));
+  PADDLE_ENFORCE_EQ(
+      grad_test_node0->InputMeta()[0][0].GetTensorMeta().dtype,
+      meta.dtype,
+      common::errors::InvalidArgument("Dtype of input tensor mismatch."));
+  PADDLE_ENFORCE_EQ(
+      grad_test_node0->InputMeta()[1][0].GetTensorMeta().dtype,
+      meta.dtype,
+      common::errors::InvalidArgument("Dtype of input tensor mismatch."));
+  PADDLE_ENFORCE_EQ(grad_test_node0->OutputMeta()[0][0].IsStopGradient(),
+                    true,
+                    common::errors::Fatal(
+                        "`grad_test_node0->OutputMeta()[0][0].IsStopGradient()"
+                        "` should be true, please related function"));
+  PADDLE_ENFORCE_EQ(grad_test_node0->OutputMeta()[1][0].IsStopGradient(),
+                    true,
+                    common::errors::Fatal(
+                        "`grad_test_node0->OutputMeta()[1][0].IsStopGradient()"
+                        "` should be true, please related function"));
+  PADDLE_ENFORCE_EQ(
+      grad_test_node0->OutputMeta()[0][0].GetTensorMeta().dtype,
+      meta.dtype,
+      common::errors::InvalidArgument("Dtype of output tensor mismatch."));
+  PADDLE_ENFORCE_EQ(
+      grad_test_node0->OutputMeta()[1][0].GetTensorMeta().dtype,
+      meta.dtype,
+      common::errors::InvalidArgument("Dtype of output tensor mismatch."));
+  PADDLE_ENFORCE_EQ(
+      grad_test_node0->OutputMeta()[0][0].DistAttr(),
+      dist_attr,
+      common::errors::InvalidArgument("DistAttr of output tensor mismatch."));
+  PADDLE_ENFORCE_EQ(
+      grad_test_node0->OutputMeta()[0][0].DistTensorGlobalDims(),
+      dist_et1.dims(),
+      common::errors::InvalidArgument("DDims of output tensor mismatch."));
+
+  VLOG(6) << "Test Default Set Meta and Get Meta";
+  auto grad_test_node2 = std::make_shared<eager_test::GradTestNode>(
+      /* val */ 5.0, /* in_num */ 1, /* out_num */ 1);
+  grad_test_node2->SetDefaultGradInOutMeta();
+  PADDLE_ENFORCE_GT(
+      grad_test_node2->OutputMeta()[0].size(),
+      0UL,
+      common::errors::InvalidArgument("Size of output not greater than 0."));
+  CHECK(grad_test_node2->OutputMeta()[0][0].IsStopGradient() == false);
+  PADDLE_ENFORCE_EQ(
+      grad_test_node2->OutputMeta()[0].size(),
+      1UL,
+      common::errors::InvalidArgument("Size of output mismatch. Expected 1."));
+
+  VLOG(6) << "Test Gradient Hook";
+  auto gradient_hook = [](const paddle::Tensor& et) -> paddle::Tensor {
+    paddle::Tensor res;
+    phi::DenseTensorMeta meta =
+        phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 1}));
+    std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
+        std::make_unique<paddle::experimental::DefaultAllocator>(
+            phi::CPUPlace())
+            .get(),
+        meta);
+    auto* dt_ptr = dt->mutable_data<float>(phi::CPUPlace());
+    dt_ptr[0] = 6.0f;
+    auto* et_ptr =
+        std::dynamic_pointer_cast<phi::DenseTensor>(et.impl())->data<float>();
+    dt_ptr[0] += et_ptr[0];
+    res.set_impl(dt);
+    VLOG(6) << "Running Gradient Hook";
+    return res;
+  };
+  int64_t hook_id = grad_test_node0->RegisterGradientHook(
+      0, 0, std::make_shared<egr::CppTensorHook>(gradient_hook));
+
+  if (is_remove_gradient_hook) {
+    // Remove GradientHook
+    grad_test_node0->RemoveGradientHook(hook_id);
+  }
+
+  // Check results
+  auto grad_hook_res = grad_test_node0->ApplyGradientHooks(grads);
+  PADDLE_ENFORCE_EQ(
+      std::dynamic_pointer_cast<phi::DenseTensor>(grad_hook_res[0][0].impl())
+          ->data<float>()[0],
+      is_remove_gradient_hook ? 5.0 : 11.0,
+      common::errors::InvalidArgument(
+          "Data of grad hook res mismatch. Expected 5.0 or 11.0."));
+}
+
+TEST(GradNodeInfo, GradNodeBase) {
+  TestGradNodeBase(true);
+  TestGradNodeBase(false);
+}
+
+TEST(GradNodeInfo, Edge) {
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 1}));
+  std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  paddle::Tensor et1(dt);
+
+  auto grad_test_node0 = std::make_shared<eager_test::GradTestNode>(5, 2, 2);
+  auto auto_grad1 = std::make_shared<egr::AutogradMeta>();
+  VLOG(6) << "Test Construct Edge";
+  egr::Edge edge0 = egr::Edge();
+  PADDLE_ENFORCE_EQ(
+      edge0.IsInitialized(),
+      false,
+      common::errors::Fatal("`edge0.IsInitialized()` should be "
+                            "false, please check related function"));
+  egr::Edge edge1 = egr::Edge(grad_test_node0, size_t(0), size_t(0));
+  PADDLE_ENFORCE_EQ(
+      edge1.IsInitialized(),
+      true,
+      common::errors::Fatal("`edge1.IsInitialized()` should be "
+                            "true, please check related function"));
+  egr::Edge edge2 =
+      egr::Edge(grad_test_node0, std::make_pair(size_t(1), size_t(0)));
+  VLOG(6) << "Test Set Edge's Grad Node";
+  auto* grad_node = edge1.GetGradNode();
+  et1.set_autograd_meta(auto_grad1);
+  grad_node->SetGradInMeta(et1, 0);
+
+  PADDLE_ENFORCE_EQ(
+      grad_node->InputMeta().size(),
+      2UL,
+      common::errors::InvalidArgument("Size of input mismatch. Expected 2."));
+  std::vector<egr::AutogradMeta*> metas = {auto_grad1.get()};
+  PADDLE_ENFORCE_EQ(
+      grad_node->InputMeta()[0][0].IsStopGradient(),
+      true,
+      common::errors::Fatal("`grad_node->InputMeta()[0][0].IsStopGradient()` "
+                            "should be true, please check related function"));
+  VLOG(6) << "Test Get/Set Edge Rank Info";
+  PADDLE_ENFORCE_EQ(
+      edge2.GetEdgeRankInfo().first,
+      1UL,
+      common::errors::InvalidArgument("Edge rank info mismatch. Expected 1."));
+  PADDLE_ENFORCE_EQ(
+      edge2.GetEdgeRankInfo().second,
+      0UL,
+      common::errors::InvalidArgument("Edge rank info mismatch. Expected 0."));
+  edge2.SetEdgeRankInfo(2, 3);
+  PADDLE_ENFORCE_EQ(
+      edge2.GetEdgeRankInfo().first,
+      2UL,
+      common::errors::InvalidArgument("Edge rank info mismatch. Expected 2."));
+  PADDLE_ENFORCE_EQ(
+      edge2.GetEdgeRankInfo().second,
+      3UL,
+      common::errors::InvalidArgument("Edge rank info mismatch. Expected 3."));
+  edge2.SetEdgeRankInfo(std::make_pair(size_t(4), size_t(5)));
+  PADDLE_ENFORCE_EQ(
+      edge2.GetEdgeRankInfo().first,
+      4UL,
+      common::errors::InvalidArgument("Edge rank info mismatch. Expected 4."));
+  PADDLE_ENFORCE_EQ(
+      edge2.GetEdgeRankInfo().second,
+      5UL,
+      common::errors::InvalidArgument("Edge rank info mismatch. Expected 5."));
+}

test/cpp/eager/data_structure_tests/grad_node_test.h

@@ -0,0 +1,65 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/fluid/eager/autograd_meta.h"
+#include "paddle/fluid/eager/eager_tensor.h"
+#include "paddle/fluid/eager/grad_node_info.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+namespace egr {
+class TensorWrapper;
+}
+
+namespace eager_test {
+class GradTestNode : public egr::GradNodeBase {
+ public:
+  ~GradTestNode() override = default;
+  GradTestNode(float val, int in_num, int out_num)
+      : GradNodeBase(in_num, out_num), val_(val) {}
+  GradTestNode() : GradNodeBase() { val_ = 1.0; }
+  std::string name() override { return "GradTestNode"; }
+  paddle::small_vector<std::vector<paddle::Tensor>, egr::kSlotSmallVectorSize>
+  operator()(paddle::small_vector<std::vector<paddle::Tensor>,
+                                  egr::kSlotSmallVectorSize>& grads,  // NOLINT
+             bool create_graph = false,
+             bool is_new_grad = false) override {
+    val_ = std::dynamic_pointer_cast<phi::DenseTensor>(grads[0][0].impl())
+               ->data<float>()[0];
+    phi::DenseTensorMeta meta =
+        phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 1}));
+    std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
+        std::make_unique<paddle::experimental::DefaultAllocator>(
+            phi::CPUPlace())
+            .get(),
+        meta);
+    auto* dt_ptr = dt->mutable_data<float>(phi::CPUPlace());
+    dt_ptr[0] = 6.0f;
+    paddle::Tensor et1(dt);
+    paddle::small_vector<std::vector<paddle::Tensor>, egr::kSlotSmallVectorSize>
+        res = {{et1}};
+    return res;
+  }
+  void ClearTensorWrappers() override { VLOG(6) << "Do nothing here now"; }
+
+  std::shared_ptr<GradNodeBase> Copy() const override {
+    {
+      auto copied_node = std::shared_ptr<GradTestNode>(new GradTestNode(*this));
+      return copied_node;
+    }
+  }
+
+  float val_;
+};
+}  // namespace eager_test

test/cpp/eager/data_structure_tests/grad_tensor_holder_test.cc

@@ -0,0 +1,206 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/eager/grad_tensor_holder.h"
+
+#include <sstream>
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/eager/eager_tensor.h"
+#include "paddle/fluid/eager/grad_node_info.h"
+#include "paddle/fluid/eager/utils.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/selected_rows.h"
+
+PD_DECLARE_KERNEL(full_like, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(add, CPU, ALL_LAYOUT);
+
+// TODO(jiabin): remove nolint here!!!
+using namespace egr;  // NOLINT
+
+TEST(GradTensorHolder, Constructor) {
+  std::vector<GradSlotMeta> slot_meta(1);
+  GradTensorHolder grad_tensor_holder = GradTensorHolder({slot_meta});
+  GradTensorHolder grad_tensor_holder2 = GradTensorHolder(grad_tensor_holder);
+
+  // Construct Eager Tensor
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({2, 2}));
+  std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  paddle::Tensor et = paddle::Tensor(dt);
+
+  paddle::small_vector<std::vector<paddle::Tensor>, kSlotSmallVectorSize>
+      inputs;
+  inputs.push_back({et});
+  grad_tensor_holder2.SetBuffers(std::move(inputs));
+}
+
+TEST(GradTensorHolder, Interfaces) {
+  // Construct Eager Tensor
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 1}));
+  std::shared_ptr<phi::DenseTensor> dt0 = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  dt0->mutable_data<float>(phi::CPUPlace())[0] = 10.0;
+  paddle::Tensor et0 = paddle::Tensor(dt0);
+
+  std::shared_ptr<phi::DenseTensor> dt1 = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  dt1->mutable_data<float>(phi::CPUPlace())[0] = 20.0;
+  paddle::Tensor et1 = paddle::Tensor(dt1);
+
+  // Constructor empty GradTensorHolder
+  std::vector<GradSlotMeta> slot_meta(1);
+  GradTensorHolder grad_tensor_holder =
+      GradTensorHolder({slot_meta, slot_meta});
+  egr::EagerUtils::autograd_meta(&et0);
+  // add():
+  // fill one
+  grad_tensor_holder.CopyValueFromTensor(0, 0, et0, true);
+
+  // accumulation
+  grad_tensor_holder.add(1, 0, et0);
+  grad_tensor_holder.add(1, 0, et1);
+
+  // Buffers()
+  const auto& buffers = grad_tensor_holder.Buffers();
+  PADDLE_ENFORCE_EQ(static_cast<int>(buffers.size()),
+                    2,
+                    common::errors::InvalidArgument(
+                        "The size of buffers should be 2, but received %d.",
+                        static_cast<int>(buffers.size())));
+  PADDLE_ENFORCE_EQ(
+      static_cast<int>(buffers[0].size()),
+      1,
+      common::errors::InvalidArgument(
+          "The size of the first buffer should be 1, but received %d.",
+          static_cast<int>(buffers[0].size())));
+  PADDLE_ENFORCE_EQ(
+      static_cast<int>(buffers[1].size()),
+      1,
+      common::errors::InvalidArgument(
+          "The size of the second buffer should be 1, but received %d.",
+          static_cast<int>(buffers[1].size())));
+
+  // operator[]
+  const auto& holder_et0 = grad_tensor_holder[0][0];
+  const auto& holder_et1 = grad_tensor_holder[1][0];
+
+  auto* holder_et0_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(holder_et0.impl())
+          ->data<float>();
+  auto* holder_et1_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(holder_et1.impl())
+          ->data<float>();
+
+  PADDLE_ENFORCE_EQ(
+      holder_et0_ptr[0],
+      1.0f,
+      common::errors::InvalidArgument(
+          "The value of holder_et0_ptr[0] should be 1.0f, but received %f.",
+          holder_et0_ptr[0]));
+  PADDLE_ENFORCE_EQ(
+      holder_et1_ptr[0],
+      30.0f,
+      common::errors::InvalidArgument(
+          "The value of holder_et1_ptr[0] should be 30.0f, but received %f.",
+          holder_et1_ptr[0]));
+}
+
+TEST(GradTensorHolder, SelectedRowsMergeAdd) {
+  phi::CPUPlace cpu;
+
+  std::vector<int64_t> rows{0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+  int64_t table_size = 10;
+  int64_t embedding_width = 10;
+
+  auto sr1 = std::make_shared<phi::SelectedRows>(rows, table_size);
+  auto sr2 = std::make_shared<phi::SelectedRows>(rows, table_size);
+
+  // initialize a sparse table 1
+  sr1->mutable_value()->Resize(
+      common::make_ddim({table_size, embedding_width}));
+  auto* data_sr1 = sr1->mutable_value()->mutable_data<float>(cpu);
+  for (int64_t i = 0; i < table_size; ++i) {
+    for (int64_t j = 0; j < embedding_width; ++j) {
+      data_sr1[i * embedding_width + j] = static_cast<float>(i);
+    }
+  }
+
+  // initialize a sparse table 2
+  sr2->mutable_value()->Resize(
+      common::make_ddim({table_size, embedding_width}));
+  auto* data_sr2 = sr2->mutable_value()->mutable_data<float>(cpu);
+  for (int64_t i = 0; i < table_size; ++i) {
+    for (int64_t j = 0; j < embedding_width; ++j) {
+      data_sr2[i * embedding_width + j] = static_cast<float>(i);
+    }
+  }
+  // new 2 phi::Tensor
+  paddle::Tensor t1(sr1);
+  paddle::Tensor t2(sr2);
+
+  // Constructor empty GradTensorHolder
+  std::vector<GradSlotMeta> slot_meta(1);
+  GradTensorHolder grad_tensor_holder =
+      GradTensorHolder({slot_meta, slot_meta});
+
+  // accumulation
+  grad_tensor_holder.add(0, 0, t1);
+  grad_tensor_holder.add(0, 0, t2);
+
+  // Buffers()
+  const auto& buffers = grad_tensor_holder.Buffers();
+  PADDLE_ENFORCE_EQ(static_cast<int>(buffers.size()),
+                    2,
+                    common::errors::InvalidArgument(
+                        "The size of buffers should be 2, but received %d.",
+                        static_cast<int>(buffers.size())));
+  PADDLE_ENFORCE_EQ(
+      static_cast<int>(buffers[0].size()),
+      1,
+      common::errors::InvalidArgument(
+          "The size of the first buffer should be 1, but received %d.",
+          static_cast<int>(buffers[0].size())));
+  PADDLE_ENFORCE_EQ(
+      static_cast<int>(buffers[1].size()),
+      1,
+      common::errors::InvalidArgument(
+          "The size of the second buffer should be 1, but received %d.",
+          static_cast<int>(buffers[1].size())));
+
+  // operator[]
+  const auto& holder_et0 = grad_tensor_holder[0][0];
+
+  auto* tmp_buffer_tensor =
+      static_cast<phi::SelectedRows*>(holder_et0.impl().get());
+  auto* tmp_buffer_data_sr =
+      tmp_buffer_tensor->mutable_value()->mutable_data<float>(cpu);
+
+  // verify the MergeAdd result (accumulation result)
+  for (int64_t i = 0; i < table_size; ++i) {
+    for (int64_t j = 0; j < embedding_width; ++j) {
+      EXPECT_EQ(tmp_buffer_data_sr[i * embedding_width + j],
+                (static_cast<float>(i) + static_cast<float>(i)));
+    }
+  }
+}

test/cpp/eager/data_structure_tests/tensor_wrapper_test.cc

@@ -0,0 +1,108 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/eager/tensor_wrapper.h"
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/fluid/eager/utils.h"
+#include "test/cpp/eager/data_structure_tests/grad_node_test.h"
+
+TEST(TensorWrapper, Basic) {
+  VLOG(6) << "Test Full reserved";
+  paddle::Tensor et1;
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 2}));
+  std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  auto* dt_ptr = dt->mutable_data<float>(phi::CPUPlace());
+  dt_ptr[0] = 5.0f;
+  dt_ptr[1] = 10.0f;
+  et1.set_impl(dt);
+  // Create grad node;
+  auto grad_test_node0 = std::make_shared<eager_test::GradTestNode>(
+      /* val */ 5.0, /* in_num */ 2, /* out_num */ 2);
+  egr::Edge edge0(grad_test_node0, 1, 2);
+  auto auto_grad0 = std::make_shared<egr::AutogradMeta>(edge0);
+  et1.set_autograd_meta(auto_grad0);
+  et1.set_name("et1");
+  auto tw0 = egr::TensorWrapper(et1);
+  auto recover_et1 = tw0.recover();
+  if (VLOG_IS_ON(7)) {
+    PADDLE_ENFORCE_EQ(
+        recover_et1.name(),
+        std::string("et1@saved"),
+        common::errors::InvalidArgument(
+            "Recovered tensor name should be 'et1@saved', but received %s.",
+            recover_et1.name().c_str()));
+  }
+  PADDLE_ENFORCE_EQ(egr::EagerUtils::OutRankInfo(recover_et1).first,
+                    egr::EagerUtils::OutRankInfo(et1).first,
+                    common::errors::InvalidArgument(
+                        "The OutRankInfo first element of the recovered tensor "
+                        "does not match the original tensor."));
+  PADDLE_ENFORCE_EQ(egr::EagerUtils::OutRankInfo(recover_et1).second,
+                    egr::EagerUtils::OutRankInfo(et1).second,
+                    common::errors::InvalidArgument(
+                        "The OutRankInfo second element of the recovered "
+                        "tensor does not match the original tensor."));
+  VLOG(6) << "Test reconstruct";
+  paddle::Tensor et2;
+  phi::DenseTensorMeta meta2 =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 2}));
+  std::shared_ptr<phi::DenseTensor> dt2 = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta2);
+  auto* dt_ptr2 = dt->mutable_data<float>(phi::CPUPlace());
+  dt_ptr2[0] = 6.0f;
+  dt_ptr2[1] = 11.0f;
+  et2.set_impl(dt2);
+  et2.set_name("et2");
+  auto grad_test_node1 =
+      std::make_shared<eager_test::GradTestNode>(/* val */ 5.0, 2, 2);
+  egr::Edge edge1(grad_test_node1, 1, 2);
+  auto auto_grad1 = std::make_shared<egr::AutogradMeta>(edge1);
+  et2.set_autograd_meta(auto_grad1);
+  auto tw1 = egr::TensorWrapper(et2, false);
+  auto recover_et2 = tw1.recover();
+  if (VLOG_IS_ON(7)) {
+    PADDLE_ENFORCE_EQ(
+        recover_et2.name(),
+        std::string("et2@Saved"),
+        common::errors::InvalidArgument(
+            "Recovered tensor name should be 'et2@Saved', but received %s.",
+            recover_et2.name().c_str()));
+  }
+  PADDLE_ENFORCE_EQ(egr::EagerUtils::OutRankInfo(recover_et2).first,
+                    egr::EagerUtils::OutRankInfo(et2).first,
+                    common::errors::InvalidArgument(
+                        "The OutRankInfo first element of the recovered tensor "
+                        "does not match the original tensor."));
+  PADDLE_ENFORCE_EQ(egr::EagerUtils::OutRankInfo(recover_et2).second,
+                    egr::EagerUtils::OutRankInfo(et2).second,
+                    common::errors::InvalidArgument(
+                        "The OutRankInfo second element of the recovered "
+                        "tensor does not match the original tensor."));
+  // Test Raw recover
+  paddle::Tensor et3;
+  auto tw2 = egr::TensorWrapper(et3);
+  PADDLE_ENFORCE_EQ(
+      tw2.recover().initialized(),
+      false,
+      common::errors::Fatal(
+          "Variable `tw2` should not be initialized after recover"));
+}

test/cpp/eager/performance_tests/CMakeLists.txt

@@ -0,0 +1,35 @@
+if(NOT (NOT WITH_PYTHON AND ON_INFER))
+  if(WITH_CINN)
+    set(eager_deps ${eager_deps} python)
+  endif()
+  cc_library(performance_benchmark_utils SRCS benchmark_utils.cc)
+  add_dependencies(
+    performance_benchmark_utils
+    ${eager_deps}
+    ${fluid_deps}
+    ${generated_deps}
+    eager_scale
+    scale_node
+    generated_op
+    generated_static_op
+    dygraph_function
+    eager_prim_api)
+
+  paddle_test(test_egr_performance_benchmark_eager_cpu SRCS
+              benchmark_eager_cpu.cc DEPS performance_benchmark_utils)
+  paddle_test(test_egr_performance_benchmark_fluid_cpu SRCS
+              benchmark_fluid_cpu.cc DEPS performance_benchmark_utils)
+
+  if(WITH_GPU)
+    paddle_test(test_egr_performance_benchmark_eager_cuda SRCS
+                benchmark_eager_cuda.cc DEPS performance_benchmark_utils)
+    paddle_test(test_egr_performance_benchmark_fluid_cuda SRCS
+                benchmark_fluid_cuda.cc DEPS performance_benchmark_utils)
+  endif()
+
+  if(WITH_ONNXRUNTIME AND WIN32)
+    # Copy onnxruntime for some c++ test in Windows, since the test will
+    # be build only in CI, so suppose the generator in Windows is Ninja.
+    copy_onnx(performance_benchmark_utils)
+  endif()
+endif()

test/cpp/eager/performance_tests/benchmark_eager_cpu.cc

@@ -0,0 +1,242 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Eager Dygraph
+
+#include <paddle/fluid/framework/op_registry.h>
+
+#include <chrono>
+
+#include "gtest/gtest.h"
+#include "paddle/common/flags.h"
+#include "paddle/fluid/eager/api/all.h"
+#include "paddle/fluid/eager/autograd_meta.h"
+#include "paddle/fluid/eager/backward.h"
+#include "paddle/fluid/imperative/tracer.h"
+#include "test/cpp/eager/performance_tests/benchmark_utils.h"
+#include "test/cpp/eager/test_utils.h"
+
+#ifdef WITH_GPERFTOOLS
+#include "gperftools/profiler.h"
+#endif
+
+#include "paddle/phi/core/kernel_registry.h"
+
+using namespace egr;            // NOLINT
+using namespace egr_utils_api;  // NOLINT
+
+TEST(Benchmark, EagerScaleCPU) {
+  // Prepare Device Contexts
+  eager_test::InitEnv(phi::CPUPlace());
+
+  for (const std::string mode : {"Accuracy", "Performance"}) {
+    phi::DDim ddim = common::make_ddim({2, 4, 4, 4});
+    paddle::Tensor tensor =
+        eager_test::CreateTensorWithValue(ddim,
+                                          phi::CPUPlace(),
+                                          phi::DataType::FLOAT32,
+                                          phi::DataLayout::NCHW,
+                                          5.0,
+                                          true);
+    RetainGradForTensor(tensor);
+
+    if (mode == "Accuracy") {
+      benchmark_eager_scale(tensor, true /* accuracy_check*/);
+
+    } else if (mode == "Performance") {
+      auto t_start = std::chrono::high_resolution_clock::now();
+#ifdef WITH_GPERFTOOLS
+      ProfilerStart("eager_scale_cpu.out");
+#endif
+      benchmark_eager_scale(tensor);
+
+#ifdef WITH_GPERFTOOLS
+      ProfilerStop();
+#endif
+      auto t_end = std::chrono::high_resolution_clock::now();
+      double elapsed_time_ms =
+          std::chrono::duration<double, std::milli>(t_end - t_start).count();
+
+      std::cout << "Duration: " << elapsed_time_ms << " ms" << std::endl;
+
+    } else {
+      PADDLE_THROW(common::errors::Fatal("Unknown benchmark mode"));
+    }
+  }
+}
+
+TEST(Benchmark, EagerMatmulCPU) {
+  // Prepare Device Contexts
+  eager_test::InitEnv(phi::CPUPlace());
+
+  for (const std::string mode : {"Accuracy", "Performance"}) {
+    phi::DDim ddimX = common::make_ddim({2, 2});
+    paddle::Tensor X = eager_test::CreateTensorWithValue(ddimX,
+                                                         phi::CPUPlace(),
+                                                         phi::DataType::FLOAT32,
+                                                         phi::DataLayout::NCHW,
+                                                         1.0,
+                                                         true);
+    RetainGradForTensor(X);
+
+    phi::DDim ddimY = common::make_ddim({2, 2});
+    paddle::Tensor Y = eager_test::CreateTensorWithValue(ddimY,
+                                                         phi::CPUPlace(),
+                                                         phi::DataType::FLOAT32,
+                                                         phi::DataLayout::NCHW,
+                                                         2.0,
+                                                         true);
+    RetainGradForTensor(Y);
+
+    if (mode == "Accuracy") {
+      benchmark_eager_matmul(X, Y, true /* accuracy_check */);
+
+    } else if (mode == "Performance") {
+      auto t_start = std::chrono::high_resolution_clock::now();
+#ifdef WITH_GPERFTOOLS
+      ProfilerStart("eager_matmul_cpu.out");
+#endif
+      benchmark_eager_matmul(X, Y);
+
+#ifdef WITH_GPERFTOOLS
+      ProfilerStop();
+#endif
+      auto t_end = std::chrono::high_resolution_clock::now();
+      double elapsed_time_ms =
+          std::chrono::duration<double, std::milli>(t_end - t_start).count();
+      std::cout << "Duration: " << elapsed_time_ms << " ms" << std::endl;
+
+    } else {
+      PADDLE_THROW(common::errors::Fatal("Unknown benchmark mode"));
+    }
+  }
+}
+
+TEST(Benchmark, EagerIntermediateMatmulCPU) {
+  // Prepare Device Contexts
+  eager_test::InitEnv(phi::CPUPlace());
+
+  auto tracer = std::make_shared<paddle::imperative::Tracer>();
+  paddle::imperative::SetCurrentTracer(tracer);
+
+  for (const std::string mode : {"Accuracy", "Performance"}) {
+    phi::DDim ddimX = common::make_ddim({2, 2});
+    paddle::Tensor X = eager_test::CreateTensorWithValue(ddimX,
+                                                         phi::CPUPlace(),
+                                                         phi::DataType::FLOAT32,
+                                                         phi::DataLayout::NCHW,
+                                                         1.0,
+                                                         true);
+    RetainGradForTensor(X);
+
+    phi::DDim ddimY = common::make_ddim({2, 2});
+    paddle::Tensor Y = eager_test::CreateTensorWithValue(ddimY,
+                                                         phi::CPUPlace(),
+                                                         phi::DataType::FLOAT32,
+                                                         phi::DataLayout::NCHW,
+                                                         2.0,
+                                                         true);
+    RetainGradForTensor(Y);
+
+    if (mode == "Accuracy") {
+      benchmark_eager_intermediate_matmul(X, Y, true /* accuracy_check */);
+
+    } else if (mode == "Performance") {
+      auto t_start = std::chrono::high_resolution_clock::now();
+#ifdef WITH_GPERFTOOLS
+      ProfilerStart("eager_intermediate_matmul_cpu.out");
+#endif
+      benchmark_eager_intermediate_matmul(X, Y);
+
+#ifdef WITH_GPERFTOOLS
+      ProfilerStop();
+#endif
+      auto t_end = std::chrono::high_resolution_clock::now();
+      double elapsed_time_ms =
+          std::chrono::duration<double, std::milli>(t_end - t_start).count();
+      std::cout << "Duration: " << elapsed_time_ms << " ms" << std::endl;
+
+    } else {
+      PADDLE_THROW(common::errors::Fatal("Unknown benchmark mode"));
+    }
+  }
+}
+
+TEST(Benchmark, EagerIntermediateMLPCPU) {
+  // Prepare Device Contexts
+  eager_test::InitEnv(phi::CPUPlace());
+
+  auto tracer = std::make_shared<paddle::imperative::Tracer>();
+  paddle::imperative::SetCurrentTracer(tracer);
+
+  for (const std::string mode : {"Accuracy", "Performance"}) {
+    phi::DDim ddimX = common::make_ddim({MLP_M, MLP_N});
+    paddle::Tensor X = eager_test::CreateTensorWithValue(ddimX,
+                                                         phi::CPUPlace(),
+                                                         phi::DataType::FLOAT32,
+                                                         phi::DataLayout::NCHW,
+                                                         MLP_X_VAL,
+                                                         true);
+    RetainGradForTensor(X);
+
+    std::vector<paddle::Tensor> Ws;
+    std::vector<paddle::Tensor> Bs;
+    for (size_t i = 0; i < MLP_NUM_LINEAR; i++) {
+      phi::DDim ddimW = common::make_ddim({MLP_N, MLP_K});
+      paddle::Tensor W =
+          eager_test::CreateTensorWithValue(ddimW,
+                                            phi::CPUPlace(),
+                                            phi::DataType::FLOAT32,
+                                            phi::DataLayout::NCHW,
+                                            MLP_W_VAL,
+                                            true);
+      RetainGradForTensor(W);
+
+      phi::DDim ddimB = common::make_ddim({MLP_K});
+      paddle::Tensor B =
+          eager_test::CreateTensorWithValue(ddimB,
+                                            phi::CPUPlace(),
+                                            phi::DataType::FLOAT32,
+                                            phi::DataLayout::NCHW,
+                                            MLP_B_VAL,
+                                            true);
+      RetainGradForTensor(B);
+
+      Ws.emplace_back(std::move(W));
+      Bs.emplace_back(std::move(B));
+    }
+
+    if (mode == "Accuracy") {
+      benchmark_eager_intermediate_mlp(X, Ws, Bs, true /* accuracy_check */);
+
+    } else if (mode == "Performance") {
+      auto t_start = std::chrono::high_resolution_clock::now();
+#ifdef WITH_GPERFTOOLS
+      ProfilerStart("eager_intermediate_mlp_cpu.out");
+#endif
+      benchmark_eager_intermediate_mlp(X, Ws, Bs);
+
+#ifdef WITH_GPERFTOOLS
+      ProfilerStop();
+#endif
+      auto t_end = std::chrono::high_resolution_clock::now();
+      double elapsed_time_ms =
+          std::chrono::duration<double, std::milli>(t_end - t_start).count();
+      std::cout << "Duration: " << elapsed_time_ms << " ms" << std::endl;
+
+    } else {
+      PADDLE_THROW(common::errors::Fatal("Unknown benchmark mode"));
+    }
+  }
+}

test/cpp/eager/performance_tests/benchmark_eager_cuda.cc

@@ -0,0 +1,257 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Eager Dygraph
+#include <paddle/fluid/framework/op_registry.h>
+
+#include <chrono>
+
+#include "gtest/gtest.h"
+#include "paddle/common/flags.h"
+#include "paddle/fluid/eager/api/all.h"
+#include "paddle/fluid/eager/autograd_meta.h"
+#include "paddle/fluid/eager/backward.h"
+#include "paddle/fluid/imperative/tracer.h"
+#include "test/cpp/eager/performance_tests/benchmark_utils.h"
+#include "test/cpp/eager/test_utils.h"
+
+#ifdef WITH_GPERFTOOLS
+#include "gperftools/profiler.h"
+#endif
+
+#include "paddle/phi/core/kernel_registry.h"
+
+using namespace egr;            // NOLINT
+using namespace egr_utils_api;  // NOLINT
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+
+TEST(Benchmark, EagerScaleCUDA) {
+  eager_test::InitEnv(phi::GPUPlace());
+
+  for (const std::string mode : {"Accuracy", "WarmUp", "Performance"}) {
+    phi::DDim ddim = common::make_ddim({2, 4, 4, 4});
+    paddle::Tensor tensor =
+        eager_test::CreateTensorWithValue(ddim,
+                                          phi::GPUPlace(),
+                                          phi::DataType::FLOAT32,
+                                          phi::DataLayout::NCHW,
+                                          5.0 /*value*/,
+                                          true /*is_leaf*/);
+    RetainGradForTensor(tensor);
+
+    if (mode == "Accuracy") {
+      benchmark_eager_scale(tensor, true /* accuracy_check */);
+
+    } else if (mode == "WarmUp") {
+      benchmark_eager_scale(tensor);
+
+    } else if (mode == "Performance") {
+      auto t_start = std::chrono::high_resolution_clock::now();
+#ifdef WITH_GPERFTOOLS
+      ProfilerStart("eager_scale_cuda.out");
+#endif
+      benchmark_eager_scale(tensor);
+
+#ifdef WITH_GPERFTOOLS
+      ProfilerStop();
+#endif
+      auto t_end = std::chrono::high_resolution_clock::now();
+      double elapsed_time_ms =
+          std::chrono::duration<double, std::milli>(t_end - t_start).count();
+      std::cout << "Duration: " << elapsed_time_ms << " ms" << std::endl;
+
+    } else {
+      PADDLE_THROW(common::errors::Fatal("Unknown benchmark mode"));
+    }
+  }
+}
+
+TEST(Benchmark, EagerMatmulCUDA) {
+  phi::GPUPlace place;
+  eager_test::InitEnv(place);
+
+  for (const std::string mode : {"Accuracy", "WarmUp", "Performance"}) {
+    phi::DDim ddimX = common::make_ddim({2, 2});
+    paddle::Tensor X = eager_test::CreateTensorWithValue(ddimX,
+                                                         phi::GPUPlace(),
+                                                         phi::DataType::FLOAT32,
+                                                         phi::DataLayout::NCHW,
+                                                         1.0,
+                                                         true);
+    RetainGradForTensor(X);
+
+    phi::DDim ddimY = common::make_ddim({2, 2});
+    paddle::Tensor Y = eager_test::CreateTensorWithValue(ddimY,
+                                                         phi::GPUPlace(),
+                                                         phi::DataType::FLOAT32,
+                                                         phi::DataLayout::NCHW,
+                                                         2.0,
+                                                         true);
+    RetainGradForTensor(Y);
+
+    if (mode == "Accuracy") {
+      benchmark_eager_matmul(X, Y, true /* accuracy_check */);
+
+    } else if (mode == "WarmUp") {
+      benchmark_eager_matmul(X, Y);
+
+    } else if (mode == "Performance") {
+      auto t_start = std::chrono::high_resolution_clock::now();
+#ifdef WITH_GPERFTOOLS
+      ProfilerStart("eager_matmul_cuda.out");
+#endif
+      benchmark_eager_matmul(X, Y);
+
+#ifdef WITH_GPERFTOOLS
+      ProfilerStop();
+#endif
+      auto t_end = std::chrono::high_resolution_clock::now();
+      double elapsed_time_ms =
+          std::chrono::duration<double, std::milli>(t_end - t_start).count();
+      std::cout << "Duration: " << elapsed_time_ms << " ms" << std::endl;
+
+    } else {
+      PADDLE_THROW(common::errors::Fatal("Unknown benchmark mode"));
+    }
+  }
+}
+
+TEST(Benchmark, EagerIntermediateMatmulCUDA) {
+  phi::GPUPlace place;
+  eager_test::InitEnv(place);
+
+  auto tracer = std::make_shared<paddle::imperative::Tracer>();
+  tracer->SetExpectedPlace(place);
+  paddle::imperative::SetCurrentTracer(tracer);
+
+  for (const std::string mode : {"Accuracy", "WarmUp", "Performance"}) {
+    phi::DDim ddimX = common::make_ddim({2, 2});
+    paddle::Tensor X = eager_test::CreateTensorWithValue(ddimX,
+                                                         phi::GPUPlace(),
+                                                         phi::DataType::FLOAT32,
+                                                         phi::DataLayout::NCHW,
+                                                         1.0,
+                                                         true);
+    RetainGradForTensor(X);
+
+    phi::DDim ddimY = common::make_ddim({2, 2});
+    paddle::Tensor Y = eager_test::CreateTensorWithValue(ddimY,
+                                                         phi::GPUPlace(),
+                                                         phi::DataType::FLOAT32,
+                                                         phi::DataLayout::NCHW,
+                                                         2.0,
+                                                         true);
+    RetainGradForTensor(Y);
+
+    if (mode == "Accuracy") {
+      benchmark_eager_intermediate_matmul(X, Y, true /* accuracy_check */);
+
+    } else if (mode == "WarmUp") {
+      benchmark_eager_intermediate_matmul(X, Y);
+
+    } else if (mode == "Performance") {
+      auto t_start = std::chrono::high_resolution_clock::now();
+#ifdef WITH_GPERFTOOLS
+      ProfilerStart("eager_intermediate_matmul_cuda.out");
+#endif
+      benchmark_eager_intermediate_matmul(X, Y);
+
+#ifdef WITH_GPERFTOOLS
+      ProfilerStop();
+#endif
+      auto t_end = std::chrono::high_resolution_clock::now();
+      double elapsed_time_ms =
+          std::chrono::duration<double, std::milli>(t_end - t_start).count();
+      std::cout << "Duration: " << elapsed_time_ms << " ms" << std::endl;
+
+    } else {
+      PADDLE_THROW(common::errors::Fatal("Unknown benchmark mode"));
+    }
+  }
+}
+
+TEST(Benchmark, EagerIntermediateMLPCUDA) {
+  phi::GPUPlace place;
+  eager_test::InitEnv(place);
+
+  auto tracer = std::make_shared<paddle::imperative::Tracer>();
+  tracer->SetExpectedPlace(place);
+  paddle::imperative::SetCurrentTracer(tracer);
+
+  for (const std::string mode : {"Accuracy", "WarmUp", "Performance"}) {
+    phi::DDim ddimX = common::make_ddim({MLP_M, MLP_N});
+    paddle::Tensor X = eager_test::CreateTensorWithValue(ddimX,
+                                                         phi::GPUPlace(),
+                                                         phi::DataType::FLOAT32,
+                                                         phi::DataLayout::NCHW,
+                                                         MLP_X_VAL,
+                                                         true);
+    RetainGradForTensor(X);
+
+    std::vector<paddle::Tensor> Ws;
+    std::vector<paddle::Tensor> Bs;
+    for (size_t i = 0; i < MLP_NUM_LINEAR; i++) {
+      phi::DDim ddimW = common::make_ddim({MLP_N, MLP_K});
+      paddle::Tensor W =
+          eager_test::CreateTensorWithValue(ddimW,
+                                            phi::GPUPlace(),
+                                            phi::DataType::FLOAT32,
+                                            phi::DataLayout::NCHW,
+                                            MLP_W_VAL,
+                                            true);
+      RetainGradForTensor(W);
+
+      phi::DDim ddimB = common::make_ddim({MLP_K});
+      paddle::Tensor B =
+          eager_test::CreateTensorWithValue(ddimB,
+                                            phi::GPUPlace(),
+                                            phi::DataType::FLOAT32,
+                                            phi::DataLayout::NCHW,
+                                            MLP_B_VAL,
+                                            true);
+      RetainGradForTensor(B);
+
+      Ws.emplace_back(std::move(W));
+      Bs.emplace_back(std::move(B));
+    }
+
+    if (mode == "Accuracy") {
+      benchmark_eager_intermediate_mlp(X, Ws, Bs, true /* accuracy_check */);
+
+    } else if (mode == "WarmUp") {
+      benchmark_eager_intermediate_mlp(X, Ws, Bs);
+
+    } else if (mode == "Performance") {
+      auto t_start = std::chrono::high_resolution_clock::now();
+#ifdef WITH_GPERFTOOLS
+      ProfilerStart("eager_intermediate_mlp_cuda.out");
+#endif
+      benchmark_eager_intermediate_mlp(X, Ws, Bs);
+
+#ifdef WITH_GPERFTOOLS
+      ProfilerStop();
+#endif
+      auto t_end = std::chrono::high_resolution_clock::now();
+      double elapsed_time_ms =
+          std::chrono::duration<double, std::milli>(t_end - t_start).count();
+      std::cout << "Duration: " << elapsed_time_ms << " ms" << std::endl;
+
+    } else {
+      PADDLE_THROW(common::errors::Fatal("Unknown benchmark mode"));
+    }
+  }
+}
+
+#endif  // PADDLE_WITH_CUDA || PADDLE_WITH_HIP

test/cpp/eager/performance_tests/benchmark_fluid_cpu.cc

@@ -0,0 +1,230 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <paddle/fluid/framework/op_registry.h>
+
+#include <chrono>
+#include <iostream>
+#include <memory>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/fluid/imperative/basic_engine.h"
+#include "paddle/fluid/imperative/tracer.h"
+#include "paddle/phi/core/memory/memcpy.h"
+#include "test/cpp/eager/performance_tests/benchmark_utils.h"
+#include "test/cpp/eager/test_utils.h"
+
+#ifdef WITH_GPERFTOOLS
+#include "gperftools/profiler.h"
+#endif
+
+#include "paddle/phi/core/kernel_registry.h"
+
+namespace paddle {
+namespace imperative {
+
+TEST(Benchmark, FluidScaleCPU) {
+  // Prepare Device Contexts
+  phi::CPUPlace place;
+  eager_test::InitEnv(place);
+
+  for (const std::string mode : {"Accuracy", "Performance"}) {
+    std::shared_ptr<imperative::VarBase> X(new imperative::VarBase(true, "X"));
+    X->SetOverriddenStopGradient(false);
+
+    std::vector<float> src_data(128, 5.0);
+    std::vector<int64_t> dims = {2, 4, 4, 4};
+
+    auto* x_tensor = X->MutableVar()->GetMutable<phi::DenseTensor>();
+    x_tensor->Resize(common::make_ddim(dims));
+    auto* mutable_x = x_tensor->mutable_data<float>(place);
+    paddle::memory::Copy(place,
+                         mutable_x,
+                         place,
+                         src_data.data(),
+                         sizeof(float) * src_data.size());
+
+    if (mode == "Accuracy") {
+      benchmark_fluid_scale(X, phi::Place(place), true /* accuracy_check */);
+
+    } else if (mode == "Performance") {
+      auto t_start = std::chrono::high_resolution_clock::now();
+#ifdef WITH_GPERFTOOLS
+      ProfilerStart("fluid_scale_cpu.out");
+#endif
+      benchmark_fluid_scale(X, phi::Place(place));
+
+#ifdef WITH_GPERFTOOLS
+      ProfilerStop();
+#endif
+      auto t_end = std::chrono::high_resolution_clock::now();
+      double elapsed_time_ms =
+          std::chrono::duration<double, std::milli>(t_end - t_start).count();
+      std::cout << "Duration: " << elapsed_time_ms << " ms" << std::endl;
+
+    } else {
+      PADDLE_THROW(common::errors::Fatal("Unknown benchmark mode"));
+    }
+  }
+}
+
+TEST(Benchmark, FluidMatmulCPU) {
+  // Prepare Device Contexts
+  phi::CPUPlace place;
+  eager_test::InitEnv(place);
+
+  for (const std::string mode : {"Accuracy", "Performance"}) {
+    std::shared_ptr<imperative::VarBase> X(new imperative::VarBase(true, "X"));
+    X->SetOverriddenStopGradient(false);
+    std::shared_ptr<imperative::VarBase> Y(new imperative::VarBase(true, "Y"));
+    Y->SetOverriddenStopGradient(false);
+
+    std::vector<float> x_src_data(4, 1.0);
+    std::vector<float> y_src_data(4, 2.0);
+    std::vector<int64_t> dims = {2, 2};
+
+    auto* x_tensor = X->MutableVar()->GetMutable<phi::DenseTensor>();
+    x_tensor->Resize(common::make_ddim(dims));
+    auto* mutable_x = x_tensor->mutable_data<float>(place);
+    paddle::memory::Copy(place,
+                         mutable_x,
+                         place,
+                         x_src_data.data(),
+                         sizeof(float) * x_src_data.size());
+
+    auto* y_tensor = Y->MutableVar()->GetMutable<phi::DenseTensor>();
+    y_tensor->Resize(common::make_ddim(dims));
+    auto* mutable_y = y_tensor->mutable_data<float>(place);
+    paddle::memory::Copy(place,
+                         mutable_y,
+                         place,
+                         y_src_data.data(),
+                         sizeof(float) * y_src_data.size());
+
+    if (mode == "Accuracy") {
+      benchmark_fluid_matmul(
+          X, Y, phi::Place(place), true /* accuracy_check */);
+
+    } else if (mode == "Performance") {
+      auto t_start = std::chrono::high_resolution_clock::now();
+#ifdef WITH_GPERFTOOLS
+      ProfilerStart("fluid_matmul_cpu.out");
+#endif
+      benchmark_fluid_matmul(X, Y, phi::Place(place));
+
+#ifdef WITH_GPERFTOOLS
+      ProfilerStop();
+#endif
+      auto t_end = std::chrono::high_resolution_clock::now();
+      double elapsed_time_ms =
+          std::chrono::duration<double, std::milli>(t_end - t_start).count();
+
+      std::cout << "Duration: " << elapsed_time_ms << " ms" << std::endl;
+
+    } else {
+      PADDLE_THROW(common::errors::Fatal("Unknown benchmark mode"));
+    }
+  }
+}
+
+TEST(Benchmark, FluidMLPCPU) {
+  // Prepare Device Contexts
+  phi::CPUPlace place;
+  eager_test::InitEnv(place);
+
+  for (const std::string mode : {"Accuracy", "Performance"}) {
+    std::vector<float> x_src_data(MLP_M * MLP_N, MLP_X_VAL);
+    std::vector<float> w_src_data(MLP_N * MLP_K, MLP_W_VAL);
+    std::vector<float> b_src_data(MLP_K, MLP_B_VAL);
+
+    std::vector<int64_t> x_dims = {MLP_M, MLP_N};
+    std::vector<int64_t> w_dims = {MLP_N, MLP_K};
+    std::vector<int64_t> b_dims = {MLP_K};
+
+    std::shared_ptr<imperative::VarBase> X(new imperative::VarBase(true, "X"));
+    X->SetOverriddenStopGradient(false);
+
+    auto* x_tensor = X->MutableVar()->GetMutable<phi::DenseTensor>();
+    x_tensor->Resize(common::make_ddim(x_dims));
+    auto* mutable_x = x_tensor->mutable_data<float>(place);
+    paddle::memory::Copy(place,
+                         mutable_x,
+                         place,
+                         x_src_data.data(),
+                         sizeof(float) * x_src_data.size());
+
+    std::vector<std::shared_ptr<imperative::VarBase>> Ws;
+    std::vector<std::shared_ptr<imperative::VarBase>> Bs;
+    for (size_t i = 0; i < MLP_NUM_LINEAR; i++) {
+      std::shared_ptr<imperative::VarBase> W(
+          new imperative::VarBase(true, "W"));
+      W->SetOverriddenStopGradient(false);
+      std::shared_ptr<imperative::VarBase> B(
+          new imperative::VarBase(true, "B"));
+      B->SetOverriddenStopGradient(false);
+
+      auto* w_tensor = W->MutableVar()->GetMutable<phi::DenseTensor>();
+      w_tensor->Resize(common::make_ddim(w_dims));
+      auto* mutable_w = w_tensor->mutable_data<float>(place);
+      paddle::memory::Copy(place,
+                           mutable_w,
+                           place,
+                           w_src_data.data(),
+                           sizeof(float) * w_src_data.size());
+
+      auto* b_tensor = B->MutableVar()->GetMutable<phi::DenseTensor>();
+      b_tensor->Resize(common::make_ddim(b_dims));
+      auto* mutable_b = b_tensor->mutable_data<float>(place);
+      paddle::memory::Copy(place,
+                           mutable_b,
+                           place,
+                           b_src_data.data(),
+                           sizeof(float) * b_src_data.size());
+
+      Ws.emplace_back(std::move(W));
+      Bs.emplace_back(std::move(B));
+    }
+
+    if (mode == "Accuracy") {
+      benchmark_fluid_mlp(
+          X, Ws, Bs, phi::Place(place), true /* accuracy_check */);
+
+    } else if (mode == "Performance") {
+      auto t_start = std::chrono::high_resolution_clock::now();
+#ifdef WITH_GPERFTOOLS
+      ProfilerStart("fluid_mlp_cpu.out");
+#endif
+      benchmark_fluid_mlp(X, Ws, Bs, phi::Place(place));
+
+#ifdef WITH_GPERFTOOLS
+      ProfilerStop();
+#endif
+      auto t_end = std::chrono::high_resolution_clock::now();
+      double elapsed_time_ms =
+          std::chrono::duration<double, std::milli>(t_end - t_start).count();
+
+      std::cout << "Duration: " << elapsed_time_ms << " ms" << std::endl;
+
+    } else {
+      PADDLE_THROW(common::errors::Fatal("Unknown benchmark mode"));
+    }
+  }
+}
+
+}  // namespace imperative
+}  // namespace paddle

test/cpp/eager/performance_tests/benchmark_fluid_cuda.cc

@@ -0,0 +1,258 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <paddle/fluid/framework/op_registry.h>
+
+#include <chrono>
+#include <iostream>
+#include <memory>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/fluid/imperative/basic_engine.h"
+#include "paddle/fluid/imperative/tracer.h"
+#include "paddle/phi/core/memory/memcpy.h"
+#include "test/cpp/eager/performance_tests/benchmark_utils.h"
+#include "test/cpp/eager/test_utils.h"
+
+#ifdef WITH_GPERFTOOLS
+#include "gperftools/profiler.h"
+#endif
+
+#include "paddle/phi/core/kernel_registry.h"
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+namespace paddle {
+namespace imperative {
+
+TEST(Benchmark, FluidScaleCUDA) {
+  // Prepare Device Contexts
+  phi::GPUPlace place;
+  eager_test::InitEnv(place);
+
+  for (const std::string mode : {"Accuracy", "WarmUp", "Performance"}) {
+    std::shared_ptr<imperative::VarBase> X(new imperative::VarBase(true, "X"));
+    X->SetOverriddenStopGradient(false);
+
+    std::vector<float> src_data(128, 5.0);
+    std::vector<int64_t> dims = {2, 4, 4, 4};
+
+    auto* x_tensor = X->MutableVar()->GetMutable<phi::DenseTensor>();
+    x_tensor->Resize(common::make_ddim(dims));
+    auto* mutable_x = x_tensor->mutable_data<float>(place);
+
+    phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+    auto* dev_ctx = dynamic_cast<phi::GPUContext*>(pool.Get(place));
+    auto stream = dev_ctx->stream();
+    paddle::memory::Copy(place,
+                         mutable_x,
+                         phi::CPUPlace(),
+                         src_data.data(),
+                         sizeof(float) * src_data.size(),
+                         stream);
+
+    if (mode == "Accuracy") {
+      benchmark_fluid_scale(X, phi::Place(place), true /* accuracy_check */);
+
+    } else if (mode == "WarmUp") {
+      benchmark_fluid_scale(X, phi::Place(place));
+
+    } else if (mode == "Performance") {
+      auto t_start = std::chrono::high_resolution_clock::now();
+#ifdef WITH_GPERFTOOLS
+      ProfilerStart("fluid_scale_cuda.out");
+#endif
+      benchmark_fluid_scale(X, phi::Place(place));
+
+#ifdef WITH_GPERFTOOLS
+      ProfilerStop();
+#endif
+      auto t_end = std::chrono::high_resolution_clock::now();
+      double elapsed_time_ms =
+          std::chrono::duration<double, std::milli>(t_end - t_start).count();
+      std::cout << "Duration: " << elapsed_time_ms << " ms" << std::endl;
+
+    } else {
+      PADDLE_THROW(common::errors::Fatal("Unknown benchmark mode"));
+    }
+  }
+}
+
+TEST(Benchmark, FluidMatmulCUDA) {
+  // Prepare Device Contexts
+  phi::GPUPlace place;
+  eager_test::InitEnv(place);
+
+  for (const std::string mode : {"Accuracy", "WarmUp", "Performance"}) {
+    std::shared_ptr<imperative::VarBase> X(new imperative::VarBase(true, "X"));
+    X->SetOverriddenStopGradient(false);
+    std::shared_ptr<imperative::VarBase> Y(new imperative::VarBase(true, "Y"));
+    Y->SetOverriddenStopGradient(false);
+
+    std::vector<float> x_src_data(4, 1.0);
+    std::vector<float> y_src_data(4, 2.0);
+    std::vector<int64_t> dims = {2, 2};
+
+    phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+    auto* dev_ctx = dynamic_cast<phi::GPUContext*>(pool.Get(place));
+    auto stream = dev_ctx->stream();
+
+    auto* x_tensor = X->MutableVar()->GetMutable<phi::DenseTensor>();
+    x_tensor->Resize(common::make_ddim(dims));
+    auto* mutable_x = x_tensor->mutable_data<float>(place);
+    paddle::memory::Copy(place,
+                         mutable_x,
+                         phi::CPUPlace(),
+                         x_src_data.data(),
+                         sizeof(float) * x_src_data.size(),
+                         stream);
+
+    auto* y_tensor = Y->MutableVar()->GetMutable<phi::DenseTensor>();
+    y_tensor->Resize(common::make_ddim(dims));
+    auto* mutable_y = y_tensor->mutable_data<float>(place);
+    paddle::memory::Copy(place,
+                         mutable_y,
+                         phi::CPUPlace(),
+                         y_src_data.data(),
+                         sizeof(float) * y_src_data.size(),
+                         stream);
+
+    if (mode == "Accuracy") {
+      benchmark_fluid_matmul(
+          X, Y, phi::Place(place), true /* accuracy_check */);
+
+    } else if (mode == "WarmUp") {
+      benchmark_fluid_matmul(X, Y, phi::Place(place));
+
+    } else if (mode == "Performance") {
+      auto t_start = std::chrono::high_resolution_clock::now();
+#ifdef WITH_GPERFTOOLS
+      ProfilerStart("fluid_matmul_cuda.out");
+#endif
+      benchmark_fluid_matmul(X, Y, phi::Place(place));
+
+#ifdef WITH_GPERFTOOLS
+      ProfilerStop();
+#endif
+      auto t_end = std::chrono::high_resolution_clock::now();
+      double elapsed_time_ms =
+          std::chrono::duration<double, std::milli>(t_end - t_start).count();
+      std::cout << "Duration: " << elapsed_time_ms << " ms" << std::endl;
+
+    } else {
+      PADDLE_THROW(common::errors::Fatal("Unknown benchmark mode"));
+    }
+  }
+}
+
+TEST(Benchmark, FluidMLPCUDA) {
+  // Prepare Device Contexts
+  phi::GPUPlace place;
+  eager_test::InitEnv(place);
+
+  for (const std::string mode : {"Accuracy", "WarmUp", "Performance"}) {
+    phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+    auto* dev_ctx = dynamic_cast<phi::GPUContext*>(pool.Get(place));
+    auto stream = dev_ctx->stream();
+
+    std::vector<float> x_src_data(MLP_M * MLP_N, MLP_X_VAL);
+    std::vector<float> w_src_data(MLP_N * MLP_K, MLP_W_VAL);
+    std::vector<float> b_src_data(MLP_K, MLP_B_VAL);
+
+    std::vector<int64_t> x_dims = {MLP_M, MLP_N};
+    std::vector<int64_t> w_dims = {MLP_N, MLP_K};
+    std::vector<int64_t> b_dims = {MLP_K};
+
+    std::shared_ptr<imperative::VarBase> X(new imperative::VarBase(true, "X"));
+    X->SetOverriddenStopGradient(false);
+
+    auto* x_tensor = X->MutableVar()->GetMutable<phi::DenseTensor>();
+    x_tensor->Resize(common::make_ddim(x_dims));
+    auto* mutable_x = x_tensor->mutable_data<float>(place);
+    paddle::memory::Copy(place,
+                         mutable_x,
+                         phi::CPUPlace(),
+                         x_src_data.data(),
+                         sizeof(float) * x_src_data.size(),
+                         stream);
+
+    std::vector<std::shared_ptr<imperative::VarBase>> Ws;
+    std::vector<std::shared_ptr<imperative::VarBase>> Bs;
+    for (size_t i = 0; i < MLP_NUM_LINEAR; i++) {
+      std::shared_ptr<imperative::VarBase> W(
+          new imperative::VarBase(true, "W"));
+      W->SetOverriddenStopGradient(false);
+      std::shared_ptr<imperative::VarBase> B(
+          new imperative::VarBase(true, "B"));
+      B->SetOverriddenStopGradient(false);
+
+      auto* w_tensor = W->MutableVar()->GetMutable<phi::DenseTensor>();
+      w_tensor->Resize(common::make_ddim(w_dims));
+      auto* mutable_w = w_tensor->mutable_data<float>(place);
+      paddle::memory::Copy(place,
+                           mutable_w,
+                           phi::CPUPlace(),
+                           w_src_data.data(),
+                           sizeof(float) * w_src_data.size(),
+                           stream);
+
+      auto* b_tensor = B->MutableVar()->GetMutable<phi::DenseTensor>();
+      b_tensor->Resize(common::make_ddim(b_dims));
+      auto* mutable_b = b_tensor->mutable_data<float>(place);
+      paddle::memory::Copy(place,
+                           mutable_b,
+                           phi::CPUPlace(),
+                           b_src_data.data(),
+                           sizeof(float) * b_src_data.size(),
+                           stream);
+
+      Ws.emplace_back(std::move(W));
+      Bs.emplace_back(std::move(B));
+    }
+
+    if (mode == "Accuracy") {
+      benchmark_fluid_mlp(
+          X, Ws, Bs, phi::Place(place), true /* accuracy_check */);
+
+    } else if (mode == "WarmUp") {
+      benchmark_fluid_mlp(X, Ws, Bs, phi::Place(place));
+
+    } else if (mode == "Performance") {
+      auto t_start = std::chrono::high_resolution_clock::now();
+#ifdef WITH_GPERFTOOLS
+      ProfilerStart("fluid_mlp_cuda.out");
+#endif
+      benchmark_fluid_mlp(X, Ws, Bs, phi::Place(place));
+
+#ifdef WITH_GPERFTOOLS
+      ProfilerStop();
+#endif
+      auto t_end = std::chrono::high_resolution_clock::now();
+      double elapsed_time_ms =
+          std::chrono::duration<double, std::milli>(t_end - t_start).count();
+
+      std::cout << "Duration: " << elapsed_time_ms << " ms" << std::endl;
+
+    } else {
+      PADDLE_THROW(common::errors::Fatal("Unknown benchmark mode"));
+    }
+  }
+}
+
+}  // namespace imperative
+}  // namespace paddle
+#endif  // PADDLE_WITH_CUDA || PADDLE_WITH_HIP

test/cpp/eager/performance_tests/benchmark_utils.cc

@@ -0,0 +1,348 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test/cpp/eager/performance_tests/benchmark_utils.h"
+
+#include <iostream>
+#include <memory>
+#include <set>
+#include <string>
+#include <vector>
+
+// Eager
+#include "paddle/fluid/eager/api/all.h"
+#include "paddle/fluid/eager/autograd_meta.h"
+#include "paddle/fluid/eager/backward.h"
+#include "paddle/fluid/eager/utils.h"
+#include "test/cpp/eager/test_utils.h"
+
+// Eager Generated
+#include "paddle/fluid/eager/api/generated/eager_generated/forwards/dygraph_functions.h"
+#include "paddle/fluid/eager/api/generated/fluid_generated/dygraph_forward_api.h"
+
+// Fluid
+#include "paddle/fluid/framework/op_registry.h"
+#include "paddle/fluid/imperative/basic_engine.h"
+#include "paddle/fluid/imperative/tracer.h"
+#include "paddle/phi/core/memory/memcpy.h"
+
+static size_t max_num_benchmark_runs = 4000;
+
+namespace egr {
+
+/* --------------------- */
+/* ---- Eager Scale ---- */
+/* --------------------- */
+void benchmark_eager_scale(const paddle::Tensor& tensor, bool accuracy_check) {
+  paddle::Tensor input_tensor = tensor;
+  float scale = 2.0;
+  float bias = 3.0;
+
+  size_t max_num_runs = accuracy_check ? 10 : max_num_benchmark_runs;
+  for (size_t i = 0; i < max_num_runs; i++) {
+    input_tensor = egr::scale(input_tensor,
+                              scale,
+                              bias,
+                              true /*bias_after_scale*/,
+                              true /*trace_backward*/);
+  }
+
+  std::vector<paddle::Tensor> target_tensors = {input_tensor};
+  Backward(target_tensors, {});
+
+  if (accuracy_check) {
+    // Examine Forward Grad (w.r.t max_num_runs = 10)
+    eager_test::CompareTensorWithValue<float>(input_tensor, 8189.0);
+    // Examine Backward Grad (w.r.t max_num_runs = 10)
+    eager_test::CompareGradTensorWithValue<float>(tensor, 1024.0);
+  }
+}
+
+void benchmark_eager_matmul(const paddle::Tensor& X,
+                            const paddle::Tensor& Y,
+                            bool accuracy_check) {
+  paddle::Tensor input_tensor0 = X;
+
+  size_t max_num_runs = accuracy_check ? 2 : max_num_benchmark_runs;
+  for (size_t i = 0; i < max_num_runs; i++) {
+    input_tensor0 = matmul_ad_func(input_tensor0, Y, false, false);
+  }
+
+  std::vector<paddle::Tensor> target_tensors = {input_tensor0};
+  Backward(target_tensors, {});
+
+  if (accuracy_check) {
+    // Examine Forward Grad (w.r.t max_num_runs = 2)
+    eager_test::CompareTensorWithValue<float>(input_tensor0, 16);
+    // Examine Backward Grad (w.r.t max_num_runs = 2)
+    eager_test::CompareGradTensorWithValue<float>(X, 16);
+    eager_test::CompareGradTensorWithValue<float>(Y, 16);
+  }
+}
+
+/* ----------------------------------- */
+/* ---- Eager Intermediate Matmul ---- */
+/* ----------------------------------- */
+void benchmark_eager_intermediate_matmul(const paddle::Tensor& X,
+                                         const paddle::Tensor& Y,
+                                         bool accuracy_check) {
+  paddle::Tensor input_tensor0 = X;
+
+  size_t max_num_runs = accuracy_check ? 2 : max_num_benchmark_runs;
+  for (size_t i = 0; i < max_num_runs; i++) {
+    input_tensor0 = matmul_v2_dygraph_function(
+        input_tensor0, Y, {{"trans_x", false}, {"trans_y", false}});
+  }
+
+  std::vector<paddle::Tensor> target_tensors = {input_tensor0};
+  Backward(target_tensors, {});
+
+  if (accuracy_check) {
+    // Examine Forward Grad (w.r.t max_num_runs = 2)
+    eager_test::CompareTensorWithValue<float>(input_tensor0, 16);
+    // Examine Backward Grad (w.r.t max_num_runs = 2)
+    eager_test::CompareGradTensorWithValue<float>(X, 16);
+    eager_test::CompareGradTensorWithValue<float>(Y, 16);
+  }
+}
+
+/* -------------------------------- */
+/* ---- Eager Intermediate MLP ---- */
+/* -------------------------------- */
+void benchmark_eager_intermediate_mlp(const paddle::Tensor& X,
+                                      const std::vector<paddle::Tensor>& Ws,
+                                      const std::vector<paddle::Tensor>& Bs,
+                                      bool accuracy_check) {
+  paddle::Tensor input0 = X;
+
+  for (size_t i = 0; i < MLP_NUM_LINEAR; i++) {
+    paddle::Tensor Out = matmul_v2_dygraph_function(
+        input0, Ws[i], {{"trans_x", false}, {"trans_y", false}});
+
+    input0 = elementwise_add_dygraph_function(Out, Bs[i], {});
+  }
+
+  paddle::Tensor Out =
+      reduce_sum_dygraph_function(input0, {{"reduce_all", true}});
+
+  std::vector<paddle::Tensor> target_tensors = {Out};
+  Backward(target_tensors, {});
+
+  if (accuracy_check) {
+    std::unordered_map<std::string, float> result =
+        compute_mlp_expected_results();
+
+    // Examine Forward Grad (w.r.t max_num_runs = 2)
+    eager_test::CompareTensorWithValue<float>(Out, result["Out"]);
+
+    // Examine Backward Grad (w.r.t max_num_runs = 2)
+    eager_test::CompareGradTensorWithValue<float>(X, result["GradX"]);
+    eager_test::CompareGradTensorWithValue<float>(Ws[0], result["GradW"]);
+  }
+}
+
+}  // namespace egr
+
+namespace paddle {
+namespace imperative {
+
+static void FluidCheckTensorValue(const std::shared_ptr<imperative::VarBase>& X,
+                                  const phi::Place& place,
+                                  float value) {
+  auto* tensor = X->MutableVar()->GetMutable<phi::DenseTensor>();
+  float* t_ptr = tensor->mutable_data<float>(place);
+  std::vector<float> host_data(tensor->numel());
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  if (place == phi::GPUPlace()) {
+    phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+    auto* dev_ctx = dynamic_cast<phi::GPUContext*>(pool.Get(place));
+    auto stream = dev_ctx->stream();
+
+    paddle::memory::Copy(phi::CPUPlace(),
+                         host_data.data(),
+                         phi::GPUPlace(),
+                         t_ptr,
+                         sizeof(float) * tensor->numel(),
+                         stream);
+    t_ptr = host_data.data();
+  }
+#endif
+
+  VLOG(6) << "Tensor Value: " << t_ptr[0] << ", Expected Value: " << value;
+  PADDLE_ENFORCE(
+      t_ptr[0] == value,
+      common::errors::Fatal(
+          "Detected numerical Error, Expected %f but got %f", value, t_ptr[0]));
+}
+
+static void FluidCheckGradTensorValue(
+    const std::shared_ptr<imperative::VarBase>& X,
+    const phi::Place& place,
+    float value) {
+  auto* grad_tensor = X->MutableGradVar()->GetMutable<phi::DenseTensor>();
+  float* g_ptr = grad_tensor->mutable_data<float>(place);
+  std::vector<float> g_host_data(grad_tensor->numel());
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  if (place == phi::GPUPlace()) {
+    phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+    auto* dev_ctx = dynamic_cast<phi::GPUContext*>(pool.Get(place));
+    auto stream = dev_ctx->stream();
+
+    paddle::memory::Copy(phi::CPUPlace(),
+                         g_host_data.data(),
+                         phi::GPUPlace(),
+                         g_ptr,
+                         sizeof(float) * grad_tensor->numel(),
+                         stream);
+    g_ptr = g_host_data.data();
+  }
+#endif
+
+  VLOG(6) << "Tensor Value: " << g_ptr[0] << ", Expected Value: " << value;
+  PADDLE_ENFORCE(
+      g_ptr[0] == value,
+      common::errors::Fatal(
+          "Detected numerical Error, Expected %f but got %f", value, g_ptr[0]));
+}
+
+/* --------------------- */
+/* ---- Fluid Scale ---- */
+/* --------------------- */
+// TODO(jiabin): Change this and remove nolint
+void benchmark_fluid_scale(const std::shared_ptr<imperative::VarBase>& X,
+                           const phi::Place& place,
+                           bool accuracy_check) {
+  imperative::Tracer tracer;
+  framework::AttributeMap attrs;
+
+  attrs["use_onednn"] = false;
+  attrs["scale"] = 2;
+  attrs["bias"] = 3;
+  attrs["bias_after_scale"] = true;
+
+  std::shared_ptr<imperative::VarBase> tmp_out = X;
+
+  size_t max_num_runs = accuracy_check ? 10 : max_num_benchmark_runs;
+  for (size_t i = 0; i < max_num_runs; i++) {
+    imperative::NameVarBaseMap ins = {{"X", {tmp_out}}};
+    imperative::NameVarBaseMap outs = {
+        {"Out", {std::make_shared<imperative::VarBase>(true, "Out")}}};
+
+    tracer.TraceOp<VarBase>("scale", ins, outs, attrs, place, true);
+
+    tmp_out = outs["Out"][0];
+  }
+
+  auto* engine = tracer.GetEngine();
+  std::vector<std::shared_ptr<imperative::VarBase>> grad_tensors{nullptr};
+  engine->Init({tmp_out}, grad_tensors, false /*retain_graph*/);
+  engine->Execute();
+
+  if (accuracy_check) {
+    FluidCheckTensorValue(tmp_out, place, 8189.0);
+    FluidCheckGradTensorValue(X, place, 1024.0);
+  }
+}
+
+/* ---------------------- */
+/* ---- Fluid Matmul ---- */
+/* ---------------------- */
+void benchmark_fluid_matmul(const std::shared_ptr<imperative::VarBase>& X,
+                            const std::shared_ptr<imperative::VarBase>& Y,
+                            const phi::Place& place,
+                            bool accuracy_check) {
+  imperative::Tracer tracer;
+
+  std::shared_ptr<imperative::VarBase> tmp_out = X;
+
+  size_t max_num_runs = accuracy_check ? 2 : max_num_benchmark_runs;
+  for (size_t i = 0; i < max_num_runs; i++) {
+    framework::AttributeMap attrs;
+    imperative::NameVarBaseMap ins = {{"X", {tmp_out}}, {"Y", {Y}}};
+    imperative::NameVarBaseMap outs = {
+        {"Out", {std::make_shared<imperative::VarBase>(true, "Out")}}};
+
+    tracer.TraceOp<VarBase>("matmul_v2", ins, outs, attrs, place, true);
+
+    tmp_out = outs["Out"][0];
+  }
+
+  auto* engine = tracer.GetEngine();
+  std::vector<std::shared_ptr<imperative::VarBase>> grad_tensors{nullptr};
+  engine->Init({tmp_out}, grad_tensors, false /*retain_graph*/);
+  engine->Execute();
+
+  if (accuracy_check) {
+    FluidCheckTensorValue(tmp_out, place, 16);
+    FluidCheckGradTensorValue(X, place, 16);
+    FluidCheckGradTensorValue(Y, place, 16);
+  }
+}
+
+/* ------------------- */
+/* ---- Fluid MLP ---- */
+/* ------------------- */
+void benchmark_fluid_mlp(
+    const std::shared_ptr<imperative::VarBase>& X,
+    const std::vector<std::shared_ptr<imperative::VarBase>>& Ws,
+    const std::vector<std::shared_ptr<imperative::VarBase>>& Bs,
+    const phi::Place& place,
+    bool accuracy_check) {
+  imperative::Tracer tracer;
+
+  imperative::NameVarBaseMap ins;
+  imperative::NameVarBaseMap outs;
+  framework::AttributeMap attrs;
+  std::shared_ptr<imperative::VarBase> input0 = X;
+  for (size_t i = 0; i < MLP_NUM_LINEAR; i++) {
+    // Matmul0
+    ins = {{"X", {input0}}, {"Y", {Ws[0]}}};
+    outs = {{"Out", {std::make_shared<imperative::VarBase>(true, "Out")}}};
+
+    tracer.TraceOp<VarBase>("matmul_v2", ins, outs, attrs, place, true);
+
+    // EW-Add0
+    ins = {{"X", outs["Out"]}, {"Y", {Bs[i]}}};
+    outs = {{"Out", {std::make_shared<imperative::VarBase>(true, "Out")}}};
+
+    tracer.TraceOp<VarBase>("elementwise_add", ins, outs, attrs, place, true);
+    input0 = outs["Out"][0];
+  }
+
+  // ReduceSum
+  ins = {{"X", {input0}}};
+  outs = {{"Out", {std::make_shared<imperative::VarBase>(true, "Out")}}};
+  attrs = {{"reduce_all", true}};
+
+  tracer.TraceOp<VarBase>("reduce_sum", ins, outs, attrs, place, true);
+
+  auto* engine = tracer.GetEngine();
+  std::vector<std::shared_ptr<imperative::VarBase>> grad_tensors{nullptr};
+  engine->Init(outs["Out"], grad_tensors, false /*retain_graph*/);
+  engine->Execute();
+
+  if (accuracy_check) {
+    std::unordered_map<std::string, float> result =
+        egr::compute_mlp_expected_results();
+
+    FluidCheckTensorValue(outs["Out"][0], place, result["Out"]);
+    FluidCheckGradTensorValue(X, place, result["GradX"]);
+    FluidCheckGradTensorValue(Ws[0], place, result["GradW"]);
+  }
+}
+
+}  // namespace imperative
+}  // namespace paddle

test/cpp/eager/performance_tests/benchmark_utils.h

@@ -0,0 +1,95 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <math.h>
+
+#include "paddle/fluid/eager/eager_tensor.h"
+#include "paddle/fluid/imperative/layer.h"
+#include "paddle/phi/api/all.h"
+
+/* MLP Configurations */
+// Out1 = X[M, N] x W[N, K] + B[K]
+// ... x MLP_NUM_LINEAR
+// Out  = ReduceSum(OutN)
+#define MLP_M 4
+#define MLP_N 16
+#define MLP_K MLP_N
+#define MLP_X_VAL 1.0
+#define MLP_W_VAL 2.0
+#define MLP_B_VAL 3.0
+#define MLP_NUM_LINEAR 1000
+
+namespace egr {
+
+inline std::unordered_map<std::string, float> compute_mlp_expected_results() {
+  float Out = MLP_X_VAL;
+  for (size_t i = 0; i < MLP_NUM_LINEAR; i++) {
+    Out = Out * MLP_W_VAL * MLP_N + MLP_B_VAL;
+  }
+  Out = Out * MLP_M * MLP_N;
+
+  float GradX = 1.0 * pow((MLP_W_VAL * MLP_N), MLP_NUM_LINEAR);
+  float GradW0 =
+      1.0 * pow((MLP_W_VAL * MLP_N), (MLP_NUM_LINEAR - 1)) * MLP_X_VAL * MLP_M;
+  return {{"Out", Out}, {"GradX", GradX}, {"GradW", GradW0}};
+}
+
+/* ---- Eager Scale ---- */
+void benchmark_eager_scale(const paddle::Tensor& tensor,
+                           bool accuracy_check = false);
+
+/* ---- Eager MatMul ---- */
+void benchmark_eager_matmul(const paddle::Tensor& X,
+                            const paddle::Tensor& Y,
+                            bool accuracy_check = false);
+
+void benchmark_eager_intermediate_matmul(const paddle::Tensor& X,
+                                         const paddle::Tensor& Y,
+                                         bool accuracy_check = false);
+
+void benchmark_eager_intermediate_mlp(const paddle::Tensor& X,
+                                      const std::vector<paddle::Tensor>& Ws,
+                                      const std::vector<paddle::Tensor>& Bs,
+                                      bool accuracy_check = false);
+
+}  // namespace egr
+
+namespace paddle {
+namespace imperative {
+/* ---- Fluid Scale ---- */
+// TODO(jiabin): Change this and remove nolint
+void benchmark_fluid_scale(
+    const std::shared_ptr<imperative::VarBase>& X,  // NOLINT
+    const phi::Place& place,
+    bool accuracy_check = false);
+
+/* ---- Fluid MatMul ---- */
+void benchmark_fluid_matmul(
+    const std::shared_ptr<imperative::VarBase>& X,
+    const std::shared_ptr<imperative::VarBase>& Y,  // NOLINT
+    const phi::Place& place,
+    bool accuracy_check = false);
+
+/* ---- Fluid MLP ---- */
+void benchmark_fluid_mlp(
+    const std::shared_ptr<imperative::VarBase>& X,
+    const std::vector<std::shared_ptr<imperative::VarBase>>& Ws,
+    const std::vector<std::shared_ptr<imperative::VarBase>>& Bs,
+    const phi::Place& place,
+    bool accuracy_check = false);
+
+}  // namespace imperative
+}  // namespace paddle

test/cpp/eager/task_tests/CMakeLists.txt

@@ -0,0 +1,20 @@
+paddle_test(test_egr_task_nan_inf_utils SRCS nan_inf_utils_test.cc DEPS common)
+
+if(NOT ((NOT WITH_PYTHON) AND ON_INFER))
+  paddle_test(test_egr_task_hook SRCS hook_test.cc)
+  paddle_test(test_egr_task_backward SRCS backward_test.cc)
+  paddle_test(test_egr_task_grad SRCS grad_test.cc)
+  paddle_test(test_egr_task_fwd_bwd_joint SRCS fwd_bwd_joint_test.cc DEPS phi)
+  paddle_test(test_egr_task_cross_batch SRCS cross_batch_accumulation_test.cc)
+  paddle_test(test_egr_task_hook_intermediate SRCS hook_test_intermediate.cc)
+  paddle_test(test_egr_task_autocodegen SRCS generated_test.cc)
+  paddle_test(test_egr_task_tensor_utils SRCS tensor_utils_test.cc)
+  paddle_test(test_egr_task_eager_utils SRCS eager_utils_test.cc)
+  paddle_test(test_egr_task_forward_autograd SRCS forward_autograd_test.cc)
+endif()
+
+if(WITH_ONNXRUNTIME AND WIN32)
+  # Copy onnxruntime for some c++ test in Windows, since the test will
+  # be build only in CI, so suppose the generator in Windows is Ninja.
+  copy_onnx(test_egr_task_nan_inf_utils)
+endif()

test/cpp/eager/task_tests/backward_test.cc

@@ -0,0 +1,345 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/eager/backward.h"
+
+#include <sstream>
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/fluid/eager/accumulation/accumulation_node.h"
+#include "paddle/fluid/eager/api/all.h"
+#include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h"
+#include "paddle/fluid/eager/autograd_meta.h"
+#include "paddle/fluid/eager/grad_node_info.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/tensor_meta.h"
+#include "test/cpp/eager/test_utils.h"
+
+PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(add, CPU, ALL_LAYOUT);
+
+namespace egr {
+
+TEST(Backward, SingleNodeEmptyGrad) {
+  // Prepare Device Contexts
+  eager_test::InitEnv(phi::CPUPlace());
+
+  // Prepare Inputs
+  phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
+
+  // Create Target Tensor
+  paddle::Tensor target_tensor =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        1.0 /*value*/,
+                                        false /*is_leaf*/);
+
+  paddle::Tensor leaf_tensor;
+  {
+    // Create Scale Node
+    auto node0_ptr = std::make_shared<GradNodeScale>(1, 1);
+    node0_ptr->SetAttributes_scale(5.0 /*scale*/);
+
+    // Set grad in/out meta
+    node0_ptr->SetDefaultGradInOutMeta();
+    AutogradMeta* auto_grad_meta = EagerUtils::autograd_meta(&target_tensor);
+    auto_grad_meta->SetGradNode(
+        std::dynamic_pointer_cast<GradNodeBase>(node0_ptr));
+    auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
+    auto_grad_meta->SetStopGradient(false);
+
+    AutogradMeta* auto_grad_meta1 = EagerUtils::autograd_meta(&leaf_tensor);
+
+    // Connect Tensor and AccumulationNode via AutoGradMeta
+    auto acc_node_ptr =
+        std::make_shared<egr::GradNodeAccumulation>(leaf_tensor);
+
+    auto_grad_meta1->SetGradNode(
+        std::dynamic_pointer_cast<GradNodeBase>(acc_node_ptr));
+    auto_grad_meta1->SetSingleOutRankWithSlot(0, 0);
+    auto_grad_meta1->SetStopGradient(false);
+
+    node0_ptr->SetGradOutMeta({leaf_tensor}, 0);
+  }
+  std::vector<paddle::Tensor> outs = {target_tensor};
+  // Run Backward
+  Backward(outs, {});
+
+  // Check Output Value
+  eager_test::CompareGradTensorWithValue<float>(leaf_tensor, 5.0);
+}
+
+TEST(Backward, SingleNodeCustomGrad) {
+  // Prepare Device Contexts
+  eager_test::InitEnv(phi::CPUPlace());
+
+  // Prepare Inputs
+  std::vector<paddle::Tensor> target_tensors;
+  phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
+
+  // Create Target Tensor
+  paddle::Tensor tensor =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        1.0 /*value*/,
+                                        false /*is_leaf*/);
+  target_tensors.emplace_back(std::move(tensor));
+
+  std::vector<paddle::Tensor> grad_tensors;
+  // Create Grad Tensor
+  paddle::Tensor grad_tensor =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        10.0 /*value*/,
+                                        false /*is_leaf*/);
+  grad_tensors.emplace_back(std::move(grad_tensor));
+
+  paddle::Tensor leaf_tensor;
+  {
+    // Create Scale Node
+    auto node0_ptr = std::make_shared<GradNodeScale>(1, 1);
+    node0_ptr->SetAttributes_scale(5.0 /*scale*/);
+
+    // Set grad in/out meta
+    node0_ptr->SetDefaultGradInOutMeta();
+
+    // Connect Tensor and Node via AutoGradMeta
+    AutogradMeta* auto_grad_meta =
+        EagerUtils::autograd_meta(&(target_tensors[0]));
+    auto_grad_meta->SetGradNode(
+        std::dynamic_pointer_cast<GradNodeBase>(node0_ptr));
+    auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
+    auto_grad_meta->SetStopGradient(false);
+
+    AutogradMeta* auto_grad_meta1 = EagerUtils::autograd_meta(&leaf_tensor);
+    // Connect Tensor and AccumulationNode via AutoGradMeta
+    auto acc_node_ptr =
+        std::make_shared<egr::GradNodeAccumulation>(leaf_tensor);
+
+    auto_grad_meta1->SetGradNode(
+        std::dynamic_pointer_cast<GradNodeBase>(acc_node_ptr));
+    auto_grad_meta1->SetSingleOutRankWithSlot(0, 0);
+    auto_grad_meta1->SetStopGradient(false);
+    node0_ptr->SetGradOutMeta({leaf_tensor}, 0);
+  }
+
+  // Run Backward
+  Backward(target_tensors, grad_tensors);
+
+  // Check Output Value
+  eager_test::CompareGradTensorWithValue<float>(leaf_tensor, 50.0);
+}
+
+/*
+Node1
+  |
+Node0
+  |
+ inp0
+*/
+TEST(Backward, LinearNodes) {
+  // Prepare Device Contexts
+  eager_test::InitEnv(phi::CPUPlace());
+
+  // Prepare Inputs
+  std::vector<paddle::Tensor> target_tensors;
+  phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
+
+  // Create Target Tensor
+  paddle::Tensor tensor =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        1.0 /*value*/,
+                                        false /*is_leaf*/);
+  target_tensors.emplace_back(std::move(tensor));
+
+  paddle::Tensor leaf_tensor;
+  {
+    // Create Node0
+    auto node0_ptr = std::make_shared<GradNodeScale>(1, 1);
+    node0_ptr->SetAttributes_scale(5.0 /*scale*/);
+
+    // Set grad in/out meta for node0
+    node0_ptr->SetDefaultGradInOutMeta();
+
+    // Create Node1
+    auto node1_ptr = std::make_shared<GradNodeScale>(1, 1);
+    node1_ptr->SetAttributes_scale(10.0 /*scale*/);
+
+    // Set grad in/out meta for node1
+    node1_ptr->SetDefaultGradInOutMeta();
+
+    // Connect Input Tensor and Node0 via AutoGradMeta
+    AutogradMeta* auto_grad_meta =
+        EagerUtils::autograd_meta(&(target_tensors[0]));
+    auto_grad_meta->SetGradNode(
+        std::dynamic_pointer_cast<GradNodeBase>(node0_ptr));
+    auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
+    auto_grad_meta->SetStopGradient(false);
+    // Connect Node0 -> Node1 via Edge
+    auto tmp_tensor = paddle::Tensor();
+    auto* meta0 = EagerUtils::autograd_meta(&tmp_tensor);
+    meta0->SetStopGradient(false);
+    meta0->SetSingleOutRankWithSlot(0, 0);
+    meta0->SetGradNode(node1_ptr);
+    node0_ptr->SetGradOutMeta(tmp_tensor, 0);
+
+    AutogradMeta* auto_grad_meta1 = EagerUtils::autograd_meta(&leaf_tensor);
+    // Connect Tensor and AccumulationNode via AutoGradMeta
+    auto acc_node_ptr =
+        std::make_shared<egr::GradNodeAccumulation>(leaf_tensor);
+
+    auto_grad_meta1->SetGradNode(
+        std::dynamic_pointer_cast<GradNodeBase>(acc_node_ptr));
+    auto_grad_meta1->SetSingleOutRankWithSlot(0, 0);
+
+    auto_grad_meta1->SetStopGradient(false);
+    node1_ptr->SetGradOutMeta(leaf_tensor, 0);
+  }
+
+  // Use Empty Grad Tensor
+  Backward(target_tensors, {});
+
+  // Check Output Value
+  eager_test::CompareGradTensorWithValue<float>(leaf_tensor, 50.0);
+}
+
+/*
+    Node2
+    |   |
+Node0   Node1
+  |      |
+ inp0   inp1
+*/
+TEST(Backward, WithAccumulation) {
+  // Prepare Device Contexts
+  eager_test::InitEnv(phi::CPUPlace());
+
+  // Prepare Inputs
+  phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
+
+  // Create Target Tensor
+  std::vector<paddle::Tensor> target_tensors;
+  paddle::Tensor tensor0 =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        1.0 /*value*/,
+                                        false /*is_leaf*/);
+  paddle::Tensor tensor1 =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        1.0 /*value*/,
+                                        false /*is_leaf*/);
+  target_tensors.emplace_back(std::move(tensor0));
+  target_tensors.emplace_back(std::move(tensor1));
+
+  // Create Grad Tensor
+  std::vector<paddle::Tensor> grad_tensors;
+  paddle::Tensor grad_tensor0 =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        5.0 /*value*/,
+                                        false /*is_leaf*/);
+  paddle::Tensor grad_tensor1 =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        10.0 /*value*/,
+                                        false /*is_leaf*/);
+  grad_tensors.emplace_back(std::move(grad_tensor0));
+  grad_tensors.emplace_back(std::move(grad_tensor1));
+
+  paddle::Tensor leaf_tensor;
+  {
+    // Create Node0
+    auto node0_ptr = std::make_shared<GradNodeScale>(1, 1);
+    node0_ptr->SetAttributes_scale(5.0 /*scale*/);
+    node0_ptr->SetDefaultGradInOutMeta();
+
+    // Create Node1
+    auto node1_ptr = std::make_shared<GradNodeScale>(1, 1);
+    node1_ptr->SetAttributes_scale(10.0 /*scale*/);
+    node1_ptr->SetDefaultGradInOutMeta();
+    // Create Node2
+    auto node2_ptr = std::make_shared<GradNodeScale>(1, 1);
+    node2_ptr->SetAttributes_scale(20.0 /*scale*/);
+    node2_ptr->SetDefaultGradInOutMeta();
+    // Connect Inp0 and Node0 via AutoGradMeta
+    AutogradMeta* auto_grad_meta0 =
+        EagerUtils::autograd_meta(&(target_tensors[0]));
+    auto_grad_meta0->SetGradNode(
+        std::dynamic_pointer_cast<GradNodeBase>(node0_ptr));
+    auto_grad_meta0->SetSingleOutRankWithSlot(0, 0);
+    auto_grad_meta0->SetStopGradient(false);
+    // Connect Inp1 and Node1 via AutoGradMeta
+    AutogradMeta* auto_grad_meta1 =
+        EagerUtils::autograd_meta(&(target_tensors[1]));
+    auto_grad_meta1->SetGradNode(
+        std::dynamic_pointer_cast<GradNodeBase>(node1_ptr));
+    auto_grad_meta1->SetSingleOutRankWithSlot(0, 0);
+    auto_grad_meta1->SetStopGradient(false);
+
+    // Connect Node0 -> Node2 via Edge
+    auto tmp_tensor0 = paddle::Tensor();
+    auto* meta0 = EagerUtils::autograd_meta(&tmp_tensor0);
+    meta0->SetStopGradient(false);
+    meta0->SetSingleOutRankWithSlot(0, 0);
+    meta0->SetGradNode(node2_ptr);
+    node0_ptr->SetGradOutMeta(tmp_tensor0, 0);
+
+    // Connect Node1 -> Node2 via Edge
+    auto tmp_tensor1 = paddle::Tensor();
+    auto* meta1 = EagerUtils::autograd_meta(&tmp_tensor1);
+    meta1->SetStopGradient(false);
+    meta1->SetSingleOutRankWithSlot(0, 0);
+    meta1->SetGradNode(node2_ptr);
+    node1_ptr->SetGradOutMeta(tmp_tensor1, 0);
+
+    AutogradMeta* auto_grad_meta2 = EagerUtils::autograd_meta(&leaf_tensor);
+    // Connect Tensor and AccumulationNode via AutoGradMeta
+    auto acc_node_ptr =
+        std::make_shared<egr::GradNodeAccumulation>(leaf_tensor);
+
+    auto_grad_meta2->SetGradNode(
+        std::dynamic_pointer_cast<GradNodeBase>(acc_node_ptr));
+    auto_grad_meta2->SetSingleOutRankWithSlot(0, 0);
+
+    auto_grad_meta2->SetStopGradient(false);
+    std::vector<egr::AutogradMeta*> res2 = {auto_grad_meta2};
+    node2_ptr->SetGradOutMeta(leaf_tensor, 0);
+  }
+
+  Backward(target_tensors, grad_tensors);
+
+  eager_test::CompareGradTensorWithValue<float>(leaf_tensor, 2500.0);
+}
+
+}  // namespace egr

test/cpp/eager/task_tests/cross_batch_accumulation_test.cc

@@ -0,0 +1,83 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <sstream>
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/fluid/eager/accumulation/accumulation_node.h"
+#include "paddle/fluid/eager/api/all.h"
+#include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h"
+#include "paddle/fluid/eager/autograd_meta.h"
+#include "paddle/fluid/eager/backward.h"
+#include "paddle/fluid/eager/grad_node_info.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/tensor_meta.h"
+#include "test/cpp/eager/test_utils.h"
+
+PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
+
+namespace egr {
+
+TEST(CrossBatchAccumulation, SingleScaleNode) {
+  eager_test::InitEnv(phi::CPUPlace());
+
+  std::vector<paddle::Tensor> target_tensors;
+  phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
+
+  paddle::Tensor tensor =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        1.0 /*value*/,
+                                        false /*is_leaf*/);
+  target_tensors.emplace_back(std::move(tensor));
+  paddle::Tensor& target_tensor = target_tensors[0];
+
+  paddle::Tensor leaf_tensor = paddle::Tensor();
+
+  auto scale_node_ptr = std::make_shared<GradNodeScale>(1, 1);
+  scale_node_ptr->SetAttributes_scale(5.0 /*scale*/);
+
+  scale_node_ptr->SetDefaultGradInOutMeta();
+
+  AutogradMeta* auto_grad_meta = EagerUtils::autograd_meta(&target_tensor);
+  auto_grad_meta->SetGradNode(
+      std::dynamic_pointer_cast<GradNodeBase>(scale_node_ptr));
+  auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
+  auto_grad_meta->SetStopGradient(false);
+  egr_utils_api::RetainGradForTensor(target_tensor);  // result: 1.0
+
+  AutogradMeta* meta = EagerUtils::autograd_meta(&leaf_tensor);
+  auto acc_node_ptr = std::make_shared<GradNodeAccumulation>(leaf_tensor);
+  meta->SetStopGradient(false);
+  meta->SetSingleOutRankWithSlot(0, 0);
+  meta->SetGradNode(acc_node_ptr);
+  std::vector<egr::AutogradMeta*> res = {meta};
+  scale_node_ptr->SetGradOutMeta(leaf_tensor, 0);
+
+  Backward(target_tensors, {});
+
+  eager_test::CompareGradTensorWithValue<float>(target_tensor, 1.0);
+  eager_test::CompareGradTensorWithValue<float>(leaf_tensor, 5.0);
+
+  Backward(target_tensors, {});
+
+  eager_test::CompareGradTensorWithValue<float>(target_tensor, 1.0);
+  eager_test::CompareGradTensorWithValue<float>(leaf_tensor, 10.0);
+}
+
+}  // namespace egr

test/cpp/eager/task_tests/eager_utils_test.cc

@@ -0,0 +1,567 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <sstream>
+
+#include "gtest/gtest.h"
+#include "paddle/common/flags.h"
+#include "paddle/fluid/eager/accumulation/accumulation_node.h"
+#include "paddle/fluid/eager/eager_tensor.h"
+#include "paddle/fluid/eager/grad_node_info.h"
+#include "paddle/fluid/eager/utils.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/core/enforce.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "test/cpp/eager/data_structure_tests/grad_node_test.h"
+#include "test/cpp/eager/test_utils.h"
+COMMON_DECLARE_bool(tensor_md5_checksum_use_binary_format);
+
+PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
+
+namespace egr {
+
+TEST(EagerUtils, AutoGradMeta) {
+  // Construct Eager Tensor
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 1}));
+  std::shared_ptr<phi::DenseTensor> dt0 = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  dt0->mutable_data<float>(phi::CPUPlace())[0] = 10.0;
+  paddle::Tensor et0 = paddle::Tensor(dt0);
+
+  std::shared_ptr<phi::DenseTensor> dt1 = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  dt1->mutable_data<float>(phi::CPUPlace())[0] = 20.0;
+  paddle::Tensor et1 = paddle::Tensor(dt1);
+
+  // unsafe_autograd_meta()
+  // autograd_meta()
+  AutogradMeta* autograd_meta0 = EagerUtils::autograd_meta(&et0);
+  AutogradMeta* autograd_meta1 = EagerUtils::autograd_meta(&et1);
+
+  AutogradMeta* unsafe_autograd_meta_after =
+      EagerUtils::unsafe_autograd_meta(et0);
+  PADDLE_ENFORCE_NOT_NULL(
+      unsafe_autograd_meta_after,
+      common::errors::PreconditionNotMet(
+          "Unsafe autograd meta after should not be null."));
+
+  // NOTE: Since autograd_meta will be copied make sure it's not null
+  std::vector<paddle::Tensor> ets = {et0, et1};
+  auto test_node = std::make_shared<eager_test::GradTestNode>();
+
+  std::vector<AutogradMeta*> autograd_metas = EagerUtils::autograd_meta(&ets);
+  std::vector<AutogradMeta*> unsafe_autograd_metas =
+      EagerUtils::unsafe_autograd_meta(ets);
+  PADDLE_ENFORCE_NOT_NULL(unsafe_autograd_metas[0],
+                          common::errors::PreconditionNotMet(
+                              "Unsafe autograd metas should not be null."));
+  PADDLE_ENFORCE_NOT_NULL(unsafe_autograd_metas[1],
+                          common::errors::PreconditionNotMet(
+                              "Unsafe autograd metas should not be null."));
+
+  // Set Autograd Meta
+  autograd_meta0->SetSingleOutRankWithSlot(0, 1);
+
+  autograd_meta0->SetGradNode(test_node);
+
+  // OutRankInfo()
+  std::pair<size_t, size_t> out_rank_info0 = EagerUtils::OutRankInfo(et0);
+  PADDLE_ENFORCE_EQ(
+      static_cast<int>(out_rank_info0.first),
+      0UL,
+      common::errors::InvalidArgument("The first element of out rank info "
+                                      "mismatch. Expected 0 but received %d.",
+                                      static_cast<int>(out_rank_info0.first)));
+  PADDLE_ENFORCE_EQ(
+      static_cast<int>(out_rank_info0.second),
+      1UL,
+      common::errors::InvalidArgument("The second element of out rank info "
+                                      "mismatch. Expected 1 but received %d.",
+                                      static_cast<int>(out_rank_info0.second)));
+
+  // grad_node()
+  std::shared_ptr<GradNodeBase> grad_node0 = EagerUtils::grad_node(et0);
+  PADDLE_ENFORCE_NOT_NULL(
+      grad_node0.get(),
+      common::errors::PreconditionNotMet("Grad of node should not be null."));
+
+  EagerUtils::SetHistory(autograd_meta1, test_node);
+  EagerUtils::SetHistory(autograd_meta1, test_node);
+  std::shared_ptr<GradNodeBase> grad_node1 = EagerUtils::grad_node(et1);
+  PADDLE_ENFORCE_NOT_NULL(
+      grad_node1.get(),
+      common::errors::PreconditionNotMet("Grad of node should not be null."));
+
+  // SetOutRankWithSlot()
+  EagerUtils::SetOutRankWithSlot(autograd_meta1, 0);
+  std::pair<size_t, size_t> out_rank_info1 = EagerUtils::OutRankInfo(et1);
+  PADDLE_ENFORCE_EQ(
+      static_cast<int>(out_rank_info1.first),
+      0UL,
+      common::errors::InvalidArgument("The first element of out rank info "
+                                      "mismatch. Expected 0 but received %d.",
+                                      static_cast<int>(out_rank_info1.first)));
+  PADDLE_ENFORCE_EQ(
+      static_cast<int>(out_rank_info1.second),
+      0UL,
+      common::errors::InvalidArgument("The second element of out rank info "
+                                      "mismatch. Expected 0 but received %d.",
+                                      static_cast<int>(out_rank_info1.second)));
+
+  EagerUtils::SetOutRankWithSlot(&autograd_metas, 0);
+  std::pair<size_t, size_t> out_rank_info2 = EagerUtils::OutRankInfo(et0);
+  PADDLE_ENFORCE_EQ(
+      static_cast<int>(out_rank_info2.first),
+      0UL,
+      common::errors::InvalidArgument("The first element of out rank info "
+                                      "mismatch. Expected 0 but received %d.",
+                                      static_cast<int>(out_rank_info2.first)));
+  PADDLE_ENFORCE_EQ(
+      static_cast<int>(out_rank_info2.second),
+      0UL,
+      common::errors::InvalidArgument("The second element of out rank info "
+                                      "mismatch. Expected 0 but received %d.",
+                                      static_cast<int>(out_rank_info2.second)));
+
+  std::pair<size_t, size_t> out_rank_info3 = EagerUtils::OutRankInfo(et1);
+  PADDLE_ENFORCE_EQ(
+      static_cast<int>(out_rank_info3.first),
+      0UL,
+      common::errors::InvalidArgument("The first element of out rank info "
+                                      "mismatch. Expected 0 but received %d.",
+                                      static_cast<int>(out_rank_info3.first)));
+  PADDLE_ENFORCE_EQ(
+      static_cast<int>(out_rank_info3.second),
+      1UL,
+      common::errors::InvalidArgument("The second element of out rank info "
+                                      "mismatch. Expected 1 but received %d.",
+                                      static_cast<int>(out_rank_info3.second)));
+}
+
+template <typename T>
+paddle::Tensor CreateTestCPUTensor(T val, const phi::DDim& ddim) {
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, ddim);
+  paddle::Tensor tensor;
+  std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  auto* dt_ptr = dt->mutable_data<T>(phi::CPUPlace());
+  for (int64_t i = 0; i < dt->numel(); i++) {
+    dt_ptr[i] = val;
+  }
+  tensor.set_impl(dt);
+  return tensor;
+}
+
+TEST(EagerUtils, ComputeRequireGrad) {
+  auto auto_grad0 = std::make_shared<egr::AutogradMeta>();
+  auto auto_grad1 = std::make_shared<egr::AutogradMeta>();
+  auto auto_grad2 = std::make_shared<egr::AutogradMeta>();
+  auto auto_grad3 = std::make_shared<egr::AutogradMeta>();
+  PADDLE_ENFORCE_EQ(
+      auto_grad0->NumericStopGradient(),
+      -1,
+      common::errors::InvalidArgument("The NumericStopGradient of auto grad "
+                                      "mismatch. Expected -1 but received %d.",
+                                      auto_grad0->NumericStopGradient()));
+  VLOG(6) << "Single Test ComputeRequireGrad";
+  auto_grad0->SetStopGradient(true);
+  PADDLE_ENFORCE_EQ(egr::EagerUtils::ComputeRequireGrad(true, auto_grad0.get()),
+                    false,
+                    ::common::errors::InvalidArgument(
+                        "Expected ComputeRequireGrad(true, auto_grad0) to be "
+                        "false, but it is true."));
+  PADDLE_ENFORCE_EQ(
+      egr::EagerUtils::ComputeRequireGrad(false, auto_grad0.get()),
+      false,
+      ::common::errors::InvalidArgument(
+          "Expected ComputeRequireGrad(false, auto_grad0) to be false, but it "
+          "is true."));
+  auto_grad0->SetStopGradient(false);
+  PADDLE_ENFORCE_EQ(
+      egr::EagerUtils::ComputeRequireGrad(false, auto_grad0.get()),
+      false,
+      ::common::errors::InvalidArgument(
+          "Expected ComputeRequireGrad(false, auto_grad0) to be false, but it "
+          "is true."));
+
+  PADDLE_ENFORCE_EQ(egr::EagerUtils::ComputeRequireGrad(true, auto_grad0.get()),
+                    true,
+                    ::common::errors::InvalidArgument(
+                        "Expected ComputeRequireGrad(true, auto_grad0) to be "
+                        "true, but it is false."));
+
+  VLOG(6) << "Multi Test ComputeRequireGrad";
+  auto_grad0->SetStopGradient(false);
+  auto_grad1->SetStopGradient(true);
+  PADDLE_ENFORCE_EQ(egr::EagerUtils::ComputeRequireGrad(
+                        true, auto_grad0.get(), auto_grad1.get()),
+                    true,
+                    ::common::errors::InvalidArgument(
+                        "Expected ComputeRequireGrad(true, auto_grad0, "
+                        "auto_grad1) to be true, but it is false."));
+
+  PADDLE_ENFORCE_EQ(egr::EagerUtils::ComputeRequireGrad(
+                        false, auto_grad0.get(), auto_grad1.get()),
+                    false,
+                    ::common::errors::InvalidArgument(
+                        "Expected ComputeRequireGrad(false, auto_grad0, "
+                        "auto_grad1) to be false, but it is true."));
+  auto_grad0->SetStopGradient(true);
+  PADDLE_ENFORCE_EQ(egr::EagerUtils::ComputeRequireGrad(
+                        true, auto_grad0.get(), auto_grad1.get()),
+                    false,
+                    ::common::errors::InvalidArgument(
+                        "Expected ComputeRequireGrad(true, auto_grad0, "
+                        "auto_grad1) to be false, but it is true."));
+
+  PADDLE_ENFORCE_EQ(egr::EagerUtils::ComputeRequireGrad(
+                        false, auto_grad0.get(), auto_grad1.get()),
+                    false,
+                    ::common::errors::InvalidArgument(
+                        "Expected ComputeRequireGrad(false, auto_grad0, "
+                        "auto_grad1) to be false, but it is true."));
+}
+
+TEST(EagerUtils, PassStopGradient) {
+  auto auto_grad0 = std::make_shared<egr::AutogradMeta>();
+  auto auto_grad1 = std::make_shared<egr::AutogradMeta>();
+  auto auto_grad2 = std::make_shared<egr::AutogradMeta>();
+  auto auto_grad3 = std::make_shared<egr::AutogradMeta>();
+  PADDLE_ENFORCE_EQ(
+      auto_grad0->NumericStopGradient(),
+      -1,
+      common::errors::InvalidArgument("The NumericStopGradient of auto grad "
+                                      "mismatch. Expected -1 but received %d.",
+                                      auto_grad0->NumericStopGradient()));
+  VLOG(6) << "Test PassStopGradient";
+  egr::EagerUtils::PassStopGradient(false, auto_grad0.get());
+  PADDLE_ENFORCE_EQ(
+      auto_grad0->StopGradient(),
+      false,
+      ::common::errors::InvalidArgument(
+          "Expected auto_grad0->StopGradient() to be false, but received %d.",
+          auto_grad0->StopGradient()));
+  egr::EagerUtils::PassStopGradient(true,
+                                    auto_grad0.get(),
+                                    auto_grad1.get(),
+                                    auto_grad2.get(),
+                                    auto_grad3.get());
+  PADDLE_ENFORCE_EQ(
+      auto_grad0->StopGradient(),
+      true,
+      ::common::errors::InvalidArgument(
+          "Expected auto_grad0->StopGradient() to be true, but received %d.",
+          auto_grad0->StopGradient()));
+  PADDLE_ENFORCE_EQ(
+      auto_grad1->StopGradient(),
+      true,
+      ::common::errors::InvalidArgument(
+          "Expected auto_grad1->StopGradient() to be true, but received %d.",
+          auto_grad1->StopGradient()));
+  PADDLE_ENFORCE_EQ(
+      auto_grad2->StopGradient(),
+      true,
+      ::common::errors::InvalidArgument(
+          "Expected auto_grad2->StopGradient() to be true, but received %d.",
+          auto_grad2->StopGradient()));
+  PADDLE_ENFORCE_EQ(
+      auto_grad3->StopGradient(),
+      true,
+      ::common::errors::InvalidArgument(
+          "Expected auto_grad3->StopGradient() to be true, but received %d.",
+          auto_grad3->StopGradient()));
+}
+
+TEST(EagerUtils, TrySyncToVar) {
+  phi::DDim ddim = common::make_ddim({2, 4, 4, 4});
+  auto tensor = CreateTestCPUTensor(5.0f, ddim);
+  std::vector<std::shared_ptr<egr::EagerVariable>> var_bases = {
+      egr::EagerUtils::TrySyncToVar(tensor)};
+
+  paddle::framework::Variable* var = var_bases[0]->MutableVar();
+  const auto& framework_tensor = var->Get<phi::DenseTensor>();
+
+  const float* ptr = framework_tensor.data<float>();
+  VLOG(6) << "Check Value for SyncToVarsSingle";
+  PADDLE_ENFORCE_EQ(framework_tensor.numel(),
+                    tensor.numel(),
+                    common::errors::InvalidArgument(
+                        "The numel of framework tensor and numel of "
+                        "tensor should be the same, but received %d and %d.",
+                        framework_tensor.numel(),
+                        tensor.numel()));
+
+  for (int i = 0; i < framework_tensor.numel(); i++) {
+    PADDLE_ENFORCE_EQ(ptr[i],
+                      5.0f,
+                      common::errors::InvalidArgument(
+                          "The numel of framework tensor mismatch. "
+                          "Expected 5.0 but received %f.",
+                          ptr[i]));
+  }
+}
+
+TEST(EagerUtils, TrySyncToVars) {
+  phi::DDim ddim = common::make_ddim({2, 4, 4, 4});
+  std::vector<paddle::Tensor> tensors = {CreateTestCPUTensor(1.0f, ddim),
+                                         CreateTestCPUTensor(2.0f, ddim)};
+
+  std::vector<std::shared_ptr<egr::EagerVariable>> var_bases =
+      egr::EagerUtils::TrySyncToVars(tensors);
+
+  {
+    paddle::framework::Variable* var = var_bases[0]->MutableVar();
+    const auto& framework_tensor = var->Get<phi::DenseTensor>();
+
+    const float* ptr = framework_tensor.data<float>();
+    PADDLE_ENFORCE_EQ(
+        framework_tensor.numel(),
+        tensors[0].numel(),
+        common::errors::InvalidArgument(
+            "The numel of framework tensor and numel "
+            "of tensor should be the same, but received %d and %d.",
+            framework_tensor.numel(),
+            tensors[0].numel()));
+
+    for (int i = 0; i < framework_tensor.numel(); i++) {
+      PADDLE_ENFORCE_EQ(ptr[i],
+                        1.0,
+                        common::errors::InvalidArgument(
+                            "The numel of framework tensor mismatch. Expected "
+                            "1.0 but received %f.",
+                            ptr[i]));
+    }
+  }
+
+  {
+    paddle::framework::Variable* var = var_bases[1]->MutableVar();
+    const auto& framework_tensor = var->Get<phi::DenseTensor>();
+
+    const float* ptr = framework_tensor.data<float>();
+    VLOG(6) << "Check Value for SyncToVarsMultiple";
+    PADDLE_ENFORCE_EQ(
+        framework_tensor.numel(),
+        tensors[0].numel(),
+        common::errors::InvalidArgument(
+            "The numel of framework tensor and numel "
+            "of tensor should be the same, but received %d and %d.",
+            framework_tensor.numel(),
+            tensors[0].numel()));
+
+    for (int i = 0; i < framework_tensor.numel(); i++) {
+      PADDLE_ENFORCE_EQ(ptr[i],
+                        2.0,
+                        common::errors::InvalidArgument(
+                            "The numel of framework tensor mismatch. Expected "
+                            "2.0 but received %f.",
+                            ptr[i]));
+    }
+  }
+}
+
+TEST(EagerUtils, CreateVars) {
+  VLOG(6) << "Check CreateVars";
+  std::vector<std::shared_ptr<egr::EagerVariable>> outs =
+      egr::EagerUtils::CreateVars(2);
+  PADDLE_ENFORCE_EQ(
+      outs.size(),
+      2UL,
+      common::errors::InvalidArgument(
+          "Size of outs mismatch. Expected 2 but received %d.", outs.size()));
+  PADDLE_ENFORCE_EQ(
+      outs[0]->Var().IsInitialized(),
+      false,
+      ::common::errors::AlreadyExists("Expected the first variable to be "
+                                      "uninitialized, but already exists."));
+}
+
+TEST(EagerUtils, GetGradAccumulationNode) {
+  VLOG(6) << "Check GetGradAccumulationNode";
+  paddle::Tensor t0("test_tensor");
+  ASSERT_EQ(egr::EagerUtils::GetGradAccumulationNode(t0), nullptr);
+  auto autograd_ptr0 = egr::EagerUtils::autograd_meta(&t0);
+  autograd_ptr0->SetStopGradient(true);
+  ASSERT_EQ(egr::EagerUtils::GetGradAccumulationNode(t0), nullptr);
+  autograd_ptr0->SetStopGradient(false);
+  auto res = std::dynamic_pointer_cast<egr::GradNodeAccumulation>(
+      egr::EagerUtils::GetGradAccumulationNode(t0));
+  ASSERT_TRUE(res != nullptr);
+  auto res2 = egr::EagerUtils::GetGradAccumulationNode(t0);
+  ASSERT_EQ(res2.get(), res.get());
+  autograd_ptr0->SetStopGradient(true);
+  auto res3 = egr::EagerUtils::GetGradAccumulationNode(t0);
+  ASSERT_EQ(res3, nullptr);
+  autograd_ptr0->SetStopGradient(false);
+  autograd_ptr0->SetGradNode(
+      std::make_shared<eager_test::GradTestNode>(1, 2.0, 3));
+  ASSERT_ANY_THROW(egr::EagerUtils::GetGradAccumulationNode(t0));
+}
+
+TEST(EagerUtils, FillZeroForEmptyOptionalGradInput) {
+  paddle::small_vector<std::vector<paddle::Tensor>, egr::kSlotSmallVectorSize>
+      grads = {std::vector<paddle::Tensor>(1)};
+  paddle::small_vector<std::vector<GradSlotMeta>, egr::kSlotSmallVectorSize>
+      slot_metas = {std::vector<GradSlotMeta>(1)};
+
+  phi::DenseTensorMeta tensor_meta;
+  tensor_meta.dtype = phi::DataType::FLOAT32;
+  tensor_meta.dims = {2, 4};
+  slot_metas[0][0].SetTensorMeta(tensor_meta);
+  slot_metas[0][0].SetPlace(phi::CPUPlace());
+
+  EagerUtils::FillZeroForEmptyOptionalGradInput(&grads[0], slot_metas[0]);
+  eager_test::CompareTensorWithValue<float>(grads[0][0], 0.0);
+}
+TEST(EagerUtils, SetTensorName) {
+  std::string unique_api_name = "Test";
+  std::string var_name = "out";
+  phi::DDim ddim = common::make_ddim({2, 4, 4, 4});
+  std::vector<paddle::Tensor> tensors = {CreateTestCPUTensor(1.0f, ddim),
+                                         CreateTestCPUTensor(2.0f, ddim)};
+  paddle::optional<paddle::Tensor> optional_t;
+  optional_t = tensors[0];
+  paddle::Tensor* t = &(optional_t.get());
+
+  auto generate_tensor_name = [](const std::string& unique_api_name,
+                                 const std::string& var_name,
+                                 const paddle::Tensor* t) {
+    std::ostringstream oss;
+    oss << unique_api_name << "_" << var_name << "_" << t->dtype() << "_";
+    for (int i = 0; i < t->dims().size(); ++i) {
+      if (i != 0) {
+        oss << "x";
+      }
+      oss << t->dims()[i];
+    }
+    return oss.str();
+  };
+  // Gen refer name
+  std::string refer_name = generate_tensor_name(unique_api_name, var_name, t);
+  // test paddle::optional<paddle::Tensor>* tensor
+  egr::SetTensorName(unique_api_name, var_name, &optional_t);
+  ASSERT_TRUE(t->name() == refer_name);
+  refer_name = generate_tensor_name(
+      unique_api_name, var_name + "_" + std::to_string(0), t);
+  // test std::vector<paddle::Tensor>* tensors
+  egr::SetTensorName(unique_api_name, var_name, &tensors);
+  ASSERT_TRUE(tensors[0].name() == refer_name);
+  // test paddle::optional<std::vector<paddle::Tensor>>* tensors
+  paddle::optional<std::vector<paddle::Tensor>> opt_tensors = tensors;
+  egr::SetTensorName(unique_api_name, var_name, &opt_tensors);
+  ASSERT_TRUE(tensors[0].name() == refer_name);
+}
+TEST(EagerUtils, SetGradTensorName) {
+  phi::DDim ddim = common::make_ddim({2, 4});
+  std::vector<paddle::Tensor> tensors = {CreateTestCPUTensor(1.0f, ddim)};
+  paddle::small_vector<std::vector<GradSlotMeta>, egr::kSlotSmallVectorSize>
+      slot_metas = {std::vector<GradSlotMeta>(1)};
+
+  phi::DenseTensorMeta tensor_meta;
+  tensor_meta.dtype = phi::DataType::FLOAT32;
+  tensor_meta.dims = {2, 4};
+  slot_metas[0][0].SetTensorMeta(tensor_meta);
+  slot_metas[0][0].SetPlace(phi::CPUPlace());
+
+  egr::SetGradTensorName(&tensors, 0, slot_metas);
+  std::string refer_name = "@Grad";
+  ASSERT_TRUE(tensors[0].name() == refer_name);
+}
+
+TEST(EagerUtils, SaveTensorMD5CheckSumToFile) {
+#define EXPECT_SAVE_TENSOR_MD5_CHECKSUM_FAILURE(t)              \
+  try {                                                         \
+    egr::SaveTensorMD5CheckSumToFile("", t);                    \
+    FAIL() << "Expected std::exception";                        \
+  } catch (const std::exception& e) {                           \
+    std::string error_str = e.what();                           \
+    EXPECT_NE(error_str.find("Cannot open file  for writing."), \
+              std::string::npos);                               \
+  } catch (...) {                                               \
+    FAIL() << "Unexpected error";                               \
+  }
+
+#define EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(t)                \
+  try {                                                           \
+    egr::SaveTensorMD5CheckSumToFile("test_md5_checksum.txt", t); \
+  } catch (const std::exception& e) {                             \
+    FAIL() << "Unexpected error: " << e.what();                   \
+  } catch (...) {                                                 \
+    FAIL() << "Unexpected error";                                 \
+  }
+
+  // Test the invalid file name
+  phi::DDim ddim = common::make_ddim({20, 40});
+  paddle::Tensor t = CreateTestCPUTensor(1.0f, ddim);
+  EXPECT_SAVE_TENSOR_MD5_CHECKSUM_FAILURE(t)
+  paddle::optional<paddle::Tensor> optional_t;
+  optional_t = CreateTestCPUTensor<double>(1.0, ddim);
+  EXPECT_SAVE_TENSOR_MD5_CHECKSUM_FAILURE(optional_t)
+  // Test the vector input
+  std::vector<paddle::Tensor> tensors = {CreateTestCPUTensor<int64_t>(1, ddim),
+                                         CreateTestCPUTensor<int64_t>(1, ddim)};
+  EXPECT_SAVE_TENSOR_MD5_CHECKSUM_FAILURE(tensors)
+  paddle::optional<std::vector<paddle::Tensor>> opt_tensors =
+      std::vector<paddle::Tensor>{CreateTestCPUTensor<bool>(true, ddim),
+                                  CreateTestCPUTensor<bool>(false, ddim)};
+  EXPECT_SAVE_TENSOR_MD5_CHECKSUM_FAILURE(opt_tensors)
+  // test the different data type
+  EXPECT_SAVE_TENSOR_MD5_CHECKSUM_FAILURE(CreateTestCPUTensor<int>(1, ddim))
+  EXPECT_SAVE_TENSOR_MD5_CHECKSUM_FAILURE(
+      CreateTestCPUTensor<phi::float16>(static_cast<phi::float16>(1), ddim))
+  EXPECT_SAVE_TENSOR_MD5_CHECKSUM_FAILURE(
+      CreateTestCPUTensor<int32_t>(static_cast<int32_t>(1), ddim))
+  paddle::Tensor complex64_t =
+      CreateTestCPUTensor(phi::complex64(1.0f, 2.0f), ddim);
+  paddle::Tensor complex128_t =
+      CreateTestCPUTensor(phi::complex128(1.0f, 2.0f), ddim);
+#if defined(PADDLE_WITH_CUDA)
+  EXPECT_SAVE_TENSOR_MD5_CHECKSUM_FAILURE(
+      CreateTestCPUTensor<phi::bfloat16>(static_cast<phi::bfloat16>(1), ddim))
+  EXPECT_SAVE_TENSOR_MD5_CHECKSUM_FAILURE(
+      CreateTestCPUTensor<phi::float8_e4m3fn>(
+          static_cast<phi::float8_e4m3fn>(1), ddim))
+  EXPECT_SAVE_TENSOR_MD5_CHECKSUM_FAILURE(CreateTestCPUTensor<phi::float8_e5m2>(
+      static_cast<phi::float8_e5m2>(1), ddim))
+#endif
+
+#ifndef _WIN32
+  // test save to file
+  EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(t)
+  EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(optional_t)
+  EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(tensors)
+  EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(opt_tensors)
+  EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(complex64_t)
+  EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(complex128_t)
+  // test  using binary format
+  FLAGS_tensor_md5_checksum_use_binary_format = true;
+  EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(t)
+  EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(optional_t)
+  EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(tensors)
+  EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(opt_tensors)
+  EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(complex64_t)
+  EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(complex128_t)
+  // test Fake dist tensor
+  t.set_impl(std::make_shared<phi::distributed::DistTensor>());
+  EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(t)
+#endif
+}
+}  // namespace egr

test/cpp/eager/task_tests/forward_autograd_test.cc

@@ -0,0 +1,362 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <sstream>
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/fluid/eager/api/all.h"
+#include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h"
+#include "paddle/fluid/eager/autograd_meta.h"
+#include "paddle/fluid/eager/grad_node_info.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/tensor_meta.h"
+#include "test/cpp/eager/test_utils.h"
+
+PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
+
+namespace egr {
+
+TEST(Forward, SingleNode) {
+  // Prepare Device Contexts
+  eager_test::InitEnv(phi::CPUPlace());
+
+  // Prepare Inputs
+  std::vector<paddle::Tensor> target_tensors;
+  phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
+
+  // Create Target Tensor
+  paddle::Tensor t = eager_test::CreateTensorWithValue(ddim,
+                                                       phi::CPUPlace(),
+                                                       phi::DataType::FLOAT32,
+                                                       phi::DataLayout::NCHW,
+                                                       5.0 /*value*/,
+                                                       false /*is_leaf*/);
+  target_tensors.emplace_back(std::move(t));
+  paddle::Tensor& tensor = target_tensors[0];
+  EagerUtils::autograd_meta(&tensor)->SetStopGradient(false);
+
+  // Run Forward
+  float scale = 2.0;
+  float bias = 3.0;
+  paddle::Tensor out = egr::scale(
+      tensor, scale, bias, true /*bias_after_scale*/, true /*trace_backward*/);
+
+  // Examine Forward Output
+  eager_test::CompareTensorWithValue<float>(out, 13.0);
+
+  // Examine GradNode
+  {
+    // 1. GradNode
+    AutogradMeta* meta = EagerUtils::autograd_meta(&out);
+    GradNodeBase* grad_node = meta->GradNode();
+    GradNodeScale* scale_node = dynamic_cast<GradNodeScale*>(grad_node);
+
+    CHECK_NOTNULL(scale_node);
+    PADDLE_ENFORCE_EQ(
+        static_cast<int>(meta->OutRankInfo().first),
+        0,
+        common::errors::InvalidArgument(
+            "static_cast<int>(meta->OutRankInfo().first) is not 0"));
+    PADDLE_ENFORCE_EQ(
+        static_cast<int>(meta->OutRankInfo().second),
+        0,
+        common::errors::InvalidArgument(
+            "static_cast<int>(meta->OutRankInfo().second) is not 0"));
+  }
+}
+
+/*
+ inp
+  |
+Node0
+  |
+Node1
+  |
+ out
+*/
+TEST(Forward, LinearNodes) {
+  eager_test::InitEnv(phi::CPUPlace());
+
+  // Prepare Inputs
+  std::vector<paddle::Tensor> target_tensors;
+  phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
+
+  // Create Target Tensor
+  paddle::Tensor t = eager_test::CreateTensorWithValue(ddim,
+                                                       phi::CPUPlace(),
+                                                       phi::DataType::FLOAT32,
+                                                       phi::DataLayout::NCHW,
+                                                       5.0 /*value*/,
+                                                       false /*is_leaf*/);
+  target_tensors.emplace_back(std::move(t));
+  paddle::Tensor& tensor = target_tensors[0];
+  EagerUtils::autograd_meta(&tensor)->SetStopGradient(false);
+
+  // Run Forward Node 0
+  float scale0 = 2.0;
+  float bias0 = 3.0;
+  paddle::Tensor out0 = egr::scale(tensor,
+                                   scale0,
+                                   bias0,
+                                   true /*bias_after_scale*/,
+                                   true /*trace_backward*/);
+
+  // Run Forward Node 1
+  float scale1 = 5.0;
+  float bias1 = 10.0;
+  paddle::Tensor out1 = egr::scale(
+      out0, scale1, bias1, true /*bias_after_scale*/, true /*trace_backward*/);
+
+  // Examine Forward Output 0
+  eager_test::CompareTensorWithValue<float>(out0, 13.0);
+
+  // Examine Forward Output 1
+  eager_test::CompareTensorWithValue<float>(out1, 75.0);
+
+  // Examine GradNode
+  {
+    // 1. GradNode
+    // Node 0
+    AutogradMeta* meta0 = EagerUtils::autograd_meta(&out0);
+    GradNodeBase* grad_node0 = meta0->GradNode();
+    GradNodeScale* scale_node0 = dynamic_cast<GradNodeScale*>(grad_node0);
+
+    CHECK_NOTNULL(scale_node0);
+    PADDLE_ENFORCE_EQ(
+        static_cast<int>(meta0->OutRankInfo().first),
+        0,
+        common::errors::InvalidArgument(
+            "static_cast<int>(meta0->OutRankInfo().first) is not 0"));
+    PADDLE_ENFORCE_EQ(
+        static_cast<int>(meta0->OutRankInfo().second),
+        0,
+        common::errors::InvalidArgument(
+            "static_cast<int>(meta0->OutRankInfo().second) is not 0"));
+
+    // Node 1
+    AutogradMeta* meta1 = EagerUtils::autograd_meta(&out1);
+    GradNodeBase* grad_node1 = meta1->GradNode();
+    GradNodeScale* scale_node1 = dynamic_cast<GradNodeScale*>(grad_node1);
+
+    CHECK_NOTNULL(scale_node1);
+    PADDLE_ENFORCE_EQ(
+        static_cast<int>(meta1->OutRankInfo().first),
+        0,
+        common::errors::InvalidArgument(
+            "static_cast<int>(meta1->OutRankInfo().first) is not 0"));
+    PADDLE_ENFORCE_EQ(
+        static_cast<int>(meta1->OutRankInfo().second),
+        0,
+        common::errors::InvalidArgument(
+            "static_cast<int>(meta1->OutRankInfo().second) is not 0"));
+
+    // 2. TensorWrapper: No TensorWrapper for ScaleNode
+    // 3. NextEdges: Node 1 -> Node 0
+    const paddle::small_vector<std::vector<GradSlotMeta>,
+                               egr::kSlotSmallVectorSize>& node1_metas =
+        grad_node1->OutputMeta();
+    const auto& node1_meta = node1_metas[0];
+
+    PADDLE_ENFORCE_EQ(
+        static_cast<int>(node1_meta[0].GetEdge().GetEdgeRankInfo().first),
+        0,
+        common::errors::InvalidArgument(
+            "static_cast<int>(node1_meta[0].GetEdge().GetEdgeRankInfo().first)"
+            "is not 0"));
+    PADDLE_ENFORCE_EQ(
+        static_cast<int>(node1_meta[0].GetEdge().GetEdgeRankInfo().second),
+        0,
+        common::errors::InvalidArgument(
+            "static_cast<int>(node1_meta[0].GetEdge().GetEdgeRankInfo().second)"
+            "is not 0"));
+    PADDLE_ENFORCE_EQ(node1_meta[0].GetEdge().GetGradNode(),
+                      grad_node0,
+                      common::errors::InvalidArgument(
+                          "node1_meta[0].GetEdge().GetGradNode() "
+                          "is not equal with grad_node0, "
+                          "the value of grad_node0 is %d "
+                          "and node1_meta[0].GetEdge().GetGradNode() is %d",
+                          grad_node0,
+                          node1_meta[0].GetEdge().GetGradNode()));
+  }
+}
+
+/*
+       inp
+        |
+      Node0
+    ____|____
+    |       |
+  Node1   Node2
+    |       |
+   out1    out2
+*/
+TEST(Forward, BranchedNodes) {
+  eager_test::InitEnv(phi::CPUPlace());
+
+  // Prepare Inputs
+  std::vector<paddle::Tensor> target_tensors;
+  phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
+
+  // Create Target Tensor
+  paddle::Tensor t = eager_test::CreateTensorWithValue(ddim,
+                                                       phi::CPUPlace(),
+                                                       phi::DataType::FLOAT32,
+                                                       phi::DataLayout::NCHW,
+                                                       5.0 /*value*/,
+                                                       false /*is_leaf*/);
+  target_tensors.emplace_back(std::move(t));
+  paddle::Tensor& tensor = target_tensors[0];
+  EagerUtils::autograd_meta(&tensor)->SetStopGradient(false);
+
+  // Run Forward Node 0
+  float scale0 = 2.0;
+  float bias0 = 3.0;
+  paddle::Tensor out0 = egr::scale(tensor,
+                                   scale0,
+                                   bias0,
+                                   true /*bias_after_scale*/,
+                                   true /*trace_backward*/);
+
+  // Run Forward Node 1
+  float scale1 = 5.0;
+  float bias1 = 10.0;
+  paddle::Tensor out1 = egr::scale(
+      out0, scale1, bias1, true /*bias_after_scale*/, true /*trace_backward*/);
+
+  // Run Forward Node 2
+  float scale2 = 10.0;
+  float bias2 = 20.0;
+  paddle::Tensor out2 = egr::scale(
+      out0, scale2, bias2, true /*bias_after_scale*/, true /*trace_backward*/);
+
+  // Examine Forward Output 0
+  eager_test::CompareTensorWithValue<float>(out0, 13.0);
+
+  // Examine Forward Output 1
+  eager_test::CompareTensorWithValue<float>(out1, 75.0);
+
+  // Examine Forward Output 2
+  eager_test::CompareTensorWithValue<float>(out2, 150.0);
+
+  // Examine GradNode
+  {
+    // 1. GradNode
+    // Node 0
+    AutogradMeta* meta0 = EagerUtils::autograd_meta(&out0);
+    GradNodeBase* grad_node0 = meta0->GradNode();
+    GradNodeScale* scale_node0 = dynamic_cast<GradNodeScale*>(grad_node0);
+
+    CHECK_NOTNULL(scale_node0);
+    PADDLE_ENFORCE_EQ(
+        static_cast<int>(meta0->OutRankInfo().first),
+        0,
+        common::errors::InvalidArgument(
+            "static_cast<int>(meta0->OutRankInfo().first) is not 0"));
+    PADDLE_ENFORCE_EQ(
+        static_cast<int>(meta0->OutRankInfo().second),
+        0,
+        common::errors::InvalidArgument(
+            "static_cast<int>(meta0->OutRankInfo().second) is not 0"));
+
+    // Node 1
+    AutogradMeta* meta1 = EagerUtils::autograd_meta(&out1);
+    GradNodeBase* grad_node1 = meta1->GradNode();
+    GradNodeScale* scale_node1 = dynamic_cast<GradNodeScale*>(grad_node1);
+
+    CHECK_NOTNULL(scale_node1);
+    PADDLE_ENFORCE_EQ(
+        static_cast<int>(meta1->OutRankInfo().first),
+        0,
+        common::errors::InvalidArgument(
+            "static_cast<int>(meta1->OutRankInfo().first) is not 0"));
+    PADDLE_ENFORCE_EQ(
+        static_cast<int>(meta1->OutRankInfo().second),
+        0,
+        common::errors::InvalidArgument(
+            "static_cast<int>(meta1->OutRankInfo().second) is not 0"));
+
+    // Node 2
+    AutogradMeta* meta2 = EagerUtils::autograd_meta(&out2);
+    GradNodeBase* grad_node2 = meta2->GradNode();
+    GradNodeScale* scale_node2 = dynamic_cast<GradNodeScale*>(grad_node2);
+
+    CHECK_NOTNULL(scale_node2);
+    PADDLE_ENFORCE_EQ(
+        static_cast<int>(meta2->OutRankInfo().first),
+        0,
+        common::errors::InvalidArgument(
+            "static_cast<int>(meta2->OutRankInfo().first) is not 0"));
+    PADDLE_ENFORCE_EQ(
+        static_cast<int>(meta2->OutRankInfo().second),
+        0,
+        common::errors::InvalidArgument(
+            "static_cast<int>(meta2->OutRankInfo().second) is not 0"));
+
+    // 2. TensorWrapper: No TensorWrapper for ScaleNode
+    // 3. NextEdges
+    // Node 1 -> Node 0
+    const paddle::small_vector<std::vector<GradSlotMeta>, kSlotSmallVectorSize>&
+        node1_metas = grad_node1->OutputMeta();
+    const Edge& node1_edge = node1_metas[0][0].GetEdge();
+
+    PADDLE_ENFORCE_EQ(
+        static_cast<int>(node1_edge.GetEdgeRankInfo().first),
+        0,
+        common::errors::InvalidArgument(
+            "static_cast<int>(node1_edge.GetEdgeRankInfo().first) is not 0"));
+    PADDLE_ENFORCE_EQ(
+        static_cast<int>(node1_edge.GetEdgeRankInfo().second),
+        0,
+        common::errors::InvalidArgument(
+            "static_cast<int>(node1_edge.GetEdgeRankInfo().second) is not 0"));
+    PADDLE_ENFORCE_EQ(
+        node1_edge.GetGradNode(),
+        grad_node0,
+        common::errors::InvalidArgument(
+            "node1_edge.GetGradNode() is not equal with grad_node0"
+            "the value of node1_edge.GetGradNode() is %d and grad_node0 is %d",
+            node1_edge.GetGradNode(),
+            grad_node0));
+
+    // Node 2 -> Node 0
+    const paddle::small_vector<std::vector<egr::GradSlotMeta>,
+                               egr::kSlotSmallVectorSize>& node2_metas =
+        grad_node2->OutputMeta();
+    const Edge& node2_edge = node2_metas[0][0].GetEdge();
+
+    PADDLE_ENFORCE_EQ(
+        static_cast<int>(node2_edge.GetEdgeRankInfo().first),
+        0,
+        common::errors::InvalidArgument(
+            "static_cast<int>(node2_edge.GetEdgeRankInfo().first) is not 0"));
+    PADDLE_ENFORCE_EQ(
+        static_cast<int>(node2_edge.GetEdgeRankInfo().second),
+        0,
+        common::errors::InvalidArgument(
+            "static_cast<int>(node2_edge.GetEdgeRankInfo().second) is not 0"));
+    PADDLE_ENFORCE_EQ(
+        node2_edge.GetGradNode(),
+        grad_node0,
+        common::errors::InvalidArgument(
+            "node2_edge.GetGradNode() is not equal with grad_node0"
+            "the value of node2_edge.GetGradNode() is %d and grad_node0 is %d",
+            node2_edge.GetGradNode(),
+            grad_node0));
+  }
+}
+
+}  // namespace egr

test/cpp/eager/task_tests/fwd_bwd_joint_test.cc

@@ -0,0 +1,457 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <sstream>
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/fluid/eager/accumulation/accumulation_node.h"
+#include "paddle/fluid/eager/api/all.h"
+#include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h"
+#include "paddle/fluid/eager/autograd_meta.h"
+#include "paddle/fluid/eager/backward.h"
+#include "paddle/fluid/eager/grad_node_info.h"
+#include "paddle/fluid/eager/hooks.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/tensor_meta.h"
+#include "test/cpp/eager/test_utils.h"
+
+PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(add, CPU, ALL_LAYOUT);
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+PD_DECLARE_KERNEL(full, GPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(add, KPS, ALL_LAYOUT);
+#endif
+
+namespace egr {
+
+paddle::Tensor hook_function(const paddle::Tensor& t) {
+  auto t_dense = std::dynamic_pointer_cast<phi::DenseTensor>(t.impl());
+
+  auto ret_meta = phi::DenseTensorMeta(
+      t_dense->dtype(), t_dense->dims(), t_dense->layout());
+  auto place = t_dense->place();
+  size_t bytes_size =
+      common::product(t_dense->dims()) * SizeOf(t_dense->dtype());
+  auto ret_dense = std::make_shared<phi::DenseTensor>(
+      paddle::memory::Alloc(place, bytes_size), std::move(ret_meta));
+
+  float* t_ptr = t_dense->mutable_data<float>(place);
+  float* ret_ptr = ret_dense->mutable_data<float>(place);
+  for (int i = 0; i < ret_dense->numel(); i++) {
+    ret_ptr[i] = t_ptr[i] + 5.0f;
+  }
+
+  auto ret_impl = std::dynamic_pointer_cast<phi::TensorBase>(ret_dense);
+  paddle::Tensor ret = paddle::Tensor();
+  ret.set_impl(ret_impl);
+
+  return ret;
+}
+
+TEST(FwdBwdJoint, SingleNode) {
+  eager_test::InitEnv(phi::CPUPlace());
+
+  // 1. Prepare Input
+  phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
+  paddle::Tensor tensor =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        5.0 /*value*/,
+                                        true /*is_leaf*/);
+  egr_utils_api::RetainGradForTensor(tensor);
+
+  // 3. Run Forward
+  float scale = 2.0;
+  float bias = 3.0;
+  paddle::Tensor out = egr::scale(
+      tensor, scale, bias, true /*bias_after_scale*/, true /*trace_backward*/);
+
+  // Examine Forward Output
+  eager_test::CompareTensorWithValue<float>(out, 13.0);
+
+  std::vector<paddle::Tensor> outs = {out};
+  // 4. Run Backward
+  Backward(outs, {});
+
+  VLOG(7) << "Target Grad is: "
+          << std::static_pointer_cast<phi::DenseTensor>(
+                 EagerUtils::unsafe_autograd_meta(tensor)->Grad().impl())
+                 ->data<float>()[0];
+  // Examine Backward Grad
+  eager_test::CompareGradTensorWithValue<float>(tensor, 2.0);
+}
+
+/*
+ inp
+  |
+Node0
+  |
+Node1
+  |
+ out
+*/
+TEST(FwdBwdJoint, LinearNodes) {
+  eager_test::InitEnv(phi::CPUPlace());
+
+  // 1. Prepare Input
+  phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
+  paddle::Tensor tensor =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        5.0 /*value*/,
+                                        true /*is_leaf*/);
+  egr_utils_api::RetainGradForTensor(tensor);
+
+  // 3. Run Forward
+  // Run Forward Node 0
+  float scale0 = 2.0;
+  float bias0 = 3.0;
+  paddle::Tensor out0 = egr::scale(tensor,
+                                   scale0,
+                                   bias0,
+                                   true /*bias_after_scale*/,
+                                   true /*trace_backward*/);
+
+  // Run Forward Node 1
+  float scale1 = 5.0;
+  float bias1 = 10.0;
+  paddle::Tensor out1 = egr::scale(
+      out0, scale1, bias1, true /*bias_after_scale*/, true /*trace_backward*/);
+
+  // Examine Forward Output 0
+  eager_test::CompareTensorWithValue<float>(out0, 13.0);
+
+  // Examine Forward Output 1
+  eager_test::CompareTensorWithValue<float>(out1, 75.0);
+
+  std::vector<paddle::Tensor> outs = {out1};
+  // 4. Run Backward
+  Backward(outs, {});
+
+  // Examine Backward Grad
+  eager_test::CompareGradTensorWithValue<float>(tensor, 10.0);
+}
+
+/*
+       inp
+        |
+      Node0
+    ____|____
+    |       |
+  Node1   Node2
+    |       |
+   out1    out2
+*/
+TEST(FwdBwdJoint, BranchedNodes) {
+  eager_test::InitEnv(phi::CPUPlace());
+
+  // 1. Prepare Input
+  phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
+  paddle::Tensor tensor =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        5.0 /*value*/,
+                                        true /*is_leaf*/);
+  egr_utils_api::RetainGradForTensor(tensor);
+
+  // 3. Run Forward
+  // Run Forward Node 0
+  float scale0 = 2.0;
+  float bias0 = 3.0;
+  paddle::Tensor out0 = egr::scale(tensor,
+                                   scale0,
+                                   bias0,
+                                   true /*bias_after_scale*/,
+                                   true /*trace_backward*/);
+
+  // Run Forward Node 1
+  float scale1 = 5.0;
+  float bias1 = 10.0;
+  paddle::Tensor out1 = egr::scale(
+      out0, scale1, bias1, true /*bias_after_scale*/, true /*trace_backward*/);
+
+  // Run Forward Node 2
+  float scale2 = 10.0;
+  float bias2 = 20.0;
+  paddle::Tensor out2 = egr::scale(
+      out0, scale2, bias2, true /*bias_after_scale*/, true /*trace_backward*/);
+
+  // Examine Forward Output 0
+  eager_test::CompareTensorWithValue<float>(out0, 13.0);
+
+  // Examine Forward Output 1
+  eager_test::CompareTensorWithValue<float>(out1, 75.0);
+
+  // Examine Forward Output 2
+  {
+    auto dense_out = std::dynamic_pointer_cast<phi::DenseTensor>(out2.impl());
+    float* ptr = dense_out->mutable_data<float>(phi::CPUPlace());
+    for (int i = 0; i < 20; i++) {
+      PADDLE_ENFORCE(ptr[i] == 150.0,
+                     common::errors::Fatal(
+                         "Detected numerical Error, Expected %f but got %f",
+                         150.0,
+                         ptr[i]));
+    }
+  }
+
+  // 4. Run Backward
+  std::vector<paddle::Tensor> outs = {out1, out2};
+  Backward(outs, {});
+
+  // Examine Backward Grad
+  eager_test::CompareGradTensorWithValue<float>(tensor, 30.0);
+}
+
+/*
+       inp
+        |
+      Node0
+    ____|____
+    |       |
+  Node1   Node2
+    |       |
+   out1    out2
+*/
+TEST(FwdBwdJoint, GradientHook) {
+  eager_test::InitEnv(phi::CPUPlace());
+
+  // 1. Prepare Input
+  phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
+  paddle::Tensor tensor =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        5.0 /*value*/,
+                                        true /*is_leaf*/);
+  egr_utils_api::RetainGradForTensor(tensor);
+
+  // 3. Run Forward
+  // Run Forward Node 0
+  float scale0 = 2.0;
+  float bias0 = 3.0;
+  paddle::Tensor out0 = egr::scale(tensor,
+                                   scale0,
+                                   bias0,
+                                   true /*bias_after_scale*/,
+                                   true /*trace_backward*/);
+  egr_utils_api::RetainGradForTensor(out0);  // hook: +5
+  egr_utils_api::RegisterGradientHookForTensor(out0,
+                                               hook_function);  // hook: +5
+
+  // Run Forward Node 1
+  float scale1 = 5.0;
+  float bias1 = 10.0;
+  paddle::Tensor out1 = egr::scale(
+      out0, scale1, bias1, true /*bias_after_scale*/, true /*trace_backward*/);
+  egr_utils_api::RetainGradForTensor(out1);  // hook: +5
+  egr_utils_api::RegisterGradientHookForTensor(out1,
+                                               hook_function);  // hook: +5
+
+  // Run Forward Node 2
+  float scale2 = 10.0;
+  float bias2 = 20.0;
+  paddle::Tensor out2 = egr::scale(
+      out0, scale2, bias2, true /*bias_after_scale*/, true /*trace_backward*/);
+  egr_utils_api::RetainGradForTensor(out2);  // hook: +5
+  egr_utils_api::RegisterGradientHookForTensor(out2,
+                                               hook_function);  // hook: +5
+
+  // 4. Run Backward
+  std::vector<paddle::Tensor> outs = {out1, out2};
+  Backward(outs, {});
+
+  // Examine Backward Grad
+  // leaf grad
+  eager_test::CompareGradTensorWithValue<float>(tensor, 190.0);
+
+  // out0 grad
+  eager_test::CompareGradTensorWithValue<float>(out0, 90.0);
+
+  // out1 grad
+  eager_test::CompareGradTensorWithValue<float>(out1, 1.0);
+
+  // out2 grad
+  eager_test::CompareGradTensorWithValue<float>(out2, 1.0);
+}
+
+/*
+       inp
+        |
+      Node0
+    ____|____
+    |       |
+  Node1   Node2
+    |       |
+   out1    out2
+*/
+TEST(FwdBwdJoint, CrossBatchAccumulation) {
+  eager_test::InitEnv(phi::CPUPlace());
+
+  // 1. Prepare Input
+  phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
+  paddle::Tensor tensor =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        5.0 /*value*/,
+                                        true /*is_leaf*/);
+  egr_utils_api::RetainGradForTensor(tensor);
+
+  // 3. Run Forward
+  // Run Forward Node 0
+  float scale0 = 2.0;
+  float bias0 = 3.0;
+  paddle::Tensor out0 = egr::scale(tensor,
+                                   scale0,
+                                   bias0,
+                                   true /*bias_after_scale*/,
+                                   true /*trace_backward*/);
+
+  // Run Forward Node 1
+  float scale1 = 5.0;
+  float bias1 = 10.0;
+  paddle::Tensor out1 = egr::scale(
+      out0, scale1, bias1, true /*bias_after_scale*/, true /*trace_backward*/);
+
+  // Run Forward Node 2
+  float scale2 = 10.0;
+  float bias2 = 20.0;
+  paddle::Tensor out2 = egr::scale(
+      out0, scale2, bias2, true /*bias_after_scale*/, true /*trace_backward*/);
+
+  // 4. Run Backward
+  std::vector<paddle::Tensor> outs = {out1, out2};
+  Backward(outs, {});
+
+  // Examine Backward Grad
+  eager_test::CompareGradTensorWithValue<float>(tensor, 30.0);
+
+  // Cross Batch Accumulation
+  Backward(outs, {});
+
+  // Examine Backward Grad
+  eager_test::CompareGradTensorWithValue<float>(tensor, 60.0);
+}
+
+/* ---------------------------------------------------- */
+/* ---------------------- CUDA Tests ------------------ */
+/* ---------------------------------------------------- */
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+TEST(FwdBwdJoint, SingleNodeCUDA) {
+  eager_test::InitEnv(phi::GPUPlace());
+
+  // 1. Prepare Input
+  phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
+  paddle::Tensor tensor =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::GPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        5.0 /*value*/,
+                                        true /*is_leaf*/);
+  egr_utils_api::RetainGradForTensor(tensor);
+
+  // 3. Run Forward
+  float scale = 2.0;
+  float bias = 3.0;
+  paddle::Tensor out = egr::scale(
+      tensor, scale, bias, true /*bias_after_scale*/, true /*trace_backward*/);
+
+  // Examine Forward Output
+  eager_test::CompareTensorWithValue<float>(out, 13.0);
+
+  std::vector<paddle::Tensor> outs = {out};
+  // 4. Run Backward
+  Backward(outs, {});
+
+  // Examine Backward Grad
+  eager_test::CompareGradTensorWithValue<float>(tensor, 2.0);
+}
+
+/*
+       inp
+        |
+      Node0
+    ____|____
+    |       |
+  Node1   Node2
+    |       |
+   out1    out2
+*/
+TEST(FwdBwdJoint, BranchedNodesCUDA) {
+  eager_test::InitEnv(phi::GPUPlace());
+
+  // 1. Prepare Input
+  phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
+  paddle::Tensor tensor =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::GPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        5.0 /*value*/,
+                                        true /*is_leaf*/);
+  egr_utils_api::RetainGradForTensor(tensor);
+
+  // 3. Run Forward
+  // Run Forward Node 0
+  float scale0 = 2.0;
+  float bias0 = 3.0;
+  paddle::Tensor out0 = egr::scale(tensor,
+                                   scale0,
+                                   bias0,
+                                   true /*bias_after_scale*/,
+                                   true /*trace_backward*/);
+
+  // Run Forward Node 1
+  float scale1 = 5.0;
+  float bias1 = 10.0;
+  paddle::Tensor out1 = egr::scale(
+      out0, scale1, bias1, true /*bias_after_scale*/, true /*trace_backward*/);
+
+  // Run Forward Node 2
+  float scale2 = 10.0;
+  float bias2 = 20.0;
+  paddle::Tensor out2 = egr::scale(
+      out0, scale2, bias2, true /*bias_after_scale*/, true /*trace_backward*/);
+
+  // Examine Forward Output 0
+  eager_test::CompareTensorWithValue<float>(out0, 13.0);
+  // Examine Forward Output 1
+  eager_test::CompareTensorWithValue<float>(out1, 75.0);
+  // Examine Forward Output 2
+  eager_test::CompareTensorWithValue<float>(out2, 150.0);
+
+  // TODO(jiabin): fix this with add functor
+  // 4. Run Backward
+  std::vector<paddle::Tensor> outs = {out1, out2};
+  Backward(outs, {});
+
+  // Examine Backward Grad
+  eager_test::CompareGradTensorWithValue<float>(tensor, 30.0);
+}
+#endif
+
+}  // namespace egr

test/cpp/eager/task_tests/generated_test.cc

@@ -0,0 +1,141 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Eager Dygraph
+
+#include <chrono>
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/eager/api/all.h"
+#include "paddle/fluid/eager/api/generated/fluid_generated/dygraph_forward_api.h"
+#include "paddle/fluid/eager/autograd_meta.h"
+#include "paddle/fluid/eager/backward.h"
+#include "paddle/fluid/eager/utils.h"
+#include "paddle/fluid/imperative/tracer.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "test/cpp/eager/test_utils.h"
+
+PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(matmul, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(matmul_grad, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(add, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(add_grad, CPU, ALL_LAYOUT);
+
+namespace egr {
+
+TEST(Generated, Sigmoid) {
+  // Prepare Device Contexts
+  eager_test::InitEnv(phi::CPUPlace());
+  VLOG(6) << "Init Env";
+  // 1. Prepare Input
+  phi::DDim ddim = common::make_ddim({2, 4, 4, 4});
+  VLOG(6) << "Make Dim";
+  paddle::Tensor tensor =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        0.0,
+                                        true);
+  VLOG(6) << "Make paddle::Tensor";
+  egr_utils_api::RetainGradForTensor(tensor);
+  VLOG(6) << "Retain Grad for Tensor";
+  auto output_tensor = sigmoid_dygraph_function(tensor, {});
+  VLOG(6) << "Run Backward";
+  eager_test::CompareTensorWithValue<float>(output_tensor, 0.5);
+
+  std::vector<paddle::Tensor> target_tensors = {output_tensor};
+  VLOG(6) << "Running Backward";
+  Backward(target_tensors, {});
+
+  VLOG(6) << "Finish Backward";
+  eager_test::CompareGradTensorWithValue<float>(tensor, 0.25);
+}
+
+TEST(Generated, Matmul_v2) {
+  // Prepare Device Contexts
+  eager_test::InitEnv(phi::CPUPlace());
+
+  auto tracer = std::make_shared<paddle::imperative::Tracer>();
+  paddle::imperative::SetCurrentTracer(tracer);
+
+  // 1. Prepare Input
+  phi::DDim ddimX = common::make_ddim({4, 16});
+  paddle::Tensor X = eager_test::CreateTensorWithValue(ddimX,
+                                                       phi::CPUPlace(),
+                                                       phi::DataType::FLOAT32,
+                                                       phi::DataLayout::NCHW,
+                                                       3.0,
+                                                       true);
+  egr_utils_api::RetainGradForTensor(X);
+
+  phi::DDim ddimY = common::make_ddim({16, 20});
+  paddle::Tensor Y = eager_test::CreateTensorWithValue(ddimY,
+                                                       phi::CPUPlace(),
+                                                       phi::DataType::FLOAT32,
+                                                       phi::DataLayout::NCHW,
+                                                       2.0,
+                                                       true);
+  egr_utils_api::RetainGradForTensor(Y);
+
+  auto output_tensor = matmul_v2_dygraph_function(
+      X, Y, {{"trans_x", false}, {"trans_y", false}});
+
+  eager_test::CompareTensorWithValue<float>(output_tensor, 96);
+
+  std::vector<paddle::Tensor> target_tensors = {output_tensor};
+  Backward(target_tensors, {});
+
+  eager_test::CompareGradTensorWithValue<float>(X, 2.0 * 20);
+  eager_test::CompareGradTensorWithValue<float>(Y, 3.0 * 4);
+}
+
+TEST(Generated, ElementwiseAdd) {
+  // Prepare Device Contexts
+  eager_test::InitEnv(phi::CPUPlace());
+
+  auto tracer = std::make_shared<paddle::imperative::Tracer>();
+  paddle::imperative::SetCurrentTracer(tracer);
+
+  // 1. Prepare Input
+  phi::DDim ddimX = common::make_ddim({4, 16});
+  paddle::Tensor X = eager_test::CreateTensorWithValue(ddimX,
+                                                       phi::CPUPlace(),
+                                                       phi::DataType::FLOAT32,
+                                                       phi::DataLayout::NCHW,
+                                                       3.0,
+                                                       true);
+  egr_utils_api::RetainGradForTensor(X);
+
+  phi::DDim ddimY = common::make_ddim({4, 16});
+  paddle::Tensor Y = eager_test::CreateTensorWithValue(ddimY,
+                                                       phi::CPUPlace(),
+                                                       phi::DataType::FLOAT32,
+                                                       phi::DataLayout::NCHW,
+                                                       2.0,
+                                                       true);
+  egr_utils_api::RetainGradForTensor(Y);
+
+  auto output_tensor = elementwise_add_dygraph_function(X, Y, {});
+
+  eager_test::CompareTensorWithValue<float>(output_tensor, 5);
+
+  std::vector<paddle::Tensor> target_tensors = {output_tensor};
+  Backward(target_tensors, {});
+
+  eager_test::CompareGradTensorWithValue<float>(X, 1.0);
+  eager_test::CompareGradTensorWithValue<float>(Y, 1.0);
+}
+
+}  // namespace egr

test/cpp/eager/task_tests/grad_test.cc

@@ -0,0 +1,371 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <sstream>
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/fluid/eager/accumulation/accumulation_node.h"
+#include "paddle/fluid/eager/api/all.h"
+#include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h"
+#include "paddle/fluid/eager/autograd_meta.h"
+#include "paddle/fluid/eager/backward.h"
+#include "paddle/fluid/eager/grad_node_info.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/tensor_meta.h"
+#include "test/cpp/eager/test_utils.h"
+
+PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(add, CPU, ALL_LAYOUT);
+
+namespace egr {
+
+TEST(Grad, SingleNodeEmptyGrad) {
+  // Prepare Device Contexts
+  eager_test::InitEnv(phi::CPUPlace());
+
+  // Prepare Inputs
+  phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
+
+  // Create Target Tensor (output)
+  paddle::Tensor output_tensor =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        1.0 /*value*/,
+                                        false /*is_leaf*/);
+
+  // Create input tensor
+  const paddle::Tensor leaf_tensor =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        1.0 /*value*/,
+                                        true /*is_leaf*/);
+
+  {
+    // Create Scale Node
+    auto node0_ptr = std::make_shared<GradNodeScale>(1, 1);
+    node0_ptr->SetAttributes_scale(5.0 /*scale*/);
+
+    // Set grad in/out meta
+    node0_ptr->SetDefaultGradInOutMeta();
+
+    // Output_tensor set GradNode、OutRank、StopGradient properties
+    AutogradMeta* auto_grad_meta = EagerUtils::autograd_meta(&output_tensor);
+    auto_grad_meta->SetGradNode(
+        std::dynamic_pointer_cast<GradNodeBase>(node0_ptr));
+    auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
+    auto_grad_meta->SetStopGradient(false);
+
+    // Get autograd_meta from input tensor
+    AutogradMeta* auto_grad_meta1 =
+        EagerUtils::unsafe_autograd_meta(leaf_tensor);
+
+    // Connect Tensor and AccumulationNode via AutoGradMeta
+    auto acc_node_ptr =
+        std::make_shared<egr::GradNodeAccumulation>(leaf_tensor);
+
+    // input tensor set GradNode、OutRank、StopGradient properties
+    auto_grad_meta1->SetGradNode(
+        std::dynamic_pointer_cast<GradNodeBase>(acc_node_ptr));
+    auto_grad_meta1->SetSingleOutRankWithSlot(0, 0);
+    auto_grad_meta1->SetStopGradient(false);
+
+    // grad_node Add Edges
+    std::vector<egr::AutogradMeta*> res = {auto_grad_meta1};
+    node0_ptr->SetGradOutMeta(leaf_tensor, 0);
+  }
+  std::vector<paddle::Tensor> outs = {output_tensor};
+
+  // Run Grad
+  auto result = Grad(outs, {leaf_tensor}, {});
+  // Check Output Value
+  eager_test::CompareTensorWithValue<float>(result[0], 5.0);
+}
+
+TEST(Grad, SingleNodeCustomGrad) {
+  // Prepare Device Contexts
+  eager_test::InitEnv(phi::CPUPlace());
+
+  // Prepare Inputs
+  std::vector<paddle::Tensor> target_tensors;
+  phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
+
+  // Create Target Tensor
+  paddle::Tensor tensor =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        1.0 /*value*/,
+                                        false /*is_leaf*/);
+  target_tensors.emplace_back(std::move(tensor));
+
+  std::vector<paddle::Tensor> grad_tensors;
+  // Create Grad Tensor
+  paddle::Tensor grad_tensor =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        10.0 /*value*/,
+                                        false /*is_leaf*/);
+  grad_tensors.emplace_back(std::move(grad_tensor));
+
+  paddle::Tensor leaf_tensor =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        1.0 /*value*/,
+                                        true /*is_leaf*/);
+
+  {
+    // Create Scale Node
+    auto node0_ptr = std::make_shared<GradNodeScale>(1, 1);
+    node0_ptr->SetAttributes_scale(5.0 /*scale*/);
+
+    // Set grad in/out meta
+    node0_ptr->SetDefaultGradInOutMeta();
+
+    // Connect Tensor and Node via AutoGradMeta
+    AutogradMeta* auto_grad_meta =
+        EagerUtils::autograd_meta(&(target_tensors[0]));
+    auto_grad_meta->SetGradNode(
+        std::dynamic_pointer_cast<GradNodeBase>(node0_ptr));
+    auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
+    auto_grad_meta->SetStopGradient(false);
+
+    AutogradMeta* auto_grad_meta1 = EagerUtils::autograd_meta(&leaf_tensor);
+    // Connect Tensor and AccumulationNode via AutoGradMeta
+    auto acc_node_ptr =
+        std::make_shared<egr::GradNodeAccumulation>(leaf_tensor);
+
+    auto_grad_meta1->SetGradNode(
+        std::dynamic_pointer_cast<GradNodeBase>(acc_node_ptr));
+    auto_grad_meta1->SetSingleOutRankWithSlot(0, 0);
+    auto_grad_meta1->SetStopGradient(false);
+    std::vector<egr::AutogradMeta*> res = {auto_grad_meta1};
+    node0_ptr->SetGradOutMeta(leaf_tensor, 0);
+  }
+
+  auto result = Grad(target_tensors, {leaf_tensor}, grad_tensors);
+
+  // Check Output Value
+  eager_test::CompareTensorWithValue<float>(result[0], 50.0);
+}
+
+/*
+Node1
+  |
+Node0
+  |
+ { } // empty grad tensor
+*/
+TEST(Grad, LinearNodes) {
+  // Prepare Device Contexts
+  eager_test::InitEnv(phi::CPUPlace());
+
+  // Prepare Target Tensor
+  std::vector<paddle::Tensor> target_tensors;
+  phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
+
+  // Create Target Tensor
+  paddle::Tensor tensor =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        1.0 /*value*/,
+                                        false /*is_leaf*/);
+  target_tensors.emplace_back(std::move(tensor));
+
+  paddle::Tensor leaf_tensor =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        1.0 /*value*/,
+                                        true /*is_leaf*/);
+  {
+    // Create Node0
+    auto node0_ptr = std::make_shared<GradNodeScale>(1, 1);
+    node0_ptr->SetAttributes_scale(5.0 /*scale*/);
+
+    // Set grad in/out meta for node0
+    node0_ptr->SetDefaultGradInOutMeta();
+
+    // Create Node1
+    auto node1_ptr = std::make_shared<GradNodeScale>(1, 1);
+    node1_ptr->SetAttributes_scale(10.0 /*scale*/);
+
+    // Set grad in/out meta for node1
+    node1_ptr->SetDefaultGradInOutMeta();
+
+    // Connect Input Tensor and Node0 via AutoGradMeta
+    AutogradMeta* auto_grad_meta =
+        EagerUtils::autograd_meta(&(target_tensors[0]));
+    auto_grad_meta->SetGradNode(
+        std::dynamic_pointer_cast<GradNodeBase>(node0_ptr));
+    auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
+    auto_grad_meta->SetStopGradient(false);
+    // Connect Node0 -> Node1 via Edge
+    auto tmp_tensor = paddle::Tensor();
+    auto* meta0 = EagerUtils::autograd_meta(&tmp_tensor);
+    meta0->SetStopGradient(false);
+    meta0->SetSingleOutRankWithSlot(0, 0);
+    meta0->SetGradNode(node1_ptr);
+    node0_ptr->SetGradOutMeta(tmp_tensor, 0);
+
+    AutogradMeta* auto_grad_meta1 = EagerUtils::autograd_meta(&leaf_tensor);
+    // Connect Tensor and AccumulationNode via AutoGradMeta
+    auto acc_node_ptr =
+        std::make_shared<egr::GradNodeAccumulation>(leaf_tensor);
+
+    auto_grad_meta1->SetGradNode(
+        std::dynamic_pointer_cast<GradNodeBase>(acc_node_ptr));
+    auto_grad_meta1->SetSingleOutRankWithSlot(0, 0);
+
+    auto_grad_meta1->SetStopGradient(false);
+    node1_ptr->SetGradOutMeta(leaf_tensor, 0);
+  }
+
+  // Use Empty Grad Tensor
+  auto result = Grad(target_tensors, {leaf_tensor}, {});
+
+  // Check Output Value
+  eager_test::CompareTensorWithValue<float>(result[0], 50.0);
+}
+
+/*
+    Node2
+    |   |
+Node0   Node1
+  |      |
+ in0   in1
+*/
+TEST(Grad, WithAccumulation) {
+  // Prepare Device Contexts
+  eager_test::InitEnv(phi::CPUPlace());
+
+  // Prepare Inputs
+  phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
+
+  // Create Target Tensor
+  std::vector<paddle::Tensor> target_tensors;
+  paddle::Tensor tensor0 =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        1.0 /*value*/,
+                                        false /*is_leaf*/);
+  paddle::Tensor tensor1 =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        1.0 /*value*/,
+                                        false /*is_leaf*/);
+  target_tensors.emplace_back(std::move(tensor0));
+  target_tensors.emplace_back(std::move(tensor1));
+
+  // Create Grad Tensor
+  std::vector<paddle::Tensor> grad_tensors;
+  paddle::Tensor grad_tensor0 =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        5.0 /*value*/,
+                                        false /*is_leaf*/);
+  paddle::Tensor grad_tensor1 =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        10.0 /*value*/,
+                                        false /*is_leaf*/);
+  grad_tensors.emplace_back(std::move(grad_tensor0));
+  grad_tensors.emplace_back(std::move(grad_tensor1));
+
+  paddle::Tensor leaf_tensor;
+  {
+    // Create Node0
+    auto node0_ptr = std::make_shared<GradNodeScale>(1, 1);
+    node0_ptr->SetAttributes_scale(5.0 /*scale*/);
+    node0_ptr->SetDefaultGradInOutMeta();
+
+    // Create Node1
+    auto node1_ptr = std::make_shared<GradNodeScale>(1, 1);
+    node1_ptr->SetAttributes_scale(10.0 /*scale*/);
+    node1_ptr->SetDefaultGradInOutMeta();
+    // Create Node2
+    auto node2_ptr = std::make_shared<GradNodeScale>(1, 1);
+    node2_ptr->SetAttributes_scale(20.0 /*scale*/);
+    node2_ptr->SetDefaultGradInOutMeta();
+    // Connect Inp0 and Node0 via AutoGradMeta
+    AutogradMeta* auto_grad_meta0 =
+        EagerUtils::autograd_meta(&(target_tensors[0]));
+    auto_grad_meta0->SetGradNode(
+        std::dynamic_pointer_cast<GradNodeBase>(node0_ptr));
+    auto_grad_meta0->SetSingleOutRankWithSlot(0, 0);
+    auto_grad_meta0->SetStopGradient(false);
+    // Connect Inp1 and Node1 via AutoGradMeta
+    AutogradMeta* auto_grad_meta1 =
+        EagerUtils::autograd_meta(&(target_tensors[1]));
+    auto_grad_meta1->SetGradNode(
+        std::dynamic_pointer_cast<GradNodeBase>(node1_ptr));
+    auto_grad_meta1->SetSingleOutRankWithSlot(0, 0);
+    auto_grad_meta1->SetStopGradient(false);
+
+    // Connect Node0 -> Node2 via Edge
+    auto tmp_tensor0 = paddle::Tensor();
+    auto* meta0 = EagerUtils::autograd_meta(&tmp_tensor0);
+    meta0->SetStopGradient(false);
+    meta0->SetSingleOutRankWithSlot(0, 0);
+    meta0->SetGradNode(node2_ptr);
+    node0_ptr->SetGradOutMeta(tmp_tensor0, 0);
+
+    // Connect Node1 -> Node2 via Edge
+    auto tmp_tensor1 = paddle::Tensor();
+    auto meta1 = EagerUtils::autograd_meta(&tmp_tensor1);
+    meta1->SetStopGradient(false);
+    meta1->SetSingleOutRankWithSlot(0, 0);
+    meta1->SetGradNode(node2_ptr);
+    node1_ptr->SetGradOutMeta(tmp_tensor1, 0);
+
+    AutogradMeta* auto_grad_meta2 = EagerUtils::autograd_meta(&leaf_tensor);
+    // Connect Tensor and AccumulationNode via AutoGradMeta
+    auto acc_node_ptr =
+        std::make_shared<egr::GradNodeAccumulation>(leaf_tensor);
+
+    auto_grad_meta2->SetGradNode(
+        std::dynamic_pointer_cast<GradNodeBase>(acc_node_ptr));
+    auto_grad_meta2->SetSingleOutRankWithSlot(0, 0);
+
+    auto_grad_meta2->SetStopGradient(false);
+    node2_ptr->SetGradOutMeta(leaf_tensor, 0);
+  }
+
+  auto result = Grad(target_tensors, {leaf_tensor}, grad_tensors);
+
+  eager_test::CompareTensorWithValue<float>(result[0], 2500.0);
+}
+
+}  // namespace egr

test/cpp/eager/task_tests/hook_test.cc

@@ -0,0 +1,200 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <sstream>
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/fluid/eager/accumulation/accumulation_node.h"
+#include "paddle/fluid/eager/api/all.h"
+#include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h"
+#include "paddle/fluid/eager/autograd_meta.h"
+#include "paddle/fluid/eager/backward.h"
+#include "paddle/fluid/eager/grad_node_info.h"
+#include "paddle/fluid/eager/hooks.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/tensor_meta.h"
+#include "test/cpp/eager/test_utils.h"
+
+PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
+
+namespace egr {
+
+paddle::Tensor hook_function(const paddle::Tensor& t) {
+  auto t_dense = std::dynamic_pointer_cast<phi::DenseTensor>(t.impl());
+
+  auto ret_meta = phi::DenseTensorMeta(
+      t_dense->dtype(), t_dense->dims(), t_dense->layout());
+  auto place = t_dense->place();
+  size_t bytes_size =
+      common::product(t_dense->dims()) * SizeOf(t_dense->dtype());
+  auto ret_dense = std::make_shared<phi::DenseTensor>(
+      paddle::memory::Alloc(place, bytes_size), std::move(ret_meta));
+
+  float* t_ptr = t_dense->mutable_data<float>(place);
+  float* ret_ptr = ret_dense->mutable_data<float>(place);
+  for (int i = 0; i < ret_dense->numel(); i++) {
+    ret_ptr[i] = t_ptr[i] + 3.0f;
+  }
+
+  auto ret_impl = std::dynamic_pointer_cast<phi::TensorBase>(ret_dense);
+  paddle::Tensor ret = paddle::Tensor();
+  ret.set_impl(ret_impl);
+
+  return ret;
+}
+
+TEST(RetainGrad, HookBeforeRetainGrad) {
+  eager_test::InitEnv(phi::CPUPlace());
+
+  // Prepare Inputs
+  std::vector<paddle::Tensor> target_tensors;
+  phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
+
+  // Create Target Tensor
+  paddle::Tensor tensor =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        1.0 /*value*/,
+                                        false /*is_leaf*/);
+  target_tensors.emplace_back(std::move(tensor));
+  paddle::Tensor& target_tensor = target_tensors[0];
+
+  // Create ScaleNode
+  auto scale_node_ptr = std::make_shared<GradNodeScale>(1, 1);
+  scale_node_ptr->SetAttributes_scale(5.0 /*scale*/);
+
+  // Set grad in/out meta for node0
+  scale_node_ptr->SetDefaultGradInOutMeta();
+
+  // Connect Input Tensor and ScaleNode via AutoGradMeta
+  // Apply RetainGrad
+  {
+    // ScaleNode Hook: +3
+
+    auto auto_grad_meta = std::make_shared<AutogradMeta>();
+    auto_grad_meta->SetGradNode(
+        std::dynamic_pointer_cast<GradNodeBase>(scale_node_ptr));
+    auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
+    auto_grad_meta->SetStopGradient(false);
+    target_tensor.set_autograd_meta(
+        std::dynamic_pointer_cast<paddle::AbstractAutogradMeta>(
+            auto_grad_meta));
+
+    egr_utils_api::RegisterGradientHookForTensor(target_tensor, hook_function);
+    egr_utils_api::RetainGradForTensor(
+        target_tensor);  // result: 1.0 + 3.0 = 4.0
+    egr_utils_api::RetainGradForTensor(
+        target_tensor);  // result: 1.0 + 3.0 = 4.0
+  }
+
+  // Retain Grad for leaf tensor1
+  paddle::Tensor leaf_tensor = paddle::Tensor();
+  {
+    // AccumulationNode Hook: +3
+    auto tmp_tensor0 = paddle::Tensor();
+    auto auto_grad_meta = EagerUtils::autograd_meta(&tmp_tensor0);
+
+    auto acc_node_ptr = std::make_shared<GradNodeAccumulation>(tmp_tensor0);
+
+    auto_grad_meta->SetStopGradient(false);
+    auto_grad_meta->SetGradNode(acc_node_ptr);
+    auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
+    std::vector<egr::AutogradMeta*> res = {auto_grad_meta};
+    scale_node_ptr->SetGradOutMeta(tmp_tensor0, 0);
+
+    leaf_tensor.set_autograd_meta(
+        std::dynamic_pointer_cast<paddle::AbstractAutogradMeta>(
+            tmp_tensor0.mutable_autograd_meta()));
+
+    egr_utils_api::RegisterGradientHookForTensor(leaf_tensor, hook_function);
+    egr_utils_api::RetainGradForTensor(
+        leaf_tensor);  // result: 4.0*5.0 + 3.0 = 23.0
+  }
+
+  Backward(target_tensors, {});
+
+  eager_test::CompareGradTensorWithValue<float>(target_tensor, 4.0);
+  eager_test::CompareGradTensorWithValue<float>(leaf_tensor, 23.0);
+}
+
+TEST(RetainGrad, HookAfterRetainGrad) {
+  eager_test::InitEnv(phi::CPUPlace());
+
+  // Prepare Inputs
+  std::vector<paddle::Tensor> target_tensors;
+  phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
+
+  // Create Target Tensor
+  paddle::Tensor tensor =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        1.0 /*value*/,
+                                        false /*is_leaf*/);
+  target_tensors.emplace_back(std::move(tensor));
+  paddle::Tensor& target_tensor = target_tensors[0];
+
+  // Create ScaleNode
+  auto scale_node_ptr = std::make_shared<GradNodeScale>(1, 1);
+  scale_node_ptr->SetAttributes_scale(5.0 /*scale*/);
+  // Set grad in/out meta for node0
+  scale_node_ptr->SetDefaultGradInOutMeta();
+
+  // Connect Input Tensor and ScaleNode via AutoGradMeta
+  // Apply RetainGrad
+  {
+    // ScaleNode Hook: +3
+
+    auto auto_grad_meta = std::make_shared<AutogradMeta>();
+    auto_grad_meta->SetGradNode(
+        std::dynamic_pointer_cast<GradNodeBase>(scale_node_ptr));
+    auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
+    auto_grad_meta->SetStopGradient(false);
+    target_tensor.set_autograd_meta(
+        std::dynamic_pointer_cast<paddle::AbstractAutogradMeta>(
+            auto_grad_meta));
+
+    egr_utils_api::RetainGradForTensor(target_tensor);  // result: 1.0
+    egr_utils_api::RegisterGradientHookForTensor(target_tensor, hook_function);
+  }
+
+  // Retain Grad for leaf tensor1
+  paddle::Tensor leaf_tensor = paddle::Tensor();
+  {
+    // AccumulationNode Hook: +3
+    auto tmp_tensor0 = paddle::Tensor();
+    auto auto_grad_meta = EagerUtils::autograd_meta(&tmp_tensor0);
+    auto acc_node_ptr = std::make_shared<GradNodeAccumulation>(tmp_tensor0);
+    auto_grad_meta->SetGradNode(acc_node_ptr);
+    auto_grad_meta->SetStopGradient(false);
+    scale_node_ptr->SetGradOutMeta(tmp_tensor0, 0);
+
+    auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
+    leaf_tensor.set_autograd_meta(
+        std::dynamic_pointer_cast<paddle::AbstractAutogradMeta>(
+            tmp_tensor0.mutable_autograd_meta()));
+
+    egr_utils_api::RegisterGradientHookForTensor(leaf_tensor, hook_function);
+  }
+
+  Backward(target_tensors, {});
+  eager_test::CompareGradTensorWithValue<float>(target_tensor, 1.0);
+  eager_test::CompareGradTensorWithValue<float>(leaf_tensor, 23.0);
+}
+}  // namespace egr

test/cpp/eager/task_tests/hook_test_intermediate.cc

@@ -0,0 +1,327 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <sstream>
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/eager/api/all.h"
+#include "paddle/fluid/eager/api/generated/fluid_generated/dygraph_forward_api.h"
+#include "paddle/fluid/eager/backward.h"
+#include "paddle/fluid/eager/grad_node_info.h"
+#include "paddle/fluid/eager/hooks.h"
+#include "paddle/fluid/imperative/tracer.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "test/cpp/eager/test_utils.h"
+
+PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(matmul, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(matmul_grad, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(add, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(add_grad, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(sigmoid, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(sigmoid_grad, CPU, ALL_LAYOUT);
+
+namespace egr {
+
+paddle::Tensor hook_function(const paddle::Tensor& t) {
+  auto t_dense = std::dynamic_pointer_cast<phi::DenseTensor>(t.impl());
+
+  auto ret_meta = phi::DenseTensorMeta(
+      t_dense->dtype(), t_dense->dims(), t_dense->layout());
+  auto place = t_dense->place();
+  size_t bytes_size =
+      common::product(t_dense->dims()) * SizeOf(t_dense->dtype());
+  auto ret_dense = std::make_shared<phi::DenseTensor>(
+      paddle::memory::Alloc(place, bytes_size), std::move(ret_meta));
+
+  float* t_ptr = t_dense->mutable_data<float>(place);
+  float* ret_ptr = ret_dense->mutable_data<float>(place);
+  for (int i = 0; i < ret_dense->numel(); i++) {
+    ret_ptr[i] = t_ptr[i] + 3.0f;
+  }
+
+  auto ret_impl = std::dynamic_pointer_cast<phi::TensorBase>(ret_dense);
+  paddle::Tensor ret = paddle::Tensor();
+  ret.set_impl(ret_impl);
+
+  return ret;
+}
+
+void test_sigmoid(bool is_remove_gradient_hook) {
+  // Prepare Device Contexts
+  VLOG(6) << "Init Env";
+  eager_test::InitEnv(phi::CPUPlace());
+
+  VLOG(6) << "Make Dim";
+  phi::DDim ddim = common::make_ddim({2, 4, 4, 4});
+
+  VLOG(6) << "Make paddle::Tensor";
+  paddle::Tensor tensor =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        0.0,
+                                        true);
+
+  VLOG(6) << "Make ReduceHook function";
+  auto reduce_hook = [&]() -> void {
+    auto* t_ptr = std::dynamic_pointer_cast<phi::DenseTensor>(tensor.impl())
+                      ->data<float>();
+    for (int i = 0; i < tensor.numel(); i++) {
+      t_ptr[i] = 100.0;  // set to 100.0
+    }
+  };
+
+  VLOG(6) << "Retain Grad for Tensor";
+  egr_utils_api::RetainGradForTensor(tensor);
+
+  VLOG(6) << "Register GradientHook for Tensor";
+  int64_t hook_id =
+      egr_utils_api::RegisterGradientHookForTensor(tensor, hook_function);
+
+  VLOG(6) << "Register ReduceHook for Tensor";
+  egr_utils_api::RegisterReduceHookForTensor(tensor, reduce_hook);
+
+  VLOG(6) << "Running Forward";
+  auto output_tensor = sigmoid_dygraph_function(tensor, {});
+  VLOG(6) << "Finish Forward";
+
+  eager_test::CompareTensorWithValue<float>(output_tensor, 0.5);
+
+  std::vector<paddle::Tensor> target_tensors = {output_tensor};
+
+  if (is_remove_gradient_hook) {
+    std::shared_ptr<GradNodeBase> grad_node_tmp = EagerUtils::grad_node(tensor);
+    grad_node_tmp->RemoveGradientHook(hook_id);
+  }
+
+  VLOG(6) << "Running Backward";
+  Backward(target_tensors, {});
+  VLOG(6) << "Finish Backward";
+
+  eager_test::CompareGradTensorWithValue<float>(
+      tensor, is_remove_gradient_hook ? 0.25 : 0.25 + 3.0);
+
+  VLOG(6) << "Checking ReduceHook results";
+  for (int i = 0; i < tensor.numel(); i++) {
+    PADDLE_ENFORCE_EQ(
+        std::dynamic_pointer_cast<phi::DenseTensor>(tensor.impl())
+            ->data<float>()[i],
+        static_cast<float>(100.0f),
+        common::errors::InvalidArgument(
+            "Required tensor.impl()->data[%d] should be equal to 100.0 . ", i));
+  }
+  VLOG(6) << "After Tests";
+}
+
+void test_elementwiseAdd(bool is_remove_gradient_hook) {
+  // Prepare Device Contexts
+  eager_test::InitEnv(phi::CPUPlace());
+
+  auto tracer = std::make_shared<paddle::imperative::Tracer>();
+  paddle::imperative::SetCurrentTracer(tracer);
+
+  // 1. Prepare Input
+  phi::DDim ddimX = common::make_ddim({4, 16});
+  paddle::Tensor X = eager_test::CreateTensorWithValue(ddimX,
+                                                       phi::CPUPlace(),
+                                                       phi::DataType::FLOAT32,
+                                                       phi::DataLayout::NCHW,
+                                                       3.0,
+                                                       true);
+  egr_utils_api::RetainGradForTensor(X);
+
+  phi::DDim ddimY = common::make_ddim({4, 16});
+  paddle::Tensor Y = eager_test::CreateTensorWithValue(ddimY,
+                                                       phi::CPUPlace(),
+                                                       phi::DataType::FLOAT32,
+                                                       phi::DataLayout::NCHW,
+                                                       2.0,
+                                                       true);
+
+  auto reduce_hook = [&]() -> void {
+    auto* t_ptr =
+        std::dynamic_pointer_cast<phi::DenseTensor>(Y.impl())->data<float>();
+    for (int i = 0; i < Y.numel(); i++) {
+      t_ptr[i] = 100.0;  // set to 100.0
+    }
+  };
+
+  egr_utils_api::RetainGradForTensor(Y);
+  int64_t hook_id =
+      egr_utils_api::RegisterGradientHookForTensor(Y, hook_function);
+  egr_utils_api::RegisterReduceHookForTensor(Y, reduce_hook);
+
+  auto output_tensor = elementwise_add_dygraph_function(X, Y, {});
+
+  eager_test::CompareTensorWithValue<float>(output_tensor, 5);
+  std::vector<paddle::Tensor> target_tensors = {output_tensor};
+
+  if (is_remove_gradient_hook) {
+    std::shared_ptr<GradNodeBase> grad_node_tmp = EagerUtils::grad_node(Y);
+    grad_node_tmp->RemoveGradientHook(hook_id);
+  }
+
+  Backward(target_tensors, {});
+
+  eager_test::CompareGradTensorWithValue<float>(X, 1.0);
+  eager_test::CompareGradTensorWithValue<float>(
+      Y, is_remove_gradient_hook ? 1.0 : 1.0 + 3.0);
+
+  // Checking ReduceHook results
+  for (int i = 0; i < Y.numel(); i++) {
+    PADDLE_ENFORCE_EQ(
+        std::dynamic_pointer_cast<phi::DenseTensor>(Y.impl())->data<float>()[i],
+        static_cast<float>(100.0f),
+        common::errors::InvalidArgument(
+            "Required Y.impl()->data[%d] should be equal to 100.0 . ", i));
+  }
+}
+
+void test_matmul(bool is_remove_gradient_hook) {
+  // Prepare Device Contexts
+  eager_test::InitEnv(phi::CPUPlace());
+
+  auto tracer = std::make_shared<paddle::imperative::Tracer>();
+  paddle::imperative::SetCurrentTracer(tracer);
+
+  // 1. Prepare Input
+  phi::DDim ddimX = common::make_ddim({4, 16});
+  paddle::Tensor X = eager_test::CreateTensorWithValue(ddimX,
+                                                       phi::CPUPlace(),
+                                                       phi::DataType::FLOAT32,
+                                                       phi::DataLayout::NCHW,
+                                                       3.0,
+                                                       true);
+  egr_utils_api::RetainGradForTensor(X);
+
+  phi::DDim ddimY = common::make_ddim({16, 20});
+  paddle::Tensor Y = eager_test::CreateTensorWithValue(ddimY,
+                                                       phi::CPUPlace(),
+                                                       phi::DataType::FLOAT32,
+                                                       phi::DataLayout::NCHW,
+                                                       2.0,
+                                                       true);
+
+  auto reduce_hook = [&]() -> void {
+    auto* t_ptr =
+        std::dynamic_pointer_cast<phi::DenseTensor>(Y.impl())->data<float>();
+    for (int i = 0; i < Y.numel(); i++) {
+      t_ptr[i] = 100.0;  // set to 100.0
+    }
+  };
+
+  egr_utils_api::RetainGradForTensor(Y);
+  int64_t hook_id =
+      egr_utils_api::RegisterGradientHookForTensor(Y, hook_function);
+  egr_utils_api::RegisterReduceHookForTensor(Y, reduce_hook);
+
+  auto output_tensor = matmul_v2_dygraph_function(
+      X, Y, {{"trans_x", false}, {"trans_y", false}});
+
+  eager_test::CompareTensorWithValue<float>(output_tensor, 96);
+  std::vector<paddle::Tensor> target_tensors = {output_tensor};
+
+  if (is_remove_gradient_hook) {
+    std::shared_ptr<GradNodeBase> grad_node_tmp = EagerUtils::grad_node(Y);
+    grad_node_tmp->RemoveGradientHook(hook_id);
+  }
+
+  Backward(target_tensors, {});
+
+  eager_test::CompareGradTensorWithValue<float>(X, 2.0 * 20);
+  eager_test::CompareGradTensorWithValue<float>(
+      Y, is_remove_gradient_hook ? 3.0 * 4 : 3.0 * 4 + 3);
+
+  // Checking ReduceHook results
+  for (int i = 0; i < Y.numel(); i++) {
+    PADDLE_ENFORCE_EQ(
+        std::dynamic_pointer_cast<phi::DenseTensor>(Y.impl())->data<float>()[i],
+        static_cast<float>(100.0f),
+        common::errors::InvalidArgument(
+            "Required Y.impl()->data[%d] should be equal to 100.0 . ", i));
+  }
+}
+
+void test_backward_final_hooks() {
+  // Prepare Device Contexts
+  VLOG(6) << "Init Env";
+  eager_test::InitEnv(phi::CPUPlace());
+
+  VLOG(6) << "Make paddle::Tensor";
+  phi::DDim ddimX = common::make_ddim({4, 16});
+  paddle::Tensor X = eager_test::CreateTensorWithValue(ddimX,
+                                                       phi::CPUPlace(),
+                                                       phi::DataType::FLOAT32,
+                                                       phi::DataLayout::NCHW,
+                                                       3.0,
+                                                       true);
+  phi::DDim ddimY = common::make_ddim({16, 20});
+  egr_utils_api::RetainGradForTensor(X);
+
+  paddle::Tensor Y = eager_test::CreateTensorWithValue(ddimY,
+                                                       phi::CPUPlace(),
+                                                       phi::DataType::FLOAT32,
+                                                       phi::DataLayout::NCHW,
+                                                       2.0,
+                                                       true);
+
+  VLOG(6) << "Make ReduceHook function";
+  auto backward_final_hook = [&]() -> void {
+    auto* t_ptr =
+        std::dynamic_pointer_cast<phi::DenseTensor>(X.impl())->data<float>();
+    VLOG(6) << "Run Target Backward Hook";
+    for (int i = 0; i < X.numel(); i++) {
+      t_ptr[i] = 100.0;  // set to 100.0
+    }
+  };
+  VLOG(6) << "Register Backward Final Hook";
+  egr_utils_api::RegisterBackwardFinalHook(backward_final_hook);
+
+  VLOG(6) << "Running Forward";
+  auto output_tensor = matmul_v2_dygraph_function(
+      X, Y, {{"trans_x", false}, {"trans_y", false}});
+  auto res = sigmoid_dygraph_function(output_tensor, {});
+  VLOG(6) << "Finish Forward";
+
+  eager_test::CompareTensorWithValue<float>(X, 3.0);
+
+  std::vector<paddle::Tensor> target_tensors = {output_tensor};
+
+  VLOG(6) << "Running Backward";
+  Backward(target_tensors, {});
+  VLOG(6) << "Finish Backward";
+  eager_test::CompareTensorWithValue<float>(X, 100.0);
+}
+
+TEST(Hook_intermediate, Sigmoid) {
+  // True or false represents whether to call RemoveGradientHook
+  test_sigmoid(true);
+  test_sigmoid(false);
+}
+
+TEST(Hook_intermediate, ElementwiseAdd) {
+  test_elementwiseAdd(true);
+  test_elementwiseAdd(false);
+}
+
+TEST(Hook_intermediate, Matmul_v2) {
+  test_matmul(true);
+  test_matmul(false);
+}
+
+TEST(Hook_intermediate, BackwardFinal) { test_backward_final_hooks(); }
+}  // namespace egr

test/cpp/eager/task_tests/nan_inf_utils_test.cc

@@ -0,0 +1,169 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/eager/nan_inf_utils.h"
+
+#include <iostream>
+#include <limits>
+#include <ostream>
+#include <string>
+#include <tuple>
+
+#include "gtest/gtest.h"
+#include "paddle/common/flags.h"
+#include "paddle/fluid/framework/tensor_util.h"
+#include "paddle/fluid/platform/enforce.h"
+#include "paddle/phi/api/include/api.h"
+#include "paddle/phi/api/include/strings_api.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(strings_empty, CPU, ALL_LAYOUT);
+
+COMMON_DECLARE_string(check_nan_inf_blacklist);
+
+namespace egr {
+
+using paddle_flags::FLAGS_check_nan_inf_blacklist;
+#define CHECK_NAN_INF(tensors)                                               \
+  {                                                                          \
+    bool caught_exception = false;                                           \
+    try {                                                                    \
+      CheckTensorHasNanOrInf("nan_inf_test", tensors);                       \
+    } catch (paddle::platform::EnforceNotMet & error) {                      \
+      caught_exception = true;                                               \
+      std::string ex_msg = error.what();                                     \
+      EXPECT_TRUE(ex_msg.find("There are NAN or INF") != std::string::npos); \
+    }                                                                        \
+    EXPECT_TRUE(caught_exception);                                           \
+  }
+
+#define CHECK_NO_NAN_INF(tensors)                                            \
+  {                                                                          \
+    bool caught_exception = false;                                           \
+    try {                                                                    \
+      CheckTensorHasNanOrInf("nan_inf_test", tensors);                       \
+    } catch (paddle::platform::EnforceNotMet & error) {                      \
+      caught_exception = true;                                               \
+      std::string ex_msg = error.what();                                     \
+      EXPECT_TRUE(ex_msg.find("There are NAN or INF") != std::string::npos); \
+    }                                                                        \
+    EXPECT_FALSE(caught_exception);                                          \
+  }
+
+#define CHECK_APINAME_SKIP(api_name, tensor)            \
+  {                                                     \
+    bool caught_exception = false;                      \
+    try {                                               \
+      CheckTensorHasNanOrInf(api_name, tensor);         \
+    } catch (paddle::platform::EnforceNotMet & error) { \
+      caught_exception = true;                          \
+    }                                                   \
+    EXPECT_FALSE(caught_exception);                     \
+  }
+
+#define CHECK_APINAME_NO_SKIP(api_name, tensor)         \
+  {                                                     \
+    bool caught_exception = false;                      \
+    try {                                               \
+      CheckTensorHasNanOrInf(api_name, tensor);         \
+    } catch (paddle::platform::EnforceNotMet & error) { \
+      caught_exception = true;                          \
+    }                                                   \
+    EXPECT_TRUE(caught_exception);                      \
+  }
+
+TEST(NanInfUtils, BlacklistSkipCheck) {
+  auto nan_tensor = paddle::experimental::full(
+      {3, 4}, std::numeric_limits<double>::quiet_NaN(), phi::DataType::FLOAT64);
+
+  FLAGS_check_nan_inf_blacklist = "";
+  CHECK_APINAME_SKIP("empty", nan_tensor);
+
+  // Test that "empty_like" always skips regardless of blacklist
+  FLAGS_check_nan_inf_blacklist = "";
+  CHECK_APINAME_SKIP("empty_like", nan_tensor);
+
+  // Test with empty blacklist (default behavior)
+  FLAGS_check_nan_inf_blacklist = "";
+  CHECK_APINAME_NO_SKIP("some_op", nan_tensor);
+
+  // Test with single op in blacklist
+  FLAGS_check_nan_inf_blacklist = "single_op";
+  CHECK_APINAME_SKIP("single_op", nan_tensor);
+  CHECK_APINAME_NO_SKIP("other_op", nan_tensor);
+
+  // Even when blacklist is set, these should still skip
+  CHECK_APINAME_SKIP("empty", nan_tensor);
+  CHECK_APINAME_SKIP("empty_like", nan_tensor);
+
+  // blacklist="op1,op2,op3" and op is in blacklist
+  FLAGS_check_nan_inf_blacklist = "op1,op2,op3";
+  CHECK_APINAME_SKIP("op1", nan_tensor);
+  CHECK_APINAME_SKIP("op2", nan_tensor);
+  CHECK_APINAME_SKIP("op3", nan_tensor);
+  // not in blacklist, should perform nan_or_inf check
+  CHECK_APINAME_NO_SKIP("op4", nan_tensor);
+
+  FLAGS_check_nan_inf_blacklist = "";
+}
+
+TEST(NanInfUtils, Functions) {
+  // test all methods
+  auto tensor = paddle::experimental::full(
+      {3, 4}, std::numeric_limits<double>::quiet_NaN(), phi::DataType::FLOAT64);
+  CHECK_NAN_INF(tensor);
+  auto tensor1 = paddle::experimental::full(
+      {3, 4}, std::numeric_limits<double>::quiet_NaN(), phi::DataType::FLOAT64);
+  auto two_tensors = std::make_tuple(tensor, tensor1);
+  CHECK_NAN_INF(two_tensors);
+  auto tensor2 = paddle::experimental::full(
+      {3, 4}, std::numeric_limits<double>::quiet_NaN(), phi::DataType::FLOAT64);
+  auto three_tensors = std::make_tuple(tensor, tensor1, tensor2);
+  CHECK_NAN_INF(three_tensors);
+  auto tensor3 = paddle::experimental::full(
+      {3, 4}, std::numeric_limits<double>::quiet_NaN(), phi::DataType::FLOAT64);
+  auto four_tensors = std::make_tuple(tensor, tensor1, tensor2, tensor3);
+  CHECK_NAN_INF(four_tensors);
+  auto tensor4 = paddle::experimental::full(
+      {3, 4}, std::numeric_limits<double>::quiet_NaN(), phi::DataType::FLOAT64);
+  auto five_tensors =
+      std::make_tuple(tensor, tensor1, tensor2, tensor3, tensor4);
+  CHECK_NAN_INF(five_tensors);
+  auto tensor5 = paddle::experimental::full(
+      {3, 4}, std::numeric_limits<double>::quiet_NaN(), phi::DataType::FLOAT64);
+  auto six_tensors =
+      std::make_tuple(tensor, tensor1, tensor2, tensor3, tensor4, tensor5);
+  CHECK_NAN_INF(six_tensors);
+  std::vector<paddle::Tensor> tensor_vec;
+  tensor_vec.emplace_back(tensor);
+  tensor_vec.emplace_back(tensor1);
+  CHECK_NAN_INF(tensor_vec);
+  paddle::small_vector<std::vector<paddle::Tensor>, egr::kSlotSmallVectorSize>
+      small_vec;
+  small_vec.emplace_back(tensor_vec);
+  CHECK_NAN_INF(small_vec);
+  // test selected_rows
+  paddle::Tensor tensor_sr;
+  auto sr = std::make_shared<phi::SelectedRows>();
+  *sr->mutable_value() =
+      *(static_cast<const phi::DenseTensor*>(tensor.impl().get()));
+  tensor_sr.set_impl(sr);
+  CHECK_NAN_INF(tensor_sr);
+  // test other tensor
+  auto tensor_str = paddle::experimental::strings::empty({3, 4});
+  CHECK_NO_NAN_INF(tensor_str);
+}
+
+}  // namespace egr

test/cpp/eager/task_tests/tensor_utils_test.cc

@@ -0,0 +1,67 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <sstream>
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/eager/eager_tensor.h"
+#include "paddle/fluid/eager/grad_node_info.h"
+#include "paddle/fluid/eager/grad_tensor_holder.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "test/cpp/eager/test_utils.h"
+
+PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
+
+namespace egr {
+
+TEST(TensorUtils, Test) {
+  // Prepare Device Contexts
+  eager_test::InitEnv(phi::CPUPlace());
+
+  // Prepare Inputs
+  std::vector<paddle::Tensor> target_tensors;
+  phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
+
+  // Create Target Tensor
+  paddle::Tensor t = eager_test::CreateTensorWithValue(ddim,
+                                                       phi::CPUPlace(),
+                                                       phi::DataType::FLOAT32,
+                                                       phi::DataLayout::NCHW,
+                                                       5.0 /*value*/,
+                                                       true /*is_leaf*/);
+
+  paddle::Tensor t_grad =
+      eager_test::CreateTensorWithValue(ddim,
+                                        phi::CPUPlace(),
+                                        phi::DataType::FLOAT32,
+                                        phi::DataLayout::NCHW,
+                                        1.0 /*value*/,
+                                        false /*is_leaf*/);
+
+  PADDLE_ENFORCE_EQ(
+      EagerUtils::IsLeafTensor(t),
+      true,
+      common::errors::InvalidArgument("The tensor t is not a leaf tensor."));
+
+  // Test Utils
+  eager_test::CompareTensorWithValue<float>(t, 5.0);
+
+  egr::AutogradMeta* meta = egr::EagerUtils::autograd_meta(&t);
+  *meta->MutableGrad() = t_grad;
+
+  eager_test::CompareGradTensorWithValue<float>(t, 1.0);
+}
+
+}  // namespace egr

test/cpp/eager/test_utils.h

@@ -0,0 +1,136 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/fluid/eager/accumulation/accumulation_node.h"
+#include "paddle/fluid/eager/api/utils/global_utils.h"
+#include "paddle/fluid/eager/autograd_meta.h"
+#include "paddle/fluid/eager/eager_tensor.h"
+#include "paddle/fluid/eager/utils.h"
+#include "paddle/fluid/platform/init.h"
+#include "paddle/phi/api/all.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/memory/memcpy.h"
+#include "paddle/phi/core/platform/device_context.h"
+#include "paddle/phi/core/tensor_meta.h"
+
+namespace eager_test {
+
+inline paddle::Tensor CreateTensorWithValue(const phi::DDim& ddim,
+                                            const phi::Place& place,
+                                            const phi::DataType& dtype,
+                                            const phi::DataLayout& layout,
+                                            float value,
+                                            bool is_leaf = true) {
+  paddle::Tensor out =
+      paddle::experimental::full(common::vectorize(ddim),
+                                 paddle::experimental::Scalar(value),
+                                 dtype,
+                                 place);
+
+  auto meta = egr::EagerUtils::autograd_meta(&out);
+  if (is_leaf) {
+    auto accumulation_node = std::make_shared<egr::GradNodeAccumulation>(out);
+    meta->SetGradNode(accumulation_node);
+    meta->SetStopGradient(false);
+  }
+
+  return out;
+}
+
+template <typename T>
+bool CompareGradTensorWithValue(const paddle::Tensor& target, T value) {
+  egr::AutogradMeta* meta = egr::EagerUtils::unsafe_autograd_meta(target);
+  auto grad_dense =
+      std::dynamic_pointer_cast<phi::DenseTensor>(meta->Grad().impl());
+  T* ptr = grad_dense->data<T>();
+
+  std::vector<T> host_data(grad_dense->numel());
+  if (phi::is_gpu_place(grad_dense->place())) {
+#ifdef PADDLE_WITH_CUDA
+    phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+    auto* dev_ctx = dynamic_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+    auto stream = dev_ctx->stream();
+
+    paddle::memory::Copy(phi::CPUPlace(),
+                         host_data.data(),
+                         phi::GPUPlace(),
+                         ptr,
+                         sizeof(T) * grad_dense->numel(),
+                         stream);
+    ptr = host_data.data();
+#endif
+  }
+  VLOG(6) << "CompareGradTensorWithValue";
+  for (int i = 0; i < grad_dense->numel(); i++) {
+    PADDLE_ENFORCE(value == ptr[i],
+                   common::errors::PreconditionNotMet(
+                       "Numerical Error in Compare Grad Variable With Value of "
+                       "%d, we expected got value: %f, but got: %f instead. "
+                       "Please check it later.",
+                       i,
+                       value,
+                       ptr[i]));
+  }
+  return true;
+}
+
+template <typename T>
+bool CompareTensorWithValue(const paddle::Tensor& target, T value) {
+  // TODO(jiabin): Support Selected Rows later
+  auto dense_t = std::dynamic_pointer_cast<phi::DenseTensor>(target.impl());
+  T* ptr = dense_t->data<T>();
+
+  std::vector<T> host_data(dense_t->numel());
+  if (phi::is_gpu_place(dense_t->place())) {
+#ifdef PADDLE_WITH_CUDA
+    phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+    auto* dev_ctx = dynamic_cast<phi::GPUContext*>(pool.Get(phi::GPUPlace()));
+    auto stream = dev_ctx->stream();
+
+    paddle::memory::Copy(phi::CPUPlace(),
+                         host_data.data(),
+                         phi::GPUPlace(),
+                         ptr,
+                         sizeof(T) * dense_t->numel(),
+                         stream);
+    ptr = host_data.data();
+#endif
+  }
+
+  VLOG(6) << "CompareTensorWithValue";
+  for (int i = 0; i < dense_t->numel(); i++) {
+    PADDLE_ENFORCE(value == ptr[i],
+                   common::errors::PreconditionNotMet(
+                       "Numerical Error in Compare Grad Variable With Value of "
+                       "%d, we expected got value: %f, but got: %f instead. "
+                       "Please check it later.",
+                       i,
+                       value,
+                       ptr[i]));
+  }
+  return true;
+}
+
+inline void InitEnv(phi::Place place) {
+  // Prepare Device Contexts
+  // Init DeviceContextPool
+  paddle::framework::InitDevices();
+
+  // Init Tracer Place
+  egr::Controller::Instance().SetExpectedPlace(place);
+}
+
+}  // namespace eager_test