chore: import upstream snapshot with attribution

test/cpp/eager/data_structure_tests/CMakeLists.txt

@@ -0,0 +1,32 @@
+if(WITH_CINN)
+  set(eager_deps ${eager_deps} python)
+endif()
+if(NOT WIN32)
+  cc_test(
+    test_egr_ds_eager_tensor
+    SRCS eager_tensor_test.cc
+    DEPS final_dygraph_function ${eager_deps})
+endif()
+cc_test(
+  test_egr_ds_auotgrad_meta
+  SRCS autograd_meta_test.cc
+  DEPS final_dygraph_function ${eager_deps})
+
+if(NOT ((NOT WITH_PYTHON) AND ON_INFER))
+  cc_test(
+    test_egr_ds_grad_tensor_holder
+    SRCS grad_tensor_holder_test.cc
+    DEPS conditional_block_op ${eager_deps} ${generated_deps})
+  cc_test(
+    test_egr_ds_grad_node_info
+    SRCS grad_node_info_test.cc
+    DEPS conditional_block_op ${eager_deps} ${generated_deps})
+  cc_test(
+    test_egr_ds_accumulation_node
+    SRCS accumulation_node_test.cc
+    DEPS conditional_block_op ${eager_deps} ${generated_deps})
+  cc_test(
+    test_egr_ds_tensor_wrapper
+    SRCS tensor_wrapper_test.cc
+    DEPS conditional_block_op ${eager_deps} ${generated_deps})
+endif()

test/cpp/eager/data_structure_tests/accumulation_node_test.cc

@@ -0,0 +1,417 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/eager/accumulation/accumulation_node.h"
+
+#include <sstream>
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/eager/api/utils/hook_utils.h"
+#include "paddle/fluid/eager/eager_tensor.h"
+#include "paddle/fluid/eager/grad_node_info.h"
+#include "paddle/fluid/eager/grad_tensor_holder.h"
+#include "paddle/fluid/eager/hooks.h"
+#include "paddle/fluid/eager/utils.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/selected_rows.h"
+
+// TODO(jiabin): remove nolint here!!!
+using namespace egr;  // NOLINT
+TEST(AccumulationNode, SelectedRowsAddToTensor) {
+  // Construct Eager Tensor
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 1}));
+  std::vector<int64_t> rows = {0};
+  std::shared_ptr<phi::SelectedRows> sr0 =
+      std::make_shared<phi::SelectedRows>(rows, 1);
+  sr0->mutable_value()->Resize(common::make_ddim({1, 1}));
+  sr0->mutable_value()->mutable_data<float>(phi::CPUPlace())[0] =
+      static_cast<float>(10.0f);
+  paddle::Tensor et0 = paddle::Tensor(sr0);
+  std::shared_ptr<phi::DenseTensor> dt1 = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  dt1->mutable_data<float>(phi::CPUPlace())[0] = static_cast<float>(20.0f);
+  paddle::Tensor et1 = paddle::Tensor(dt1);
+  std::shared_ptr<phi::DenseTensor> input_dt =
+      std::make_shared<phi::DenseTensor>(
+          std::make_unique<paddle::experimental::DefaultAllocator>(
+              phi::CPUPlace())
+              .get(),
+          meta);
+  paddle::Tensor input_et = paddle::Tensor(input_dt);
+  auto grad_meta = EagerUtils::autograd_meta(&input_et);
+  // Initialize Grad Tensor
+  std::shared_ptr<phi::SelectedRows> grad_dt =
+      std::make_shared<phi::SelectedRows>(rows, 1);
+  grad_dt->mutable_value()->Resize(common::make_ddim({1, 1}));
+  grad_dt->mutable_value()->mutable_data<float>(phi::CPUPlace())[0] =
+      static_cast<float>(0.0f);
+  grad_meta->MutableGrad()->set_impl(grad_dt);
+  // AccumulationNode
+  auto node = std::make_shared<GradNodeAccumulation>(input_et);
+  grad_meta->SetGradNode(node);
+  grad_meta->SetStopGradient(false);
+  // operator()
+  paddle::small_vector<std::vector<paddle::Tensor>, kSlotSmallVectorSize>
+      et0_vec = {{et0}};
+  paddle::Tensor ret_et0 = node->operator()(et0_vec)[0][0];
+  auto* ret_et0_ptr =
+      std::dynamic_pointer_cast<phi::SelectedRows>(ret_et0.impl())
+          ->value()
+          .data<float>();
+  PADDLE_ENFORCE_EQ(ret_et0_ptr[0],
+                    static_cast<float>(10.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the selected rows "
+                        "should be 10.0f."));
+  paddle::small_vector<std::vector<paddle::Tensor>, kSlotSmallVectorSize>
+      et1_vec = {{et1}};
+  paddle::Tensor ret_et1 = node->operator()(et1_vec)[0][0];
+  auto* ret_et1_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(ret_et1.impl())
+          ->data<float>();
+  PADDLE_ENFORCE_EQ(ret_et1_ptr[0],
+                    static_cast<float>(20.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the dense tensor "
+                        "should be 20.0f"));
+  // Check Retain Grad
+  PADDLE_ENFORCE_EQ(
+      std::dynamic_pointer_cast<phi::SelectedRows>(et0.impl())
+          ->value()
+          .data<float>()[0],
+      static_cast<float>(10.0f),
+      common::errors::InvalidArgument(
+          "The value of the first element of the dense tensor should be "
+          "10.0f"));
+
+  paddle::Tensor* grad = EagerUtils::mutable_grad(input_et);
+  auto* grad_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(grad->impl())->data<float>();
+  PADDLE_ENFORCE_EQ(grad_ptr[0],
+                    static_cast<float>(30.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the dense tensor "
+                        "should be 30.0f"));
+}
+
+TEST(AccumulationNode, SelectedRowsMerge) {
+  // Construct Eager Tensor
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 1}));
+  std::vector<int64_t> rows = {0};
+  std::shared_ptr<phi::SelectedRows> sr0 =
+      std::make_shared<phi::SelectedRows>(rows, 1);
+  sr0->mutable_value()->Resize(common::make_ddim({1, 1}));
+  sr0->mutable_value()->mutable_data<float>(phi::CPUPlace())[0] =
+      static_cast<float>(10.0f);
+  paddle::Tensor et0 = paddle::Tensor(sr0);
+  std::shared_ptr<phi::SelectedRows> sr1 =
+      std::make_shared<phi::SelectedRows>(rows, 1);
+  sr1->mutable_value()->Resize(common::make_ddim({1, 1}));
+  sr1->mutable_value()->mutable_data<float>(phi::CPUPlace())[0] =
+      static_cast<float>(20.0f);
+  paddle::Tensor et1 = paddle::Tensor(sr1);
+  std::shared_ptr<phi::DenseTensor> input_dt =
+      std::make_shared<phi::DenseTensor>(
+          std::make_unique<paddle::experimental::DefaultAllocator>(
+              phi::CPUPlace())
+              .get(),
+          meta);
+  paddle::Tensor input_et = paddle::Tensor(input_dt);
+  auto grad_meta = EagerUtils::autograd_meta(&input_et);
+  // Initialize Grad Tensor
+  std::shared_ptr<phi::SelectedRows> grad_dt =
+      std::make_shared<phi::SelectedRows>(rows, 1);
+  grad_dt->mutable_value()->Resize(common::make_ddim({1, 1}));
+  grad_dt->mutable_value()->mutable_data<float>(phi::CPUPlace())[0] =
+      static_cast<float>(0.0f);
+  grad_meta->MutableGrad()->set_impl(grad_dt);
+  // AccumulationNode
+  auto node = std::make_shared<GradNodeAccumulation>(input_et);
+  grad_meta->SetGradNode(node);
+  grad_meta->SetStopGradient(false);
+  // operator()
+  paddle::small_vector<std::vector<paddle::Tensor>, kSlotSmallVectorSize>
+      et0_vec = {{et0}};
+  paddle::Tensor ret_et0 = node->operator()(et0_vec)[0][0];
+  auto* ret_et0_ptr =
+      std::dynamic_pointer_cast<phi::SelectedRows>(ret_et0.impl())
+          ->value()
+          .data<float>();
+  PADDLE_ENFORCE_EQ(ret_et0_ptr[0],
+                    static_cast<float>(10.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the selected rows "
+                        "should be 10.0f."));
+  paddle::small_vector<std::vector<paddle::Tensor>, kSlotSmallVectorSize>
+      et1_vec = {{et1}};
+  paddle::Tensor ret_et1 = node->operator()(et1_vec)[0][0];
+  auto* ret_et1_ptr =
+      std::dynamic_pointer_cast<phi::SelectedRows>(ret_et1.impl())
+          ->value()
+          .data<float>();
+  PADDLE_ENFORCE_EQ(ret_et1_ptr[0],
+                    static_cast<float>(20.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the selected rows "
+                        "should be 20.0f."));
+  // Check Retain Grad
+  PADDLE_ENFORCE_EQ(std::dynamic_pointer_cast<phi::SelectedRows>(et0.impl())
+                        ->value()
+                        .data<float>()[0],
+                    static_cast<float>(10.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the selected rows "
+                        "should be 10.0f."));
+  paddle::Tensor* grad = EagerUtils::mutable_grad(input_et);
+  auto* grad_ptr = std::dynamic_pointer_cast<phi::SelectedRows>(grad->impl())
+                       ->value()
+                       .data<float>();
+  PADDLE_ENFORCE_EQ(grad_ptr[0],
+                    static_cast<float>(30.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the selected rows "
+                        "should be 30.0f."));
+}
+
+TEST(AccumulationNode, SelectedRowsAddTensor) {
+  // Construct Eager Tensor
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 1}));
+  std::vector<int64_t> rows = {0};
+  std::shared_ptr<phi::SelectedRows> sr0 =
+      std::make_shared<phi::SelectedRows>(rows, 1);
+  sr0->mutable_value()->Resize(common::make_ddim({1, 1}));
+  sr0->mutable_value()->mutable_data<float>(phi::CPUPlace())[0] =
+      static_cast<float>(10.0f);
+  paddle::Tensor et0 = paddle::Tensor(sr0);
+  std::shared_ptr<phi::SelectedRows> sr1 =
+      std::make_shared<phi::SelectedRows>(rows, 1);
+  sr1->mutable_value()->Resize(common::make_ddim({1, 1}));
+  sr1->mutable_value()->mutable_data<float>(phi::CPUPlace())[0] =
+      static_cast<float>(20.0f);
+  paddle::Tensor et1 = paddle::Tensor(sr1);
+  std::shared_ptr<phi::DenseTensor> input_dt =
+      std::make_shared<phi::DenseTensor>(
+          std::make_unique<paddle::experimental::DefaultAllocator>(
+              phi::CPUPlace())
+              .get(),
+          meta);
+  paddle::Tensor input_et = paddle::Tensor(input_dt);
+  auto grad_meta = EagerUtils::autograd_meta(&input_et);
+  // Initialize Grad Tensor
+  std::shared_ptr<phi::DenseTensor> grad_dt =
+      std::make_shared<phi::DenseTensor>(
+          std::make_unique<paddle::experimental::DefaultAllocator>(
+              phi::CPUPlace())
+              .get(),
+          meta);
+  grad_dt->mutable_data<float>(phi::CPUPlace())[0] = static_cast<float>(0.0f);
+  grad_meta->MutableGrad()->set_impl(grad_dt);
+  // AccumulationNode
+  auto node = std::make_shared<GradNodeAccumulation>(input_et);
+  grad_meta->SetGradNode(node);
+  grad_meta->SetStopGradient(false);
+  // operator()
+  paddle::small_vector<std::vector<paddle::Tensor>, kSlotSmallVectorSize>
+      et0_vec = {{et0}};
+  paddle::Tensor ret_et0 = node->operator()(et0_vec)[0][0];
+  auto* ret_et0_ptr =
+      std::dynamic_pointer_cast<phi::SelectedRows>(ret_et0.impl())
+          ->value()
+          .data<float>();
+  PADDLE_ENFORCE_EQ(ret_et0_ptr[0],
+                    static_cast<float>(10.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the selected rows "
+                        "should be 10.0f."));
+  paddle::small_vector<std::vector<paddle::Tensor>, kSlotSmallVectorSize>
+      et1_vec = {{et1}};
+  paddle::Tensor ret_et1 = node->operator()(et1_vec)[0][0];
+  auto* ret_et1_ptr =
+      std::dynamic_pointer_cast<phi::SelectedRows>(ret_et1.impl())
+          ->value()
+          .data<float>();
+  PADDLE_ENFORCE_EQ(ret_et1_ptr[0],
+                    static_cast<float>(20.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the selected rows "
+                        "should be 20.0f"));
+  // Check Retain Grad
+  PADDLE_ENFORCE_EQ(std::dynamic_pointer_cast<phi::SelectedRows>(et0.impl())
+                        ->value()
+                        .data<float>()[0],
+                    static_cast<float>(10.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the selected rows "
+                        "should be 10.0f"));
+  paddle::Tensor* grad = EagerUtils::mutable_grad(input_et);
+  auto* grad_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(grad->impl())->data<float>();
+  PADDLE_ENFORCE_EQ(grad_ptr[0],
+                    static_cast<float>(30.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the dense tensor "
+                        "should be 30.0f"));
+}
+
+TEST(AccumulationNode, Tensor) {
+  // Construct Eager Tensor
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT16, common::make_ddim({1, 1}));
+  std::shared_ptr<phi::DenseTensor> dt0 = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  dt0->mutable_data<phi::dtype::float16>(phi::CPUPlace())[0] =
+      phi::dtype::float16(10.0f);
+  paddle::Tensor et0 = paddle::Tensor(dt0);
+
+  std::shared_ptr<phi::DenseTensor> dt1 = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+
+  dt1->mutable_data<phi::dtype::float16>(phi::CPUPlace())[0] =
+      phi::dtype::float16(20.0f);
+  paddle::Tensor et1 = paddle::Tensor(dt1);
+
+  std::shared_ptr<phi::DenseTensor> input_dt =
+      std::make_shared<phi::DenseTensor>(
+          std::make_unique<paddle::experimental::DefaultAllocator>(
+              phi::CPUPlace())
+              .get(),
+          meta);
+  paddle::Tensor input_et = paddle::Tensor(input_dt);
+  auto grad_meta = EagerUtils::autograd_meta(&input_et);
+
+  // Initialize Grad Tensor
+  std::shared_ptr<phi::DenseTensor> grad_dt =
+      std::make_shared<phi::DenseTensor>(
+          std::make_unique<paddle::experimental::DefaultAllocator>(
+              phi::CPUPlace())
+              .get(),
+          meta);
+  grad_dt->mutable_data<phi::dtype::float16>(phi::CPUPlace())[0] =
+      phi::dtype::float16(0.0f);
+  grad_meta->MutableGrad()->set_impl(grad_dt);
+
+  // AccumulationNode
+  auto node = std::make_shared<GradNodeAccumulation>(input_et);
+  grad_meta->SetGradNode(node);
+  grad_meta->SetStopGradient(false);
+
+  // operator()
+  paddle::small_vector<std::vector<paddle::Tensor>, kSlotSmallVectorSize>
+      et0_vec = {{et0}};
+  paddle::Tensor ret_et0 = node->operator()(et0_vec)[0][0];
+  auto* ret_et0_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(ret_et0.impl())
+          ->data<phi::dtype::float16>();
+  PADDLE_ENFORCE_EQ(ret_et0_ptr[0],
+                    phi::dtype::float16(10.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the dense tensor "
+                        "should be 10.0f."));
+  paddle::small_vector<std::vector<paddle::Tensor>, kSlotSmallVectorSize>
+      et1_vec = {{et1}};
+  paddle::Tensor ret_et1 = node->operator()(et1_vec)[0][0];
+
+  auto* ret_et1_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(ret_et1.impl())
+          ->data<phi::dtype::float16>();
+  PADDLE_ENFORCE_EQ(ret_et1_ptr[0],
+                    phi::dtype::float16(20.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the dense tensor "
+                        "should be 20.0f"));
+  // Check Retain Grad
+  PADDLE_ENFORCE_EQ(std::dynamic_pointer_cast<phi::DenseTensor>(et0.impl())
+                        ->data<phi::dtype::float16>()[0],
+                    phi::dtype::float16(10.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the dense tensor "
+                        "should be 10.0f"));
+  paddle::Tensor* grad = EagerUtils::mutable_grad(input_et);
+  auto* grad_ptr = std::dynamic_pointer_cast<phi::DenseTensor>(grad->impl())
+                       ->data<phi::dtype::float16>();
+  PADDLE_ENFORCE_EQ(grad_ptr[0],
+                    phi::dtype::float16(30.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the dense tensor "
+                        "should be 30.0f"));
+
+  // Reduce Hook case 1: Call RegisterReduceHook and run operator()
+  VLOG(6) << "Test Reduce Hook";
+  PADDLE_ENFORCE_EQ(std::dynamic_pointer_cast<phi::DenseTensor>(et0.impl())
+                        ->data<phi::dtype::float16>()[0],
+                    phi::dtype::float16(10.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the dense tensor "
+                        "should be 10.0f"));
+  auto reduce_hook_1 = [&]() -> void {
+    auto* input_et_ptr =
+        std::dynamic_pointer_cast<phi::DenseTensor>(input_et.impl())
+            ->mutable_data<phi::dtype::float16>(phi::CPUPlace());
+    input_et_ptr[0] = 36.0;
+    VLOG(6) << "Running Reduce Hook";
+  };
+
+  node->RegisterReduceHook(std::make_shared<egr::CppVoidHook>(reduce_hook_1));
+
+  // operator()
+  paddle::Tensor _ret = node->operator()(et0_vec)[0][0];
+
+  // Check operator() result, should be 36.0
+  auto* _ret_ptr = std::dynamic_pointer_cast<phi::DenseTensor>(_ret.impl())
+                       ->data<phi::dtype::float16>();
+  PADDLE_ENFORCE_EQ(_ret_ptr[0],
+                    phi::dtype::float16(10.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the dense tensor "
+                        "should be 10.0f"));
+
+  // Check Retain Grad, should be 36.0
+  auto* _ret_input_et_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(input_et.impl())
+          ->data<phi::dtype::float16>();
+  PADDLE_ENFORCE_EQ(_ret_input_et_ptr[0],
+                    phi::dtype::float16(36.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the dense tensor "
+                        "should be 36.0f"));
+  // Reduce Hook case 2: Call RegisterReduceHook and ApplyReduceHooks directly
+  VLOG(6) << "Test Reduce Hook";
+  auto reduce_hook_2 = [&]() -> void {
+    auto* ret_et0_ptr =
+        std::dynamic_pointer_cast<phi::DenseTensor>(et0.impl())
+            ->mutable_data<phi::dtype::float16>(phi::CPUPlace());
+    ret_et0_ptr[0] = 100.0;  // set to 100.0
+    VLOG(6) << "Running Reduce Hook";
+  };
+  node->RegisterReduceHook(std::make_shared<egr::CppVoidHook>(reduce_hook_2));
+  node->ApplyReduceHooks();
+
+  // Check ApplyReduceHooks result
+  PADDLE_ENFORCE_EQ(std::dynamic_pointer_cast<phi::DenseTensor>(et0.impl())
+                        ->data<phi::dtype::float16>()[0],
+                    phi::dtype::float16(100.0f),
+                    common::errors::InvalidArgument(
+                        "The value of the first element of the dense tensor "
+                        "should be 100.0f"));
+}

test/cpp/eager/data_structure_tests/autograd_meta_test.cc

@@ -0,0 +1,151 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/eager/autograd_meta.h"
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/fluid/eager/eager_tensor.h"
+#include "paddle/fluid/eager/grad_node_info.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "test/cpp/eager/data_structure_tests/grad_node_test.h"
+
+TEST(AutogradMeta, Constructor) {
+  paddle::Tensor et1;
+  auto auto_grad = std::make_shared<egr::AutogradMeta>();
+  et1.set_autograd_meta(auto_grad);
+  auto* tmp_auto = static_cast<egr::AutogradMeta*>(et1.get_autograd_meta());
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->OutRankInfo().first,
+      size_t(0),
+      common::errors::InvalidArgument(
+          "The first element of OutRankInfo should be 0, but received %d.",
+          tmp_auto->OutRankInfo().first));
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->OutRankInfo().second,
+      size_t(0),
+      common::errors::InvalidArgument(
+          "The second element of OutRankInfo should be 0, but received %d.",
+          tmp_auto->OutRankInfo().second));
+  CHECK(tmp_auto->IsInitialized() == false);
+}
+
+TEST(AutogradMeta, MemberFunction) {
+  paddle::Tensor et1;
+  auto auto_grad = std::make_shared<egr::AutogradMeta>();
+  et1.set_autograd_meta(auto_grad);
+  auto* tmp_auto = static_cast<egr::AutogradMeta*>(et1.get_autograd_meta());
+  VLOG(6) << "Test Grad";
+  PADDLE_ENFORCE_EQ(tmp_auto->Grad().defined(),
+                    false,
+                    common::errors::Fatal("grad should not be defined now"));
+  auto* grad_t = tmp_auto->MutableGrad();
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 2}));
+  std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  auto* dt_ptr = dt->mutable_data<float>(phi::CPUPlace());
+  dt_ptr[0] = 5.0f;
+  dt_ptr[1] = 10.0f;
+  grad_t->set_impl(dt);
+  VLOG(6) << "Test Mutable Grad";
+  auto impl_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(tmp_auto->Grad().impl());
+  PADDLE_ENFORCE_EQ(
+      impl_ptr->data<float>()[0],
+      5.0f,
+      common::errors::InvalidArgument(
+          "The first element of grad tensor should be 5.0, but received %f.",
+          impl_ptr->data<float>()[0]));
+  PADDLE_ENFORCE_EQ(
+      impl_ptr->data<float>()[1],
+      10.0f,
+      common::errors::InvalidArgument(
+          "The second element of grad tensor should be 10.0, but received %f.",
+          impl_ptr->data<float>()[1]));
+  VLOG(6) << "Test IsInitialized";
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->IsInitialized(),
+      false,
+      common::errors::Fatal(
+          "egr::AutogradMeta variable tmp_auto should not be initialized now"));
+  VLOG(6) << "Test GradNodeSetter Getter";
+  auto grad_node = std::make_shared<eager_test::GradTestNode>();
+  tmp_auto->SetGradNode(grad_node);
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->IsInitialized(),
+      true,
+      common::errors::Fatal(
+          "egr::AutogradMeta variable tmp_auto should be initialized now"));
+  auto tmp_grad_node = tmp_auto->GetMutableGradNode();
+  std::dynamic_pointer_cast<eager_test::GradTestNode>(tmp_grad_node)->val_ =
+      5.0;
+  PADDLE_ENFORCE_EQ(
+      dynamic_cast<eager_test::GradTestNode*>(tmp_auto->GradNode())->val_,
+      5.0,
+      common::errors::InvalidArgument(
+          "The value of GradTestNode should be 5.0, but received %f.",
+          dynamic_cast<eager_test::GradTestNode*>(tmp_auto->GradNode())->val_));
+  VLOG(6) << "Test rank Setter Getter";
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->OutRankInfo().first,
+      size_t(0),
+      common::errors::InvalidArgument(
+          "The first element of OutRankInfo should be 0, but received %d.",
+          tmp_auto->OutRankInfo().first));
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->OutRankInfo().second,
+      size_t(0),
+      common::errors::InvalidArgument(
+          "The second element of OutRankInfo should be 0, but received %d.",
+          tmp_auto->OutRankInfo().second));
+  tmp_auto->SetSingleOutRankWithSlot(2, 3);
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->OutRankInfo().first,
+      size_t(2),
+      common::errors::InvalidArgument(
+          "The first element of OutRankInfo should be 2, but received %d.",
+          tmp_auto->OutRankInfo().first));
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->OutRankInfo().second,
+      size_t(3),
+      common::errors::InvalidArgument(
+          "The second element of OutRankInfo should be 3, but received %d.",
+          tmp_auto->OutRankInfo().second));
+  VLOG(6) << "Test stop gradient Setter Getter";
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->NumericStopGradient(),
+      -1,
+      common::errors::InvalidArgument(
+          "The NumericStopGradient value should be -1, but received %d.",
+          tmp_auto->NumericStopGradient()));
+  tmp_auto->SetStopGradient(true);
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->StopGradient(),
+      true,
+      common::errors::Fatal("tmp_auto->StopGradient() should be true now"));
+  VLOG(6) << "Test Persistable Setter Getter";
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->Persistable(),
+      false,
+      common::errors::Fatal("tmp_auto->Persistable() should be false now"));
+  tmp_auto->SetPersistable(true);
+  PADDLE_ENFORCE_EQ(
+      tmp_auto->Persistable(),
+      true,
+      common::errors::Fatal(
+          "tmp_auto->Persistable() should be true now after SetPersistable()"));
+}

test/cpp/eager/data_structure_tests/eager_tensor_test.cc

@@ -0,0 +1,439 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/eager/eager_tensor.h"
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/common/layout.h"
+#include "paddle/fluid/imperative/var_helper.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+namespace eager_test {
+using AbstractAutogradMeta = paddle::AbstractAutogradMeta;
+class AutogradMetaTest : public AbstractAutogradMeta {
+ public:
+  explicit AutogradMetaTest(int val) : val_(val) {}
+  int val_ = 0;
+};
+}  // namespace eager_test
+
+TEST(Tensor, Constructor) {
+  paddle::Tensor et1 = paddle::Tensor();
+  paddle::Tensor et2 = paddle::Tensor("et2");
+
+  PADDLE_ENFORCE_EQ(et1.defined(),
+                    false,
+                    common::errors::InvalidArgument("Tensor et1 should be "
+                                                    "undefined."));
+  PADDLE_ENFORCE_EQ(et2.name(),
+                    "et2",
+                    common::errors::InvalidArgument("Tensor name should be "
+                                                    "'et2'."));
+
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 2}));
+  std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  auto* dt_ptr = dt->mutable_data<float>(phi::CPUPlace());
+  dt_ptr[0] = 5.0f;
+  dt_ptr[1] = 10.0f;
+  paddle::Tensor et3 = paddle::Tensor(dt);
+  auto* et3_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(et3.impl())->data<float>();
+  PADDLE_ENFORCE_EQ(et3_ptr[0],
+                    5.0f,
+                    common::errors::InvalidArgument("First element should be "
+                                                    "5.0f."));
+  PADDLE_ENFORCE_EQ(et3_ptr[1],
+                    10.0f,
+                    common::errors::InvalidArgument("Second element should be "
+                                                    "10.0f."));
+  // copy constructor
+  paddle::Tensor et4(et3);
+  auto* et4_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(et4.impl())->data<float>();
+  PADDLE_ENFORCE_EQ(et4_ptr[0],
+                    5.0f,
+                    common::errors::InvalidArgument("First element should be "
+                                                    "5.0f."));
+  PADDLE_ENFORCE_EQ(et4_ptr[1],
+                    10.0f,
+                    common::errors::InvalidArgument("Second element should be "
+                                                    "10.0f."));
+  paddle::Tensor et5(std::move(et4));
+  auto* et5_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(et5.impl())->data<float>();
+  PADDLE_ENFORCE_EQ(et5_ptr[0],
+                    5.0f,
+                    common::errors::InvalidArgument("First element should be "
+                                                    "5.0f."));
+  PADDLE_ENFORCE_EQ(et5_ptr[1],
+                    10.0f,
+                    common::errors::InvalidArgument("Second element should be "
+                                                    "10.0f."));
+}
+
+TEST(Tensor, MemberFunction) {
+  paddle::Tensor et3;
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 2}));
+  std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  auto* dt_ptr = dt->mutable_data<float>(phi::CPUPlace());
+  dt_ptr[0] = 5.0f;
+  dt_ptr[1] = 10.0f;
+  VLOG(6) << "Make Dense Tensor";
+  et3.set_name("et3");
+  VLOG(6) << "Set Name";
+  PADDLE_ENFORCE_EQ(et3.name(),
+                    "et3",
+                    common::errors::InvalidArgument("Tensor name should be "
+                                                    "'et3'."));
+  PADDLE_ENFORCE_EQ(et3.defined(),
+                    false,
+                    common::errors::InvalidArgument("Tensor et3 should be "
+                                                    "undefined."));
+  et3.set_impl(dt);
+  VLOG(6) << "Set impl";
+  PADDLE_ENFORCE_EQ(et3.initialized(),
+                    true,
+                    common::errors::InvalidArgument("Tensor et3 should be "
+                                                    "initialized."));
+  PADDLE_ENFORCE_EQ(et3.is_cpu(),
+                    true,
+                    common::errors::InvalidArgument("Tensor et3 should be "
+                                                    "on CPU."));
+  PADDLE_ENFORCE_EQ(et3.is_gpu(),
+                    false,
+                    common::errors::InvalidArgument("Tensor et3 should not be "
+                                                    "on GPU."));
+  PADDLE_ENFORCE_EQ(et3.numel(),
+                    2,
+                    common::errors::InvalidArgument("Tensor et3 should have "
+                                                    "2 elements."));
+  auto expected_dim = common::make_ddim({1, 2});
+  PADDLE_ENFORCE_EQ(
+      et3.dims(),
+      expected_dim,
+      common::errors::InvalidArgument("Tensor dimensions should be "
+                                      "{1, 2}."));
+  PADDLE_ENFORCE_EQ(et3.type(),
+                    phi::DataType::FLOAT32,
+                    common::errors::InvalidArgument("Tensor data type should "
+                                                    "be FLOAT32."));
+  PADDLE_ENFORCE_EQ(et3.layout(),
+                    phi::DataLayout::NCHW,
+                    common::errors::InvalidArgument("Tensor layout should be "
+                                                    "NCHW."));
+  CHECK(phi::is_cpu_place(et3.place()));
+  VLOG(6) << "Get impl";
+  auto* dt3_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(et3.impl())->data<float>();
+  PADDLE_ENFORCE_EQ(dt3_ptr[0],
+                    5.0f,
+                    common::errors::InvalidArgument("First element should be "
+                                                    "5.0f."));
+  PADDLE_ENFORCE_EQ(dt3_ptr[1],
+                    10.0f,
+                    common::errors::InvalidArgument("Second element should be "
+                                                    "10.0f."));
+  paddle::Tensor et4 = et3;
+  VLOG(6) << "copy =";
+  PADDLE_ENFORCE_EQ(et4.initialized(),
+                    true,
+                    common::errors::InvalidArgument("Tensor et4 should be "
+                                                    "initialized."));
+  auto* dt4_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(et4.impl())->data<float>();
+  PADDLE_ENFORCE_EQ(dt4_ptr[0],
+                    5.0f,
+                    common::errors::InvalidArgument("First element should be "
+                                                    "5.0f."));
+  PADDLE_ENFORCE_EQ(dt4_ptr[1],
+                    10.0f,
+                    common::errors::InvalidArgument("Second element should be "
+                                                    "10.0f."));
+  VLOG(6) << "move =";
+  paddle::Tensor et5 = std::move(et4);
+  auto* dt5_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(et5.impl())->data<float>();
+  PADDLE_ENFORCE_EQ(dt5_ptr[0],
+                    5.0f,
+                    common::errors::InvalidArgument("First element should be "
+                                                    "5.0f."));
+  PADDLE_ENFORCE_EQ(dt5_ptr[1],
+                    10.0f,
+                    common::errors::InvalidArgument("Second element should be "
+                                                    "10.0f."));
+  VLOG(6) << "AutogradMeta";
+  auto autograd_meta_test = std::make_shared<eager_test::AutogradMetaTest>(2);
+  et3.set_autograd_meta(autograd_meta_test);
+  auto* tmp_autograd_meta_test =
+      static_cast<eager_test::AutogradMetaTest*>(et3.get_autograd_meta());
+  PADDLE_ENFORCE_EQ(tmp_autograd_meta_test->val_,
+                    2,
+                    common::errors::InvalidArgument("AutogradMetaTest value "
+                                                    "should be 2."));
+}
+
+TEST(EagerVariable, Constructor) {
+  paddle::Tensor t3;
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 2}));
+  std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  auto* dt_ptr = dt->mutable_data<float>(phi::CPUPlace());
+  dt_ptr[0] = 5.0f;
+  dt_ptr[1] = 10.0f;
+  VLOG(6) << "Make Dense Tensor";
+  t3.set_name("t3");
+  VLOG(6) << "Set Name";
+  PADDLE_ENFORCE_EQ(t3.name(),
+                    "t3",
+                    common::errors::InvalidArgument("Tensor name should be "
+                                                    "'t3'."));
+  PADDLE_ENFORCE_EQ(
+      t3.defined(),
+      false,
+      common::errors::InvalidArgument(
+          "Tensor t3 should be undefined but got %d.", t3.defined()));
+  t3.set_impl(dt);
+
+  egr::EagerVariable et3 = egr::EagerVariable(t3);
+  VLOG(6) << "SyncToVar";
+  PADDLE_ENFORCE_EQ(et3.Var().Get<phi::DenseTensor>().data<float>()[0],
+                    5.0f,
+                    common::errors::InvalidArgument("First element should be "
+                                                    "5.0f."));
+  PADDLE_ENFORCE_EQ(et3.Var().Get<phi::DenseTensor>().data<float>()[1],
+                    10.0f,
+                    common::errors::InvalidArgument("Second element should be "
+                                                    "10.0f."));
+  VLOG(6) << "SyncToTensor";
+  paddle::Tensor t4;
+  t4.set_impl(et3.GetTensorBase());
+  PADDLE_ENFORCE_EQ(
+      t4.initialized(),
+      true,
+      common::errors::InvalidArgument(
+          "Tensor t4 should be initialized but got %d.", t4.initialized()));
+
+  VLOG(6) << "Check Tensor";
+  auto* dt3_tmp_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(t4.impl())->data<float>();
+  PADDLE_ENFORCE_EQ(dt3_tmp_ptr[0],
+                    5.0f,
+                    common::errors::InvalidArgument("First element should be "
+                                                    "5.0f."));
+  PADDLE_ENFORCE_EQ(dt3_tmp_ptr[1],
+                    10.0f,
+                    common::errors::InvalidArgument("Second element should be "
+                                                    "10.0f."));
+  t4.reset();
+  PADDLE_ENFORCE_EQ(
+      t4.defined(),
+      false,
+      common::errors::InvalidArgument(
+          "Tensor t4 should be undefined but got %d.", t4.defined()));
+
+  VLOG(6) << "Check Tensor Copy_";
+  std::vector<int64_t> rows = {1, 2};
+  std::vector<int64_t> dims = {2};
+  paddle::Tensor t7(std::make_shared<phi::SelectedRows>(rows, 2));
+  std::dynamic_pointer_cast<phi::SelectedRows>(t7.impl())
+      ->mutable_value()
+      ->Resize(common::make_ddim(dims));
+  auto* dt7_tmp_ptr = std::dynamic_pointer_cast<phi::SelectedRows>(t7.impl())
+                          ->mutable_value()
+                          ->mutable_data<float>(phi::CPUPlace());
+  dt7_tmp_ptr[0] = 6.0f;
+  dt7_tmp_ptr[1] = 11.0f;
+
+  paddle::Tensor t8;
+  paddle::Tensor t5;
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  paddle::Tensor t6;
+  paddle::Tensor t9;
+  VLOG(6) << "Check Tensor Copy_ Selected Rows";
+  t8.copy_(t7, phi::GPUPlace(0), false);
+  t9.copy_(t8, phi::CPUPlace(), false);
+  auto* dt9_tmp_ptr = std::dynamic_pointer_cast<phi::SelectedRows>(t9.impl())
+                          ->value()
+                          .data<float>();
+  PADDLE_ENFORCE_EQ(dt9_tmp_ptr[0],
+                    6.0f,
+                    common::errors::InvalidArgument("First element should be "
+                                                    "6.0f."));
+  PADDLE_ENFORCE_EQ(dt9_tmp_ptr[1],
+                    11.0f,
+                    common::errors::InvalidArgument("Second element should be "
+                                                    "11.0f."));
+  PADDLE_ENFORCE_EQ(
+      std::dynamic_pointer_cast<phi::SelectedRows>(t9.impl())->height(),
+      2,
+      common::errors::InvalidArgument("SelectedRows height should "
+                                      "be 2."));
+
+  VLOG(6) << "Check Tensor Copy_ Dense Tensor";
+  t5.copy_(t3, phi::GPUPlace(0), false);
+  t6.copy_(t5, phi::CPUPlace(), false);
+  auto* dt6_tmp_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(t6.impl())->data<float>();
+  PADDLE_ENFORCE_EQ(dt6_tmp_ptr[0],
+                    5.0f,
+                    common::errors::InvalidArgument("First element should be "
+                                                    "5.0f."));
+  PADDLE_ENFORCE_EQ(dt6_tmp_ptr[1],
+                    10.0f,
+                    common::errors::InvalidArgument("Second element should be "
+                                                    "10.0f."));
+#else
+  t5.copy_(t3, phi::CPUPlace(), false);
+  auto* dt5_tmp_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(t5.impl())->data<float>();
+  PADDLE_ENFORCE_EQ(dt5_tmp_ptr[0],
+                    5.0f,
+                    common::errors::InvalidArgument("First element should be "
+                                                    "5.0f."));
+  PADDLE_ENFORCE_EQ(dt5_tmp_ptr[1],
+                    10.0f,
+                    common::errors::InvalidArgument("Second element should be "
+                                                    "10.0f."));
+#endif
+
+  VLOG(6) << "Finish";
+}
+
+TEST(EagerVariable, DataLayout) {
+  paddle::Tensor tensor;
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32,
+                           common::make_ddim({1, 1, 1, 1}),
+                           phi::DataLayout::UNDEFINED);
+  std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  auto* dt_ptr = dt->mutable_data<float>(phi::CPUPlace());
+  dt_ptr[0] = 5.0f;
+  dt_ptr[1] = 5.0f;
+  dt_ptr[2] = 5.0f;
+  dt_ptr[3] = 5.0f;
+  tensor.set_impl(dt);
+  auto eager_var = std::make_shared<egr::EagerVariable>(tensor);
+  auto layout = paddle::imperative::GetDataLayout(eager_var);
+  PADDLE_ENFORCE_EQ(layout,
+                    phi::DataLayout::UNDEFINED,
+                    common::errors::InvalidArgument("Data layout should be "
+                                                    "UNDEFINED."));
+  paddle::imperative::SetDataLayout(eager_var, phi::DataLayout::NCHW);
+  layout = paddle::imperative::GetDataLayout(eager_var);
+  PADDLE_ENFORCE_EQ(layout,
+                    phi::DataLayout::NCHW,
+                    common::errors::InvalidArgument("Data layout should be "
+                                                    "NCHW."));
+}
+
+TEST(VariableCompatTensor, MemberFunction) {
+  egr::VariableCompatTensor var_tensor;
+  // test GetMutable and Get
+  var_tensor.GetMutable<phi::Vocab>();
+  auto& vocab = var_tensor.Get<phi::Vocab>();
+  EXPECT_EQ(vocab.size(), 0UL);
+  bool caught_exception = false;
+  try {
+    var_tensor.GetMutable<phi::Strings>();
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("The Variable type must be") != std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+  // test Type and IsType
+  EXPECT_TRUE(var_tensor.IsType<phi::Vocab>());
+  EXPECT_EQ(var_tensor.Type(),
+            static_cast<int>(paddle::framework::proto::VarType::VOCAB));
+  // test valid and initialized
+  EXPECT_TRUE(var_tensor.IsInitialized());
+  EXPECT_TRUE(var_tensor.valid());
+  EXPECT_TRUE(var_tensor.initialized());
+  // test name
+  EXPECT_EQ(var_tensor.name(), "VariableCompatTensor");
+  // test other throw error methods
+  caught_exception = false;
+  try {
+    var_tensor.numel();
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("numel") != std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+  caught_exception = false;
+  try {
+    var_tensor.dims();
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("dims") != std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+  caught_exception = false;
+  try {
+    var_tensor.dtype();
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("dtype") != std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+  caught_exception = false;
+  try {
+    var_tensor.layout();
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("layout") != std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+  caught_exception = false;
+  try {
+    var_tensor.place();
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("place") != std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+  caught_exception = false;
+  try {
+    var_tensor.AllocateFrom(nullptr, phi::DataType::UNDEFINED);
+  } catch (paddle::platform::EnforceNotMet& error) {
+    caught_exception = true;
+    std::string ex_msg = error.what();
+    EXPECT_TRUE(ex_msg.find("AllocateFrom") != std::string::npos);
+  }
+  EXPECT_TRUE(caught_exception);
+  // test Clear
+  var_tensor.Clear();
+  EXPECT_FALSE(var_tensor.IsInitialized());
+}

test/cpp/eager/data_structure_tests/grad_node_info_test.cc

@@ -0,0 +1,257 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/eager/grad_node_info.h"
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/fluid/eager/autograd_meta.h"
+#include "paddle/fluid/eager/eager_tensor.h"
+#include "paddle/fluid/eager/hooks.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/core/enforce.h"
+#include "test/cpp/eager/data_structure_tests/grad_node_test.h"
+
+TEST(GradNodeInfo, GradSlotMeta) {
+  auto grad_slot = egr::GradSlotMeta();
+  VLOG(6) << "Set SetStopGradient";
+  grad_slot.SetStopGradient();
+  PADDLE_ENFORCE_EQ(
+      grad_slot.IsStopGradient(),
+      true,
+      common::errors::Fatal("`grad_slot.IsStopGradient()` should be "
+                            "true, please check related function"));
+}
+
+void TestGradNodeBase(bool is_remove_gradient_hook) {
+  VLOG(6) << "Construct Grad Node";
+  auto grad_test_node0 = std::make_shared<eager_test::GradTestNode>(
+      /* val */ 5.0, /* in_num */ 2, /* out_num */ 2);
+  auto grad_test_node1 = std::make_shared<eager_test::GradTestNode>();
+  paddle::small_vector<std::vector<paddle::Tensor>, egr::kSlotSmallVectorSize>
+      grads;
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 1}));
+  std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  auto* dt_ptr = dt->mutable_data<float>(phi::CPUPlace());
+  dt_ptr[0] = 5.0f;
+  paddle::Tensor et1(dt);
+  grads = {{et1}};
+
+  std::vector<int64_t> mesh_shape = {2, 2};
+  std::vector<int64_t> process_ids = {0, 1, 2, 3};
+  std::vector<std::string> dim_names = {"dp", "mp"};
+  phi::distributed::ProcessMesh process_mesh(
+      mesh_shape, process_ids, dim_names);
+  std::vector<int64_t> dim_mapping = {-1, 1};
+  phi::distributed::TensorDistAttr dist_attr =
+      phi::distributed::TensorDistAttr();
+  dist_attr.set_process_mesh(process_mesh);
+  dist_attr.set_dims_mapping(dim_mapping);
+  dist_attr.set_dynamic_dims(std::vector<bool>(mesh_shape.size(), false));
+
+  std::shared_ptr<phi::distributed::DistTensor> dist_dt1 =
+      std::make_shared<phi::distributed::DistTensor>(
+          phi::distributed::DistTensor(dt, dist_attr));
+  paddle::Tensor dist_et1(dist_dt1);
+
+  VLOG(6) << "Test Grad Node Call";
+  auto res = (*grad_test_node0)(grads);
+  PADDLE_ENFORCE_EQ(
+      std::dynamic_pointer_cast<phi::DenseTensor>(res[0][0].impl())
+          ->data<float>()[0],
+      6.0f,
+      common::errors::InvalidArgument("Data of grads mismatch. Expected 6.0."));
+  egr::Edge tmp_edge1(grad_test_node1, 3, 4);
+  auto auto_grad1 = std::make_shared<egr::AutogradMeta>(tmp_edge1);
+  et1.set_autograd_meta(auto_grad1);
+
+  VLOG(6) << "Test Set Meta and Get Meta";
+  auto_grad1->SetStopGradient(true);
+  grad_test_node0->SetGradInMeta(et1, 0);
+  grad_test_node0->SetGradInMeta({et1}, 1);
+  grad_test_node0->SetGradOutMeta(et1, 0);
+  grad_test_node0->SetGradOutMeta({et1}, 1);
+  grad_test_node0->SetGradOutMeta(dist_et1, 0, dist_attr, dist_et1.dims());
+  PADDLE_ENFORCE_EQ(
+      grad_test_node0->InputMeta()[0].size(),
+      1UL,
+      common::errors::InvalidArgument("Size of input mismatch. Expected 1."));
+  PADDLE_ENFORCE_EQ(
+      grad_test_node0->InputMeta()[1].size(),
+      1UL,
+      common::errors::InvalidArgument("Size of input mismatch. Expected 1."));
+  PADDLE_ENFORCE_EQ(
+      grad_test_node0->InputMeta()[0][0].GetTensorMeta().dtype,
+      meta.dtype,
+      common::errors::InvalidArgument("Dtype of input tensor mismatch."));
+  PADDLE_ENFORCE_EQ(
+      grad_test_node0->InputMeta()[1][0].GetTensorMeta().dtype,
+      meta.dtype,
+      common::errors::InvalidArgument("Dtype of input tensor mismatch."));
+  PADDLE_ENFORCE_EQ(grad_test_node0->OutputMeta()[0][0].IsStopGradient(),
+                    true,
+                    common::errors::Fatal(
+                        "`grad_test_node0->OutputMeta()[0][0].IsStopGradient()"
+                        "` should be true, please related function"));
+  PADDLE_ENFORCE_EQ(grad_test_node0->OutputMeta()[1][0].IsStopGradient(),
+                    true,
+                    common::errors::Fatal(
+                        "`grad_test_node0->OutputMeta()[1][0].IsStopGradient()"
+                        "` should be true, please related function"));
+  PADDLE_ENFORCE_EQ(
+      grad_test_node0->OutputMeta()[0][0].GetTensorMeta().dtype,
+      meta.dtype,
+      common::errors::InvalidArgument("Dtype of output tensor mismatch."));
+  PADDLE_ENFORCE_EQ(
+      grad_test_node0->OutputMeta()[1][0].GetTensorMeta().dtype,
+      meta.dtype,
+      common::errors::InvalidArgument("Dtype of output tensor mismatch."));
+  PADDLE_ENFORCE_EQ(
+      grad_test_node0->OutputMeta()[0][0].DistAttr(),
+      dist_attr,
+      common::errors::InvalidArgument("DistAttr of output tensor mismatch."));
+  PADDLE_ENFORCE_EQ(
+      grad_test_node0->OutputMeta()[0][0].DistTensorGlobalDims(),
+      dist_et1.dims(),
+      common::errors::InvalidArgument("DDims of output tensor mismatch."));
+
+  VLOG(6) << "Test Default Set Meta and Get Meta";
+  auto grad_test_node2 = std::make_shared<eager_test::GradTestNode>(
+      /* val */ 5.0, /* in_num */ 1, /* out_num */ 1);
+  grad_test_node2->SetDefaultGradInOutMeta();
+  PADDLE_ENFORCE_GT(
+      grad_test_node2->OutputMeta()[0].size(),
+      0UL,
+      common::errors::InvalidArgument("Size of output not greater than 0."));
+  CHECK(grad_test_node2->OutputMeta()[0][0].IsStopGradient() == false);
+  PADDLE_ENFORCE_EQ(
+      grad_test_node2->OutputMeta()[0].size(),
+      1UL,
+      common::errors::InvalidArgument("Size of output mismatch. Expected 1."));
+
+  VLOG(6) << "Test Gradient Hook";
+  auto gradient_hook = [](const paddle::Tensor& et) -> paddle::Tensor {
+    paddle::Tensor res;
+    phi::DenseTensorMeta meta =
+        phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 1}));
+    std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
+        std::make_unique<paddle::experimental::DefaultAllocator>(
+            phi::CPUPlace())
+            .get(),
+        meta);
+    auto* dt_ptr = dt->mutable_data<float>(phi::CPUPlace());
+    dt_ptr[0] = 6.0f;
+    auto* et_ptr =
+        std::dynamic_pointer_cast<phi::DenseTensor>(et.impl())->data<float>();
+    dt_ptr[0] += et_ptr[0];
+    res.set_impl(dt);
+    VLOG(6) << "Running Gradient Hook";
+    return res;
+  };
+  int64_t hook_id = grad_test_node0->RegisterGradientHook(
+      0, 0, std::make_shared<egr::CppTensorHook>(gradient_hook));
+
+  if (is_remove_gradient_hook) {
+    // Remove GradientHook
+    grad_test_node0->RemoveGradientHook(hook_id);
+  }
+
+  // Check results
+  auto grad_hook_res = grad_test_node0->ApplyGradientHooks(grads);
+  PADDLE_ENFORCE_EQ(
+      std::dynamic_pointer_cast<phi::DenseTensor>(grad_hook_res[0][0].impl())
+          ->data<float>()[0],
+      is_remove_gradient_hook ? 5.0 : 11.0,
+      common::errors::InvalidArgument(
+          "Data of grad hook res mismatch. Expected 5.0 or 11.0."));
+}
+
+TEST(GradNodeInfo, GradNodeBase) {
+  TestGradNodeBase(true);
+  TestGradNodeBase(false);
+}
+
+TEST(GradNodeInfo, Edge) {
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 1}));
+  std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  paddle::Tensor et1(dt);
+
+  auto grad_test_node0 = std::make_shared<eager_test::GradTestNode>(5, 2, 2);
+  auto auto_grad1 = std::make_shared<egr::AutogradMeta>();
+  VLOG(6) << "Test Construct Edge";
+  egr::Edge edge0 = egr::Edge();
+  PADDLE_ENFORCE_EQ(
+      edge0.IsInitialized(),
+      false,
+      common::errors::Fatal("`edge0.IsInitialized()` should be "
+                            "false, please check related function"));
+  egr::Edge edge1 = egr::Edge(grad_test_node0, size_t(0), size_t(0));
+  PADDLE_ENFORCE_EQ(
+      edge1.IsInitialized(),
+      true,
+      common::errors::Fatal("`edge1.IsInitialized()` should be "
+                            "true, please check related function"));
+  egr::Edge edge2 =
+      egr::Edge(grad_test_node0, std::make_pair(size_t(1), size_t(0)));
+  VLOG(6) << "Test Set Edge's Grad Node";
+  auto* grad_node = edge1.GetGradNode();
+  et1.set_autograd_meta(auto_grad1);
+  grad_node->SetGradInMeta(et1, 0);
+
+  PADDLE_ENFORCE_EQ(
+      grad_node->InputMeta().size(),
+      2UL,
+      common::errors::InvalidArgument("Size of input mismatch. Expected 2."));
+  std::vector<egr::AutogradMeta*> metas = {auto_grad1.get()};
+  PADDLE_ENFORCE_EQ(
+      grad_node->InputMeta()[0][0].IsStopGradient(),
+      true,
+      common::errors::Fatal("`grad_node->InputMeta()[0][0].IsStopGradient()` "
+                            "should be true, please check related function"));
+  VLOG(6) << "Test Get/Set Edge Rank Info";
+  PADDLE_ENFORCE_EQ(
+      edge2.GetEdgeRankInfo().first,
+      1UL,
+      common::errors::InvalidArgument("Edge rank info mismatch. Expected 1."));
+  PADDLE_ENFORCE_EQ(
+      edge2.GetEdgeRankInfo().second,
+      0UL,
+      common::errors::InvalidArgument("Edge rank info mismatch. Expected 0."));
+  edge2.SetEdgeRankInfo(2, 3);
+  PADDLE_ENFORCE_EQ(
+      edge2.GetEdgeRankInfo().first,
+      2UL,
+      common::errors::InvalidArgument("Edge rank info mismatch. Expected 2."));
+  PADDLE_ENFORCE_EQ(
+      edge2.GetEdgeRankInfo().second,
+      3UL,
+      common::errors::InvalidArgument("Edge rank info mismatch. Expected 3."));
+  edge2.SetEdgeRankInfo(std::make_pair(size_t(4), size_t(5)));
+  PADDLE_ENFORCE_EQ(
+      edge2.GetEdgeRankInfo().first,
+      4UL,
+      common::errors::InvalidArgument("Edge rank info mismatch. Expected 4."));
+  PADDLE_ENFORCE_EQ(
+      edge2.GetEdgeRankInfo().second,
+      5UL,
+      common::errors::InvalidArgument("Edge rank info mismatch. Expected 5."));
+}

test/cpp/eager/data_structure_tests/grad_node_test.h

@@ -0,0 +1,65 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/fluid/eager/autograd_meta.h"
+#include "paddle/fluid/eager/eager_tensor.h"
+#include "paddle/fluid/eager/grad_node_info.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+namespace egr {
+class TensorWrapper;
+}
+
+namespace eager_test {
+class GradTestNode : public egr::GradNodeBase {
+ public:
+  ~GradTestNode() override = default;
+  GradTestNode(float val, int in_num, int out_num)
+      : GradNodeBase(in_num, out_num), val_(val) {}
+  GradTestNode() : GradNodeBase() { val_ = 1.0; }
+  std::string name() override { return "GradTestNode"; }
+  paddle::small_vector<std::vector<paddle::Tensor>, egr::kSlotSmallVectorSize>
+  operator()(paddle::small_vector<std::vector<paddle::Tensor>,
+                                  egr::kSlotSmallVectorSize>& grads,  // NOLINT
+             bool create_graph = false,
+             bool is_new_grad = false) override {
+    val_ = std::dynamic_pointer_cast<phi::DenseTensor>(grads[0][0].impl())
+               ->data<float>()[0];
+    phi::DenseTensorMeta meta =
+        phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 1}));
+    std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
+        std::make_unique<paddle::experimental::DefaultAllocator>(
+            phi::CPUPlace())
+            .get(),
+        meta);
+    auto* dt_ptr = dt->mutable_data<float>(phi::CPUPlace());
+    dt_ptr[0] = 6.0f;
+    paddle::Tensor et1(dt);
+    paddle::small_vector<std::vector<paddle::Tensor>, egr::kSlotSmallVectorSize>
+        res = {{et1}};
+    return res;
+  }
+  void ClearTensorWrappers() override { VLOG(6) << "Do nothing here now"; }
+
+  std::shared_ptr<GradNodeBase> Copy() const override {
+    {
+      auto copied_node = std::shared_ptr<GradTestNode>(new GradTestNode(*this));
+      return copied_node;
+    }
+  }
+
+  float val_;
+};
+}  // namespace eager_test

test/cpp/eager/data_structure_tests/grad_tensor_holder_test.cc

@@ -0,0 +1,206 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/eager/grad_tensor_holder.h"
+
+#include <sstream>
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/eager/eager_tensor.h"
+#include "paddle/fluid/eager/grad_node_info.h"
+#include "paddle/fluid/eager/utils.h"
+#include "paddle/phi/api/lib/utils/allocator.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/selected_rows.h"
+
+PD_DECLARE_KERNEL(full_like, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(add, CPU, ALL_LAYOUT);
+
+// TODO(jiabin): remove nolint here!!!
+using namespace egr;  // NOLINT
+
+TEST(GradTensorHolder, Constructor) {
+  std::vector<GradSlotMeta> slot_meta(1);
+  GradTensorHolder grad_tensor_holder = GradTensorHolder({slot_meta});
+  GradTensorHolder grad_tensor_holder2 = GradTensorHolder(grad_tensor_holder);
+
+  // Construct Eager Tensor
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({2, 2}));
+  std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  paddle::Tensor et = paddle::Tensor(dt);
+
+  paddle::small_vector<std::vector<paddle::Tensor>, kSlotSmallVectorSize>
+      inputs;
+  inputs.push_back({et});
+  grad_tensor_holder2.SetBuffers(std::move(inputs));
+}
+
+TEST(GradTensorHolder, Interfaces) {
+  // Construct Eager Tensor
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 1}));
+  std::shared_ptr<phi::DenseTensor> dt0 = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  dt0->mutable_data<float>(phi::CPUPlace())[0] = 10.0;
+  paddle::Tensor et0 = paddle::Tensor(dt0);
+
+  std::shared_ptr<phi::DenseTensor> dt1 = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  dt1->mutable_data<float>(phi::CPUPlace())[0] = 20.0;
+  paddle::Tensor et1 = paddle::Tensor(dt1);
+
+  // Constructor empty GradTensorHolder
+  std::vector<GradSlotMeta> slot_meta(1);
+  GradTensorHolder grad_tensor_holder =
+      GradTensorHolder({slot_meta, slot_meta});
+  egr::EagerUtils::autograd_meta(&et0);
+  // add():
+  // fill one
+  grad_tensor_holder.CopyValueFromTensor(0, 0, et0, true);
+
+  // accumulation
+  grad_tensor_holder.add(1, 0, et0);
+  grad_tensor_holder.add(1, 0, et1);
+
+  // Buffers()
+  const auto& buffers = grad_tensor_holder.Buffers();
+  PADDLE_ENFORCE_EQ(static_cast<int>(buffers.size()),
+                    2,
+                    common::errors::InvalidArgument(
+                        "The size of buffers should be 2, but received %d.",
+                        static_cast<int>(buffers.size())));
+  PADDLE_ENFORCE_EQ(
+      static_cast<int>(buffers[0].size()),
+      1,
+      common::errors::InvalidArgument(
+          "The size of the first buffer should be 1, but received %d.",
+          static_cast<int>(buffers[0].size())));
+  PADDLE_ENFORCE_EQ(
+      static_cast<int>(buffers[1].size()),
+      1,
+      common::errors::InvalidArgument(
+          "The size of the second buffer should be 1, but received %d.",
+          static_cast<int>(buffers[1].size())));
+
+  // operator[]
+  const auto& holder_et0 = grad_tensor_holder[0][0];
+  const auto& holder_et1 = grad_tensor_holder[1][0];
+
+  auto* holder_et0_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(holder_et0.impl())
+          ->data<float>();
+  auto* holder_et1_ptr =
+      std::dynamic_pointer_cast<phi::DenseTensor>(holder_et1.impl())
+          ->data<float>();
+
+  PADDLE_ENFORCE_EQ(
+      holder_et0_ptr[0],
+      1.0f,
+      common::errors::InvalidArgument(
+          "The value of holder_et0_ptr[0] should be 1.0f, but received %f.",
+          holder_et0_ptr[0]));
+  PADDLE_ENFORCE_EQ(
+      holder_et1_ptr[0],
+      30.0f,
+      common::errors::InvalidArgument(
+          "The value of holder_et1_ptr[0] should be 30.0f, but received %f.",
+          holder_et1_ptr[0]));
+}
+
+TEST(GradTensorHolder, SelectedRowsMergeAdd) {
+  phi::CPUPlace cpu;
+
+  std::vector<int64_t> rows{0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+  int64_t table_size = 10;
+  int64_t embedding_width = 10;
+
+  auto sr1 = std::make_shared<phi::SelectedRows>(rows, table_size);
+  auto sr2 = std::make_shared<phi::SelectedRows>(rows, table_size);
+
+  // initialize a sparse table 1
+  sr1->mutable_value()->Resize(
+      common::make_ddim({table_size, embedding_width}));
+  auto* data_sr1 = sr1->mutable_value()->mutable_data<float>(cpu);
+  for (int64_t i = 0; i < table_size; ++i) {
+    for (int64_t j = 0; j < embedding_width; ++j) {
+      data_sr1[i * embedding_width + j] = static_cast<float>(i);
+    }
+  }
+
+  // initialize a sparse table 2
+  sr2->mutable_value()->Resize(
+      common::make_ddim({table_size, embedding_width}));
+  auto* data_sr2 = sr2->mutable_value()->mutable_data<float>(cpu);
+  for (int64_t i = 0; i < table_size; ++i) {
+    for (int64_t j = 0; j < embedding_width; ++j) {
+      data_sr2[i * embedding_width + j] = static_cast<float>(i);
+    }
+  }
+  // new 2 phi::Tensor
+  paddle::Tensor t1(sr1);
+  paddle::Tensor t2(sr2);
+
+  // Constructor empty GradTensorHolder
+  std::vector<GradSlotMeta> slot_meta(1);
+  GradTensorHolder grad_tensor_holder =
+      GradTensorHolder({slot_meta, slot_meta});
+
+  // accumulation
+  grad_tensor_holder.add(0, 0, t1);
+  grad_tensor_holder.add(0, 0, t2);
+
+  // Buffers()
+  const auto& buffers = grad_tensor_holder.Buffers();
+  PADDLE_ENFORCE_EQ(static_cast<int>(buffers.size()),
+                    2,
+                    common::errors::InvalidArgument(
+                        "The size of buffers should be 2, but received %d.",
+                        static_cast<int>(buffers.size())));
+  PADDLE_ENFORCE_EQ(
+      static_cast<int>(buffers[0].size()),
+      1,
+      common::errors::InvalidArgument(
+          "The size of the first buffer should be 1, but received %d.",
+          static_cast<int>(buffers[0].size())));
+  PADDLE_ENFORCE_EQ(
+      static_cast<int>(buffers[1].size()),
+      1,
+      common::errors::InvalidArgument(
+          "The size of the second buffer should be 1, but received %d.",
+          static_cast<int>(buffers[1].size())));
+
+  // operator[]
+  const auto& holder_et0 = grad_tensor_holder[0][0];
+
+  auto* tmp_buffer_tensor =
+      static_cast<phi::SelectedRows*>(holder_et0.impl().get());
+  auto* tmp_buffer_data_sr =
+      tmp_buffer_tensor->mutable_value()->mutable_data<float>(cpu);
+
+  // verify the MergeAdd result (accumulation result)
+  for (int64_t i = 0; i < table_size; ++i) {
+    for (int64_t j = 0; j < embedding_width; ++j) {
+      EXPECT_EQ(tmp_buffer_data_sr[i * embedding_width + j],
+                (static_cast<float>(i) + static_cast<float>(i)));
+    }
+  }
+}

test/cpp/eager/data_structure_tests/tensor_wrapper_test.cc

@@ -0,0 +1,108 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/eager/tensor_wrapper.h"
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/fluid/eager/utils.h"
+#include "test/cpp/eager/data_structure_tests/grad_node_test.h"
+
+TEST(TensorWrapper, Basic) {
+  VLOG(6) << "Test Full reserved";
+  paddle::Tensor et1;
+  phi::DenseTensorMeta meta =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 2}));
+  std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta);
+  auto* dt_ptr = dt->mutable_data<float>(phi::CPUPlace());
+  dt_ptr[0] = 5.0f;
+  dt_ptr[1] = 10.0f;
+  et1.set_impl(dt);
+  // Create grad node;
+  auto grad_test_node0 = std::make_shared<eager_test::GradTestNode>(
+      /* val */ 5.0, /* in_num */ 2, /* out_num */ 2);
+  egr::Edge edge0(grad_test_node0, 1, 2);
+  auto auto_grad0 = std::make_shared<egr::AutogradMeta>(edge0);
+  et1.set_autograd_meta(auto_grad0);
+  et1.set_name("et1");
+  auto tw0 = egr::TensorWrapper(et1);
+  auto recover_et1 = tw0.recover();
+  if (VLOG_IS_ON(7)) {
+    PADDLE_ENFORCE_EQ(
+        recover_et1.name(),
+        std::string("et1@saved"),
+        common::errors::InvalidArgument(
+            "Recovered tensor name should be 'et1@saved', but received %s.",
+            recover_et1.name().c_str()));
+  }
+  PADDLE_ENFORCE_EQ(egr::EagerUtils::OutRankInfo(recover_et1).first,
+                    egr::EagerUtils::OutRankInfo(et1).first,
+                    common::errors::InvalidArgument(
+                        "The OutRankInfo first element of the recovered tensor "
+                        "does not match the original tensor."));
+  PADDLE_ENFORCE_EQ(egr::EagerUtils::OutRankInfo(recover_et1).second,
+                    egr::EagerUtils::OutRankInfo(et1).second,
+                    common::errors::InvalidArgument(
+                        "The OutRankInfo second element of the recovered "
+                        "tensor does not match the original tensor."));
+  VLOG(6) << "Test reconstruct";
+  paddle::Tensor et2;
+  phi::DenseTensorMeta meta2 =
+      phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 2}));
+  std::shared_ptr<phi::DenseTensor> dt2 = std::make_shared<phi::DenseTensor>(
+      std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
+          .get(),
+      meta2);
+  auto* dt_ptr2 = dt->mutable_data<float>(phi::CPUPlace());
+  dt_ptr2[0] = 6.0f;
+  dt_ptr2[1] = 11.0f;
+  et2.set_impl(dt2);
+  et2.set_name("et2");
+  auto grad_test_node1 =
+      std::make_shared<eager_test::GradTestNode>(/* val */ 5.0, 2, 2);
+  egr::Edge edge1(grad_test_node1, 1, 2);
+  auto auto_grad1 = std::make_shared<egr::AutogradMeta>(edge1);
+  et2.set_autograd_meta(auto_grad1);
+  auto tw1 = egr::TensorWrapper(et2, false);
+  auto recover_et2 = tw1.recover();
+  if (VLOG_IS_ON(7)) {
+    PADDLE_ENFORCE_EQ(
+        recover_et2.name(),
+        std::string("et2@Saved"),
+        common::errors::InvalidArgument(
+            "Recovered tensor name should be 'et2@Saved', but received %s.",
+            recover_et2.name().c_str()));
+  }
+  PADDLE_ENFORCE_EQ(egr::EagerUtils::OutRankInfo(recover_et2).first,
+                    egr::EagerUtils::OutRankInfo(et2).first,
+                    common::errors::InvalidArgument(
+                        "The OutRankInfo first element of the recovered tensor "
+                        "does not match the original tensor."));
+  PADDLE_ENFORCE_EQ(egr::EagerUtils::OutRankInfo(recover_et2).second,
+                    egr::EagerUtils::OutRankInfo(et2).second,
+                    common::errors::InvalidArgument(
+                        "The OutRankInfo second element of the recovered "
+                        "tensor does not match the original tensor."));
+  // Test Raw recover
+  paddle::Tensor et3;
+  auto tw2 = egr::TensorWrapper(et3);
+  PADDLE_ENFORCE_EQ(
+      tw2.recover().initialized(),
+      false,
+      common::errors::Fatal(
+          "Variable `tw2` should not be initialized after recover"));
+}