chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
Executable
+20
@@ -0,0 +1,20 @@
|
||||
paddle_test(test_egr_task_nan_inf_utils SRCS nan_inf_utils_test.cc DEPS common)
|
||||
|
||||
if(NOT ((NOT WITH_PYTHON) AND ON_INFER))
|
||||
paddle_test(test_egr_task_hook SRCS hook_test.cc)
|
||||
paddle_test(test_egr_task_backward SRCS backward_test.cc)
|
||||
paddle_test(test_egr_task_grad SRCS grad_test.cc)
|
||||
paddle_test(test_egr_task_fwd_bwd_joint SRCS fwd_bwd_joint_test.cc DEPS phi)
|
||||
paddle_test(test_egr_task_cross_batch SRCS cross_batch_accumulation_test.cc)
|
||||
paddle_test(test_egr_task_hook_intermediate SRCS hook_test_intermediate.cc)
|
||||
paddle_test(test_egr_task_autocodegen SRCS generated_test.cc)
|
||||
paddle_test(test_egr_task_tensor_utils SRCS tensor_utils_test.cc)
|
||||
paddle_test(test_egr_task_eager_utils SRCS eager_utils_test.cc)
|
||||
paddle_test(test_egr_task_forward_autograd SRCS forward_autograd_test.cc)
|
||||
endif()
|
||||
|
||||
if(WITH_ONNXRUNTIME AND WIN32)
|
||||
# Copy onnxruntime for some c++ test in Windows, since the test will
|
||||
# be build only in CI, so suppose the generator in Windows is Ninja.
|
||||
copy_onnx(test_egr_task_nan_inf_utils)
|
||||
endif()
|
||||
@@ -0,0 +1,345 @@
|
||||
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/fluid/eager/backward.h"
|
||||
|
||||
#include <sstream>
|
||||
|
||||
#include "glog/logging.h"
|
||||
#include "gtest/gtest.h"
|
||||
#include "paddle/fluid/eager/accumulation/accumulation_node.h"
|
||||
#include "paddle/fluid/eager/api/all.h"
|
||||
#include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h"
|
||||
#include "paddle/fluid/eager/autograd_meta.h"
|
||||
#include "paddle/fluid/eager/grad_node_info.h"
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/core/tensor_meta.h"
|
||||
#include "test/cpp/eager/test_utils.h"
|
||||
|
||||
PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
|
||||
PD_DECLARE_KERNEL(add, CPU, ALL_LAYOUT);
|
||||
|
||||
namespace egr {
|
||||
|
||||
TEST(Backward, SingleNodeEmptyGrad) {
|
||||
// Prepare Device Contexts
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
// Prepare Inputs
|
||||
phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
|
||||
|
||||
// Create Target Tensor
|
||||
paddle::Tensor target_tensor =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
1.0 /*value*/,
|
||||
false /*is_leaf*/);
|
||||
|
||||
paddle::Tensor leaf_tensor;
|
||||
{
|
||||
// Create Scale Node
|
||||
auto node0_ptr = std::make_shared<GradNodeScale>(1, 1);
|
||||
node0_ptr->SetAttributes_scale(5.0 /*scale*/);
|
||||
|
||||
// Set grad in/out meta
|
||||
node0_ptr->SetDefaultGradInOutMeta();
|
||||
AutogradMeta* auto_grad_meta = EagerUtils::autograd_meta(&target_tensor);
|
||||
auto_grad_meta->SetGradNode(
|
||||
std::dynamic_pointer_cast<GradNodeBase>(node0_ptr));
|
||||
auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
|
||||
auto_grad_meta->SetStopGradient(false);
|
||||
|
||||
AutogradMeta* auto_grad_meta1 = EagerUtils::autograd_meta(&leaf_tensor);
|
||||
|
||||
// Connect Tensor and AccumulationNode via AutoGradMeta
|
||||
auto acc_node_ptr =
|
||||
std::make_shared<egr::GradNodeAccumulation>(leaf_tensor);
|
||||
|
||||
auto_grad_meta1->SetGradNode(
|
||||
std::dynamic_pointer_cast<GradNodeBase>(acc_node_ptr));
|
||||
auto_grad_meta1->SetSingleOutRankWithSlot(0, 0);
|
||||
auto_grad_meta1->SetStopGradient(false);
|
||||
|
||||
node0_ptr->SetGradOutMeta({leaf_tensor}, 0);
|
||||
}
|
||||
std::vector<paddle::Tensor> outs = {target_tensor};
|
||||
// Run Backward
|
||||
Backward(outs, {});
|
||||
|
||||
// Check Output Value
|
||||
eager_test::CompareGradTensorWithValue<float>(leaf_tensor, 5.0);
|
||||
}
|
||||
|
||||
TEST(Backward, SingleNodeCustomGrad) {
|
||||
// Prepare Device Contexts
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
// Prepare Inputs
|
||||
std::vector<paddle::Tensor> target_tensors;
|
||||
phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
|
||||
|
||||
// Create Target Tensor
|
||||
paddle::Tensor tensor =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
1.0 /*value*/,
|
||||
false /*is_leaf*/);
|
||||
target_tensors.emplace_back(std::move(tensor));
|
||||
|
||||
std::vector<paddle::Tensor> grad_tensors;
|
||||
// Create Grad Tensor
|
||||
paddle::Tensor grad_tensor =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
10.0 /*value*/,
|
||||
false /*is_leaf*/);
|
||||
grad_tensors.emplace_back(std::move(grad_tensor));
|
||||
|
||||
paddle::Tensor leaf_tensor;
|
||||
{
|
||||
// Create Scale Node
|
||||
auto node0_ptr = std::make_shared<GradNodeScale>(1, 1);
|
||||
node0_ptr->SetAttributes_scale(5.0 /*scale*/);
|
||||
|
||||
// Set grad in/out meta
|
||||
node0_ptr->SetDefaultGradInOutMeta();
|
||||
|
||||
// Connect Tensor and Node via AutoGradMeta
|
||||
AutogradMeta* auto_grad_meta =
|
||||
EagerUtils::autograd_meta(&(target_tensors[0]));
|
||||
auto_grad_meta->SetGradNode(
|
||||
std::dynamic_pointer_cast<GradNodeBase>(node0_ptr));
|
||||
auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
|
||||
auto_grad_meta->SetStopGradient(false);
|
||||
|
||||
AutogradMeta* auto_grad_meta1 = EagerUtils::autograd_meta(&leaf_tensor);
|
||||
// Connect Tensor and AccumulationNode via AutoGradMeta
|
||||
auto acc_node_ptr =
|
||||
std::make_shared<egr::GradNodeAccumulation>(leaf_tensor);
|
||||
|
||||
auto_grad_meta1->SetGradNode(
|
||||
std::dynamic_pointer_cast<GradNodeBase>(acc_node_ptr));
|
||||
auto_grad_meta1->SetSingleOutRankWithSlot(0, 0);
|
||||
auto_grad_meta1->SetStopGradient(false);
|
||||
node0_ptr->SetGradOutMeta({leaf_tensor}, 0);
|
||||
}
|
||||
|
||||
// Run Backward
|
||||
Backward(target_tensors, grad_tensors);
|
||||
|
||||
// Check Output Value
|
||||
eager_test::CompareGradTensorWithValue<float>(leaf_tensor, 50.0);
|
||||
}
|
||||
|
||||
/*
|
||||
Node1
|
||||
|
|
||||
Node0
|
||||
|
|
||||
inp0
|
||||
*/
|
||||
TEST(Backward, LinearNodes) {
|
||||
// Prepare Device Contexts
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
// Prepare Inputs
|
||||
std::vector<paddle::Tensor> target_tensors;
|
||||
phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
|
||||
|
||||
// Create Target Tensor
|
||||
paddle::Tensor tensor =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
1.0 /*value*/,
|
||||
false /*is_leaf*/);
|
||||
target_tensors.emplace_back(std::move(tensor));
|
||||
|
||||
paddle::Tensor leaf_tensor;
|
||||
{
|
||||
// Create Node0
|
||||
auto node0_ptr = std::make_shared<GradNodeScale>(1, 1);
|
||||
node0_ptr->SetAttributes_scale(5.0 /*scale*/);
|
||||
|
||||
// Set grad in/out meta for node0
|
||||
node0_ptr->SetDefaultGradInOutMeta();
|
||||
|
||||
// Create Node1
|
||||
auto node1_ptr = std::make_shared<GradNodeScale>(1, 1);
|
||||
node1_ptr->SetAttributes_scale(10.0 /*scale*/);
|
||||
|
||||
// Set grad in/out meta for node1
|
||||
node1_ptr->SetDefaultGradInOutMeta();
|
||||
|
||||
// Connect Input Tensor and Node0 via AutoGradMeta
|
||||
AutogradMeta* auto_grad_meta =
|
||||
EagerUtils::autograd_meta(&(target_tensors[0]));
|
||||
auto_grad_meta->SetGradNode(
|
||||
std::dynamic_pointer_cast<GradNodeBase>(node0_ptr));
|
||||
auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
|
||||
auto_grad_meta->SetStopGradient(false);
|
||||
// Connect Node0 -> Node1 via Edge
|
||||
auto tmp_tensor = paddle::Tensor();
|
||||
auto* meta0 = EagerUtils::autograd_meta(&tmp_tensor);
|
||||
meta0->SetStopGradient(false);
|
||||
meta0->SetSingleOutRankWithSlot(0, 0);
|
||||
meta0->SetGradNode(node1_ptr);
|
||||
node0_ptr->SetGradOutMeta(tmp_tensor, 0);
|
||||
|
||||
AutogradMeta* auto_grad_meta1 = EagerUtils::autograd_meta(&leaf_tensor);
|
||||
// Connect Tensor and AccumulationNode via AutoGradMeta
|
||||
auto acc_node_ptr =
|
||||
std::make_shared<egr::GradNodeAccumulation>(leaf_tensor);
|
||||
|
||||
auto_grad_meta1->SetGradNode(
|
||||
std::dynamic_pointer_cast<GradNodeBase>(acc_node_ptr));
|
||||
auto_grad_meta1->SetSingleOutRankWithSlot(0, 0);
|
||||
|
||||
auto_grad_meta1->SetStopGradient(false);
|
||||
node1_ptr->SetGradOutMeta(leaf_tensor, 0);
|
||||
}
|
||||
|
||||
// Use Empty Grad Tensor
|
||||
Backward(target_tensors, {});
|
||||
|
||||
// Check Output Value
|
||||
eager_test::CompareGradTensorWithValue<float>(leaf_tensor, 50.0);
|
||||
}
|
||||
|
||||
/*
|
||||
Node2
|
||||
| |
|
||||
Node0 Node1
|
||||
| |
|
||||
inp0 inp1
|
||||
*/
|
||||
TEST(Backward, WithAccumulation) {
|
||||
// Prepare Device Contexts
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
// Prepare Inputs
|
||||
phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
|
||||
|
||||
// Create Target Tensor
|
||||
std::vector<paddle::Tensor> target_tensors;
|
||||
paddle::Tensor tensor0 =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
1.0 /*value*/,
|
||||
false /*is_leaf*/);
|
||||
paddle::Tensor tensor1 =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
1.0 /*value*/,
|
||||
false /*is_leaf*/);
|
||||
target_tensors.emplace_back(std::move(tensor0));
|
||||
target_tensors.emplace_back(std::move(tensor1));
|
||||
|
||||
// Create Grad Tensor
|
||||
std::vector<paddle::Tensor> grad_tensors;
|
||||
paddle::Tensor grad_tensor0 =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
5.0 /*value*/,
|
||||
false /*is_leaf*/);
|
||||
paddle::Tensor grad_tensor1 =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
10.0 /*value*/,
|
||||
false /*is_leaf*/);
|
||||
grad_tensors.emplace_back(std::move(grad_tensor0));
|
||||
grad_tensors.emplace_back(std::move(grad_tensor1));
|
||||
|
||||
paddle::Tensor leaf_tensor;
|
||||
{
|
||||
// Create Node0
|
||||
auto node0_ptr = std::make_shared<GradNodeScale>(1, 1);
|
||||
node0_ptr->SetAttributes_scale(5.0 /*scale*/);
|
||||
node0_ptr->SetDefaultGradInOutMeta();
|
||||
|
||||
// Create Node1
|
||||
auto node1_ptr = std::make_shared<GradNodeScale>(1, 1);
|
||||
node1_ptr->SetAttributes_scale(10.0 /*scale*/);
|
||||
node1_ptr->SetDefaultGradInOutMeta();
|
||||
// Create Node2
|
||||
auto node2_ptr = std::make_shared<GradNodeScale>(1, 1);
|
||||
node2_ptr->SetAttributes_scale(20.0 /*scale*/);
|
||||
node2_ptr->SetDefaultGradInOutMeta();
|
||||
// Connect Inp0 and Node0 via AutoGradMeta
|
||||
AutogradMeta* auto_grad_meta0 =
|
||||
EagerUtils::autograd_meta(&(target_tensors[0]));
|
||||
auto_grad_meta0->SetGradNode(
|
||||
std::dynamic_pointer_cast<GradNodeBase>(node0_ptr));
|
||||
auto_grad_meta0->SetSingleOutRankWithSlot(0, 0);
|
||||
auto_grad_meta0->SetStopGradient(false);
|
||||
// Connect Inp1 and Node1 via AutoGradMeta
|
||||
AutogradMeta* auto_grad_meta1 =
|
||||
EagerUtils::autograd_meta(&(target_tensors[1]));
|
||||
auto_grad_meta1->SetGradNode(
|
||||
std::dynamic_pointer_cast<GradNodeBase>(node1_ptr));
|
||||
auto_grad_meta1->SetSingleOutRankWithSlot(0, 0);
|
||||
auto_grad_meta1->SetStopGradient(false);
|
||||
|
||||
// Connect Node0 -> Node2 via Edge
|
||||
auto tmp_tensor0 = paddle::Tensor();
|
||||
auto* meta0 = EagerUtils::autograd_meta(&tmp_tensor0);
|
||||
meta0->SetStopGradient(false);
|
||||
meta0->SetSingleOutRankWithSlot(0, 0);
|
||||
meta0->SetGradNode(node2_ptr);
|
||||
node0_ptr->SetGradOutMeta(tmp_tensor0, 0);
|
||||
|
||||
// Connect Node1 -> Node2 via Edge
|
||||
auto tmp_tensor1 = paddle::Tensor();
|
||||
auto* meta1 = EagerUtils::autograd_meta(&tmp_tensor1);
|
||||
meta1->SetStopGradient(false);
|
||||
meta1->SetSingleOutRankWithSlot(0, 0);
|
||||
meta1->SetGradNode(node2_ptr);
|
||||
node1_ptr->SetGradOutMeta(tmp_tensor1, 0);
|
||||
|
||||
AutogradMeta* auto_grad_meta2 = EagerUtils::autograd_meta(&leaf_tensor);
|
||||
// Connect Tensor and AccumulationNode via AutoGradMeta
|
||||
auto acc_node_ptr =
|
||||
std::make_shared<egr::GradNodeAccumulation>(leaf_tensor);
|
||||
|
||||
auto_grad_meta2->SetGradNode(
|
||||
std::dynamic_pointer_cast<GradNodeBase>(acc_node_ptr));
|
||||
auto_grad_meta2->SetSingleOutRankWithSlot(0, 0);
|
||||
|
||||
auto_grad_meta2->SetStopGradient(false);
|
||||
std::vector<egr::AutogradMeta*> res2 = {auto_grad_meta2};
|
||||
node2_ptr->SetGradOutMeta(leaf_tensor, 0);
|
||||
}
|
||||
|
||||
Backward(target_tensors, grad_tensors);
|
||||
|
||||
eager_test::CompareGradTensorWithValue<float>(leaf_tensor, 2500.0);
|
||||
}
|
||||
|
||||
} // namespace egr
|
||||
@@ -0,0 +1,83 @@
|
||||
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <sstream>
|
||||
|
||||
#include "glog/logging.h"
|
||||
#include "gtest/gtest.h"
|
||||
#include "paddle/fluid/eager/accumulation/accumulation_node.h"
|
||||
#include "paddle/fluid/eager/api/all.h"
|
||||
#include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h"
|
||||
#include "paddle/fluid/eager/autograd_meta.h"
|
||||
#include "paddle/fluid/eager/backward.h"
|
||||
#include "paddle/fluid/eager/grad_node_info.h"
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/core/tensor_meta.h"
|
||||
#include "test/cpp/eager/test_utils.h"
|
||||
|
||||
PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
|
||||
|
||||
namespace egr {
|
||||
|
||||
TEST(CrossBatchAccumulation, SingleScaleNode) {
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
std::vector<paddle::Tensor> target_tensors;
|
||||
phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
|
||||
|
||||
paddle::Tensor tensor =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
1.0 /*value*/,
|
||||
false /*is_leaf*/);
|
||||
target_tensors.emplace_back(std::move(tensor));
|
||||
paddle::Tensor& target_tensor = target_tensors[0];
|
||||
|
||||
paddle::Tensor leaf_tensor = paddle::Tensor();
|
||||
|
||||
auto scale_node_ptr = std::make_shared<GradNodeScale>(1, 1);
|
||||
scale_node_ptr->SetAttributes_scale(5.0 /*scale*/);
|
||||
|
||||
scale_node_ptr->SetDefaultGradInOutMeta();
|
||||
|
||||
AutogradMeta* auto_grad_meta = EagerUtils::autograd_meta(&target_tensor);
|
||||
auto_grad_meta->SetGradNode(
|
||||
std::dynamic_pointer_cast<GradNodeBase>(scale_node_ptr));
|
||||
auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
|
||||
auto_grad_meta->SetStopGradient(false);
|
||||
egr_utils_api::RetainGradForTensor(target_tensor); // result: 1.0
|
||||
|
||||
AutogradMeta* meta = EagerUtils::autograd_meta(&leaf_tensor);
|
||||
auto acc_node_ptr = std::make_shared<GradNodeAccumulation>(leaf_tensor);
|
||||
meta->SetStopGradient(false);
|
||||
meta->SetSingleOutRankWithSlot(0, 0);
|
||||
meta->SetGradNode(acc_node_ptr);
|
||||
std::vector<egr::AutogradMeta*> res = {meta};
|
||||
scale_node_ptr->SetGradOutMeta(leaf_tensor, 0);
|
||||
|
||||
Backward(target_tensors, {});
|
||||
|
||||
eager_test::CompareGradTensorWithValue<float>(target_tensor, 1.0);
|
||||
eager_test::CompareGradTensorWithValue<float>(leaf_tensor, 5.0);
|
||||
|
||||
Backward(target_tensors, {});
|
||||
|
||||
eager_test::CompareGradTensorWithValue<float>(target_tensor, 1.0);
|
||||
eager_test::CompareGradTensorWithValue<float>(leaf_tensor, 10.0);
|
||||
}
|
||||
|
||||
} // namespace egr
|
||||
@@ -0,0 +1,567 @@
|
||||
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <sstream>
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "paddle/common/flags.h"
|
||||
#include "paddle/fluid/eager/accumulation/accumulation_node.h"
|
||||
#include "paddle/fluid/eager/eager_tensor.h"
|
||||
#include "paddle/fluid/eager/grad_node_info.h"
|
||||
#include "paddle/fluid/eager/utils.h"
|
||||
#include "paddle/phi/api/lib/utils/allocator.h"
|
||||
#include "paddle/phi/core/enforce.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "test/cpp/eager/data_structure_tests/grad_node_test.h"
|
||||
#include "test/cpp/eager/test_utils.h"
|
||||
COMMON_DECLARE_bool(tensor_md5_checksum_use_binary_format);
|
||||
|
||||
PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
|
||||
|
||||
namespace egr {
|
||||
|
||||
TEST(EagerUtils, AutoGradMeta) {
|
||||
// Construct Eager Tensor
|
||||
phi::DenseTensorMeta meta =
|
||||
phi::DenseTensorMeta(phi::DataType::FLOAT32, common::make_ddim({1, 1}));
|
||||
std::shared_ptr<phi::DenseTensor> dt0 = std::make_shared<phi::DenseTensor>(
|
||||
std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
|
||||
.get(),
|
||||
meta);
|
||||
dt0->mutable_data<float>(phi::CPUPlace())[0] = 10.0;
|
||||
paddle::Tensor et0 = paddle::Tensor(dt0);
|
||||
|
||||
std::shared_ptr<phi::DenseTensor> dt1 = std::make_shared<phi::DenseTensor>(
|
||||
std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
|
||||
.get(),
|
||||
meta);
|
||||
dt1->mutable_data<float>(phi::CPUPlace())[0] = 20.0;
|
||||
paddle::Tensor et1 = paddle::Tensor(dt1);
|
||||
|
||||
// unsafe_autograd_meta()
|
||||
// autograd_meta()
|
||||
AutogradMeta* autograd_meta0 = EagerUtils::autograd_meta(&et0);
|
||||
AutogradMeta* autograd_meta1 = EagerUtils::autograd_meta(&et1);
|
||||
|
||||
AutogradMeta* unsafe_autograd_meta_after =
|
||||
EagerUtils::unsafe_autograd_meta(et0);
|
||||
PADDLE_ENFORCE_NOT_NULL(
|
||||
unsafe_autograd_meta_after,
|
||||
common::errors::PreconditionNotMet(
|
||||
"Unsafe autograd meta after should not be null."));
|
||||
|
||||
// NOTE: Since autograd_meta will be copied make sure it's not null
|
||||
std::vector<paddle::Tensor> ets = {et0, et1};
|
||||
auto test_node = std::make_shared<eager_test::GradTestNode>();
|
||||
|
||||
std::vector<AutogradMeta*> autograd_metas = EagerUtils::autograd_meta(&ets);
|
||||
std::vector<AutogradMeta*> unsafe_autograd_metas =
|
||||
EagerUtils::unsafe_autograd_meta(ets);
|
||||
PADDLE_ENFORCE_NOT_NULL(unsafe_autograd_metas[0],
|
||||
common::errors::PreconditionNotMet(
|
||||
"Unsafe autograd metas should not be null."));
|
||||
PADDLE_ENFORCE_NOT_NULL(unsafe_autograd_metas[1],
|
||||
common::errors::PreconditionNotMet(
|
||||
"Unsafe autograd metas should not be null."));
|
||||
|
||||
// Set Autograd Meta
|
||||
autograd_meta0->SetSingleOutRankWithSlot(0, 1);
|
||||
|
||||
autograd_meta0->SetGradNode(test_node);
|
||||
|
||||
// OutRankInfo()
|
||||
std::pair<size_t, size_t> out_rank_info0 = EagerUtils::OutRankInfo(et0);
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(out_rank_info0.first),
|
||||
0UL,
|
||||
common::errors::InvalidArgument("The first element of out rank info "
|
||||
"mismatch. Expected 0 but received %d.",
|
||||
static_cast<int>(out_rank_info0.first)));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(out_rank_info0.second),
|
||||
1UL,
|
||||
common::errors::InvalidArgument("The second element of out rank info "
|
||||
"mismatch. Expected 1 but received %d.",
|
||||
static_cast<int>(out_rank_info0.second)));
|
||||
|
||||
// grad_node()
|
||||
std::shared_ptr<GradNodeBase> grad_node0 = EagerUtils::grad_node(et0);
|
||||
PADDLE_ENFORCE_NOT_NULL(
|
||||
grad_node0.get(),
|
||||
common::errors::PreconditionNotMet("Grad of node should not be null."));
|
||||
|
||||
EagerUtils::SetHistory(autograd_meta1, test_node);
|
||||
EagerUtils::SetHistory(autograd_meta1, test_node);
|
||||
std::shared_ptr<GradNodeBase> grad_node1 = EagerUtils::grad_node(et1);
|
||||
PADDLE_ENFORCE_NOT_NULL(
|
||||
grad_node1.get(),
|
||||
common::errors::PreconditionNotMet("Grad of node should not be null."));
|
||||
|
||||
// SetOutRankWithSlot()
|
||||
EagerUtils::SetOutRankWithSlot(autograd_meta1, 0);
|
||||
std::pair<size_t, size_t> out_rank_info1 = EagerUtils::OutRankInfo(et1);
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(out_rank_info1.first),
|
||||
0UL,
|
||||
common::errors::InvalidArgument("The first element of out rank info "
|
||||
"mismatch. Expected 0 but received %d.",
|
||||
static_cast<int>(out_rank_info1.first)));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(out_rank_info1.second),
|
||||
0UL,
|
||||
common::errors::InvalidArgument("The second element of out rank info "
|
||||
"mismatch. Expected 0 but received %d.",
|
||||
static_cast<int>(out_rank_info1.second)));
|
||||
|
||||
EagerUtils::SetOutRankWithSlot(&autograd_metas, 0);
|
||||
std::pair<size_t, size_t> out_rank_info2 = EagerUtils::OutRankInfo(et0);
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(out_rank_info2.first),
|
||||
0UL,
|
||||
common::errors::InvalidArgument("The first element of out rank info "
|
||||
"mismatch. Expected 0 but received %d.",
|
||||
static_cast<int>(out_rank_info2.first)));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(out_rank_info2.second),
|
||||
0UL,
|
||||
common::errors::InvalidArgument("The second element of out rank info "
|
||||
"mismatch. Expected 0 but received %d.",
|
||||
static_cast<int>(out_rank_info2.second)));
|
||||
|
||||
std::pair<size_t, size_t> out_rank_info3 = EagerUtils::OutRankInfo(et1);
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(out_rank_info3.first),
|
||||
0UL,
|
||||
common::errors::InvalidArgument("The first element of out rank info "
|
||||
"mismatch. Expected 0 but received %d.",
|
||||
static_cast<int>(out_rank_info3.first)));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(out_rank_info3.second),
|
||||
1UL,
|
||||
common::errors::InvalidArgument("The second element of out rank info "
|
||||
"mismatch. Expected 1 but received %d.",
|
||||
static_cast<int>(out_rank_info3.second)));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
paddle::Tensor CreateTestCPUTensor(T val, const phi::DDim& ddim) {
|
||||
phi::DenseTensorMeta meta =
|
||||
phi::DenseTensorMeta(phi::DataType::FLOAT32, ddim);
|
||||
paddle::Tensor tensor;
|
||||
std::shared_ptr<phi::DenseTensor> dt = std::make_shared<phi::DenseTensor>(
|
||||
std::make_unique<paddle::experimental::DefaultAllocator>(phi::CPUPlace())
|
||||
.get(),
|
||||
meta);
|
||||
auto* dt_ptr = dt->mutable_data<T>(phi::CPUPlace());
|
||||
for (int64_t i = 0; i < dt->numel(); i++) {
|
||||
dt_ptr[i] = val;
|
||||
}
|
||||
tensor.set_impl(dt);
|
||||
return tensor;
|
||||
}
|
||||
|
||||
TEST(EagerUtils, ComputeRequireGrad) {
|
||||
auto auto_grad0 = std::make_shared<egr::AutogradMeta>();
|
||||
auto auto_grad1 = std::make_shared<egr::AutogradMeta>();
|
||||
auto auto_grad2 = std::make_shared<egr::AutogradMeta>();
|
||||
auto auto_grad3 = std::make_shared<egr::AutogradMeta>();
|
||||
PADDLE_ENFORCE_EQ(
|
||||
auto_grad0->NumericStopGradient(),
|
||||
-1,
|
||||
common::errors::InvalidArgument("The NumericStopGradient of auto grad "
|
||||
"mismatch. Expected -1 but received %d.",
|
||||
auto_grad0->NumericStopGradient()));
|
||||
VLOG(6) << "Single Test ComputeRequireGrad";
|
||||
auto_grad0->SetStopGradient(true);
|
||||
PADDLE_ENFORCE_EQ(egr::EagerUtils::ComputeRequireGrad(true, auto_grad0.get()),
|
||||
false,
|
||||
::common::errors::InvalidArgument(
|
||||
"Expected ComputeRequireGrad(true, auto_grad0) to be "
|
||||
"false, but it is true."));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
egr::EagerUtils::ComputeRequireGrad(false, auto_grad0.get()),
|
||||
false,
|
||||
::common::errors::InvalidArgument(
|
||||
"Expected ComputeRequireGrad(false, auto_grad0) to be false, but it "
|
||||
"is true."));
|
||||
auto_grad0->SetStopGradient(false);
|
||||
PADDLE_ENFORCE_EQ(
|
||||
egr::EagerUtils::ComputeRequireGrad(false, auto_grad0.get()),
|
||||
false,
|
||||
::common::errors::InvalidArgument(
|
||||
"Expected ComputeRequireGrad(false, auto_grad0) to be false, but it "
|
||||
"is true."));
|
||||
|
||||
PADDLE_ENFORCE_EQ(egr::EagerUtils::ComputeRequireGrad(true, auto_grad0.get()),
|
||||
true,
|
||||
::common::errors::InvalidArgument(
|
||||
"Expected ComputeRequireGrad(true, auto_grad0) to be "
|
||||
"true, but it is false."));
|
||||
|
||||
VLOG(6) << "Multi Test ComputeRequireGrad";
|
||||
auto_grad0->SetStopGradient(false);
|
||||
auto_grad1->SetStopGradient(true);
|
||||
PADDLE_ENFORCE_EQ(egr::EagerUtils::ComputeRequireGrad(
|
||||
true, auto_grad0.get(), auto_grad1.get()),
|
||||
true,
|
||||
::common::errors::InvalidArgument(
|
||||
"Expected ComputeRequireGrad(true, auto_grad0, "
|
||||
"auto_grad1) to be true, but it is false."));
|
||||
|
||||
PADDLE_ENFORCE_EQ(egr::EagerUtils::ComputeRequireGrad(
|
||||
false, auto_grad0.get(), auto_grad1.get()),
|
||||
false,
|
||||
::common::errors::InvalidArgument(
|
||||
"Expected ComputeRequireGrad(false, auto_grad0, "
|
||||
"auto_grad1) to be false, but it is true."));
|
||||
auto_grad0->SetStopGradient(true);
|
||||
PADDLE_ENFORCE_EQ(egr::EagerUtils::ComputeRequireGrad(
|
||||
true, auto_grad0.get(), auto_grad1.get()),
|
||||
false,
|
||||
::common::errors::InvalidArgument(
|
||||
"Expected ComputeRequireGrad(true, auto_grad0, "
|
||||
"auto_grad1) to be false, but it is true."));
|
||||
|
||||
PADDLE_ENFORCE_EQ(egr::EagerUtils::ComputeRequireGrad(
|
||||
false, auto_grad0.get(), auto_grad1.get()),
|
||||
false,
|
||||
::common::errors::InvalidArgument(
|
||||
"Expected ComputeRequireGrad(false, auto_grad0, "
|
||||
"auto_grad1) to be false, but it is true."));
|
||||
}
|
||||
|
||||
TEST(EagerUtils, PassStopGradient) {
|
||||
auto auto_grad0 = std::make_shared<egr::AutogradMeta>();
|
||||
auto auto_grad1 = std::make_shared<egr::AutogradMeta>();
|
||||
auto auto_grad2 = std::make_shared<egr::AutogradMeta>();
|
||||
auto auto_grad3 = std::make_shared<egr::AutogradMeta>();
|
||||
PADDLE_ENFORCE_EQ(
|
||||
auto_grad0->NumericStopGradient(),
|
||||
-1,
|
||||
common::errors::InvalidArgument("The NumericStopGradient of auto grad "
|
||||
"mismatch. Expected -1 but received %d.",
|
||||
auto_grad0->NumericStopGradient()));
|
||||
VLOG(6) << "Test PassStopGradient";
|
||||
egr::EagerUtils::PassStopGradient(false, auto_grad0.get());
|
||||
PADDLE_ENFORCE_EQ(
|
||||
auto_grad0->StopGradient(),
|
||||
false,
|
||||
::common::errors::InvalidArgument(
|
||||
"Expected auto_grad0->StopGradient() to be false, but received %d.",
|
||||
auto_grad0->StopGradient()));
|
||||
egr::EagerUtils::PassStopGradient(true,
|
||||
auto_grad0.get(),
|
||||
auto_grad1.get(),
|
||||
auto_grad2.get(),
|
||||
auto_grad3.get());
|
||||
PADDLE_ENFORCE_EQ(
|
||||
auto_grad0->StopGradient(),
|
||||
true,
|
||||
::common::errors::InvalidArgument(
|
||||
"Expected auto_grad0->StopGradient() to be true, but received %d.",
|
||||
auto_grad0->StopGradient()));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
auto_grad1->StopGradient(),
|
||||
true,
|
||||
::common::errors::InvalidArgument(
|
||||
"Expected auto_grad1->StopGradient() to be true, but received %d.",
|
||||
auto_grad1->StopGradient()));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
auto_grad2->StopGradient(),
|
||||
true,
|
||||
::common::errors::InvalidArgument(
|
||||
"Expected auto_grad2->StopGradient() to be true, but received %d.",
|
||||
auto_grad2->StopGradient()));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
auto_grad3->StopGradient(),
|
||||
true,
|
||||
::common::errors::InvalidArgument(
|
||||
"Expected auto_grad3->StopGradient() to be true, but received %d.",
|
||||
auto_grad3->StopGradient()));
|
||||
}
|
||||
|
||||
TEST(EagerUtils, TrySyncToVar) {
|
||||
phi::DDim ddim = common::make_ddim({2, 4, 4, 4});
|
||||
auto tensor = CreateTestCPUTensor(5.0f, ddim);
|
||||
std::vector<std::shared_ptr<egr::EagerVariable>> var_bases = {
|
||||
egr::EagerUtils::TrySyncToVar(tensor)};
|
||||
|
||||
paddle::framework::Variable* var = var_bases[0]->MutableVar();
|
||||
const auto& framework_tensor = var->Get<phi::DenseTensor>();
|
||||
|
||||
const float* ptr = framework_tensor.data<float>();
|
||||
VLOG(6) << "Check Value for SyncToVarsSingle";
|
||||
PADDLE_ENFORCE_EQ(framework_tensor.numel(),
|
||||
tensor.numel(),
|
||||
common::errors::InvalidArgument(
|
||||
"The numel of framework tensor and numel of "
|
||||
"tensor should be the same, but received %d and %d.",
|
||||
framework_tensor.numel(),
|
||||
tensor.numel()));
|
||||
|
||||
for (int i = 0; i < framework_tensor.numel(); i++) {
|
||||
PADDLE_ENFORCE_EQ(ptr[i],
|
||||
5.0f,
|
||||
common::errors::InvalidArgument(
|
||||
"The numel of framework tensor mismatch. "
|
||||
"Expected 5.0 but received %f.",
|
||||
ptr[i]));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(EagerUtils, TrySyncToVars) {
|
||||
phi::DDim ddim = common::make_ddim({2, 4, 4, 4});
|
||||
std::vector<paddle::Tensor> tensors = {CreateTestCPUTensor(1.0f, ddim),
|
||||
CreateTestCPUTensor(2.0f, ddim)};
|
||||
|
||||
std::vector<std::shared_ptr<egr::EagerVariable>> var_bases =
|
||||
egr::EagerUtils::TrySyncToVars(tensors);
|
||||
|
||||
{
|
||||
paddle::framework::Variable* var = var_bases[0]->MutableVar();
|
||||
const auto& framework_tensor = var->Get<phi::DenseTensor>();
|
||||
|
||||
const float* ptr = framework_tensor.data<float>();
|
||||
PADDLE_ENFORCE_EQ(
|
||||
framework_tensor.numel(),
|
||||
tensors[0].numel(),
|
||||
common::errors::InvalidArgument(
|
||||
"The numel of framework tensor and numel "
|
||||
"of tensor should be the same, but received %d and %d.",
|
||||
framework_tensor.numel(),
|
||||
tensors[0].numel()));
|
||||
|
||||
for (int i = 0; i < framework_tensor.numel(); i++) {
|
||||
PADDLE_ENFORCE_EQ(ptr[i],
|
||||
1.0,
|
||||
common::errors::InvalidArgument(
|
||||
"The numel of framework tensor mismatch. Expected "
|
||||
"1.0 but received %f.",
|
||||
ptr[i]));
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
paddle::framework::Variable* var = var_bases[1]->MutableVar();
|
||||
const auto& framework_tensor = var->Get<phi::DenseTensor>();
|
||||
|
||||
const float* ptr = framework_tensor.data<float>();
|
||||
VLOG(6) << "Check Value for SyncToVarsMultiple";
|
||||
PADDLE_ENFORCE_EQ(
|
||||
framework_tensor.numel(),
|
||||
tensors[0].numel(),
|
||||
common::errors::InvalidArgument(
|
||||
"The numel of framework tensor and numel "
|
||||
"of tensor should be the same, but received %d and %d.",
|
||||
framework_tensor.numel(),
|
||||
tensors[0].numel()));
|
||||
|
||||
for (int i = 0; i < framework_tensor.numel(); i++) {
|
||||
PADDLE_ENFORCE_EQ(ptr[i],
|
||||
2.0,
|
||||
common::errors::InvalidArgument(
|
||||
"The numel of framework tensor mismatch. Expected "
|
||||
"2.0 but received %f.",
|
||||
ptr[i]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST(EagerUtils, CreateVars) {
|
||||
VLOG(6) << "Check CreateVars";
|
||||
std::vector<std::shared_ptr<egr::EagerVariable>> outs =
|
||||
egr::EagerUtils::CreateVars(2);
|
||||
PADDLE_ENFORCE_EQ(
|
||||
outs.size(),
|
||||
2UL,
|
||||
common::errors::InvalidArgument(
|
||||
"Size of outs mismatch. Expected 2 but received %d.", outs.size()));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
outs[0]->Var().IsInitialized(),
|
||||
false,
|
||||
::common::errors::AlreadyExists("Expected the first variable to be "
|
||||
"uninitialized, but already exists."));
|
||||
}
|
||||
|
||||
TEST(EagerUtils, GetGradAccumulationNode) {
|
||||
VLOG(6) << "Check GetGradAccumulationNode";
|
||||
paddle::Tensor t0("test_tensor");
|
||||
ASSERT_EQ(egr::EagerUtils::GetGradAccumulationNode(t0), nullptr);
|
||||
auto autograd_ptr0 = egr::EagerUtils::autograd_meta(&t0);
|
||||
autograd_ptr0->SetStopGradient(true);
|
||||
ASSERT_EQ(egr::EagerUtils::GetGradAccumulationNode(t0), nullptr);
|
||||
autograd_ptr0->SetStopGradient(false);
|
||||
auto res = std::dynamic_pointer_cast<egr::GradNodeAccumulation>(
|
||||
egr::EagerUtils::GetGradAccumulationNode(t0));
|
||||
ASSERT_TRUE(res != nullptr);
|
||||
auto res2 = egr::EagerUtils::GetGradAccumulationNode(t0);
|
||||
ASSERT_EQ(res2.get(), res.get());
|
||||
autograd_ptr0->SetStopGradient(true);
|
||||
auto res3 = egr::EagerUtils::GetGradAccumulationNode(t0);
|
||||
ASSERT_EQ(res3, nullptr);
|
||||
autograd_ptr0->SetStopGradient(false);
|
||||
autograd_ptr0->SetGradNode(
|
||||
std::make_shared<eager_test::GradTestNode>(1, 2.0, 3));
|
||||
ASSERT_ANY_THROW(egr::EagerUtils::GetGradAccumulationNode(t0));
|
||||
}
|
||||
|
||||
TEST(EagerUtils, FillZeroForEmptyOptionalGradInput) {
|
||||
paddle::small_vector<std::vector<paddle::Tensor>, egr::kSlotSmallVectorSize>
|
||||
grads = {std::vector<paddle::Tensor>(1)};
|
||||
paddle::small_vector<std::vector<GradSlotMeta>, egr::kSlotSmallVectorSize>
|
||||
slot_metas = {std::vector<GradSlotMeta>(1)};
|
||||
|
||||
phi::DenseTensorMeta tensor_meta;
|
||||
tensor_meta.dtype = phi::DataType::FLOAT32;
|
||||
tensor_meta.dims = {2, 4};
|
||||
slot_metas[0][0].SetTensorMeta(tensor_meta);
|
||||
slot_metas[0][0].SetPlace(phi::CPUPlace());
|
||||
|
||||
EagerUtils::FillZeroForEmptyOptionalGradInput(&grads[0], slot_metas[0]);
|
||||
eager_test::CompareTensorWithValue<float>(grads[0][0], 0.0);
|
||||
}
|
||||
TEST(EagerUtils, SetTensorName) {
|
||||
std::string unique_api_name = "Test";
|
||||
std::string var_name = "out";
|
||||
phi::DDim ddim = common::make_ddim({2, 4, 4, 4});
|
||||
std::vector<paddle::Tensor> tensors = {CreateTestCPUTensor(1.0f, ddim),
|
||||
CreateTestCPUTensor(2.0f, ddim)};
|
||||
paddle::optional<paddle::Tensor> optional_t;
|
||||
optional_t = tensors[0];
|
||||
paddle::Tensor* t = &(optional_t.get());
|
||||
|
||||
auto generate_tensor_name = [](const std::string& unique_api_name,
|
||||
const std::string& var_name,
|
||||
const paddle::Tensor* t) {
|
||||
std::ostringstream oss;
|
||||
oss << unique_api_name << "_" << var_name << "_" << t->dtype() << "_";
|
||||
for (int i = 0; i < t->dims().size(); ++i) {
|
||||
if (i != 0) {
|
||||
oss << "x";
|
||||
}
|
||||
oss << t->dims()[i];
|
||||
}
|
||||
return oss.str();
|
||||
};
|
||||
// Gen refer name
|
||||
std::string refer_name = generate_tensor_name(unique_api_name, var_name, t);
|
||||
// test paddle::optional<paddle::Tensor>* tensor
|
||||
egr::SetTensorName(unique_api_name, var_name, &optional_t);
|
||||
ASSERT_TRUE(t->name() == refer_name);
|
||||
refer_name = generate_tensor_name(
|
||||
unique_api_name, var_name + "_" + std::to_string(0), t);
|
||||
// test std::vector<paddle::Tensor>* tensors
|
||||
egr::SetTensorName(unique_api_name, var_name, &tensors);
|
||||
ASSERT_TRUE(tensors[0].name() == refer_name);
|
||||
// test paddle::optional<std::vector<paddle::Tensor>>* tensors
|
||||
paddle::optional<std::vector<paddle::Tensor>> opt_tensors = tensors;
|
||||
egr::SetTensorName(unique_api_name, var_name, &opt_tensors);
|
||||
ASSERT_TRUE(tensors[0].name() == refer_name);
|
||||
}
|
||||
TEST(EagerUtils, SetGradTensorName) {
|
||||
phi::DDim ddim = common::make_ddim({2, 4});
|
||||
std::vector<paddle::Tensor> tensors = {CreateTestCPUTensor(1.0f, ddim)};
|
||||
paddle::small_vector<std::vector<GradSlotMeta>, egr::kSlotSmallVectorSize>
|
||||
slot_metas = {std::vector<GradSlotMeta>(1)};
|
||||
|
||||
phi::DenseTensorMeta tensor_meta;
|
||||
tensor_meta.dtype = phi::DataType::FLOAT32;
|
||||
tensor_meta.dims = {2, 4};
|
||||
slot_metas[0][0].SetTensorMeta(tensor_meta);
|
||||
slot_metas[0][0].SetPlace(phi::CPUPlace());
|
||||
|
||||
egr::SetGradTensorName(&tensors, 0, slot_metas);
|
||||
std::string refer_name = "@Grad";
|
||||
ASSERT_TRUE(tensors[0].name() == refer_name);
|
||||
}
|
||||
|
||||
TEST(EagerUtils, SaveTensorMD5CheckSumToFile) {
|
||||
#define EXPECT_SAVE_TENSOR_MD5_CHECKSUM_FAILURE(t) \
|
||||
try { \
|
||||
egr::SaveTensorMD5CheckSumToFile("", t); \
|
||||
FAIL() << "Expected std::exception"; \
|
||||
} catch (const std::exception& e) { \
|
||||
std::string error_str = e.what(); \
|
||||
EXPECT_NE(error_str.find("Cannot open file for writing."), \
|
||||
std::string::npos); \
|
||||
} catch (...) { \
|
||||
FAIL() << "Unexpected error"; \
|
||||
}
|
||||
|
||||
#define EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(t) \
|
||||
try { \
|
||||
egr::SaveTensorMD5CheckSumToFile("test_md5_checksum.txt", t); \
|
||||
} catch (const std::exception& e) { \
|
||||
FAIL() << "Unexpected error: " << e.what(); \
|
||||
} catch (...) { \
|
||||
FAIL() << "Unexpected error"; \
|
||||
}
|
||||
|
||||
// Test the invalid file name
|
||||
phi::DDim ddim = common::make_ddim({20, 40});
|
||||
paddle::Tensor t = CreateTestCPUTensor(1.0f, ddim);
|
||||
EXPECT_SAVE_TENSOR_MD5_CHECKSUM_FAILURE(t)
|
||||
paddle::optional<paddle::Tensor> optional_t;
|
||||
optional_t = CreateTestCPUTensor<double>(1.0, ddim);
|
||||
EXPECT_SAVE_TENSOR_MD5_CHECKSUM_FAILURE(optional_t)
|
||||
// Test the vector input
|
||||
std::vector<paddle::Tensor> tensors = {CreateTestCPUTensor<int64_t>(1, ddim),
|
||||
CreateTestCPUTensor<int64_t>(1, ddim)};
|
||||
EXPECT_SAVE_TENSOR_MD5_CHECKSUM_FAILURE(tensors)
|
||||
paddle::optional<std::vector<paddle::Tensor>> opt_tensors =
|
||||
std::vector<paddle::Tensor>{CreateTestCPUTensor<bool>(true, ddim),
|
||||
CreateTestCPUTensor<bool>(false, ddim)};
|
||||
EXPECT_SAVE_TENSOR_MD5_CHECKSUM_FAILURE(opt_tensors)
|
||||
// test the different data type
|
||||
EXPECT_SAVE_TENSOR_MD5_CHECKSUM_FAILURE(CreateTestCPUTensor<int>(1, ddim))
|
||||
EXPECT_SAVE_TENSOR_MD5_CHECKSUM_FAILURE(
|
||||
CreateTestCPUTensor<phi::float16>(static_cast<phi::float16>(1), ddim))
|
||||
EXPECT_SAVE_TENSOR_MD5_CHECKSUM_FAILURE(
|
||||
CreateTestCPUTensor<int32_t>(static_cast<int32_t>(1), ddim))
|
||||
paddle::Tensor complex64_t =
|
||||
CreateTestCPUTensor(phi::complex64(1.0f, 2.0f), ddim);
|
||||
paddle::Tensor complex128_t =
|
||||
CreateTestCPUTensor(phi::complex128(1.0f, 2.0f), ddim);
|
||||
#if defined(PADDLE_WITH_CUDA)
|
||||
EXPECT_SAVE_TENSOR_MD5_CHECKSUM_FAILURE(
|
||||
CreateTestCPUTensor<phi::bfloat16>(static_cast<phi::bfloat16>(1), ddim))
|
||||
EXPECT_SAVE_TENSOR_MD5_CHECKSUM_FAILURE(
|
||||
CreateTestCPUTensor<phi::float8_e4m3fn>(
|
||||
static_cast<phi::float8_e4m3fn>(1), ddim))
|
||||
EXPECT_SAVE_TENSOR_MD5_CHECKSUM_FAILURE(CreateTestCPUTensor<phi::float8_e5m2>(
|
||||
static_cast<phi::float8_e5m2>(1), ddim))
|
||||
#endif
|
||||
|
||||
#ifndef _WIN32
|
||||
// test save to file
|
||||
EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(t)
|
||||
EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(optional_t)
|
||||
EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(tensors)
|
||||
EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(opt_tensors)
|
||||
EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(complex64_t)
|
||||
EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(complex128_t)
|
||||
// test using binary format
|
||||
FLAGS_tensor_md5_checksum_use_binary_format = true;
|
||||
EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(t)
|
||||
EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(optional_t)
|
||||
EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(tensors)
|
||||
EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(opt_tensors)
|
||||
EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(complex64_t)
|
||||
EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(complex128_t)
|
||||
// test Fake dist tensor
|
||||
t.set_impl(std::make_shared<phi::distributed::DistTensor>());
|
||||
EXPECT_SAVE_TENSOR_MD5_CHECKSUM_SUCCESS(t)
|
||||
#endif
|
||||
}
|
||||
} // namespace egr
|
||||
@@ -0,0 +1,362 @@
|
||||
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <sstream>
|
||||
|
||||
#include "glog/logging.h"
|
||||
#include "gtest/gtest.h"
|
||||
#include "paddle/fluid/eager/api/all.h"
|
||||
#include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h"
|
||||
#include "paddle/fluid/eager/autograd_meta.h"
|
||||
#include "paddle/fluid/eager/grad_node_info.h"
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/core/tensor_meta.h"
|
||||
#include "test/cpp/eager/test_utils.h"
|
||||
|
||||
PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
|
||||
|
||||
namespace egr {
|
||||
|
||||
TEST(Forward, SingleNode) {
|
||||
// Prepare Device Contexts
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
// Prepare Inputs
|
||||
std::vector<paddle::Tensor> target_tensors;
|
||||
phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
|
||||
|
||||
// Create Target Tensor
|
||||
paddle::Tensor t = eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
5.0 /*value*/,
|
||||
false /*is_leaf*/);
|
||||
target_tensors.emplace_back(std::move(t));
|
||||
paddle::Tensor& tensor = target_tensors[0];
|
||||
EagerUtils::autograd_meta(&tensor)->SetStopGradient(false);
|
||||
|
||||
// Run Forward
|
||||
float scale = 2.0;
|
||||
float bias = 3.0;
|
||||
paddle::Tensor out = egr::scale(
|
||||
tensor, scale, bias, true /*bias_after_scale*/, true /*trace_backward*/);
|
||||
|
||||
// Examine Forward Output
|
||||
eager_test::CompareTensorWithValue<float>(out, 13.0);
|
||||
|
||||
// Examine GradNode
|
||||
{
|
||||
// 1. GradNode
|
||||
AutogradMeta* meta = EagerUtils::autograd_meta(&out);
|
||||
GradNodeBase* grad_node = meta->GradNode();
|
||||
GradNodeScale* scale_node = dynamic_cast<GradNodeScale*>(grad_node);
|
||||
|
||||
CHECK_NOTNULL(scale_node);
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(meta->OutRankInfo().first),
|
||||
0,
|
||||
common::errors::InvalidArgument(
|
||||
"static_cast<int>(meta->OutRankInfo().first) is not 0"));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(meta->OutRankInfo().second),
|
||||
0,
|
||||
common::errors::InvalidArgument(
|
||||
"static_cast<int>(meta->OutRankInfo().second) is not 0"));
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
inp
|
||||
|
|
||||
Node0
|
||||
|
|
||||
Node1
|
||||
|
|
||||
out
|
||||
*/
|
||||
TEST(Forward, LinearNodes) {
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
// Prepare Inputs
|
||||
std::vector<paddle::Tensor> target_tensors;
|
||||
phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
|
||||
|
||||
// Create Target Tensor
|
||||
paddle::Tensor t = eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
5.0 /*value*/,
|
||||
false /*is_leaf*/);
|
||||
target_tensors.emplace_back(std::move(t));
|
||||
paddle::Tensor& tensor = target_tensors[0];
|
||||
EagerUtils::autograd_meta(&tensor)->SetStopGradient(false);
|
||||
|
||||
// Run Forward Node 0
|
||||
float scale0 = 2.0;
|
||||
float bias0 = 3.0;
|
||||
paddle::Tensor out0 = egr::scale(tensor,
|
||||
scale0,
|
||||
bias0,
|
||||
true /*bias_after_scale*/,
|
||||
true /*trace_backward*/);
|
||||
|
||||
// Run Forward Node 1
|
||||
float scale1 = 5.0;
|
||||
float bias1 = 10.0;
|
||||
paddle::Tensor out1 = egr::scale(
|
||||
out0, scale1, bias1, true /*bias_after_scale*/, true /*trace_backward*/);
|
||||
|
||||
// Examine Forward Output 0
|
||||
eager_test::CompareTensorWithValue<float>(out0, 13.0);
|
||||
|
||||
// Examine Forward Output 1
|
||||
eager_test::CompareTensorWithValue<float>(out1, 75.0);
|
||||
|
||||
// Examine GradNode
|
||||
{
|
||||
// 1. GradNode
|
||||
// Node 0
|
||||
AutogradMeta* meta0 = EagerUtils::autograd_meta(&out0);
|
||||
GradNodeBase* grad_node0 = meta0->GradNode();
|
||||
GradNodeScale* scale_node0 = dynamic_cast<GradNodeScale*>(grad_node0);
|
||||
|
||||
CHECK_NOTNULL(scale_node0);
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(meta0->OutRankInfo().first),
|
||||
0,
|
||||
common::errors::InvalidArgument(
|
||||
"static_cast<int>(meta0->OutRankInfo().first) is not 0"));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(meta0->OutRankInfo().second),
|
||||
0,
|
||||
common::errors::InvalidArgument(
|
||||
"static_cast<int>(meta0->OutRankInfo().second) is not 0"));
|
||||
|
||||
// Node 1
|
||||
AutogradMeta* meta1 = EagerUtils::autograd_meta(&out1);
|
||||
GradNodeBase* grad_node1 = meta1->GradNode();
|
||||
GradNodeScale* scale_node1 = dynamic_cast<GradNodeScale*>(grad_node1);
|
||||
|
||||
CHECK_NOTNULL(scale_node1);
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(meta1->OutRankInfo().first),
|
||||
0,
|
||||
common::errors::InvalidArgument(
|
||||
"static_cast<int>(meta1->OutRankInfo().first) is not 0"));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(meta1->OutRankInfo().second),
|
||||
0,
|
||||
common::errors::InvalidArgument(
|
||||
"static_cast<int>(meta1->OutRankInfo().second) is not 0"));
|
||||
|
||||
// 2. TensorWrapper: No TensorWrapper for ScaleNode
|
||||
// 3. NextEdges: Node 1 -> Node 0
|
||||
const paddle::small_vector<std::vector<GradSlotMeta>,
|
||||
egr::kSlotSmallVectorSize>& node1_metas =
|
||||
grad_node1->OutputMeta();
|
||||
const auto& node1_meta = node1_metas[0];
|
||||
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(node1_meta[0].GetEdge().GetEdgeRankInfo().first),
|
||||
0,
|
||||
common::errors::InvalidArgument(
|
||||
"static_cast<int>(node1_meta[0].GetEdge().GetEdgeRankInfo().first)"
|
||||
"is not 0"));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(node1_meta[0].GetEdge().GetEdgeRankInfo().second),
|
||||
0,
|
||||
common::errors::InvalidArgument(
|
||||
"static_cast<int>(node1_meta[0].GetEdge().GetEdgeRankInfo().second)"
|
||||
"is not 0"));
|
||||
PADDLE_ENFORCE_EQ(node1_meta[0].GetEdge().GetGradNode(),
|
||||
grad_node0,
|
||||
common::errors::InvalidArgument(
|
||||
"node1_meta[0].GetEdge().GetGradNode() "
|
||||
"is not equal with grad_node0, "
|
||||
"the value of grad_node0 is %d "
|
||||
"and node1_meta[0].GetEdge().GetGradNode() is %d",
|
||||
grad_node0,
|
||||
node1_meta[0].GetEdge().GetGradNode()));
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
inp
|
||||
|
|
||||
Node0
|
||||
____|____
|
||||
| |
|
||||
Node1 Node2
|
||||
| |
|
||||
out1 out2
|
||||
*/
|
||||
TEST(Forward, BranchedNodes) {
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
// Prepare Inputs
|
||||
std::vector<paddle::Tensor> target_tensors;
|
||||
phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
|
||||
|
||||
// Create Target Tensor
|
||||
paddle::Tensor t = eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
5.0 /*value*/,
|
||||
false /*is_leaf*/);
|
||||
target_tensors.emplace_back(std::move(t));
|
||||
paddle::Tensor& tensor = target_tensors[0];
|
||||
EagerUtils::autograd_meta(&tensor)->SetStopGradient(false);
|
||||
|
||||
// Run Forward Node 0
|
||||
float scale0 = 2.0;
|
||||
float bias0 = 3.0;
|
||||
paddle::Tensor out0 = egr::scale(tensor,
|
||||
scale0,
|
||||
bias0,
|
||||
true /*bias_after_scale*/,
|
||||
true /*trace_backward*/);
|
||||
|
||||
// Run Forward Node 1
|
||||
float scale1 = 5.0;
|
||||
float bias1 = 10.0;
|
||||
paddle::Tensor out1 = egr::scale(
|
||||
out0, scale1, bias1, true /*bias_after_scale*/, true /*trace_backward*/);
|
||||
|
||||
// Run Forward Node 2
|
||||
float scale2 = 10.0;
|
||||
float bias2 = 20.0;
|
||||
paddle::Tensor out2 = egr::scale(
|
||||
out0, scale2, bias2, true /*bias_after_scale*/, true /*trace_backward*/);
|
||||
|
||||
// Examine Forward Output 0
|
||||
eager_test::CompareTensorWithValue<float>(out0, 13.0);
|
||||
|
||||
// Examine Forward Output 1
|
||||
eager_test::CompareTensorWithValue<float>(out1, 75.0);
|
||||
|
||||
// Examine Forward Output 2
|
||||
eager_test::CompareTensorWithValue<float>(out2, 150.0);
|
||||
|
||||
// Examine GradNode
|
||||
{
|
||||
// 1. GradNode
|
||||
// Node 0
|
||||
AutogradMeta* meta0 = EagerUtils::autograd_meta(&out0);
|
||||
GradNodeBase* grad_node0 = meta0->GradNode();
|
||||
GradNodeScale* scale_node0 = dynamic_cast<GradNodeScale*>(grad_node0);
|
||||
|
||||
CHECK_NOTNULL(scale_node0);
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(meta0->OutRankInfo().first),
|
||||
0,
|
||||
common::errors::InvalidArgument(
|
||||
"static_cast<int>(meta0->OutRankInfo().first) is not 0"));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(meta0->OutRankInfo().second),
|
||||
0,
|
||||
common::errors::InvalidArgument(
|
||||
"static_cast<int>(meta0->OutRankInfo().second) is not 0"));
|
||||
|
||||
// Node 1
|
||||
AutogradMeta* meta1 = EagerUtils::autograd_meta(&out1);
|
||||
GradNodeBase* grad_node1 = meta1->GradNode();
|
||||
GradNodeScale* scale_node1 = dynamic_cast<GradNodeScale*>(grad_node1);
|
||||
|
||||
CHECK_NOTNULL(scale_node1);
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(meta1->OutRankInfo().first),
|
||||
0,
|
||||
common::errors::InvalidArgument(
|
||||
"static_cast<int>(meta1->OutRankInfo().first) is not 0"));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(meta1->OutRankInfo().second),
|
||||
0,
|
||||
common::errors::InvalidArgument(
|
||||
"static_cast<int>(meta1->OutRankInfo().second) is not 0"));
|
||||
|
||||
// Node 2
|
||||
AutogradMeta* meta2 = EagerUtils::autograd_meta(&out2);
|
||||
GradNodeBase* grad_node2 = meta2->GradNode();
|
||||
GradNodeScale* scale_node2 = dynamic_cast<GradNodeScale*>(grad_node2);
|
||||
|
||||
CHECK_NOTNULL(scale_node2);
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(meta2->OutRankInfo().first),
|
||||
0,
|
||||
common::errors::InvalidArgument(
|
||||
"static_cast<int>(meta2->OutRankInfo().first) is not 0"));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(meta2->OutRankInfo().second),
|
||||
0,
|
||||
common::errors::InvalidArgument(
|
||||
"static_cast<int>(meta2->OutRankInfo().second) is not 0"));
|
||||
|
||||
// 2. TensorWrapper: No TensorWrapper for ScaleNode
|
||||
// 3. NextEdges
|
||||
// Node 1 -> Node 0
|
||||
const paddle::small_vector<std::vector<GradSlotMeta>, kSlotSmallVectorSize>&
|
||||
node1_metas = grad_node1->OutputMeta();
|
||||
const Edge& node1_edge = node1_metas[0][0].GetEdge();
|
||||
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(node1_edge.GetEdgeRankInfo().first),
|
||||
0,
|
||||
common::errors::InvalidArgument(
|
||||
"static_cast<int>(node1_edge.GetEdgeRankInfo().first) is not 0"));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(node1_edge.GetEdgeRankInfo().second),
|
||||
0,
|
||||
common::errors::InvalidArgument(
|
||||
"static_cast<int>(node1_edge.GetEdgeRankInfo().second) is not 0"));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
node1_edge.GetGradNode(),
|
||||
grad_node0,
|
||||
common::errors::InvalidArgument(
|
||||
"node1_edge.GetGradNode() is not equal with grad_node0"
|
||||
"the value of node1_edge.GetGradNode() is %d and grad_node0 is %d",
|
||||
node1_edge.GetGradNode(),
|
||||
grad_node0));
|
||||
|
||||
// Node 2 -> Node 0
|
||||
const paddle::small_vector<std::vector<egr::GradSlotMeta>,
|
||||
egr::kSlotSmallVectorSize>& node2_metas =
|
||||
grad_node2->OutputMeta();
|
||||
const Edge& node2_edge = node2_metas[0][0].GetEdge();
|
||||
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(node2_edge.GetEdgeRankInfo().first),
|
||||
0,
|
||||
common::errors::InvalidArgument(
|
||||
"static_cast<int>(node2_edge.GetEdgeRankInfo().first) is not 0"));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
static_cast<int>(node2_edge.GetEdgeRankInfo().second),
|
||||
0,
|
||||
common::errors::InvalidArgument(
|
||||
"static_cast<int>(node2_edge.GetEdgeRankInfo().second) is not 0"));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
node2_edge.GetGradNode(),
|
||||
grad_node0,
|
||||
common::errors::InvalidArgument(
|
||||
"node2_edge.GetGradNode() is not equal with grad_node0"
|
||||
"the value of node2_edge.GetGradNode() is %d and grad_node0 is %d",
|
||||
node2_edge.GetGradNode(),
|
||||
grad_node0));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace egr
|
||||
@@ -0,0 +1,457 @@
|
||||
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <sstream>
|
||||
|
||||
#include "glog/logging.h"
|
||||
#include "gtest/gtest.h"
|
||||
#include "paddle/fluid/eager/accumulation/accumulation_node.h"
|
||||
#include "paddle/fluid/eager/api/all.h"
|
||||
#include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h"
|
||||
#include "paddle/fluid/eager/autograd_meta.h"
|
||||
#include "paddle/fluid/eager/backward.h"
|
||||
#include "paddle/fluid/eager/grad_node_info.h"
|
||||
#include "paddle/fluid/eager/hooks.h"
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/core/tensor_meta.h"
|
||||
#include "test/cpp/eager/test_utils.h"
|
||||
|
||||
PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
|
||||
PD_DECLARE_KERNEL(add, CPU, ALL_LAYOUT);
|
||||
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
|
||||
PD_DECLARE_KERNEL(full, GPU, ALL_LAYOUT);
|
||||
PD_DECLARE_KERNEL(add, KPS, ALL_LAYOUT);
|
||||
#endif
|
||||
|
||||
namespace egr {
|
||||
|
||||
paddle::Tensor hook_function(const paddle::Tensor& t) {
|
||||
auto t_dense = std::dynamic_pointer_cast<phi::DenseTensor>(t.impl());
|
||||
|
||||
auto ret_meta = phi::DenseTensorMeta(
|
||||
t_dense->dtype(), t_dense->dims(), t_dense->layout());
|
||||
auto place = t_dense->place();
|
||||
size_t bytes_size =
|
||||
common::product(t_dense->dims()) * SizeOf(t_dense->dtype());
|
||||
auto ret_dense = std::make_shared<phi::DenseTensor>(
|
||||
paddle::memory::Alloc(place, bytes_size), std::move(ret_meta));
|
||||
|
||||
float* t_ptr = t_dense->mutable_data<float>(place);
|
||||
float* ret_ptr = ret_dense->mutable_data<float>(place);
|
||||
for (int i = 0; i < ret_dense->numel(); i++) {
|
||||
ret_ptr[i] = t_ptr[i] + 5.0f;
|
||||
}
|
||||
|
||||
auto ret_impl = std::dynamic_pointer_cast<phi::TensorBase>(ret_dense);
|
||||
paddle::Tensor ret = paddle::Tensor();
|
||||
ret.set_impl(ret_impl);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
TEST(FwdBwdJoint, SingleNode) {
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
// 1. Prepare Input
|
||||
phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
|
||||
paddle::Tensor tensor =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
5.0 /*value*/,
|
||||
true /*is_leaf*/);
|
||||
egr_utils_api::RetainGradForTensor(tensor);
|
||||
|
||||
// 3. Run Forward
|
||||
float scale = 2.0;
|
||||
float bias = 3.0;
|
||||
paddle::Tensor out = egr::scale(
|
||||
tensor, scale, bias, true /*bias_after_scale*/, true /*trace_backward*/);
|
||||
|
||||
// Examine Forward Output
|
||||
eager_test::CompareTensorWithValue<float>(out, 13.0);
|
||||
|
||||
std::vector<paddle::Tensor> outs = {out};
|
||||
// 4. Run Backward
|
||||
Backward(outs, {});
|
||||
|
||||
VLOG(7) << "Target Grad is: "
|
||||
<< std::static_pointer_cast<phi::DenseTensor>(
|
||||
EagerUtils::unsafe_autograd_meta(tensor)->Grad().impl())
|
||||
->data<float>()[0];
|
||||
// Examine Backward Grad
|
||||
eager_test::CompareGradTensorWithValue<float>(tensor, 2.0);
|
||||
}
|
||||
|
||||
/*
|
||||
inp
|
||||
|
|
||||
Node0
|
||||
|
|
||||
Node1
|
||||
|
|
||||
out
|
||||
*/
|
||||
TEST(FwdBwdJoint, LinearNodes) {
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
// 1. Prepare Input
|
||||
phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
|
||||
paddle::Tensor tensor =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
5.0 /*value*/,
|
||||
true /*is_leaf*/);
|
||||
egr_utils_api::RetainGradForTensor(tensor);
|
||||
|
||||
// 3. Run Forward
|
||||
// Run Forward Node 0
|
||||
float scale0 = 2.0;
|
||||
float bias0 = 3.0;
|
||||
paddle::Tensor out0 = egr::scale(tensor,
|
||||
scale0,
|
||||
bias0,
|
||||
true /*bias_after_scale*/,
|
||||
true /*trace_backward*/);
|
||||
|
||||
// Run Forward Node 1
|
||||
float scale1 = 5.0;
|
||||
float bias1 = 10.0;
|
||||
paddle::Tensor out1 = egr::scale(
|
||||
out0, scale1, bias1, true /*bias_after_scale*/, true /*trace_backward*/);
|
||||
|
||||
// Examine Forward Output 0
|
||||
eager_test::CompareTensorWithValue<float>(out0, 13.0);
|
||||
|
||||
// Examine Forward Output 1
|
||||
eager_test::CompareTensorWithValue<float>(out1, 75.0);
|
||||
|
||||
std::vector<paddle::Tensor> outs = {out1};
|
||||
// 4. Run Backward
|
||||
Backward(outs, {});
|
||||
|
||||
// Examine Backward Grad
|
||||
eager_test::CompareGradTensorWithValue<float>(tensor, 10.0);
|
||||
}
|
||||
|
||||
/*
|
||||
inp
|
||||
|
|
||||
Node0
|
||||
____|____
|
||||
| |
|
||||
Node1 Node2
|
||||
| |
|
||||
out1 out2
|
||||
*/
|
||||
TEST(FwdBwdJoint, BranchedNodes) {
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
// 1. Prepare Input
|
||||
phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
|
||||
paddle::Tensor tensor =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
5.0 /*value*/,
|
||||
true /*is_leaf*/);
|
||||
egr_utils_api::RetainGradForTensor(tensor);
|
||||
|
||||
// 3. Run Forward
|
||||
// Run Forward Node 0
|
||||
float scale0 = 2.0;
|
||||
float bias0 = 3.0;
|
||||
paddle::Tensor out0 = egr::scale(tensor,
|
||||
scale0,
|
||||
bias0,
|
||||
true /*bias_after_scale*/,
|
||||
true /*trace_backward*/);
|
||||
|
||||
// Run Forward Node 1
|
||||
float scale1 = 5.0;
|
||||
float bias1 = 10.0;
|
||||
paddle::Tensor out1 = egr::scale(
|
||||
out0, scale1, bias1, true /*bias_after_scale*/, true /*trace_backward*/);
|
||||
|
||||
// Run Forward Node 2
|
||||
float scale2 = 10.0;
|
||||
float bias2 = 20.0;
|
||||
paddle::Tensor out2 = egr::scale(
|
||||
out0, scale2, bias2, true /*bias_after_scale*/, true /*trace_backward*/);
|
||||
|
||||
// Examine Forward Output 0
|
||||
eager_test::CompareTensorWithValue<float>(out0, 13.0);
|
||||
|
||||
// Examine Forward Output 1
|
||||
eager_test::CompareTensorWithValue<float>(out1, 75.0);
|
||||
|
||||
// Examine Forward Output 2
|
||||
{
|
||||
auto dense_out = std::dynamic_pointer_cast<phi::DenseTensor>(out2.impl());
|
||||
float* ptr = dense_out->mutable_data<float>(phi::CPUPlace());
|
||||
for (int i = 0; i < 20; i++) {
|
||||
PADDLE_ENFORCE(ptr[i] == 150.0,
|
||||
common::errors::Fatal(
|
||||
"Detected numerical Error, Expected %f but got %f",
|
||||
150.0,
|
||||
ptr[i]));
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Run Backward
|
||||
std::vector<paddle::Tensor> outs = {out1, out2};
|
||||
Backward(outs, {});
|
||||
|
||||
// Examine Backward Grad
|
||||
eager_test::CompareGradTensorWithValue<float>(tensor, 30.0);
|
||||
}
|
||||
|
||||
/*
|
||||
inp
|
||||
|
|
||||
Node0
|
||||
____|____
|
||||
| |
|
||||
Node1 Node2
|
||||
| |
|
||||
out1 out2
|
||||
*/
|
||||
TEST(FwdBwdJoint, GradientHook) {
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
// 1. Prepare Input
|
||||
phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
|
||||
paddle::Tensor tensor =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
5.0 /*value*/,
|
||||
true /*is_leaf*/);
|
||||
egr_utils_api::RetainGradForTensor(tensor);
|
||||
|
||||
// 3. Run Forward
|
||||
// Run Forward Node 0
|
||||
float scale0 = 2.0;
|
||||
float bias0 = 3.0;
|
||||
paddle::Tensor out0 = egr::scale(tensor,
|
||||
scale0,
|
||||
bias0,
|
||||
true /*bias_after_scale*/,
|
||||
true /*trace_backward*/);
|
||||
egr_utils_api::RetainGradForTensor(out0); // hook: +5
|
||||
egr_utils_api::RegisterGradientHookForTensor(out0,
|
||||
hook_function); // hook: +5
|
||||
|
||||
// Run Forward Node 1
|
||||
float scale1 = 5.0;
|
||||
float bias1 = 10.0;
|
||||
paddle::Tensor out1 = egr::scale(
|
||||
out0, scale1, bias1, true /*bias_after_scale*/, true /*trace_backward*/);
|
||||
egr_utils_api::RetainGradForTensor(out1); // hook: +5
|
||||
egr_utils_api::RegisterGradientHookForTensor(out1,
|
||||
hook_function); // hook: +5
|
||||
|
||||
// Run Forward Node 2
|
||||
float scale2 = 10.0;
|
||||
float bias2 = 20.0;
|
||||
paddle::Tensor out2 = egr::scale(
|
||||
out0, scale2, bias2, true /*bias_after_scale*/, true /*trace_backward*/);
|
||||
egr_utils_api::RetainGradForTensor(out2); // hook: +5
|
||||
egr_utils_api::RegisterGradientHookForTensor(out2,
|
||||
hook_function); // hook: +5
|
||||
|
||||
// 4. Run Backward
|
||||
std::vector<paddle::Tensor> outs = {out1, out2};
|
||||
Backward(outs, {});
|
||||
|
||||
// Examine Backward Grad
|
||||
// leaf grad
|
||||
eager_test::CompareGradTensorWithValue<float>(tensor, 190.0);
|
||||
|
||||
// out0 grad
|
||||
eager_test::CompareGradTensorWithValue<float>(out0, 90.0);
|
||||
|
||||
// out1 grad
|
||||
eager_test::CompareGradTensorWithValue<float>(out1, 1.0);
|
||||
|
||||
// out2 grad
|
||||
eager_test::CompareGradTensorWithValue<float>(out2, 1.0);
|
||||
}
|
||||
|
||||
/*
|
||||
inp
|
||||
|
|
||||
Node0
|
||||
____|____
|
||||
| |
|
||||
Node1 Node2
|
||||
| |
|
||||
out1 out2
|
||||
*/
|
||||
TEST(FwdBwdJoint, CrossBatchAccumulation) {
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
// 1. Prepare Input
|
||||
phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
|
||||
paddle::Tensor tensor =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
5.0 /*value*/,
|
||||
true /*is_leaf*/);
|
||||
egr_utils_api::RetainGradForTensor(tensor);
|
||||
|
||||
// 3. Run Forward
|
||||
// Run Forward Node 0
|
||||
float scale0 = 2.0;
|
||||
float bias0 = 3.0;
|
||||
paddle::Tensor out0 = egr::scale(tensor,
|
||||
scale0,
|
||||
bias0,
|
||||
true /*bias_after_scale*/,
|
||||
true /*trace_backward*/);
|
||||
|
||||
// Run Forward Node 1
|
||||
float scale1 = 5.0;
|
||||
float bias1 = 10.0;
|
||||
paddle::Tensor out1 = egr::scale(
|
||||
out0, scale1, bias1, true /*bias_after_scale*/, true /*trace_backward*/);
|
||||
|
||||
// Run Forward Node 2
|
||||
float scale2 = 10.0;
|
||||
float bias2 = 20.0;
|
||||
paddle::Tensor out2 = egr::scale(
|
||||
out0, scale2, bias2, true /*bias_after_scale*/, true /*trace_backward*/);
|
||||
|
||||
// 4. Run Backward
|
||||
std::vector<paddle::Tensor> outs = {out1, out2};
|
||||
Backward(outs, {});
|
||||
|
||||
// Examine Backward Grad
|
||||
eager_test::CompareGradTensorWithValue<float>(tensor, 30.0);
|
||||
|
||||
// Cross Batch Accumulation
|
||||
Backward(outs, {});
|
||||
|
||||
// Examine Backward Grad
|
||||
eager_test::CompareGradTensorWithValue<float>(tensor, 60.0);
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------- */
|
||||
/* ---------------------- CUDA Tests ------------------ */
|
||||
/* ---------------------------------------------------- */
|
||||
|
||||
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
|
||||
TEST(FwdBwdJoint, SingleNodeCUDA) {
|
||||
eager_test::InitEnv(phi::GPUPlace());
|
||||
|
||||
// 1. Prepare Input
|
||||
phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
|
||||
paddle::Tensor tensor =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::GPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
5.0 /*value*/,
|
||||
true /*is_leaf*/);
|
||||
egr_utils_api::RetainGradForTensor(tensor);
|
||||
|
||||
// 3. Run Forward
|
||||
float scale = 2.0;
|
||||
float bias = 3.0;
|
||||
paddle::Tensor out = egr::scale(
|
||||
tensor, scale, bias, true /*bias_after_scale*/, true /*trace_backward*/);
|
||||
|
||||
// Examine Forward Output
|
||||
eager_test::CompareTensorWithValue<float>(out, 13.0);
|
||||
|
||||
std::vector<paddle::Tensor> outs = {out};
|
||||
// 4. Run Backward
|
||||
Backward(outs, {});
|
||||
|
||||
// Examine Backward Grad
|
||||
eager_test::CompareGradTensorWithValue<float>(tensor, 2.0);
|
||||
}
|
||||
|
||||
/*
|
||||
inp
|
||||
|
|
||||
Node0
|
||||
____|____
|
||||
| |
|
||||
Node1 Node2
|
||||
| |
|
||||
out1 out2
|
||||
*/
|
||||
TEST(FwdBwdJoint, BranchedNodesCUDA) {
|
||||
eager_test::InitEnv(phi::GPUPlace());
|
||||
|
||||
// 1. Prepare Input
|
||||
phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
|
||||
paddle::Tensor tensor =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::GPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
5.0 /*value*/,
|
||||
true /*is_leaf*/);
|
||||
egr_utils_api::RetainGradForTensor(tensor);
|
||||
|
||||
// 3. Run Forward
|
||||
// Run Forward Node 0
|
||||
float scale0 = 2.0;
|
||||
float bias0 = 3.0;
|
||||
paddle::Tensor out0 = egr::scale(tensor,
|
||||
scale0,
|
||||
bias0,
|
||||
true /*bias_after_scale*/,
|
||||
true /*trace_backward*/);
|
||||
|
||||
// Run Forward Node 1
|
||||
float scale1 = 5.0;
|
||||
float bias1 = 10.0;
|
||||
paddle::Tensor out1 = egr::scale(
|
||||
out0, scale1, bias1, true /*bias_after_scale*/, true /*trace_backward*/);
|
||||
|
||||
// Run Forward Node 2
|
||||
float scale2 = 10.0;
|
||||
float bias2 = 20.0;
|
||||
paddle::Tensor out2 = egr::scale(
|
||||
out0, scale2, bias2, true /*bias_after_scale*/, true /*trace_backward*/);
|
||||
|
||||
// Examine Forward Output 0
|
||||
eager_test::CompareTensorWithValue<float>(out0, 13.0);
|
||||
// Examine Forward Output 1
|
||||
eager_test::CompareTensorWithValue<float>(out1, 75.0);
|
||||
// Examine Forward Output 2
|
||||
eager_test::CompareTensorWithValue<float>(out2, 150.0);
|
||||
|
||||
// TODO(jiabin): fix this with add functor
|
||||
// 4. Run Backward
|
||||
std::vector<paddle::Tensor> outs = {out1, out2};
|
||||
Backward(outs, {});
|
||||
|
||||
// Examine Backward Grad
|
||||
eager_test::CompareGradTensorWithValue<float>(tensor, 30.0);
|
||||
}
|
||||
#endif
|
||||
|
||||
} // namespace egr
|
||||
@@ -0,0 +1,141 @@
|
||||
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Eager Dygraph
|
||||
|
||||
#include <chrono>
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "paddle/fluid/eager/api/all.h"
|
||||
#include "paddle/fluid/eager/api/generated/fluid_generated/dygraph_forward_api.h"
|
||||
#include "paddle/fluid/eager/autograd_meta.h"
|
||||
#include "paddle/fluid/eager/backward.h"
|
||||
#include "paddle/fluid/eager/utils.h"
|
||||
#include "paddle/fluid/imperative/tracer.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "test/cpp/eager/test_utils.h"
|
||||
|
||||
PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
|
||||
PD_DECLARE_KERNEL(matmul, CPU, ALL_LAYOUT);
|
||||
PD_DECLARE_KERNEL(matmul_grad, CPU, ALL_LAYOUT);
|
||||
PD_DECLARE_KERNEL(add, CPU, ALL_LAYOUT);
|
||||
PD_DECLARE_KERNEL(add_grad, CPU, ALL_LAYOUT);
|
||||
|
||||
namespace egr {
|
||||
|
||||
TEST(Generated, Sigmoid) {
|
||||
// Prepare Device Contexts
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
VLOG(6) << "Init Env";
|
||||
// 1. Prepare Input
|
||||
phi::DDim ddim = common::make_ddim({2, 4, 4, 4});
|
||||
VLOG(6) << "Make Dim";
|
||||
paddle::Tensor tensor =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
0.0,
|
||||
true);
|
||||
VLOG(6) << "Make paddle::Tensor";
|
||||
egr_utils_api::RetainGradForTensor(tensor);
|
||||
VLOG(6) << "Retain Grad for Tensor";
|
||||
auto output_tensor = sigmoid_dygraph_function(tensor, {});
|
||||
VLOG(6) << "Run Backward";
|
||||
eager_test::CompareTensorWithValue<float>(output_tensor, 0.5);
|
||||
|
||||
std::vector<paddle::Tensor> target_tensors = {output_tensor};
|
||||
VLOG(6) << "Running Backward";
|
||||
Backward(target_tensors, {});
|
||||
|
||||
VLOG(6) << "Finish Backward";
|
||||
eager_test::CompareGradTensorWithValue<float>(tensor, 0.25);
|
||||
}
|
||||
|
||||
TEST(Generated, Matmul_v2) {
|
||||
// Prepare Device Contexts
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
auto tracer = std::make_shared<paddle::imperative::Tracer>();
|
||||
paddle::imperative::SetCurrentTracer(tracer);
|
||||
|
||||
// 1. Prepare Input
|
||||
phi::DDim ddimX = common::make_ddim({4, 16});
|
||||
paddle::Tensor X = eager_test::CreateTensorWithValue(ddimX,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
3.0,
|
||||
true);
|
||||
egr_utils_api::RetainGradForTensor(X);
|
||||
|
||||
phi::DDim ddimY = common::make_ddim({16, 20});
|
||||
paddle::Tensor Y = eager_test::CreateTensorWithValue(ddimY,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
2.0,
|
||||
true);
|
||||
egr_utils_api::RetainGradForTensor(Y);
|
||||
|
||||
auto output_tensor = matmul_v2_dygraph_function(
|
||||
X, Y, {{"trans_x", false}, {"trans_y", false}});
|
||||
|
||||
eager_test::CompareTensorWithValue<float>(output_tensor, 96);
|
||||
|
||||
std::vector<paddle::Tensor> target_tensors = {output_tensor};
|
||||
Backward(target_tensors, {});
|
||||
|
||||
eager_test::CompareGradTensorWithValue<float>(X, 2.0 * 20);
|
||||
eager_test::CompareGradTensorWithValue<float>(Y, 3.0 * 4);
|
||||
}
|
||||
|
||||
TEST(Generated, ElementwiseAdd) {
|
||||
// Prepare Device Contexts
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
auto tracer = std::make_shared<paddle::imperative::Tracer>();
|
||||
paddle::imperative::SetCurrentTracer(tracer);
|
||||
|
||||
// 1. Prepare Input
|
||||
phi::DDim ddimX = common::make_ddim({4, 16});
|
||||
paddle::Tensor X = eager_test::CreateTensorWithValue(ddimX,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
3.0,
|
||||
true);
|
||||
egr_utils_api::RetainGradForTensor(X);
|
||||
|
||||
phi::DDim ddimY = common::make_ddim({4, 16});
|
||||
paddle::Tensor Y = eager_test::CreateTensorWithValue(ddimY,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
2.0,
|
||||
true);
|
||||
egr_utils_api::RetainGradForTensor(Y);
|
||||
|
||||
auto output_tensor = elementwise_add_dygraph_function(X, Y, {});
|
||||
|
||||
eager_test::CompareTensorWithValue<float>(output_tensor, 5);
|
||||
|
||||
std::vector<paddle::Tensor> target_tensors = {output_tensor};
|
||||
Backward(target_tensors, {});
|
||||
|
||||
eager_test::CompareGradTensorWithValue<float>(X, 1.0);
|
||||
eager_test::CompareGradTensorWithValue<float>(Y, 1.0);
|
||||
}
|
||||
|
||||
} // namespace egr
|
||||
@@ -0,0 +1,371 @@
|
||||
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <sstream>
|
||||
|
||||
#include "glog/logging.h"
|
||||
#include "gtest/gtest.h"
|
||||
#include "paddle/fluid/eager/accumulation/accumulation_node.h"
|
||||
#include "paddle/fluid/eager/api/all.h"
|
||||
#include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h"
|
||||
#include "paddle/fluid/eager/autograd_meta.h"
|
||||
#include "paddle/fluid/eager/backward.h"
|
||||
#include "paddle/fluid/eager/grad_node_info.h"
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/core/tensor_meta.h"
|
||||
#include "test/cpp/eager/test_utils.h"
|
||||
|
||||
PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
|
||||
PD_DECLARE_KERNEL(add, CPU, ALL_LAYOUT);
|
||||
|
||||
namespace egr {
|
||||
|
||||
TEST(Grad, SingleNodeEmptyGrad) {
|
||||
// Prepare Device Contexts
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
// Prepare Inputs
|
||||
phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
|
||||
|
||||
// Create Target Tensor (output)
|
||||
paddle::Tensor output_tensor =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
1.0 /*value*/,
|
||||
false /*is_leaf*/);
|
||||
|
||||
// Create input tensor
|
||||
const paddle::Tensor leaf_tensor =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
1.0 /*value*/,
|
||||
true /*is_leaf*/);
|
||||
|
||||
{
|
||||
// Create Scale Node
|
||||
auto node0_ptr = std::make_shared<GradNodeScale>(1, 1);
|
||||
node0_ptr->SetAttributes_scale(5.0 /*scale*/);
|
||||
|
||||
// Set grad in/out meta
|
||||
node0_ptr->SetDefaultGradInOutMeta();
|
||||
|
||||
// Output_tensor set GradNode、OutRank、StopGradient properties
|
||||
AutogradMeta* auto_grad_meta = EagerUtils::autograd_meta(&output_tensor);
|
||||
auto_grad_meta->SetGradNode(
|
||||
std::dynamic_pointer_cast<GradNodeBase>(node0_ptr));
|
||||
auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
|
||||
auto_grad_meta->SetStopGradient(false);
|
||||
|
||||
// Get autograd_meta from input tensor
|
||||
AutogradMeta* auto_grad_meta1 =
|
||||
EagerUtils::unsafe_autograd_meta(leaf_tensor);
|
||||
|
||||
// Connect Tensor and AccumulationNode via AutoGradMeta
|
||||
auto acc_node_ptr =
|
||||
std::make_shared<egr::GradNodeAccumulation>(leaf_tensor);
|
||||
|
||||
// input tensor set GradNode、OutRank、StopGradient properties
|
||||
auto_grad_meta1->SetGradNode(
|
||||
std::dynamic_pointer_cast<GradNodeBase>(acc_node_ptr));
|
||||
auto_grad_meta1->SetSingleOutRankWithSlot(0, 0);
|
||||
auto_grad_meta1->SetStopGradient(false);
|
||||
|
||||
// grad_node Add Edges
|
||||
std::vector<egr::AutogradMeta*> res = {auto_grad_meta1};
|
||||
node0_ptr->SetGradOutMeta(leaf_tensor, 0);
|
||||
}
|
||||
std::vector<paddle::Tensor> outs = {output_tensor};
|
||||
|
||||
// Run Grad
|
||||
auto result = Grad(outs, {leaf_tensor}, {});
|
||||
// Check Output Value
|
||||
eager_test::CompareTensorWithValue<float>(result[0], 5.0);
|
||||
}
|
||||
|
||||
TEST(Grad, SingleNodeCustomGrad) {
|
||||
// Prepare Device Contexts
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
// Prepare Inputs
|
||||
std::vector<paddle::Tensor> target_tensors;
|
||||
phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
|
||||
|
||||
// Create Target Tensor
|
||||
paddle::Tensor tensor =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
1.0 /*value*/,
|
||||
false /*is_leaf*/);
|
||||
target_tensors.emplace_back(std::move(tensor));
|
||||
|
||||
std::vector<paddle::Tensor> grad_tensors;
|
||||
// Create Grad Tensor
|
||||
paddle::Tensor grad_tensor =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
10.0 /*value*/,
|
||||
false /*is_leaf*/);
|
||||
grad_tensors.emplace_back(std::move(grad_tensor));
|
||||
|
||||
paddle::Tensor leaf_tensor =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
1.0 /*value*/,
|
||||
true /*is_leaf*/);
|
||||
|
||||
{
|
||||
// Create Scale Node
|
||||
auto node0_ptr = std::make_shared<GradNodeScale>(1, 1);
|
||||
node0_ptr->SetAttributes_scale(5.0 /*scale*/);
|
||||
|
||||
// Set grad in/out meta
|
||||
node0_ptr->SetDefaultGradInOutMeta();
|
||||
|
||||
// Connect Tensor and Node via AutoGradMeta
|
||||
AutogradMeta* auto_grad_meta =
|
||||
EagerUtils::autograd_meta(&(target_tensors[0]));
|
||||
auto_grad_meta->SetGradNode(
|
||||
std::dynamic_pointer_cast<GradNodeBase>(node0_ptr));
|
||||
auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
|
||||
auto_grad_meta->SetStopGradient(false);
|
||||
|
||||
AutogradMeta* auto_grad_meta1 = EagerUtils::autograd_meta(&leaf_tensor);
|
||||
// Connect Tensor and AccumulationNode via AutoGradMeta
|
||||
auto acc_node_ptr =
|
||||
std::make_shared<egr::GradNodeAccumulation>(leaf_tensor);
|
||||
|
||||
auto_grad_meta1->SetGradNode(
|
||||
std::dynamic_pointer_cast<GradNodeBase>(acc_node_ptr));
|
||||
auto_grad_meta1->SetSingleOutRankWithSlot(0, 0);
|
||||
auto_grad_meta1->SetStopGradient(false);
|
||||
std::vector<egr::AutogradMeta*> res = {auto_grad_meta1};
|
||||
node0_ptr->SetGradOutMeta(leaf_tensor, 0);
|
||||
}
|
||||
|
||||
auto result = Grad(target_tensors, {leaf_tensor}, grad_tensors);
|
||||
|
||||
// Check Output Value
|
||||
eager_test::CompareTensorWithValue<float>(result[0], 50.0);
|
||||
}
|
||||
|
||||
/*
|
||||
Node1
|
||||
|
|
||||
Node0
|
||||
|
|
||||
{ } // empty grad tensor
|
||||
*/
|
||||
TEST(Grad, LinearNodes) {
|
||||
// Prepare Device Contexts
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
// Prepare Target Tensor
|
||||
std::vector<paddle::Tensor> target_tensors;
|
||||
phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
|
||||
|
||||
// Create Target Tensor
|
||||
paddle::Tensor tensor =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
1.0 /*value*/,
|
||||
false /*is_leaf*/);
|
||||
target_tensors.emplace_back(std::move(tensor));
|
||||
|
||||
paddle::Tensor leaf_tensor =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
1.0 /*value*/,
|
||||
true /*is_leaf*/);
|
||||
{
|
||||
// Create Node0
|
||||
auto node0_ptr = std::make_shared<GradNodeScale>(1, 1);
|
||||
node0_ptr->SetAttributes_scale(5.0 /*scale*/);
|
||||
|
||||
// Set grad in/out meta for node0
|
||||
node0_ptr->SetDefaultGradInOutMeta();
|
||||
|
||||
// Create Node1
|
||||
auto node1_ptr = std::make_shared<GradNodeScale>(1, 1);
|
||||
node1_ptr->SetAttributes_scale(10.0 /*scale*/);
|
||||
|
||||
// Set grad in/out meta for node1
|
||||
node1_ptr->SetDefaultGradInOutMeta();
|
||||
|
||||
// Connect Input Tensor and Node0 via AutoGradMeta
|
||||
AutogradMeta* auto_grad_meta =
|
||||
EagerUtils::autograd_meta(&(target_tensors[0]));
|
||||
auto_grad_meta->SetGradNode(
|
||||
std::dynamic_pointer_cast<GradNodeBase>(node0_ptr));
|
||||
auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
|
||||
auto_grad_meta->SetStopGradient(false);
|
||||
// Connect Node0 -> Node1 via Edge
|
||||
auto tmp_tensor = paddle::Tensor();
|
||||
auto* meta0 = EagerUtils::autograd_meta(&tmp_tensor);
|
||||
meta0->SetStopGradient(false);
|
||||
meta0->SetSingleOutRankWithSlot(0, 0);
|
||||
meta0->SetGradNode(node1_ptr);
|
||||
node0_ptr->SetGradOutMeta(tmp_tensor, 0);
|
||||
|
||||
AutogradMeta* auto_grad_meta1 = EagerUtils::autograd_meta(&leaf_tensor);
|
||||
// Connect Tensor and AccumulationNode via AutoGradMeta
|
||||
auto acc_node_ptr =
|
||||
std::make_shared<egr::GradNodeAccumulation>(leaf_tensor);
|
||||
|
||||
auto_grad_meta1->SetGradNode(
|
||||
std::dynamic_pointer_cast<GradNodeBase>(acc_node_ptr));
|
||||
auto_grad_meta1->SetSingleOutRankWithSlot(0, 0);
|
||||
|
||||
auto_grad_meta1->SetStopGradient(false);
|
||||
node1_ptr->SetGradOutMeta(leaf_tensor, 0);
|
||||
}
|
||||
|
||||
// Use Empty Grad Tensor
|
||||
auto result = Grad(target_tensors, {leaf_tensor}, {});
|
||||
|
||||
// Check Output Value
|
||||
eager_test::CompareTensorWithValue<float>(result[0], 50.0);
|
||||
}
|
||||
|
||||
/*
|
||||
Node2
|
||||
| |
|
||||
Node0 Node1
|
||||
| |
|
||||
in0 in1
|
||||
*/
|
||||
TEST(Grad, WithAccumulation) {
|
||||
// Prepare Device Contexts
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
// Prepare Inputs
|
||||
phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
|
||||
|
||||
// Create Target Tensor
|
||||
std::vector<paddle::Tensor> target_tensors;
|
||||
paddle::Tensor tensor0 =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
1.0 /*value*/,
|
||||
false /*is_leaf*/);
|
||||
paddle::Tensor tensor1 =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
1.0 /*value*/,
|
||||
false /*is_leaf*/);
|
||||
target_tensors.emplace_back(std::move(tensor0));
|
||||
target_tensors.emplace_back(std::move(tensor1));
|
||||
|
||||
// Create Grad Tensor
|
||||
std::vector<paddle::Tensor> grad_tensors;
|
||||
paddle::Tensor grad_tensor0 =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
5.0 /*value*/,
|
||||
false /*is_leaf*/);
|
||||
paddle::Tensor grad_tensor1 =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
10.0 /*value*/,
|
||||
false /*is_leaf*/);
|
||||
grad_tensors.emplace_back(std::move(grad_tensor0));
|
||||
grad_tensors.emplace_back(std::move(grad_tensor1));
|
||||
|
||||
paddle::Tensor leaf_tensor;
|
||||
{
|
||||
// Create Node0
|
||||
auto node0_ptr = std::make_shared<GradNodeScale>(1, 1);
|
||||
node0_ptr->SetAttributes_scale(5.0 /*scale*/);
|
||||
node0_ptr->SetDefaultGradInOutMeta();
|
||||
|
||||
// Create Node1
|
||||
auto node1_ptr = std::make_shared<GradNodeScale>(1, 1);
|
||||
node1_ptr->SetAttributes_scale(10.0 /*scale*/);
|
||||
node1_ptr->SetDefaultGradInOutMeta();
|
||||
// Create Node2
|
||||
auto node2_ptr = std::make_shared<GradNodeScale>(1, 1);
|
||||
node2_ptr->SetAttributes_scale(20.0 /*scale*/);
|
||||
node2_ptr->SetDefaultGradInOutMeta();
|
||||
// Connect Inp0 and Node0 via AutoGradMeta
|
||||
AutogradMeta* auto_grad_meta0 =
|
||||
EagerUtils::autograd_meta(&(target_tensors[0]));
|
||||
auto_grad_meta0->SetGradNode(
|
||||
std::dynamic_pointer_cast<GradNodeBase>(node0_ptr));
|
||||
auto_grad_meta0->SetSingleOutRankWithSlot(0, 0);
|
||||
auto_grad_meta0->SetStopGradient(false);
|
||||
// Connect Inp1 and Node1 via AutoGradMeta
|
||||
AutogradMeta* auto_grad_meta1 =
|
||||
EagerUtils::autograd_meta(&(target_tensors[1]));
|
||||
auto_grad_meta1->SetGradNode(
|
||||
std::dynamic_pointer_cast<GradNodeBase>(node1_ptr));
|
||||
auto_grad_meta1->SetSingleOutRankWithSlot(0, 0);
|
||||
auto_grad_meta1->SetStopGradient(false);
|
||||
|
||||
// Connect Node0 -> Node2 via Edge
|
||||
auto tmp_tensor0 = paddle::Tensor();
|
||||
auto* meta0 = EagerUtils::autograd_meta(&tmp_tensor0);
|
||||
meta0->SetStopGradient(false);
|
||||
meta0->SetSingleOutRankWithSlot(0, 0);
|
||||
meta0->SetGradNode(node2_ptr);
|
||||
node0_ptr->SetGradOutMeta(tmp_tensor0, 0);
|
||||
|
||||
// Connect Node1 -> Node2 via Edge
|
||||
auto tmp_tensor1 = paddle::Tensor();
|
||||
auto meta1 = EagerUtils::autograd_meta(&tmp_tensor1);
|
||||
meta1->SetStopGradient(false);
|
||||
meta1->SetSingleOutRankWithSlot(0, 0);
|
||||
meta1->SetGradNode(node2_ptr);
|
||||
node1_ptr->SetGradOutMeta(tmp_tensor1, 0);
|
||||
|
||||
AutogradMeta* auto_grad_meta2 = EagerUtils::autograd_meta(&leaf_tensor);
|
||||
// Connect Tensor and AccumulationNode via AutoGradMeta
|
||||
auto acc_node_ptr =
|
||||
std::make_shared<egr::GradNodeAccumulation>(leaf_tensor);
|
||||
|
||||
auto_grad_meta2->SetGradNode(
|
||||
std::dynamic_pointer_cast<GradNodeBase>(acc_node_ptr));
|
||||
auto_grad_meta2->SetSingleOutRankWithSlot(0, 0);
|
||||
|
||||
auto_grad_meta2->SetStopGradient(false);
|
||||
node2_ptr->SetGradOutMeta(leaf_tensor, 0);
|
||||
}
|
||||
|
||||
auto result = Grad(target_tensors, {leaf_tensor}, grad_tensors);
|
||||
|
||||
eager_test::CompareTensorWithValue<float>(result[0], 2500.0);
|
||||
}
|
||||
|
||||
} // namespace egr
|
||||
@@ -0,0 +1,200 @@
|
||||
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <sstream>
|
||||
|
||||
#include "glog/logging.h"
|
||||
#include "gtest/gtest.h"
|
||||
#include "paddle/fluid/eager/accumulation/accumulation_node.h"
|
||||
#include "paddle/fluid/eager/api/all.h"
|
||||
#include "paddle/fluid/eager/api/generated/eager_generated/backwards/scale_node.h"
|
||||
#include "paddle/fluid/eager/autograd_meta.h"
|
||||
#include "paddle/fluid/eager/backward.h"
|
||||
#include "paddle/fluid/eager/grad_node_info.h"
|
||||
#include "paddle/fluid/eager/hooks.h"
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "paddle/phi/core/tensor_meta.h"
|
||||
#include "test/cpp/eager/test_utils.h"
|
||||
|
||||
PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
|
||||
|
||||
namespace egr {
|
||||
|
||||
paddle::Tensor hook_function(const paddle::Tensor& t) {
|
||||
auto t_dense = std::dynamic_pointer_cast<phi::DenseTensor>(t.impl());
|
||||
|
||||
auto ret_meta = phi::DenseTensorMeta(
|
||||
t_dense->dtype(), t_dense->dims(), t_dense->layout());
|
||||
auto place = t_dense->place();
|
||||
size_t bytes_size =
|
||||
common::product(t_dense->dims()) * SizeOf(t_dense->dtype());
|
||||
auto ret_dense = std::make_shared<phi::DenseTensor>(
|
||||
paddle::memory::Alloc(place, bytes_size), std::move(ret_meta));
|
||||
|
||||
float* t_ptr = t_dense->mutable_data<float>(place);
|
||||
float* ret_ptr = ret_dense->mutable_data<float>(place);
|
||||
for (int i = 0; i < ret_dense->numel(); i++) {
|
||||
ret_ptr[i] = t_ptr[i] + 3.0f;
|
||||
}
|
||||
|
||||
auto ret_impl = std::dynamic_pointer_cast<phi::TensorBase>(ret_dense);
|
||||
paddle::Tensor ret = paddle::Tensor();
|
||||
ret.set_impl(ret_impl);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
TEST(RetainGrad, HookBeforeRetainGrad) {
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
// Prepare Inputs
|
||||
std::vector<paddle::Tensor> target_tensors;
|
||||
phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
|
||||
|
||||
// Create Target Tensor
|
||||
paddle::Tensor tensor =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
1.0 /*value*/,
|
||||
false /*is_leaf*/);
|
||||
target_tensors.emplace_back(std::move(tensor));
|
||||
paddle::Tensor& target_tensor = target_tensors[0];
|
||||
|
||||
// Create ScaleNode
|
||||
auto scale_node_ptr = std::make_shared<GradNodeScale>(1, 1);
|
||||
scale_node_ptr->SetAttributes_scale(5.0 /*scale*/);
|
||||
|
||||
// Set grad in/out meta for node0
|
||||
scale_node_ptr->SetDefaultGradInOutMeta();
|
||||
|
||||
// Connect Input Tensor and ScaleNode via AutoGradMeta
|
||||
// Apply RetainGrad
|
||||
{
|
||||
// ScaleNode Hook: +3
|
||||
|
||||
auto auto_grad_meta = std::make_shared<AutogradMeta>();
|
||||
auto_grad_meta->SetGradNode(
|
||||
std::dynamic_pointer_cast<GradNodeBase>(scale_node_ptr));
|
||||
auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
|
||||
auto_grad_meta->SetStopGradient(false);
|
||||
target_tensor.set_autograd_meta(
|
||||
std::dynamic_pointer_cast<paddle::AbstractAutogradMeta>(
|
||||
auto_grad_meta));
|
||||
|
||||
egr_utils_api::RegisterGradientHookForTensor(target_tensor, hook_function);
|
||||
egr_utils_api::RetainGradForTensor(
|
||||
target_tensor); // result: 1.0 + 3.0 = 4.0
|
||||
egr_utils_api::RetainGradForTensor(
|
||||
target_tensor); // result: 1.0 + 3.0 = 4.0
|
||||
}
|
||||
|
||||
// Retain Grad for leaf tensor1
|
||||
paddle::Tensor leaf_tensor = paddle::Tensor();
|
||||
{
|
||||
// AccumulationNode Hook: +3
|
||||
auto tmp_tensor0 = paddle::Tensor();
|
||||
auto auto_grad_meta = EagerUtils::autograd_meta(&tmp_tensor0);
|
||||
|
||||
auto acc_node_ptr = std::make_shared<GradNodeAccumulation>(tmp_tensor0);
|
||||
|
||||
auto_grad_meta->SetStopGradient(false);
|
||||
auto_grad_meta->SetGradNode(acc_node_ptr);
|
||||
auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
|
||||
std::vector<egr::AutogradMeta*> res = {auto_grad_meta};
|
||||
scale_node_ptr->SetGradOutMeta(tmp_tensor0, 0);
|
||||
|
||||
leaf_tensor.set_autograd_meta(
|
||||
std::dynamic_pointer_cast<paddle::AbstractAutogradMeta>(
|
||||
tmp_tensor0.mutable_autograd_meta()));
|
||||
|
||||
egr_utils_api::RegisterGradientHookForTensor(leaf_tensor, hook_function);
|
||||
egr_utils_api::RetainGradForTensor(
|
||||
leaf_tensor); // result: 4.0*5.0 + 3.0 = 23.0
|
||||
}
|
||||
|
||||
Backward(target_tensors, {});
|
||||
|
||||
eager_test::CompareGradTensorWithValue<float>(target_tensor, 4.0);
|
||||
eager_test::CompareGradTensorWithValue<float>(leaf_tensor, 23.0);
|
||||
}
|
||||
|
||||
TEST(RetainGrad, HookAfterRetainGrad) {
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
// Prepare Inputs
|
||||
std::vector<paddle::Tensor> target_tensors;
|
||||
phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
|
||||
|
||||
// Create Target Tensor
|
||||
paddle::Tensor tensor =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
1.0 /*value*/,
|
||||
false /*is_leaf*/);
|
||||
target_tensors.emplace_back(std::move(tensor));
|
||||
paddle::Tensor& target_tensor = target_tensors[0];
|
||||
|
||||
// Create ScaleNode
|
||||
auto scale_node_ptr = std::make_shared<GradNodeScale>(1, 1);
|
||||
scale_node_ptr->SetAttributes_scale(5.0 /*scale*/);
|
||||
// Set grad in/out meta for node0
|
||||
scale_node_ptr->SetDefaultGradInOutMeta();
|
||||
|
||||
// Connect Input Tensor and ScaleNode via AutoGradMeta
|
||||
// Apply RetainGrad
|
||||
{
|
||||
// ScaleNode Hook: +3
|
||||
|
||||
auto auto_grad_meta = std::make_shared<AutogradMeta>();
|
||||
auto_grad_meta->SetGradNode(
|
||||
std::dynamic_pointer_cast<GradNodeBase>(scale_node_ptr));
|
||||
auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
|
||||
auto_grad_meta->SetStopGradient(false);
|
||||
target_tensor.set_autograd_meta(
|
||||
std::dynamic_pointer_cast<paddle::AbstractAutogradMeta>(
|
||||
auto_grad_meta));
|
||||
|
||||
egr_utils_api::RetainGradForTensor(target_tensor); // result: 1.0
|
||||
egr_utils_api::RegisterGradientHookForTensor(target_tensor, hook_function);
|
||||
}
|
||||
|
||||
// Retain Grad for leaf tensor1
|
||||
paddle::Tensor leaf_tensor = paddle::Tensor();
|
||||
{
|
||||
// AccumulationNode Hook: +3
|
||||
auto tmp_tensor0 = paddle::Tensor();
|
||||
auto auto_grad_meta = EagerUtils::autograd_meta(&tmp_tensor0);
|
||||
auto acc_node_ptr = std::make_shared<GradNodeAccumulation>(tmp_tensor0);
|
||||
auto_grad_meta->SetGradNode(acc_node_ptr);
|
||||
auto_grad_meta->SetStopGradient(false);
|
||||
scale_node_ptr->SetGradOutMeta(tmp_tensor0, 0);
|
||||
|
||||
auto_grad_meta->SetSingleOutRankWithSlot(0, 0);
|
||||
leaf_tensor.set_autograd_meta(
|
||||
std::dynamic_pointer_cast<paddle::AbstractAutogradMeta>(
|
||||
tmp_tensor0.mutable_autograd_meta()));
|
||||
|
||||
egr_utils_api::RegisterGradientHookForTensor(leaf_tensor, hook_function);
|
||||
}
|
||||
|
||||
Backward(target_tensors, {});
|
||||
eager_test::CompareGradTensorWithValue<float>(target_tensor, 1.0);
|
||||
eager_test::CompareGradTensorWithValue<float>(leaf_tensor, 23.0);
|
||||
}
|
||||
} // namespace egr
|
||||
@@ -0,0 +1,327 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <sstream>
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "paddle/fluid/eager/api/all.h"
|
||||
#include "paddle/fluid/eager/api/generated/fluid_generated/dygraph_forward_api.h"
|
||||
#include "paddle/fluid/eager/backward.h"
|
||||
#include "paddle/fluid/eager/grad_node_info.h"
|
||||
#include "paddle/fluid/eager/hooks.h"
|
||||
#include "paddle/fluid/imperative/tracer.h"
|
||||
#include "paddle/phi/core/dense_tensor.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "test/cpp/eager/test_utils.h"
|
||||
|
||||
PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
|
||||
PD_DECLARE_KERNEL(matmul, CPU, ALL_LAYOUT);
|
||||
PD_DECLARE_KERNEL(matmul_grad, CPU, ALL_LAYOUT);
|
||||
PD_DECLARE_KERNEL(add, CPU, ALL_LAYOUT);
|
||||
PD_DECLARE_KERNEL(add_grad, CPU, ALL_LAYOUT);
|
||||
PD_DECLARE_KERNEL(sigmoid, CPU, ALL_LAYOUT);
|
||||
PD_DECLARE_KERNEL(sigmoid_grad, CPU, ALL_LAYOUT);
|
||||
|
||||
namespace egr {
|
||||
|
||||
paddle::Tensor hook_function(const paddle::Tensor& t) {
|
||||
auto t_dense = std::dynamic_pointer_cast<phi::DenseTensor>(t.impl());
|
||||
|
||||
auto ret_meta = phi::DenseTensorMeta(
|
||||
t_dense->dtype(), t_dense->dims(), t_dense->layout());
|
||||
auto place = t_dense->place();
|
||||
size_t bytes_size =
|
||||
common::product(t_dense->dims()) * SizeOf(t_dense->dtype());
|
||||
auto ret_dense = std::make_shared<phi::DenseTensor>(
|
||||
paddle::memory::Alloc(place, bytes_size), std::move(ret_meta));
|
||||
|
||||
float* t_ptr = t_dense->mutable_data<float>(place);
|
||||
float* ret_ptr = ret_dense->mutable_data<float>(place);
|
||||
for (int i = 0; i < ret_dense->numel(); i++) {
|
||||
ret_ptr[i] = t_ptr[i] + 3.0f;
|
||||
}
|
||||
|
||||
auto ret_impl = std::dynamic_pointer_cast<phi::TensorBase>(ret_dense);
|
||||
paddle::Tensor ret = paddle::Tensor();
|
||||
ret.set_impl(ret_impl);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void test_sigmoid(bool is_remove_gradient_hook) {
|
||||
// Prepare Device Contexts
|
||||
VLOG(6) << "Init Env";
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
VLOG(6) << "Make Dim";
|
||||
phi::DDim ddim = common::make_ddim({2, 4, 4, 4});
|
||||
|
||||
VLOG(6) << "Make paddle::Tensor";
|
||||
paddle::Tensor tensor =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
0.0,
|
||||
true);
|
||||
|
||||
VLOG(6) << "Make ReduceHook function";
|
||||
auto reduce_hook = [&]() -> void {
|
||||
auto* t_ptr = std::dynamic_pointer_cast<phi::DenseTensor>(tensor.impl())
|
||||
->data<float>();
|
||||
for (int i = 0; i < tensor.numel(); i++) {
|
||||
t_ptr[i] = 100.0; // set to 100.0
|
||||
}
|
||||
};
|
||||
|
||||
VLOG(6) << "Retain Grad for Tensor";
|
||||
egr_utils_api::RetainGradForTensor(tensor);
|
||||
|
||||
VLOG(6) << "Register GradientHook for Tensor";
|
||||
int64_t hook_id =
|
||||
egr_utils_api::RegisterGradientHookForTensor(tensor, hook_function);
|
||||
|
||||
VLOG(6) << "Register ReduceHook for Tensor";
|
||||
egr_utils_api::RegisterReduceHookForTensor(tensor, reduce_hook);
|
||||
|
||||
VLOG(6) << "Running Forward";
|
||||
auto output_tensor = sigmoid_dygraph_function(tensor, {});
|
||||
VLOG(6) << "Finish Forward";
|
||||
|
||||
eager_test::CompareTensorWithValue<float>(output_tensor, 0.5);
|
||||
|
||||
std::vector<paddle::Tensor> target_tensors = {output_tensor};
|
||||
|
||||
if (is_remove_gradient_hook) {
|
||||
std::shared_ptr<GradNodeBase> grad_node_tmp = EagerUtils::grad_node(tensor);
|
||||
grad_node_tmp->RemoveGradientHook(hook_id);
|
||||
}
|
||||
|
||||
VLOG(6) << "Running Backward";
|
||||
Backward(target_tensors, {});
|
||||
VLOG(6) << "Finish Backward";
|
||||
|
||||
eager_test::CompareGradTensorWithValue<float>(
|
||||
tensor, is_remove_gradient_hook ? 0.25 : 0.25 + 3.0);
|
||||
|
||||
VLOG(6) << "Checking ReduceHook results";
|
||||
for (int i = 0; i < tensor.numel(); i++) {
|
||||
PADDLE_ENFORCE_EQ(
|
||||
std::dynamic_pointer_cast<phi::DenseTensor>(tensor.impl())
|
||||
->data<float>()[i],
|
||||
static_cast<float>(100.0f),
|
||||
common::errors::InvalidArgument(
|
||||
"Required tensor.impl()->data[%d] should be equal to 100.0 . ", i));
|
||||
}
|
||||
VLOG(6) << "After Tests";
|
||||
}
|
||||
|
||||
void test_elementwiseAdd(bool is_remove_gradient_hook) {
|
||||
// Prepare Device Contexts
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
auto tracer = std::make_shared<paddle::imperative::Tracer>();
|
||||
paddle::imperative::SetCurrentTracer(tracer);
|
||||
|
||||
// 1. Prepare Input
|
||||
phi::DDim ddimX = common::make_ddim({4, 16});
|
||||
paddle::Tensor X = eager_test::CreateTensorWithValue(ddimX,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
3.0,
|
||||
true);
|
||||
egr_utils_api::RetainGradForTensor(X);
|
||||
|
||||
phi::DDim ddimY = common::make_ddim({4, 16});
|
||||
paddle::Tensor Y = eager_test::CreateTensorWithValue(ddimY,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
2.0,
|
||||
true);
|
||||
|
||||
auto reduce_hook = [&]() -> void {
|
||||
auto* t_ptr =
|
||||
std::dynamic_pointer_cast<phi::DenseTensor>(Y.impl())->data<float>();
|
||||
for (int i = 0; i < Y.numel(); i++) {
|
||||
t_ptr[i] = 100.0; // set to 100.0
|
||||
}
|
||||
};
|
||||
|
||||
egr_utils_api::RetainGradForTensor(Y);
|
||||
int64_t hook_id =
|
||||
egr_utils_api::RegisterGradientHookForTensor(Y, hook_function);
|
||||
egr_utils_api::RegisterReduceHookForTensor(Y, reduce_hook);
|
||||
|
||||
auto output_tensor = elementwise_add_dygraph_function(X, Y, {});
|
||||
|
||||
eager_test::CompareTensorWithValue<float>(output_tensor, 5);
|
||||
std::vector<paddle::Tensor> target_tensors = {output_tensor};
|
||||
|
||||
if (is_remove_gradient_hook) {
|
||||
std::shared_ptr<GradNodeBase> grad_node_tmp = EagerUtils::grad_node(Y);
|
||||
grad_node_tmp->RemoveGradientHook(hook_id);
|
||||
}
|
||||
|
||||
Backward(target_tensors, {});
|
||||
|
||||
eager_test::CompareGradTensorWithValue<float>(X, 1.0);
|
||||
eager_test::CompareGradTensorWithValue<float>(
|
||||
Y, is_remove_gradient_hook ? 1.0 : 1.0 + 3.0);
|
||||
|
||||
// Checking ReduceHook results
|
||||
for (int i = 0; i < Y.numel(); i++) {
|
||||
PADDLE_ENFORCE_EQ(
|
||||
std::dynamic_pointer_cast<phi::DenseTensor>(Y.impl())->data<float>()[i],
|
||||
static_cast<float>(100.0f),
|
||||
common::errors::InvalidArgument(
|
||||
"Required Y.impl()->data[%d] should be equal to 100.0 . ", i));
|
||||
}
|
||||
}
|
||||
|
||||
void test_matmul(bool is_remove_gradient_hook) {
|
||||
// Prepare Device Contexts
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
auto tracer = std::make_shared<paddle::imperative::Tracer>();
|
||||
paddle::imperative::SetCurrentTracer(tracer);
|
||||
|
||||
// 1. Prepare Input
|
||||
phi::DDim ddimX = common::make_ddim({4, 16});
|
||||
paddle::Tensor X = eager_test::CreateTensorWithValue(ddimX,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
3.0,
|
||||
true);
|
||||
egr_utils_api::RetainGradForTensor(X);
|
||||
|
||||
phi::DDim ddimY = common::make_ddim({16, 20});
|
||||
paddle::Tensor Y = eager_test::CreateTensorWithValue(ddimY,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
2.0,
|
||||
true);
|
||||
|
||||
auto reduce_hook = [&]() -> void {
|
||||
auto* t_ptr =
|
||||
std::dynamic_pointer_cast<phi::DenseTensor>(Y.impl())->data<float>();
|
||||
for (int i = 0; i < Y.numel(); i++) {
|
||||
t_ptr[i] = 100.0; // set to 100.0
|
||||
}
|
||||
};
|
||||
|
||||
egr_utils_api::RetainGradForTensor(Y);
|
||||
int64_t hook_id =
|
||||
egr_utils_api::RegisterGradientHookForTensor(Y, hook_function);
|
||||
egr_utils_api::RegisterReduceHookForTensor(Y, reduce_hook);
|
||||
|
||||
auto output_tensor = matmul_v2_dygraph_function(
|
||||
X, Y, {{"trans_x", false}, {"trans_y", false}});
|
||||
|
||||
eager_test::CompareTensorWithValue<float>(output_tensor, 96);
|
||||
std::vector<paddle::Tensor> target_tensors = {output_tensor};
|
||||
|
||||
if (is_remove_gradient_hook) {
|
||||
std::shared_ptr<GradNodeBase> grad_node_tmp = EagerUtils::grad_node(Y);
|
||||
grad_node_tmp->RemoveGradientHook(hook_id);
|
||||
}
|
||||
|
||||
Backward(target_tensors, {});
|
||||
|
||||
eager_test::CompareGradTensorWithValue<float>(X, 2.0 * 20);
|
||||
eager_test::CompareGradTensorWithValue<float>(
|
||||
Y, is_remove_gradient_hook ? 3.0 * 4 : 3.0 * 4 + 3);
|
||||
|
||||
// Checking ReduceHook results
|
||||
for (int i = 0; i < Y.numel(); i++) {
|
||||
PADDLE_ENFORCE_EQ(
|
||||
std::dynamic_pointer_cast<phi::DenseTensor>(Y.impl())->data<float>()[i],
|
||||
static_cast<float>(100.0f),
|
||||
common::errors::InvalidArgument(
|
||||
"Required Y.impl()->data[%d] should be equal to 100.0 . ", i));
|
||||
}
|
||||
}
|
||||
|
||||
void test_backward_final_hooks() {
|
||||
// Prepare Device Contexts
|
||||
VLOG(6) << "Init Env";
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
VLOG(6) << "Make paddle::Tensor";
|
||||
phi::DDim ddimX = common::make_ddim({4, 16});
|
||||
paddle::Tensor X = eager_test::CreateTensorWithValue(ddimX,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
3.0,
|
||||
true);
|
||||
phi::DDim ddimY = common::make_ddim({16, 20});
|
||||
egr_utils_api::RetainGradForTensor(X);
|
||||
|
||||
paddle::Tensor Y = eager_test::CreateTensorWithValue(ddimY,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
2.0,
|
||||
true);
|
||||
|
||||
VLOG(6) << "Make ReduceHook function";
|
||||
auto backward_final_hook = [&]() -> void {
|
||||
auto* t_ptr =
|
||||
std::dynamic_pointer_cast<phi::DenseTensor>(X.impl())->data<float>();
|
||||
VLOG(6) << "Run Target Backward Hook";
|
||||
for (int i = 0; i < X.numel(); i++) {
|
||||
t_ptr[i] = 100.0; // set to 100.0
|
||||
}
|
||||
};
|
||||
VLOG(6) << "Register Backward Final Hook";
|
||||
egr_utils_api::RegisterBackwardFinalHook(backward_final_hook);
|
||||
|
||||
VLOG(6) << "Running Forward";
|
||||
auto output_tensor = matmul_v2_dygraph_function(
|
||||
X, Y, {{"trans_x", false}, {"trans_y", false}});
|
||||
auto res = sigmoid_dygraph_function(output_tensor, {});
|
||||
VLOG(6) << "Finish Forward";
|
||||
|
||||
eager_test::CompareTensorWithValue<float>(X, 3.0);
|
||||
|
||||
std::vector<paddle::Tensor> target_tensors = {output_tensor};
|
||||
|
||||
VLOG(6) << "Running Backward";
|
||||
Backward(target_tensors, {});
|
||||
VLOG(6) << "Finish Backward";
|
||||
eager_test::CompareTensorWithValue<float>(X, 100.0);
|
||||
}
|
||||
|
||||
TEST(Hook_intermediate, Sigmoid) {
|
||||
// True or false represents whether to call RemoveGradientHook
|
||||
test_sigmoid(true);
|
||||
test_sigmoid(false);
|
||||
}
|
||||
|
||||
TEST(Hook_intermediate, ElementwiseAdd) {
|
||||
test_elementwiseAdd(true);
|
||||
test_elementwiseAdd(false);
|
||||
}
|
||||
|
||||
TEST(Hook_intermediate, Matmul_v2) {
|
||||
test_matmul(true);
|
||||
test_matmul(false);
|
||||
}
|
||||
|
||||
TEST(Hook_intermediate, BackwardFinal) { test_backward_final_hooks(); }
|
||||
} // namespace egr
|
||||
@@ -0,0 +1,169 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/fluid/eager/nan_inf_utils.h"
|
||||
|
||||
#include <iostream>
|
||||
#include <limits>
|
||||
#include <ostream>
|
||||
#include <string>
|
||||
#include <tuple>
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "paddle/common/flags.h"
|
||||
#include "paddle/fluid/framework/tensor_util.h"
|
||||
#include "paddle/fluid/platform/enforce.h"
|
||||
#include "paddle/phi/api/include/api.h"
|
||||
#include "paddle/phi/api/include/strings_api.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
|
||||
PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
|
||||
PD_DECLARE_KERNEL(strings_empty, CPU, ALL_LAYOUT);
|
||||
|
||||
COMMON_DECLARE_string(check_nan_inf_blacklist);
|
||||
|
||||
namespace egr {
|
||||
|
||||
using paddle_flags::FLAGS_check_nan_inf_blacklist;
|
||||
#define CHECK_NAN_INF(tensors) \
|
||||
{ \
|
||||
bool caught_exception = false; \
|
||||
try { \
|
||||
CheckTensorHasNanOrInf("nan_inf_test", tensors); \
|
||||
} catch (paddle::platform::EnforceNotMet & error) { \
|
||||
caught_exception = true; \
|
||||
std::string ex_msg = error.what(); \
|
||||
EXPECT_TRUE(ex_msg.find("There are NAN or INF") != std::string::npos); \
|
||||
} \
|
||||
EXPECT_TRUE(caught_exception); \
|
||||
}
|
||||
|
||||
#define CHECK_NO_NAN_INF(tensors) \
|
||||
{ \
|
||||
bool caught_exception = false; \
|
||||
try { \
|
||||
CheckTensorHasNanOrInf("nan_inf_test", tensors); \
|
||||
} catch (paddle::platform::EnforceNotMet & error) { \
|
||||
caught_exception = true; \
|
||||
std::string ex_msg = error.what(); \
|
||||
EXPECT_TRUE(ex_msg.find("There are NAN or INF") != std::string::npos); \
|
||||
} \
|
||||
EXPECT_FALSE(caught_exception); \
|
||||
}
|
||||
|
||||
#define CHECK_APINAME_SKIP(api_name, tensor) \
|
||||
{ \
|
||||
bool caught_exception = false; \
|
||||
try { \
|
||||
CheckTensorHasNanOrInf(api_name, tensor); \
|
||||
} catch (paddle::platform::EnforceNotMet & error) { \
|
||||
caught_exception = true; \
|
||||
} \
|
||||
EXPECT_FALSE(caught_exception); \
|
||||
}
|
||||
|
||||
#define CHECK_APINAME_NO_SKIP(api_name, tensor) \
|
||||
{ \
|
||||
bool caught_exception = false; \
|
||||
try { \
|
||||
CheckTensorHasNanOrInf(api_name, tensor); \
|
||||
} catch (paddle::platform::EnforceNotMet & error) { \
|
||||
caught_exception = true; \
|
||||
} \
|
||||
EXPECT_TRUE(caught_exception); \
|
||||
}
|
||||
|
||||
TEST(NanInfUtils, BlacklistSkipCheck) {
|
||||
auto nan_tensor = paddle::experimental::full(
|
||||
{3, 4}, std::numeric_limits<double>::quiet_NaN(), phi::DataType::FLOAT64);
|
||||
|
||||
FLAGS_check_nan_inf_blacklist = "";
|
||||
CHECK_APINAME_SKIP("empty", nan_tensor);
|
||||
|
||||
// Test that "empty_like" always skips regardless of blacklist
|
||||
FLAGS_check_nan_inf_blacklist = "";
|
||||
CHECK_APINAME_SKIP("empty_like", nan_tensor);
|
||||
|
||||
// Test with empty blacklist (default behavior)
|
||||
FLAGS_check_nan_inf_blacklist = "";
|
||||
CHECK_APINAME_NO_SKIP("some_op", nan_tensor);
|
||||
|
||||
// Test with single op in blacklist
|
||||
FLAGS_check_nan_inf_blacklist = "single_op";
|
||||
CHECK_APINAME_SKIP("single_op", nan_tensor);
|
||||
CHECK_APINAME_NO_SKIP("other_op", nan_tensor);
|
||||
|
||||
// Even when blacklist is set, these should still skip
|
||||
CHECK_APINAME_SKIP("empty", nan_tensor);
|
||||
CHECK_APINAME_SKIP("empty_like", nan_tensor);
|
||||
|
||||
// blacklist="op1,op2,op3" and op is in blacklist
|
||||
FLAGS_check_nan_inf_blacklist = "op1,op2,op3";
|
||||
CHECK_APINAME_SKIP("op1", nan_tensor);
|
||||
CHECK_APINAME_SKIP("op2", nan_tensor);
|
||||
CHECK_APINAME_SKIP("op3", nan_tensor);
|
||||
// not in blacklist, should perform nan_or_inf check
|
||||
CHECK_APINAME_NO_SKIP("op4", nan_tensor);
|
||||
|
||||
FLAGS_check_nan_inf_blacklist = "";
|
||||
}
|
||||
|
||||
TEST(NanInfUtils, Functions) {
|
||||
// test all methods
|
||||
auto tensor = paddle::experimental::full(
|
||||
{3, 4}, std::numeric_limits<double>::quiet_NaN(), phi::DataType::FLOAT64);
|
||||
CHECK_NAN_INF(tensor);
|
||||
auto tensor1 = paddle::experimental::full(
|
||||
{3, 4}, std::numeric_limits<double>::quiet_NaN(), phi::DataType::FLOAT64);
|
||||
auto two_tensors = std::make_tuple(tensor, tensor1);
|
||||
CHECK_NAN_INF(two_tensors);
|
||||
auto tensor2 = paddle::experimental::full(
|
||||
{3, 4}, std::numeric_limits<double>::quiet_NaN(), phi::DataType::FLOAT64);
|
||||
auto three_tensors = std::make_tuple(tensor, tensor1, tensor2);
|
||||
CHECK_NAN_INF(three_tensors);
|
||||
auto tensor3 = paddle::experimental::full(
|
||||
{3, 4}, std::numeric_limits<double>::quiet_NaN(), phi::DataType::FLOAT64);
|
||||
auto four_tensors = std::make_tuple(tensor, tensor1, tensor2, tensor3);
|
||||
CHECK_NAN_INF(four_tensors);
|
||||
auto tensor4 = paddle::experimental::full(
|
||||
{3, 4}, std::numeric_limits<double>::quiet_NaN(), phi::DataType::FLOAT64);
|
||||
auto five_tensors =
|
||||
std::make_tuple(tensor, tensor1, tensor2, tensor3, tensor4);
|
||||
CHECK_NAN_INF(five_tensors);
|
||||
auto tensor5 = paddle::experimental::full(
|
||||
{3, 4}, std::numeric_limits<double>::quiet_NaN(), phi::DataType::FLOAT64);
|
||||
auto six_tensors =
|
||||
std::make_tuple(tensor, tensor1, tensor2, tensor3, tensor4, tensor5);
|
||||
CHECK_NAN_INF(six_tensors);
|
||||
std::vector<paddle::Tensor> tensor_vec;
|
||||
tensor_vec.emplace_back(tensor);
|
||||
tensor_vec.emplace_back(tensor1);
|
||||
CHECK_NAN_INF(tensor_vec);
|
||||
paddle::small_vector<std::vector<paddle::Tensor>, egr::kSlotSmallVectorSize>
|
||||
small_vec;
|
||||
small_vec.emplace_back(tensor_vec);
|
||||
CHECK_NAN_INF(small_vec);
|
||||
// test selected_rows
|
||||
paddle::Tensor tensor_sr;
|
||||
auto sr = std::make_shared<phi::SelectedRows>();
|
||||
*sr->mutable_value() =
|
||||
*(static_cast<const phi::DenseTensor*>(tensor.impl().get()));
|
||||
tensor_sr.set_impl(sr);
|
||||
CHECK_NAN_INF(tensor_sr);
|
||||
// test other tensor
|
||||
auto tensor_str = paddle::experimental::strings::empty({3, 4});
|
||||
CHECK_NO_NAN_INF(tensor_str);
|
||||
}
|
||||
|
||||
} // namespace egr
|
||||
@@ -0,0 +1,67 @@
|
||||
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <sstream>
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "paddle/fluid/eager/eager_tensor.h"
|
||||
#include "paddle/fluid/eager/grad_node_info.h"
|
||||
#include "paddle/fluid/eager/grad_tensor_holder.h"
|
||||
#include "paddle/phi/api/lib/utils/allocator.h"
|
||||
#include "paddle/phi/core/kernel_registry.h"
|
||||
#include "test/cpp/eager/test_utils.h"
|
||||
|
||||
PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);
|
||||
|
||||
namespace egr {
|
||||
|
||||
TEST(TensorUtils, Test) {
|
||||
// Prepare Device Contexts
|
||||
eager_test::InitEnv(phi::CPUPlace());
|
||||
|
||||
// Prepare Inputs
|
||||
std::vector<paddle::Tensor> target_tensors;
|
||||
phi::DDim ddim = common::make_ddim({4, 16, 16, 32});
|
||||
|
||||
// Create Target Tensor
|
||||
paddle::Tensor t = eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
5.0 /*value*/,
|
||||
true /*is_leaf*/);
|
||||
|
||||
paddle::Tensor t_grad =
|
||||
eager_test::CreateTensorWithValue(ddim,
|
||||
phi::CPUPlace(),
|
||||
phi::DataType::FLOAT32,
|
||||
phi::DataLayout::NCHW,
|
||||
1.0 /*value*/,
|
||||
false /*is_leaf*/);
|
||||
|
||||
PADDLE_ENFORCE_EQ(
|
||||
EagerUtils::IsLeafTensor(t),
|
||||
true,
|
||||
common::errors::InvalidArgument("The tensor t is not a leaf tensor."));
|
||||
|
||||
// Test Utils
|
||||
eager_test::CompareTensorWithValue<float>(t, 5.0);
|
||||
|
||||
egr::AutogradMeta* meta = egr::EagerUtils::autograd_meta(&t);
|
||||
*meta->MutableGrad() = t_grad;
|
||||
|
||||
eager_test::CompareGradTensorWithValue<float>(t, 1.0);
|
||||
}
|
||||
|
||||
} // namespace egr
|
||||
Reference in New Issue
Block a user