chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:40:42 +08:00
commit e25996e7db
15472 changed files with 3536181 additions and 0 deletions
@@ -0,0 +1,9 @@
paddle_test(test_builtin_parameter SRCS test_builtin_parameter.cc)
paddle_test(save_load_version_compat_test SRCS save_load_version_compat_test.cc
DEPS test_dialect)
if(WITH_ONNXRUNTIME AND WIN32)
# Copy onnxruntime for some c++ test in Windows, since the test will
# be build only in CI, so suppose the generator in Windows is Ninja.
copy_onnx(test_builtin_parameter)
endif()
@@ -0,0 +1,32 @@
op_pair_patches:
- op_pair : [pd_op.full, pd_op.scale]
actions:
- action : add_value
object : [1,2]
type : pir::DenseTensorType
data : [pir::Float32Type,[1],"NCHW",[],0]
- action : delete_value
object : [1, 2]
- op_pair : [test1.operation4, test1.operation3]
actions:
- action : delete_value
object : [0, 0]
- op_pair : [test1.operation1, test1.operation2]
actions:
- action : add_value
object : [1, 0]
type : pir::DenseTensorType
data : [pir::Int32Type,[1],"NCHW",[],0]
op_patches:
- op_name : builtin.parameter
actions :
- action : add_attr
object : new_attribute
type : pir::StrAttribute
data : "new.attribute"
- op_name : pd_op.fetch
actions :
- action : delete_input
object : 0
- action : delete_attr
object : col
@@ -0,0 +1,110 @@
op_patches:
- op_name : pd_op.data
actions:
- action : modify_output_attr
object : stop_gradient
type : pir::ArrayAttribute
data :
- type: pir::BoolAttribute
data: "false"
- op_name : builtin.constant
actions:
- action : modify_attr
object : value
type : pir::FloatAttribute
data : 1.0
- action : modify_output_type
object : 0
type : pir::Float64Type
data : pir::Float32Type
- op_name : builtin.parameter
actions :
- action : modify_attr
object : parameter_name
type : pir::StrAttribute
data : "fc_0"
- action : modify_output_attr
object : stop_gradient
type: pir::BoolAttribute
data : "false"
- op_name : pd_op.fetch
actions :
- action : add_output
object : 1
type : pir::DenseTensorType
data : [pir::Float32Type,[-1,30],"NCHW",[],0]
- op_name : test.operation1
actions :
- action : modify_attr
object : op1_attr1
type : pir::StrAttribute
data : "op1_attr1_value"
- action : delete_attr
object : op1_attr2
- action : add_attr
object : op1_attr3
type : pir::StrAttribute
data : "op1_attr3_value"
- action : modify_name
data : test1.operation1
- op_name : test.operation2
actions :
- action : add_output
object : 0
type : pir::DenseTensorType
data : [pir::Float32Type,[-1],"NCHW",[],0]
- action : add_output_attr
object : op2_attr1
type : pir::StrAttribute
data : "op2_attr1_value"
- op_name : test1.operation3
actions :
- action : delete_input
object : 0
- op_name : test1.operation4
actions :
- action : modify_attr
object : op4_attr1
type : pir::ArrayAttribute
data :
- type: pir::Int64Attribute
- type: pir::Int64Attribute
- action : add_attr
object : op4_attr2
type : paddle::dialect::DataTypeAttribute
data : phi::DataType::UNDEFINED
- action : add_attr
object : op4_attr3
type : paddle::dialect::PlaceAttribute
data : [0,0,""]
- action : add_attr
object : op4_attr4
type : paddle::dialect::IntArrayAttribute
data : [2, 2, 2]
- action : add_attr
object : op4_attr5
type : paddle::dialect::ScalarAttribute
data :
- type: phi::DataType::COMPLEX64
data: [1.0, 1.0]
- action : add_attr
object : op4_attr6
type : pir::TypeAttribute
data : pir::Float64Type
- action : add_attr
object : op4_attr7
type : paddle::dialect::DataLayoutAttribute
data : phi::DataLayout::kNHWC
attr_patches:
- attr_name : pir::FloatAttribute
actions:
- action : modify_name
type : pir::DoubleAttribute
type_patches:
- type_name : pir::Float32Type
actions:
- action : modify_name
type : pir::Float64Type
@@ -0,0 +1,413 @@
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <gtest/gtest.h>
#include <stdio.h>
#include <iostream>
#include <map>
#include <sstream>
#include <string>
#include <vector>
#include "paddle/common/enforce.h"
#include "paddle/fluid/pir/dialect/operator/ir/op_attribute.h"
#include "paddle/fluid/pir/dialect/operator/ir/op_dialect.h"
#include "paddle/fluid/pir/dialect/operator/ir/op_type.h"
#include "paddle/fluid/pir/dialect/operator/utils/utils.h"
#include "paddle/fluid/pir/serialize_deserialize/include/interface.h"
#include "paddle/fluid/pir/serialize_deserialize/include/ir_deserialize.h"
#include "paddle/fluid/pir/serialize_deserialize/include/schema.h"
#include "paddle/fluid/pir/serialize_deserialize/include/version_compat.h"
#include "paddle/phi/common/port.h"
#include "paddle/phi/core/tensor_meta.h"
#include "paddle/pir/include/core/block.h"
#include "paddle/pir/include/core/builder.h"
#include "paddle/pir/include/core/builtin_attribute.h"
#include "paddle/pir/include/core/builtin_dialect.h"
#include "paddle/pir/include/core/builtin_op.h"
#include "paddle/pir/include/core/dialect.h"
#include "paddle/pir/include/core/ir_context.h"
#include "paddle/pir/include/core/ir_printer.h"
#include "paddle/pir/include/core/op_base.h"
#include "paddle/pir/include/core/program.h"
#include "paddle/pir/include/core/region.h"
#include "paddle/pir/include/core/utils.h"
#include "test/cpp/pir/tools/test1_dialect.h"
#include "test/cpp/pir/tools/test_dialect.h"
#include "test/cpp/pir/tools/test_op.h"
#include "test/cpp/pir/tools/test_pir_utils.h"
#define PROGRAM "program"
#define BASE_CODE "base_code"
#define MAGIC "magic"
#define PIRVERSION "version"
#define TRAINABLE "trainable"
#define PIR "pir"
namespace {
void RegisterTestDialectIdsForTest() {
auto *dialect_id_map = pir::DialectIdMap::Instance();
dialect_id_map->insert(test::TestDialect::name(), "-1");
dialect_id_map->insert(test1::Test1Dialect::name(), "-2");
}
} // namespace
// Test for building patches.
TEST(save_load_version_compat, op_patch_test) {
RegisterTestDialectIdsForTest();
// (1) Init environment.
pir::IrContext *ctx = pir::IrContext::Instance();
// (2) Create an empty program object
pir::Program program(ctx);
// pir::Program *program = new pir::Program();
EXPECT_EQ(program.block()->empty(), true);
const uint64_t pir_version = 0;
pir::PatchBuilder builder(pir_version);
builder.SetFileVersion(1);
std::string patch_path = "patch";
VLOG(8) << "Patch path: " << patch_path;
builder.BuildPatch(2, 2, patch_path);
}
bool ReadModuleForTest(const std::string &file_path,
pir::Program *program,
uint64_t pir_version) {
std::ifstream f(file_path);
Json data = Json::parse(f);
pir::PatchBuilder builder(pir_version);
if (data.contains(BASE_CODE) && data[BASE_CODE].contains(MAGIC) &&
data[BASE_CODE][MAGIC] == PIR) {
uint64_t file_version =
data.at(BASE_CODE).at(PIRVERSION).template get<uint64_t>();
if (file_version != pir_version) {
builder.SetFileVersion(file_version);
std::string patch_path = "patch";
VLOG(8) << "Patch path: " << patch_path;
builder.BuildPatch(2, 2, patch_path);
}
} else {
PADDLE_THROW(::common::errors::InvalidArgument("Invalid model file: %s.",
file_path));
}
pir::ProgramReader reader(pir_version);
reader.RecoverProgram(&(data[PROGRAM]), program, &builder);
if (data[BASE_CODE].contains(TRAINABLE)) {
return data[BASE_CODE][TRAINABLE].get<bool>();
} else {
return false;
}
}
// Test for attribute patch and op attribute modification.
TEST(save_load_version_compat, attribute_patch_test1) {
RegisterTestDialectIdsForTest();
pir::IrContext *ctx = pir::IrContext::Instance();
ctx->GetOrRegisterDialect<test::TestDialect>();
ctx->GetOrRegisterDialect<paddle::dialect::OperatorDialect>();
pir::Program program(ctx);
auto block = program.block();
pir::Builder builder(ctx, block);
// Create a float32 DenseTensor Parameter and save into Program
pir::Type fp32_dtype = pir::Float32Type::get(ctx);
// Def a = ParameterOp("a")
std::string op1_name = pir::ParameterOp::name();
pir::OpInfo op1_info = ctx->GetRegisteredOpInfo(op1_name);
std::unordered_map<std::string, pir::Attribute> op1_attribute{
{"parameter_name", pir::StrAttribute::get(ctx, "a")}};
pir::Operation *op1 =
pir::Operation::Create({}, op1_attribute, {fp32_dtype}, op1_info);
program.block()->push_back(op1);
// Def b = Constant("b")
std::string op2_name = std::string(pir::ConstantOp::name());
pir::OpInfo op2_info = ctx->GetRegisteredOpInfo(op2_name);
pir::AttributeMap attr_map;
attr_map.insert(std::pair<std::string, pir::Attribute>(
"value", pir::FloatAttribute::get(ctx, 2.0)));
pir::Operation *op2 =
pir::Operation::Create({}, attr_map, {fp32_dtype}, op2_info);
program.block()->push_back(op2);
// Save the program into file
pir::WriteModule(
program, "./test_save_load", true, false, true, /*pir_version*/ 1);
// Load the program from file
pir::Program new_program(ctx);
ReadModuleForTest("./test_save_load", &new_program, 2);
EXPECT_EQ(new_program.block()->front().name(), op1->name());
EXPECT_EQ(new_program.block()->back().name(), op2->name());
// In patch yaml, the value of attribute "parameter_name" in builtin.parameter
// is changed into "fc_0"
EXPECT_EQ(new_program.block()
->front()
.attribute("parameter_name")
.dyn_cast<::pir::StrAttribute>()
.AsString(),
"fc_0");
// In patch yaml, the value of attribute "stop_gradient" in builtin.parameter
// is changed into false
EXPECT_EQ(new_program.block()
->front()
.attribute("stop_gradient")
.dyn_cast<::pir::ArrayAttribute>()
.AsVector()[0]
.dyn_cast<::pir::BoolAttribute>()
.data(),
false);
// In patch yaml, the value of attribute "value" in builtin.constant is
// changed into 1.0 Also, the type of attribute "FloatAttribute" is changed
// into "DoubleAttribute".
EXPECT_EQ(new_program.block()
->back()
.attribute("value")
.dyn_cast<::pir::DoubleAttribute>()
.data(),
1.0);
// In patch yaml, the type of Float32Type is changed into Float64Type
EXPECT_EQ(new_program.block()->front().result(0).type(),
pir::Float64Type::get(ctx));
EXPECT_EQ(new_program.block()->back().result(0).type(),
pir::Float64Type::get(ctx));
}
TEST(save_load_version_compat, attribute_patch_test2) {
RegisterTestDialectIdsForTest();
pir::IrContext *ctx = pir::IrContext::Instance();
ctx->GetOrRegisterDialect<test1::Test1Dialect>();
ctx->GetOrRegisterDialect<paddle::dialect::OperatorDialect>();
pir::Program program(ctx);
auto block = program.block();
pir::Builder builder(ctx, block);
pir::Type dtype = pir::Float32Type::get(ctx);
// Get registered operations.
pir::OpInfo op4_info = ctx->GetRegisteredOpInfo(test1::Operation4::name());
std::unordered_map<std::string, pir::Attribute> op4_attribute{
{"op4_attr1",
pir::ArrayAttribute::get(ctx, {pir::Int32Attribute::get(ctx, 3)})}};
pir::Operation *op4 =
pir::Operation::Create({}, op4_attribute, {dtype}, op4_info);
block->push_back(op4);
// Save the program into file
pir::WriteModule(
program, "./test_save_load", /*pir_version*/ 1, true, false, true);
// Load the program from file
pir::Program new_program(ctx);
ReadModuleForTest("./test_save_load", &new_program, 2);
EXPECT_EQ(new_program.block()
->front()
.attribute("op4_attr1")
.dyn_cast<::pir::ArrayAttribute>()
.AsVector()[0]
.dyn_cast<::pir::Int64Attribute>()
.data(),
3);
EXPECT_EQ(new_program.block()
->front()
.attribute("op4_attr2")
.dyn_cast<::paddle::dialect::DataTypeAttribute>()
.data(),
phi::DataType::UNDEFINED);
EXPECT_EQ(new_program.block()
->front()
.attribute("op4_attr3")
.dyn_cast<::paddle::dialect::PlaceAttribute>()
.data()
.GetType(),
phi::AllocationType::UNDEFINED);
EXPECT_EQ(new_program.block()
->front()
.attribute("op4_attr4")
.dyn_cast<::paddle::dialect::IntArrayAttribute>()
.data()
.size(),
3);
EXPECT_EQ(new_program.block()
->front()
.attribute("op4_attr5")
.dyn_cast<::paddle::dialect::ScalarAttribute>()
.data()
.to<phi::dtype::complex<float>>()
.real,
1.0);
EXPECT_EQ(new_program.block()
->front()
.attribute("op4_attr6")
.dyn_cast<::pir::TypeAttribute>()
.data()
.isa<pir::Float64Type>(),
true);
EXPECT_EQ(new_program.block()
->front()
.attribute("op4_attr7")
.dyn_cast<::paddle::dialect::DataLayoutAttribute>()
.data(),
phi::DataLayout::NHWC);
}
// Test for op I/O and op attribute modification.
TEST(save_load_version_compat, op_patch_test1) {
RegisterTestDialectIdsForTest();
pir::IrContext *ctx = pir::IrContext::Instance();
ctx->GetOrRegisterDialect<test::TestDialect>();
ctx->GetOrRegisterDialect<test1::Test1Dialect>();
ctx->GetOrRegisterDialect<paddle::dialect::OperatorDialect>();
pir::Program program(ctx);
auto block = program.block();
pir::Builder builder(ctx, block);
pir::Type dtype = pir::Float32Type::get(ctx);
// Get registered operations.
pir::OpInfo op1_info = ctx->GetRegisteredOpInfo(test::Operation1::name());
std::unordered_map<std::string, pir::Attribute> op1_attribute{
{"op1_attr1", pir::StrAttribute::get(ctx, "op1_attr1")},
{"op1_attr2", pir::StrAttribute::get(ctx, "op1_attr2")}};
pir::Operation *op1 =
pir::Operation::Create({}, op1_attribute, {dtype}, op1_info);
block->push_back(op1);
pir::OpInfo op2_info = ctx->GetRegisteredOpInfo(test::Operation2::name());
pir::Operation *op2 = pir::Operation::Create({}, {}, {}, op2_info);
program.block()->push_back(op2);
pir::WriteModule(
program, "./test_save_load", true, false, true, /*pir_version*/ 1);
// Load the program from file
pir::Program new_program(ctx);
ReadModuleForTest("./test_save_load", &new_program, 2);
// In patch yaml, the type of Float32Type is changed into Float64Type
EXPECT_EQ(new_program.block()->front().result(0).type(),
pir::Float64Type::get(ctx));
// In patch yaml, the value of attribute "op1_attr1" in test.operation1 is
// changed into "op1_attr1_value".
EXPECT_EQ(new_program.block()
->front()
.attribute("op1_attr1")
.dyn_cast<::pir::StrAttribute>()
.AsString(),
"op1_attr1_value");
// In patch yaml, the attribute "op1_attr2" is deleted from test.operation1.
EXPECT_EQ(new_program.block()->front().HasAttribute("op1_attr2"), false);
// In patch yaml, a new attribute "op1_attr3" is added into test.operation1.
EXPECT_EQ(new_program.block()
->front()
.attribute("op1_attr3")
.dyn_cast<::pir::StrAttribute>()
.AsString(),
"op1_attr3_value");
// In patch yaml, an output has been added to test.operation2, so the number
// of results is not 0.
EXPECT_EQ(new_program.block()->back().results().empty(), false);
}
// Test for the combination of op I/O and op_pair patch for deleting value.
TEST(save_load_version_compat, op_patch_test2) {
RegisterTestDialectIdsForTest();
pir::IrContext *ctx = pir::IrContext::Instance();
ctx->GetOrRegisterDialect<test::TestDialect>();
ctx->GetOrRegisterDialect<test1::Test1Dialect>();
ctx->GetOrRegisterDialect<paddle::dialect::OperatorDialect>();
pir::Program program(ctx);
auto block = program.block();
pir::Builder builder(ctx, block);
pir::Type dtype = pir::Float32Type::get(ctx);
pir::OpInfo op1_info = ctx->GetRegisteredOpInfo(test1::Operation4::name());
pir::OpInfo op2_info = ctx->GetRegisteredOpInfo(test1::Operation3::name());
pir::Operation *op1 =
pir::Operation::Create({}, {}, {dtype, dtype}, op1_info);
block->push_back(op1);
pir::Operation *op2 = pir::Operation::Create(
{op1->result(0), op1->result(1)}, {}, {dtype}, op2_info);
block->push_back(op2);
pir::WriteModule(
program, "./test_save_load", true, false, true, /*pir_version*/ 1);
// Load the program from file
pir::Program new_program(ctx);
ReadModuleForTest("./test_save_load", &new_program, 2);
// After applying the op_pair patch which deleted the output of
// test1.operation4 and the input of test1.operation3 the number of results of
// test1.operation4 is changed into 1.
EXPECT_EQ(new_program.block()->front().num_results(), (uint64_t)1);
// And with the combination of the op_patch which deleted another input of
// test1.operation3, the number of operands of test1.operation3 is changed
// into 0.
EXPECT_EQ(new_program.block()->back().operands().empty(), true);
}
// Test for op_pair patch for adding value.
TEST(save_load_version_compat, op_patch_test3) {
RegisterTestDialectIdsForTest();
pir::IrContext *ctx = pir::IrContext::Instance();
ctx->GetOrRegisterDialect<test::TestDialect>();
ctx->GetOrRegisterDialect<test1::Test1Dialect>();
ctx->GetOrRegisterDialect<paddle::dialect::OperatorDialect>();
pir::Program program(ctx);
auto block = program.block();
pir::Builder builder(ctx, block);
pir::Type dtype = pir::Float32Type::get(ctx);
pir::OpInfo op1_info = ctx->GetRegisteredOpInfo(test1::Operation1::name());
pir::OpInfo op2_info = ctx->GetRegisteredOpInfo(test1::Operation2::name());
std::unordered_map<std::string, pir::Attribute> op1_attribute{
{"op1_attr1", pir::StrAttribute::get(ctx, "op1_attr1")},
{"op1_attr3", pir::StrAttribute::get(ctx, "op1_attr3")}};
pir::Operation *op1 =
pir::Operation::Create({}, op1_attribute, {dtype}, op1_info);
block->push_back(op1);
pir::Operation *op2 = pir::Operation::Create({}, {}, {dtype}, op2_info);
block->push_back(op2);
pir::WriteModule(
program, "./test_save_load", true, false, true, /*pir_version*/ 1);
// Load the program from file
pir::Program new_program(ctx);
ReadModuleForTest("./test_save_load", &new_program, 2);
// After applying the op_pair patch which deleted the output of
// test1.operation4 and the input of test1.operation3 the number of results of
// test1.operation4 is changed into 1.
EXPECT_EQ(new_program.block()->front().num_results(), (uint64_t)2);
// And with the combination of the op_patch which deleted another input of
// test1.operation3, the number of operands of test1.operation3 is changed
// into 0.
EXPECT_EQ(new_program.block()->back().num_operands(), (uint64_t)1);
}
@@ -0,0 +1,62 @@
// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <memory>
#include "paddle/fluid/pir/dialect/operator/ir/op_dialect.h"
#include "paddle/fluid/pir/dialect/operator/ir/pd_op.h"
#include "paddle/fluid/pir/serialize_deserialize/include/interface.h"
#include "paddle/pir/include/core/builtin_dialect.h"
#include "paddle/pir/include/core/operation.h"
#include "paddle/pir/include/core/program.h"
TEST(SaveTest, uncompelteted_parameter) {
pir::IrContext* ctx = pir::IrContext::Instance();
ctx->GetOrRegisterDialect<paddle::dialect::OperatorDialect>();
ctx->GetOrRegisterDialect<pir::BuiltinDialect>();
pir::Program program(ctx);
pir::Builder builder = pir::Builder(ctx, program.block());
paddle::dialect::FullOp full_op1 =
builder.Build<paddle::dialect::FullOp>(std::vector<int64_t>{64, 64}, 1.5);
pir::OpInfo op_info = ctx->GetRegisteredOpInfo(pir::ParameterOp::name());
std::vector<pir::Value> inputs;
std::vector<pir::Type> output_types;
output_types.push_back(full_op1.out().type());
pir::AttributeMap attributes;
attributes.insert(
{"parameter_name", pir::StrAttribute::get(ctx, "test_param")});
pir::Operation* op =
pir::Operation::Create(inputs, attributes, output_types, op_info);
program.block()->push_back(op);
pir::WriteModule(
program, "./test_param", true, false, true, /*pir_version*/ 0);
pir::Program new_program(ctx);
pir::ReadModule("./test_param", &new_program, /*pir_version*/ 0);
pir::Operation& new_op = new_program.block()->back();
EXPECT_EQ(new_op.attribute("is_distributed").isa<pir::ArrayAttribute>(),
true);
EXPECT_EQ(new_op.attribute("is_parameter").isa<pir::ArrayAttribute>(), true);
EXPECT_EQ(new_op.attribute("need_clip").isa<pir::ArrayAttribute>(), true);
EXPECT_EQ(new_op.attribute("parameter_name").isa<pir::StrAttribute>(), true);
EXPECT_EQ(new_op.attribute("persistable").isa<pir::ArrayAttribute>(), true);
EXPECT_EQ(new_op.attribute("stop_gradient").isa<pir::ArrayAttribute>(), true);
EXPECT_EQ(new_op.attribute("trainable").isa<pir::ArrayAttribute>(), true);
}