chore: import upstream snapshot with attribution

test/cpp/fluid/framework/ir/onednn/CMakeLists.txt

@@ -0,0 +1,62 @@
+# OneDNN IR Pass Tests
+
+cc_test(
+  test_depthwise_conv_onednn_pass
+  SRCS depthwise_conv_onednn_pass_test.cc
+  DEPS depthwise_conv_onednn_pass)
+
+cc_test(
+  test_int8_scale_calculation_onednn_pass
+  SRCS int8_scale_calculation_onednn_pass_test.cc
+  DEPS int8_scale_calculation_onednn_pass pass_test_util)
+
+cc_test(
+  test_params_quantization_onednn_pass
+  SRCS params_quantization_onednn_pass_test.cc
+  DEPS params_quantization_onednn_pass)
+
+cc_test(
+  test_onednn_placement_pass
+  SRCS onednn_placement_pass_test.cc
+  DEPS onednn_placement_pass)
+
+cc_test(
+  test_compute_propagate_scales_onednn_pass
+  SRCS compute_propagate_scales_onednn_pass_test.cc
+  DEPS compute_propagate_scales_onednn_pass naive_executor)
+
+if(WITH_ONNXRUNTIME AND WIN32)
+  # Copy onnxruntime for some c++ test in Windows, since the test will
+  # be build only in CI, so suppose the generator in Windows is Ninja.
+  copy_onnx(test_compute_propagate_scales_onednn_pass)
+endif()
+
+cc_test(
+  test_cpu_quantize_placement_pass
+  SRCS cpu_quantize_placement_pass_test.cc
+  DEPS cpu_quantize_placement_pass)
+
+cc_test(
+  test_cpu_quantize_pass
+  SRCS cpu_quantize_pass_test.cc
+  DEPS cpu_quantize_pass naive_executor)
+
+cc_test(
+  test_cpu_quantize_squash_pass
+  SRCS cpu_quantize_squash_pass_test.cc
+  DEPS cpu_quantize_squash_pass naive_executor)
+
+cc_test(
+  test_shuffle_channel_onednn_detect_pass
+  SRCS shuffle_channel_onednn_detect_pass_test.cc
+  DEPS shuffle_channel_onednn_detect_pass)
+
+cc_test(
+  test_cpu_bfloat16_placement_pass
+  SRCS cpu_bfloat16_placement_pass_test.cc
+  DEPS cpu_bfloat16_placement_pass)
+
+cc_test(
+  test_cpu_bfloat16_pass
+  SRCS cpu_bfloat16_pass_test.cc
+  DEPS cpu_bfloat16_pass)

test/cpp/fluid/framework/ir/onednn/compute_propagate_scales_onednn_pass_test.cc

@@ -0,0 +1,347 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+#include <unordered_map>
+
+#include "paddle/fluid/framework/ir/onednn/compute_propagate_scales_onednn_pass.h"
+#include "paddle/fluid/framework/naive_executor.h"
+#include "paddle/phi/common/place.h"
+
+namespace paddle::framework::ir {
+
+const std::array<float, 10> positive_and_negative_values = {-0.0482659,
+                                                            -0.0102493,
+                                                            -0.00794221,
+                                                            -0.00387115,
+                                                            -0.00674586,
+                                                            -0.0495346,
+                                                            0.0629528,
+                                                            -0.00531285,
+                                                            -0.0230353,
+                                                            0.0269089};
+
+const std::vector<std::vector<float>> wx = {
+    {0.04347931, -0.5643393, 0.7551297, 0.26713502, 0.8055306, 0.91144973},
+    {0.01707571, 0.12741385, 0.15419468, 0.66127586, 0.46821925, 0.9665961},
+    {0.40393898, 0.884427, -0.5853097, 0.5840954, 0.9170512, 0.98245513}};
+const std::vector<std::vector<float>> wh = {
+    {0.42484227, -0.9025513, 0.17087583, 0.8403284, 0.03325734, 0.92331886},
+    {0.32630175, 0.41691914, 0.99848574, 0.3504407, 0.06707559, 0.62239844}};
+
+const std::vector<double> gru_scales = {
+    2.35381475, 1.08304947, 1.32427582, 1.19001095, 1.00151656, 1.01785819};
+
+const std::vector<double> lstm_scales = {
+    2.35381475, 1.10797026, 1.00151656, 1.19001095, 1.09045166, 1.01785819};
+
+static const std::initializer_list<std::string> conv_variable_names{
+    "conv_in", "filter", "bias", "conv_out"};
+
+static const std::initializer_list<std::string> rnn_variable_names{
+    "x", "wx", "wh", "b", "h", "c"};
+
+class ComputePropagateScalesOnednnPassTest : public testing::Test {
+ public:
+  ComputePropagateScalesOnednnPassTest() {  // NOLINT
+    pass = std::make_unique<ComputePropagateScalesOnednnPass>();
+  }
+
+  std::vector<float> GetScales(phi::DenseTensor* tensor, int axis) const {
+    return pass->GetScales(tensor, axis);
+  }
+
+  void ComputeVarScales(ir::Graph* graph,
+                        Scope* scope,
+                        const std::unordered_set<std::string> ops,
+                        const std::string& weight_name,
+                        const int axis,
+                        StringPairMap* var_quant_scales) const {
+    pass->ComputeVarScales(
+        graph, scope, ops, weight_name, axis, var_quant_scales);
+  }
+
+  void ComputeGruWeightScales(ir::Graph* graph,
+                              Scope* scope,
+                              const std::string& wx_name,
+                              const std::string& wh_name,
+                              StringPairMap* var_quant_scales) const {
+    pass->ComputeGruWeightScales(
+        graph, scope, wx_name, wh_name, var_quant_scales);
+  }
+
+  void ComputeLstmWeightScales(ir::Graph* graph,
+                               Scope* scope,
+                               std::string wx_name,
+                               std::string wh_name,
+                               StringPairMap* var_quant_scales) const {
+    pass->ComputeLstmWeightScales(
+        graph, scope, wx_name, wh_name, var_quant_scales);
+  }
+
+  void UpdateReluOutputScales(ir::Graph* graph,
+                              StringPairMap* var_quant_scales) const {
+    pass->UpdateReluOutputScales(graph, var_quant_scales);
+  }
+
+  void InitTensorHolder(Scope* scope,
+                        const phi::Place& place,
+                        const std::string& var_name) {
+    auto x = scope->Var(var_name);
+    auto tensor = x->GetMutable<phi::DenseTensor>();
+    auto tensor_size = 1;
+    if (var_name == "filter") {
+      tensor_size = positive_and_negative_values.size();
+    } else if (var_name == "wx") {
+      tensor_size = wx.size();
+    } else if (var_name == "wh") {
+      tensor_size = wh.size();
+    }
+    tensor->mutable_data(
+        place, phi::TransToPhiDataType(proto::VarType::FP32), tensor_size);
+  }
+
+  void PrepareGraph(ir::Graph* graph,
+                    const ProgramDesc& prog,
+                    Scope* scope,
+                    const std::initializer_list<std::string>& variable_names) {
+    auto place = phi::CPUPlace();
+    NaiveExecutor exe{place};
+    exe.CreateVariables(prog, 0, true, scope);
+
+    for (auto& v : variable_names) {
+      InitTensorHolder(scope, place, v.c_str());
+    }
+    graph->SetNotOwned(kParamScopeAttr, scope);
+  }
+
+  void ComputeRnnWeightScalesTest(const std::string& type,
+                                  const framework::ProgramDesc& prog,
+                                  std::vector<double> scales) {
+    ir::Graph* graph(new ir::Graph(prog));
+    Scope scope;
+
+    PrepareGraph(graph, prog, &scope, rnn_variable_names);
+
+    std::string wx_name = "WeightX";
+    std::string wh_name = "WeightH";
+    std::string wx_var_names = "wx";
+    std::string wh_var_names = "wh";
+
+    StringPairMap var_quant_scales;
+
+    auto* wx_var = scope.FindVar(wx_var_names);
+    auto* wx_tensor = wx_var->GetMutable<phi::DenseTensor>();
+    wx_tensor->Resize(common::make_dim(wx.size(), wx[0].size()));
+    for (size_t i = 0; i < wx.size(); i++)
+      std::copy(
+          begin(wx[i]),
+          end(wx[i]),
+          wx_tensor->mutable_data<float>(phi::CPUPlace()) + i * wx[0].size());
+
+    auto* wh_var = scope.FindVar(wh_var_names);
+    auto* wh_tensor = wh_var->GetMutable<phi::DenseTensor>();
+    wh_tensor->Resize(common::make_dim(wh.size(), wh[0].size()));
+    for (size_t i = 0; i < wh.size(); i++)
+      std::copy(
+          begin(wh[i]),
+          end(wh[i]),
+          wh_tensor->mutable_data<float>(phi::CPUPlace()) + i * wh[0].size());
+    if (type == "gru") {  // NOLINT
+      ComputeGruWeightScales(
+          graph, &scope, wx_name, wh_name, &var_quant_scales);
+    } else {
+      ComputeLstmWeightScales(
+          graph, &scope, wx_name, wh_name, &var_quant_scales);
+    }
+    bool is_unsigned;
+    phi::DenseTensor wx_result_tensor;
+
+    std::tie(is_unsigned, wx_result_tensor) = var_quant_scales[wx_var_names];
+    ASSERT_EQ(is_unsigned, false);
+    ASSERT_EQ(wx_result_tensor.numel(), static_cast<int64_t>(scales.size()));
+    for (int64_t i = 0; i < wx_result_tensor.numel(); i++) {
+      ASSERT_FLOAT_EQ(wx_result_tensor.data<float>()[i], scales[i]);
+    }
+  }
+
+  void UpdateReluOutputScaleTest(
+      const framework::ProgramDesc& prog,
+      StringPairMap* var_quant_scales,
+      const std::initializer_list<std::string>& variable_names) {
+    ir::Graph* graph(new ir::Graph(prog));
+    Scope scope;
+
+    PrepareGraph(graph, prog, &scope, conv_variable_names);
+
+    UpdateReluOutputScales(graph, var_quant_scales);
+
+    for (auto& var_name : variable_names) {
+      auto iter = var_quant_scales->find(var_name);
+      ASSERT_NE(iter, var_quant_scales->end());
+      ASSERT_EQ((*var_quant_scales)[var_name].first, true);
+    }
+  }
+
+ private:
+  std::unique_ptr<ComputePropagateScalesOnednnPass> pass;
+};
+
+void SetOp(ProgramDesc* prog,
+           const std::string& type,
+           const std::string& name,
+           const std::vector<std::string>& inputs,
+           const std::vector<std::string>& outputs,
+           const std::unordered_map<std::string, std::string>& attrs = {}) {
+  auto* op = prog->MutableBlock(0)->AppendOp();
+  op->SetType(type);
+  op->SetAttr("use_onednn", true);
+  op->SetAttr("name", name);
+  if (!attrs.empty())
+    for (auto& attr : attrs) op->SetAttr(attr.first, attr.second);
+
+  if (type == "conv2d") {
+    op->SetInput("Input", {inputs[0]});
+    if (inputs.size() > 1) op->SetInput("Filter", {inputs[1]});
+    if (inputs.size() > 2) op->SetInput("Bias", {inputs[2]});
+    op->SetOutput("Output", {outputs[0]});
+  } else if (type == "fusion_gru" || type == "fusion_lstm") {
+    op->SetInput("X", {inputs[0]});
+    op->SetInput("WeightX", {inputs[1]});
+    op->SetInput("WeightH", {inputs[2]});
+    op->SetOutput("Hidden", {outputs[0]});
+    if (type == "fusion_lstm") op->SetOutput("Cell", {outputs[1]});
+  }
+}
+
+ProgramDesc BuildConv2dProgramDesc() {
+  ProgramDesc prog;
+  for (auto& v : conv_variable_names) {
+    prog.MutableBlock(0)->Var(v);
+  }
+  SetOp(&prog, "conv2d", "Conv2d", {"conv_in", "filter", "bias"}, {"conv_out"});
+
+  return prog;
+}
+
+ProgramDesc BuildConv2dReluProgramDesc() {
+  ProgramDesc prog;
+  for (auto& v : conv_variable_names) {
+    prog.MutableBlock(0)->Var(v);
+  }
+  std::unordered_map<std::string, std::string> attrs = {
+      {"fuse_activation", "relu"}};
+  SetOp(&prog,
+        "conv2d",
+        "Conv2d",
+        {"conv_in", "filter", "bias"},
+        {"conv_out"},
+        attrs);
+
+  return prog;
+}
+
+ProgramDesc BuildFusionGruProgramDesc() {
+  ProgramDesc prog;
+  for (auto& v : rnn_variable_names) {
+    prog.MutableBlock(0)->Var(v);
+  }
+  SetOp(&prog, "fusion_gru", "Fusion_gru", {"x", "wx", "wh"}, {"h"});
+
+  return prog;
+}
+
+ProgramDesc BuildFusionLstmProgramDesc() {
+  ProgramDesc prog;
+  for (auto& v : rnn_variable_names) {
+    prog.MutableBlock(0)->Var(v);
+  }
+  SetOp(&prog, "fusion_lstm", "Fusion_lstm", {"x", "wx", "wh"}, {"h", "c"});
+
+  return prog;
+}
+
+TEST_F(ComputePropagateScalesOnednnPassTest, get_scales_function) {
+  const auto& values = positive_and_negative_values;
+  float max_val = *std::max_element(values.begin(), values.end());
+
+  phi::DenseTensor var_tensor;
+  var_tensor.Resize(common::make_dim(values.size(), 1));
+  std::copy(begin(values),
+            end(values),
+            var_tensor.mutable_data<float>(phi::CPUPlace()));
+  std::vector<float> results = GetScales(&var_tensor, 0);
+
+  ASSERT_EQ(results.size(), std::size_t(1));
+  ASSERT_EQ(results[0], (1.f / max_val));
+}
+
+TEST_F(ComputePropagateScalesOnednnPassTest, compute_var_scales) {
+  auto prog = BuildConv2dProgramDesc();
+  const auto& values = positive_and_negative_values;
+  ir::Graph* graph(new ir::Graph(prog));
+  Scope scope;
+
+  PrepareGraph(graph, prog, &scope, conv_variable_names);
+
+  std::initializer_list<std::string> ops = {"conv2d", "depthwise_conv2d"};
+  std::string weight_name = "Filter";
+  std::string weight_var_name = "filter";
+
+  auto axis = 1;
+  StringPairMap var_quant_scales;
+
+  auto* var = scope.FindVar(weight_var_name);
+  auto* weight_tensor = var->GetMutable<phi::DenseTensor>();
+  weight_tensor->Resize(common::make_dim(1, values.size()));
+  std::copy(begin(values),
+            end(values),
+            weight_tensor->mutable_data<float>(phi::CPUPlace()));
+
+  auto max_val = *std::max_element(values.begin(), values.end());
+
+  ComputeVarScales(graph, &scope, ops, weight_name, axis, &var_quant_scales);
+
+  bool is_unsigned;
+  phi::DenseTensor result_tensor;
+
+  std::tie(is_unsigned, result_tensor) = var_quant_scales[weight_var_name];
+
+  ASSERT_EQ(is_unsigned, false);
+  ASSERT_EQ(result_tensor.numel(), 1);
+  ASSERT_FLOAT_EQ(result_tensor.data<float>()[0], (1.0 / max_val));
+}
+
+TEST_F(ComputePropagateScalesOnednnPassTest, compute_gru_weight_scales) {
+  ComputeRnnWeightScalesTest("gru", BuildFusionGruProgramDesc(), gru_scales);
+}
+
+TEST_F(ComputePropagateScalesOnednnPassTest, compute_lstm_weight_scales) {
+  ComputeRnnWeightScalesTest("lstm", BuildFusionLstmProgramDesc(), lstm_scales);
+}
+
+TEST_F(ComputePropagateScalesOnednnPassTest, update_relu_output_scales) {
+  StringPairMap var_quant_scales;
+  for (auto& var_name : conv_variable_names) {
+    phi::DenseTensor tensor;
+    auto* data = tensor.mutable_data<float>({1}, phi::CPUPlace());
+    data[0] = 10;
+    auto pair = std::make_pair(false, tensor);
+    var_quant_scales.insert(std::make_pair(var_name, pair));
+  }
+  UpdateReluOutputScaleTest(
+      BuildConv2dReluProgramDesc(), &var_quant_scales, {"conv_out"});
+}
+
+}  // namespace paddle::framework::ir

test/cpp/fluid/framework/ir/onednn/cpu_bfloat16_pass_test.cc

@@ -0,0 +1,233 @@
+// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include "paddle/fluid/framework/ir/onednn/cpu_bfloat16_pass.h"
+#include "paddle/fluid/imperative/type_defs.h"
+
+namespace paddle::framework::ir {
+
+void SetOp(ProgramDesc* prog,
+           const std::string& type,
+           const std::string& name,
+           const std::vector<std::string>& inputs,
+           const std::vector<std::string>& outputs,
+           bool use_onednn,
+           const std::string& onednn_data_type = "float32") {
+  auto* op = prog->MutableBlock(0)->AppendOp();
+  op->SetType(type);
+  op->SetAttr("use_onednn", use_onednn);
+  op->SetAttr("name", name);
+
+  if (type == "conv2d") {
+    op->SetInput("Input", {inputs[0]});
+    op->SetOutput("Output", {outputs[0]});
+    op->SetAttr("onednn_data_type", onednn_data_type);
+  } else if (type == "pool2d" || type == "transpose2" || type == "reshape2" ||
+             type == "dropout") {
+    op->SetInput("X", {inputs[0]});
+    op->SetOutput("Out", {outputs[0]});
+    if (type != "dropout") op->SetAttr("onednn_data_type", onednn_data_type);
+  } else if (type == "fc") {
+    op->SetInput("Input", {inputs[0]});
+    op->SetOutput("Out", {outputs[0]});
+    op->SetAttr("onednn_data_type", onednn_data_type);
+  } else if (type == "concat" || type == "sum" || type == "split") {
+    op->SetInput("X", inputs);
+    op->SetOutput("Out", outputs);
+    op->SetAttr("onednn_data_type", onednn_data_type);
+  } else if (type == "matmul" || type == "elementwise_add" ||
+             type == "elementwise_mul") {
+    op->SetInput("X", {inputs[0]});
+    if (inputs.size() > 1) op->SetInput("Y", {inputs[1]});
+    op->SetOutput("Out", {outputs[0]});
+    op->SetAttr("onednn_data_type", onednn_data_type);
+  } else if (type == "layer_norm") {
+    op->SetInput("X", {inputs[0]});
+    op->SetOutput("Y", {outputs[0]});
+    op->SetAttr("onednn_data_type", onednn_data_type);
+  }
+}
+
+static const std::initializer_list<std::string> variable_names{
+    "z", "a", "b", "c", "d", "e", "f", "g", "h", "i"};
+
+void MainTest(const ProgramDesc& prog,
+              const int& quant_count,
+              const int& dequant_count,
+              const int& added_nodes_count) {
+  auto graph = std::make_unique<ir::Graph>(prog);
+  auto pass = PassRegistry::Instance().Get("cpu_bfloat16_pass");
+
+  int original_nodes_num = static_cast<int>(graph->Nodes().size());
+  graph.reset(pass->Apply(graph.release()));
+  int current_nodes_num = static_cast<int>(graph->Nodes().size());
+
+  int quantize_nodes_count = 0;
+  int dequantize_nodes_count = 0;
+  for (auto* node : graph->Nodes()) {
+    if (node->IsOp()) {
+      auto* op = node->Op();
+      if (op->Type() == "quantize") {
+        quantize_nodes_count++;
+      } else if (op->Type() == "dequantize") {
+        dequantize_nodes_count++;
+      }
+    }
+  }
+  EXPECT_EQ(quantize_nodes_count, quant_count);
+  EXPECT_EQ(dequantize_nodes_count, dequant_count);
+  EXPECT_EQ(original_nodes_num + added_nodes_count, current_nodes_num);
+}
+
+ProgramDesc BuildProgramDescConv(bool use_onednn) {
+  ProgramDesc prog;
+  for (auto& v : variable_names) {
+    prog.MutableBlock(0)->Var(v);
+  }
+  SetOp(&prog, "dropout", "Dropout", {"a"}, {"b"}, use_onednn, "float32");
+  SetOp(&prog, "conv2d", "Conv1", {"b"}, {"c"}, use_onednn, "bfloat16");
+  SetOp(&prog, "pool2d", "Pool", {"c"}, {"d"}, use_onednn, "bfloat16");
+  SetOp(&prog, "conv2d", "Conv2", {"d"}, {"e"}, use_onednn, "bfloat16");
+  SetOp(&prog, "transpose2", "Transpose", {"e"}, {"f"}, use_onednn, "float32");
+
+  return prog;
+}
+
+TEST(CpuBfloat16Pass, convolution) {
+  bool use_onednn = true;
+  int quant_op = 3;
+  int dequant_op = 3;
+  // each added op consists of 2 nodes
+  int added_nodes = quant_op * 2 + dequant_op * 2;
+  MainTest(BuildProgramDescConv(use_onednn), quant_op, dequant_op, added_nodes);
+}
+
+ProgramDesc BuildProgramDescDoubleInput(bool use_onednn) {
+  ProgramDesc prog;
+  for (auto& v : variable_names) {
+    prog.MutableBlock(0)->Var(v);
+  }
+  SetOp(&prog, "dropout", "Dropout", {"a"}, {"b"}, use_onednn, "float32");
+  SetOp(&prog, "matmul", "Matmul", {"b", "b"}, {"c"}, use_onednn, "bfloat16");
+  SetOp(&prog, "transpose2", "Transpose", {"d"}, {"e"}, use_onednn, "float32");
+  SetOp(&prog,
+        "elementwise_add",
+        "ElementwiseAdd",
+        {"c", "e"},
+        {"f"},
+        use_onednn,
+        "bfloat16");
+  SetOp(&prog, "reshape2", "Reshape", {"f"}, {"g"}, use_onednn, "bfloat16");
+
+  return prog;
+}
+
+TEST(CpuBfloat16Pass, double_input_ops) {
+  bool use_onednn = true;
+  int quant_op = 4;
+  int dequant_op = 3;
+  // each added op consists of 2 nodes
+  int added_nodes = quant_op * 2 + dequant_op * 2;
+  MainTest(BuildProgramDescDoubleInput(use_onednn),
+           quant_op,
+           dequant_op,
+           added_nodes);
+}
+
+ProgramDesc BuildProgramDescDuplicatedInput(bool use_onednn) {
+  ProgramDesc prog;
+  for (auto& v : variable_names) {
+    prog.MutableBlock(0)->Var(v);
+  }
+  SetOp(&prog, "dropout", "Dropout1", {"a"}, {"b"}, use_onednn, "float32");
+  SetOp(&prog, "dropout", "Dropout2", {"c"}, {"d"}, use_onednn, "float32");
+  SetOp(&prog, "concat", "Concat", {"b", "d"}, {"e"}, use_onednn, "bfloat16");
+  SetOp(&prog, "transpose2", "Transpose", {"f"}, {"g"}, use_onednn, "float32");
+  SetOp(&prog, "sum", "Sum", {"e", "g"}, {"h"}, use_onednn, "bfloat16");
+  SetOp(&prog, "reshape2", "Reshape", {"h"}, {"i"}, use_onednn, "bfloat16");
+
+  return prog;
+}
+
+TEST(CpuBfloat16Pass, duplicated_input_ops) {
+  bool use_onednn = true;
+  int quant_op = 5;
+  int dequant_op = 3;
+  // each added op consists of 2 nodes
+  int added_nodes = quant_op * 2 + dequant_op * 2;
+  MainTest(BuildProgramDescDuplicatedInput(use_onednn),
+           quant_op,
+           dequant_op,
+           added_nodes);
+}
+
+ProgramDesc BuildProgramDescDuplicatedOutput(bool use_onednn) {
+  ProgramDesc prog;
+  for (auto& v : variable_names) {
+    prog.MutableBlock(0)->Var(v);
+  }
+  SetOp(&prog, "dropout", "Dropout", {"a"}, {"b"}, use_onednn, "float32");
+  SetOp(&prog, "split", "Split", {"b"}, {"c", "d"}, use_onednn, "bfloat16");
+  SetOp(&prog, "transpose2", "Transpose", {"c"}, {"e"}, use_onednn, "float32");
+  SetOp(&prog, "reshape2", "Reshape", {"d"}, {"f"}, use_onednn, "bfloat16");
+
+  return prog;
+}
+
+TEST(CpuBfloat16Pass, duplicated_output_ops) {
+  bool use_onednn = true;
+  int quant_op = 2;
+  int dequant_op = 3;
+  // each added op consists of 2 nodes
+  int added_nodes = quant_op * 2 + dequant_op * 2;
+  MainTest(BuildProgramDescDuplicatedOutput(use_onednn),
+           quant_op,
+           dequant_op,
+           added_nodes);
+}
+
+ProgramDesc BuildProgramDescDoubleOutputs(bool use_onednn) {
+  ProgramDesc prog;
+  for (auto& v : variable_names) {
+    prog.MutableBlock(0)->Var(v);
+  }
+  SetOp(
+      &prog, "layer_norm", "LayerNorm1", {"a"}, {"b"}, use_onednn, "bfloat16");
+  SetOp(&prog, "dropout", "Dropout1", {"b"}, {"c"}, use_onednn, "float32");
+  SetOp(&prog, "transpose2", "Transpose", {"b"}, {"d"}, use_onednn, "bfloat16");
+  SetOp(
+      &prog, "layer_norm", "LayerNorm2", {"d"}, {"e"}, use_onednn, "bfloat16");
+  SetOp(&prog, "reshape2", "Reshape", {"e"}, {"f"}, use_onednn, "float32");
+  SetOp(&prog, "dropout", "Dropout2", {"e"}, {"g"}, use_onednn, "float32");
+
+  return prog;
+}
+
+TEST(CpuBfloat16Pass, double_outputs_ops) {
+  bool use_onednn = true;
+  int quant_op = 3;
+  int dequant_op = 3;
+  // each added op consists of 2 nodes
+  int added_nodes = quant_op * 2 + dequant_op * 2;
+  MainTest(BuildProgramDescDoubleOutputs(use_onednn),
+           quant_op,
+           dequant_op,
+           added_nodes);
+}
+
+}  // namespace paddle::framework::ir
+
+USE_PASS(cpu_bfloat16_pass);

test/cpp/fluid/framework/ir/onednn/cpu_bfloat16_placement_pass_test.cc

@@ -0,0 +1,177 @@
+// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include "paddle/fluid/framework/ir/onednn/cpu_bfloat16_placement_pass.h"
+#include "paddle/fluid/platform/onednn_helper.h"
+
+namespace paddle::framework::ir {
+
+void SetOp(ProgramDesc* prog,
+           const std::string& type,
+           const std::string& name,
+           const std::vector<std::string>& inputs,
+           const std::vector<std::string>& outputs,
+           const std::string& onednn_data_type = "float32",
+           const bool use_onednn = true) {
+  auto* op = prog->MutableBlock(0)->AppendOp();
+
+  op->SetType(type);
+  if (type != "reshape2") op->SetAttr("use_onednn", use_onednn);
+  op->SetAttr("onednn_data_type", onednn_data_type);
+
+  if (type == "conv2d") {
+    op->SetAttr("name", name);
+    op->SetInput("Input", {inputs[0]});
+  } else if (type == "gelu") {
+    op->SetInput("X", inputs);
+  } else if (type == "concat") {
+    op->SetAttr("axis", 1);
+    op->SetInput("X", {inputs[0], inputs[1]});
+  } else if (type == "pool2d") {
+    op->SetInput("X", {inputs[0]});
+  } else if (type == "transpose2") {
+    op->SetInput("X", {inputs[0]});
+  } else if (type == "reshape2") {
+    op->SetInput("X", {inputs[0]});
+  } else if (type == "sum") {
+    op->SetInput("X", {inputs[0], inputs[1]});
+  } else {
+    FAIL() << "Unexpected operator type.";
+  }
+  op->SetOutput("Out", {outputs[0]});
+}
+
+// operator                      onednn_data_type
+// ---------------------------------------
+// (a,b)->concat->c              float32
+// c->conv->f                    float32
+// f->relu->g                    float32
+// g->pool->h                    float32
+// h->conv->k                    float32
+// k->pool->l                    float32
+ProgramDesc BuildProgramDesc() {
+  ProgramDesc prog;
+
+  for (auto& v : std::vector<std::string>({"a",
+                                           "b",
+                                           "c",
+                                           "f",
+                                           "g",
+                                           "h",
+                                           "k",
+                                           "l",
+                                           "m",
+                                           "n",
+                                           "o",
+                                           "p",
+                                           "r",
+                                           "s"})) {
+    prog.MutableBlock(0)->Var(v)->SetDataType(proto::VarType::FP32);
+  }
+
+  SetOp(&prog, "concat", "concat1", {"a", "b"}, {"c"});
+  SetOp(&prog, "conv2d", "conv1", {"c"}, {"f"});
+  SetOp(&prog, "gelu", "gelu1", {"f"}, {"g"});
+  SetOp(&prog, "pool2d", "pool1", {"g"}, {"h"});
+  SetOp(&prog, "conv2d", "conv2", {"h"}, {"k"});
+  SetOp(&prog, "pool2d", "pool2", {"k"}, {"l"});
+  SetOp(&prog, "concat", "concat2", {"l", "m"}, {"n"});
+  SetOp(&prog, "transpose2", "transpose", {"n"}, {"o"});
+  SetOp(&prog, "reshape2", "reshape", {"o"}, {"p"});
+  SetOp(&prog, "sum", "sum", {"p", "r"}, {"s"});
+
+  return prog;
+}
+
+void MainTest(std::initializer_list<std::string> bfloat16_enabled_op_types,
+              unsigned expected_bfloat16_data_type_count,
+              const ProgramDesc& prog) {
+  std::unique_ptr<ir::Graph> graph(new ir::Graph(prog));
+
+  auto pass = PassRegistry::Instance().Get("cpu_bfloat16_placement_pass");
+  pass->Set("bfloat16_enabled_op_types",
+            new std::unordered_set<std::string>(bfloat16_enabled_op_types));
+
+  graph.reset(pass->Apply(graph.release()));
+
+  unsigned bfloat16_data_type_count = 0;
+
+  for (auto* node : graph->Nodes()) {
+    if (node->IsOp()) {
+      if (platform::HasOpBFLOAT16DataType(node->Op())) {
+        ++bfloat16_data_type_count;
+      }
+    }
+  }
+
+  EXPECT_EQ(bfloat16_data_type_count, expected_bfloat16_data_type_count);
+}
+
+void DefaultAttrTest(unsigned expected_bfloat16_data_type_count,
+                     const ProgramDesc& prog) {
+  std::unique_ptr<ir::Graph> graph(new ir::Graph(prog));
+  auto pass = PassRegistry::Instance().Get("cpu_bfloat16_placement_pass");
+  graph.reset(pass->Apply(graph.release()));
+
+  unsigned bfloat16_data_type_count = 0;
+  for (auto* node : graph->Nodes()) {
+    if (node->IsOp()) {
+      if (platform::HasOpBFLOAT16DataType(node->Op())) {
+        ++bfloat16_data_type_count;
+      }
+    }
+  }
+  EXPECT_EQ(bfloat16_data_type_count, expected_bfloat16_data_type_count);
+}
+
+TEST(Bfloat16PlacementPass, enable_all) {
+  MainTest(
+      {"conv2d", "pool2d", "gelu", "concat", "sum"}, 8, BuildProgramDesc());
+}
+
+TEST(Bfloat16PlacementPass, enabled_conv_and_pool) {
+  // 2 conv2d + 2 pool2 - 1 orphaned conv2d
+  MainTest({"conv2d", "pool2d"}, 3, BuildProgramDesc());
+}
+
+TEST(Bfloat16PlacementPass, default_attr_value) {
+  DefaultAttrTest(10, BuildProgramDesc());
+}
+
+ProgramDesc BuildProgramDescWithDataType() {
+  ProgramDesc prog;
+
+  for (auto& v : std::vector<std::string>({"a", "b", "c", "d", "e"})) {
+    if (v == "a") {
+      prog.MutableBlock(0)->Var(v)->SetDataType(proto::VarType::INT32);
+    } else {
+      prog.MutableBlock(0)->Var(v)->SetDataType(proto::VarType::FP32);
+    }
+  }
+
+  SetOp(&prog, "conv2d", "conv1", {"a"}, {"b"});
+  SetOp(&prog, "pool2d", "pool1", {"b"}, {"c"});
+  SetOp(&prog, "concat", "concat1", {"c", "d"}, {"e"});
+  return prog;
+}
+
+TEST(Bfloat16PlacementPass, check_data_types) {
+  DefaultAttrTest(2, BuildProgramDescWithDataType());
+}
+
+}  // namespace paddle::framework::ir
+
+USE_PASS(cpu_bfloat16_placement_pass);

test/cpp/fluid/framework/ir/onednn/cpu_quantize_pass_test.cc

@@ -0,0 +1,911 @@
+// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include <unordered_map>
+
+#include "paddle/fluid/framework/ir/onednn/cpu_quantize_pass.h"  // NOLINT
+#include "paddle/fluid/framework/naive_executor.h"
+#include "paddle/fluid/imperative/type_defs.h"
+#include "paddle/phi/common/place.h"
+
+namespace paddle {
+namespace framework {
+namespace ir {
+
+static float const SCALE = 2.f;
+static int const S8_MAX = 127;
+static int const U8_MAX = 255;
+
+void SetOp(ProgramDesc* prog,
+           const std::string& type,
+           const std::string& name,
+           const std::vector<std::string>& inputs,
+           const std::vector<std::string>& outputs,
+           bool use_onednn,
+           const std::string& onednn_data_type = "float32") {
+  auto* op = prog->MutableBlock(0)->AppendOp();
+  op->SetType(type);
+  op->SetAttr("use_onednn", use_onednn);
+  op->SetAttr("name", name);
+  if (type != "dropout" && type != "quantize" && type != "dequantize") {
+    op->SetAttr("onednn_data_type", onednn_data_type);
+  }
+
+  if (type == "conv2d") {
+    op->SetInput("Input", {inputs[0]});
+    op->SetInput("Filter", {inputs[1]});
+    if (inputs.size() > 2)
+      op->SetInput("Bias", {inputs[2]});
+    else
+      op->SetInput("Bias", {});
+    if (inputs.size() > 3) {
+      op->SetInput("ResidualData", {inputs[3]});
+      op->SetAttr("fuse_residual_connection", true);
+    } else {
+      op->SetInput("ResidualData", {});
+      op->SetAttr("fuse_residual_connection", false);
+    }
+    op->SetOutput("Output", {outputs[0]});
+  } else if (type == "pool2d" || type == "fused_transpose" ||
+             type == "reshape2" || type == "nearest_interp" ||
+             type == "nearest_interp_v2" || type == "dropout") {
+    op->SetInput("X", {inputs[0]});
+    op->SetOutput("Out", {outputs[0]});
+  } else if (type == "slice") {
+    op->SetInput("Input", {inputs[0]});
+    op->SetOutput("Out", {outputs[0]});
+  } else if (type == "split") {
+    op->SetInput("X", {inputs[0]});
+    op->SetOutput("Out", {outputs});
+  } else if (type == "fc") {
+    op->SetInput("Input", {inputs[0]});
+    if (inputs.size() > 1) op->SetInput("W", {inputs[1]});
+    if (inputs.size() > 2) op->SetInput("Bias", {inputs[2]});
+    op->SetOutput("Out", {outputs[0]});
+    op->SetAttr("Scale_in", 1.0f);
+    op->SetAttr("Scale_out", 1.0f);
+    op->SetAttr("Scale_weights", std::vector<float>{1.0f});
+  } else if (type == "concat") {
+    op->SetInput("X", inputs);
+    op->SetOutput("Out", outputs);
+  } else if (type == "dequantize") {
+    op->SetInput("Input", {inputs[0]});
+    op->SetOutput("Output", {outputs[0]});
+    op->SetAttr("Scale", 1.0f);
+  } else if (type == "matmul" || type == "matmul_v2" ||
+             type == "fused_matmul") {
+    op->SetInput("X", {inputs[0]});
+    if (inputs.size() > 1) op->SetInput("Y", {inputs[1]});
+    if (inputs.size() > 2) op->SetInput("ResidualData", {inputs[2]});
+    op->SetOutput("Out", {outputs[0]});
+    op->SetAttr("Scale_x", 1.0f);
+    op->SetAttr("Scale_y", 1.0f);
+    op->SetAttr("Scale_out", 1.0f);
+  } else if (type == "fused_elementwise_add" ||
+             type == "fused_elementwise_sub" ||
+             type == "fused_elementwise_mul") {
+    op->SetInput("X", {inputs[0]});
+    if (inputs.size() > 1) op->SetInput("Y", {inputs[1]});
+    op->SetOutput("Out", {outputs[0]});
+    op->SetAttr("scale_x", 1.0f);
+    op->SetAttr("scale_y", 1.0f);
+    op->SetAttr("scale_out", 1.0f);
+  } else if (type == "fusion_gru") {
+    op->SetInput("X", {inputs[0]});
+    op->SetInput("Bias", {inputs[1]});
+    op->SetInput("WeightX", {inputs[2]});
+    op->SetInput("WeightH", {inputs[3]});
+    op->SetOutput("Hidden", {outputs[0]});
+    op->SetAttr("Scale_data", 1.0f);
+    op->SetAttr("Shift_data", 0.0f);
+    op->SetAttr("Weight_scale", std::vector<float>{1.0f});
+  } else if (type == "fusion_lstm") {
+    op->SetInput("X", {inputs[0]});
+    op->SetInput("Bias", {inputs[1]});
+    op->SetInput("WeightX", {inputs[2]});
+    op->SetInput("WeightH", {inputs[3]});
+
+    op->SetOutput("Hidden", {outputs[0]});
+    op->SetOutput("Cell", {outputs[1]});
+
+    op->SetAttr("Scale_data", 1.0f);
+    op->SetAttr("Shift_data", 0.0f);
+    op->SetAttr("Weight_scale", std::vector<float>{1.0f});
+  }
+}
+
+void InitTensorHolder(Scope* scope,
+                      const phi::Place& place,
+                      const char* var_name) {
+  auto x = scope->Var(var_name);
+  auto tensor = x->GetMutable<phi::DenseTensor>();
+  tensor->mutable_data(place, phi::TransToPhiDataType(proto::VarType::FP32), 1);
+}
+
+void PreparePass(std::unique_ptr<ir::Graph>* graph,
+                 const ProgramDesc& prog,
+                 const std::vector<std::string> variable_names,
+                 int* original_nodes_num,
+                 int* current_nodes_num,
+                 std::string var_without_scale = "",
+                 std::string var_signed = "") {
+  auto place = phi::CPUPlace();
+  NaiveExecutor exe{place};
+  Scope scope;
+  exe.CreateVariables(prog, 0, true, &scope);
+  auto* scales = new VarQuantScale();
+  for (auto& v : variable_names) {
+    if (v.compare(var_without_scale) == 0) continue;
+    InitTensorHolder(&scope, place, v.c_str());
+    phi::DenseTensor tensor;
+    tensor.Resize({1});
+    auto* ptr = tensor.mutable_data<double>(place);
+    ptr[0] = SCALE;
+    (*scales)[v] = std::make_pair(v == var_signed, std::move(tensor));
+  }
+
+  (*graph)->SetNotOwned(kParamScopeAttr, &scope);
+  std::unique_ptr<Pass> pass =
+      PassRegistry::Instance().Get("cpu_quantize_pass");
+  pass->Set("quant_var_scales", scales);
+
+  *original_nodes_num = (*graph)->Nodes().size();
+  (*graph).reset(pass->Apply((*graph).release()));
+  *current_nodes_num = (*graph)->Nodes().size();
+}
+
+void CheckScales(const OpDesc* op, float scale, float shift) {
+  std::string type = op->Type();
+  std::vector<std::string> scale_names;
+  if (type == "conv2d" || type == "fused_conv2d" || type == "fc") {
+    EXPECT_EQ(op->GetAttrIfExists<std::vector<float>>("Scale_weights")[0],
+              scale);
+    scale_names.push_back("Scale_in");
+    scale_names.push_back("Scale_out");
+  } else if (type == "fused_matmul") {
+    scale_names.push_back("Scale_x");
+    scale_names.push_back("Scale_y");
+    scale_names.push_back("Scale_out");
+    auto const& names = op->InputNames();
+    if (std::find(names.begin(), names.end(), "ResidualData") != names.end())
+      scale_names.push_back("Scale_in_eltwise");
+  } else if (type == "fused_elementwise_add" ||
+             type == "fused_elementwise_sub" ||
+             type == "fused_elementwise_mul") {
+    scale_names.push_back("scale_x");
+    scale_names.push_back("scale_y");
+    scale_names.push_back("scale_out");
+  } else if (type == "fusion_gru" || type == "fusion_lstm") {
+    EXPECT_EQ(op->GetAttrIfExists<float>("Shift_data"), shift);
+    EXPECT_EQ(op->GetAttrIfExists<std::vector<float>>("Scale_weights")[0],
+              scale);
+    EXPECT_EQ(op->GetAttrIfExists<bool>("force_fp32_output"), true);
+    scale_names.push_back("Scale_data");
+  }
+
+  for (auto const& scale_name : scale_names) {
+    EXPECT_EQ(op->GetAttrIfExists<float>(scale_name), scale);
+  }
+}
+
+void MainTest(const ProgramDesc& prog,
+              const std::vector<std::string> variable_names,
+              std::unordered_map<std::string, int> expected_operators,
+              const int added_nodes_count,
+              float scale = 1.f,
+              float shift = 1.f,
+              std::string var_without_scale = "",
+              std::string var_signed = "") {
+  std::unique_ptr<ir::Graph> graph(new ir::Graph(prog));
+  int original_nodes_num, current_nodes_num;
+  PreparePass(&graph,
+              prog,
+              variable_names,
+              &original_nodes_num,
+              &current_nodes_num,
+              var_without_scale,
+              var_signed);
+  for (auto* node : graph->Nodes()) {
+    if (node->IsOp()) {
+      auto* op = node->Op();
+      if (expected_operators.count(op->Type()) > 0) {
+        expected_operators[op->Type()]--;
+        if (op->GetAttrIfExists<std::string>("onednn_data_type") == "int8")
+          CheckScales(op, scale, shift);
+      }
+    }
+  }
+  for (auto const& pair : expected_operators) {
+    EXPECT_EQ(pair.second, 0);
+  }
+  EXPECT_EQ(original_nodes_num + added_nodes_count, current_nodes_num);
+}
+
+static const std::initializer_list<std::string> variable_names{"a",
+                                                               "w1",
+                                                               "c",
+                                                               "d",
+                                                               "w2",
+                                                               "e",
+                                                               "f",
+                                                               "g",
+                                                               "h",
+                                                               "w3",
+                                                               "b1",
+                                                               "i",
+                                                               "j",
+                                                               "w4",
+                                                               "b2",
+                                                               "w5",
+                                                               "b3"};
+// (a,w1)->Conv1->c and c->Pool1->d
+//
+// (d,w2)->Conv2->e and e->Pool2->f
+//
+// d->Dropout1->g and (g, w5, b3)->Fc1->h and (h,w3,b1,i)->Conv3->j
+//
+// (d,w4, b2)->Conv4->i
+ProgramDesc BuildProgramDesc(bool use_onednn,
+                             const std::string& onednn_data_type) {
+  ProgramDesc prog;
+  for (auto& v : variable_names) {
+    auto* var = prog.MutableBlock(0)->Var(v);
+    if (v.find("w") == 0 || v.find("b") == 0) {
+      var->SetPersistable(true);
+    }
+  }
+
+  SetOp(&prog,
+        "conv2d",
+        "Conv1",
+        {"a", "w1"},
+        {"c"},
+        use_onednn,
+        onednn_data_type);
+  SetOp(&prog, "pool2d", "Pool1", {"c"}, {"d"}, use_onednn, onednn_data_type);
+
+  SetOp(&prog,
+        "conv2d",
+        "Conv2",
+        {"d", "w2"},
+        {"e"},
+        use_onednn,
+        onednn_data_type);
+  SetOp(&prog, "pool2d", "Pool2", {"e"}, {"f"}, use_onednn, onednn_data_type);
+
+  SetOp(&prog, "dropout", "Dropout1", {"d"}, {"g"}, use_onednn);
+  SetOp(&prog,
+        "fc",
+        "Fc1",
+        {"g", "w5", "b3"},
+        {"h"},
+        use_onednn,
+        onednn_data_type);
+  SetOp(&prog,
+        "conv2d",
+        "Conv3",
+        {"h", "w3", "b1", "i"},
+        {"j"},
+        use_onednn,
+        onednn_data_type);
+
+  SetOp(&prog,
+        "conv2d",
+        "Conv4",
+        {"c", "w4", "b2"},
+        {"i"},
+        use_onednn,
+        onednn_data_type);
+
+  return prog;
+}
+
+TEST(CpuQuantizePass, quantize) {
+  bool use_onednn = true;
+  std::string onednn_data_type = "int8";
+  // (a->QUANT1->IN1,w1)->Conv1->OUT1->DEQUANT1->c and
+  // c->QUANT2->IN2->Pool1->OUT2->DEQUANT2->d
+  //
+  // (d->QUANT3->IN3,w2)->Conv2->OUT3->DEQUANT3->e and
+  // e->QUANT4->IN4->Pool2->OUT4->DEQUANT4->f
+  //
+  // d->Dropout1->g and (g->QUANT8->IN8,w5,b3)->Fc1->OUT7->DEQUANT7->h and
+  // (h->QUANT5->IN5,w3,b1,i->QUANT6->IN6)->Conv3->OUT5->DEQUANT5->j
+  //
+  // (d->QUANT7->IN7,w4, b2)->Conv4->DEQUANT6->OUT6->i
+  // Insert nodes: 8 Quant + 8 IN + 7 OUT + 7 DEQUANT
+  int added_nodes = 8 + 8 + 7 + 7;
+  std::unordered_map<std::string, int> expected_operators = {
+      {"fused_conv2d", 4}, {"pool2d", 2}, {"quantize", 8}, {"dequantize", 7}};
+  MainTest(BuildProgramDesc(use_onednn, onednn_data_type),
+           variable_names,
+           expected_operators,
+           added_nodes,
+           SCALE * S8_MAX);
+}
+
+TEST(CpuQuantizePass, do_not_quantize) {
+  bool use_onednn = true;
+  std::string onednn_data_type = "float32";
+  int added_nodes = 0;
+  std::unordered_map<std::string, int> expected_operators = {
+      {"fused_conv2d", 4}, {"pool2d", 2}, {"quantize", 0}, {"dequantize", 0}};
+  MainTest(BuildProgramDesc(use_onednn, onednn_data_type),
+           variable_names,
+           expected_operators,
+           added_nodes,
+           1.0f);
+}
+
+static const std::initializer_list<std::string> variable_names_concat = {
+    "a1", "b1", "a2", "b2", "c", "d"};
+
+// a1->Pool1->b1
+// a2->Pool2->b2
+// (b1,b2)->Concat->c
+// c->Pool3->d
+ProgramDesc BuildProgramDescConcat() {
+  ProgramDesc prog;
+
+  SetOp(&prog, "pool2d", "Pool1", {"a1"}, {"b1"}, true, "float32");
+  SetOp(&prog, "pool2d", "Pool2", {"a2"}, {"b2"}, true, "float32");
+  SetOp(&prog, "concat", "Concat", {"b1", "b2"}, {"c"}, true, "int8");
+  SetOp(&prog, "pool2d", "Pool3", {"c"}, {"d"}, true, "float32");
+
+  return prog;
+}
+
+TEST(CpuQuantizePass, concat) {
+  // a1->Pool1->b1
+  // a2->Pool2->b2
+  // (b1->QUANT1->IN1, b2->QUANT2->IN2)->Concat->c
+  // c->OUT1->DEQUANT1->Pool3->d
+  int added_nodes = 6;
+  std::unordered_map<std::string, int> expected_operators = {
+      {"pool2d", 3}, {"concat", 1}, {"quantize", 2}, {"dequantize", 1}};
+  MainTest(BuildProgramDescConcat(),
+           variable_names_concat,
+           expected_operators,
+           added_nodes);
+}
+
+static const std::initializer_list<std::string> variable_names_fusion_gru = {
+    "x", "wx", "wh", "b", "h"};
+
+// (x, wx, wh, b)->Fusion_gru->h
+ProgramDesc BuildProgramDescFusionGru() {
+  ProgramDesc prog;
+  for (auto& v : variable_names_fusion_gru) {
+    auto* var = prog.MutableBlock(0)->Var(v);
+    if (v.find("wx") == 0 || v.find("wh") || v.find("b")) {
+      var->SetPersistable(true);
+    }
+  }
+
+  SetOp(&prog,
+        "fusion_gru",
+        "Fusion_gru",
+        {"x", "wx", "wh", "b"},
+        {"h"},
+        true,
+        "int8");
+
+  return prog;
+}
+
+static const std::initializer_list<std::string> variable_names_fusion_lstm = {
+    "x", "wx", "wh", "b", "h", "c"};
+
+// (x, wx, wh, b)->Fusion_lstm_1->h
+ProgramDesc BuildProgramDescFusionLSTM() {
+  ProgramDesc prog;
+  for (auto& v : variable_names_fusion_lstm) {
+    auto* var = prog.MutableBlock(0)->Var(v);
+    if (v.find("wx") == 0 || v.find("wh") || v.find("b")) {
+      var->SetPersistable(true);
+    }
+  }
+
+  SetOp(&prog,
+        "fusion_lstm",
+        "Fusion_lstm_1",
+        {"x", "wx", "wh", "b"},
+        {"h", "c"},
+        true,
+        "int8");
+
+  return prog;
+}
+
+TEST(CpuQuantizePass, fusion_gru) {
+  // (x, wx, wh, b)->Fusion_gru->h
+
+  // 1 Quant + 1 IN + 0 DeQuant + 0 OUT
+  int added_nodes = 1 + 1 + 0 + 0;
+  std::unordered_map<std::string, int> expected_operators = {
+      {"fusion_gru", 1}, {"quantize", 1}, {"dequantize", 0}};
+  MainTest(BuildProgramDescFusionGru(),
+           variable_names_fusion_gru,
+           expected_operators,
+           added_nodes,
+           SCALE * S8_MAX,
+           128);
+}
+
+TEST(CpuQuantizePass, fusion_lstm) {
+  // (x, wx, wh, b)->Fusion_lstm->h
+
+  // 1 Quant + 1 IN + 0 DeQuant + 0 OUT
+  int added_nodes = 1 + 1 + 0 + 0;
+  std::unordered_map<std::string, int> expected_operators = {
+      {"fusion_lstm", 1}, {"quantize", 1}, {"dequantize", 0}};
+  MainTest(BuildProgramDescFusionLSTM(),
+           variable_names_fusion_lstm,
+           expected_operators,
+           added_nodes,
+           SCALE * S8_MAX,
+           128.);
+}
+
+static const std::initializer_list<std::string> variable_names_immutable_ops = {
+    "a", "w1", "b", "c", "d", "e", "f", "g"};
+
+// a->Dequantize->b
+// b->Tested Op->c
+// c->Dropout->d
+void TestImmutableOp(const std::string tested_op) {
+  ProgramDesc prog;
+  for (auto& v : variable_names_immutable_ops) {
+    prog.MutableBlock(0)->Var(v)->SetDataType(proto::VarType::FP32);
+  }
+  SetOp(&prog, "dequantize", "Dequantize1", {"a"}, {"b"}, true);
+  SetOp(&prog, tested_op, tested_op, {"b"}, {"c"}, true, "int8");
+  SetOp(&prog, "dropout", "Dropout", {"c"}, {"d"}, true, "float32");
+
+  // a->Dequantize->b
+  // b2->Quant->b3->Tested Op->c1->Dequant->c2
+  // c2->Dropout->d
+  // 1 Quant + 1 IN + 1 DeQuant + 1 OUT
+  int added_nodes = 4;
+  std::unordered_map<std::string, int> expected_operators = {
+      {tested_op, 1}, {"quantize", 1}, {"dequantize", 2}};
+  MainTest(prog,
+           variable_names_immutable_ops,
+           expected_operators,
+           added_nodes,
+           SCALE * S8_MAX);
+}
+
+// a->Dropout1->b
+// b->Tested Op->c
+// c->Dropout2->d
+void TestImmutableOpBetweenNonQuantizedOp(const std::string tested_op) {
+  ProgramDesc prog;
+  for (auto& v : variable_names_immutable_ops) {
+    prog.MutableBlock(0)->Var(v);
+  }
+
+  SetOp(&prog, "dropout", "Dropout1", {"a"}, {"b"}, true, "float32");
+  SetOp(&prog, tested_op, tested_op, {"b"}, {"c"}, true, "int8");
+  SetOp(&prog, "dropout", "Dropout2", {"c"}, {"d"}, true, "float32");
+
+  // 0 Quant + 0 IN + 0 DeQuant + 0 OUT
+  int added_nodes = 0;
+  std::unordered_map<std::string, int> expected_operators = {
+      {tested_op, 1}, {"dropout", 2}, {"quantize", 0}, {"dequantize", 0}};
+  MainTest(prog,
+           variable_names_immutable_ops,
+           expected_operators,
+           added_nodes,
+           SCALE * S8_MAX);
+}
+
+// a->Dropout1->b
+// b->TestedOp1(won't be quantized)->c
+//    c->Dropout2->d
+//    c->TestedOp2(will be quantized)->e
+//        e->Pool2d1(will be quantized)->f
+//        e->Pool2d2(will be quantized)->g
+void TestImmutableOpWithManyOutputs(const std::string tested_op) {
+  ProgramDesc prog;
+  for (auto& v : variable_names_immutable_ops) {
+    prog.MutableBlock(0)->Var(v)->SetDataType(proto::VarType::FP32);
+  }
+
+  SetOp(&prog, "dropout", "Dropout1", {"a"}, {"b"}, true, "float32");
+  SetOp(&prog,
+        tested_op,
+        std::string(tested_op + "1"),
+        {"b"},
+        {"c"},
+        true,
+        "int8");
+  SetOp(&prog, "dropout", "Dropout2", {"c"}, {"d"}, true, "float32");
+  SetOp(&prog,
+        tested_op,
+        std::string(tested_op + "2"),
+        {"c"},
+        {"e"},
+        true,
+        "int8");
+  SetOp(&prog, "pool2d", "Pool2d1", {"e"}, {"f"}, true, "int8");
+  SetOp(&prog, "pool2d", "Pool2d2", {"e"}, {"g"}, true, "int8");
+
+  // 3 Quant + 3 IN + 3 DeQuant + 3 OUT
+  int added_nodes = 12;
+  std::unordered_map<std::string, int> expected_operators = {{tested_op, 2},
+                                                             {"dropout", 2},
+                                                             {"pool2d", 2},
+                                                             {"quantize", 3},
+                                                             {"dequantize", 3}};
+  MainTest(prog,
+           variable_names_immutable_ops,
+           expected_operators,
+           added_nodes,
+           SCALE * S8_MAX);
+}
+
+const std::vector<std::string> immutables = {"reshape2",
+                                             "fused_transpose",
+                                             "slice",
+                                             "nearest_interp",
+                                             "nearest_interp_v2",
+                                             "split"};
+
+class TestImmutables : public testing::TestWithParam<std::string> {};
+
+TEST_P(TestImmutables, immutable_basic) {  // NOLINT
+  TestImmutableOp(GetParam());
+}
+
+TEST_P(TestImmutables, immutable_between_non_quantized) {  // NOLINT
+  TestImmutableOpBetweenNonQuantizedOp(GetParam());
+}
+
+TEST_P(TestImmutables, immutable_many_outputs) {  // NOLINT
+  TestImmutableOpWithManyOutputs(GetParam());
+}
+
+INSTANTIATE_TEST_CASE_P(
+    CpuQuantizePass,
+    TestImmutables,
+    testing::ValuesIn(immutables),
+    [](const ::testing::TestParamInfo<TestImmutables::ParamType>& info) {
+      std::string name = info.param;
+      return name;
+    });
+
+static const std::initializer_list<std::string> variable_names_matmul = {
+    "a", "b", "c", "d", "e", "f", "g", "h"};
+
+ProgramDesc BuildProgramDescMatmul() {
+  ProgramDesc prog;
+  for (auto& v : variable_names_matmul) {
+    prog.MutableBlock(0)->Var(v);
+  }
+  SetOp(&prog, "dequantize", "Dequantize1", {"a"}, {"b"}, true);
+  SetOp(&prog, "dequantize", "Dequantize2", {"c"}, {"d"}, true);
+  SetOp(&prog, "fused_matmul", "FusedMatmul", {"b", "d"}, {"e"}, true, "int8");
+  SetOp(&prog, "dropout", "Dropout", {"e"}, {"f"}, true, "float32");
+
+  return prog;
+}
+
+ProgramDesc BuildProgramDescMatmulResidual() {
+  ProgramDesc prog;
+  for (auto& v : variable_names_matmul) {
+    prog.MutableBlock(0)->Var(v);
+  }
+  SetOp(&prog, "dequantize", "Dequantize1", {"a"}, {"b"}, true);
+  SetOp(&prog, "dequantize", "Dequantize2", {"c"}, {"d"}, true);
+  SetOp(&prog, "dequantize", "Dequantize3", {"e"}, {"f"}, true);
+  SetOp(&prog,
+        "fused_matmul",
+        "FusedMatmul",
+        {"b", "d", "f"},
+        {"g"},
+        true,
+        "int8");
+  SetOp(&prog, "dropout", "Dropout", {"g"}, {"h"}, true, "float32");
+
+  return prog;
+}
+
+TEST(CpuQuantizePass, matmul) {
+  // 2 Quant + 2 IN + 1 DeQuant + 1 OUT
+  int added_nodes = 6;
+  std::unordered_map<std::string, int> expected_operators = {
+      {"fused_matmul", 1}, {"quantize", 2}, {"dequantize", 3}};
+  MainTest(BuildProgramDescMatmul(),
+           variable_names_matmul,
+           expected_operators,
+           added_nodes,
+           SCALE * S8_MAX);
+}
+
+TEST(CpuQuantizePass, matmul_residual) {
+  // 3 Quant + 3 IN + 1 DeQuant + 1 OUT
+  int added_nodes = 8;
+  std::unordered_map<std::string, int> expected_operators = {
+      {"fused_matmul", 1}, {"quantize", 3}, {"dequantize", 4}};
+  MainTest(BuildProgramDescMatmulResidual(),
+           variable_names_matmul,
+           expected_operators,
+           added_nodes,
+           SCALE * S8_MAX);
+}
+
+static const std::initializer_list<std::string> variable_names_elementwise = {
+    "a", "b", "c", "d", "e", "f"};
+
+ProgramDesc BuildProgramDescElementwise(const std::string elementwise_type,
+                                        const std::string elementwise_name) {
+  ProgramDesc prog;
+  for (auto& v : variable_names_elementwise) {
+    prog.MutableBlock(0)->Var(v);
+  }
+  SetOp(&prog, "dequantize", "Dequantize1", {"a"}, {"b"}, true);
+  SetOp(&prog, "dequantize", "Dequantize2", {"c"}, {"d"}, true);
+  SetOp(&prog,
+        elementwise_type,
+        elementwise_name,
+        {"b", "d"},
+        {"e"},
+        true,
+        "int8");
+  SetOp(&prog, "dropout", "Dropout", {"e"}, {"f"}, true, "float32");
+
+  return prog;
+}
+
+void TestElementwise(std::vector<std::string> elementwise) {
+  // 2 Quant + 2 IN + 1 DeQuant + 1 OUT
+  int added_nodes = 6;
+  std::unordered_map<std::string, int> expected_operators = {
+      {elementwise[0], 1}, {"quantize", 2}, {"dequantize", 3}};
+  MainTest(BuildProgramDescElementwise(elementwise[0], elementwise[1]),
+           variable_names_elementwise,
+           expected_operators,
+           added_nodes,
+           SCALE * S8_MAX);
+}
+
+void TestElementwiseOutputScaleMissing(std::vector<std::string> elementwise) {
+  int added_nodes = 0;
+  std::unordered_map<std::string, int> expected_operators = {
+      {elementwise[0], 1}, {"quantize", 0}, {"dequantize", 2}};
+  MainTest(BuildProgramDescElementwise(elementwise[0], elementwise[1]),
+           variable_names_elementwise,
+           expected_operators,
+           added_nodes,
+           1.f,
+           1.f,
+           "e");
+}
+
+void TestElementwiseUnsignedAndSignedInput(
+    std::vector<std::string> elementwise) {
+  int added_nodes = 0;
+  std::unordered_map<std::string, int> expected_operators = {
+      {elementwise[0], 1}, {"quantize", 0}, {"dequantize", 2}};
+  MainTest(BuildProgramDescElementwise(elementwise[0], elementwise[1]),
+           variable_names_elementwise,
+           expected_operators,
+           added_nodes,
+           1.f,
+           1.f,
+           "",
+           "b");
+}
+
+const std::vector<std::vector<std::string>> elementwises = {
+    {"fused_elementwise_add", "FusedElementwiseAdd"},
+    {"fused_elementwise_mul", "FusedElementwiseMul"},
+    {"fused_elementwise_sub", "FusedElementwiseSub"}};
+
+class TestElementwises
+    : public testing::TestWithParam<std::vector<std::string>> {};
+
+TEST_P(TestElementwises, elementwise_basic) {  // NOLIN
+  TestElementwise(GetParam());
+}
+
+TEST_P(TestElementwises, elementwise_output_scale_missing) {  // NOLINT
+  TestElementwiseOutputScaleMissing(GetParam());
+}
+
+TEST_P(TestElementwises, elementwise_unsigned_and_signed_input) {  // NOLINT
+  TestElementwiseUnsignedAndSignedInput(GetParam());
+}
+
+INSTANTIATE_TEST_CASE_P(
+    CpuQuantizePass,
+    TestElementwises,
+    testing::ValuesIn(elementwises),
+    [](const ::testing::TestParamInfo<TestElementwises::ParamType>& info) {
+      std::string name = info.param[0];
+      return name;
+    });
+
+const std::vector<std::string> churn_out_vars(ProgramDesc* prog,
+                                              const std::string& prefix,
+                                              int number) {
+  auto v = std::vector<std::string>();
+  for (int i = 0; i < number; ++i) {
+    auto name = prefix + std::to_string(i);
+    prog->MutableBlock(0)->Var(name);
+    v.push_back(name);
+  }
+  return v;
+}
+
+void create_vars(ProgramDesc* prog,
+                 const std::initializer_list<std::string>& names) {
+  for (auto const& name : names) prog->MutableBlock(0)->Var(name);
+}
+
+void SetMultiGruOp(ProgramDesc* prog,
+                   const std::string x,
+                   const std::vector<std::string> wx,
+                   const std::vector<std::string> wh,
+                   const std::vector<std::string> b,
+                   const std::string h,
+                   int layers) {
+  auto* op = prog->MutableBlock(0)->AppendOp();
+  op->SetType("multi_gru");
+  op->SetInput("X", {x});
+  op->SetInput("WeightX", wx);
+  op->SetInput("WeightH", wh);
+  op->SetInput("Bias", b);
+  op->SetOutput("Hidden", {h});
+  op->SetAttr("layers", layers);
+  op->SetAttr("origin_mode", false);
+  op->SetAttr("use_onednn", true);
+  op->SetAttr("name", std::string("Multi_gru"));
+  op->SetAttr("onednn_data_type", std::string("int8"));
+  op->SetAttr("Scale_data", 1.0f);
+  op->SetAttr("Shift_data", 0.0f);
+}
+
+void MainTestMultiGru(int layers) {
+  ProgramDesc prog;
+
+  // Create variables
+  create_vars(&prog, {"x", "h"});
+  const std::vector<std::string> wx = churn_out_vars(&prog, "wx", 2 * layers);
+  const std::vector<std::string> wh = churn_out_vars(&prog, "wh", 2 * layers);
+  const std::vector<std::string> b = churn_out_vars(&prog, "b", 2 * layers);
+
+  std::vector<std::string> all_vars;
+  all_vars.reserve(wx.size() + wh.size() + b.size() + 2);
+  all_vars.insert(all_vars.end(), wx.begin(), wx.end());
+  all_vars.insert(all_vars.end(), wh.begin(), wh.end());
+  all_vars.insert(all_vars.end(), b.begin(), b.end());
+  all_vars.push_back("x");
+  all_vars.push_back("h");
+
+  // Prepare program descriptor
+  SetMultiGruOp(&prog, "x", wx, wh, b, "h", layers);
+
+  // Prepare and run the pass
+  std::unique_ptr<ir::Graph> graph(new ir::Graph(prog));
+  int original_nodes_num, current_nodes_num;
+  PreparePass(&graph, prog, all_vars, &original_nodes_num, &current_nodes_num);
+
+  // Verify graph after quantization
+  float scale = 2 * 127;
+  float shift = 128;
+  int quantize_nodes_count = 0;
+  int dequantize_nodes_count = 0;
+  int multi_gru_nodes_count = 0;
+  for (auto* node : graph->Nodes()) {
+    if (node->IsOp()) {
+      auto* op = node->Op();
+      if (op->Type() == "multi_gru") {
+        multi_gru_nodes_count++;
+
+        auto op_name = PADDLE_GET_CONST(std::string, op->GetAttr("name"));
+        EXPECT_EQ(PADDLE_GET_CONST(float, op->GetAttr("Scale_data")), scale)
+            << "Scale_data for node '" + op_name + "'.";
+        EXPECT_EQ(PADDLE_GET_CONST(float, op->GetAttr("Shift_data")), shift)
+            << "Shift_data for node '" + op_name + "'.";
+        EXPECT_EQ(op->Input("Scale_weights").size(), 2u * layers)
+            << "Scale_weights for node '" + op_name + "'.";
+        EXPECT_EQ(PADDLE_GET_CONST(bool, op->GetAttr("force_fp32_output")),
+                  true)
+            << "force_fp32_output for node '" + op_name + "'.";
+      } else if (op->Type() == "quantize") {
+        quantize_nodes_count++;
+      } else if (op->Type() == "dequantize") {
+        dequantize_nodes_count++;
+      }
+    }
+  }
+
+  int multi_gru_count = 1;
+  int quant_count = 1;
+  int quant_out_count = 1;
+  int dequant_count = 0;
+  int dequant_out_count = 0;
+  int scale_weights_count = 2 * layers;
+  int added_nodes_count = quant_count + quant_out_count + scale_weights_count +
+                          dequant_count + dequant_out_count;
+
+  EXPECT_EQ(multi_gru_nodes_count, multi_gru_count);
+  EXPECT_EQ(quantize_nodes_count, quant_count);
+  EXPECT_EQ(dequantize_nodes_count, dequant_count);
+  EXPECT_EQ(original_nodes_num + added_nodes_count, current_nodes_num);
+}
+
+TEST(CpuQuantizePass, multi_gru_1) {
+  int layers = 1;
+  MainTestMultiGru(layers);
+}
+
+TEST(CpuQuantizePass, multi_gru_2) {
+  int layers = 2;
+  MainTestMultiGru(layers);
+}
+
+TEST(CpuQuantizePass, multi_gru_3) {
+  int layers = 3;
+  MainTestMultiGru(layers);
+}
+
+static const std::initializer_list<std::string>
+    variable_names_multi_inputs_outputs = {"a", "b", "c1", "c2", "d", "e"};
+
+// a->Pool->b
+// b->Split->c1, c2
+// (c1, c2, c1, c2)->Concat->d
+// d->Pool->e
+ProgramDesc BuildProgramDescMulti() {
+  ProgramDesc prog;
+  for (auto& v : variable_names_multi_inputs_outputs) {
+    prog.MutableBlock(0)->Var(v)->SetDataType(proto::VarType::FP32);
+  }
+
+  SetOp(&prog, "pool2d", "Pool", {"a"}, {"b"}, true, "float32");
+  SetOp(&prog, "split", "Split", {"b"}, {"c1", "c2"}, true, "int8");
+  SetOp(
+      &prog, "concat", "Concat", {"c1", "c2", "c1", "c2"}, {"d"}, true, "int8");
+  SetOp(&prog, "pool2d", "Pool2", {"d"}, {"e"}, true, "float32");
+
+  return prog;
+}
+
+TEST(CpuQuantizePass, multi_inputs_outputs_ops) {
+  // a->QUANT1->Split
+  //               b1->DEQUANT->OUT->QUANT
+  //               b2->DEQUANT->OUT->QUANT
+  // (b1, b2, b1, b2)->Concat->c->DEQUANT->Pool->d
+  int added_nodes = 6 * 2;
+  std::unordered_map<std::string, int> expected_operators = {{"pool2d", 2},
+                                                             {"concat", 1},
+                                                             {"split", 1},
+                                                             {"quantize", 3},
+                                                             {"dequantize", 3}};
+  MainTest(BuildProgramDescMulti(),
+           variable_names_multi_inputs_outputs,
+           expected_operators,
+           added_nodes);
+}
+
+}  // namespace ir
+}  // namespace framework
+}  // namespace paddle
+
+USE_PASS(cpu_quantize_pass);

test/cpp/fluid/framework/ir/onednn/cpu_quantize_placement_pass_test.cc

@@ -0,0 +1,185 @@
+// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include "paddle/fluid/framework/ir/onednn/cpu_quantize_placement_pass.h"
+#include "paddle/fluid/platform/onednn_helper.h"
+
+namespace paddle {
+namespace framework {
+namespace ir {
+
+void SetOp(ProgramDesc* prog,
+           const std::string& type,
+           const std::string& name,
+           const std::vector<std::string>& inputs,
+           const std::vector<std::string>& outputs,
+           const std::string& onednn_data_type = "float32") {
+  auto* op = prog->MutableBlock(0)->AppendOp();
+
+  op->SetType(type);
+  op->SetAttr("use_onednn", true);
+  op->SetAttr("onednn_data_type", onednn_data_type);
+
+  if (type == "conv2d") {
+    op->SetAttr("name", name);
+    op->SetInput("Input", {inputs[0]});
+    op->SetInput("Filter", {inputs[1]});
+    op->SetInput("Bias", {inputs[2]});
+  } else if (type == "relu") {
+    op->SetInput("X", inputs);
+  } else if (type == "concat") {
+    op->SetAttr("axis", 1);
+    op->SetInput("X", {inputs[0], inputs[1]});
+  } else if (type == "pool2d") {
+    op->SetInput("X", {inputs[0]});
+  } else {
+    FAIL() << "Unexpected operator type.";
+  }
+  op->SetOutput("Out", {outputs[0]});
+}
+
+// operator                      onednn_data_type
+// ---------------------------------------
+// (a,b)->concat->c              none
+// (c,weights,bias)->conv->f     false
+// f->relu->g                    none
+// g->pool->h                    false
+// (h,weights2,bias2)->conv->k   false
+// k->pool->l                    false
+ProgramDesc BuildProgramDesc() {
+  ProgramDesc prog;
+
+  for (auto& v : std::vector<std::string>({"a",
+                                           "b",
+                                           "c",
+                                           "weights",
+                                           "bias",
+                                           "f",
+                                           "g",
+                                           "h",
+                                           "weights2",
+                                           "bias2",
+                                           "k",
+                                           "l"})) {
+    auto* var = prog.MutableBlock(0)->Var(v);
+    var->SetType(proto::VarType::SELECTED_ROWS);
+    if (v == "weights" || v == "bias") {
+      var->SetPersistable(true);
+    }
+  }
+
+  SetOp(&prog, "concat", "concat1", {"a", "b"}, {"c"}, "float32");
+  SetOp(&prog, "conv2d", "conv1", {"c", "weights", "bias"}, {"f"}, "float32");
+  SetOp(&prog, "relu", "relu1", {"f"}, {"g"}, "float32");
+  SetOp(&prog, "pool2d", "pool1", {"g"}, {"h"}, "float32");
+  SetOp(&prog, "conv2d", "conv2", {"h", "weights2", "bias2"}, {"k"}, "float32");
+  SetOp(&prog, "pool2d", "pool2", {"k"}, {"l"}, "float32");
+
+  return prog;
+}
+
+void MainTest(std::initializer_list<std::string> quantize_enabled_op_types,
+              std::initializer_list<int> quantize_excluded_op_ids,
+              unsigned expected_int8_data_type_count) {
+  auto prog = BuildProgramDesc();
+
+  std::unique_ptr<ir::Graph> graph(new ir::Graph(prog));
+
+  auto pass = PassRegistry::Instance().Get("cpu_quantize_placement_pass");
+  pass->Set("quantize_enabled_op_types",
+            new std::unordered_set<std::string>(quantize_enabled_op_types));
+  pass->Set("quantize_excluded_op_ids",
+            new std::unordered_set<int>(quantize_excluded_op_ids));
+
+  graph.reset(pass->Apply(graph.release()));
+
+  unsigned int8_data_type_count = 0;
+
+  for (auto* node : graph->Nodes()) {
+    if (node->IsOp()) {
+      if (platform::HasOpINT8DataType(node->Op())) {
+        ++int8_data_type_count;
+      }
+    }
+  }
+
+  EXPECT_EQ(int8_data_type_count, expected_int8_data_type_count);
+}
+
+void DefaultAttrTest(unsigned expected_int8_data_type_count) {
+  auto prog = BuildProgramDesc();
+  std::unique_ptr<ir::Graph> graph(new ir::Graph(prog));
+  auto pass = PassRegistry::Instance().Get("cpu_quantize_placement_pass");
+  graph.reset(pass->Apply(graph.release()));
+
+  unsigned int8_data_type_count = 0;
+  for (auto* node : graph->Nodes()) {
+    if (node->IsOp()) {
+      if (platform::HasOpINT8DataType(node->Op())) {
+        ++int8_data_type_count;
+      }
+    }
+  }
+  EXPECT_EQ(int8_data_type_count, expected_int8_data_type_count);
+}
+
+TEST(QuantizerPlacementPass, enabled_pool) { MainTest({"pool2d"}, {}, 2); }
+
+TEST(QuantizerPlacementPass, enabled_conv_excluded_one) {
+  MainTest({"conv2d"}, {4}, 1);
+}
+
+TEST(QuantizerPlacementPass, empty_list) {
+  // all operators except relu should be quantized
+  MainTest({}, {}, 5);
+}
+
+TEST(QuantizerPlacementPass, default_attr_value) {
+  // all operators except relu should be quantized
+  DefaultAttrTest(5);
+}
+
+void EnabledOpTypesTest(
+    std::initializer_list<std::string> quantize_enabled_op_types,
+    std::string missing_op) {
+  auto prog = BuildProgramDesc();
+  std::unique_ptr<ir::Graph> graph(new ir::Graph(prog));
+
+  auto pass = PassRegistry::Instance().Get("cpu_quantize_placement_pass");
+  pass->Set("quantize_enabled_op_types",
+            new std::unordered_set<std::string>(quantize_enabled_op_types));
+
+  try {
+    graph.reset(pass->Apply(graph.release()));
+  } catch (paddle::platform::EnforceNotMet& err) {
+    std::string ex_msg = err.what();
+    std::string expected_msg =
+        "Pass attribute quantize_enabled_op_types contains operator " +
+        missing_op + " that is not supported by OneDNN quantization.";
+    EXPECT_TRUE(ex_msg.find(expected_msg) != std::string::npos);
+  }
+}
+
+TEST(QuantizerPlacementPass, unsupported_op_type) {
+  // Dropout op is not supported by OneDNN quantization
+  EnabledOpTypesTest({"conv2d", "dropout"}, "dropout");
+}
+
+}  // namespace ir
+}  // namespace framework
+}  // namespace paddle
+
+USE_PASS(cpu_quantize_placement_pass);

test/cpp/fluid/framework/ir/onednn/depthwise_conv_onednn_pass_test.cc

@@ -0,0 +1,157 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include "paddle/fluid/framework/ir/onednn/depthwise_conv_onednn_pass.h"
+#include "paddle/fluid/framework/op_version_registry.h"
+
+namespace paddle::framework::ir {
+
+void SetOp(ProgramDesc* prog,
+           const std::string& type,
+           const std::string& name,
+           const std::vector<std::string>& inputs,
+           const std::vector<std::string>& outputs,
+           bool use_onednn = false) {
+  auto* op = prog->MutableBlock(0)->AppendOp();
+  op->SetType(type);
+  op->SetAttr("use_onednn", use_onednn);
+  op->SetAttr("name", name);
+  op->SetAttr("groups", 1);
+  op->SetAttr("padding_algorithm", std::string("EXPLICIT"));
+  op->SetAttr("data_format", std::string("NCHW"));
+  op->SetAttr("strides", std::vector<int>({1, 1}));
+  op->SetAttr("dilations", std::vector<int>({1, 1}));
+  op->SetAttr("paddings", std::vector<int>({0, 0}));
+  op->SetInput("Input", {inputs[0]});
+  op->SetInput("Filter", {inputs[1]});
+  op->SetInput("Bias", {inputs[2]});
+  op->SetOutput("Output", outputs);
+}
+
+// (a, weights, bias)->depthwise conv onednn->b
+// (b, weights2, bias2)->depthwise conv no onednn->c
+// (c, weights3, bias3)->conv onednn->d
+// (d, weights3, bias3)->conv no onednn->e
+ProgramDesc BuildProgramDesc() {
+  ProgramDesc prog;
+  for (auto& v : std::vector<std::string>({"a",
+                                           "b",
+                                           "c",
+                                           "d",
+                                           "e",
+                                           "weights",
+                                           "bias",
+                                           "weights2",
+                                           "bias2",
+                                           "weights3",
+                                           "bias3",
+                                           "weights4",
+                                           "bias4"})) {
+    auto* var = prog.MutableBlock(0)->Var(v);
+    var->SetType(proto::VarType::SELECTED_ROWS);
+    if (v == "weights" || v == "bias" || v == "weights2" || v == "bias2" ||
+        v == "weights3" || v == "bias3" || v == "weights4" || v == "bias4") {
+      var->SetPersistable(true);
+    }
+  }
+
+  // depthwise conv with MKL-DNN
+  SetOp(&prog,
+        "depthwise_conv2d",
+        "conv1",
+        std::vector<std::string>({"a", "weights", "bias"}),
+        std::vector<std::string>({"b"}),
+        true);
+  // depthwise conv without MKL-DNN
+  SetOp(&prog,
+        "depthwise_conv2d",
+        "conv2",
+        std::vector<std::string>({"b", "weights2", "bias2"}),
+        std::vector<std::string>({"c"}),
+        false);
+  // conv with MKL-DNN
+  SetOp(&prog,
+        "conv2d",
+        "conv3",
+        std::vector<std::string>({"c", "weights3", "bias3"}),
+        std::vector<std::string>({"d"}),
+        true);
+  // conv without MKL-dNN
+  SetOp(&prog,
+        "conv2d",
+        "conv4",
+        std::vector<std::string>({"d", "weights4", "bias4"}),
+        std::vector<std::string>({"e"}),
+        false);
+
+  return prog;
+}
+
+TEST(DepthwiseConvOneDNNPass, pass_op_version_check) {
+  ASSERT_TRUE(
+      paddle::framework::compatible::PassVersionCheckerRegistrar::GetInstance()
+          .IsPassCompatible("depthwise_conv_onednn_pass"));
+}
+
+TEST(DepthwiseConvOneDNNPass, basic) {
+  auto prog = BuildProgramDesc();
+
+  std::unique_ptr<ir::Graph> graph(new ir::Graph(prog));
+
+  auto pass = PassRegistry::Instance().Get("depthwise_conv_onednn_pass");
+
+  struct counters {
+    int onednn_depthwise_conv_nodes;
+    int other_depthwise_conv_nodes;
+    int onednn_conv_nodes;
+    int other_conv_nodes;
+  };
+
+  counters before{1, 1, 1, 1};
+
+  graph.reset(pass->Apply(graph.release()));
+
+  // initialize counters before loop
+  counters after{0, 0, 0, 0};
+
+  for (auto* node : graph->Nodes()) {
+    if (node->IsOp()) {
+      auto* op = node->Op();
+      if (op->Type() == "conv2d") {
+        if (PADDLE_GET_CONST(bool, op->GetAttr("use_onednn")))
+          after.onednn_conv_nodes++;
+        else
+          after.other_conv_nodes++;
+      } else if (op->Type() == "depthwise_conv2d") {
+        if (PADDLE_GET_CONST(bool, op->GetAttr("use_onednn")))
+          after.onednn_depthwise_conv_nodes++;
+        else
+          after.other_depthwise_conv_nodes++;
+      }
+    }
+  }
+
+  EXPECT_EQ(after.other_depthwise_conv_nodes,
+            before.other_depthwise_conv_nodes);
+  EXPECT_EQ(after.other_conv_nodes, before.other_conv_nodes);
+  EXPECT_EQ(after.onednn_depthwise_conv_nodes,
+            before.onednn_depthwise_conv_nodes - 1);
+  EXPECT_EQ(after.onednn_conv_nodes, before.onednn_conv_nodes + 1);
+}
+
+}  // namespace paddle::framework::ir
+
+USE_PASS(depthwise_conv_onednn_pass);

test/cpp/fluid/framework/ir/onednn/int8_scale_calculation_onednn_pass_test.cc

@@ -0,0 +1,152 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include "paddle/fluid/framework/ir/onednn/int8_scale_calculation_onednn_pass.h"
+
+namespace paddle::framework::ir {
+
+void SetOp(ProgramDesc* prog,
+           const std::string& type,
+           const std::string& name,
+           const std::vector<std::string>& inputs,
+           const std::vector<std::string>& outputs,
+           std::vector<float> scale_weights = {1.5f}) {  // NOLINT
+  auto* op = prog->MutableBlock(0)->AppendOp();
+
+  op->SetType(type);
+  if (type == "conv2d") {
+    op->SetAttr("use_onednn", true);
+    op->SetAttr("name", name);
+    op->SetAttr("strides", std::vector<int>({1, 1}));
+    op->SetAttr("groups", 1);
+    op->SetAttr("paddings", std::vector<int>({0, 0}));
+    op->SetAttr("padding_algorithm", std::string("EXPLICIT"));
+    op->SetAttr("dilations", std::vector<int>({1, 1}));
+    op->SetAttr("data_format", std::string("NCHW"));
+    op->SetInput("Input", {inputs[0]});
+    op->SetInput("Filter", {inputs[1]});
+    if (inputs.size() > 2)
+      op->SetInput("Bias", {inputs[2]});
+    else
+      op->SetInput("Bias", {});
+
+    op->SetOutput("Output", outputs);
+    op->SetAttr("Scale_in", 1.0f);
+    op->SetAttr("Scale_out", 1.0f);
+    op->SetAttr("Scale_weights", scale_weights);
+    op->SetAttr("use_onednn", true);
+    op->SetAttr("onednn_data_type", std::string("int8"));
+  } else {
+    FAIL() << "Unexpected operator type.";
+  }
+}
+
+ProgramDesc BuildProgramDesc(bool convWithExistingBias,
+                             std::vector<float> scale_weights = {1.5f}) {
+  ProgramDesc prog;
+  std::vector<std::string> nodes{"c", "weights", "f"};
+  if (convWithExistingBias) nodes.push_back("conv_bias");
+  for (auto& v : nodes) {
+    auto* var = prog.MutableBlock(0)->Var(v);
+    var->SetType(proto::VarType::DENSE_TENSOR);
+    if (v == "weights") {
+      var->SetPersistable(true);
+      var->SetShape({1, static_cast<int>(scale_weights.size()), 1, 1});
+    }
+  }
+
+  if (convWithExistingBias || scale_weights.size() > 1) {
+    SetOp(&prog,
+          "conv2d",
+          "conv",
+          std::vector<std::string>({"c", "weights", "conv_bias"}),
+          std::vector<std::string>({"f"}),
+          scale_weights);
+  } else {
+    SetOp(&prog,
+          "conv2d",
+          "conv",
+          std::vector<std::string>({"c", "weights"}),
+          std::vector<std::string>({"f"}));
+  }
+
+  return prog;
+}
+
+void MainTest(bool convWithExistingBias,
+              int removed_nodes_count,
+              float scale,
+              std::vector<float> scale_weights = {1.5f}) {  // NOLINT
+  auto prog = BuildProgramDesc(convWithExistingBias, scale_weights);
+  std::unique_ptr<ir::Graph> graph(new ir::Graph(prog));
+  auto pass =
+      PassRegistry::Instance().Get("int8_scale_calculation_onednn_pass");
+  int original_nodes_num = graph->Nodes().size();
+  graph.reset(pass->Apply(graph.release()));
+  int current_nodes_num = graph->Nodes().size();
+
+  EXPECT_EQ(original_nodes_num, current_nodes_num);
+
+  for (auto* node : graph->Nodes()) {
+    if (node->IsOp() && node->Op()->Type() == "conv2d") {
+      auto* op = node->Op();
+      ASSERT_TRUE(op->HasAttr("use_mkldnn") || op->HasAttr("use_onednn"));
+
+      EXPECT_EQ(op->GetAttrIfExists<std::vector<float>>("Scale_weights"),
+                scale_weights);
+      EXPECT_EQ(op->GetAttrIfExists<float>("Scale_in"), scale);
+      EXPECT_EQ(op->GetAttrIfExists<float>("Scale_out"), scale);
+
+      EXPECT_EQ(op->GetAttrIfExists<float>("Sum_scale"), scale);
+      EXPECT_EQ(
+          op->GetAttrIfExists<std::vector<float>>("Output_shift_scale")[0],
+          scale / scale_weights[0]);
+      EXPECT_EQ(op->GetAttrIfExists<float>("Activation_scale"), scale);
+
+      if (convWithExistingBias) {
+        EXPECT_EQ(op->GetAttrIfExists<std::vector<float>>("Bias_scales")[0],
+                  scale * scale_weights[0]);
+      }
+    }
+  }
+  EXPECT_EQ(original_nodes_num - removed_nodes_count, current_nodes_num);
+}
+
+TEST(Int8ScaleCalculationOnednnPass, int8_scale_calculation_with_no_bias) {
+  auto scale = 1.0f;
+  int removed_nodes_count = 0;
+  auto scale_weights = {1.5f};
+  MainTest(false, removed_nodes_count, scale, scale_weights);
+}
+
+TEST(Int8ScaleCalculationOnednnPass, int8_scale_calculation_with_bias) {
+  auto scale = 1.0f;
+  int removed_nodes_count = 0;
+  auto scale_weights = {1.5f};
+  MainTest(true, removed_nodes_count, scale, scale_weights);
+}
+
+TEST(Int8ScaleCalculationOnednnPass,
+     int8_scale_calculation_with_bias_scale_weights) {
+  auto scale = 1.0f;
+  int removed_nodes_count = 0;
+  std::vector<float> scale_weights = {1.5f, 2.3f};
+  MainTest(true, removed_nodes_count, scale, scale_weights);
+}
+
+}  // namespace paddle::framework::ir
+
+USE_PASS(int8_scale_calculation_onednn_pass);

test/cpp/fluid/framework/ir/onednn/onednn_placement_pass_test.cc

@@ -0,0 +1,187 @@
+// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include "paddle/fluid/framework/ir/onednn/onednn_placement_pass.h"
+#include "paddle/fluid/framework/ir/pass_tester_helper.h"
+#include "paddle/utils/tribool.h"
+
+namespace paddle::framework::ir {
+
+class PlacementPassTest {
+ private:
+  void SetOp(ProgramDesc* prog,
+             const std::string& type,
+             const std::string& name,
+             const std::vector<std::string>& inputs,
+             const std::vector<std::string>& outputs,
+             paddle::tribool use_onednn) {
+    auto* op = prog->MutableBlock(0)->AppendOp();
+
+    op->SetType(type);
+
+    if (!paddle::indeterminate(use_onednn))
+      op->SetAttr("use_onednn", use_onednn);
+
+    if (type == "conv2d") {
+      op->SetAttr("name", name);
+      op->SetInput("Input", {inputs[0]});
+      op->SetInput("Filter", {inputs[1]});
+      op->SetInput("Bias", {inputs[2]});
+    } else if (type == "relu") {
+      op->SetInput("X", inputs);
+    } else if (type == "concat") {
+      op->SetAttr("axis", 1);
+      op->SetInput("X", {inputs[0], inputs[1]});
+    } else if (type == "pool2d") {
+      op->SetInput("X", {inputs[0]});
+    } else {
+      FAIL() << "Unexpected operator type.";
+    }
+    op->SetOutput("Out", {outputs[0]});
+  }
+
+  // operator                      use_onednn
+  // ---------------------------------------
+  // (a,b)->concat->c              none
+  // (c,weights,bias)->conv->f     none
+  // f->relu->g                    false
+  // g->pool->h                    false
+  // (h,weights2,bias2)->conv->k   true
+  // k->relu->l                    true
+  ProgramDesc BuildProgramDesc() {
+    ProgramDesc prog;
+
+    for (auto& v : std::vector<std::string>({"a",
+                                             "b",
+                                             "c",
+                                             "weights",
+                                             "bias",
+                                             "f",
+                                             "g",
+                                             "h",
+                                             "weights2",
+                                             "bias2",
+                                             "k",
+                                             "l"})) {
+      auto* var = prog.MutableBlock(0)->Var(v);
+      var->SetType(proto::VarType::SELECTED_ROWS);
+      var->SetDataType(framework::proto::VarType::FP32);
+      if (v == "weights" || v == "bias") {
+        var->SetPersistable(true);
+      }
+    }
+
+    SetOp(&prog,
+          "concat",
+          "concat1",
+          std::vector<std::string>({"a", "b"}),
+          std::vector<std::string>({"c"}),
+          paddle::indeterminate);
+    SetOp(&prog,
+          "conv2d",
+          "conv1",
+          std::vector<std::string>({"c", "weights", "bias"}),
+          std::vector<std::string>({"f"}),
+          paddle::indeterminate);
+    SetOp(&prog,
+          "relu",
+          "relu1",
+          std::vector<std::string>({"f"}),
+          std::vector<std::string>({"g"}),
+          false);
+    SetOp(&prog,
+          "pool2d",
+          "pool1",
+          std::vector<std::string>({"g"}),
+          std::vector<std::string>({"h"}),
+          false);
+    SetOp(&prog,
+          "conv2d",
+          "conv2",
+          std::vector<std::string>({"h", "weights2", "bias2"}),
+          std::vector<std::string>({"k"}),
+          true);
+    SetOp(&prog,
+          "relu",
+          "relu2",
+          std::vector<std::string>({"k"}),
+          std::vector<std::string>({"l"}),
+          true);
+
+    return prog;
+  }
+
+ public:
+  void MainTest(std::initializer_list<std::string> onednn_enabled_op_types,
+                unsigned expected_use_onednn_true_count) {
+    auto prog = BuildProgramDesc();
+    RegisterOpKernel({"conv2d", "pool2d", "concat", "relu"});
+    std::unique_ptr<ir::Graph> graph(new ir::Graph(prog));
+
+    auto pass = PassRegistry::Instance().Get("onednn_placement_pass");
+
+    pass->Set("onednn_enabled_op_types",
+              new std::unordered_set<std::string>(onednn_enabled_op_types));
+
+    graph.reset(pass->Apply(graph.release()));
+
+    unsigned use_onednn_true_count = 0;
+
+    for (auto* node : graph->Nodes()) {
+      if (node->IsOp()) {
+        auto* op = node->Op();
+        if ((op->HasAttr("use_mkldnn") &&
+             PADDLE_GET_CONST(bool, op->GetAttr("use_mkldnn"))) ||
+            (op->HasAttr("use_onednn") &&
+             PADDLE_GET_CONST(bool, op->GetAttr("use_onednn")))) {
+          ++use_onednn_true_count;
+        }
+      }
+    }
+
+    EXPECT_EQ(use_onednn_true_count, expected_use_onednn_true_count);
+  }
+
+  void PlacementNameTest() {
+    auto pass = PassRegistry::Instance().Get("onednn_placement_pass");
+    EXPECT_EQ(static_cast<PlacementPassBase*>(pass.get())->GetPlacementName(),
+              "ONEDNN");
+  }
+};
+
+TEST(ONEDNNPlacementPass, enable_conv_relu) {
+  // 2 conv (1 conv is always true) + 2 relu (1 relu is always true) + 0 pool
+  PlacementPassTest().MainTest({"conv2d", "relu"}, 4);
+}
+
+TEST(ONEDNNPlacementPass, enable_relu_pool) {
+  // 1 conv (1 conv is always true) + 2 relu (1 relu is always true) + 1 pool
+  PlacementPassTest().MainTest({"relu", "pool2d"}, 4);
+}
+
+TEST(ONEDNNPlacementPass, enable_all) {
+  // 2 conv (1 conv is always true) + 2 relu (1 relu is always true) + 1 pool +
+  // 1 concat
+  PlacementPassTest().MainTest({}, 6);
+}
+
+TEST(ONEDNNPlacementPass, placement_name) {
+  PlacementPassTest().PlacementNameTest();
+}
+
+}  // namespace paddle::framework::ir
+
+USE_PASS(onednn_placement_pass);

test/cpp/fluid/framework/ir/onednn/params_quantization_onednn_pass_test.cc

@@ -0,0 +1,383 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include "paddle/fluid/framework/ir/onednn/params_quantization_onednn_pass.h"  // NOLINT
+#include "paddle/fluid/imperative/type_defs.h"
+#include "paddle/phi/common/place.h"
+
+namespace paddle::framework::ir {
+namespace {
+struct Data {
+  Data() = default;
+
+  Data(std::vector<int64_t>&& data_shape, std::vector<float>&& raw_data)
+      : shape(std::move(data_shape)), data(std::move(raw_data)) {
+    auto size_from_shape = std::accumulate(
+        shape.begin(), shape.end(), 1, std::multiplies<int64_t>());
+    PADDLE_ENFORCE_EQ(
+        size_from_shape,
+        data.size(),
+        common::errors::InvalidArgument("Shape size doesn't match data size."));
+  }
+
+  const std::vector<int64_t>& getShape() const { return shape; }
+  const std::vector<float>& getData() const { return data; }
+
+ private:
+  const std::vector<int64_t> shape{};
+  const std::vector<float> data{};
+};
+
+struct TestScope {
+  void CreateTensor(const std::string& var_name, const Data& data) {
+    auto variable = scope.Var(var_name);
+    auto tensor = variable->GetMutable<phi::DenseTensor>();
+    tensor->Resize(common::make_ddim(data.getShape()));
+    auto dptr = tensor->mutable_data<float>(place);
+    std::copy(data.getData().begin(), data.getData().end(), dptr);
+  }
+
+  const phi::DenseTensor& GetTensor(const std::string& input) const {
+    Variable* var = scope.FindVar(input);
+    return var->Get<phi::DenseTensor>();
+  }
+
+  framework::Scope* Scope() { return &scope; }
+
+ private:
+  framework::Scope scope;
+  CPUPlace place;
+};
+
+struct ProgramStrategy {
+  virtual ~ProgramStrategy() = default;
+
+  std::unique_ptr<Graph> CreateGraph() {
+    CreateProgram();
+    auto graph = std::make_unique<ir::Graph>(program);
+    graph->SetNotOwned(kParamScopeAttr, test_scope.Scope());
+    return graph;
+  }
+
+  void CheckGraph(const std::unique_ptr<ir::Graph>& graph) const {
+    for (auto* node : graph->Nodes()) {
+      if (node->IsOp()) {
+        CheckOp(*node->Op());
+      }
+    }
+  }
+
+ protected:
+  virtual void CreateProgram() = 0;
+
+  virtual void CheckOp(const OpDesc& op) const = 0;
+
+  VarDesc* AddInput(OpDesc* op,
+                    std::string input_name,
+                    const Data& data,
+                    const std::string user_var_name = "") {
+    std::string var_name = user_var_name;
+    if (var_name.empty()) {
+      var_name = input_name + "_var";
+    }
+    op->SetInput(input_name, {var_name});
+    auto var = program.MutableBlock(0)->Var(var_name);
+    var->SetShape(data.getShape());
+    test_scope.CreateTensor(var_name, data);
+    return var;
+  }
+
+  void AddOutput(OpDesc* op,
+                 std::string output_name,
+                 const Data& data,
+                 const std::string user_var_name = "") {
+    std::string var_name = user_var_name;
+    if (var_name.empty()) {
+      var_name = output_name + "_var";
+    }
+    op->SetOutput(output_name, {var_name});
+    program.MutableBlock(0)->Var(var_name);
+    test_scope.CreateTensor(var_name, data);
+  }
+
+ protected:
+  TestScope test_scope;
+  ProgramDesc program;
+};
+
+struct ConvProgramStrategy : public ProgramStrategy {
+  ConvProgramStrategy(Data&& input,
+                      Data&& filter,
+                      Data&& output,
+                      std::vector<float>&& scale_weights,
+                      int groups = 1,
+                      Data&& bias = Data(),
+                      std::vector<float>&& scale_bias = {},
+                      bool share_weight = false)
+      : input(std::move(input)),
+        filter(std::move(filter)),
+        output(std::move(output)),
+        scale_weights(std::move(scale_weights)),
+        groups(std::move(groups)),
+        bias(std::move(bias)),
+        scale_bias(std::move(scale_bias)),
+        share_weight(std::move(share_weight)) {}
+
+ protected:
+  OpDesc* CreateBasicConvOp(const std::string conv_name = "Conv1") {
+    auto op = program.MutableBlock(0)->AppendOp();
+    op->SetType("fused_conv2d");
+    op->SetAttr("use_onednn", true);
+    op->SetAttr("name", conv_name);
+    op->SetAttr("onednn_data_type", std::string{"int8"});
+    op->SetAttr("data_format", std::string{"NCHW"});
+    op->SetAttr("dilations", std::vector<int>({1, 1}));
+    op->SetAttr("paddings", std::vector<int>({1, 1}));
+    op->SetAttr("strides", std::vector<int>({1, 1}));
+    return op;
+  }
+
+ protected:
+  void CreateProgram() override {
+    OpDesc* op = CreateBasicConvOp();
+    AddInput(op, "Input", input);
+    AddInput(op, "Filter", filter)->SetPersistable(true);
+    AddOutput(op, "Output", output);
+
+    op->SetAttr("Scale_weights", scale_weights);
+    op->SetAttr("Scale_in", 1.0f);
+    op->SetAttr("groups", groups);
+
+    if (HasBias()) {
+      AddInput(op, "Bias", bias);
+      op->SetAttr("Bias_scales", scale_bias);
+    }
+
+    if (share_weight) {
+      OpDesc* op2 = CreateBasicConvOp("Conv2");
+      AddInput(op2, "Input", input);
+      AddInput(op2, "Filter", filter)->SetPersistable(true);
+      AddOutput(op2, "Output", output, "output2");
+      op2->SetAttr("Scale_weights", scale_weights);
+      op2->SetAttr("Scale_in", 1.0f);
+      op2->SetAttr("groups", groups);
+      if (HasBias()) {
+        AddInput(op2, "Bias", bias, "Bias2");
+        op2->SetAttr("Bias_scales", scale_bias);
+      }
+    }
+  }
+
+  void CheckOp(const OpDesc& op) const override {
+    CheckFilter(op);
+    if (HasBias()) {
+      CheckBias(op);
+    }
+  }
+
+  bool HasBias() const { return !bias.getData().empty(); }
+
+  void CheckFilter(const OpDesc& op) const {
+    EXPECT_EQ(op.GetAttrIfExists<std::vector<float>>("Scale_weights"),
+              std::vector<float>(1, 1));
+
+    auto filter_inputs = op.Input("Filter");
+    ASSERT_EQ(filter_inputs.size(), 1ul);
+
+    auto tensor = test_scope.GetTensor(filter_inputs[0]);
+    ASSERT_EQ(tensor.dtype(), phi::DataType::INT8);
+
+    auto filter_ptr = tensor.data<int8_t>();
+    ASSERT_NE(filter_ptr, nullptr);
+    auto length = tensor.numel() / scale_weights.size();
+    for (int64_t i = 0; i < tensor.numel(); i++) {
+      EXPECT_EQ(filter_ptr[i],
+                static_cast<int8_t>(std::round(filter.getData()[i] *
+                                               scale_weights[i / length])));
+    }
+  }
+
+  void CheckBias(const OpDesc& op) const {
+    EXPECT_EQ(op.GetAttrIfExists<std::vector<float>>("Bias_scales"),
+              std::vector<float>(1, 1));
+
+    auto bias_inputs = op.Input("Bias");
+    ASSERT_EQ(bias_inputs.size(), 1ul);
+
+    auto tensor = test_scope.GetTensor(bias_inputs[0]);
+    auto bias_ptr = tensor.data<int32_t>();
+    ASSERT_NE(bias_ptr, nullptr);
+    auto length = tensor.numel() / scale_bias.size();
+    for (int64_t i = 0; i < tensor.numel(); i++) {
+      EXPECT_EQ(bias_ptr[i],
+                static_cast<int32_t>(
+                    std::round(bias.getData()[i] * scale_bias[i / length])));
+    }
+  }
+
+ private:
+  const Data input;
+  const Data filter;
+  const Data output;
+  const std::vector<float> scale_weights;
+  const int groups;
+  const Data bias;
+  const std::vector<float> scale_bias;
+  const bool share_weight;
+};
+
+struct ParamsQuantizationOnednnPassTestFixture : public ::testing::Test {
+  void RunPassTest(std::unique_ptr<ProgramStrategy> program) {
+    auto graph = program->CreateGraph();
+
+    auto pass = PassRegistry::Instance().Get("params_quantization_onednn_pass");
+    graph.reset(pass->Apply(graph.release()));
+
+    program->CheckGraph(graph);
+  }
+};
+
+Data GenericInput() { return Data({1, 4, 1, 1}, {1.5f, 1.5f, 1.5f, 1.5f}); }
+Data GenericOutput() { return GenericInput(); }
+
+TEST_F(ParamsQuantizationOnednnPassTestFixture, conv_without_bias_o1i1h1w1) {
+  auto program =
+      std::make_unique<ConvProgramStrategy>(GenericInput(),
+                                            Data({1, 1, 1, 1}, {1.5f}),
+                                            GenericOutput(),
+                                            std::vector<float>{2.f});
+  RunPassTest(std::move(program));
+}
+
+TEST_F(ParamsQuantizationOnednnPassTestFixture, conv_without_bias_2o1i1h1w) {
+  auto program =
+      std::make_unique<ConvProgramStrategy>(GenericInput(),
+                                            Data({2, 1, 1, 1}, {1.5f, 1.5f}),
+                                            GenericOutput(),
+                                            std::vector<float>{2.f, 4.f});
+  RunPassTest(std::move(program));
+}
+
+TEST_F(ParamsQuantizationOnednnPassTestFixture, conv_without_bias_2o2i2h2w) {
+  auto program =
+      std::make_unique<ConvProgramStrategy>(GenericInput(),
+                                            Data({2, 2, 2, 2},
+                                                 {1.5f,
+                                                  1.5f,
+                                                  1.5f,
+                                                  1.5f,
+                                                  1.5f,
+                                                  1.5f,
+                                                  1.5f,
+                                                  1.5f,
+                                                  1.5f,
+                                                  1.5f,
+                                                  1.5f,
+                                                  1.5f,
+                                                  1.5f,
+                                                  1.5f,
+                                                  1.5f,
+                                                  1.5f}),
+                                            GenericOutput(),
+                                            std::vector<float>{2.f, 4.f});
+  RunPassTest(std::move(program));
+}
+
+TEST_F(ParamsQuantizationOnednnPassTestFixture, conv_without_bias_2g2o2i1h1w) {
+  auto program = std::make_unique<ConvProgramStrategy>(
+      GenericInput(),
+      Data({2, 2, 2, 1, 1}, {1.5f, 1.5f, 1.5f, 1.5f, 1.5f, 1.5f, 1.5f, 1.5f}),
+      GenericOutput(),
+      std::vector<float>{2.f, 2.f, 2.f, 2.f},
+      2);
+  RunPassTest(std::move(program));
+}
+
+TEST_F(ParamsQuantizationOnednnPassTestFixture, conv_without_bias_2g2o1i1h1w) {
+  auto program = std::make_unique<ConvProgramStrategy>(
+      GenericInput(),
+      Data({2, 2, 1, 1, 1}, {1.5f, 1.5f, 1.5f, 1.5f}),
+      GenericOutput(),
+      std::vector<float>{2.f, 2.f, 2.f, 2.f},
+      2);
+  RunPassTest(std::move(program));
+}
+
+TEST_F(ParamsQuantizationOnednnPassTestFixture, conv_with_bias_1o1i1h1w) {
+  auto program =
+      std::make_unique<ConvProgramStrategy>(GenericInput(),
+                                            Data({1, 1, 1, 1}, {1.5f}),
+                                            GenericOutput(),
+                                            std::vector<float>{2.f},
+                                            1,
+                                            Data({1, 1, 1, 1}, {1.5f}),
+                                            std::vector<float>{2.f});
+  RunPassTest(std::move(program));
+}
+
+TEST_F(ParamsQuantizationOnednnPassTestFixture, conv_with_bias_2o1i1h1w) {
+  auto program =
+      std::make_unique<ConvProgramStrategy>(GenericInput(),
+                                            Data({2, 1, 1, 1}, {1.5f, 1.5f}),
+                                            GenericOutput(),
+                                            std::vector<float>{2.f, 4.f},
+                                            1,
+                                            Data({2, 1, 1, 1}, {1.5f, 1.5f}),
+                                            std::vector<float>{2.f, 4.f});
+  RunPassTest(std::move(program));
+}
+
+TEST_F(ParamsQuantizationOnednnPassTestFixture, conv_with_bias_2g2o1i1h1w) {
+  auto program = std::make_unique<ConvProgramStrategy>(
+      GenericInput(),
+      Data({4, 1, 1, 1}, {1.5f, 1.5f, 1.5f, 1.5f}),
+      GenericOutput(),
+      std::vector<float>{2.f, 2.f, 4.f, 4.f},
+      2,
+      Data({4, 1, 1, 1}, {1.5f, 1.5f, 1.5f, 1.5f}),
+      std::vector<float>{2.f, 2.f, 4.f, 4.f});
+  RunPassTest(std::move(program));
+}
+
+TEST_F(ParamsQuantizationOnednnPassTestFixture, conv_with_bias_2g2o2i1h1w) {
+  auto program = std::make_unique<ConvProgramStrategy>(
+      GenericInput(),
+      Data({2, 2, 2, 1, 1}, {1.5f, 1.5f, 1.5f, 1.5f, 1.5f, 1.5f, 1.5f, 1.5f}),
+      GenericOutput(),
+      std::vector<float>{2.f, 2.f, 4.f, 4.f},
+      2,
+      Data({2, 2, 1, 1, 1}, {1.5f, 1.5f, 1.5f, 1.5f}),
+      std::vector<float>{2.f, 2.f, 4.f, 4.f});
+  RunPassTest(std::move(program));
+}
+
+TEST_F(ParamsQuantizationOnednnPassTestFixture, conv_with_bias_2g2o2i1h1ws) {
+  auto program = std::make_unique<ConvProgramStrategy>(
+      GenericInput(),
+      Data({2, 2, 2, 1, 1}, {1.5f, 1.5f, 1.5f, 1.5f, 1.5f, 1.5f, 1.5f, 1.5f}),
+      GenericOutput(),
+      std::vector<float>{2.f, 2.f, 4.f, 4.f},
+      2,
+      Data({2, 2, 1, 1, 1}, {1.5f, 1.5f, 1.5f, 1.5f}),
+      std::vector<float>{2.f, 2.f, 4.f, 4.f},
+      true);
+  RunPassTest(std::move(program));
+}
+
+}  // namespace
+}  // namespace paddle::framework::ir
+
+USE_PASS(params_quantization_onednn_pass);

test/cpp/fluid/framework/ir/onednn/shuffle_channel_onednn_detect_pass_test.cc

@@ -0,0 +1,84 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include <vector>
+
+#include "paddle/fluid/framework/ir/onednn/shuffle_channel_onednn_detect_pass.h"
+#include "paddle/fluid/framework/ir/pass_tester_helper.h"
+
+namespace paddle::framework::ir {
+
+void AddVarToScope(Scope* param_scope,
+                   const std::string& name,
+                   const DDim& dims) {
+  auto* tensor = param_scope->Var(name)->GetMutable<phi::DenseTensor>();
+  tensor->Resize(dims);
+  tensor->mutable_data<float>(phi::CPUPlace());
+}
+
+Scope* CreateParamScope() {
+  auto param_scope = new Scope();
+  AddVarToScope(param_scope, "prog_x", {1, 128, 52, 52});
+  return param_scope;
+}
+
+void MainTest() {
+  Layers layers;
+  auto prog_x = layers.data("prog_x", {1, 128, 52, 52});
+  auto first_reshape2 = layers.reshape2(prog_x, {-1, 2, 64, 52, 52}, true);
+  first_reshape2->SetShape({-1, 2, 64, 52, 52});
+  auto transpose2 = layers.transpose2(first_reshape2, {0, 2, 1, 3, 4}, true);
+  transpose2->SetShape({-1, 64, 2, 52, 52});
+  auto second_reshape2 = layers.reshape2(transpose2, {-1, 128, 52, 52}, true);
+  second_reshape2->SetShape({-1, 128, 52, 52});
+
+  std::unique_ptr<ir::Graph> graph(new ir::Graph(layers.main_program()));
+  graph->Set("__param_scope__", CreateParamScope());
+
+  int added_nodes = 1;    // shuffle_channel
+  int removed_nodes = 5;  // 2 * reshape, reshape_out, transpose, transpose_out
+
+  int original_nodes_num = graph->Nodes().size();
+  auto pass =
+      PassRegistry::Instance().Get("shuffle_channel_onednn_detect_pass");
+  graph.reset(pass->Apply(graph.release()));
+  int current_nodes_num = graph->Nodes().size();
+
+  EXPECT_EQ(current_nodes_num,
+            original_nodes_num + added_nodes - removed_nodes);
+  EXPECT_EQ(GetNumOpNodes(graph, "reshape2"), 0);
+  EXPECT_EQ(GetNumOpNodes(graph, "transpose2"), 0);
+  EXPECT_EQ(GetNumOpNodes(graph, "shuffle_channel"), 1);
+
+  for (const auto* node : graph->Nodes()) {
+    if (node->IsOp() && node->Op()->Type() == "shuffle_channel") {
+      const auto* op = node->Op();
+      ASSERT_TRUE(op->HasAttr("use_mkldnn") || op->HasAttr("use_onednn"));
+      EXPECT_TRUE((op->HasAttr("use_mkldnn") &&
+                   PADDLE_GET_CONST(bool, op->GetAttr("use_mkldnn"))) ||
+                  (op->HasAttr("use_onednn") &&
+                   PADDLE_GET_CONST(bool, op->GetAttr("use_onednn"))));
+    }
+  }
+}
+
+TEST(ShuffleChannelOneDNNDetectPass, ShuffleChannelOneDNNDetectPassTest) {
+  MainTest();
+}
+
+}  // namespace paddle::framework::ir
+
+USE_PASS(shuffle_channel_onednn_detect_pass);