chore: import upstream snapshot with attribution

test/cpp/inference/CMakeLists.txt

@@ -0,0 +1,11 @@
+add_definitions(-DPADDLE_DLL_EXPORT)
+if(WITH_TESTING)
+  include(test.cmake) # some generic cmake function for inference
+  include(test_cases.cmake)
+endif()
+
+add_subdirectory(analysis)
+add_subdirectory(api)
+if(TENSORRT_FOUND AND WITH_TENSORRT)
+  add_subdirectory(tensorrt)
+endif()

test/cpp/inference/analysis/CMakeLists.txt

@@ -0,0 +1,39 @@
+function(inference_analysis_test_build TARGET)
+  if(WITH_TESTING)
+    set(options "")
+    set(oneValueArgs "")
+    set(multiValueArgs SRCS EXTRA_DEPS)
+    cmake_parse_arguments(analysis_test "${options}" "${oneValueArgs}"
+                          "${multiValueArgs}" ${ARGN})
+    inference_base_test_build(${TARGET} SRCS ${analysis_test_SRCS} DEPS
+                              ${analysis_test_EXTRA_DEPS})
+  endif()
+endfunction()
+
+function(inference_analysis_test_run TARGET)
+  if(WITH_TESTING)
+    set(options "")
+    set(oneValueArgs "")
+    set(multiValueArgs COMMAND ARGS)
+    cmake_parse_arguments(analysis_test "${options}" "${oneValueArgs}"
+                          "${multiValueArgs}" ${ARGN})
+    inference_base_test_run(${TARGET} COMMAND ${analysis_test_COMMAND} ARGS
+                            ${analysis_test_ARGS})
+    set_tests_properties(${TARGET} PROPERTIES LABELS "RUN_TYPE=INFER")
+  endif()
+endfunction()
+
+function(inference_analysis_test TARGET)
+  if(WITH_TESTING)
+    set(options "")
+    set(oneValueArgs "")
+    set(multiValueArgs SRCS ARGS EXTRA_DEPS)
+    cmake_parse_arguments(analysis_test "${options}" "${oneValueArgs}"
+                          "${multiValueArgs}" ${ARGN})
+    inference_base_test_build(${TARGET} SRCS ${analysis_test_SRCS} DEPS
+                              ${analysis_test_EXTRA_DEPS})
+    inference_base_test_run(${TARGET} COMMAND ${TARGET} ARGS
+                            ${analysis_test_ARGS})
+    set_tests_properties(${TARGET} PROPERTIES LABELS "RUN_TYPE=INFER")
+  endif()
+endfunction()

test/cpp/inference/api/analyzer_bfloat16_image_classification_tester.cc

@@ -0,0 +1,60 @@
+/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <fstream>
+#include <iostream>
+
+#include "paddle/fluid/inference/api/paddle_analysis_config.h"
+#include "paddle/phi/backends/cpu/cpu_info.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+PD_DEFINE_bool(enable_onednn, true, "Enable ONEDNN");
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+
+void SetConfig(AnalysisConfig *cfg) {
+  std::ifstream model_file(FLAGS_infer_model + "/__model__");
+  if (model_file.good())
+    cfg->SetModel(FLAGS_infer_model);
+  else
+    cfg->SetModel(FLAGS_infer_model + "/inference.pdmodel",
+                  FLAGS_infer_model + "/inference.pdiparams");
+  cfg->DisableGpu();
+  cfg->SwitchIrOptim();
+  cfg->SwitchSpecifyInputNames();
+  cfg->SetCpuMathLibraryNumThreads(FLAGS_num_threads);
+  if (!FLAGS_enable_onednn) cfg->DisableONEDNN();
+}
+
+TEST(Analyzer_bfloat16_image_classification, bfloat16) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  AnalysisConfig b_cfg;
+  SetConfig(&b_cfg);
+
+  // read data from file and prepare batches with test data
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInputs(&input_slots_all);
+  if (FLAGS_enable_onednn && FLAGS_enable_bf16 &&
+      phi::backends::cpu::MayIUse(phi::backends::cpu::cpu_isa_t::avx512_bf16)) {
+    b_cfg.EnableOnednnBfloat16();
+  } else {
+    FLAGS_enable_bf16 = false;
+  }
+  CompareBFloat16AndAnalysis(&cfg, &b_cfg, input_slots_all);
+}
+
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_capi_exp_gpu_tester.cc

@@ -0,0 +1,183 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+
+#include <string>
+#include <vector>
+
+#if defined(PADDLE_WITH_CUDA)
+#include <cuda_runtime.h>
+#endif
+
+#include "paddle/common/flags.h"
+#include "paddle/fluid/inference/capi_exp/pd_inference_api.h"
+
+PD_DEFINE_string(infer_model, "", "model path");
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+
+TEST(PD_Config, gpu_interface) {
+  std::string model_dir = FLAGS_infer_model + "/mobilenet";
+  std::string prog_file = model_dir + "/__model__";
+  std::string param_file = model_dir + "/__params__";
+  std::string opt_cache_dir = FLAGS_infer_model + "/OptimCacheDir";
+  const char* ops_name = "conv_2d";
+
+  PD_Config* config = PD_ConfigCreate();
+  PD_ConfigSetModel(config, prog_file.c_str(), param_file.c_str());
+  PD_ConfigSetOptimCacheDir(config, opt_cache_dir.c_str());
+
+  PD_ConfigEnableUseGpu(config, 100, 0, 0);
+  bool use_gpu = PD_ConfigUseGpu(config);
+  EXPECT_TRUE(use_gpu);
+  int init_size = PD_ConfigMemoryPoolInitSizeMb(config);
+  EXPECT_EQ(init_size, 100);
+  int gpu_device_id = PD_ConfigGpuDeviceId(config);
+  EXPECT_EQ(gpu_device_id, 0);
+  float frac = PD_ConfigFractionOfGpuMemoryForPool(config);
+  LOG(INFO) << frac;
+  PD_ConfigEnableCudnn(config);
+  bool cudnn = PD_ConfigCudnnEnabled(config);
+  EXPECT_TRUE(cudnn);
+
+  PD_ConfigEnableTensorRtEngine(
+      config, 1 << 20, 1, 3, PD_PRECISION_INT8, FALSE, TRUE);
+  bool trt_enable = PD_ConfigTensorRtEngineEnabled(config);
+  EXPECT_TRUE(trt_enable);
+
+  const char* tensor_name = "image";
+  std::array<size_t, 1> shapes_num = {4};
+  std::array<int32_t, 4> min_shape = {1, 3, 36, 36};
+  std::array<int32_t, 4> max_shape = {1, 3, 224, 224};
+  std::array<int32_t, 4> opt_shape = {1, 3, 224, 224};
+  int32_t* min_shape_ptr = min_shape.data();
+  int32_t* max_shape_ptr = max_shape.data();
+  int32_t* opt_shape_ptr = opt_shape.data();
+  PD_ConfigSetTrtDynamicShapeInfo(config,
+                                  1,
+                                  &tensor_name,
+                                  shapes_num.data(),
+                                  &min_shape_ptr,
+                                  &max_shape_ptr,
+                                  &opt_shape_ptr,
+                                  FALSE);
+  PD_ConfigDisableTensorRtOPs(config, 1, &ops_name);
+  PD_ConfigEnableVarseqlen(config);
+  bool oss_enabled = PD_ConfigTensorRtOssEnabled(config);
+  EXPECT_TRUE(oss_enabled);
+
+  PD_ConfigEnableTensorRtDla(config, 4);
+  bool dla_enabled = PD_ConfigTensorRtDlaEnabled(config);
+  EXPECT_TRUE(dla_enabled);
+
+  PD_ConfigEnableGpuMultiStream(config);
+  bool thread_local_thread = PD_ConfigThreadLocalStreamEnabled(config);
+  EXPECT_TRUE(thread_local_thread);
+
+#if defined(PADDLE_WITH_CUDA)
+  {
+    cudaStream_t external_stream;
+    cudaStreamCreate(&external_stream);
+    PD_ConfigSetExecStream(config, external_stream);
+  }
+#endif
+
+  PD_ConfigDisableGpu(config);
+  PD_ConfigDestroy(config);
+}
+
+TEST(PD_Config, use_gpu) {
+  std::string model_dir = FLAGS_infer_model + "/mobilenet";
+  PD_Config* config = PD_ConfigCreate();
+
+  PD_ConfigDisableGpu(config);
+  PD_ConfigSetCpuMathLibraryNumThreads(config, 10);
+  int num_thread = PD_ConfigGetCpuMathLibraryNumThreads(config);
+  EXPECT_EQ(num_thread, 10);
+
+  PD_ConfigSwitchIrDebug(config, TRUE);
+  PD_ConfigSetModelDir(config, model_dir.c_str());
+  PD_ConfigSetOptimCacheDir(config,
+                            (FLAGS_infer_model + "/OptimCacheDir").c_str());
+  const char* model_dir_ = PD_ConfigGetModelDir(config);
+  LOG(INFO) << model_dir_;
+
+  PD_ConfigEnableUseGpu(config, 100, 0, 0);
+  bool use_gpu = PD_ConfigUseGpu(config);
+  EXPECT_TRUE(use_gpu);
+  int device_id = PD_ConfigGpuDeviceId(config);
+  EXPECT_EQ(device_id, 0);
+  int init_size = PD_ConfigMemoryPoolInitSizeMb(config);
+  EXPECT_EQ(init_size, 100);
+
+  float frac = PD_ConfigFractionOfGpuMemoryForPool(config);
+  LOG(INFO) << frac;
+
+  PD_ConfigEnableCudnn(config);
+  bool cudnn = PD_ConfigCudnnEnabled(config);
+  EXPECT_TRUE(cudnn);
+
+  PD_ConfigSwitchIrOptim(config, TRUE);
+  bool ir_optim = PD_ConfigIrOptim(config);
+  EXPECT_TRUE(ir_optim);
+
+  PD_ConfigEnableTensorRtEngine(
+      config, 1 << 20, 1, 3, PD_PRECISION_FLOAT32, FALSE, FALSE);
+  bool trt_enable = PD_ConfigTensorRtEngineEnabled(config);
+  EXPECT_TRUE(trt_enable);
+  PD_ConfigEnableMemoryOptim(config, true);
+  bool memory_optim_enable = PD_ConfigMemoryOptimEnabled(config);
+  EXPECT_TRUE(memory_optim_enable);
+  PD_ConfigEnableProfile(config);
+  bool profiler_enable = PD_ConfigProfileEnabled(config);
+  EXPECT_TRUE(profiler_enable);
+  PD_ConfigSetInvalid(config);
+  bool is_valid = PD_ConfigIsValid(config);
+  EXPECT_FALSE(is_valid);
+  PD_ConfigDestroy(config);
+}
+
+TEST(PD_Config, trt_int8) {
+  std::string model_dir = FLAGS_infer_model + "/mobilenet";
+  PD_Config* config = PD_ConfigCreate();
+  PD_ConfigEnableUseGpu(config, 100, 0, 0);
+  PD_ConfigEnableTensorRtEngine(
+      config, 1 << 20, 1, 3, PD_PRECISION_INT8, FALSE, TRUE);
+  bool trt_enable = PD_ConfigTensorRtEngineEnabled(config);
+  EXPECT_TRUE(trt_enable);
+  PD_ConfigDestroy(config);
+}
+
+TEST(PD_Config, trt_fp16) {
+  std::string model_dir = FLAGS_infer_model + "/mobilenet";
+  PD_Config* config = PD_ConfigCreate();
+  PD_ConfigEnableUseGpu(config, 100, 0, 0);
+  PD_ConfigEnableTensorRtEngine(
+      config, 1 << 20, 1, 3, PD_PRECISION_HALF, FALSE, FALSE);
+  bool trt_enable = PD_ConfigTensorRtEngineEnabled(config);
+  EXPECT_TRUE(trt_enable);
+  PD_Predictor* predictor = PD_PredictorCreate(config);
+  PD_PredictorDestroy(predictor);
+}
+
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_capi_exp_int_tester.cc

@@ -0,0 +1,95 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+
+#include <string>
+#include <vector>
+
+#include "paddle/common/flags.h"
+#include "paddle/fluid/inference/capi_exp/pd_inference_api.h"
+
+PD_DEFINE_string(infer_model, "", "model path");
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+
+void predictor_run() {
+  std::string model_dir = FLAGS_infer_model;
+  PD_Config* config = PD_ConfigCreate();
+  PD_ConfigDisableGpu(config);
+  PD_ConfigSetCpuMathLibraryNumThreads(config, 10);
+  PD_ConfigSwitchIrDebug(config, TRUE);
+  PD_ConfigSetModel(config,
+                    (model_dir + "/inference.pdmodel").c_str(),
+                    (model_dir + "/inference.pdiparams").c_str());
+  PD_Predictor* predictor = PD_PredictorCreate(config);
+  PD_OneDimArrayCstr* input_names = PD_PredictorGetInputNames(predictor);
+  LOG(INFO) << "The inputs' size is: " << input_names->size;
+  EXPECT_EQ(input_names->size, 1u);
+  PD_IOInfos* in_infos = PD_PredictorGetInputInfos(predictor);
+  EXPECT_EQ(in_infos->size, 1u);
+  PD_IOInfos* out_infos = PD_PredictorGetOutputInfos(predictor);
+
+  std::array<int32_t, 4> shape_0 = {1, 3, 224, 224};
+  std::array<float, 1 * 3 * 224 * 224> data_0 = {0};
+  PD_Tensor* input_0 = PD_PredictorGetInputHandle(predictor, "x");
+  PD_TensorReshape(input_0, 4, shape_0.data());
+  PD_TensorCopyFromCpuFloat(input_0, data_0.data());
+  LOG(INFO) << "Run Inference in CAPI encapsulation. ";
+  EXPECT_TRUE(PD_PredictorRun(predictor));
+
+  PD_OneDimArrayCstr* output_names = PD_PredictorGetOutputNames(predictor);
+  LOG(INFO) << "output size is: " << output_names->size;
+  for (size_t index = 0; index < output_names->size; ++index) {
+    LOG(INFO) << "output[" << index
+              << "]'s name is: " << output_names->data[index];
+    PD_Tensor* output =
+        PD_PredictorGetOutputHandle(predictor, output_names->data[index]);
+    PD_OneDimArrayInt32* shape = PD_TensorGetShape(output);
+    LOG(INFO) << "output[" << index << "]'s shape_size is: " << shape->size;
+    int32_t out_size = 1;
+    for (size_t i = 0; i < shape->size; ++i) {
+      LOG(INFO) << "output[" << index << "]'s shape is: " << shape->data[i];
+      out_size = out_size * shape->data[i];
+    }
+    float* out_data = new float[out_size];
+    PD_TensorCopyToCpuFloat(output, out_data);
+    LOG(INFO) << "output[" << index << "]'s DATA is: " << out_data[0];
+    delete[] out_data;
+    PD_OneDimArrayInt32Destroy(shape);
+    PD_TensorDestroy(output);
+  }
+  PD_PredictorClearIntermediateTensor(predictor);
+  PD_PredictorTryShrinkMemory(predictor);
+  PD_OneDimArrayCstrDestroy(output_names);
+  PD_TensorDestroy(input_0);
+  PD_OneDimArrayCstrDestroy(input_names);
+  PD_IOInfosDestroy(in_infos);
+  PD_IOInfosDestroy(out_infos);
+  PD_PredictorDestroy(predictor);
+}
+
+#ifdef PADDLE_WITH_DNNL
+TEST(PD_PredictorRun, predictor_run) { predictor_run(); }
+#endif
+
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_capi_exp_ner_tester.cc

@@ -0,0 +1,113 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+
+#include <string>
+#include <vector>
+
+#include "paddle/common/flags.h"
+#include "paddle/fluid/inference/capi_exp/pd_inference_api.h"
+
+PD_DEFINE_string(infer_model, "", "model path");
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+
+TEST(PD_PredictorRun, predictor_run) {
+  auto model_dir = FLAGS_infer_model;
+  PD_Config *config = PD_ConfigCreate();
+  PD_ConfigSetModel(config,
+                    (model_dir + "/__model__").c_str(),
+                    (model_dir + "/param").c_str());
+  PD_ConfigDisableGpu(config);
+
+  PD_Predictor *predictor = PD_PredictorCreate(config);
+  size_t input_num = PD_PredictorGetInputNum(predictor);
+  LOG(INFO) << "Input num: " << input_num;
+  size_t output_num = PD_PredictorGetOutputNum(predictor);
+  LOG(INFO) << "Output num: " << output_num;
+
+  PD_OneDimArrayCstr *input_names = PD_PredictorGetInputNames(predictor);
+  EXPECT_EQ(input_names->size, 2u);
+  LOG(INFO) << "Predictor start run!";
+  PD_Tensor *inputs[2];  // NOLINT
+  inputs[0] = PD_PredictorGetInputHandle(predictor, input_names->data[0]);
+  inputs[1] = PD_PredictorGetInputHandle(predictor, input_names->data[1]);
+  LOG(INFO) << "Predictor start run!";
+  // inputs[0]: word, use lod memory in stack
+  std::array<int32_t, 2> shape_0 = {11, 1};
+  std::array<int64_t, 11 * 1> data_0 = {
+      12673, 9763, 905, 284, 45, 7474, 20, 17, 1, 4, 9};
+  std::array<size_t, 2> lod_layer_0 = {0, 11};
+  PD_OneDimArraySize layer_0;
+  layer_0.size = 2;
+  layer_0.data = lod_layer_0.data();
+  PD_OneDimArraySize *layer_0_ptr = &layer_0;
+  PD_TwoDimArraySize lod_0;
+  lod_0.size = 1;
+  lod_0.data = &layer_0_ptr;
+  PD_TensorReshape(inputs[0], 2, shape_0.data());
+  PD_TensorCopyFromCpuInt64(inputs[0], data_0.data());
+  PD_TensorSetLod(inputs[0], &lod_0);
+
+  // inputs[1]: mention, use lod memory in heap
+  std::array<int32_t, 2> shape_1 = {11, 1};
+  std::array<int64_t, 11 * 1> data_1 = {27, 0, 0, 33, 34, 33, 0, 0, 0, 1, 2};
+  PD_TwoDimArraySize *lod_1_ptr = new PD_TwoDimArraySize();
+  lod_1_ptr->size = 1;
+  lod_1_ptr->data = new PD_OneDimArraySize *[1];
+  lod_1_ptr->data[0] = new PD_OneDimArraySize();
+  lod_1_ptr->data[0]->size = 2;
+  lod_1_ptr->data[0]->data = new size_t[2];
+  lod_1_ptr->data[0]->data[0] = 0;
+  lod_1_ptr->data[0]->data[1] = 11;
+
+  PD_TensorReshape(inputs[1], 2, shape_1.data());
+  PD_TensorCopyFromCpuInt64(inputs[1], data_1.data());
+  PD_TensorSetLod(inputs[1], lod_1_ptr);
+  // retrieve the lod memory
+  delete[] lod_1_ptr->data[0]->data;
+  delete lod_1_ptr->data[0];
+  delete[] lod_1_ptr->data;
+  delete lod_1_ptr;
+  lod_1_ptr = nullptr;
+
+  LOG(INFO) << "Predictor start run!";
+  bool success = PD_PredictorRun(predictor);
+  EXPECT_TRUE(success);
+  LOG(INFO) << "Predictor run success!";
+  PD_OneDimArrayCstr *output_names = PD_PredictorGetOutputNames(predictor);
+  PD_Tensor *output =
+      PD_PredictorGetOutputHandle(predictor, output_names->data[0]);
+  PD_TwoDimArraySize *output_lod = PD_TensorGetLod(output);
+
+  PD_TwoDimArraySizeDestroy(output_lod);
+  PD_TensorDestroy(output);
+  PD_OneDimArrayCstrDestroy(output_names);
+
+  PD_TensorDestroy(inputs[0]);
+  PD_TensorDestroy(inputs[1]);
+  PD_OneDimArrayCstrDestroy(input_names);
+  PD_PredictorDestroy(predictor);
+}
+
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_capi_exp_pd_config_tester.cc

@@ -0,0 +1,116 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+
+#include <string>
+#include <vector>
+
+#include "paddle/common/flags.h"
+#include "paddle/fluid/inference/capi_exp/pd_inference_api.h"
+
+PD_DEFINE_string(infer_model, "", "model path");
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+
+TEST(PD_Config, interface) {
+  std::string model_dir = FLAGS_infer_model + "/mobilenet";
+  std::string prog_file = model_dir + "/__model__";
+  std::string param_file = model_dir + "/__params__";
+  std::string opt_cache_dir = FLAGS_infer_model + "/OptimCacheDir";
+
+  PD_Config* config = PD_ConfigCreate();
+  PD_ConfigSetModelDir(config, model_dir.c_str());
+  std::string model_dir_ = PD_ConfigGetModelDir(config);
+  EXPECT_EQ(model_dir, model_dir_);
+
+  PD_ConfigSetModel(config, prog_file.c_str(), param_file.c_str());
+  PD_ConfigSetProgFile(config, prog_file.c_str());
+  PD_ConfigSetParamsFile(config, param_file.c_str());
+  PD_ConfigSetOptimCacheDir(config, opt_cache_dir.c_str());
+  std::string prog_file_ = PD_ConfigGetProgFile(config);
+  std::string param_file_ = PD_ConfigGetParamsFile(config);
+  EXPECT_EQ(prog_file, prog_file_);
+  EXPECT_EQ(param_file, param_file_);
+
+  PD_ConfigDisableFCPadding(config);
+  bool fc_padding = PD_ConfigUseFcPadding(config);
+  EXPECT_FALSE(fc_padding);
+
+  PD_ConfigDisableGpu(config);
+  PD_ConfigSwitchIrOptim(config, TRUE);
+  bool ir_optim = PD_ConfigIrOptim(config);
+  EXPECT_TRUE(ir_optim);
+
+  PD_ConfigEnableMemoryOptim(config, true);
+  bool memory_enabled = PD_ConfigMemoryOptimEnabled(config);
+  EXPECT_TRUE(memory_enabled);
+
+  PD_ConfigSwitchIrDebug(config, TRUE);
+#ifdef PADDLE_WITH_DNNL
+  const char* ops_name = "conv_2d";
+  PD_ConfigEnableONEDNN(config);
+  PD_ConfigSetOnednnOp(config, 1, &ops_name);
+  PD_ConfigSetOnednnCacheCapacity(config, 100);
+  bool onednn_enabled = PD_ConfigOnednnEnabled(config);
+  EXPECT_TRUE(onednn_enabled);
+
+  PD_ConfigSetCpuMathLibraryNumThreads(config, 10);
+  int32_t cpu_threads = PD_ConfigGetCpuMathLibraryNumThreads(config);
+  EXPECT_EQ(cpu_threads, 10);
+
+  PD_ConfigEnableOnednnBfloat16(config);
+  PD_ConfigSetBfloat16Op(config, 1, &ops_name);
+
+  PD_ConfigEnableOnednnInt8(config);
+  bool onednn_int8_enabled = PD_ConfigOnednnInt8Enabled(config);
+  EXPECT_TRUE(onednn_int8_enabled);
+#endif
+
+  PD_ConfigEnableONNXRuntime(config);
+  bool onnxruntime_enabled = PD_ConfigONNXRuntimeEnabled(config);
+#ifdef PADDLE_WITH_ONNXRUNTIME
+  EXPECT_TRUE(onnxruntime_enabled);
+#else
+  EXPECT_FALSE(onnxruntime_enabled);
+#endif
+  PD_ConfigDisableONNXRuntime(config);
+  bool onnxruntime_disabled = PD_ConfigONNXRuntimeEnabled(config);
+  EXPECT_FALSE(onnxruntime_disabled);
+  PD_ConfigEnableORTOptimization(config);
+
+  PD_ConfigEnableProfile(config);
+  bool profile_enabled = PD_ConfigProfileEnabled(config);
+  EXPECT_TRUE(profile_enabled);
+
+  PD_ConfigDisableGlogInfo(config);
+  bool glog_disabled = PD_ConfigGlogInfoDisabled(config);
+  EXPECT_TRUE(glog_disabled);
+
+  PD_ConfigSetInvalid(config);
+  bool is_valid = PD_ConfigIsValid(config);
+  EXPECT_FALSE(is_valid);
+
+  PD_ConfigDestroy(config);
+}
+
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_capi_exp_pd_tensor_tester.cc

@@ -0,0 +1,211 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+
+#include <fstream>
+#include <iostream>
+#include <numeric>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "paddle/common/flags.h"
+#include "paddle/fluid/inference/capi_exp/pd_inference_api.h"
+
+PD_DEFINE_string(infer_model, "", "model path");
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+
+void PD_run() {
+  auto model_dir = FLAGS_infer_model;
+  PD_Config* config = PD_ConfigCreate();
+  PD_ConfigSetModel(config,
+                    (model_dir + "/__model__").c_str(),
+                    (model_dir + "/__params__").c_str());
+  PD_Predictor* predictor = PD_PredictorCreate(config);
+  PD_OneDimArrayCstr* input_names = PD_PredictorGetInputNames(predictor);
+  PD_Tensor* tensor =
+      PD_PredictorGetInputHandle(predictor, input_names->data[0]);
+
+  std::array<int32_t, 4> shapes = {1, 3, 224, 224};
+  std::vector<float> input(1 * 3 * 224 * 224, 0);
+  int32_t size;
+  PD_PlaceType place;
+  PD_TensorReshape(tensor, 4, shapes.data());
+  PD_TensorCopyFromCpuFloat(tensor, input.data());
+  PD_TensorDataFloat(tensor, &place, &size);
+  PD_TensorMutableDataFloat(tensor, place);
+
+  PD_TwoDimArraySize lod;
+  lod.size = 0;
+  lod.data = nullptr;
+  PD_TensorSetLod(tensor, &lod);
+
+  PD_PredictorRun(predictor);
+
+  std::vector<float> out_data;
+  PD_OneDimArrayCstr* output_names = PD_PredictorGetOutputNames(predictor);
+  PD_Tensor* output_tensor =
+      PD_PredictorGetOutputHandle(predictor, output_names->data[0]);
+  PD_OneDimArrayInt32* output_shape = PD_TensorGetShape(output_tensor);
+  int32_t out_num = std::accumulate(output_shape->data,
+                                    output_shape->data + output_shape->size,
+                                    1,
+                                    std::multiplies<>());
+  out_data.resize(out_num);
+  PD_TensorCopyToCpuFloat(output_tensor, out_data.data());
+  LOG(INFO) << "Output tensor name is: " << PD_TensorGetName(output_tensor);
+  PD_DataType data_type = PD_TensorGetDataType(output_tensor);
+  EXPECT_EQ(data_type, PD_DATA_FLOAT32);
+
+  PD_TwoDimArraySize* out_lod = PD_TensorGetLod(output_tensor);
+
+  PD_TwoDimArraySizeDestroy(out_lod);
+  PD_OneDimArrayInt32Destroy(output_shape);
+  PD_TensorDestroy(output_tensor);
+  PD_OneDimArrayCstrDestroy(output_names);
+  PD_TensorDestroy(tensor);
+  PD_OneDimArrayCstrDestroy(input_names);
+  PD_PredictorDestroy(predictor);
+}
+TEST(PD_Tensor, PD_run) { PD_run(); }
+
+TEST(PD_Tensor, int32) {
+  auto model_dir = FLAGS_infer_model;
+  PD_Config* config = PD_ConfigCreate();
+  PD_ConfigSetModel(config,
+                    (model_dir + "/__model__").c_str(),
+                    (model_dir + "/__params__").c_str());
+  PD_Predictor* predictor = PD_PredictorCreate(config);
+  PD_OneDimArrayCstr* input_names = PD_PredictorGetInputNames(predictor);
+  PD_Tensor* tensor =
+      PD_PredictorGetInputHandle(predictor, input_names->data[0]);
+  std::array<int32_t, 4> shapes = {1, 3, 224, 224};
+  std::vector<int32_t> input(1 * 3 * 224 * 224, 0);
+  int32_t size;
+  PD_PlaceType place;
+  PD_TensorReshape(tensor, 4, shapes.data());
+  PD_TensorCopyFromCpuInt32(tensor, input.data());
+  int32_t* data_ptr = PD_TensorDataInt32(tensor, &place, &size);
+  EXPECT_EQ(place, PD_PLACE_CPU);
+  EXPECT_EQ(size, 1 * 3 * 224 * 224);
+  int32_t* mutable_data_ptr = PD_TensorMutableDataInt32(tensor, place);
+  EXPECT_EQ(data_ptr, mutable_data_ptr);
+
+  PD_DataType data_type = PD_TensorGetDataType(tensor);
+  EXPECT_EQ(data_type, PD_DATA_INT32);
+  PD_TensorCopyToCpuInt32(tensor, input.data());
+
+  PD_TensorDestroy(tensor);
+  PD_OneDimArrayCstrDestroy(input_names);
+  PD_PredictorDestroy(predictor);
+}
+
+TEST(PD_Tensor, int64) {
+  auto model_dir = FLAGS_infer_model;
+  PD_Config* config = PD_ConfigCreate();
+  PD_ConfigSetModel(config,
+                    (model_dir + "/__model__").c_str(),
+                    (model_dir + "/__params__").c_str());
+  PD_Predictor* predictor = PD_PredictorCreate(config);
+  PD_OneDimArrayCstr* input_names = PD_PredictorGetInputNames(predictor);
+  PD_Tensor* tensor =
+      PD_PredictorGetInputHandle(predictor, input_names->data[0]);
+  std::array<int32_t, 4> shapes = {1, 3, 224, 224};
+  std::vector<int64_t> input(1 * 3 * 224 * 224, 0);
+  int32_t size;
+  PD_PlaceType place;
+  PD_TensorReshape(tensor, 4, shapes.data());
+  PD_TensorCopyFromCpuInt64(tensor, input.data());
+  int64_t* data_ptr = PD_TensorDataInt64(tensor, &place, &size);
+  EXPECT_EQ(place, PD_PLACE_CPU);
+  EXPECT_EQ(size, 1 * 3 * 224 * 224);
+  int64_t* mutable_data_ptr = PD_TensorMutableDataInt64(tensor, place);
+  EXPECT_EQ(data_ptr, mutable_data_ptr);
+
+  PD_DataType data_type = PD_TensorGetDataType(tensor);
+  EXPECT_EQ(data_type, PD_DATA_INT64);
+  PD_TensorCopyToCpuInt64(tensor, input.data());
+
+  PD_TensorDestroy(tensor);
+  PD_OneDimArrayCstrDestroy(input_names);
+  PD_PredictorDestroy(predictor);
+}
+
+TEST(PD_Tensor, uint8) {
+  auto model_dir = FLAGS_infer_model;
+  PD_Config* config = PD_ConfigCreate();
+  PD_ConfigSetModel(config,
+                    (model_dir + "/__model__").c_str(),
+                    (model_dir + "/__params__").c_str());
+  PD_Predictor* predictor = PD_PredictorCreate(config);
+  PD_OneDimArrayCstr* input_names = PD_PredictorGetInputNames(predictor);
+  PD_Tensor* tensor =
+      PD_PredictorGetInputHandle(predictor, input_names->data[0]);
+  std::array<int32_t, 4> shapes = {1, 3, 224, 224};
+  std::array<uint8_t, 1 * 3 * 224 * 224> input = {0};
+  int32_t size;
+  PD_PlaceType place;
+  PD_TensorReshape(tensor, 4, shapes.data());
+  PD_TensorCopyFromCpuUint8(tensor, input.data());
+  uint8_t* data_ptr = PD_TensorDataUint8(tensor, &place, &size);
+  EXPECT_EQ(place, PD_PLACE_CPU);
+  EXPECT_EQ(size, 1 * 3 * 224 * 224);
+  uint8_t* mutable_data_ptr = PD_TensorMutableDataUint8(tensor, place);
+  EXPECT_EQ(data_ptr, mutable_data_ptr);
+
+  PD_DataType data_type = PD_TensorGetDataType(tensor);
+  EXPECT_EQ(data_type, PD_DATA_UINT8);
+  PD_TensorCopyToCpuUint8(tensor, input.data());
+
+  PD_TensorDestroy(tensor);
+  PD_OneDimArrayCstrDestroy(input_names);
+  PD_PredictorDestroy(predictor);
+}
+
+std::string read_file(std::string filename) {
+  std::ifstream file(filename);
+  return std::string((std::istreambuf_iterator<char>(file)),
+                     std::istreambuf_iterator<char>());
+}
+
+TEST(PD_Tensor, from_buffer) {
+  PD_Config* config = PD_ConfigCreate();
+  std::string prog_file = FLAGS_infer_model + "/__model__";
+  std::string params_file = FLAGS_infer_model + "/__params__";
+
+  std::string prog_str = read_file(prog_file);
+  std::string params_str = read_file(params_file);
+
+  PD_ConfigSetModelBuffer(config,
+                          prog_str.c_str(),
+                          prog_str.size(),
+                          params_str.c_str(),
+                          params_str.size());
+
+  bool model_from_memory = PD_ConfigModelFromMemory(config);
+  EXPECT_TRUE(model_from_memory);
+  PD_ConfigDestroy(config);
+}
+
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_capi_exp_pd_threads_tester.cc

@@ -0,0 +1,117 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+
+#include <fstream>
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "paddle/common/flags.h"
+#include "paddle/fluid/inference/capi_exp/pd_inference_api.h"
+
+PD_DEFINE_string(infer_model, "", "model path");
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+
+typedef struct RunParameter {
+  PD_Predictor* predictor;
+  int32_t* shapes;
+  size_t shape_size;
+  float* input_data;
+  int32_t out_size;
+  std::vector<float> out_data;
+  int32_t thread_index;
+} RunParameter;
+
+void* run(void* thread_param) {
+  struct RunParameter* param = (struct RunParameter*)thread_param;
+  LOG(INFO) << "Thread " << param->thread_index << " start run!";
+  PD_OneDimArrayCstr* input_names = PD_PredictorGetInputNames(param->predictor);
+  PD_Tensor* tensor =
+      PD_PredictorGetInputHandle(param->predictor, input_names->data[0]);
+  PD_TensorReshape(tensor, param->shape_size, param->shapes);
+  PD_TensorCopyFromCpuFloat(tensor, param->input_data);
+  PD_PredictorRun(param->predictor);
+  PD_OneDimArrayCstr* output_names =
+      PD_PredictorGetOutputNames(param->predictor);
+  PD_Tensor* output_tensor =
+      PD_PredictorGetOutputHandle(param->predictor, output_names->data[0]);
+  PD_OneDimArrayInt32* output_shape = PD_TensorGetShape(output_tensor);
+  param->out_size = 1;
+  for (size_t index = 0; index < output_shape->size; ++index) {
+    param->out_size = param->out_size * output_shape->data[index];
+  }
+  PD_OneDimArrayInt32Destroy(output_shape);
+  param->out_data.resize(param->out_size);
+  PD_TensorCopyToCpuFloat(output_tensor, param->out_data.data());
+  PD_TensorDestroy(output_tensor);
+  PD_OneDimArrayCstrDestroy(output_names);
+  PD_TensorDestroy(tensor);
+  PD_OneDimArrayCstrDestroy(input_names);
+  LOG(INFO) << "Thread " << param->thread_index << " end run!";
+  return nullptr;
+}
+void threads_run(int thread_num) {
+  auto model_dir = FLAGS_infer_model;
+  PD_Config* config = PD_ConfigCreate();
+  PD_ConfigSetModel(config,
+                    (model_dir + "/__model__").c_str(),
+                    (model_dir + "/__params__").c_str());
+  PD_Predictor* predictor = PD_PredictorCreate(config);
+
+  std::vector<pthread_t> threads(thread_num);
+  std::vector<RunParameter> params(thread_num);
+
+  std::array<int32_t, 4> shapes = {1, 3, 224, 224};
+  std::vector<float> input(1 * 3 * 224 * 224, 0);
+  for (int i = 0; i < thread_num; ++i) {
+    params[i].predictor = PD_PredictorClone(predictor);
+    params[i].shapes = shapes.data();
+    params[i].shape_size = 4;
+    params[i].input_data = input.data();
+    params[i].out_size = 0;
+    params[i].thread_index = i;
+    pthread_create(&(threads[i]), nullptr, run, &(params[i]));
+  }
+  for (int i = 0; i < thread_num; ++i) {
+    pthread_join(threads[i], nullptr);
+  }
+  ASSERT_GT(params[0].out_size, 0);
+
+  for (int i = 1; i < thread_num; ++i) {
+    ASSERT_EQ(params[i].out_size, params[0].out_size);
+    for (int j = 0; j < params[i].out_size; ++j) {
+      ASSERT_EQ(params[i].out_data[j], params[0].out_data[j]);
+    }
+  }
+  for (int i = 0; i < thread_num; ++i) {
+    PD_PredictorDestroy(params[i].predictor);
+  }
+  PD_PredictorDestroy(predictor);
+}
+
+TEST(PD_Predictor, PD_multi_threads_run) { threads_run(10); }
+
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_capi_exp_tester.cc

@@ -0,0 +1,90 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <string>
+#include <vector>
+
+#include "paddle/common/flags.h"
+#include "paddle/fluid/inference/capi_exp/pd_config.h"
+#include "paddle/fluid/inference/capi_exp/pd_inference_api.h"
+#include "paddle/fluid/inference/capi_exp/pd_utils.h"
+
+PD_DEFINE_string(infer_model, "", "model path");
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+
+void predictor_run() {
+  std::string model_dir = FLAGS_infer_model;
+  std::string prog_file = model_dir + "/model";
+  std::string params_file = model_dir + "/params";
+  PD_Config *config = PD_ConfigCreate();
+  PD_ConfigDisableGpu(config);
+  PD_ConfigSetCpuMathLibraryNumThreads(config, 10);
+  PD_ConfigSwitchIrDebug(config, TRUE);
+  PD_ConfigSetModel(config, prog_file.c_str(), params_file.c_str());
+  PD_Cstr *config_summary = PD_ConfigSummary(config);
+  LOG(INFO) << config_summary->data;
+
+  PD_Predictor *predictor = PD_PredictorCreate(config);
+  PD_Tensor *tensor = PD_PredictorGetInputHandle(predictor, "data");
+
+  const int batch_size = 1;
+  const int channels = 3;
+  const int height = 318;
+  const int width = 318;
+  float *input = new float[batch_size * channels * height * width]();
+
+  std::array<int32_t, 4> shape = {batch_size, channels, height, width};
+  PD_TensorReshape(tensor, 4, shape.data());
+  PD_TensorCopyFromCpuFloat(tensor, input);
+  EXPECT_TRUE(PD_PredictorRun(predictor));
+
+  delete[] input;
+  PD_TensorDestroy(tensor);
+  PD_CstrDestroy(config_summary);
+  PD_PredictorDestroy(predictor);
+}
+
+TEST(PD_PredictorRun, predictor_run) { predictor_run(); }
+
+#ifdef PADDLE_WITH_DNNL
+TEST(PD_Config, profile_onednn) {
+  std::string model_dir = FLAGS_infer_model;
+  std::string prog_file = model_dir + "/model";
+  std::string params_file = model_dir + "/params";
+  PD_Config *config = PD_ConfigCreate();
+  PD_ConfigDisableGpu(config);
+  PD_ConfigSetCpuMathLibraryNumThreads(config, 10);
+  PD_ConfigSwitchIrDebug(config, TRUE);
+  PD_ConfigEnableONEDNN(config);
+  bool onednn_enable = PD_ConfigOnednnEnabled(config);
+  EXPECT_TRUE(onednn_enable);
+  PD_ConfigEnableOnednnBfloat16(config);
+  PD_ConfigSetOnednnCacheCapacity(config, 0);
+  PD_ConfigSetModel(config, prog_file.c_str(), params_file.c_str());
+  PD_ConfigDestroy(config);
+}
+#endif
+
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_capi_exp_xpu_tester.cc

@@ -0,0 +1,66 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+
+#include <string>
+#include <vector>
+
+#include "paddle/common/flags.h"
+#include "paddle/fluid/inference/capi_exp/pd_inference_api.h"
+
+PD_DEFINE_string(infer_model, "", "model path");
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+
+#ifdef PADDLE_WITH_XPU
+TEST(PD_Config, use_xpu) {
+  std::string model_dir = FLAGS_infer_model + "/mobilenet";
+  PD_Config *config = PD_Config();
+  PD_ConfigSwitchIrDebug(config, TRUE);
+  PD_ConfigSetModelDir(config, model_dir.c_str());
+  PD_ConfigSetOptimCacheDir(config,
+                            (FLAGS_infer_model + "/OptimCacheDir").c_str());
+  const char *model_dir_ = PD_ConfigGetModelDir(config);
+  LOG(INFO) << model_dir_;
+  PD_ConfigEnableXpu(config, 0xfffc00);
+  bool use_xpu = PD_ConfigUseXpu(config);
+  EXPECT_TRUE(use_xpu);
+  int32_t device_id = PD_ConfigXpuDeviceId(config);
+  EXPECT_EQ(device_id, 0);
+  PD_ConfigSwitchIrOptim(config, TRUE);
+  bool ir_optim = PD_IrOptim(config);
+  EXPECT_TRUE(ir_optim);
+  PD_ConfigEnableMemoryOptim(config, true);
+  bool memory_optim_enable = PD_ConfigMemoryOptimEnabled(config);
+  EXPECT_TRUE(memory_optim_enable);
+  PD_ConfigEnableProfile(config);
+  bool profiler_enable = PD_ConfigProfileEnabled(config);
+  EXPECT_TRUE(profiler_enable);
+  PD_SetInValid(config);
+  bool is_valid = PD_ConfigIsValid(config);
+  EXPECT_FALSE(is_valid);
+  PD_ConfigDestroy(config);
+}
+#endif
+
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_capi_gpu_tester.cc

@@ -0,0 +1,181 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include <string>
+#include <vector>
+
+#include "paddle/fluid/inference/capi/paddle_c_api.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+
+TEST(PD_AnalysisConfig, use_gpu) {
+  std::string model_dir = FLAGS_infer_model + "/mobilenet";
+  PD_AnalysisConfig *config = PD_NewAnalysisConfig();
+
+  PD_DisableGpu(config);
+  PD_SetCpuMathLibraryNumThreads(config, 10);
+  int num_thread = PD_CpuMathLibraryNumThreads(config);
+  PADDLE_ENFORCE_EQ(
+      10,
+      num_thread,
+      common::errors::InvalidArgument("The num of thread should be "
+                                      "equal to 10, but got %d.",
+                                      num_thread));
+  PD_SwitchSpecifyInputNames(config, true);
+  PD_SwitchIrDebug(config, true);
+  PD_SetModel(config, model_dir.c_str(), nullptr);
+  PD_SetOptimCacheDir(config, (FLAGS_infer_model + "/OptimCacheDir").c_str());
+  const char *model_dir_ = PD_ModelDir(config);
+  LOG(INFO) << model_dir_;
+  PD_EnableUseGpu(config, 100, 0);
+  bool use_gpu = PD_UseGpu(config);
+  PADDLE_ENFORCE_EQ(use_gpu,
+                    true,
+                    common::errors::InvalidArgument(
+                        "GPU is not enabled. "
+                        "The configuration indicates that GPU should be used, "
+                        "but it is currently disabled. "
+                        "Please check your configuration settings and ensure "
+                        "that GPU is properly enabled."));
+  int device = PD_GpuDeviceId(config);
+  PADDLE_ENFORCE_EQ(device,
+                    0,
+                    common::errors::InvalidArgument(
+                        "The device ID is incorrect. "
+                        "Expected device ID is 0, but received %d. "
+                        "Please check your device configuration and "
+                        "ensure the correct device ID is used.",
+                        device));
+  int init_size = PD_MemoryPoolInitSizeMb(config);
+  PADDLE_ENFORCE_EQ(init_size,
+                    100,
+                    common::errors::InvalidArgument(
+                        "The initial size of the memory pool is incorrect. "
+                        "Expected size is 100 MB, but received %d MB. "
+                        "Please check your configuration settings and ensure "
+                        "the correct memory pool size is set.",
+                        init_size));
+  float frac = PD_FractionOfGpuMemoryForPool(config);
+  LOG(INFO) << frac;
+  PD_EnableCUDNN(config);
+  bool cudnn = PD_CudnnEnabled(config);
+  PADDLE_ENFORCE_EQ(cudnn,
+                    true,
+                    common::errors::InvalidArgument(
+                        "cuDNN is not enabled. "
+                        "The configuration indicates that cuDNN should be "
+                        "enabled, but it is currently disabled. "
+                        "Please check your configuration settings and ensure "
+                        "that cuDNN is properly enabled."));
+  PD_SwitchIrOptim(config, true);
+  bool ir_optim = PD_IrOptim(config);
+  PADDLE_ENFORCE_EQ(ir_optim,
+                    true,
+                    common::errors::InvalidArgument(
+                        "IR optimization is not enabled. "
+                        "The configuration indicates that IR optimization "
+                        "should be enabled, but it is currently disabled. "
+                        "Please check your configuration settings and ensure "
+                        "that IR optimization is properly enabled."));
+  PD_EnableTensorRtEngine(
+      config, 1 << 20, 1, 3, Precision::kFloat32, false, false);
+  bool trt_enable = PD_TensorrtEngineEnabled(config);
+  PADDLE_ENFORCE_EQ(trt_enable,
+                    true,
+                    common::errors::InvalidArgument(
+                        "TensorRT engine is not enabled. "
+                        "The configuration indicates that TensorRT engine "
+                        "should be enabled, but it is currently disabled. "
+                        "Please check your configuration settings and ensure "
+                        "that TensorRT engine is properly enabled."));
+  PD_EnableMemoryOptim(config);
+  bool memory_optim_enable = PD_MemoryOptimEnabled(config);
+  PADDLE_ENFORCE_EQ(memory_optim_enable,
+                    true,
+                    common::errors::InvalidArgument(
+                        "Memory optimization is not enabled. "
+                        "The configuration indicates that memory optimization "
+                        "should be enabled, but it is currently disabled. "
+                        "Please check your configuration settings and ensure "
+                        "that memory optimization is properly enabled."));
+  PD_EnableProfile(config);
+  bool profiler_enable = PD_ProfileEnabled(config);
+  PADDLE_ENFORCE_EQ(profiler_enable,
+                    true,
+                    common::errors::InvalidArgument(
+                        "Profiler is not enabled. "
+                        "The configuration indicates that the profiler should "
+                        "be enabled, but it is currently disabled. "
+                        "Please check your configuration settings and ensure "
+                        "that the profiler is properly enabled."));
+  PD_SetInValid(config);
+  bool is_valid = PD_IsValid(config);
+  PADDLE_ENFORCE_EQ(
+      is_valid,
+      true,
+      common::errors::InvalidArgument(
+          "Configuration is not valid. "
+          "The configuration should be valid, but it is currently invalid. "
+          "Please check your configuration settings and ensure they are "
+          "correct."));
+  PD_DeleteAnalysisConfig(config);
+}
+
+TEST(PD_AnalysisConfig, trt_int8) {
+  std::string model_dir = FLAGS_infer_model + "/mobilenet";
+  PD_AnalysisConfig *config = PD_NewAnalysisConfig();
+  PD_EnableUseGpu(config, 100, 0);
+  PD_EnableTensorRtEngine(config, 1 << 20, 1, 3, Precision::kInt8, false, true);
+  bool trt_enable = PD_TensorrtEngineEnabled(config);
+  PADDLE_ENFORCE_EQ(trt_enable,
+                    true,
+                    common::errors::InvalidArgument(
+                        "TensorRT engine is not enabled. "
+                        "The configuration indicates that TensorRT engine "
+                        "should be enabled, but it is currently disabled. "
+                        "Please check your configuration settings and ensure "
+                        "that TensorRT engine is properly enabled."));
+  PD_DeleteAnalysisConfig(config);
+}
+
+TEST(PD_AnalysisConfig, trt_fp16) {
+  std::string model_dir = FLAGS_infer_model + "/mobilenet";
+  PD_AnalysisConfig *config = PD_NewAnalysisConfig();
+  PD_EnableUseGpu(config, 100, 0);
+  PD_EnableTensorRtEngine(
+      config, 1 << 20, 1, 3, Precision::kHalf, false, false);
+  bool trt_enable = PD_TensorrtEngineEnabled(config);
+  PADDLE_ENFORCE_EQ(trt_enable,
+                    true,
+                    common::errors::InvalidArgument(
+                        "TensorRT engine is not enabled. "
+                        "The configuration indicates that TensorRT engine "
+                        "should be enabled, but it is currently disabled. "
+                        "Please check your configuration settings and ensure "
+                        "that TensorRT engine is properly enabled."));
+  PD_Predictor *predictor = PD_NewPredictor(config);
+  PD_DeletePredictor(predictor);
+  PD_DeleteAnalysisConfig(config);
+}
+
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_capi_int_tester.cc

@@ -0,0 +1,105 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include <string>
+#include <vector>
+
+#include "paddle/fluid/inference/capi/paddle_c_api.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+
+void zero_copy_run() {
+  std::string model_dir = FLAGS_infer_model;
+  PD_AnalysisConfig *config = PD_NewAnalysisConfig();
+  PD_DisableGpu(config);
+  PD_SetCpuMathLibraryNumThreads(config, 10);
+  PD_SwitchSpecifyInputNames(config, true);
+  PD_SwitchIrDebug(config, true);
+  PD_SetModel(config, model_dir.c_str(), nullptr);
+  bool use_feed_fetch = PD_UseFeedFetchOpsEnabled(config);
+  PADDLE_ENFORCE_EQ(
+      use_feed_fetch, false, common::errors::PreconditionNotMet("NO"));
+  bool specify_input_names = PD_SpecifyInputName(config);
+  PADDLE_ENFORCE_EQ(
+      specify_input_names, true, common::errors::PreconditionNotMet("NO"));
+  const int batch_size = 1;
+  const int channels = 3;
+  const int height = 224;
+  const int width = 224;
+  float input[batch_size * channels * height * width] = {0};
+  int shape[4] = {batch_size, channels, height, width};
+  int shape_size = 4;
+  int in_size = 2;
+  int out_size;
+  PD_ZeroCopyData *inputs = new PD_ZeroCopyData[2];
+  PD_ZeroCopyData *outputs = nullptr;
+  inputs[0].data = static_cast<void *>(input);
+  inputs[0].dtype = PD_FLOAT32;
+  inputs[0].name = new char[6];
+  inputs[0].name[0] = 'i';
+  inputs[0].name[1] = 'm';
+  inputs[0].name[2] = 'a';
+  inputs[0].name[3] = 'g';
+  inputs[0].name[4] = 'e';
+  inputs[0].name[5] = '\0';
+  inputs[0].shape = shape;
+  inputs[0].shape_size = shape_size;
+
+  int *label = new int[1];
+  label[0] = 0;
+  inputs[1].data = static_cast<void *>(label);
+  inputs[1].dtype = PD_INT64;
+  inputs[1].name = new char[6];
+  inputs[1].name[0] = 'l';
+  inputs[1].name[1] = 'a';
+  inputs[1].name[2] = 'b';
+  inputs[1].name[3] = 'e';
+  inputs[1].name[4] = 'l';
+  inputs[1].name[5] = '\0';
+  int label_shape[2] = {1, 1};
+  int label_shape_size = 2;
+  inputs[1].shape = label_shape;
+  inputs[1].shape_size = label_shape_size;
+
+  PD_PredictorZeroCopyRun(config, inputs, in_size, &outputs, &out_size);
+
+  LOG(INFO) << "output size is: " << out_size;
+  LOG(INFO) << outputs[0].name;
+  for (int j = 0; j < out_size; ++j) {
+    LOG(INFO) << "output[" << j
+              << "]'s shape_size is: " << outputs[j].shape_size;
+    for (int i = 0; i < outputs[0].shape_size; ++i) {
+      LOG(INFO) << "output[" << j << "]'s shape is: " << outputs[j].shape[i];
+    }
+    LOG(INFO) << "output[" << j
+              << "]'s DATA is: " << *(static_cast<float *>(outputs[j].data));
+  }
+  delete[] outputs;
+  delete[] inputs;
+}
+
+#ifdef PADDLE_WITH_DNNL
+TEST(PD_ZeroCopyRun, zero_copy_run) { zero_copy_run(); }
+#endif
+
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_capi_ner_tester.cc

@@ -0,0 +1,125 @@
+// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include <string>
+#include <vector>
+
+#include "paddle/fluid/inference/capi/paddle_c_api.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+
+void SetConfig(PD_AnalysisConfig *config) {
+  auto model_dir = FLAGS_infer_model;
+  PD_SetModel(config,
+              (model_dir + "/__model__").c_str(),
+              (model_dir + "/param").c_str());
+  PD_SwitchSpecifyInputNames(config, true);
+  PD_DisableGpu(config);
+}
+
+TEST(PD_ZeroCopyRun, zero_copy_run) {
+  PD_AnalysisConfig *config = PD_NewAnalysisConfig();
+  SetConfig(config);
+  PD_Predictor *predictor = PD_NewPredictor(config);
+
+  int input_num = PD_GetInputNum(predictor);
+  printf("Input num: %d\n", input_num);
+  int output_num = PD_GetOutputNum(predictor);
+  printf("Output num: %d\n", output_num);
+
+  PD_ZeroCopyTensor inputs[2];
+
+  // inputs[0]: word
+  PD_InitZeroCopyTensor(&inputs[0]);
+  inputs[0].name = new char[5];
+  snprintf(inputs[0].name,
+           strlen(PD_GetInputName(predictor, 0)) + 1,
+           "%s",
+           PD_GetInputName(predictor, 0));
+
+  inputs[0].data.capacity = sizeof(int64_t) * 11 * 1;
+  inputs[0].data.length = inputs[0].data.capacity;
+  inputs[0].data.data = malloc(inputs[0].data.capacity);
+  std::vector<int64_t> ref_word(
+      {12673, 9763, 905, 284, 45, 7474, 20, 17, 1, 4, 9});
+  inputs[0].data.data = reinterpret_cast<void *>(ref_word.data());
+
+  int shape0[] = {11, 1};
+  inputs[0].shape.data = reinterpret_cast<void *>(shape0);
+  inputs[0].shape.capacity = sizeof(shape0);
+  inputs[0].shape.length = sizeof(shape0);
+  inputs[0].dtype = PD_INT64;
+
+  size_t lod0[] = {0, 11};
+  inputs[0].lod.data = reinterpret_cast<void *>(lod0);
+  inputs[0].lod.capacity = sizeof(size_t) * 2;
+  inputs[0].lod.length = sizeof(size_t) * 2;
+
+  PD_SetZeroCopyInput(predictor, &inputs[0]);
+
+  // inputs[1]: mention
+  PD_InitZeroCopyTensor(&inputs[1]);
+  inputs[1].name = new char[8];
+  snprintf(inputs[1].name,
+           strlen(PD_GetInputName(predictor, 1)) + 1,
+           "%s",
+           PD_GetInputName(predictor, 1));
+
+  inputs[1].data.capacity = sizeof(int64_t) * 11 * 1;
+  inputs[1].data.length = inputs[1].data.capacity;
+  inputs[1].data.data = malloc(inputs[1].data.capacity);
+  std::vector<int64_t> ref_mention({27, 0, 0, 33, 34, 33, 0, 0, 0, 1, 2});
+  inputs[1].data.data = reinterpret_cast<void *>(ref_mention.data());
+
+  int shape1[] = {11, 1};
+  inputs[1].shape.data = reinterpret_cast<void *>(shape1);
+  inputs[1].shape.capacity = sizeof(shape1);
+  inputs[1].shape.length = sizeof(shape1);
+  inputs[1].dtype = PD_INT64;
+
+  size_t lod1[] = {0, 11};
+  inputs[1].lod.data = reinterpret_cast<void *>(lod1);
+  inputs[1].lod.capacity = sizeof(size_t) * 2;
+  inputs[1].lod.length = sizeof(size_t) * 2;
+
+  PD_SetZeroCopyInput(predictor, &inputs[1]);
+
+  PD_ZeroCopyRun(predictor);
+  PD_ZeroCopyTensor output;
+  PD_InitZeroCopyTensor(&output);
+  output.name = new char[21];
+  snprintf(output.name,
+           strlen(PD_GetOutputName(predictor, 0)) + 1,
+           "%s",
+           PD_GetOutputName(predictor, 0));
+
+  // not necessary, just for coverage tests
+  output.lod.data = std::malloc(sizeof(size_t));
+
+  PD_GetZeroCopyOutput(predictor, &output);
+  PD_DestroyZeroCopyTensor(&output);
+  PD_DeleteAnalysisConfig(config);
+  PD_DeletePredictor(predictor);
+}
+
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_capi_pd_tensor_tester.cc

@@ -0,0 +1,173 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include <fstream>
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "paddle/common/enforce.h"
+#include "paddle/fluid/inference/capi/c_api_internal.h"
+#include "paddle/fluid/inference/capi/paddle_c_api.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+
+void PD_run() {
+  PD_AnalysisConfig* config = PD_NewAnalysisConfig();
+  std::string prog_file = FLAGS_infer_model + "/__model__";
+  std::string params_file = FLAGS_infer_model + "/__params__";
+  PD_SetModel(config, prog_file.c_str(), params_file.c_str());
+  PD_SetProgFile(config, prog_file.c_str());
+  PD_SetParamsFile(config, params_file.c_str());
+  LOG(INFO) << PD_ProgFile(config);
+  LOG(INFO) << PD_ParamsFile(config);
+  PD_Tensor* input = PD_NewPaddleTensor();
+  PD_PaddleBuf* buf = PD_NewPaddleBuf();
+  LOG(INFO) << "PaddleBuf empty: " << PD_PaddleBufEmpty(buf);
+  int batch = 1;
+  int channel = 3;
+  int height = 300;
+  int width = 300;
+  int shape[4] = {batch, channel, height, width};
+  int shape_size = 4;
+  float* data = new float[batch * channel * height * width];
+  PD_PaddleBufReset(buf,
+                    static_cast<void*>(data),
+                    sizeof(float) * (batch * channel * height * width));
+
+  char name[6] = {'i', 'm', 'a', 'g', 'e', '\0'};
+  PD_SetPaddleTensorName(input, name);
+  PD_SetPaddleTensorDType(input, PD_FLOAT32);
+  PD_SetPaddleTensorShape(input, shape, shape_size);
+  PD_SetPaddleTensorData(input, buf);
+
+  PD_Tensor* out_data = PD_NewPaddleTensor();
+  int out_size;
+  PD_PredictorRun(config, input, 1, &out_data, &out_size, 1);
+  LOG(INFO) << out_size;
+  LOG(INFO) << PD_GetPaddleTensorName(out_data);
+  LOG(INFO) << PD_GetPaddleTensorDType(out_data);
+  PD_PaddleBuf* b = PD_GetPaddleTensorData(out_data);
+  LOG(INFO) << PD_PaddleBufLength(b) / sizeof(float);
+  float* result = static_cast<float*>(PD_PaddleBufData(b));
+  LOG(INFO) << *result;
+  PD_DeletePaddleTensor(input);
+  int size;
+  const int* out_shape = PD_GetPaddleTensorShape(out_data, &size);
+  PADDLE_ENFORCE_EQ(
+      size,
+      2,
+      common::errors::InvalidArgument("The Output shape's size is NOT match."));
+  std::vector<int> ref_outshape_size({9, 6});
+  for (int i = 0; i < 2; ++i) {
+    PADDLE_ENFORCE_EQ(out_shape[i],
+                      ref_outshape_size[i],
+                      common::errors::InvalidArgument(
+                          "The Output shape's size is NOT match."));
+  }
+  PD_DeletePaddleBuf(buf);
+}
+
+TEST(PD_Tensor, PD_run) { PD_run(); }
+
+TEST(PD_Tensor, int32) {
+  PD_Tensor* input = PD_NewPaddleTensor();
+  PD_SetPaddleTensorDType(input, PD_INT32);
+  LOG(INFO) << PD_GetPaddleTensorDType(input);
+}
+
+TEST(PD_Tensor, int64) {
+  PD_Tensor* input = PD_NewPaddleTensor();
+  PD_SetPaddleTensorDType(input, PD_INT64);
+  LOG(INFO) << PD_GetPaddleTensorDType(input);
+}
+
+TEST(PD_Tensor, int8) {
+  PD_Tensor* input = PD_NewPaddleTensor();
+  PD_SetPaddleTensorDType(input, PD_UINT8);
+  LOG(INFO) << PD_GetPaddleTensorDType(input);
+}
+
+std::string read_file(std::string filename) {
+  std::ifstream file(filename);
+  return std::string((std::istreambuf_iterator<char>(file)),
+                     std::istreambuf_iterator<char>());
+}
+
+void buffer_run() {
+  PD_AnalysisConfig* config = PD_NewAnalysisConfig();
+  std::string prog_file = FLAGS_infer_model + "/__model__";
+  std::string params_file = FLAGS_infer_model + "/__params__";
+
+  std::string prog_str = read_file(prog_file);
+  std::string params_str = read_file(params_file);
+
+  PD_SetModelBuffer(config,
+                    prog_str.c_str(),
+                    prog_str.size(),
+                    params_str.c_str(),
+                    params_str.size());
+  LOG(INFO) << PD_ProgFile(config);
+  LOG(INFO) << PD_ParamsFile(config);
+  PADDLE_ENFORCE(PD_ModelFromMemory(config),
+                 common::errors::PreconditionNotMet(
+                     "PD_ModelFromMemory(config) is failed"));
+
+  PD_Tensor* input = PD_NewPaddleTensor();
+  PD_PaddleBuf* buf = PD_NewPaddleBuf();
+  LOG(INFO) << "PaddleBuf empty: " << PD_PaddleBufEmpty(buf);
+  int batch = 1;
+  int channel = 3;
+  int height = 300;
+  int width = 300;
+  int shape[4] = {batch, channel, height, width};
+  int shape_size = 4;
+  float* data = new float[batch * channel * height * width];
+  PD_PaddleBufReset(buf,
+                    static_cast<void*>(data),
+                    sizeof(float) * (batch * channel * height * width));
+
+  char name[6] = {'i', 'm', 'a', 'g', 'e', '\0'};
+  PD_SetPaddleTensorName(input, name);
+  PD_SetPaddleTensorDType(input, PD_FLOAT32);
+  PD_SetPaddleTensorShape(input, shape, shape_size);
+  PD_SetPaddleTensorData(input, buf);
+
+  PD_Tensor* out_data = PD_NewPaddleTensor();
+  int out_size;
+  PD_PredictorRun(config, input, 1, &out_data, &out_size, 1);
+  LOG(INFO) << out_size;
+  LOG(INFO) << PD_GetPaddleTensorName(out_data);
+  LOG(INFO) << PD_GetPaddleTensorDType(out_data);
+  PD_PaddleBuf* b = PD_GetPaddleTensorData(out_data);
+  LOG(INFO) << PD_PaddleBufLength(b) / sizeof(float);
+  float* result = static_cast<float*>(PD_PaddleBufData(b));
+  LOG(INFO) << *result;
+  PD_DeletePaddleTensor(input);
+  PD_DeletePaddleBuf(buf);
+}
+
+TEST(SetModelBuffer, read) { buffer_run(); }
+
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_capi_tester.cc

@@ -0,0 +1,98 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include <string>
+#include <vector>
+
+#include "paddle/fluid/inference/capi/paddle_c_api.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+
+void zero_copy_run() {
+  std::string model_dir = FLAGS_infer_model;
+  std::string prog_file = model_dir + "/model";
+  std::string params_file = model_dir + "/params";
+  PD_AnalysisConfig *config = PD_NewAnalysisConfig();
+  PD_DisableGpu(config);
+  PD_SetCpuMathLibraryNumThreads(config, 10);
+  PD_SwitchSpecifyInputNames(config, true);
+  PD_SwitchIrDebug(config, true);
+  PD_SetModel(config, prog_file.c_str(), params_file.c_str());
+  bool use_feed_fetch = PD_UseFeedFetchOpsEnabled(config);
+  EXPECT_FALSE(use_feed_fetch);
+  bool specify_input_names = PD_SpecifyInputName(config);
+  EXPECT_TRUE(specify_input_names);
+
+  const int batch_size = 1;
+  const int channels = 3;
+  const int height = 318;
+  const int width = 318;
+  float *input = new float[batch_size * channels * height * width]();
+
+  int shape[4] = {batch_size, channels, height, width};
+  int shape_size = 4;
+  int in_size = 1;
+  int out_size;
+  PD_ZeroCopyData *inputs = new PD_ZeroCopyData;
+  PD_ZeroCopyData *outputs = new PD_ZeroCopyData;
+  inputs->data = static_cast<void *>(input);
+  inputs->dtype = PD_FLOAT32;
+  inputs->name = new char[5];
+  inputs->name[0] = 'd';
+  inputs->name[1] = 'a';
+  inputs->name[2] = 't';
+  inputs->name[3] = 'a';
+  inputs->name[4] = '\0';
+  inputs->shape = shape;
+  inputs->shape_size = shape_size;
+
+  PD_PredictorZeroCopyRun(config, inputs, in_size, &outputs, &out_size);
+
+  delete[] input;
+  delete[] inputs;
+  delete[] outputs;
+}
+
+TEST(PD_PredictorZeroCopyRun, zero_copy_run) { zero_copy_run(); }
+
+#ifdef PADDLE_WITH_DNNL
+TEST(PD_AnalysisConfig, profile_onednn) {
+  std::string model_dir = FLAGS_infer_model;
+  std::string prog_file = model_dir + "/model";
+  std::string params_file = model_dir + "/params";
+  PD_AnalysisConfig *config = PD_NewAnalysisConfig();
+  PD_DisableGpu(config);
+  PD_SetCpuMathLibraryNumThreads(config, 10);
+  PD_SwitchSpecifyInputNames(config, true);
+  PD_SwitchIrDebug(config, true);
+  PD_EnableONEDNN(config);
+  bool onednn_enable = PD_OnednnEnabled(config);
+  EXPECT_TRUE(onednn_enable);
+  PD_EnableOnednnBfloat16(config);
+  PD_SetOnednnCacheCapacity(config, 0);
+  PD_SetModel(config, prog_file.c_str(), params_file.c_str());
+  PD_DeleteAnalysisConfig(config);
+}
+#endif
+
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_capi_xpu_tester.cc

@@ -0,0 +1,65 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include <string>
+#include <vector>
+
+#include "paddle/common/enforce.h"
+#include "paddle/fluid/inference/capi/paddle_c_api.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+
+#ifdef PADDLE_WITH_XPU
+TEST(PD_AnalysisConfig, use_xpu) {
+  std::string model_dir = FLAGS_infer_model + "/mobilenet";
+  PD_AnalysisConfig *config = PD_NewAnalysisConfig();
+  PD_SwitchSpecifyInputNames(config, true);
+  PD_SwitchIrDebug(config, true);
+  PD_SetModel(config, model_dir.c_str(), nullptr);
+  PD_SetOptimCacheDir(config, (FLAGS_infer_model + "/OptimCacheDir").c_str());
+  const char *model_dir_ = PD_ModelDir(config);
+  LOG(INFO) << model_dir_;
+  PD_EnableXpu(config, 0xfffc00);
+  bool use_xpu = PD_UseXpu(config);
+  PADDLE_ENFORCE_EQ(use_xpu, true, common::errors::PreconditionNotMet("NO"));
+  int device = PD_XpuDeviceId(config);
+  PADDLE_ENFORCE_EQ(device, 0, common::errors::PreconditionNotMet("NO"));
+  PD_SwitchIrOptim(config, true);
+  bool ir_optim = PD_IrOptim(config);
+  PADDLE_ENFORCE_EQ(ir_optim, true, common::errors::PreconditionNotMet("NO"));
+  PD_EnableMemoryOptim(config);
+  bool memory_optim_enable = PD_MemoryOptimEnabled(config);
+  PADDLE_ENFORCE_EQ(
+      memory_optim_enable, true, common::errors::PreconditionNotMet("NO"));
+  PD_EnableProfile(config);
+  bool profiler_enable = PD_ProfileEnabled(config);
+  PADDLE_ENFORCE_EQ(
+      profiler_enable, true, common::errors::PreconditionNotMet("NO"));
+  PD_SetInValid(config);
+  bool is_valid = PD_IsValid(config);
+  PADDLE_ENFORCE_EQ(is_valid, false, common::errors::PreconditionNotMet("NO"));
+  PD_DeleteAnalysisConfig(config);
+}
+#endif
+
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_dam_tester.cc

@@ -0,0 +1,332 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <vector>
+
+#include "paddle/fluid/inference/analysis/helper.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+const int FLAGS_max_turn_num = 1;
+
+namespace paddle {
+namespace inference {
+
+constexpr int32_t kMaxTurnLen = 50;
+
+static std::vector<float> result_data;
+
+struct DataRecord {
+  std::vector<std::vector<int64_t>> *turns;
+  std::vector<std::vector<float>> *turns_mask;
+  std::vector<std::vector<int64_t>> response;     // response data : 1
+  std::vector<std::vector<float>> response_mask;  // response mask data : 1
+  size_t batch_iter{0};
+  size_t batch_size{1};
+  size_t num_samples;  // total number of samples
+
+  DataRecord() {  // NOLINT
+    turns = new std::vector<std::vector<
+        int64_t>>[FLAGS_max_turn_num];  // turns data : FLAGS_max_turn_num
+    turns_mask = new std::vector<std::vector<
+        float>>[FLAGS_max_turn_num];  // turns mask data : FLAGS_max_turn_num
+  }
+
+  explicit DataRecord(const std::string &path, int batch_size = 1)
+      : DataRecord() {
+    this->batch_size = batch_size;
+    Load(path);
+  }
+
+  ~DataRecord() {  // NOLINT
+    delete[] turns;
+    delete[] turns_mask;
+  }
+
+  DataRecord NextBatch() {
+    DataRecord data;
+    size_t batch_end = batch_iter + batch_size;
+    // NOTE skip the final batch, if no enough data is provided.
+    if (batch_end <= response.size()) {
+      for (int i = 0; i < FLAGS_max_turn_num; ++i) {
+        data.turns[i].assign(turns[i].begin() + batch_iter,
+                             turns[i].begin() + batch_end);
+      }
+      for (int i = 0; i < FLAGS_max_turn_num; ++i) {
+        data.turns_mask[i].assign(turns_mask[i].begin() + batch_iter,
+                                  turns_mask[i].begin() + batch_end);
+      }
+      data.response.assign(response.begin() + batch_iter,
+                           response.begin() + batch_end);
+      data.response_mask.assign(response_mask.begin() + batch_iter,
+                                response_mask.begin() + batch_end);
+      PADDLE_ENFORCE_EQ(!data.response.empty(),
+                        true,
+                        common::errors::Fatal(
+                            "Variable `data` response is empty, please check"));
+      PADDLE_ENFORCE_EQ(
+          !data.response_mask.empty(),
+          true,
+          common::errors::Fatal(
+              "Variable `data` response mask is empty, please check"));
+      PADDLE_ENFORCE_EQ(data.response.size(),
+                        data.response_mask.size(),
+                        common::errors::InvalidArgument(
+                            "Required data.response.size() should be equal to "
+                            "data.response_mask.size() . "));
+    }
+    batch_iter += batch_size;
+    return data;
+  }
+
+  void Load(const std::string &path) {
+    std::ifstream file(path);
+    std::string line;
+    size_t num_lines = 0;
+    result_data.clear();
+    while (std::getline(file, line)) {
+      num_lines++;
+      std::vector<std::string> data;
+      split(line, ',', &data);
+      PADDLE_ENFORCE_EQ(data.size(),
+                        (size_t)(2 * FLAGS_max_turn_num + 3),
+                        common::errors::InvalidArgument(
+                            "Required data.size() should be equal to "
+                            "(size_t)(2 * FLAGS_max_turn_num + 3) . "));
+      // load turn data
+      std::vector<int64_t> turns_tmp[FLAGS_max_turn_num];
+      for (int i = 0; i < FLAGS_max_turn_num; ++i) {
+        split_to_int64(data[i], ' ', &turns_tmp[i]);
+        turns[i].push_back(std::move(turns_tmp[i]));
+      }
+      // load turn_mask data
+      std::vector<float> turns_mask_tmp[FLAGS_max_turn_num];
+      for (int i = 0; i < FLAGS_max_turn_num; ++i) {
+        split_to_float(data[FLAGS_max_turn_num + i], ' ', &turns_mask_tmp[i]);
+        turns_mask[i].push_back(std::move(turns_mask_tmp[i]));
+      }
+      // load response data
+      std::vector<int64_t> response_tmp;
+      split_to_int64(data[2 * FLAGS_max_turn_num], ' ', &response_tmp);
+      response.push_back(std::move(response_tmp));
+      // load response_mask data
+      std::vector<float> response_mask_tmp;
+      split_to_float(data[2 * FLAGS_max_turn_num + 1], ' ', &response_mask_tmp);
+      response_mask.push_back(std::move(response_mask_tmp));
+      // load result data
+      float result_tmp;
+      result_tmp = std::stof(data[2 * FLAGS_max_turn_num + 2]);
+      result_data.push_back(result_tmp);
+    }
+    num_samples = num_lines;
+  }
+};
+
+void PrepareInputs(std::vector<PaddleTensor> *input_slots,
+                   DataRecord *data,
+                   int batch_size) {
+  PaddleTensor turns_tensor[FLAGS_max_turn_num];       // NOLINT
+  PaddleTensor turns_mask_tensor[FLAGS_max_turn_num];  // NOLINT
+  PaddleTensor response_tensor;
+  PaddleTensor response_mask_tensor;
+  std::string turn_pre = "turn_";
+  std::string turn_mask_pre = "turn_mask_";
+
+  auto one_batch = data->NextBatch();
+  PADDLE_ENFORCE(
+      !one_batch.response.empty(),
+      ::common::errors::Fatal("The response of one batch is empty."));
+  int size = one_batch.response[0].size();
+  PADDLE_ENFORCE_EQ(size,
+                    kMaxTurnLen,
+                    common::errors::InvalidArgument(
+                        "Required size should be equal to kMaxTurnLen . "));
+  // turn tensor assignment
+  for (int i = 0; i < FLAGS_max_turn_num; ++i) {
+    turns_tensor[i].name = turn_pre + std::to_string(i);
+    turns_tensor[i].shape.assign({batch_size, size, 1});
+    turns_tensor[i].dtype = PaddleDType::INT64;
+    TensorAssignData<int64_t>(&turns_tensor[i], one_batch.turns[i]);
+  }
+  // turn mask tensor assignment
+  for (int i = 0; i < FLAGS_max_turn_num; ++i) {
+    turns_mask_tensor[i].name = turn_mask_pre + std::to_string(i);
+    turns_mask_tensor[i].shape.assign({batch_size, size, 1});
+    turns_mask_tensor[i].dtype = PaddleDType::FLOAT32;
+    TensorAssignData<float>(&turns_mask_tensor[i], one_batch.turns_mask[i]);
+  }
+  // response tensor assignment
+  response_tensor.name = "response";
+  response_tensor.shape.assign({batch_size, size, 1});
+  response_tensor.dtype = PaddleDType::INT64;
+  TensorAssignData<int64_t>(&response_tensor, one_batch.response);
+  // response mask tensor assignment
+  response_mask_tensor.name = "response_mask";
+  response_mask_tensor.shape.assign({batch_size, size, 1});
+  response_mask_tensor.dtype = PaddleDType::FLOAT32;
+  TensorAssignData<float>(&response_mask_tensor, one_batch.response_mask);
+
+  // Set inputs.
+  for (auto &item : turns_tensor) {
+    input_slots->push_back(std::move(item));
+  }
+  for (auto &item : turns_mask_tensor) {
+    input_slots->push_back(std::move(item));
+  }
+  input_slots->push_back(std::move(response_tensor));
+  input_slots->push_back(std::move(response_mask_tensor));
+}
+
+/*
+ * this model is unreasonable, it set a output tensor persistable, so
+ * ridiculous! so I disable constant_folding_pass
+ */
+
+void SetConfig(AnalysisConfig *cfg) {
+  cfg->SetModel(FLAGS_infer_model + "/__model__", FLAGS_infer_model + "/param");
+  cfg->SwitchSpecifyInputNames();
+  cfg->DeletePass("constant_folding_pass");
+  cfg->SwitchIrOptim(true);
+}
+
+void SetOptimConfig(AnalysisConfig *cfg) {
+  std::string optimModelPath = FLAGS_infer_model + "/saved_optim_model";
+  cfg->SetModel(optimModelPath + "/model", optimModelPath + "/params");
+  cfg->SwitchIrOptim(true);
+  cfg->SwitchSpecifyInputNames();
+}
+
+void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
+  DataRecord data(FLAGS_infer_data, FLAGS_batch_size);
+  std::vector<PaddleTensor> input_slots;
+  int test_batch_num =
+      FLAGS_test_all_data ? data.num_samples / FLAGS_batch_size : 1;  // NOLINT
+  LOG(INFO) << "The number of samples to be test: "
+            << test_batch_num * FLAGS_batch_size;
+  for (int bid = 0; bid < test_batch_num; ++bid) {
+    input_slots.clear();
+    PrepareInputs(&input_slots, &data, FLAGS_batch_size);
+    (*inputs).emplace_back(input_slots);
+  }
+}
+
+// Easy for profiling independently.
+void profile(bool use_onednn = false) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  if (use_onednn) {
+    cfg.EnableONEDNN();
+    // Enable all the onednn supported ops except conv3d in dam
+    std::unordered_set<std::string> op_list = {
+        "softmax", "elementwise_add", "relu", "fc"};
+    cfg.SetONEDNNOp(op_list);
+  } else {
+    cfg.DisableONEDNN();
+  }
+
+  std::vector<std::vector<PaddleTensor>> outputs;
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+
+  TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                 input_slots_all,
+                 &outputs,
+                 FLAGS_num_threads);
+
+  if (FLAGS_num_threads == 1 && !FLAGS_test_all_data) {
+    PADDLE_ENFORCE_GT(outputs.size(),
+                      0,
+                      ::common::errors::Fatal(
+                          "The size of outputs should be greater than 0."));
+    auto output = outputs.back();
+    PADDLE_ENFORCE_GT(output.size(),
+                      0,
+                      ::common::errors::Fatal(
+                          "The size of output should be greater than 0."));
+    size_t size = GetSize(output[0]);
+    PADDLE_ENFORCE_GT(size,
+                      0,
+                      ::common::errors::Fatal(
+                          "The size of output should be greater than 0."));
+    float *result = static_cast<float *>(output[0].data.data());
+    for (size_t i = 0; i < size; i++) {
+      EXPECT_NEAR(result[i], result_data[i], 1e-3);
+    }
+  }
+}
+
+TEST(Analyzer_dam, profile) { profile(); }
+#ifdef PADDLE_WITH_DNNL
+TEST(Analyzer_dam, profile_onednn) { profile(true /* use_onednn */); }
+#endif
+
+// Compare result of NativeConfig and AnalysisConfig
+void compare(bool use_onednn = false) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+  if (use_onednn) {
+    cfg.EnableONEDNN();
+    // Enable all the onednn supported ops except conv3d in dam
+    std::unordered_set<std::string> op_list = {
+        "softmax", "elementwise_add", "relu"};
+    cfg.SetONEDNNOp(op_list);
+  } else {
+    cfg.DisableONEDNN();
+  }
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+
+  CompareNativeAndAnalysis(
+      reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
+}
+
+TEST(Analyzer_dam, compare_with_dynamic_memory_optim) {
+  // The small dam will core in CI, but works in local.
+  if (FLAGS_max_turn_num == 9) {
+    AnalysisConfig cfg, cfg1;
+    DataRecord data(FLAGS_infer_data, FLAGS_batch_size);
+
+    std::vector<std::vector<PaddleTensor>> input_slots_all;
+    SetInput(&input_slots_all);
+    // Run the first time to force to update memory cache
+    SetConfig(&cfg);
+    cfg.EnableMemoryOptim();
+
+    CompareNativeAndAnalysis(
+        reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+        input_slots_all);
+  }
+}
+
+TEST(Analyzer_dam, compare) { compare(); }
+
+#ifdef PADDLE_WITH_DNNL
+TEST(Analyzer_dam, compare_onednn) { compare(true /* use_onednn */); }
+#endif
+
+// Compare Deterministic result
+TEST(Analyzer_dam, compare_determine) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+  cfg.DisableONEDNN();
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                       input_slots_all);
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_detect_tester.cc

@@ -0,0 +1,137 @@
+/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+
+#include <fstream>
+#include <iostream>
+
+#include "test/cpp/inference/api/tester_helper.h"
+
+PD_DEFINE_string(infer_shape, "", "data shape file");
+PD_DEFINE_int32(sample, 20, "number of sample");
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+
+struct Record {
+  std::vector<float> data;
+  std::vector<int32_t> shape;
+};
+
+Record ProcessALine(const std::string &line, const std::string &shape_line) {
+  VLOG(3) << "process a line";
+  std::vector<std::string> columns;
+
+  Record record;
+  std::vector<std::string> data_strs;
+  split(line, ' ', &data_strs);
+  for (auto &d : data_strs) {
+    record.data.push_back(std::stof(d));
+  }
+
+  std::vector<std::string> shape_strs;
+  split(shape_line, ' ', &shape_strs);
+  for (auto &s : shape_strs) {
+    record.shape.push_back(std::stoi(s));
+  }
+  return record;
+}
+
+void SetConfig(AnalysisConfig *cfg) {
+  cfg->SetModel(FLAGS_infer_model + "/model", FLAGS_infer_model + "/params");
+  cfg->DisableGpu();
+  cfg->SwitchIrDebug();
+  cfg->SwitchSpecifyInputNames(false);
+  cfg->SetCpuMathLibraryNumThreads(FLAGS_cpu_num_threads);
+}
+
+void SetInput(std::vector<std::vector<PaddleTensor>> *inputs,
+              const std::string &line,
+              const std::string &shape_line) {
+  auto record = ProcessALine(line, shape_line);
+
+  PaddleTensor input;
+  input.shape = record.shape;
+  input.dtype = PaddleDType::FLOAT32;
+  size_t input_size = record.data.size() * sizeof(float);
+  input.data.Resize(input_size);
+  memcpy(input.data.data(), record.data.data(), input_size);
+  std::vector<PaddleTensor> input_slots;
+  input_slots.assign({input});
+  (*inputs).emplace_back(input_slots);
+}
+
+void profile(int cache_capacity = 1) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+  cfg.EnableONEDNN();
+  cfg.SetOnednnCacheCapacity(cache_capacity);
+
+  std::vector<std::vector<PaddleTensor>> outputs;
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+
+  Timer run_timer;
+  double elapsed_time = 0;
+
+  int num_times = FLAGS_repeat;
+  int sample = FLAGS_sample;
+  auto predictor = CreatePaddlePredictor<AnalysisConfig>(cfg);
+  outputs.resize(sample);
+
+  std::vector<std::thread> threads;
+
+  std::ifstream file(FLAGS_infer_data);
+  std::ifstream infer_file(FLAGS_infer_shape);
+  std::string line;
+  std::string shape_line;
+
+  for (int i = 0; i < sample; i++) {
+    threads.emplace_back([&, i]() {
+      std::getline(file, line);
+      std::getline(infer_file, shape_line);
+      SetInput(&input_slots_all, line, shape_line);
+
+      run_timer.tic();
+      predictor->Run(input_slots_all[0], &outputs[0], FLAGS_batch_size);
+      elapsed_time += run_timer.toc();
+    });
+    threads[0].join();
+    threads.clear();
+    std::vector<std::vector<PaddleTensor>>().swap(input_slots_all);
+  }
+  file.close();
+  infer_file.close();
+
+  auto batch_latency = elapsed_time / (sample * num_times);
+  PrintTime(FLAGS_batch_size,
+            num_times,
+            FLAGS_num_threads,
+            0,
+            batch_latency,
+            sample,
+            VarType::FP32);
+}
+
+#ifdef PADDLE_WITH_DNNL
+TEST(Analyzer_detect, profile_onednn) {
+  profile(5 /* cache_capacity */);
+  profile(10 /* cache_capacity */);
+}
+#endif
+
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_dist_model_xpu_tester.cc

@@ -0,0 +1,67 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "gtest/gtest.h"
+#include "paddle/fluid/framework/block_desc.h"
+#include "paddle/fluid/framework/op_desc.h"
+#include "paddle/fluid/framework/program_desc.h"
+#include "paddle/fluid/framework/scope.h"
+#include "paddle/fluid/inference/utils/singleton.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+
+TEST(test_dist_model_xpu, dist_model_xpu) {
+  std::cout << "Analysis Predictor DistModel XPU test." << std::endl;
+  AnalysisConfig config;
+  config.SetModel(FLAGS_infer_model + "/__model__",
+                  FLAGS_infer_model + "/__params__");
+  config.EnableXpu();
+  config.SetXpuDeviceId(0);
+
+  auto predictor = paddle_infer::CreatePredictor(config);
+  int batch_size = 1;
+  int channels = 1;
+  int height = 48;
+  int width = 512;
+  int nums = batch_size * channels * height * width;
+  std::cout << "Created predictor." << std::endl;
+
+  float* input = new float[nums];
+  for (int i = 0; i < nums; ++i) input[i] = 0;
+  auto input_names = predictor->GetInputNames();
+
+  auto input_t = predictor->GetInputHandle(input_names[0]);
+  input_t->Reshape({batch_size, channels, height, width});
+  input_t->CopyFromCpu(input);
+  std::cout << "Input data." << std::endl;
+
+  predictor->Run();
+  std::cout << "Zero Copy Run." << std::endl;
+
+  std::vector<float> out_data;
+  auto output_names = predictor->GetOutputNames();
+  auto output_t = predictor->GetOutputHandle(output_names[0]);
+  std::vector<int> output_shape = output_t->shape();
+  int out_num = std::accumulate(
+      output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
+  out_data.resize(out_num);
+  output_t->CopyToCpu(out_data.data());
+  std::cout << "Output data." << std::endl;
+  delete[] input;
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_image_classification_tester.cc

@@ -0,0 +1,99 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <fstream>
+#include <iostream>
+
+#include "test/cpp/inference/api/tester_helper.h"
+
+PD_DEFINE_bool(disable_onednn_fc, false, "Disable usage of ONE-DNN's FC op");
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+
+void SetConfig(AnalysisConfig *cfg) {
+  cfg->SetModel(FLAGS_infer_model + "/model", FLAGS_infer_model + "/params");
+  cfg->DisableGpu();
+  cfg->SwitchIrOptim();
+  cfg->SwitchSpecifyInputNames();
+  cfg->SetCpuMathLibraryNumThreads(FLAGS_cpu_num_threads);
+  cfg->DeletePass("constant_folding_pass");
+}
+
+void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
+  SetFakeImageInput(inputs, FLAGS_infer_model);
+}
+
+// Easy for profiling independently.
+void profile(bool use_onednn = false) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  if (use_onednn) {
+    cfg.EnableONEDNN();
+    if (FLAGS_disable_onednn_fc) {
+      cfg.DisableOnednnFcPasses();
+    }
+  }
+  std::vector<std::vector<PaddleTensor>> outputs;
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                 input_slots_all,
+                 &outputs,
+                 FLAGS_num_threads);
+}
+
+TEST(Analyzer_resnet50, profile) { profile(); }
+#ifdef PADDLE_WITH_DNNL
+TEST(Analyzer_resnet50, profile_onednn) { profile(true /* use_onednn */); }
+#endif
+
+// Compare result of NativeConfig and AnalysisConfig
+void compare(bool use_onednn = false) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+  if (use_onednn) {
+    cfg.EnableONEDNN();
+    if (FLAGS_disable_onednn_fc) {
+      cfg.DisableOnednnFcPasses();
+    }
+  }
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareNativeAndAnalysis(
+      reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
+}
+
+TEST(Analyzer_resnet50, compare) { compare(); }
+#ifdef PADDLE_WITH_DNNL
+TEST(Analyzer_resnet50, compare_onednn) { compare(true /* use_onednn */); }
+#endif
+
+// Compare Deterministic result
+TEST(Analyzer_resnet50, compare_determine) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                       input_slots_all);
+}
+
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_lac_tester.cc

@@ -0,0 +1,191 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+
+struct DataRecord {
+  std::vector<int64_t> data;
+  std::vector<size_t> lod;
+  // for dataset and nextbatch
+  size_t batch_iter{0};
+  std::vector<std::vector<size_t>> batched_lods;
+  std::vector<std::vector<int64_t>> batched_datas;
+  std::vector<std::vector<int64_t>> datasets;
+  DataRecord() : data(), lod(), batched_lods(), batched_datas(), datasets() {}
+  explicit DataRecord(const std::string &path, int batch_size = 1)
+      : data(), lod(), batched_lods(), batched_datas(), datasets() {
+    Load(path);
+    Prepare(batch_size);
+    batch_iter = 0;
+  }
+  void Load(const std::string &path) {
+    std::ifstream file(path);
+    std::string line;
+    int num_lines = 0;
+    datasets.resize(0);
+    while (std::getline(file, line)) {
+      num_lines++;
+      std::vector<std::string> data;
+      split(line, ';', &data);
+      std::vector<int64_t> words_ids;
+      split_to_int64(data[1], ' ', &words_ids);
+      datasets.emplace_back(words_ids);
+    }
+  }
+  void Prepare(int bs) {
+    if (bs == 1) {
+      batched_datas = datasets;
+      for (auto one_sentence : datasets) {
+        batched_lods.push_back({0, one_sentence.size()});
+      }
+    } else {
+      std::vector<int64_t> one_batch;
+      std::vector<size_t> lod{0};
+      int bs_id = 0;
+      for (auto one_sentence : datasets) {
+        bs_id++;
+        one_batch.insert(
+            one_batch.end(), one_sentence.begin(), one_sentence.end());
+        lod.push_back(lod.back() + one_sentence.size());
+        if (bs_id == bs) {
+          bs_id = 0;
+          batched_datas.push_back(one_batch);
+          batched_lods.push_back(lod);
+          one_batch.clear();
+          one_batch.resize(0);
+          lod.clear();
+          lod.resize(0);
+          lod.push_back(0);
+        }
+      }
+      if (!one_batch.empty()) {
+        batched_datas.push_back(one_batch);
+        batched_lods.push_back(lod);
+      }
+    }
+  }
+
+  DataRecord NextBatch() {
+    DataRecord data;
+    data.data = batched_datas[batch_iter];
+    data.lod = batched_lods[batch_iter];
+    batch_iter++;
+    if (batch_iter >= batched_datas.size()) {
+      batch_iter = 0;
+    }
+    return data;
+  }
+};
+
+void GetOneBatch(std::vector<PaddleTensor> *input_slots,
+                 DataRecord *data,
+                 int batch_size) {
+  auto one_batch = data->NextBatch();
+  PaddleTensor input_tensor;
+  input_tensor.name = "word";
+  input_tensor.dtype = PaddleDType::INT64;
+  TensorAssignData<int64_t>(&input_tensor, {one_batch.data}, one_batch.lod);
+  PADDLE_ENFORCE_EQ(
+      batch_size,
+      static_cast<int>(one_batch.lod.size() - 1),
+      ::common::errors::Fatal("The lod size of one batch is invalid."));
+  input_slots->assign({input_tensor});
+}
+
+void SetConfig(AnalysisConfig *cfg) {
+  cfg->SetModel(FLAGS_infer_model);
+  cfg->DisableGpu();
+  cfg->SwitchSpecifyInputNames();
+  cfg->SwitchIrOptim();
+}
+
+void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
+  DataRecord data(FLAGS_infer_data, FLAGS_batch_size);
+  std::vector<PaddleTensor> input_slots;
+  int epoch = FLAGS_test_all_data ? data.batched_datas.size() : 1;
+  LOG(INFO) << "number of samples: " << epoch;
+  for (int bid = 0; bid < epoch; ++bid) {
+    GetOneBatch(&input_slots, &data, FLAGS_batch_size);
+    (*inputs).emplace_back(input_slots);
+  }
+}
+
+// Easy for profiling independently.
+TEST(Analyzer_LAC, profile) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+  std::vector<std::vector<PaddleTensor>> outputs;
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                 input_slots_all,
+                 &outputs,
+                 FLAGS_num_threads);
+
+  if (FLAGS_num_threads == 1 && !FLAGS_test_all_data) {
+    // the first inference result
+    const std::array<int64_t, 47> lac_ref_data = {
+        24, 25, 25, 25, 38, 30, 31, 14, 15, 44, 24, 25, 25, 25, 25, 25,
+        44, 24, 25, 25, 25, 36, 42, 43, 44, 14, 15, 44, 14, 15, 44, 14,
+        15, 44, 38, 39, 14, 15, 44, 22, 23, 23, 23, 23, 23, 23, 23};
+    PADDLE_ENFORCE_GT(outputs.size(),
+                      0,
+                      ::common::errors::Fatal(
+                          "The size of output should be greater than 0."));
+    auto output = outputs.back();
+    PADDLE_ENFORCE_EQ(
+        output.size(),
+        1UL,
+        ::common::errors::Fatal("The size of output should be equal to 1."));
+    size_t size = GetSize(output[0]);
+    size_t batch1_size = sizeof(lac_ref_data) / sizeof(int64_t);
+    PADDLE_ENFORCE_GE(size,
+                      batch1_size,
+                      ::common::errors::Fatal("The size of batch is invalid."));
+    int64_t *pdata = static_cast<int64_t *>(output[0].data.data());
+    for (size_t i = 0; i < batch1_size; ++i) {
+      EXPECT_EQ(pdata[i], lac_ref_data[i]);
+    }
+  }
+}
+// Compare result of NativeConfig and AnalysisConfig
+TEST(Analyzer_LAC, compare) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareNativeAndAnalysis(
+      reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
+}
+
+// Compare Deterministic result
+TEST(Analyzer_LAC, compare_determine) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                       input_slots_all);
+}
+
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_mmp_tester.cc

@@ -0,0 +1,115 @@
+// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <random>
+
+#include "paddle/fluid/framework/transfer_scope_cache.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+// Here add missing commands
+PD_DEFINE_string(infer_model2, "", "model path");
+PD_DEFINE_string(infer_model3, "", "model path");
+
+namespace paddle {
+namespace inference {
+
+// Shape of Input to models
+const int N = 1, C = 3, H = 224, W = 224;
+
+void SetConfig(AnalysisConfig* config, const std::string& infer_model) {
+  config->SetModel(infer_model + "/__model__", infer_model + "/__params__");
+  config->DisableFCPadding();
+  config->SwitchSpecifyInputNames(true);
+}
+
+std::unique_ptr<PaddlePredictor> InitializePredictor(
+    const std::string& infer_model,
+    const std::vector<float>& data,
+    bool use_onednn) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg, infer_model);
+  if (use_onednn) {
+    cfg.EnableONEDNN();
+  }
+
+  auto predictor = ::paddle::CreatePaddlePredictor<AnalysisConfig>(cfg);
+  auto input_name = predictor->GetInputNames()[0];
+  auto input = predictor->GetInputTensor(input_name);
+  std::vector<int> shape{N, C, H, W};
+  input->Reshape(std::move(shape));
+  input->copy_from_cpu(data.data());
+
+  return predictor;
+}
+
+// Compare result of NativeConfig and AnalysisConfig
+void compare(bool use_onednn = false) {
+  // Create Input to models
+  std::vector<float> data(N * C * H * W);
+  std::default_random_engine re{1234};
+  std::uniform_real_distribution<float> sampler{0.0, 1.0};
+  for (auto& v : data) {
+    v = sampler(re);
+  }
+
+  // Initialize Models predictors
+  auto predictor_1 = InitializePredictor(FLAGS_infer_model, data, use_onednn);
+  auto predictor_xx = InitializePredictor(FLAGS_infer_model2, data, use_onednn);
+  auto predictor_3 = InitializePredictor(FLAGS_infer_model3, data, use_onednn);
+
+  // Run single xx model
+  predictor_xx->ZeroCopyRun();
+  auto output =
+      predictor_xx->GetOutputTensor(predictor_xx->GetOutputNames()[0]);
+  auto output_shape = output->shape();
+  int numel = std::accumulate(
+      output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
+  std::vector<float> xx_output(numel);
+  output->copy_to_cpu(xx_output.data());
+
+  // Initialize xx model's predictor to trigger oneDNN cache clearing
+  predictor_xx = InitializePredictor(FLAGS_infer_model2, data, use_onednn);
+
+  // Run sequence of models
+  predictor_1->ZeroCopyRun();
+  predictor_xx->ZeroCopyRun();
+  predictor_3->ZeroCopyRun();
+
+  // Get again output of xx model , but when all three models were executed
+  std::vector<float> xx2_output(numel);
+  output = predictor_xx->GetOutputTensor(predictor_xx->GetOutputNames()[0]);
+  output->copy_to_cpu(xx2_output.data());
+
+  // compare results
+  auto result = std::equal(
+      xx_output.begin(),
+      xx_output.end(),
+      xx2_output.begin(),
+      [](const float& l, const float& r) { return fabs(l - r) < 1e-4; });
+
+  PADDLE_ENFORCE_EQ(
+      result,
+      true,
+      ::common::errors::Fatal("Results of model run independently "
+                              "differs from results of the same model "
+                              "run as a sequence of models"));
+}
+
+TEST(Analyzer_mmp, compare) { compare(); }
+#ifdef PADDLE_WITH_DNNL
+TEST(Analyzer_mmp, compare_onednn) { compare(true /* use_onednn */); }
+#endif
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_ner_tester.cc

@@ -0,0 +1,175 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+
+struct DataRecord {
+  std::vector<std::vector<int64_t>> word, mention;
+  std::vector<size_t> lod;  // two inputs have the same lod info.
+  size_t batch_iter{0}, batch_size{1}, num_samples;  // total number of samples
+  DataRecord() : word(), mention(), lod(), num_samples(0) {}
+  explicit DataRecord(const std::string &path, int batch_size = 1)
+      : word(), mention(), lod(), batch_size(batch_size), num_samples(0) {
+    Load(path);
+  }
+  DataRecord NextBatch() {
+    DataRecord data;
+    size_t batch_end = batch_iter + batch_size;
+    // NOTE skip the final batch, if no enough data is provided.
+    if (batch_end <= word.size()) {
+      GetInputPerBatch(word, &data.word, &data.lod, batch_iter, batch_end);
+      GetInputPerBatch(
+          mention, &data.mention, &data.lod, batch_iter, batch_end);
+    }
+    batch_iter += batch_size;
+    return data;
+  }
+  void Load(const std::string &path) {
+    std::ifstream file(path);
+    std::string line;
+    int num_lines = 0;
+    while (std::getline(file, line)) {
+      num_lines++;
+      std::vector<std::string> data;
+      split(line, ';', &data);
+      // load word data
+      std::vector<int64_t> word_data;
+      split_to_int64(data[1], ' ', &word_data);
+      // load mention data
+      std::vector<int64_t> mention_data;
+      split_to_int64(data[3], ' ', &mention_data);
+      word.push_back(std::move(word_data));
+      mention.push_back(std::move(mention_data));
+    }
+    num_samples = num_lines;
+  }
+};
+
+void PrepareInputs(std::vector<PaddleTensor> *input_slots, DataRecord *data) {
+  PaddleTensor lod_word_tensor, lod_mention_tensor;
+  lod_word_tensor.name = "word";
+  lod_mention_tensor.name = "mention";
+  auto one_batch = data->NextBatch();
+  // assign data
+  TensorAssignData<int64_t>(&lod_word_tensor, one_batch.word, one_batch.lod);
+  TensorAssignData<int64_t>(
+      &lod_mention_tensor, one_batch.mention, one_batch.lod);
+  // Set inputs.
+  input_slots->assign({lod_word_tensor, lod_mention_tensor});
+  for (auto &tensor : *input_slots) {
+    tensor.dtype = PaddleDType::INT64;
+  }
+}
+
+void SetConfig(AnalysisConfig *cfg, bool memory_load = false) {
+  if (memory_load) {
+    std::string buffer_prog, buffer_param;
+    ReadBinaryFile(FLAGS_infer_model + "/__model__", &buffer_prog);
+    ReadBinaryFile(FLAGS_infer_model + "/param", &buffer_param);
+    cfg->SetModelBuffer(&buffer_prog[0],
+                        buffer_prog.size(),
+                        &buffer_param[0],
+                        buffer_param.size());
+  } else {
+    cfg->SetModel(FLAGS_infer_model + "/__model__",
+                  FLAGS_infer_model + "/param");
+  }
+  cfg->DisableGpu();
+  cfg->SwitchSpecifyInputNames();
+  cfg->SwitchIrOptim();
+}
+
+void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
+  DataRecord data(FLAGS_infer_data, FLAGS_batch_size);
+  std::vector<PaddleTensor> input_slots;
+  int epoch =
+      FLAGS_test_all_data ? data.num_samples / FLAGS_batch_size : 1;  // NOLINT
+  LOG(INFO) << "number of samples: " << epoch * FLAGS_batch_size;
+  for (int bid = 0; bid < epoch; ++bid) {
+    PrepareInputs(&input_slots, &data);
+    (*inputs).emplace_back(input_slots);
+  }
+}
+
+// Easy for profiling independently.
+void profile(bool memory_load = false) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg, memory_load);
+  std::vector<std::vector<PaddleTensor>> outputs;
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                 input_slots_all,
+                 &outputs,
+                 FLAGS_num_threads);
+
+  if (FLAGS_num_threads == 1 && !FLAGS_test_all_data) {
+    // the first inference result
+    const std::array<int, 11> chinese_ner_result_data = {
+        30, 45, 41, 48, 17, 26, 48, 39, 38, 16, 25};
+    PADDLE_ENFORCE_GT(
+        outputs.size(),
+        0,
+        common::errors::Fatal("The size of output should be greater than 0."));
+    auto output = outputs.back();
+    PADDLE_ENFORCE_EQ(
+        output.size(),
+        1UL,
+        common::errors::Fatal("The size of output should be equal to 1."));
+    size_t size = GetSize(output[0]);
+    PADDLE_ENFORCE_GT(
+        size,
+        0,
+        common::errors::Fatal("The size of output should be greater than 0."));
+    int64_t *result = static_cast<int64_t *>(output[0].data.data());
+    for (size_t i = 0; i < std::min<size_t>(11, size); i++) {
+      EXPECT_EQ(result[i], chinese_ner_result_data[i]);
+    }
+  }
+}
+
+TEST(Analyzer_Chinese_ner, profile) { profile(); }
+
+TEST(Analyzer_Chinese_ner, profile_memory_load) {
+  profile(true /* memory_load */);
+}
+
+// Compare result of NativeConfig and AnalysisConfig
+TEST(Analyzer_Chinese_ner, compare) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareNativeAndAnalysis(
+      reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
+}
+
+// Compare Deterministic result
+TEST(Analyzer_Chinese_ner, compare_determine) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                       input_slots_all);
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_openvino_tester.cc

@@ -0,0 +1,72 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <fstream>
+#include <iostream>
+
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+
+void SetConfig(AnalysisConfig *cfg) {
+  cfg->SetModel(FLAGS_infer_model + "/inference.pdmodel",
+                FLAGS_infer_model + "/inference.pdiparams");
+  cfg->DisableGpu();
+  cfg->SwitchIrOptim();
+  cfg->EnableOpenVINOEngine(AnalysisConfig::Precision::kFloat32);
+  if (cfg->openvino_engine_enabled()) {
+    cfg->SetCpuMathLibraryNumThreads(FLAGS_cpu_num_threads);
+  }
+}
+
+void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
+  SetFakeImageInput(inputs,
+                    FLAGS_infer_model,
+                    true,
+                    "inference.pdmodel",
+                    "inference.pdiparams");
+}
+
+// Easy for profiling independently.
+void profile() {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  std::vector<std::vector<PaddleTensor>> outputs;
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                 input_slots_all,
+                 &outputs,
+                 FLAGS_num_threads);
+}
+
+TEST(Analyzer_openvino_resnet50, profile) { profile(); }
+
+#ifdef PADDLE_WITH_OPENVINO
+TEST(Analyzer_openvino_resnet50, compare_determine) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                       input_slots_all);
+}
+#endif
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_pyramid_dnn_tester.cc

@@ -0,0 +1,205 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+
+struct DataRecord {
+  std::vector<std::vector<int64_t>> query_basic, query_phrase, title_basic,
+      title_phrase;
+  std::vector<size_t> lod1, lod2, lod3, lod4;
+  size_t batch_iter{0}, batch_size{1}, num_samples;  // total number of samples
+  DataRecord() = default;
+  explicit DataRecord(const std::string &path, int batch_size = 1)
+      : batch_size(batch_size) {
+    Load(path);
+  }
+  DataRecord NextBatch() {
+    DataRecord data;
+    size_t batch_end = batch_iter + batch_size;
+    // NOTE skip the final batch, if no enough data is provided.
+    if (batch_end <= query_basic.size()) {
+      GetInputPerBatch(
+          query_basic, &data.query_basic, &data.lod1, batch_iter, batch_end);
+      GetInputPerBatch(
+          query_phrase, &data.query_phrase, &data.lod2, batch_iter, batch_end);
+      GetInputPerBatch(
+          title_basic, &data.title_basic, &data.lod3, batch_iter, batch_end);
+      GetInputPerBatch(
+          title_phrase, &data.title_phrase, &data.lod4, batch_iter, batch_end);
+    }
+    batch_iter += batch_size;
+    return data;
+  }
+  void Load(const std::string &path) {
+    std::ifstream file(path);
+    std::string line;
+    int num_lines = 0;
+    while (std::getline(file, line)) {
+      std::vector<std::string> data;
+      split(line, ';', &data);
+      // load query data
+      std::vector<int64_t> query_basic_data;
+      split_to_int64(data[1], ' ', &query_basic_data);
+      std::vector<int64_t> query_phrase_data;
+      split_to_int64(data[2], ' ', &query_phrase_data);
+      // load title data
+      std::vector<int64_t> title_basic_data;
+      split_to_int64(data[3], ' ', &title_basic_data);
+      std::vector<int64_t> title_phrase_data;
+      split_to_int64(data[4], ' ', &title_phrase_data);
+      // filter the empty data
+      bool flag =
+          data[1].size() && data[2].size() && data[3].size() && data[4].size();
+      if (flag) {
+        query_basic.push_back(std::move(query_basic_data));
+        query_phrase.push_back(std::move(query_phrase_data));
+        title_basic.push_back(std::move(title_basic_data));
+        title_phrase.push_back(std::move(title_phrase_data));
+        num_lines++;
+      }
+    }
+    num_samples = num_lines;
+  }
+};
+
+void PrepareInputs(std::vector<PaddleTensor> *input_slots,
+                   DataRecord *data,
+                   int batch_size) {
+  PaddleTensor query_basic_tensor, query_phrase_tensor, title_basic_tensor,
+      title_phrase_tensor;
+  query_basic_tensor.name = "query_basic";
+  query_phrase_tensor.name = "query_phrase";
+  title_basic_tensor.name = "pos_title_basic";
+  title_phrase_tensor.name = "pos_title_phrase";
+  auto one_batch = data->NextBatch();
+  // assign data
+  TensorAssignData<int64_t>(
+      &query_basic_tensor, one_batch.query_basic, one_batch.lod1);
+  TensorAssignData<int64_t>(
+      &query_phrase_tensor, one_batch.query_phrase, one_batch.lod2);
+  TensorAssignData<int64_t>(
+      &title_basic_tensor, one_batch.title_basic, one_batch.lod3);
+  TensorAssignData<int64_t>(
+      &title_phrase_tensor, one_batch.title_phrase, one_batch.lod4);
+  // Set inputs.
+  input_slots->assign({query_basic_tensor,
+                       query_phrase_tensor,
+                       title_basic_tensor,
+                       title_phrase_tensor});
+  for (auto &tensor : *input_slots) {
+    tensor.dtype = PaddleDType::INT64;
+  }
+}
+
+void SetConfig(AnalysisConfig *cfg) {
+  cfg->SetModel(FLAGS_infer_model);
+  cfg->DisableGpu();
+  cfg->SwitchSpecifyInputNames();
+  cfg->SwitchIrOptim();
+  cfg->SetCpuMathLibraryNumThreads(FLAGS_cpu_num_threads);
+}
+
+void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
+  DataRecord data(FLAGS_infer_data, FLAGS_batch_size);
+  std::vector<PaddleTensor> input_slots;
+  int epoch = FLAGS_test_all_data ? data.num_samples / FLAGS_batch_size : 1;
+  LOG(INFO) << "number of samples: " << epoch * FLAGS_batch_size;
+  for (int bid = 0; bid < epoch; ++bid) {
+    PrepareInputs(&input_slots, &data, FLAGS_batch_size);
+    (*inputs).emplace_back(input_slots);
+  }
+}
+
+// Easy for profiling independently.
+TEST(Analyzer_Pyramid_DNN, profile) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+  std::vector<std::vector<PaddleTensor>> outputs;
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                 input_slots_all,
+                 &outputs,
+                 FLAGS_num_threads);
+
+  if (FLAGS_num_threads == 1 && !FLAGS_test_all_data && !FLAGS_zero_copy) {
+    PADDLE_ENFORCE_GT(
+        outputs.size(),
+        0,
+        common::errors::Fatal("The size of output should be greater than 0."));
+    auto output = outputs.back();
+    PADDLE_ENFORCE_EQ(
+        output.size(),
+        1UL,
+        common::errors::Fatal("The size of output should be equal to 1."));
+    size_t size = GetSize(output[0]);
+    PADDLE_ENFORCE_GT(
+        size,
+        0,
+        common::errors::Fatal("The size of output should be greater than 0."));
+    float *result = static_cast<float *>(output[0].data.data());
+    // output is probability, which is in (0, 1).
+    for (size_t i = 0; i < size; i++) {
+      EXPECT_GT(result[i], 0);
+      EXPECT_LT(result[i], 1);
+    }
+  }
+}
+
+// Compare result of NativeConfig and AnalysisConfig
+TEST(Analyzer_Pyramid_DNN, compare) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareNativeAndAnalysis(
+      reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
+}
+
+// Compare result of AnalysisConfig and AnalysisConfig + ZeroCopy
+TEST(Analyzer_Pyramid_DNN, compare_zero_copy) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  AnalysisConfig cfg1;
+  SetConfig(&cfg1);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  std::vector<std::string> outputs_name;
+  outputs_name.emplace_back("cos_sim_2.tmp_0");
+  CompareAnalysisAndZeroCopy(reinterpret_cast<PaddlePredictor::Config *>(&cfg),
+                             reinterpret_cast<PaddlePredictor::Config *>(&cfg1),
+                             input_slots_all,
+                             outputs_name);
+}
+
+// Compare Deterministic result
+TEST(Analyzer_Pyramid_DNN, compare_determine) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                       input_slots_all);
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_quant_image_classification_tester.cc

@@ -0,0 +1,143 @@
+/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <fstream>
+#include <iostream>
+
+#include "paddle/fluid/inference/api/paddle_analysis_config.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+PD_DEFINE_bool(enable_onednn, true, "Enable ONEDNN");
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+
+void SetConfig(AnalysisConfig *cfg, std::string model_path) {
+  cfg->SetModel(model_path);
+  cfg->DisableGpu();
+  cfg->SwitchIrOptim(true);
+  cfg->SetCpuMathLibraryNumThreads(FLAGS_cpu_num_threads);
+  cfg->EnableNewIR();
+  cfg->EnableNewExecutor();
+  cfg->SetOptimizationLevel(3);
+
+  if (FLAGS_enable_onednn) cfg->EnableONEDNN();
+}
+
+template <typename T>
+class TensorReader {
+ public:
+  TensorReader(std::ifstream &file,
+               size_t beginning_offset,
+               std::vector<int> shape,
+               std::string name)
+      : file_(file),
+        position_(beginning_offset),
+        shape_(shape),
+        name_(name),
+        numel_(0) {
+    numel_ = std::accumulate(
+        shape_.begin(), shape_.end(), size_t{1}, std::multiplies<size_t>());
+  }
+
+  PaddleTensor NextBatch() {
+    PaddleTensor tensor;
+    tensor.name = name_;
+    tensor.shape = shape_;
+    tensor.dtype = GetPaddleDType<T>();
+    tensor.data.Resize(numel_ * sizeof(T));
+
+    file_.seekg(position_);
+    file_.read(static_cast<char *>(tensor.data.data()), numel_ * sizeof(T));
+    position_ = file_.tellg();
+
+    if (file_.eof()) LOG(ERROR) << name_ << ": reached end of stream";
+    if (file_.fail())
+      throw std::runtime_error(name_ + ": failed reading file.");
+
+    return tensor;
+  }
+
+ protected:
+  std::ifstream &file_;
+  size_t position_;
+  std::vector<int> shape_;
+  std::string name_;
+  size_t numel_;
+};
+
+void SetInput(std::vector<std::vector<PaddleTensor>> *inputs,
+              bool with_accuracy_layer = FLAGS_with_accuracy_layer,
+              int32_t batch_size = FLAGS_batch_size) {
+  std::ifstream file(FLAGS_infer_data, std::ios::binary);
+  if (!file) {
+    FAIL() << "Couldn't open file: " << FLAGS_infer_data;
+  }
+
+  int64_t total_images{0};
+  file.read(reinterpret_cast<char *>(&total_images), sizeof(total_images));
+  LOG(INFO) << "Total images in file: " << total_images;
+
+  std::vector<int> image_batch_shape{batch_size, 3, 224, 224};
+  std::vector<int> label_batch_shape{batch_size, 1};
+  auto images_offset_in_file = static_cast<size_t>(file.tellg());
+
+  TensorReader<float> image_reader(
+      file, images_offset_in_file, image_batch_shape, "image");
+
+  auto iterations_max = total_images / batch_size;
+  auto iterations = iterations_max;
+  if (FLAGS_iterations > 0 && FLAGS_iterations < iterations_max) {
+    iterations = FLAGS_iterations;
+  }
+
+  auto labels_offset_in_file =
+      images_offset_in_file + sizeof(float) * total_images * 3 * 224 * 224;
+
+  TensorReader<int64_t> label_reader(
+      file, labels_offset_in_file, label_batch_shape, "label");
+  for (auto i = 0; i < iterations; i++) {
+    auto images = image_reader.NextBatch();
+    std::vector<PaddleTensor> tmp_vec;
+    tmp_vec.push_back(std::move(images));
+    if (with_accuracy_layer) {
+      auto labels = label_reader.NextBatch();
+      tmp_vec.push_back(std::move(labels));
+    }
+    inputs->push_back(std::move(tmp_vec));
+  }
+}
+
+TEST(Analyzer_quant_image_classification, quantization) {
+  AnalysisConfig fp32_cfg;
+  SetConfig(&fp32_cfg, FLAGS_fp32_model);
+  fp32_cfg.EnableONEDNN();
+
+  AnalysisConfig int8_cfg;
+  SetConfig(&int8_cfg, FLAGS_int8_model);
+  if (FLAGS_enable_int8_qat) int8_cfg.EnableOnednnInt8();
+
+  // read data from file and prepare batches with test data
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+
+  // 0 is avg_cost, 1 is top1_accuracy, 2 is top5_accuracy or mAP
+  CompareAnalysisAndAnalysis(
+      &fp32_cfg, &int8_cfg, input_slots_all, FLAGS_with_accuracy_layer, 1);
+}
+
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_rnn1_tester.cc

@@ -0,0 +1,325 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test/cpp/inference/api/tester_helper.h"
+
+#include "paddle/common/enforce.h"
+
+PD_DEFINE_bool(with_precision_check, true, "turn on test");
+
+namespace paddle {
+namespace inference {
+
+using namespace framework;  // NOLINT
+
+struct DataRecord {
+  std::vector<std::vector<std::vector<float>>> link_step_data_all;
+  std::vector<std::vector<float>> week_data_all, minute_data_all;
+  std::vector<size_t> lod1, lod2, lod3;
+  std::vector<std::vector<float>> rnn_link_data, rnn_week_datas,
+      rnn_minute_datas;
+  size_t num_samples;  // total number of samples
+  size_t batch_iter{0};
+  size_t batch_size{1};
+  DataRecord() = default;
+
+  explicit DataRecord(const std::string &path, int batch_size = 1)
+      : batch_size(batch_size) {
+    Load(path);
+  }
+
+  DataRecord NextBatch() {
+    DataRecord data;
+    size_t batch_end = batch_iter + batch_size;
+    // NOTE skip the final batch, if no enough data is provided.
+    if (batch_end <= link_step_data_all.size()) {
+      data.link_step_data_all.assign(link_step_data_all.begin() + batch_iter,
+                                     link_step_data_all.begin() + batch_end);
+      data.week_data_all.assign(week_data_all.begin() + batch_iter,
+                                week_data_all.begin() + batch_end);
+      data.minute_data_all.assign(minute_data_all.begin() + batch_iter,
+                                  minute_data_all.begin() + batch_end);
+      // Prepare LoDs
+      data.lod1.push_back(0);
+      data.lod2.push_back(0);
+      data.lod3.push_back(0);
+      PADDLE_ENFORCE_EQ(!data.link_step_data_all.empty(),
+                        true,
+                        common::errors::InvalidArgument(
+                            "link_step_data_all should not be empty."));
+
+      PADDLE_ENFORCE_EQ(!data.week_data_all.empty(),
+                        true,
+                        common::errors::InvalidArgument(
+                            "week_data_all should not be empty."));
+
+      PADDLE_ENFORCE_EQ(!data.minute_data_all.empty(),
+                        true,
+                        common::errors::InvalidArgument(
+                            "minute_data_all should not be empty."));
+      PADDLE_ENFORCE_EQ(
+          data.link_step_data_all.size(),
+          data.week_data_all.size(),
+          platform::errors::InvalidArgument(
+              "The value of data.link_step_data_all.size() is not equal to the "
+              "value of data.week_data_all.size()."))
+      PADDLE_ENFORCE_EQ(
+          data.minute_data_all.size(),
+          data.link_step_data_all.size(),
+          platform::errors::InvalidArgument(
+              "The value of data.minute_data_all.size() is not equal to the "
+              "value of data.link_step_data_all.size()."))
+      for (size_t j = 0; j < data.link_step_data_all.size(); j++) {
+        for (const auto &d : data.link_step_data_all[j]) {
+          data.rnn_link_data.push_back(d);
+        }
+        data.rnn_week_datas.push_back(data.week_data_all[j]);
+        data.rnn_minute_datas.push_back(data.minute_data_all[j]);
+        // calculate lod
+        data.lod1.push_back(data.lod1.back() +
+                            data.link_step_data_all[j].size());
+        data.lod3.push_back(data.lod3.back() + 1);
+        for (size_t i = 1; i < data.link_step_data_all[j].size() + 1; i++) {
+          data.lod2.push_back(data.lod2.back() +
+                              data.link_step_data_all[j].size());
+        }
+      }
+    }
+    batch_iter += batch_size;
+    return data;
+  }
+  void Load(const std::string &path) {
+    std::ifstream file(path);
+    std::string line;
+    int num_lines = 0;
+    while (std::getline(file, line)) {
+      num_lines++;
+      std::vector<std::string> data;
+      split(line, ':', &data);
+      std::vector<std::vector<float>> link_step_data;
+      std::vector<std::string> link_datas;
+      split(data[0], '|', &link_datas);
+      for (auto &step_data : link_datas) {
+        std::vector<float> tmp;
+        split_to_float(step_data, ',', &tmp);
+        link_step_data.push_back(tmp);
+      }
+      // load week data
+      std::vector<float> week_data;
+      split_to_float(data[2], ',', &week_data);
+      // load minute data
+      std::vector<float> minute_data;
+      split_to_float(data[1], ',', &minute_data);
+      link_step_data_all.push_back(std::move(link_step_data));
+      week_data_all.push_back(std::move(week_data));
+      minute_data_all.push_back(std::move(minute_data));
+    }
+    num_samples = num_lines;
+  }
+};
+
+void PrepareInputs(std::vector<PaddleTensor> *input_slots,
+                   DataRecord *data,
+                   int batch_size) {
+  PaddleTensor lod_attention_tensor, init_zero_tensor, lod_tensor_tensor,
+      week_tensor, minute_tensor;
+  lod_attention_tensor.name = "data_lod_attention";
+  init_zero_tensor.name = "cell_init";
+  lod_tensor_tensor.name = "data";
+  week_tensor.name = "week";
+  minute_tensor.name = "minute";
+  auto one_batch = data->NextBatch();
+  std::vector<int> rnn_link_data_shape(
+      {static_cast<int>(one_batch.rnn_link_data.size()),
+       static_cast<int>(one_batch.rnn_link_data.front().size())});
+  lod_attention_tensor.shape.assign({1, 2});
+  lod_attention_tensor.lod.assign({one_batch.lod1, one_batch.lod2});
+  init_zero_tensor.shape.assign({batch_size, 15});
+  init_zero_tensor.lod.assign({one_batch.lod3});
+  lod_tensor_tensor.shape = rnn_link_data_shape;
+  lod_tensor_tensor.lod.assign({one_batch.lod1});
+  week_tensor.shape.assign(
+      {static_cast<int>(one_batch.rnn_week_datas.size()),
+       static_cast<int>(one_batch.rnn_week_datas.front().size())});
+  week_tensor.lod.assign({one_batch.lod3});
+  minute_tensor.shape.assign(
+      {static_cast<int>(one_batch.rnn_minute_datas.size()),
+       static_cast<int>(one_batch.rnn_minute_datas.front().size())});
+  minute_tensor.lod.assign({one_batch.lod3});
+  // assign data
+  TensorAssignData<float>(&lod_attention_tensor,
+                          std::vector<std::vector<float>>({{0, 0}}));
+  std::vector<float> tmp_zeros(batch_size * 15, 0.);
+  TensorAssignData<float>(&init_zero_tensor, {tmp_zeros});
+  TensorAssignData<float>(&lod_tensor_tensor, one_batch.rnn_link_data);
+  TensorAssignData<float>(&week_tensor, one_batch.rnn_week_datas);
+  TensorAssignData<float>(&minute_tensor, one_batch.rnn_minute_datas);
+  // Set inputs.
+  auto init_zero_tensor1 = init_zero_tensor;
+  init_zero_tensor1.name = "hidden_init";
+  input_slots->assign({week_tensor,
+                       init_zero_tensor,
+                       minute_tensor,
+                       init_zero_tensor1,
+                       lod_attention_tensor,
+                       lod_tensor_tensor});
+  for (auto &tensor : *input_slots) {
+    tensor.dtype = PaddleDType::FLOAT32;
+  }
+}
+
+void PrepareZeroCopyInputs(ZeroCopyTensor *lod_attention_tensor,
+                           ZeroCopyTensor *cell_init_tensor,
+                           ZeroCopyTensor *data_tensor,
+                           ZeroCopyTensor *hidden_init_tensor,
+                           ZeroCopyTensor *week_tensor,
+                           ZeroCopyTensor *minute_tensor,
+                           DataRecord *data_record,
+                           int batch_size) {
+  auto one_batch = data_record->NextBatch();
+  std::vector<int> rnn_link_data_shape(
+      {static_cast<int>(one_batch.rnn_link_data.size()),
+       static_cast<int>(one_batch.rnn_link_data.front().size())});
+  lod_attention_tensor->Reshape({1, 2});
+  lod_attention_tensor->SetLoD({one_batch.lod1, one_batch.lod2});
+
+  cell_init_tensor->Reshape({batch_size, 15});
+  cell_init_tensor->SetLoD({one_batch.lod3});
+
+  hidden_init_tensor->Reshape({batch_size, 15});
+  hidden_init_tensor->SetLoD({one_batch.lod3});
+
+  data_tensor->Reshape(rnn_link_data_shape);
+  data_tensor->SetLoD({one_batch.lod1});
+
+  week_tensor->Reshape(
+      {static_cast<int>(one_batch.rnn_week_datas.size()),
+       static_cast<int>(one_batch.rnn_week_datas.front().size())});
+  week_tensor->SetLoD({one_batch.lod3});
+
+  minute_tensor->Reshape(
+      {static_cast<int>(one_batch.rnn_minute_datas.size()),
+       static_cast<int>(one_batch.rnn_minute_datas.front().size())});
+  minute_tensor->SetLoD({one_batch.lod3});
+
+  // assign data
+  std::array<float, 2> arr0 = {0, 0};
+  std::vector<float> zeros(batch_size * 15, 0);
+  std::copy_n(arr0.data(),
+              2,
+              lod_attention_tensor->mutable_data<float>(PaddlePlace::kCPU));
+  std::copy_n(
+      arr0.data(), 2, data_tensor->mutable_data<float>(PaddlePlace::kCPU));
+  std::copy_n(zeros.begin(),
+              zeros.size(),
+              cell_init_tensor->mutable_data<float>(PaddlePlace::kCPU));
+  std::copy_n(zeros.begin(),
+              zeros.size(),
+              hidden_init_tensor->mutable_data<float>(PaddlePlace::kCPU));
+  ZeroCopyTensorAssignData(data_tensor, one_batch.rnn_link_data);
+  ZeroCopyTensorAssignData(week_tensor, one_batch.rnn_week_datas);
+  ZeroCopyTensorAssignData(minute_tensor, one_batch.rnn_minute_datas);
+}
+
+void SetConfig(AnalysisConfig *cfg) {
+  cfg->SetModel(FLAGS_infer_model + "/__model__", FLAGS_infer_model + "/param");
+  cfg->DisableGpu();
+  cfg->SwitchSpecifyInputNames();
+  cfg->SwitchIrOptim();
+}
+
+void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
+  DataRecord data(FLAGS_infer_data, FLAGS_batch_size);
+  std::vector<PaddleTensor> input_slots;
+  int epoch =
+      FLAGS_test_all_data ? data.num_samples / FLAGS_batch_size : 1;  // NOLINT
+  LOG(INFO) << "number of samples: " << epoch * FLAGS_batch_size;
+  for (int bid = 0; bid < epoch; ++bid) {
+    PrepareInputs(&input_slots, &data, FLAGS_batch_size);
+    (*inputs).emplace_back(input_slots);
+  }
+}
+
+// Easy for profiling independently.
+TEST(Analyzer_rnn1, profile) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+  cfg.DisableGpu();
+  cfg.SwitchIrDebug();
+  std::vector<std::vector<PaddleTensor>> outputs;
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                 input_slots_all,
+                 &outputs,
+                 FLAGS_num_threads);
+}
+
+// Compare result of NativeConfig and AnalysisConfig
+TEST(Analyzer_rnn1, compare) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareNativeAndAnalysis(
+      reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
+}
+
+// Compare Deterministic result
+TEST(Analyzer_rnn1, compare_determine) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                       input_slots_all);
+}
+
+// Test Multi-Thread.
+TEST(Analyzer_rnn1, multi_thread) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+  std::vector<std::vector<PaddleTensor>> outputs;
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                 input_slots_all,
+                 &outputs,
+                 2 /* multi_thread */);
+}
+
+// Compare result of AnalysisConfig and AnalysisConfig + ZeroCopy
+TEST(Analyzer_rnn1, compare_zero_copy) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  AnalysisConfig cfg1;
+  SetConfig(&cfg1);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  std::vector<std::string> outputs_name;
+  outputs_name.emplace_back("final_output.tmp_1");
+  CompareAnalysisAndZeroCopy(reinterpret_cast<PaddlePredictor::Config *>(&cfg),
+                             reinterpret_cast<PaddlePredictor::Config *>(&cfg1),
+                             input_slots_all,
+                             outputs_name);
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_rnn2_tester.cc

@@ -0,0 +1,194 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+
+using namespace framework;  // NOLINT
+static std::vector<float> result_data;
+
+struct DataRecord {
+  std::vector<std::vector<std::vector<float>>> link_step_data_all;
+  std::vector<size_t> lod;
+  std::vector<std::vector<float>> rnn_link_data;
+  size_t num_samples;  // total number of samples
+  size_t batch_iter{0};
+  size_t batch_size{1};
+  DataRecord() : link_step_data_all(), lod(), rnn_link_data(), num_samples(0) {}
+  explicit DataRecord(const std::string &path, int batch_size = 1)
+      : link_step_data_all(),
+        lod(),
+        rnn_link_data(),
+        num_samples(0),
+        batch_size(batch_size) {
+    Load(path);
+  }
+  DataRecord NextBatch() {
+    DataRecord data;
+    size_t batch_end = batch_iter + batch_size;
+    // NOTE skip the final batch, if no enough data is provided.
+    if (batch_end <= link_step_data_all.size()) {
+      data.link_step_data_all.assign(link_step_data_all.begin() + batch_iter,
+                                     link_step_data_all.begin() + batch_end);
+      // Prepare LoDs
+      data.lod.push_back(0);
+      PADDLE_ENFORCE_EQ(
+          !data.link_step_data_all.empty(),
+          true,
+          common::errors::InvalidArgument(
+              "`data.link_step_data_all` is empty, please check"));
+      for (size_t j = 0; j < data.link_step_data_all.size(); j++) {
+        for (const auto &d : data.link_step_data_all[j]) {
+          data.rnn_link_data.push_back(d);
+          // calculate lod
+          data.lod.push_back(data.lod.back() + 11);
+        }
+      }
+    }
+    batch_iter += batch_size;
+    return data;
+  }
+  void Load(const std::string &path) {
+    std::ifstream file(path);
+    std::string line;
+    int num_lines = 0;
+    result_data.clear();
+    while (std::getline(file, line)) {
+      num_lines++;
+      std::vector<std::string> data;
+      split(line, ':', &data);
+      if (num_lines % 2) {  // feature
+        std::vector<std::string> feature_data;
+        split(data[1], ' ', &feature_data);
+        std::vector<std::vector<float>> link_step_data;
+        int feature_count = 1;
+        std::vector<float> feature;
+        for (auto &step_data : feature_data) {
+          std::vector<float> tmp;
+          split_to_float(step_data, ',', &tmp);
+          feature.insert(feature.end(), tmp.begin(), tmp.end());
+          if (feature_count % 11 == 0) {  // each sample has 11 features
+            link_step_data.push_back(feature);
+            feature.clear();
+          }
+          feature_count++;
+        }
+        link_step_data_all.push_back(std::move(link_step_data));
+      } else {  // result
+        std::vector<float> tmp;
+        split_to_float(data[1], ',', &tmp);
+        result_data.insert(result_data.end(), tmp.begin(), tmp.end());
+      }
+    }
+    num_samples = num_lines / 2;
+  }
+};
+void PrepareInputs(std::vector<PaddleTensor> *input_slots,
+                   DataRecord *data,
+                   int batch_size) {
+  PaddleTensor feed_tensor;
+  feed_tensor.name = "feed";
+  auto one_batch = data->NextBatch();
+  int token_size = one_batch.rnn_link_data.size();
+  // each token has 11 features, each feature's dim is 54.
+  std::vector<int> rnn_link_data_shape({token_size * 11, 54});
+  feed_tensor.shape = rnn_link_data_shape;
+  feed_tensor.lod.assign({one_batch.lod});
+  feed_tensor.dtype = PaddleDType::FLOAT32;
+  TensorAssignData<float>(&feed_tensor, one_batch.rnn_link_data);
+  // Set inputs.
+  input_slots->assign({feed_tensor});
+}
+
+void SetConfig(AnalysisConfig *cfg) {
+  cfg->SetModel(FLAGS_infer_model + "/__model__", FLAGS_infer_model + "/param");
+  cfg->DisableGpu();
+  cfg->SwitchSpecifyInputNames();
+  cfg->SwitchIrOptim();
+}
+
+void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
+  DataRecord data(FLAGS_infer_data, FLAGS_batch_size);
+  std::vector<PaddleTensor> input_slots;
+  int epoch =
+      FLAGS_test_all_data ? data.num_samples / FLAGS_batch_size : 1;  // NOLINT
+  LOG(INFO) << "number of samples: " << epoch * FLAGS_batch_size;
+  for (int bid = 0; bid < epoch; ++bid) {
+    PrepareInputs(&input_slots, &data, FLAGS_batch_size);
+    (*inputs).emplace_back(input_slots);
+  }
+}
+
+// Easy for profiling independently.
+TEST(Analyzer_rnn2, profile) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+  std::vector<std::vector<PaddleTensor>> outputs;
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                 input_slots_all,
+                 &outputs,
+                 FLAGS_num_threads);
+
+  if (FLAGS_num_threads == 1 && !FLAGS_test_all_data) {
+    // the first inference result
+    PADDLE_ENFORCE_GT(
+        outputs.size(),
+        0,
+        common::errors::Fatal("The size of output should be greater than 0."));
+    auto output = outputs.back();
+    PADDLE_ENFORCE_GT(
+        output.size(),
+        0,
+        common::errors::Fatal("The size of output should be greater than 0."));
+    size_t size = GetSize(output[0]);
+    PADDLE_ENFORCE_GT(
+        size,
+        0,
+        common::errors::Fatal("The size of output should be greater than 0."));
+    float *result = static_cast<float *>(output[0].data.data());
+    for (size_t i = 0; i < size; i++) {
+      EXPECT_NEAR(result[i], result_data[i], 1e-3);
+    }
+  }
+}
+
+// Compare result of NativeConfig and AnalysisConfig
+TEST(Analyzer_rnn2, compare) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareNativeAndAnalysis(
+      reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
+}
+
+// Compare Deterministic result
+TEST(Analyzer_rnn2, compare_determine) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                       input_slots_all);
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_seq_conv1_tester.cc

@@ -0,0 +1,194 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+
+struct DataRecord {
+  std::vector<std::vector<int64_t>> title1, title2, title3, l1;
+  std::vector<size_t> lod1, lod2, lod3, l1_lod;
+  size_t batch_iter{0}, batch_size{1}, num_samples;  // total number of samples
+  DataRecord()
+      : title1(),
+        title2(),
+        title3(),
+        l1(),
+        lod1(),
+        lod2(),
+        lod3(),
+        l1_lod(),
+        batch_size(0),
+        num_samples(0) {}
+  explicit DataRecord(const std::string &path, int batch_size = 1)
+      : title1(),
+        title2(),
+        title3(),
+        l1(),
+        lod1(),
+        lod2(),
+        lod3(),
+        l1_lod(),
+        batch_size(batch_size),
+        num_samples(0) {
+    Load(path);
+  }
+  DataRecord NextBatch() {
+    DataRecord data;
+    size_t batch_end = batch_iter + batch_size;
+    // NOTE skip the final batch, if no enough data is provided.
+    if (batch_end <= title1.size()) {
+      GetInputPerBatch(title1, &data.title1, &data.lod1, batch_iter, batch_end);
+      GetInputPerBatch(title2, &data.title2, &data.lod2, batch_iter, batch_end);
+      GetInputPerBatch(title3, &data.title3, &data.lod3, batch_iter, batch_end);
+      GetInputPerBatch(l1, &data.l1, &data.l1_lod, batch_iter, batch_end);
+    }
+    batch_iter += batch_size;
+    return data;
+  }
+  void Load(const std::string &path) {
+    std::ifstream file(path);
+    std::string line;
+    int num_lines = 0;
+    while (std::getline(file, line)) {
+      num_lines++;
+      std::vector<std::string> data;
+      split(line, '\t', &data);
+      PADDLE_ENFORCE_GT(data.size(),
+                        4,
+                        common::errors::Fatal("The size of data is invalid."));
+      // load title1 data
+      std::vector<int64_t> title1_data;
+      split_to_int64(data[0], ' ', &title1_data);
+      // load title2 data
+      std::vector<int64_t> title2_data;
+      split_to_int64(data[1], ' ', &title2_data);
+      // load title3 data
+      std::vector<int64_t> title3_data;
+      split_to_int64(data[2], ' ', &title3_data);
+      // load l1 data
+      std::vector<int64_t> l1_data;
+      split_to_int64(data[3], ' ', &l1_data);
+      title1.push_back(std::move(title1_data));
+      title2.push_back(std::move(title2_data));
+      title3.push_back(std::move(title3_data));
+      l1.push_back(std::move(l1_data));
+    }
+    num_samples = num_lines;
+  }
+};
+
+void PrepareInputs(std::vector<PaddleTensor> *input_slots,
+                   DataRecord *data,
+                   int batch_size) {
+  PaddleTensor title1_tensor, title2_tensor, title3_tensor, l1_tensor;
+  title1_tensor.name = "title1";
+  title2_tensor.name = "title2";
+  title3_tensor.name = "title3";
+  l1_tensor.name = "l1";
+  auto one_batch = data->NextBatch();
+  // assign data
+  TensorAssignData<int64_t>(&title1_tensor, one_batch.title1, one_batch.lod1);
+  TensorAssignData<int64_t>(&title2_tensor, one_batch.title2, one_batch.lod2);
+  TensorAssignData<int64_t>(&title3_tensor, one_batch.title3, one_batch.lod3);
+  TensorAssignData<int64_t>(&l1_tensor, one_batch.l1, one_batch.l1_lod);
+  // Set inputs.
+  input_slots->assign({title1_tensor, title2_tensor, title3_tensor, l1_tensor});
+  for (auto &tensor : *input_slots) {
+    tensor.dtype = PaddleDType::INT64;
+  }
+}
+
+void SetConfig(AnalysisConfig *cfg) {
+  cfg->SetModel(FLAGS_infer_model);
+  cfg->DisableGpu();
+  cfg->SwitchSpecifyInputNames();
+  cfg->SwitchIrOptim();
+}
+
+void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
+  DataRecord data(FLAGS_infer_data, FLAGS_batch_size);
+  std::vector<PaddleTensor> input_slots;
+  int epoch =
+      FLAGS_test_all_data ? data.num_samples / FLAGS_batch_size : 1;  // NOLINT
+  LOG(INFO) << "number of samples: " << epoch * FLAGS_batch_size;
+  for (int bid = 0; bid < epoch; ++bid) {
+    PrepareInputs(&input_slots, &data, FLAGS_batch_size);
+    (*inputs).emplace_back(input_slots);
+  }
+}
+
+// Easy for profiling independently.
+TEST(Analyzer_seq_conv1, profile) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+  std::vector<std::vector<PaddleTensor>> outputs;
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                 input_slots_all,
+                 &outputs,
+                 FLAGS_num_threads);
+
+  if (FLAGS_num_threads == 1 && !FLAGS_test_all_data) {
+    // the first inference result
+    PADDLE_ENFORCE_GT(
+        outputs.size(),
+        0,
+        common::errors::Fatal("The size of output should be greater than 0."));
+    auto output = outputs.back();
+    PADDLE_ENFORCE_EQ(
+        output.size(),
+        1UL,
+        common::errors::Fatal("The size of output should be equal to 0."));
+    size_t size = GetSize(output[0]);
+    PADDLE_ENFORCE_GT(
+        size,
+        0,
+        common::errors::Fatal("The size of output should be greater than 0."));
+    float *result = static_cast<float *>(output[0].data.data());
+    // output is probability, which is in (0, 1).
+    for (size_t i = 0; i < size; i++) {
+      EXPECT_GT(result[i], 0);
+      EXPECT_LT(result[i], 1);
+    }
+  }
+}
+
+// Compare result of NativeConfig and AnalysisConfig
+TEST(Analyzer_seq_conv1, compare) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareNativeAndAnalysis(
+      reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
+}
+
+// Compare Deterministic result
+TEST(Analyzer_seq_conv1, compare_determine) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                       input_slots_all);
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_seq_pool1_compare_determine_tester.cc

@@ -0,0 +1,41 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <algorithm>
+#include <fstream>
+#include <iostream>
+
+#include "test/cpp/inference/api/analyzer_seq_pool1_tester_helper.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+namespace seq_pool1_tester {
+
+// Compare Deterministic result
+TEST(Analyzer_seq_pool1_compare_determine, compare_determine) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                       input_slots_all);
+}
+
+}  // namespace seq_pool1_tester
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_seq_pool1_compare_tester.cc

@@ -0,0 +1,40 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <algorithm>
+#include <fstream>
+#include <iostream>
+
+#include "test/cpp/inference/api/analyzer_seq_pool1_tester_helper.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+namespace seq_pool1_tester {
+
+TEST(Analyzer_seq_pool1_compare, compare) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareNativeAndAnalysis(
+      reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
+}
+
+}  // namespace seq_pool1_tester
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_seq_pool1_fuse_compare_zero_copy_tester.cc

@@ -0,0 +1,48 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <algorithm>
+#include <fstream>
+#include <iostream>
+
+#include "test/cpp/inference/api/analyzer_seq_pool1_tester_helper.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+namespace seq_pool1_tester {
+
+// Compare result of AnalysisConfig and AnalysisConfig + ZeroCopy
+TEST(Analyzer_seq_pool1_compare_zero_copy, compare_zero_copy) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  AnalysisConfig cfg1;
+  SetConfig(&cfg1);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  std::vector<std::string> outputs_name;
+  outputs_name.emplace_back(out_var_name);
+  CompareAnalysisAndZeroCopy(reinterpret_cast<PaddlePredictor::Config *>(&cfg),
+                             reinterpret_cast<PaddlePredictor::Config *>(&cfg1),
+                             input_slots_all,
+                             outputs_name);
+}
+
+}  // namespace seq_pool1_tester
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_seq_pool1_profile_tester.cc

@@ -0,0 +1,45 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <algorithm>
+#include <fstream>
+#include <iostream>
+
+#include "test/cpp/inference/api/analyzer_seq_pool1_tester_helper.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+namespace seq_pool1_tester {
+
+void profile(bool use_onednn = false) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg, use_onednn);
+
+  std::vector<std::vector<PaddleTensor>> outputs;
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                 input_slots_all,
+                 &outputs,
+                 FLAGS_num_threads);
+}
+
+TEST(Analyzer_seq_pool1_profile, profile) { profile(); }
+
+}  // namespace seq_pool1_tester
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_seq_pool1_tester_helper.h

@@ -0,0 +1,178 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+#pragma once
+#include <algorithm>
+#include <fstream>
+#include <iostream>
+#include <map>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+namespace seq_pool1_tester {
+
+// diff: similarity_norm.tmp_0, for speed: fc_4.tmp_1
+static const char out_var_name[] = "reduce_sum_0.tmp_0";
+
+// for diff: 154, for speed 111
+constexpr int num_slots = 154;
+
+struct OneSlotInBatch {
+  std::string name;
+  std::vector<std::vector<float>> data;
+  std::vector<int> shape;
+  std::vector<size_t> lod;
+};
+
+struct DataRecord {
+  std::vector<std::vector<OneSlotInBatch>> batched_data;
+  std::map<std::string, std::vector<std::vector<float>>> datasets;
+  size_t batch_iter{0}, num_samples;  // total number of samples
+
+  DataRecord() = default;
+  explicit DataRecord(const std::string &path, int batch_size = 1) {
+    Load(path);
+    Prepare(batch_size);
+  }
+
+  void Load(const std::string &path) {
+    std::ifstream file(path);
+    std::string line;
+    int num_lines = 0;
+    while (std::getline(file, line)) {
+      num_lines++;
+      std::vector<std::string> data;
+      split(line, '\t', &data);
+      std::vector<float> slot_data;
+      split_to_float(data[1], ' ', &slot_data);
+      std::string name = data[0];
+      PADDLE_ENFORCE_EQ(
+          slot_data.size() % 11,
+          0UL,
+          ::common::errors::Fatal(
+              "line %d, %s should be divisible", num_lines, name));
+      datasets[name].emplace_back(std::move(slot_data));
+    }
+    num_samples = num_lines / num_slots;
+    PADDLE_ENFORCE_EQ(
+        num_samples * num_slots,
+        static_cast<size_t>(num_lines),
+        ::common::errors::Fatal("num samples should be divisible"));
+    PADDLE_ENFORCE_GT(num_samples,
+                      0UL,
+                      ::common::errors::Fatal(
+                          "The num of samples should be greater than 0."));
+  }
+
+  void Prepare(int bs) {
+    for (auto it = datasets.begin(); it != datasets.end(); ++it) {
+      PADDLE_ENFORCE_EQ(
+          it->second.size(),
+          num_samples,
+          ::common::errors::Fatal("size of each slot should be equal"));
+    }
+    size_t num_batches = num_samples / bs;
+    EXPECT_GT(num_batches, 0UL);
+    batched_data.resize(num_batches);
+    for (auto &one_batch : batched_data) {
+      one_batch.resize(datasets.size());
+      size_t i = 0;
+      for (auto it = datasets.begin(); it != datasets.end(); ++it) {
+        auto &slot = one_batch[i];
+        slot.name = it->first;
+        slot.data.resize(bs);
+        slot.lod.resize(bs + 1);
+        slot.lod[0] = 0;
+        auto &lod = slot.lod;
+        auto &datas = it->second;
+        for (int k = 0; k < bs; ++k) {
+          size_t id = k + batch_iter * bs;
+          std::copy(datas[id].begin(),
+                    datas[id].end(),
+                    std::back_inserter(slot.data[k]));
+          size_t len = datas[id].size() / 11;
+          PADDLE_ENFORCE_EQ(
+              len * 11,
+              datas[id].size(),
+              ::common::errors::Fatal(
+                  "%s %d size should be divisible", slot.name, id));
+          lod[k + 1] = lod[k] + len;
+        }
+        slot.shape.assign({static_cast<int>(lod[bs]), 11});
+        i++;
+      }
+    }
+  }
+
+  const std::vector<OneSlotInBatch> &NextBatch() {
+    if (batch_iter >= batched_data.size() - 1) {
+      batch_iter = -1;
+    }
+    return batched_data[++batch_iter];
+  }
+};
+
+static void TensorAssignSlot(PaddleTensor *tensor, const OneSlotInBatch &slot) {
+  tensor->name = slot.name + "_embed";
+  tensor->shape = slot.shape;
+  tensor->dtype = PaddleDType::FLOAT32;
+  tensor->lod.clear();
+  tensor->lod.emplace_back(slot.lod);
+  TensorAssignData(tensor, slot.data);
+}
+
+void PrepareInputs(std::vector<PaddleTensor> *input_slots, DataRecord *data) {
+  const auto &one_batch = data->NextBatch();
+  input_slots->resize(one_batch.size());
+  for (size_t i = 0; i < one_batch.size(); ++i) {
+    auto &slot = one_batch[i];
+    TensorAssignSlot(&((*input_slots)[i]), slot);
+  }
+}
+
+void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
+  DataRecord data(FLAGS_infer_data, FLAGS_batch_size);
+  std::vector<PaddleTensor> input_slots;
+  int epoch = FLAGS_test_all_data ? data.batched_data.size() : 1;
+  LOG(INFO) << "number of samples: "
+            << data.batched_data.size() * FLAGS_batch_size;
+  for (int bid = 0; bid < epoch; ++bid) {
+    PrepareInputs(&input_slots, &data);
+    (*inputs).emplace_back(input_slots);
+  }
+}
+
+void SetConfig(AnalysisConfig *cfg, bool use_onednn = false) {
+  cfg->SetModel(FLAGS_infer_model + "/model", FLAGS_infer_model + "/params");
+  cfg->DisableGpu();
+  cfg->SwitchSpecifyInputNames();
+  cfg->SwitchIrDebug();
+  cfg->SetCpuMathLibraryNumThreads(FLAGS_cpu_num_threads);
+  if (use_onednn) {
+    cfg->EnableONEDNN();
+  }
+  // Enable seqpool_concat_fuse_pass, disabled by default since it takes much
+  // time
+  cfg->pass_builder()->InsertPass(2, "seqpool_concat_fuse_pass");
+}
+
+}  // namespace seq_pool1_tester
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_text_classification_tester.cc

@@ -0,0 +1,161 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+
+struct DataReader {
+  explicit DataReader(const std::string &path)
+      : file(new std::ifstream(path)) {}
+
+  bool NextBatch(std::vector<PaddleTensor> *input, int batch_size) {
+    PADDLE_ENFORCE_EQ(
+        batch_size,
+        1,
+        common::errors::Fatal("The size of batch should be equal to 1."));
+    std::string line;
+    PaddleTensor tensor;
+    tensor.dtype = PaddleDType::INT64;
+    tensor.lod.emplace_back(std::vector<size_t>({0}));
+    std::vector<int64_t> data;
+
+    for (int i = 0; i < batch_size; i++) {
+      if (!std::getline(*file, line)) return false;
+      inference::split_to_int64(line, ' ', &data);
+    }
+    tensor.lod.front().push_back(data.size());
+
+    tensor.data.Resize(data.size() * sizeof(int64_t));
+    PADDLE_ENFORCE_NE(
+        tensor.data.data(),
+        nullptr,
+        common::errors::Fatal("Variable `tensor.data.data()` is nullptr"));
+    PADDLE_ENFORCE_NE(
+        data.data(),
+        nullptr,
+        common::errors::Fatal("Variable `data.data()` is nullptr"));
+    memcpy(tensor.data.data(), data.data(), data.size() * sizeof(int64_t));
+    tensor.shape.push_back(data.size());
+    tensor.shape.push_back(1);
+    input->assign({tensor});
+    return true;
+  }
+
+  std::unique_ptr<std::ifstream> file = nullptr;
+};
+
+void SetConfig(AnalysisConfig *cfg) {
+  cfg->SetModel(FLAGS_infer_model);
+  cfg->DisableGpu();
+  cfg->SwitchSpecifyInputNames();
+  cfg->SwitchIrOptim();
+}
+
+void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
+  std::vector<PaddleTensor> input_slots;
+  DataReader reader(FLAGS_infer_data);
+  int num_batches = 0;
+  while (reader.NextBatch(&input_slots, FLAGS_batch_size)) {
+    (*inputs).emplace_back(input_slots);
+    ++num_batches;
+    if (!FLAGS_test_all_data) return;
+  }
+  LOG(INFO) << "total number of samples: " << num_batches * FLAGS_batch_size;
+}
+
+// Easy for profiling independently.
+TEST(Analyzer_Text_Classification, profile) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+  cfg.SwitchIrDebug();
+  std::vector<std::vector<PaddleTensor>> outputs;
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                 input_slots_all,
+                 &outputs,
+                 FLAGS_num_threads);
+
+  if (FLAGS_num_threads == 1) {
+    // Get output
+    PADDLE_ENFORCE_GT(
+        outputs.size(),
+        0,
+        common::errors::Fatal("The size of output should be greater than 0."));
+    LOG(INFO) << "get outputs " << outputs.back().size();
+    for (auto &output : outputs.back()) {
+      LOG(INFO) << "output.shape: " << to_string(output.shape);
+      // no lod ?
+      PADDLE_ENFORCE_EQ(
+          output.lod.size(),
+          0UL,
+          common::errors::InvalidArgument(
+              "The 'lod' size of 'output' should be 0, but received size %d.",
+              output.lod.size()));
+      LOG(INFO) << "output.dtype: " << output.dtype;
+      std::stringstream ss;
+      int num_data = 1;
+      for (auto i : output.shape) {
+        num_data *= i;
+      }
+
+      for (int i = 0; i < num_data; i++) {
+        ss << static_cast<float *>(output.data.data())[i] << " ";
+      }
+      LOG(INFO) << "output.data summary: " << ss.str();
+      // one batch ends
+    }
+  }
+}
+
+// Compare result of NativeConfig and AnalysisConfig
+TEST(Analyzer_Text_Classification, compare) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+  cfg.EnableMemoryOptim();
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareNativeAndAnalysis(
+      reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
+}
+
+// Compare Deterministic result
+TEST(Analyzer_Text_Classification, compare_determine) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                       input_slots_all);
+}
+
+TEST(Analyzer_Text_Classification, compare_against_embedding_fc_lstm_fused) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+  // Enable embedding_fc_lstm_fuse_pass (disabled by default)
+  cfg.pass_builder()->InsertPass(2, "embedding_fc_lstm_fuse_pass");
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareNativeAndAnalysis(
+      reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_transformer_tester_helper.h

@@ -0,0 +1,203 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+namespace transformer_tester {
+
+struct DataRecord {
+  std::vector<std::vector<int64_t>> src_word, src_pos, trg_word, init_idx;
+  std::vector<std::vector<float>> src_slf_attn_bias, init_score,
+      trg_src_attn_bias;
+  std::vector<std::vector<int32_t>> batch_data_shape;
+  std::vector<std::vector<size_t>> lod;
+  size_t batch_iter{0}, batch_size{1}, num_samples;  // total number of samples
+  DataRecord() = default;
+  explicit DataRecord(const std::string &path, int batch_size = 1)
+      : batch_size(batch_size) {
+    Load(path);
+  }
+  DataRecord NextBatch() {
+    DataRecord data;
+    size_t batch_end = batch_iter + batch_size;
+    // NOTE skip the final batch, if no enough data is provided.
+    if (batch_end <= src_word.size()) {
+      data.src_word.assign(src_word.begin() + batch_iter,
+                           src_word.begin() + batch_end);
+      data.src_pos.assign(src_pos.begin() + batch_iter,
+                          src_pos.begin() + batch_end);
+      data.src_slf_attn_bias.assign(src_slf_attn_bias.begin() + batch_iter,
+                                    src_slf_attn_bias.begin() + batch_end);
+      data.trg_word.assign(trg_word.begin() + batch_iter,
+                           trg_word.begin() + batch_end);
+      data.init_score.assign(init_score.begin() + batch_iter,
+                             init_score.begin() + batch_end);
+      data.init_idx.assign(init_idx.begin() + batch_iter,
+                           init_idx.begin() + batch_end);
+      data.trg_src_attn_bias.assign(trg_src_attn_bias.begin() + batch_iter,
+                                    trg_src_attn_bias.begin() + batch_end);
+      std::vector<int32_t> batch_shape =
+          *(batch_data_shape.begin() + batch_iter);
+      data.batch_data_shape.push_back(batch_shape);
+      data.lod.resize(2);
+      for (int i = 0; i < batch_shape[0] + 1; i++) {
+        data.lod[0].push_back(i);
+        data.lod[1].push_back(i);
+      }
+    }
+    batch_iter += batch_size;
+    return data;
+  }
+  void Load(const std::string &path) {
+    std::ifstream file(path);
+    std::string line;
+    size_t num_lines = 0;
+    while (std::getline(file, line)) {
+      num_lines++;
+      std::vector<std::string> data;
+      split(line, ',', &data);
+      PADDLE_ENFORCE_EQ(data.size(),
+                        static_cast<size_t>(8),
+                        common::errors::InvalidArgument(
+                            "The size of data should be equal to 8. "));
+      // load src_word
+      std::vector<int64_t> src_word_data;
+      split_to_int64(data[0], ' ', &src_word_data);
+      src_word.push_back(std::move(src_word_data));
+      // load src_pos
+      std::vector<int64_t> src_pos_data;
+      split_to_int64(data[1], ' ', &src_pos_data);
+      src_pos.push_back(std::move(src_pos_data));
+      // load src_slf_attn_bias
+      std::vector<float> src_slf_attn_bias_data;
+      split_to_float(data[2], ' ', &src_slf_attn_bias_data);
+      src_slf_attn_bias.push_back(std::move(src_slf_attn_bias_data));
+      // load trg_word
+      std::vector<int64_t> trg_word_data;
+      split_to_int64(data[3], ' ', &trg_word_data);
+      trg_word.push_back(std::move(trg_word_data));
+      // load init_score
+      std::vector<float> init_score_data;
+      split_to_float(data[4], ' ', &init_score_data);
+      init_score.push_back(std::move(init_score_data));
+      // load init_idx
+      std::vector<int64_t> init_idx_data;
+      split_to_int64(data[5], ' ', &init_idx_data);
+      init_idx.push_back(std::move(init_idx_data));
+      // load trg_src_attn_bias
+      std::vector<float> trg_src_attn_bias_data;
+      split_to_float(data[6], ' ', &trg_src_attn_bias_data);
+      trg_src_attn_bias.push_back(std::move(trg_src_attn_bias_data));
+      // load shape for variant data shape
+      std::vector<int> batch_data_shape_data;
+      split_to_int(data[7], ' ', &batch_data_shape_data);
+      batch_data_shape.push_back(std::move(batch_data_shape_data));
+    }
+    num_samples = num_lines;
+  }
+};
+
+void PrepareInputs(std::vector<PaddleTensor> *input_slots,
+                   DataRecord *data,
+                   int batch_size) {
+  auto one_batch = data->NextBatch();
+  batch_size = one_batch.batch_data_shape[0][0];
+  auto n_head = one_batch.batch_data_shape[0][1];
+  auto trg_seq_len = one_batch.batch_data_shape[0][2];  // 1 for inference
+  auto src_seq_len = one_batch.batch_data_shape[0][3];
+
+  PaddleTensor src_word, src_pos, src_slf_attn_bias, trg_word, init_score,
+      init_idx, trg_src_attn_bias;
+
+  src_word.name = "src_word";
+  src_word.shape.assign({batch_size, src_seq_len, 1});
+  src_word.dtype = PaddleDType::INT64;
+  TensorAssignData<int64_t>(&src_word, one_batch.src_word);
+
+  src_pos.name = "src_pos";
+  src_pos.shape.assign({batch_size, src_seq_len, 1});
+  src_pos.dtype = PaddleDType::INT64;
+  TensorAssignData<int64_t>(&src_pos, one_batch.src_pos);
+
+  src_slf_attn_bias.name = "src_slf_attn_bias";
+  src_slf_attn_bias.shape.assign(
+      {batch_size, n_head, src_seq_len, src_seq_len});
+  src_slf_attn_bias.dtype = PaddleDType::FLOAT32;
+  TensorAssignData<float>(&src_slf_attn_bias, one_batch.src_slf_attn_bias);
+
+  trg_word.name = "trg_word";
+  trg_word.shape.assign({batch_size, 1});
+  trg_word.dtype = PaddleDType::INT64;
+  trg_word.lod.assign(one_batch.lod.begin(), one_batch.lod.end());
+  TensorAssignData<int64_t>(&trg_word, one_batch.trg_word);
+
+  init_score.name = "init_score";
+  init_score.shape.assign({batch_size, 1});
+  init_score.dtype = PaddleDType::FLOAT32;
+  init_score.lod.assign(one_batch.lod.begin(), one_batch.lod.end());
+  TensorAssignData<float>(&init_score, one_batch.init_score);
+
+  init_idx.name = "init_idx";
+  init_idx.shape.assign({batch_size});
+  init_idx.dtype = PaddleDType::INT64;
+  TensorAssignData<int64_t>(&init_idx, one_batch.init_idx);
+
+  trg_src_attn_bias.name = "trg_src_attn_bias";
+  trg_src_attn_bias.shape.assign(
+      {batch_size, n_head, trg_seq_len, src_seq_len});
+  trg_src_attn_bias.dtype = PaddleDType::FLOAT32;
+  TensorAssignData<float>(&trg_src_attn_bias, one_batch.trg_src_attn_bias);
+
+  input_slots->assign({src_word,
+                       src_pos,
+                       src_slf_attn_bias,
+                       trg_word,
+                       init_score,
+                       init_idx,
+                       trg_src_attn_bias});
+}
+
+void SetConfig(AnalysisConfig *cfg) {
+  cfg->SetModel(FLAGS_infer_model + "/model", FLAGS_infer_model + "/params");
+  cfg->DisableGpu();
+  cfg->SwitchSpecifyInputNames();
+  cfg->SwitchIrOptim();
+  cfg->SetCpuMathLibraryNumThreads(FLAGS_cpu_num_threads);
+}
+
+void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
+  DataRecord data(FLAGS_infer_data, FLAGS_batch_size);
+  std::vector<PaddleTensor> input_slots;
+  int test_batch_num =
+      FLAGS_test_all_data ? data.num_samples / FLAGS_batch_size : 1;
+  LOG(INFO) << "The number of samples to be test: "
+            << test_batch_num * FLAGS_batch_size;
+  for (int bid = 0; bid < test_batch_num; ++bid) {
+    input_slots.clear();
+    PrepareInputs(&input_slots, &data, FLAGS_batch_size);
+    (*inputs).emplace_back(input_slots);
+  }
+}
+
+}  // namespace transformer_tester
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_vis_tester.cc

@@ -0,0 +1,168 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gtest/gtest.h>
+
+#include <fstream>
+#include <iostream>
+
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+
+struct Record {
+  std::vector<float> data;
+  std::vector<int32_t> shape;
+  Record() : data(), shape() {}
+};
+
+Record ProcessALine(const std::string &line) {
+  VLOG(3) << "process a line";
+  std::vector<std::string> columns;
+  split(line, '\t', &columns);
+  PADDLE_ENFORCE_EQ(columns.size(),
+                    2UL,
+                    common::errors::InvalidArgument(
+                        "data format error, should be <data>\t<shape>"));
+
+  Record record;
+  std::vector<std::string> data_strs;
+  split(columns[0], ' ', &data_strs);
+  for (auto &d : data_strs) {
+    record.data.push_back(std::stof(d));
+  }
+
+  std::vector<std::string> shape_strs;
+  split(columns[1], ' ', &shape_strs);
+  for (auto &s : shape_strs) {
+    record.shape.push_back(std::stoi(s));
+  }
+  VLOG(3) << "data size " << record.data.size();
+  VLOG(3) << "data shape size " << record.shape.size();
+  return record;
+}
+
+void SetConfig(AnalysisConfig *cfg) {
+  cfg->SetModel(FLAGS_infer_model + "/__model__",
+                FLAGS_infer_model + "/__params__");
+  cfg->DisableGpu();
+  cfg->SwitchIrDebug();
+  cfg->SwitchSpecifyInputNames(false);
+}
+
+void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
+  PADDLE_ENFORCE_EQ(FLAGS_test_all_data,
+                    0,
+                    ::common::errors::Fatal("Only have single batch of data."));
+  std::string line;
+  std::ifstream file(FLAGS_infer_data);
+  std::getline(file, line);
+  auto record = ProcessALine(line);
+
+  PaddleTensor input;
+  input.shape = record.shape;
+  input.dtype = PaddleDType::FLOAT32;
+  size_t input_size = record.data.size() * sizeof(float);
+  input.data.Resize(input_size);
+  memcpy(input.data.data(), record.data.data(), input_size);
+  std::vector<PaddleTensor> input_slots;
+  input_slots.assign({input});
+  (*inputs).emplace_back(input_slots);
+}
+
+// Easy for profiling independently.
+//  ocr, mobilenet and se_resnext50
+void profile(bool use_onednn = false) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+  if (use_onednn) {
+    cfg.EnableONEDNN();
+  }
+  // cfg.pass_builder()->TurnOnDebug();
+  std::vector<std::vector<PaddleTensor>> outputs;
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                 input_slots_all,
+                 &outputs,
+                 FLAGS_num_threads);
+  if (FLAGS_num_threads == 1 && !FLAGS_test_all_data) {
+    std::string line;
+    std::ifstream file(FLAGS_refer_result);
+    std::getline(file, line);
+    auto refer = ProcessALine(line);
+    file.close();
+
+    PADDLE_ENFORCE_GT(outputs.size(),
+                      0,
+                      ::common::errors::Fatal(
+                          "The size of output should be greater than 0."));
+    auto &output = outputs.back().front();
+    size_t numel = output.data.length() / PaddleDtypeSize(output.dtype);
+    PADDLE_ENFORCE_EQ(
+        numel,
+        refer.data.size(),
+        common::errors::InvalidArgument(
+            "value of numel is wrong, expected %d but received %d",
+            refer.data.size(),
+            numel));
+    for (size_t i = 0; i < numel; ++i) {
+      EXPECT_NEAR(
+          static_cast<float *>(output.data.data())[i], refer.data[i], 1e-5);
+    }
+  }
+}
+
+TEST(Analyzer_vis, profile) { profile(); }
+
+#ifdef PADDLE_WITH_DNNL
+TEST(Analyzer_vis, profile_onednn) { profile(true /* use_onednn */); }
+#endif
+
+// Compare result of NativeConfig and AnalysisConfig
+void compare(bool use_onednn = false) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+  if (use_onednn) {
+    cfg.EnableONEDNN();
+  }
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareNativeAndAnalysis(
+      reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
+}
+
+TEST(Analyzer_vis, compare) { compare(); }
+#ifdef PADDLE_WITH_DNNL
+TEST(Analyzer_vis, compare_onednn) { compare(true /* use_onednn */); }
+#endif
+
+// Compare Deterministic result
+TEST(Analyzer_vis, compare_determine) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                       input_slots_all);
+}
+
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_vit_ocr_tester.cc

@@ -0,0 +1,100 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <fstream>
+#include <iostream>
+
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+namespace analysis {
+
+struct Record {
+  std::vector<float> data;
+  std::vector<int32_t> shape;
+  Record() : data(), shape() {}
+};
+
+Record ProcessALine(const std::string &line) {
+  std::vector<std::string> columns;
+  split(line, '\t', &columns);
+  PADDLE_ENFORCE_EQ(columns.size(),
+                    2UL,
+                    common::errors::InvalidArgument(
+                        "Data format is invalid, should be <data>\t<shape>"));
+
+  Record record;
+  std::vector<std::string> data_strs;
+  split(columns[0], ' ', &data_strs);
+  for (auto &d : data_strs) {
+    record.data.push_back(std::stof(d));
+  }
+
+  std::vector<std::string> shape_strs;
+  split(columns[1], ' ', &shape_strs);
+  for (auto &s : shape_strs) {
+    record.shape.push_back(std::stoi(s));
+  }
+
+  return record;
+}
+
+void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
+  std::string line;
+  std::ifstream file(FLAGS_infer_data);
+  std::getline(file, line);
+  auto record = ProcessALine(line);
+
+  PaddleTensor input;
+  input.shape = record.shape;
+  input.dtype = PaddleDType::FLOAT32;
+  size_t input_size = record.data.size() * sizeof(float);
+  input.data.Resize(input_size);
+  memcpy(input.data.data(), record.data.data(), input_size);
+  std::vector<PaddleTensor> input_slots;
+  input_slots.assign({input});
+  (*inputs).emplace_back(input_slots);
+}
+
+void SetConfig(AnalysisConfig *cfg, bool use_onednn = false) {
+  cfg->SetModel(FLAGS_infer_model + "/inference.pdmodel",
+                FLAGS_infer_model + "/inference.pdiparams");
+
+  if (use_onednn) {
+    cfg->EnableONEDNN();
+    cfg->SwitchIrOptim();
+  }
+}
+
+// Compare results of NativeConfig and AnalysisConfig
+void compare(bool use_onednn = false) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg, use_onednn);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  SetInput(&input_slots_all);
+  CompareNativeAndAnalysis(
+      reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
+}
+
+TEST(Analyzer_vit_ocr, compare) { compare(); }
+
+#ifdef PADDLE_WITH_DNNL
+TEST(Analyzer_vit_ocr, compare_onednn) { compare(true /* use_onednn */); }
+#endif
+
+}  // namespace analysis
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/analyzer_zerocopy_tensor_tester.cc

@@ -0,0 +1,63 @@
+// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/framework/block_desc.h"
+#include "paddle/fluid/framework/op_desc.h"
+#include "paddle/fluid/framework/program_desc.h"
+#include "paddle/fluid/framework/scope.h"
+#include "paddle/fluid/inference/utils/singleton.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+
+TEST(test_zerocopy_tensor, zerocopy_tensor) {
+  AnalysisConfig config;
+  config.SetModel(FLAGS_infer_model + "/inference.pdmodel",
+                  FLAGS_infer_model + "/inference.pdiparams");
+
+  auto predictor = CreatePaddlePredictor(config);
+  int batch_size = 1;
+  int channels = 3;
+  int height = 224;
+  int width = 224;
+  int nums = batch_size * channels * height * width;
+
+  float* input = new float[nums];
+  for (int i = 0; i < nums; ++i) input[i] = 0;
+  auto input_names = predictor->GetInputNames();
+  PaddlePlace p = PaddlePlace::kCPU;
+  PaddlePlace* place = &p;
+  int size;
+
+  auto input_t = predictor->GetInputTensor(input_names[0]);
+  input_t->Reshape({batch_size, channels, height, width});
+  input_t->copy_from_cpu<float>(input);
+  input_t->data<float>(place, &size);
+  input_t->mutable_data<float>(p);
+
+  predictor->ZeroCopyRun();
+
+  std::vector<float> out_data;
+  auto output_names = predictor->GetOutputNames();
+  auto output_t = predictor->GetOutputTensor(output_names[0]);
+  std::vector<int> output_shape = output_t->shape();
+  int out_num = std::accumulate(
+      output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
+  out_data.resize(out_num);
+  output_t->copy_to_cpu<float>(out_data.data());
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/api_impl_tester.cc

@@ -0,0 +1,343 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include <thread>  // NOLINT
+
+#include "paddle/common/flags.h"
+#include "paddle/fluid/framework/convert_utils.h"
+#include "paddle/fluid/inference/api/api_impl.h"
+#include "test/cpp/inference/test_helper.h"
+
+#ifdef __clang__
+#define ACC_DIFF 4e-3
+#else
+#define ACC_DIFF 2e-3
+#endif
+
+PD_DEFINE_string(word2vec_dirname,
+                 "",
+                 "Directory of the word2vec inference model.");
+PD_DEFINE_string(book_dirname, "", "Directory of the book inference model.");
+
+namespace paddle {
+
+PaddleTensor LodTensorToPaddleTensor(phi::DenseTensor* t) {
+  PaddleTensor pt;
+
+  if (framework::TransToProtoVarType(t->dtype()) ==
+      framework::proto::VarType::INT64) {
+    pt.data.Reset(t->data(), t->numel() * sizeof(int64_t));
+    pt.dtype = PaddleDType::INT64;
+  } else if (framework::TransToProtoVarType(t->dtype()) ==
+             framework::proto::VarType::FP32) {
+    pt.data.Reset(t->data(), t->numel() * sizeof(float));
+    pt.dtype = PaddleDType::FLOAT32;
+  } else if (framework::TransToProtoVarType(t->dtype()) ==
+             framework::proto::VarType::INT32) {
+    pt.data.Reset(t->data(), t->numel() * sizeof(int32_t));
+    pt.dtype = PaddleDType::INT32;
+  } else {
+    PADDLE_THROW(common::errors::Unimplemented(
+        "Unsupported tensor date type. Now only supports INT64, FP32, INT32."));
+  }
+  pt.shape = common::vectorize<int>(t->dims());
+  return pt;
+}
+
+NativeConfig GetConfig() {
+  NativeConfig config;
+  config.model_dir = FLAGS_word2vec_dirname;
+  LOG(INFO) << "dirname  " << config.model_dir;
+  config.fraction_of_gpu_memory = 0.15;
+  config.device = 0;
+  return config;
+}
+
+void MainWord2Vec(const ::paddle::PaddlePlace& place) {
+  NativeConfig config = GetConfig();
+  auto predictor = CreatePaddlePredictor<NativeConfig>(config);
+  config.use_gpu = ::paddle::gpu_place_used(place);
+  config.use_xpu = ::paddle::xpu_place_used(place);
+
+  phi::DenseTensor first_word, second_word, third_word, fourth_word;
+  phi::LegacyLoD lod{{0, 1}};
+  int64_t dict_size = 2073;  // The size of dictionary
+
+  SetupDenseTensor(&first_word, lod, static_cast<int64_t>(0), dict_size - 1);
+  SetupDenseTensor(&second_word, lod, static_cast<int64_t>(0), dict_size - 1);
+  SetupDenseTensor(&third_word, lod, static_cast<int64_t>(0), dict_size - 1);
+  SetupDenseTensor(&fourth_word, lod, static_cast<int64_t>(0), dict_size - 1);
+
+  std::vector<PaddleTensor> paddle_tensor_feeds;
+  paddle_tensor_feeds.push_back(LodTensorToPaddleTensor(&first_word));
+  paddle_tensor_feeds.push_back(LodTensorToPaddleTensor(&second_word));
+  paddle_tensor_feeds.push_back(LodTensorToPaddleTensor(&third_word));
+  paddle_tensor_feeds.push_back(LodTensorToPaddleTensor(&fourth_word));
+
+  std::vector<PaddleTensor> outputs;
+  ASSERT_TRUE(predictor->Run(paddle_tensor_feeds, &outputs));
+  ASSERT_EQ(outputs.size(), 1UL);
+  size_t len = outputs[0].data.length();
+  float* data = static_cast<float*>(outputs[0].data.data());
+  for (size_t j = 0; j < len / sizeof(float); ++j) {
+    ASSERT_LT(data[j], 1.0);
+    ASSERT_GT(data[j], -1.0);
+  }
+
+  std::vector<phi::DenseTensor*> cpu_feeds;
+  cpu_feeds.push_back(&first_word);
+  cpu_feeds.push_back(&second_word);
+  cpu_feeds.push_back(&third_word);
+  cpu_feeds.push_back(&fourth_word);
+
+  framework::FetchType output1;
+  std::vector<::paddle::framework::FetchType*> cpu_fetches1;
+  cpu_fetches1.push_back(&output1);
+
+  TestInference<phi::CPUPlace>(config.model_dir, cpu_feeds, cpu_fetches1);
+
+  auto output1_tensor = PADDLE_GET(phi::DenseTensor, output1);
+  float* lod_data = output1_tensor.data<float>();
+  for (int i = 0; i < output1_tensor.numel(); ++i) {
+    EXPECT_LT(lod_data[i] - data[i], ACC_DIFF);
+    EXPECT_GT(lod_data[i] - data[i], -ACC_DIFF);
+  }
+}
+
+void MainImageClassification(const ::paddle::PaddlePlace& place) {
+  int batch_size = 2;
+  bool repeat = false;
+  NativeConfig config = GetConfig();
+  config.use_gpu = ::paddle::gpu_place_used(place);
+  config.use_xpu = ::paddle::xpu_place_used(place);
+  config.model_dir =
+      FLAGS_book_dirname + "/image_classification_resnet.inference.model";
+
+  const bool is_combined = false;
+  std::vector<std::vector<int64_t>> feed_target_shapes =
+      GetFeedTargetShapes(config.model_dir, is_combined);
+
+  phi::DenseTensor input;
+  // Use normilized image pixels as input data,
+  // which should be in the range [0.0, 1.0].
+  feed_target_shapes[0][0] = batch_size;
+  phi::DDim input_dims = common::make_ddim(feed_target_shapes[0]);
+  SetupTensor<float>(
+      &input, input_dims, static_cast<float>(0), static_cast<float>(1));
+  std::vector<phi::DenseTensor*> cpu_feeds;
+  cpu_feeds.push_back(&input);
+
+  framework::FetchType output1;
+  std::vector<framework::FetchType*> cpu_fetches1;
+  cpu_fetches1.push_back(&output1);
+
+  TestInference<phi::CPUPlace, false, true>(
+      config.model_dir, cpu_feeds, cpu_fetches1, repeat, is_combined);
+
+  auto predictor = CreatePaddlePredictor(config);
+  std::vector<PaddleTensor> paddle_tensor_feeds;
+  paddle_tensor_feeds.push_back(LodTensorToPaddleTensor(&input));
+
+  std::vector<PaddleTensor> outputs;
+  ASSERT_TRUE(predictor->Run(paddle_tensor_feeds, &outputs));
+  ASSERT_EQ(outputs.size(), 1UL);
+  size_t len = outputs[0].data.length();
+  float* data = static_cast<float*>(outputs[0].data.data());
+  float* lod_data = PADDLE_GET(phi::DenseTensor, output1).data<float>();
+  for (size_t j = 0; j < len / sizeof(float); ++j) {
+    EXPECT_NEAR(lod_data[j], data[j], ACC_DIFF);
+  }
+}
+
+void MainThreadsWord2Vec(const ::paddle::PaddlePlace& place) {
+  NativeConfig config = GetConfig();
+  config.use_gpu = ::paddle::gpu_place_used(place);
+  config.use_xpu = ::paddle::xpu_place_used(place);
+  auto main_predictor = CreatePaddlePredictor<NativeConfig>(config);
+
+  // prepare inputs data and reference results
+  constexpr int num_jobs = 3;
+  std::vector<std::vector<phi::DenseTensor>> jobs(num_jobs);
+  std::vector<std::vector<PaddleTensor>> paddle_tensor_feeds(num_jobs);
+  std::vector<framework::FetchType> refs(num_jobs);
+  for (size_t i = 0; i < jobs.size(); ++i) {
+    // each job has 4 words
+    jobs[i].resize(4);
+    for (size_t j = 0; j < 4; ++j) {
+      phi::LegacyLoD lod{{0, 1}};
+      int64_t dict_size = 2073;  // The size of dictionary
+      SetupDenseTensor(
+          &jobs[i][j], lod, static_cast<int64_t>(0), dict_size - 1);
+      paddle_tensor_feeds[i].push_back(LodTensorToPaddleTensor(&jobs[i][j]));
+    }
+
+    // get reference result of each job
+    std::vector<phi::DenseTensor*> ref_feeds;
+    std::vector<::paddle::framework::FetchType*> ref_fetches(1, &refs[i]);
+    for (auto& word : jobs[i]) {
+      ref_feeds.push_back(&word);
+    }
+    TestInference<phi::CPUPlace>(config.model_dir, ref_feeds, ref_fetches);
+  }
+
+  // create threads and each thread run 1 job
+  std::vector<std::thread> threads;
+  for (int tid = 0; tid < num_jobs; ++tid) {
+    threads.emplace_back([&, tid]() {
+      auto predictor = CreatePaddlePredictor(config);
+      auto& local_inputs = paddle_tensor_feeds[tid];
+      std::vector<PaddleTensor> local_outputs;
+      ASSERT_TRUE(predictor->Run(local_inputs, &local_outputs));
+
+      // check outputs range
+      ASSERT_EQ(local_outputs.size(), 1UL);
+      const size_t len = local_outputs[0].data.length();
+      float* data = static_cast<float*>(local_outputs[0].data.data());
+      for (size_t j = 0; j < len / sizeof(float); ++j) {
+        ASSERT_LT(data[j], 1.0);
+        ASSERT_GT(data[j], -1.0);
+      }
+
+      // check outputs correctness
+      auto ref_tensor = PADDLE_GET(phi::DenseTensor, refs[tid]);
+      float* ref_data = ref_tensor.data<float>();
+      EXPECT_EQ(ref_tensor.numel(), static_cast<int64_t>(len / sizeof(float)));
+      for (int i = 0; i < ref_tensor.numel(); ++i) {
+        EXPECT_NEAR(ref_data[i], data[i], 2e-3);
+      }
+    });
+  }
+  for (int i = 0; i < num_jobs; ++i) {
+    threads[i].join();
+  }
+}
+
+void MainThreadsImageClassification(const ::paddle::PaddlePlace& place) {
+  constexpr int num_jobs = 4;  // each job run 1 batch
+  constexpr int batch_size = 1;
+  NativeConfig config = GetConfig();
+  config.use_gpu = ::paddle::gpu_place_used(place);
+  config.use_xpu = ::paddle::xpu_place_used(place);
+  config.model_dir =
+      FLAGS_book_dirname + "/image_classification_resnet.inference.model";
+
+  auto main_predictor = CreatePaddlePredictor<NativeConfig>(config);
+  std::vector<phi::DenseTensor> jobs(num_jobs);
+  std::vector<std::vector<PaddleTensor>> paddle_tensor_feeds(num_jobs);
+  std::vector<framework::FetchType> refs(num_jobs);
+  for (size_t i = 0; i < jobs.size(); ++i) {
+    // prepare inputs
+    std::vector<std::vector<int64_t>> feed_target_shapes =
+        GetFeedTargetShapes(config.model_dir, /*is_combined*/ false);
+    feed_target_shapes[0][0] = batch_size;
+    phi::DDim input_dims = common::make_ddim(feed_target_shapes[0]);
+    SetupTensor<float>(&jobs[i], input_dims, 0.f, 1.f);
+    paddle_tensor_feeds[i].push_back(LodTensorToPaddleTensor(&jobs[i]));
+
+    // get reference result of each job
+    std::vector<phi::DenseTensor*> ref_feeds(1, &jobs[i]);
+    std::vector<framework::FetchType*> ref_fetches(1, &refs[i]);
+    TestInference<phi::CPUPlace>(config.model_dir, ref_feeds, ref_fetches);
+  }
+
+  // create threads and each thread run 1 job
+  std::vector<std::thread> threads;
+  for (int tid = 0; tid < num_jobs; ++tid) {
+    threads.emplace_back([&, tid]() {
+      auto predictor = CreatePaddlePredictor(config);
+      auto& local_inputs = paddle_tensor_feeds[tid];
+      std::vector<PaddleTensor> local_outputs;
+      ASSERT_TRUE(predictor->Run(local_inputs, &local_outputs));
+
+      // check outputs correctness
+      ASSERT_EQ(local_outputs.size(), 1UL);
+      const size_t len = local_outputs[0].data.length();
+      float* data = static_cast<float*>(local_outputs[0].data.data());
+      auto ref_tensor = PADDLE_GET(phi::DenseTensor, refs[tid]);
+      float* ref_data = ref_tensor.data<float>();
+      EXPECT_EQ((size_t)ref_tensor.numel(), len / sizeof(float));
+      for (int i = 0; i < ref_tensor.numel(); ++i) {
+        EXPECT_NEAR(ref_data[i], data[i], ACC_DIFF);
+      }
+    });
+  }
+  for (int i = 0; i < num_jobs; ++i) {
+    threads[i].join();
+  }
+}
+
+TEST(inference_api_native, word2vec_cpu) {
+  MainWord2Vec(::paddle::PaddlePlace::kCPU);
+}
+TEST(inference_api_native, word2vec_cpu_threads) {
+  MainThreadsWord2Vec(::paddle::PaddlePlace::kCPU);
+}
+TEST(inference_api_native, image_classification_cpu) {
+  MainImageClassification(::paddle::PaddlePlace::kCPU);
+}
+TEST(inference_api_native, image_classification_cpu_threads) {
+  MainThreadsImageClassification(::paddle::PaddlePlace::kCPU);
+}
+
+#ifdef PADDLE_WITH_XPU
+TEST(inference_api_native, word2vec_xpu) {
+  MainWord2Vec(::paddle::PaddlePlace::kXPU);
+}
+TEST(inference_api_native, image_classification_xpu) {
+  MainImageClassification(::paddle::PaddlePlace::kXPU);
+}
+#endif
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+TEST(inference_api_native, word2vec_gpu) {
+  MainWord2Vec(::paddle::PaddlePlace::kGPU);
+}
+// Turn off temporarily for the unstable result.
+// TEST(inference_api_native, word2vec_gpu_threads) {
+//   MainThreadsWord2Vec(::paddle::PaddlePlace::kGPU);
+// }
+TEST(inference_api_native, image_classification_gpu) {
+  MainImageClassification(::paddle::PaddlePlace::kGPU);
+}
+// Turn off temporarily for the unstable result.
+// TEST(inference_api_native, image_classification_gpu_threads) {
+//   MainThreadsImageClassification(::paddle::PaddlePlace::kGPU);
+// }
+#endif
+
+#ifdef PADDLE_WITH_DNNL
+TEST(inference_api_native, image_classification_cpu_onednn) {
+  FLAGS_use_onednn = true;
+  MainImageClassification(::paddle::PaddlePlace::kCPU);
+}
+
+TEST(inference_api_native, word2vec_cpu_onednn) {
+  FLAGS_use_onednn = true;
+  MainWord2Vec(::paddle::PaddlePlace::kCPU);
+}
+#endif
+
+TEST(PassBuilder, Delete) {
+  AnalysisConfig config;
+  config.DisableGpu();
+  config.pass_builder()->DeletePass("attention_lstm_fuse_pass");
+  const auto& passes = config.pass_builder()->AllPasses();
+  auto it = std::find(passes.begin(), passes.end(), "attention_lstm_fuse_pass");
+  ASSERT_EQ(it, passes.end());
+}
+
+}  // namespace paddle

test/cpp/inference/api/api_tester.cc

@@ -0,0 +1,112 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include <exception>
+#include <string>
+
+#include "paddle/fluid/inference/api/paddle_analysis_config.h"
+#include "paddle/fluid/inference/api/paddle_api.h"
+#include "paddle/fluid/platform/enforce.h"
+
+namespace paddle {
+
+/*
+ * Do not use this, just a demo indicating how to customize a config for a
+ * specific predictor.
+ */
+struct DemoConfig : public PaddlePredictor::Config {
+  float other_config;
+  DemoConfig() : other_config(0) {}
+};
+
+/*
+ * Do not use this, just a demo indicating how to customize a Predictor.
+ */
+class DemoPredictor : public PaddlePredictor {
+ public:
+  explicit DemoPredictor(const DemoConfig &config) {
+    LOG(INFO) << "I get other_config " << config.other_config;
+  }
+  bool Run(const std::vector<PaddleTensor> &inputs,
+           std::vector<PaddleTensor> *output_data,
+           int batch_size = 0) override {
+    LOG(INFO) << "Run";
+    return false;
+  }
+
+  std::unique_ptr<PaddlePredictor> Clone(void *stream = nullptr) override {
+    return nullptr;
+  }
+
+  ~DemoPredictor() override = default;
+};
+
+template <>
+std::unique_ptr<PaddlePredictor> CreatePaddlePredictor<DemoConfig>(
+    const DemoConfig &config) {
+  std::unique_ptr<PaddlePredictor> x(new DemoPredictor(config));
+  return x;
+}
+
+TEST(paddle_inference_api, demo) {
+  DemoConfig config;
+  config.other_config = 1.7;
+  auto predictor = CreatePaddlePredictor(config);
+  std::vector<PaddleTensor> outputs;
+  predictor->Run({}, &outputs);
+  predictor->TryShrinkMemory();
+}
+
+TEST(paddle_inference_api, get_version) {
+  LOG(INFO) << "paddle version:\n" << get_version();
+  auto version = get_version();
+  ASSERT_FALSE(version.empty());
+}
+
+TEST(paddle_inference_api, UpdateDllFlag) {
+  UpdateDllFlag("paddle_num_threads", "10");
+  try {
+    UpdateDllFlag("paddle_num_threads2", "10");
+  } catch (std::exception &e) {
+    LOG(INFO) << e.what();
+  }
+}
+
+TEST(paddle_inference_api, AnalysisConfigCopyCtor) {
+  AnalysisConfig cfg1;
+  cfg1.EnableUseGpu(10);
+#ifdef PADDLE_WITH_TENSORRT
+  cfg1.EnableTensorRtEngine();
+#endif
+  std::string delete_pass("skip_layernorm_fuse_pass");
+  cfg1.pass_builder()->DeletePass(delete_pass);
+  AnalysisConfig cfg2(cfg1);
+
+  auto passes = cfg2.pass_builder()->AllPasses();
+  for (auto const &ps : passes) {
+    PADDLE_ENFORCE_NE(ps,
+                      delete_pass,
+                      common::errors::InvalidArgument(
+                          "Required ps shouldn't be equal to delete_pass. "));
+  }
+}
+
+#ifdef PADDLE_WITH_CRYPTO
+TEST(paddle_inference_api, crypto) { paddle::MakeCipher(""); }
+#endif
+
+}  // namespace paddle

test/cpp/inference/api/config_printer.h

@@ -0,0 +1,86 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <ostream>
+#include <string>
+
+#include "paddle/fluid/inference/api/paddle_inference_api.h"
+
+namespace paddle {
+namespace inference {
+
+thread_local int num_spaces = 0;
+
+static std::string GenSpaces(int num_spaces) {
+  std::ostringstream os;
+  for (int i = 0; i < num_spaces; ++i) {
+    os << "  ";
+  }
+  return os.str();
+}
+
+std::ostream &operator<<(std::ostream &os,
+                         const PaddlePredictor::Config &config) {
+  os << GenSpaces(num_spaces) << "PaddlePredictor::Config {\n";
+  num_spaces++;
+  os << GenSpaces(num_spaces) << "model_dir: " << config.model_dir << "\n";
+  num_spaces--;
+  os << GenSpaces(num_spaces) << "}\n";
+  return os;
+}
+
+std::ostream &operator<<(std::ostream &os, const NativeConfig &config) {
+  os << GenSpaces(num_spaces) << "NativeConfig {\n";
+  num_spaces++;
+  os << *reinterpret_cast<const PaddlePredictor::Config *>(&config);
+  os << GenSpaces(num_spaces) << "use_gpu: " << config.use_gpu << "\n";
+  os << GenSpaces(num_spaces) << "device: " << config.device << "\n";
+  os << GenSpaces(num_spaces)
+     << "fraction_of_gpu_memory: " << config.fraction_of_gpu_memory << "\n";
+  os << GenSpaces(num_spaces)
+     << "specify_input_name: " << config.specify_input_name << "\n";
+  num_spaces--;
+  os << GenSpaces(num_spaces) << "}\n";
+  return os;
+}
+
+std::ostream &operator<<(std::ostream &os, const AnalysisConfig &config) {
+  os << GenSpaces(num_spaces) << "AnalysisConfig {\n";
+  num_spaces++;
+  os << config.ToNativeConfig();
+  if (!config.model_from_memory()) {
+    os << GenSpaces(num_spaces) << "prog_file: " << config.prog_file() << "\n";
+    os << GenSpaces(num_spaces) << "param_file: " << config.params_file()
+       << "\n";
+  } else {
+    os << GenSpaces(num_spaces)
+       << "prog_file and param_file: load from memory \n";
+  }
+  os << GenSpaces(num_spaces) << "enable_ir_optim: " << config.ir_optim()
+     << "\n";
+  os << GenSpaces(num_spaces)
+     << "cpu_num_threads: " << config.cpu_math_library_num_threads() << "\n";
+  os << GenSpaces(num_spaces)
+     << "use_tensorrt: " << config.tensorrt_engine_enabled() << "\n";
+  os << GenSpaces(num_spaces) << "use_onednn: " << config.onednn_enabled()
+     << "\n";
+  num_spaces--;
+  os << GenSpaces(num_spaces) << "}\n";
+  return os;
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/full_ILSVRC2012_val_preprocess.py

@@ -0,0 +1,275 @@
+#   copyright (c) 2019 paddlepaddle authors. all rights reserved.
+# licensed under the apache license, version 2.0 (the "license");
+# you may not use this file except in compliance with the license.
+# you may obtain a copy of the license at
+#
+#     http://www.apache.org/licenses/license-2.0
+#
+# unless required by applicable law or agreed to in writing, software
+# distributed under the license is distributed on an "as is" basis,
+# without warranties or conditions of any kind, either express or implied.
+# see the license for the specific language governing permissions and
+# limitations under the license.
+import argparse
+import io
+import os
+import shutil
+import sys
+import tarfile
+
+import numpy as np
+from PIL import Image
+
+from paddle.dataset.common import download
+
+np.random.seed(0)
+
+DATA_DIM = 224
+SIZE_FLOAT32 = 4
+SIZE_INT64 = 8
+FULL_SIZE_BYTES = 30106000008
+FULL_IMAGES = 50000
+FOLDER_NAME = "ILSVRC2012/"
+VALLIST_TAR_NAME = "ILSVRC2012/val_list.txt"
+
+img_mean = np.array([0.485, 0.456, 0.406]).reshape((3, 1, 1))
+img_std = np.array([0.229, 0.224, 0.225]).reshape((3, 1, 1))
+
+
+def resize_short(img, target_size):
+    percent = float(target_size) / min(img.size[0], img.size[1])
+    resized_width = int(round(img.size[0] * percent))
+    resized_height = int(round(img.size[1] * percent))
+    img = img.resize((resized_width, resized_height), Image.LANCZOS)
+    return img
+
+
+def crop_image(img, target_size, center):
+    width, height = img.size
+    size = target_size
+    if center:
+        w_start = (width - size) // 2
+        h_start = (height - size) // 2
+    else:
+        w_start = np.random.randint(0, width - size + 1)
+        h_start = np.random.randint(0, height - size + 1)
+    w_end = w_start + size
+    h_end = h_start + size
+    img = img.crop((w_start, h_start, w_end, h_end))
+    return img
+
+
+def process_image(img):
+    img = resize_short(img, target_size=256)
+    img = crop_image(img, target_size=DATA_DIM, center=True)
+    if img.mode != 'RGB':
+        img = img.convert('RGB')
+    img = np.array(img).astype('float32').transpose((2, 0, 1)) / 255
+    img -= img_mean
+    img /= img_std
+    return img
+
+
+def download_concat(cache_folder, zip_path):
+    data_urls = []
+    data_md5s = []
+    data_urls.append(
+        'https://paddle-inference-dist.bj.bcebos.com/int8/ILSVRC2012_img_val.tar.gz.partaa'
+    )
+    data_md5s.append('60f6525b0e1d127f345641d75d41f0a8')
+    data_urls.append(
+        'https://paddle-inference-dist.bj.bcebos.com/int8/ILSVRC2012_img_val.tar.gz.partab'
+    )
+    data_md5s.append('1e9f15f64e015e58d6f9ec3210ed18b5')
+    file_names = []
+    print("Downloading full ImageNet Validation dataset ...")
+    for i in range(0, len(data_urls)):
+        download(data_urls[i], cache_folder, data_md5s[i])
+        file_name = os.path.join(cache_folder, data_urls[i].split('/')[-1])
+        file_names.append(file_name)
+        print(f"Downloaded part {file_name}\n")
+    with open(zip_path, "wb") as outfile:
+        for fname in file_names:
+            shutil.copyfileobj(open(fname, 'rb'), outfile)
+
+
+def print_processbar(done_percentage):
+    done_filled = done_percentage * '='
+    empty_filled = (100 - done_percentage) * ' '
+    sys.stdout.write(f"\r[{done_filled}{empty_filled}]{done_percentage}%")
+    sys.stdout.flush()
+
+
+def convert_Imagenet_tar2bin(tar_file, output_file):
+    print('Converting 50000 images to binary file ...\n')
+    tar = tarfile.open(name=tar_file, mode='r:gz')
+
+    print_processbar(0)
+
+    dataset = {}
+    for tarInfo in tar:
+        if tarInfo.isfile() and tarInfo.name != VALLIST_TAR_NAME:
+            dataset[tarInfo.name] = tar.extractfile(tarInfo).read()
+    with open(output_file, "w+b") as ofs:
+        ofs.seek(0)
+        num = np.array(int(FULL_IMAGES)).astype('int64')
+        ofs.write(num.tobytes())
+
+        per_percentage = FULL_IMAGES // 100
+
+        val_info = tar.getmember(VALLIST_TAR_NAME)
+        val_list = tar.extractfile(val_info).read().decode("utf-8")
+        lines = val_list.splitlines()
+        idx = 0
+        for imagedata in dataset.values():
+            img = Image.open(io.BytesIO(imagedata))
+            img = process_image(img)
+            np_img = np.array(img)
+            ofs.write(np_img.astype('float32').tobytes())
+            if idx % per_percentage == 0:
+                print_processbar(idx // per_percentage)
+            idx = idx + 1
+
+        val_dict = {}
+        for line_idx, line in enumerate(lines):
+            if line_idx == FULL_IMAGES:
+                break
+            name, label = line.split()
+            val_dict[name] = label
+
+        for img_name in dataset.keys():
+            remove_len = len(FOLDER_NAME)
+            img_name_prim = img_name[remove_len:]
+            label = val_dict[img_name_prim]
+            label_int = (int)(label)
+            np_label = np.array(label_int)
+            ofs.write(np_label.astype('int64').tobytes())
+        print_processbar(100)
+    tar.close()
+    print("Conversion finished.")
+
+
+def run_convert():
+    print('Start to download and convert 50000 images to binary file...')
+    cache_folder = os.path.expanduser('~/.cache/paddle/dataset/int8/download')
+    zip_path = os.path.join(cache_folder, 'full_imagenet_val.tar.gz.partaa')
+    output_file = os.path.join(cache_folder, 'int8_full_val.bin')
+    retry = 0
+    try_limit = 3
+
+    while not (
+        os.path.exists(output_file)
+        and os.path.getsize(output_file) == FULL_SIZE_BYTES
+    ):
+        if os.path.exists(output_file):
+            sys.stderr.write(
+                f"\n\nThe existing binary file[{output_file}] is broken. Start to generate new one...\n\n"
+            )
+            os.remove(output_file)
+        if retry < try_limit:
+            retry = retry + 1
+        else:
+            raise RuntimeError(
+                f"Can not convert the dataset to binary file with try limit {try_limit}"
+            )
+        download_concat(cache_folder, zip_path)
+        convert_Imagenet_tar2bin(zip_path, output_file)
+    print(f"\nSuccess! The binary file can be found at {output_file}")
+
+
+def convert_Imagenet_local2bin(args):
+    data_dir = args.data_dir
+    label_list_path = os.path.join(args.data_dir, args.label_list)
+    bin_file_path = os.path.join(args.data_dir, args.output_file)
+    assert data_dir, 'Once set --local, user need to provide the --data_dir'
+    with open(label_list_path) as flist:
+        lines = [line.strip() for line in flist]
+        num_images = len(lines)
+
+        with open(bin_file_path, "w+b") as of:
+            of.seek(0)
+            num = np.array(int(num_images)).astype('int64')
+            of.write(num.tobytes())
+            for idx, line in enumerate(lines):
+                img_path, label = line.split()
+                img_path = os.path.join(data_dir, img_path)
+                if not os.path.exists(img_path):
+                    continue
+
+                # save image(float32) to file
+                img = Image.open(img_path)
+                img = process_image(img)
+                np_img = np.array(img)
+                of.seek(
+                    SIZE_INT64 + SIZE_FLOAT32 * DATA_DIM * DATA_DIM * 3 * idx
+                )
+                of.write(np_img.astype('float32').tobytes())
+
+                # save label(int64_t) to file
+                label_int = (int)(label)
+                np_label = np.array(label_int)
+                of.seek(
+                    SIZE_INT64
+                    + SIZE_FLOAT32 * DATA_DIM * DATA_DIM * 3 * num_images
+                    + idx * SIZE_INT64
+                )
+                of.write(np_label.astype('int64').tobytes())
+
+        # The bin file should contain
+        # number of images + all images data + all corresponding labels
+        # so the file target_size should be as follows
+        target_size = (
+            SIZE_INT64
+            + num_images * 3 * args.data_dim * args.data_dim * SIZE_FLOAT32
+            + num_images * SIZE_INT64
+        )
+        if os.path.getsize(bin_file_path) == target_size:
+            print(
+                f"Success! The user data output binary file can be found at: {bin_file_path}"
+            )
+        else:
+            print("Conversion failed!")
+
+
+def main_preprocess_Imagenet(args):
+    parser = argparse.ArgumentParser(
+        description="Convert the full Imagenet val set or local data to binary file.",
+        usage=None,
+        add_help=True,
+    )
+    parser.add_argument(
+        '--local',
+        action="store_true",
+        help="If used, user need to set --data_dir and then convert file",
+    )
+    parser.add_argument(
+        "--data_dir", default="", type=str, help="Dataset root directory"
+    )
+    parser.add_argument(
+        "--label_list",
+        type=str,
+        default="val_list.txt",
+        help="List of object labels with same sequence as denoted in the annotation file",
+    )
+    parser.add_argument(
+        "--output_file",
+        type=str,
+        default="imagenet_small.bin",
+        help="File path of the output binary file",
+    )
+    parser.add_argument(
+        "--data_dim",
+        type=int,
+        default=DATA_DIM,
+        help="Image preprocess with data_dim width and height",
+    )
+
+    args = parser.parse_args()
+    if args.local:
+        convert_Imagenet_local2bin(args)
+    else:
+        run_convert()
+
+
+if __name__ == '__main__':
+    main_preprocess_Imagenet(sys.argv)

test/cpp/inference/api/full_pascalvoc_test_preprocess.py

@@ -0,0 +1,373 @@
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import hashlib
+import os
+import sys
+import tarfile
+import xml.etree.ElementTree
+from io import StringIO
+
+import numpy as np
+from PIL import Image
+
+from paddle.dataset.common import download
+
+DATA_URL = (
+    "http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar"
+)
+DATA_DIR = os.path.expanduser("~/.cache/paddle/dataset/pascalvoc/")
+TAR_FILE = "VOCtest_06-Nov-2007.tar"
+TAR_PATH = os.path.join(DATA_DIR, TAR_FILE)
+SIZE_FLOAT32 = 4
+SIZE_INT64 = 8
+RESIZE_H = 300
+RESIZE_W = 300
+MEAN_VALUE = [127.5, 127.5, 127.5]
+AP_VERSION = '11point'
+DATA_OUT = 'pascalvoc_full.bin'
+DATA_OUT_PATH = os.path.join(DATA_DIR, DATA_OUT)
+BIN_TARGETHASH = "f6546cadc42f5ff13178b84ed29b740b"
+TAR_TARGETHASH = "b6e924de25625d8de591ea690078ad9f"
+TEST_LIST_KEY = "VOCdevkit/VOC2007/ImageSets/Main/test.txt"
+BIN_FULLSIZE = 5348678856
+
+
+def preprocess(img):
+    img_width, img_height = img.size
+    img = img.resize((RESIZE_W, RESIZE_H), Image.LANCZOS)
+    img = np.array(img)
+    # HWC to CHW
+    if len(img.shape) == 3:
+        img = np.swapaxes(img, 1, 2)
+        img = np.swapaxes(img, 1, 0)
+    # RBG to BGR
+    img = img[[2, 1, 0], :, :]
+    img = img.astype('float32')
+    img_mean = np.array(MEAN_VALUE)[:, np.newaxis, np.newaxis].astype('float32')
+    img -= img_mean
+    img = img * 0.007843
+    return img
+
+
+def convert_pascalvoc_local2bin(args):
+    data_dir = os.path.expanduser(args.data_dir)
+    label_fpath = os.path.join(data_dir, args.label_file)
+    assert data_dir, 'Once set --local, user need to provide the --data_dir'
+    flabel = open(label_fpath)
+    label_list = [line.strip() for line in flabel]
+
+    img_annotation_list_path = os.path.join(data_dir, args.img_annotation_list)
+    flist = open(img_annotation_list_path)
+    lines = [line.strip() for line in flist]
+
+    output_file_path = os.path.join(data_dir, args.output_file)
+    f1 = open(output_file_path, "w+b")
+    f1.seek(0)
+    image_nums = len(lines)
+    f1.write(np.array(image_nums).astype('int64').tobytes())
+
+    boxes = []
+    lbls = []
+    difficulties = []
+    object_nums = []
+
+    for line in lines:
+        image_path, label_path = line.split()
+        image_path = os.path.join(data_dir, image_path)
+        label_path = os.path.join(data_dir, label_path)
+
+        im = Image.open(image_path)
+        if im.mode == 'L':
+            im = im.convert('RGB')
+        im_width, im_height = im.size
+
+        im = preprocess(im)
+        np_im = np.array(im)
+        f1.write(np_im.astype('float32').tobytes())
+
+        # layout: label | xmin | ymin | xmax | ymax | difficult
+        bbox_labels = []
+        root = xml.etree.ElementTree.parse(label_path).getroot()
+
+        objects = root.findall('object')
+        objects_size = len(objects)
+        object_nums.append(objects_size)
+
+        for object in objects:
+            bbox_sample = []
+            # start from 1
+            bbox_sample.append(
+                float(label_list.index(object.find('name').text))
+            )
+            bbox = object.find('bndbox')
+            difficult = float(object.find('difficult').text)
+            bbox_sample.append(float(bbox.find('xmin').text) / im_width)
+            bbox_sample.append(float(bbox.find('ymin').text) / im_height)
+            bbox_sample.append(float(bbox.find('xmax').text) / im_width)
+            bbox_sample.append(float(bbox.find('ymax').text) / im_height)
+            bbox_sample.append(difficult)
+            bbox_labels.append(bbox_sample)
+
+        bbox_labels = np.array(bbox_labels)
+        if len(bbox_labels) == 0:
+            continue
+
+        lbls.extend(bbox_labels[:, 0])
+        boxes.extend(bbox_labels[:, 1:5])
+        difficulties.extend(bbox_labels[:, -1])
+
+    f1.write(np.array(object_nums).astype('uint64').tobytes())
+    f1.write(np.array(lbls).astype('int64').tobytes())
+    f1.write(np.array(boxes).astype('float32').tobytes())
+    f1.write(np.array(difficulties).astype('int64').tobytes())
+    f1.close()
+
+    object_nums_sum = sum(object_nums)
+    # The data should be contains
+    # number of images + all images data + an array that represent object numbers of each image
+    # + labels of all objects in images + bboxes of all objects + difficulties of all objects
+    # so the target size should be as follows:
+    target_size = (
+        SIZE_INT64
+        + image_nums * 3 * args.resize_h * args.resize_h * SIZE_FLOAT32
+        + image_nums * SIZE_INT64
+        + object_nums_sum * (SIZE_INT64 + 4 * SIZE_FLOAT32 + SIZE_INT64)
+    )
+    if os.path.getsize(output_file_path) == target_size:
+        print(
+            "Success! \nThe local data output binary file can be found at: ",
+            output_file_path,
+        )
+    else:
+        print("Conversion failed!")
+
+
+def print_processbar(done_percentage):
+    done_filled = done_percentage * '='
+    empty_filled = (100 - done_percentage) * ' '
+    sys.stdout.write(f"\r[{done_filled}{empty_filled}]{done_percentage}%")
+    sys.stdout.flush()
+
+
+def convert_pascalvoc_tar2bin(tar_path, data_out_path):
+    print("Start converting ...\n")
+    images = {}
+    gt_labels = {}
+    boxes = []
+    lbls = []
+    difficulties = []
+    object_nums = []
+
+    # map label to number (index)
+    label_list = [
+        "background",
+        "aeroplane",
+        "bicycle",
+        "bird",
+        "boat",
+        "bottle",
+        "bus",
+        "car",
+        "cat",
+        "chair",
+        "cow",
+        "diningtable",
+        "dog",
+        "horse",
+        "motorbike",
+        "person",
+        "pottedplant",
+        "sheep",
+        "sofa",
+        "train",
+        "tvmonitor",
+    ]
+    print_processbar(0)
+    # read from tar file and write to bin
+    tar = tarfile.open(tar_path, "r")
+    f_test = tar.extractfile(TEST_LIST_KEY).read()
+    lines = f_test.split('\n')
+    del lines[-1]
+    image_nums = len(lines)
+    per_percentage = image_nums / 100
+
+    f1 = open(data_out_path, "w+b")
+    f1.seek(0)
+    f1.write(np.array(image_nums).astype('int64').tobytes())
+    for tarInfo in tar:
+        if tarInfo.isfile():
+            tmp_filename = tarInfo.name
+            name_arr = tmp_filename.split('/')
+            name_prefix = name_arr[-1].split('.')[0]
+            if name_arr[-2] == 'JPEGImages' and name_prefix in lines:
+                images[name_prefix] = tar.extractfile(tarInfo).read()
+            if name_arr[-2] == 'Annotations' and name_prefix in lines:
+                gt_labels[name_prefix] = tar.extractfile(tarInfo).read()
+
+    for line_idx, name_prefix in enumerate(lines):
+        im = Image.open(StringIO(images[name_prefix]))
+        if im.mode == 'L':
+            im = im.convert('RGB')
+        im_width, im_height = im.size
+
+        im = preprocess(im)
+        np_im = np.array(im)
+        f1.write(np_im.astype('float32').tobytes())
+
+        # layout: label | xmin | ymin | xmax | ymax | difficult
+        bbox_labels = []
+        root = xml.etree.ElementTree.fromstring(gt_labels[name_prefix])
+
+        objects = root.findall('object')
+        objects_size = len(objects)
+        object_nums.append(objects_size)
+
+        for object in objects:
+            bbox_sample = []
+            bbox_sample.append(
+                float(label_list.index(object.find('name').text))
+            )
+            bbox = object.find('bndbox')
+            difficult = float(object.find('difficult').text)
+            bbox_sample.append(float(bbox.find('xmin').text) / im_width)
+            bbox_sample.append(float(bbox.find('ymin').text) / im_height)
+            bbox_sample.append(float(bbox.find('xmax').text) / im_width)
+            bbox_sample.append(float(bbox.find('ymax').text) / im_height)
+            bbox_sample.append(difficult)
+            bbox_labels.append(bbox_sample)
+
+        bbox_labels = np.array(bbox_labels)
+        if len(bbox_labels) == 0:
+            continue
+        lbls.extend(bbox_labels[:, 0])
+        boxes.extend(bbox_labels[:, 1:5])
+        difficulties.extend(bbox_labels[:, -1])
+
+        if line_idx % per_percentage:
+            print_processbar(line_idx / per_percentage)
+
+    # The data should be stored in binary in following sequence:
+    # number of images->all images data->an array that represent object numbers in each image
+    # ->labels of all objects in images->bboxes of all objects->difficulties of all objects
+    f1.write(np.array(object_nums).astype('uint64').tobytes())
+    f1.write(np.array(lbls).astype('int64').tobytes())
+    f1.write(np.array(boxes).astype('float32').tobytes())
+    f1.write(np.array(difficulties).astype('int64').tobytes())
+    f1.close()
+    print_processbar(100)
+    print("Conversion finished!\n")
+
+
+def download_pascalvoc(data_url, data_dir, tar_targethash, tar_path):
+    print("Downloading pascalvcoc test set...")
+    download(data_url, data_dir, tar_targethash)
+    if not os.path.exists(tar_path):
+        print(f"Failed in downloading pascalvoc test set. URL {data_url}\n")
+    else:
+        tmp_hash = hashlib.md5(open(tar_path, 'rb').read()).hexdigest()
+        if tmp_hash != tar_targethash:
+            print("Downloaded test set is broken, removing ...\n")
+        else:
+            print(f"Downloaded successfully. Path: {tar_path}\n")
+
+
+def run_convert():
+    try_limit = 2
+    retry = 0
+    while not (
+        os.path.exists(DATA_OUT_PATH)
+        and os.path.getsize(DATA_OUT_PATH) == BIN_FULLSIZE
+        and BIN_TARGETHASH
+        == hashlib.md5(open(DATA_OUT_PATH, 'rb').read()).hexdigest()
+    ):
+        if os.path.exists(DATA_OUT_PATH):
+            sys.stderr.write(
+                "The existing binary file is broken. It is being removed...\n"
+            )
+            os.remove(DATA_OUT_PATH)
+        if retry < try_limit:
+            retry = retry + 1
+        else:
+            download_pascalvoc(DATA_URL, DATA_DIR, TAR_TARGETHASH, TAR_PATH)
+            convert_pascalvoc_tar2bin(TAR_PATH, DATA_OUT_PATH)
+    print(f"Success!\nThe binary file can be found at {DATA_OUT_PATH}\n")
+
+
+def main_pascalvoc_preprocess(args):
+    parser = argparse.ArgumentParser(
+        description="Convert the full pascalvoc val set or local data to binary file.",
+        usage=None,
+        add_help=True,
+    )
+    parser.add_argument(
+        '--local',
+        action="store_true",
+        help="If used, user need to set --data_dir and then convert file",
+    )
+    parser.add_argument(
+        "--data_dir", default="", type=str, help="Dataset root directory"
+    )
+    parser.add_argument(
+        "--img_annotation_list",
+        type=str,
+        default="test_100.txt",
+        help="A file containing the image file path and corresponding annotation file path",
+    )
+    parser.add_argument(
+        "--label_file",
+        type=str,
+        default="label_list",
+        help="List of object labels with same sequence as denoted in the annotation file",
+    )
+    parser.add_argument(
+        "--output_file",
+        type=str,
+        default="pascalvoc_small.bin",
+        help="File path of the output binary file",
+    )
+    parser.add_argument(
+        "--resize_h",
+        type=int,
+        default=RESIZE_H,
+        help="Image preprocess with resize_h",
+    )
+    parser.add_argument(
+        "--resize_w",
+        type=int,
+        default=RESIZE_W,
+        help="Image prerocess with resize_w",
+    )
+    parser.add_argument(
+        "--mean_value",
+        type=str,
+        default=MEAN_VALUE,
+        help="Image preprocess with mean_value",
+    )
+    parser.add_argument(
+        "--ap_version",
+        type=str,
+        default=AP_VERSION,
+        help="Image preprocess with ap_version",
+    )
+    args = parser.parse_args()
+    if args.local:
+        convert_pascalvoc_local2bin(args)
+    else:
+        run_convert()
+
+
+if __name__ == "__main__":
+    main_pascalvoc_preprocess(sys.argv)

test/cpp/inference/api/gpu_ernie_half_test.cc

@@ -0,0 +1,290 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/inference/api/paddle_inference_api.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+
+using paddle::PaddleTensor;
+
+template <typename T>
+void GetValueFromStream(std::stringstream *ss, T *t) {
+  (*ss) >> (*t);
+}
+
+template <>
+void GetValueFromStream<std::string>(std::stringstream *ss, std::string *t) {
+  *t = ss->str();
+}
+
+// Split string to vector
+template <typename T>
+void Split(const std::string &line, char sep, std::vector<T> *v) {
+  std::stringstream ss;
+  T t;
+  for (auto c : line) {
+    if (c != sep) {
+      ss << c;
+    } else {
+      GetValueFromStream<T>(&ss, &t);
+      v->push_back(std::move(t));
+      ss.str({});
+      ss.clear();
+    }
+  }
+
+  if (!ss.str().empty()) {
+    GetValueFromStream<T>(&ss, &t);
+    v->push_back(std::move(t));
+    ss.str({});
+    ss.clear();
+  }
+}
+
+// Parse tensor from string
+template <typename T>
+bool ParseTensor(const std::string &field, paddle::PaddleTensor *tensor) {
+  std::vector<std::string> data;
+  Split(field, ':', &data);
+  if (data.size() < 2) return false;
+
+  std::string shape_str = data[0];
+
+  std::vector<int> shape;
+  Split(shape_str, ' ', &shape);
+
+  std::string mat_str = data[1];
+
+  std::vector<T> mat;
+  Split(mat_str, ' ', &mat);
+
+  tensor->shape = shape;
+  auto size =
+      std::accumulate(shape.begin(), shape.end(), 1, std::multiplies<int>()) *
+      sizeof(T);
+  tensor->data.Resize(size);
+  std::copy(mat.begin(), mat.end(), static_cast<T *>(tensor->data.data()));
+  tensor->dtype = GetPaddleDType<T>();
+
+  return true;
+}
+
+// Parse input tensors from string
+bool ParseLine(const std::string &line,
+               std::vector<paddle::PaddleTensor> *tensors) {
+  std::vector<std::string> fields;
+  Split(line, ';', &fields);
+
+  tensors->clear();
+  tensors->reserve(4);
+
+  int i = 0;
+  auto input_name = FLAGS_ernie_large ? "eval_placeholder_" : "placeholder_";
+  for (; i < 3; i++) {
+    paddle::PaddleTensor temp;
+    ParseTensor<int64_t>(fields[i], &temp);
+    temp.name = input_name + std::to_string(i);
+    tensors->push_back(temp);
+  }
+
+  // input_mask
+  paddle::PaddleTensor input_mask;
+  ParseTensor<float>(fields[i], &input_mask);
+  input_mask.name = input_name + std::to_string(i);
+  tensors->push_back(input_mask);
+
+  return true;
+}
+
+bool LoadInputData(std::vector<std::vector<paddle::PaddleTensor>> *inputs,
+                   int batch_size = 1) {
+  if (FLAGS_infer_data.empty()) {
+    LOG(ERROR) << "please set input data path";
+    return false;
+  }
+
+  std::ifstream fin(FLAGS_infer_data);
+  std::string line;
+  int sample = 0;
+
+  // The unit-test dataset only have 10 samples, each sample have 5 feeds.
+  while (std::getline(fin, line)) {
+    std::vector<paddle::PaddleTensor> feed_data;
+    ParseLine(line, &feed_data);
+    inputs->push_back(std::move(feed_data));
+    sample++;
+    if (!FLAGS_test_all_data && sample == batch_size) break;
+  }
+  LOG(INFO) << "number of samples: " << sample;
+  return true;
+}
+
+// Compare results
+TEST(Ernie_gpu_fp16_no_ir, compare_results) {
+  AnalysisConfig config;
+  config.SetModel(FLAGS_infer_model);
+  config.EnableUseGpu(512, 0, paddle_infer::PrecisionType::kHalf);
+  config.SwitchIrOptim(false);
+
+  auto predictor = CreatePaddlePredictor(config);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  LoadInputData(&input_slots_all);
+
+  std::ifstream fin(FLAGS_refer_result);
+  std::string line;
+  std::vector<float> ref;
+
+  while (std::getline(fin, line)) {
+    Split(line, ' ', &ref);
+  }
+
+  std::vector<PaddleTensor> outputs;
+  for (size_t i = 0; i < input_slots_all.size(); i++) {
+    outputs.clear();
+    predictor->Run(input_slots_all[i], &outputs);
+
+    auto output = outputs.front();
+    size_t outputs_size = 1;
+    for (auto dim : output.shape) {
+      outputs_size *= dim;
+    }
+    float *result = reinterpret_cast<float *>(output.data.data());
+    for (size_t j = 0; j < outputs_size; ++j) {
+      EXPECT_NEAR(ref[i * outputs_size + j], result[j], 8e-3);
+    }
+  }
+}
+
+// Compare results
+TEST(Ernie_gpu_fp16_with_ir, compare_results) {
+  AnalysisConfig config;
+  config.SetModel(FLAGS_infer_model);
+  config.EnableUseGpu(512, 0, paddle_infer::PrecisionType::kHalf);
+  config.SwitchIrOptim(true);
+  // There is a problem with the model itself, which has nothing to do with
+  // constant_folding_pass.
+  config.pass_builder()->DeletePass("constant_folding_pass");
+
+  auto predictor = CreatePaddlePredictor(config);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  LoadInputData(&input_slots_all);
+
+  std::ifstream fin(FLAGS_refer_result);
+  std::string line;
+  std::vector<float> ref;
+
+  while (std::getline(fin, line)) {
+    Split(line, ' ', &ref);
+  }
+
+  std::vector<PaddleTensor> outputs;
+  for (size_t i = 0; i < input_slots_all.size(); i++) {
+    outputs.clear();
+    predictor->Run(input_slots_all[i], &outputs);
+
+    auto output = outputs.front();
+    size_t outputs_size = 1;
+    for (auto dim : output.shape) {
+      outputs_size *= dim;
+    }
+    float *result = reinterpret_cast<float *>(output.data.data());
+    for (size_t j = 0; j < outputs_size; ++j) {
+      EXPECT_NEAR(ref[i * outputs_size + j], result[j], 2e-2);
+    }
+  }
+}
+
+// Compare results
+TEST(Ernie_gpu_bf16_no_ir, compare_results) {
+  AnalysisConfig config;
+  config.SetModel(FLAGS_infer_model);
+  config.EnableUseGpu(512, 0, paddle_infer::PrecisionType::kBf16);
+  config.SwitchIrOptim(false);
+
+  auto predictor = CreatePaddlePredictor(config);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  LoadInputData(&input_slots_all);
+
+  std::ifstream fin(FLAGS_refer_result);
+  std::string line;
+  std::vector<float> ref;
+
+  while (std::getline(fin, line)) {
+    Split(line, ' ', &ref);
+  }
+
+  std::vector<PaddleTensor> outputs;
+  for (size_t i = 0; i < input_slots_all.size(); i++) {
+    outputs.clear();
+    predictor->Run(input_slots_all[i], &outputs);
+
+    auto output = outputs.front();
+    size_t outputs_size = 1;
+    for (auto dim : output.shape) {
+      outputs_size *= dim;
+    }
+    float *result = reinterpret_cast<float *>(output.data.data());
+    for (size_t j = 0; j < outputs_size; ++j) {
+      EXPECT_NEAR(ref[i * outputs_size + j], result[j], 1e-2);
+    }
+  }
+}
+
+// Compare results
+TEST(Ernie_gpu_bf16_with_ir, compare_results) {
+  AnalysisConfig config;
+  config.SetModel(FLAGS_infer_model);
+  config.EnableUseGpu(512, 0, paddle_infer::PrecisionType::kBf16);
+  config.SwitchIrOptim(true);
+  // There is a problem with the model itself, which has nothing to do with
+  // constant_folding_pass.
+  config.pass_builder()->DeletePass("constant_folding_pass");
+
+  auto predictor = CreatePaddlePredictor(config);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  LoadInputData(&input_slots_all);
+
+  std::ifstream fin(FLAGS_refer_result);
+  std::string line;
+  std::vector<float> ref;
+
+  while (std::getline(fin, line)) {
+    Split(line, ' ', &ref);
+  }
+
+  std::vector<PaddleTensor> outputs;
+  for (size_t i = 0; i < input_slots_all.size(); i++) {
+    outputs.clear();
+    predictor->Run(input_slots_all[i], &outputs);
+
+    auto output = outputs.front();
+    size_t outputs_size = 1;
+    for (auto dim : output.shape) {
+      outputs_size *= dim;
+    }
+    float *result = reinterpret_cast<float *>(output.data.data());
+    for (size_t j = 0; j < outputs_size; ++j) {
+      EXPECT_NEAR(ref[i * outputs_size + j], result[j], 5e-3);
+    }
+  }
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/helper_test.cc

@@ -0,0 +1,35 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/fluid/inference/api/helper.h"
+
+#include "gtest/gtest.h"
+
+namespace paddle {
+
+TEST(inference_api_helper, DataType) {
+  ASSERT_TRUE(
+      paddle::inference::IsFloatVar(paddle::framework::proto::VarType::FP64));
+  ASSERT_TRUE(
+      paddle::inference::IsFloatVar(paddle::framework::proto::VarType::FP32));
+  ASSERT_TRUE(
+      paddle::inference::IsFloatVar(paddle::framework::proto::VarType::FP16));
+  ASSERT_TRUE(
+      paddle::inference::IsFloatVar(paddle::framework::proto::VarType::BF16));
+
+  ASSERT_FALSE(
+      paddle::inference::IsFloatVar(paddle::framework::proto::VarType::INT32));
+}
+
+}  // namespace paddle

test/cpp/inference/api/ipu_ernie_fp16_test.cc

@@ -0,0 +1,184 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+
+using paddle::PaddleTensor;
+
+template <typename T>
+void GetValueFromStream(std::stringstream *ss, T *t) {
+  (*ss) >> (*t);
+}
+
+template <>
+void GetValueFromStream<std::string>(std::stringstream *ss, std::string *t) {
+  *t = ss->str();
+}
+
+// Split string to vector
+template <typename T>
+void Split(const std::string &line, char sep, std::vector<T> *v) {
+  std::stringstream ss;
+  T t;
+  for (auto c : line) {
+    if (c != sep) {
+      ss << c;
+    } else {
+      GetValueFromStream<T>(&ss, &t);
+      v->push_back(std::move(t));
+      ss.str({});
+      ss.clear();
+    }
+  }
+
+  if (!ss.str().empty()) {
+    GetValueFromStream<T>(&ss, &t);
+    v->push_back(std::move(t));
+    ss.str({});
+    ss.clear();
+  }
+}
+
+// Parse tensor from string
+template <typename T>
+bool ParseTensor(const std::string &field, paddle::PaddleTensor *tensor) {
+  std::vector<std::string> data;
+  Split(field, ':', &data);
+  if (data.size() < 2) return false;
+
+  std::string shape_str = data[0];
+
+  std::vector<int> shape;
+  Split(shape_str, ' ', &shape);
+
+  std::string mat_str = data[1];
+
+  std::vector<T> mat;
+  Split(mat_str, ' ', &mat);
+
+  tensor->shape = shape;
+  auto size =
+      std::accumulate(shape.begin(), shape.end(), 1, std::multiplies<int>()) *
+      sizeof(T);
+  tensor->data.Resize(size);
+  std::copy(mat.begin(), mat.end(), static_cast<T *>(tensor->data.data()));
+  tensor->dtype = GetPaddleDType<T>();
+
+  return true;
+}
+
+// Parse input tensors from string
+bool ParseLine(const std::string &line,
+               std::vector<paddle::PaddleTensor> *tensors) {
+  std::vector<std::string> fields;
+  Split(line, ';', &fields);
+
+  tensors->clear();
+  tensors->reserve(4);
+
+  int i = 0;
+  auto input_name = FLAGS_ernie_large ? "eval_placeholder_" : "placeholder_";
+  for (; i < 3; i++) {
+    paddle::PaddleTensor temp;
+    ParseTensor<int64_t>(fields[i], &temp);
+    temp.name = input_name + std::to_string(i);
+    tensors->push_back(temp);
+  }
+
+  // input_mask
+  paddle::PaddleTensor input_mask;
+  ParseTensor<float>(fields[i], &input_mask);
+  // fp32 to fp16
+  ConvertFP32toFP16(input_mask);
+  input_mask.name = input_name + std::to_string(i);
+  tensors->push_back(input_mask);
+
+  return true;
+}
+
+bool LoadInputData(std::vector<std::vector<paddle::PaddleTensor>> *inputs,
+                   int batch_size = 1) {
+  if (FLAGS_infer_data.empty()) {
+    LOG(ERROR) << "please set input data path";
+    return false;
+  }
+
+  std::ifstream fin(FLAGS_infer_data);
+  std::string line;
+  int sample = 0;
+
+  // The unit-test dataset only have 10 samples, each sample have 5 feeds.
+  while (std::getline(fin, line)) {
+    std::vector<paddle::PaddleTensor> feed_data;
+    ParseLine(line, &feed_data);
+    inputs->push_back(std::move(feed_data));
+    sample++;
+    if (!FLAGS_test_all_data && sample == batch_size) break;
+  }
+  LOG(INFO) << "number of samples: " << sample;
+  return true;
+}
+
+void SetConfig(AnalysisConfig *cfg, int batch_size = 1) {
+  cfg->SetModel(FLAGS_infer_model);
+  // ipu_device_num, ipu_micro_batch_size, ipu_enable_pipelining
+  cfg->EnableIpu(1, batch_size, false);
+  // ipu_enable_fp16, ipu_replica_num, ipu_available_memory_proportion,
+  // ipu_enable_half_partial
+  cfg->SetIpuConfig(true, 1, 1.0, true);
+}
+
+// Compare results
+TEST(Analyzer_Ernie_ipu, compare_results) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  LoadInputData(&input_slots_all);
+
+  std::ifstream fin(FLAGS_refer_result);
+  std::string line;
+  std::vector<float> ref;
+
+  while (std::getline(fin, line)) {
+    Split(line, ' ', &ref);
+  }
+
+  auto predictor = CreateTestPredictor(
+      reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+      FLAGS_use_analysis);
+
+  std::vector<PaddleTensor> outputs;
+  for (size_t i = 0; i < input_slots_all.size(); i++) {
+    outputs.clear();
+    predictor->Run(input_slots_all[i], &outputs);
+
+    auto output = outputs.front();
+    ConvertFP16toFP32(output);
+    auto outputs_size = 1;
+    for (auto dim : output.shape) {
+      outputs_size *= dim;
+    }
+    float *fp32_data = reinterpret_cast<float *>(output.data.data());
+    for (size_t j = 0; j < outputs_size; ++j) {
+      EXPECT_NEAR(ref[i * outputs_size + j], fp32_data[j], 5e-3);
+    }
+  }
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/ipu_ernie_test.cc

@@ -0,0 +1,198 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+
+using paddle::PaddleTensor;
+
+template <typename T>
+void GetValueFromStream(std::stringstream *ss, T *t) {
+  (*ss) >> (*t);
+}
+
+template <>
+void GetValueFromStream<std::string>(std::stringstream *ss, std::string *t) {
+  *t = ss->str();
+}
+
+// Split string to vector
+template <typename T>
+void Split(const std::string &line, char sep, std::vector<T> *v) {
+  std::stringstream ss;
+  T t;
+  for (auto c : line) {
+    if (c != sep) {
+      ss << c;
+    } else {
+      GetValueFromStream<T>(&ss, &t);
+      v->push_back(std::move(t));
+      ss.str({});
+      ss.clear();
+    }
+  }
+
+  if (!ss.str().empty()) {
+    GetValueFromStream<T>(&ss, &t);
+    v->push_back(std::move(t));
+    ss.str({});
+    ss.clear();
+  }
+}
+
+// Parse tensor from string
+template <typename T>
+bool ParseTensor(const std::string &field, paddle::PaddleTensor *tensor) {
+  std::vector<std::string> data;
+  Split(field, ':', &data);
+  if (data.size() < 2) return false;
+
+  std::string shape_str = data[0];
+
+  std::vector<int> shape;
+  Split(shape_str, ' ', &shape);
+
+  std::string mat_str = data[1];
+
+  std::vector<T> mat;
+  Split(mat_str, ' ', &mat);
+
+  tensor->shape = shape;
+  auto size =
+      std::accumulate(shape.begin(), shape.end(), 1, std::multiplies<int>()) *
+      sizeof(T);
+  tensor->data.Resize(size);
+  std::copy(mat.begin(), mat.end(), static_cast<T *>(tensor->data.data()));
+  tensor->dtype = GetPaddleDType<T>();
+
+  return true;
+}
+
+// Parse input tensors from string
+bool ParseLine(const std::string &line,
+               std::vector<paddle::PaddleTensor> *tensors) {
+  std::vector<std::string> fields;
+  Split(line, ';', &fields);
+
+  tensors->clear();
+  tensors->reserve(4);
+
+  int i = 0;
+  auto input_name = FLAGS_ernie_large ? "eval_placeholder_" : "placeholder_";
+  for (; i < 3; i++) {
+    paddle::PaddleTensor temp;
+    ParseTensor<int64_t>(fields[i], &temp);
+    temp.name = input_name + std::to_string(i);
+    tensors->push_back(temp);
+  }
+
+  // input_mask
+  paddle::PaddleTensor input_mask;
+  ParseTensor<float>(fields[i], &input_mask);
+  input_mask.name = input_name + std::to_string(i);
+  tensors->push_back(input_mask);
+
+  return true;
+}
+
+bool LoadInputData(std::vector<std::vector<paddle::PaddleTensor>> *inputs,
+                   int batch_size = 1) {
+  if (FLAGS_infer_data.empty()) {
+    LOG(ERROR) << "please set input data path";
+    return false;
+  }
+
+  std::ifstream fin(FLAGS_infer_data);
+  std::string line;
+  int sample = 0;
+
+  // The unit-test dataset only have 10 samples, each sample have 5 feeds.
+  while (std::getline(fin, line)) {
+    std::vector<paddle::PaddleTensor> feed_data;
+    ParseLine(line, &feed_data);
+    inputs->push_back(std::move(feed_data));
+    sample++;
+    if (!FLAGS_test_all_data && sample == batch_size) break;
+  }
+  LOG(INFO) << "number of samples: " << sample;
+  return true;
+}
+
+void SetConfig(AnalysisConfig *cfg, int batch_size = 1) {
+  cfg->SetModel(FLAGS_infer_model);
+  // ipu_device_num, ipu_micro_batch_size, ipu_enable_pipelining
+  cfg->EnableIpu(1, batch_size, false);
+}
+
+void profile() {
+  AnalysisConfig config;
+  SetConfig(&config);
+
+  std::vector<std::vector<PaddleTensor>> outputs;
+  std::vector<std::vector<PaddleTensor>> inputs;
+  LoadInputData(&inputs);
+  TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&config),
+                 inputs,
+                 &outputs,
+                 FLAGS_num_threads);
+}
+
+// Compare Deterministic result
+TEST(Analyzer_Ernie_ipu, compare_determine) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  LoadInputData(&input_slots_all);
+  CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+                       input_slots_all);
+}
+
+// Compare results
+TEST(Analyzer_Ernie_ipu, compare_results) {
+  AnalysisConfig cfg;
+  SetConfig(&cfg);
+
+  std::vector<std::vector<PaddleTensor>> input_slots_all;
+  LoadInputData(&input_slots_all);
+
+  std::ifstream fin(FLAGS_refer_result);
+  std::string line;
+  std::vector<float> ref;
+
+  while (std::getline(fin, line)) {
+    Split(line, ' ', &ref);
+  }
+
+  auto predictor = CreateTestPredictor(
+      reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
+      FLAGS_use_analysis);
+
+  std::vector<PaddleTensor> outputs;
+  for (size_t i = 0; i < input_slots_all.size(); i++) {
+    outputs.clear();
+    predictor->Run(input_slots_all[i], &outputs);
+    auto outputs_size = outputs.front().data.length() / (sizeof(float));
+    for (size_t j = 0; j < outputs_size; ++j) {
+      EXPECT_NEAR(ref[i * outputs_size + j],
+                  static_cast<float *>(outputs[0].data.data())[j],
+                  FLAGS_accuracy);
+    }
+  }
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/ipu_multi_model_profile.cc

@@ -0,0 +1,112 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include "paddle/common/flags.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+
+void ErnieInputData(const int &total_batch_size,
+                    const bool enable_fp16,
+                    std::vector<PaddleTensor> *inputs) {
+  const int input_num = total_batch_size * 128 * 1;
+  std::vector<int64_t> placeholder_012(input_num, 1);
+  std::vector<float> placeholder_3(input_num, 1);
+
+  for (int i = 0; i < 4; i++) {
+    PaddleTensor in;
+    in.name = "placeholder_" + std::to_string(i);
+    in.shape = {total_batch_size, 128, 1};
+    if (i < 3) {
+      in.data = PaddleBuf(static_cast<void *>(placeholder_012.data()),
+                          input_num * sizeof(int64_t));
+      in.dtype = PaddleDType::INT64;
+    } else {
+      in.data = PaddleBuf(static_cast<void *>(placeholder_3.data()),
+                          input_num * sizeof(float));
+      in.dtype = PaddleDType::FLOAT32;
+      if (enable_fp16) {
+        ConvertFP32toFP16(in);
+      }
+    }
+    inputs->push_back(std::move(in));
+  }
+}
+
+void Resnet50InputData(const int &total_batch_size,
+                       const bool enable_fp16,
+                       std::vector<paddle::PaddleTensor> *inputs) {
+  const int input_num = total_batch_size * 3 * 318 * 318;
+  std::vector<float> input(input_num, 1);
+  PaddleTensor in;
+  in.shape = {total_batch_size, 3, 318, 318};
+  in.data =
+      PaddleBuf(static_cast<void *>(input.data()), input_num * sizeof(float));
+  in.dtype = PaddleDType::FLOAT32;
+  if (enable_fp16) {
+    ConvertFP32toFP16(in);
+  }
+  inputs->push_back(std::move(in));
+}
+
+// performance profile
+TEST(Analyzer_ipu_fp16, performance_profile) {
+  AnalysisConfig config;
+  std::vector<PaddleTensor> inputs;
+  std::vector<std::vector<PaddleTensor>> outputs;
+
+  int total_batch_size = FLAGS_ipu_micro_batch_size * FLAGS_ipu_replica_num;
+  if (FLAGS_ipu_enable_pipelining) {
+    // if device_num > 1 and pipelining is enabled, the total batch size =
+    // micro_batch_size * device_num(batches_per_step) * replica_num
+    total_batch_size = FLAGS_ipu_micro_batch_size * FLAGS_ipu_batches_per_step *
+                       FLAGS_ipu_replica_num;
+  }
+
+  if (FLAGS_model_name == "Resnet50") {
+    config.SetModel(FLAGS_infer_model + "/model/model",
+                    FLAGS_infer_model + "/model/params");
+    Resnet50InputData(total_batch_size, FLAGS_ipu_enable_fp16, &inputs);
+  } else if (FLAGS_model_name == "Ernie") {
+    config.SetModel(FLAGS_infer_model + "/model/");
+    ErnieInputData(total_batch_size, FLAGS_ipu_enable_fp16, &inputs);
+  } else {
+    PADDLE_THROW(common::errors::InvalidArgument(
+        "Only support Resnet50 and Ernie Currently"));
+  }
+  // ipu_device_num, ipu_micro_batch_size, ipu_enable_pipelining,
+  // ipu_batches_per_step
+  config.EnableIpu(FLAGS_ipu_device_num,
+                   FLAGS_ipu_micro_batch_size,
+                   FLAGS_ipu_enable_pipelining,
+                   FLAGS_ipu_batches_per_step);
+  // ipu_enable_fp16, ipu_replica_num, ipu_available_memory_proportion,
+  // ipu_enable_half_partial
+  config.SetIpuConfig(FLAGS_ipu_enable_fp16,
+                      FLAGS_ipu_replica_num,
+                      FLAGS_ipu_available_memory_proportion,
+                      FLAGS_ipu_enable_half_partial);
+
+  TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&config),
+                 {inputs},
+                 &outputs,
+                 1);
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/ipu_resnet50_fp16_test.cc

@@ -0,0 +1,87 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include <cmath>
+
+#include "paddle/common/flags.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+
+// Compare results with 1 batch
+TEST(Analyzer_Resnet50_ipu, compare_results_1_batch) {
+  std::string model_dir = FLAGS_infer_model + "/" + "model";
+  AnalysisConfig config;
+  // ipu_device_num, ipu_micro_batch_size, ipu_enable_pipelining
+  config.EnableIpu(1, 1, false);
+  // ipu_enable_fp16, ipu_replica_num, ipu_available_memory_proportion,
+  // ipu_enable_half_partial
+  config.SetIpuConfig(true, 1, 1.0, true);
+  config.SetModel(model_dir + "/model", model_dir + "/params");
+
+  std::vector<PaddleTensor> inputs;
+  auto predictor = CreatePaddlePredictor(config);
+  const int batch = 1;
+  const int channel = 3;
+  const int height = 318;
+  const int width = 318;
+  const int input_num = batch * channel * height * width;
+  std::vector<float> input(input_num, 1);
+
+  PaddleTensor in;
+  in.shape = {batch, channel, height, width};
+  in.data =
+      PaddleBuf(static_cast<void*>(input.data()), input_num * sizeof(float));
+  in.dtype = PaddleDType::FLOAT32;
+  ConvertFP32toFP16(in);
+  inputs.emplace_back(in);
+
+  std::vector<PaddleTensor> outputs;
+
+  ASSERT_TRUE(predictor->Run(inputs, &outputs));
+
+  const std::vector<float> truth_values = {
+      127.779f,  738.165f,  1013.22f,  -438.17f,  366.401f,  927.659f,
+      736.222f,  -633.684f, -329.927f, -430.155f, -633.062f, -146.548f,
+      -1324.28f, -1349.36f, -242.675f, 117.448f,  -801.723f, -391.514f,
+      -404.818f, 454.16f,   515.48f,   -133.031f, 69.293f,   590.096f,
+      -1434.69f, -1070.89f, 307.074f,  400.525f,  -316.12f,  -587.125f,
+      -161.056f, 800.363f,  -96.4708f, 748.706f,  868.174f,  -447.938f,
+      112.737f,  1127.2f,   47.4355f,  677.72f,   593.186f,  -336.4f,
+      551.362f,  397.823f,  78.3979f,  -715.398f, 405.969f,  404.256f,
+      246.019f,  -8.42969f, 131.365f,  -648.051f};
+
+  const size_t expected_size = 1;
+  EXPECT_EQ(outputs.size(), expected_size);
+
+  auto output = outputs.front();
+  ConvertFP16toFP32(output);
+  auto outputs_size = 1;
+  for (auto dim : output.shape) {
+    outputs_size *= dim;
+  }
+  float* fp32_data = reinterpret_cast<float*>(output.data.data());
+
+  for (size_t j = 0; j < outputs_size; j += 10) {
+    EXPECT_NEAR(
+        (fp32_data[j] - truth_values[j / 10]) / truth_values[j / 10], 0., 9e-2);
+  }
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/ipu_resnet50_test.cc

@@ -0,0 +1,178 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include <cmath>
+
+#include "paddle/common/flags.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+
+static std::vector<float> truth_values = {
+    127.779f,  738.165f,  1013.22f,  -438.17f,  366.401f,  927.659f,  736.222f,
+    -633.684f, -329.927f, -430.155f, -633.062f, -146.548f, -1324.28f, -1349.36f,
+    -242.675f, 117.448f,  -801.723f, -391.514f, -404.818f, 454.16f,   515.48f,
+    -133.031f, 69.293f,   590.096f,  -1434.69f, -1070.89f, 307.074f,  400.525f,
+    -316.12f,  -587.125f, -161.056f, 800.363f,  -96.4708f, 748.706f,  868.174f,
+    -447.938f, 112.737f,  1127.2f,   47.4355f,  677.72f,   593.186f,  -336.4f,
+    551.362f,  397.823f,  78.3979f,  -715.398f, 405.969f,  404.256f,  246.019f,
+    -8.42969f, 131.365f,  -648.051f};
+
+// Compare results with 1 batch
+TEST(Analyzer_Resnet50_ipu, compare_results_1_batch) {
+  std::string model_dir = FLAGS_infer_model + "/" + "model";
+  AnalysisConfig config;
+  // ipu_device_num, ipu_micro_batch_size, ipu_enable_pipelining
+  config.EnableIpu(1, 1, false);
+  config.SetModel(model_dir + "/model", model_dir + "/params");
+
+  std::vector<PaddleTensor> inputs;
+  auto predictor = CreatePaddlePredictor(config);
+  const int batch = 1;
+  const int channel = 3;
+  const int height = 318;
+  const int width = 318;
+  const int input_num = batch * channel * height * width;
+  std::vector<float> input(input_num, 1);
+
+  PaddleTensor in;
+  in.shape = {batch, channel, height, width};
+  in.data =
+      PaddleBuf(static_cast<void*>(input.data()), input_num * sizeof(float));
+  in.dtype = PaddleDType::FLOAT32;
+  inputs.emplace_back(in);
+
+  std::vector<PaddleTensor> outputs;
+
+  ASSERT_TRUE(predictor->Run(inputs, &outputs));
+
+  const size_t expected_size = 1;
+  EXPECT_EQ(outputs.size(), expected_size);
+  float* data_o = static_cast<float*>(outputs[0].data.data());
+
+  for (size_t j = 0; j < outputs[0].data.length() / sizeof(float); j += 10) {
+    EXPECT_NEAR(
+        (data_o[j] - truth_values[j / 10]) / truth_values[j / 10], 0., 12e-5);
+  }
+}
+
+// Compare results with 2 batch
+TEST(Analyzer_Resnet50_ipu, compare_results_2_batch) {
+  std::string model_dir = FLAGS_infer_model + "/" + "model";
+  AnalysisConfig config;
+  // ipu_device_num, ipu_micro_batch_size, ipu_enable_pipelining
+  config.EnableIpu(1, 2, false);
+  config.SetModel(model_dir + "/model", model_dir + "/params");
+
+  std::vector<PaddleTensor> inputs;
+  auto predictor = CreatePaddlePredictor(config);
+  const int batch = 2;
+  const int channel = 3;
+  const int height = 318;
+  const int width = 318;
+  const int input_num = batch * channel * height * width;
+  std::vector<float> input(input_num, 1);
+
+  PaddleTensor in;
+  in.shape = {batch, channel, height, width};
+  in.data =
+      PaddleBuf(static_cast<void*>(input.data()), input_num * sizeof(float));
+  in.dtype = PaddleDType::FLOAT32;
+  inputs.emplace_back(in);
+
+  std::vector<PaddleTensor> outputs;
+
+  ASSERT_TRUE(predictor->Run(inputs, &outputs));
+
+  const size_t expected_size = 1;
+  EXPECT_EQ(outputs.size(), expected_size);
+  float* data_o = static_cast<float*>(outputs[0].data.data());
+
+  auto num_output_per_batch = outputs[0].data.length() / sizeof(float) / 2;
+  for (size_t j = 0; j < num_output_per_batch; j += 10) {
+    EXPECT_NEAR(
+        (data_o[j] - truth_values[j / 10]) / truth_values[j / 10], 0., 12e-5);
+    EXPECT_NEAR((data_o[j + num_output_per_batch] - truth_values[j / 10]) /
+                    truth_values[j / 10],
+                0.,
+                12e-5);
+  }
+}
+
+// multi threading
+TEST(Analyzer_Resnet50_ipu, model_runtime_multi_thread) {
+  std::string model_dir = FLAGS_infer_model + "/" + "model";
+  AnalysisConfig config;
+  const int thread_num = 10;
+  // ipu_device_num, ipu_micro_batch_size, ipu_enable_pipelining
+  config.EnableIpu(1, 1, false);
+  config.SetIpuConfig(false, 1, 1.0, false, true);
+  config.SetModel(model_dir + "/model", model_dir + "/params");
+
+  auto main_predictor = CreatePaddlePredictor(config);
+  std::vector<std::vector<PaddleTensor>> inputs;
+  std::vector<std::vector<PaddleTensor>> outputs;
+  std::vector<decltype(main_predictor)> predictors;
+  std::vector<std::thread> threads;
+  outputs.resize(thread_num);
+  inputs.resize(thread_num);
+
+  const int batch = 1;
+  const int channel = 3;
+  const int height = 318;
+  const int width = 318;
+  const int input_num = batch * channel * height * width;
+  std::vector<float> input(input_num, 1);
+
+  PaddleTensor in;
+  in.shape = {batch, channel, height, width};
+  in.data =
+      PaddleBuf(static_cast<void*>(input.data()), input_num * sizeof(float));
+  in.dtype = PaddleDType::FLOAT32;
+
+  for (int i = 0; i < thread_num; ++i) {
+    inputs[i].emplace_back(in);
+    predictors.emplace_back(std::move(main_predictor->Clone()));
+  }
+
+  auto run = [](PaddlePredictor* predictor,
+                std::vector<PaddleTensor>& input,
+                std::vector<PaddleTensor>& output) {
+    ASSERT_TRUE(predictor->Run(input, &output));
+  };
+
+  for (int i = 0; i < thread_num; ++i) {
+    threads.emplace_back(
+        run, predictors[i].get(), std::ref(inputs[i]), std::ref(outputs[i]));
+  }
+
+  for (int i = 0; i < thread_num; ++i) {
+    threads[i].join();
+  }
+
+  const size_t expected_size = 1;
+  for (int i = 0; i < thread_num; ++i) {
+    EXPECT_EQ(outputs[i].size(), expected_size);
+    float* data_o = static_cast<float*>(outputs[i][0].data.data());
+
+    for (size_t j = 0; j < outputs[i][0].data.length() / sizeof(float);
+         j += 10) {
+      EXPECT_NEAR(
+          (data_o[j] - truth_values[j / 10]) / truth_values[j / 10], 0., 12e-5);
+    }
+  }
+}
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/ipu_word2vec_sample.cc

@@ -0,0 +1,82 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+/*
+ * This file contains a simple demo for how to take a model for inference with
+ * IPUs.
+ * Model: wget -q
+ * http://paddle-inference-dist.bj.bcebos.com/word2vec.inference.model.tar.gz
+ */
+
+#include <iostream>
+#include <numeric>
+#include <string>
+#include <vector>
+
+#include "glog/logging.h"
+#include "paddle/common/flags.h"
+#include "paddle/fluid/inference/api/paddle_inference_api.h"
+
+PD_DEFINE_string(infer_model, "", "Directory of the inference model.");
+
+using paddle_infer::Config;
+using paddle_infer::CreatePredictor;
+using paddle_infer::Predictor;
+
+void inference(std::string model_path,
+               bool use_ipu,
+               std::vector<float> *out_data) {
+  //# 1. Create Predictor with a config.
+  Config config;
+  config.SetModel(FLAGS_infer_model);
+  if (use_ipu) {
+    // ipu_device_num, ipu_micro_batch_size
+    config.EnableIpu(1, 4);
+  }
+  auto predictor = CreatePredictor(config);
+
+  //# 2. Prepare input/output tensor.
+  auto input_names = predictor->GetInputNames();
+  std::vector<int64_t> data{1, 2, 3, 4};
+  // For simplicity, we set all the slots with the same data.
+  for (auto input_name : input_names) {
+    auto input_tensor = predictor->GetInputHandle(input_name);
+    input_tensor->Reshape({4, 1});
+    input_tensor->CopyFromCpu(data.data());
+  }
+
+  //# 3. Run
+  predictor->Run();
+
+  //# 4. Get output.
+  auto output_names = predictor->GetOutputNames();
+  auto output_tensor = predictor->GetOutputHandle(output_names[0]);
+  std::vector<int> output_shape = output_tensor->shape();
+  int out_num = std::accumulate(
+      output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
+  out_data->resize(out_num);
+  output_tensor->CopyToCpu(out_data->data());
+}
+
+int main(int argc, char *argv[]) {
+  ::paddle::flags::ParseCommandLineFlags(&argc, &argv);
+  std::vector<float> ipu_result;
+  std::vector<float> cpu_result;
+  inference(FLAGS_infer_model, true, &ipu_result);
+  inference(FLAGS_infer_model, false, &cpu_result);
+  for (size_t i = 0; i < ipu_result.size(); i++) {
+    CHECK_NEAR(ipu_result[i], cpu_result[i], 1e-6);
+  }
+  LOG(INFO) << "Finished";
+}

test/cpp/inference/api/lite_mul_model_test.cc

@@ -0,0 +1,126 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include <cmath>
+#include <mutex>   // NOLINT
+#include <thread>  // NOLINT
+
+#include "paddle/common/flags.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+
+int test_predictor(const AnalysisConfig& config_in,
+                   Barrier* barrier = nullptr) {
+  static std::mutex mutex;
+  AnalysisConfig config{config_in};
+  std::unique_ptr<PaddlePredictor> predictor;
+  {
+    std::unique_lock<std::mutex> lock(mutex);
+    predictor = CreatePaddlePredictor(config);
+  }
+  if (barrier) {
+    barrier->Wait();
+  }
+
+  std::vector<PaddleTensor> inputs;
+  std::vector<float> input({1});
+
+  PaddleTensor in;
+  in.shape = {1, 1};
+  in.data = PaddleBuf(static_cast<void*>(input.data()), 1 * sizeof(float));
+  in.dtype = PaddleDType::FLOAT32;
+  inputs.emplace_back(in);
+
+  std::vector<PaddleTensor> outputs;
+  predictor->Run(inputs, &outputs);
+  const std::vector<float> truth_values = {-0.00621776f,
+                                           -0.00620937f,
+                                           0.00990623f,
+                                           -0.0039817f,
+                                           -0.00074315f,
+                                           0.61229795f,
+                                           -0.00491806f,
+                                           -0.00068755f,
+                                           0.18409646f,
+                                           0.30090684f};
+  const size_t expected_size = 1;
+  EXPECT_EQ(outputs.size(), expected_size);
+  float* data_o = static_cast<float*>(outputs[0].data.data());
+  for (size_t j = 0; j < outputs[0].data.length() / sizeof(float); ++j) {
+    EXPECT_LT(std::abs(data_o[j] - truth_values[j]), 10e-6);
+  }
+  return 0;
+}
+
+int test_predictor_zero_copy(const AnalysisConfig& config_in,
+                             Barrier* barrier = nullptr) {
+  static std::mutex mutex;
+  AnalysisConfig config{config_in};
+  std::unique_ptr<PaddlePredictor> predictor;
+  {
+    std::unique_lock<std::mutex> lock(mutex);
+    predictor = CreatePaddlePredictor(config);
+  }
+  if (barrier) {
+    barrier->Wait();
+  }
+
+  std::vector<float> input({1});
+  auto in_tensor =
+      predictor->GetInputTensor(predictor->GetInputNames().front());
+  in_tensor->Reshape({1, 1});
+  in_tensor->copy_from_cpu(input.data());
+
+  predictor->ZeroCopyRun();
+
+  auto out_tensor =
+      predictor->GetOutputTensor(predictor->GetOutputNames().front());
+  std::vector<float> data_o(10);
+  out_tensor->copy_to_cpu(data_o.data());
+
+  const std::vector<float> truth_values = {-0.00621776f,
+                                           -0.00620937f,
+                                           0.00990623f,
+                                           -0.0039817f,
+                                           -0.00074315f,
+                                           0.61229795f,
+                                           -0.00491806f,
+                                           -0.00068755f,
+                                           0.18409646f,
+                                           0.30090684f};
+  const size_t expected_size = 1;
+  EXPECT_EQ(predictor->GetOutputNames().size(), expected_size);
+  for (size_t j = 0; j < truth_values.size(); ++j) {
+    EXPECT_LT(std::abs(data_o[j] - truth_values[j]), 10e-6);
+  }
+  return 0;
+}
+
+#ifdef PADDLE_WITH_XPU
+TEST(AnalysisPredictor, native_xpu) {
+  AnalysisConfig config;
+  config.EnableXpu();
+  config.SetModel(FLAGS_infer_model + "/" + "mul_model");
+  test_predictor(config);
+  test_predictor_zero_copy(config);
+}
+#endif
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/onnxruntime_predictor_tester.cc

@@ -0,0 +1,81 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include <string>
+#include <thread>  // NOLINT
+#include <vector>
+
+#include "paddle/fluid/framework/ir/pass.h"
+#include "paddle/fluid/framework/tensor.h"
+#include "paddle/fluid/inference/api/helper.h"
+#include "paddle/fluid/inference/api/onnxruntime_predictor.h"
+#include "paddle/fluid/inference/api/paddle_api.h"
+#include "paddle/fluid/inference/api/paddle_inference_api.h"
+#include "paddle/fluid/inference/utils/io_utils.h"
+#include "paddle/phi/backends/cpu/cpu_info.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+PD_DEFINE_string(dirname, "", "dirname to tests.");
+
+namespace paddle {
+
+TEST(ONNXRuntimePredictor, onnxruntime_on) {
+  AnalysisConfig config;
+  config.SetModel(FLAGS_dirname + "/inference.pdmodel",
+                  FLAGS_dirname + "/inference.pdiparams");
+  config.EnableONNXRuntime();
+  config.EnableORTOptimization();
+  config.SetCpuMathLibraryNumThreads(2);
+  LOG(INFO) << config.Summary();
+
+  auto _predictor =
+      CreatePaddlePredictor<AnalysisConfig,
+                            paddle::PaddleEngineKind::kONNXRuntime>(config);
+  ASSERT_TRUE(_predictor);
+  auto* predictor = static_cast<ONNXRuntimePredictor*>(_predictor.get());
+
+  ASSERT_TRUE(predictor);
+  ASSERT_TRUE(!predictor->Clone());
+  // Dummy Input Data
+  std::vector<int64_t> input_shape = {-1, 3, 224, 224};
+  std::vector<float> input_data(1 * 3 * 224 * 224, 1.0);
+  std::vector<float> out_data;
+  out_data.resize(1000);
+
+  // testing all interfaces
+  auto input_names = predictor->GetInputNames();
+  auto output_names = predictor->GetOutputNames();
+  auto get_input_shape = predictor->GetInputTensorShape();
+
+  ASSERT_EQ(input_names.size(), 1UL);
+  ASSERT_EQ(output_names.size(), 1UL);
+  ASSERT_EQ(input_names[0], "inputs");
+  ASSERT_EQ(output_names[0], "save_infer_model/scale_0.tmp_1");
+  ASSERT_EQ(get_input_shape["inputs"], input_shape);
+
+  auto input_tensor = predictor->GetInputTensor(input_names[0]);
+  input_tensor->Reshape({1, 3, 224, 224});
+  auto output_tensor = predictor->GetOutputTensor(output_names[0]);
+
+  input_tensor->CopyFromCpu(input_data.data());
+  ASSERT_TRUE(predictor->ZeroCopyRun());
+  output_tensor->CopyToCpu(out_data.data());
+
+  predictor->TryShrinkMemory();
+}
+
+}  // namespace paddle

test/cpp/inference/api/paddle_infer_api_copy_tensor_tester.cc

@@ -0,0 +1,560 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <cuda_runtime.h>
+#include <gtest/gtest.h>
+
+#include <array>
+#include <cstring>
+#include <functional>
+#include <numeric>
+
+#include "glog/logging.h"
+#include "paddle/common/flags.h"
+#include "paddle/fluid/inference/api/paddle_infer_contrib.h"
+#include "paddle/phi/common/float16.h"
+#include "test/cpp/inference/api/trt_test_helper.h"
+
+namespace paddle_infer {
+
+class InferApiTesterUtils {
+ public:
+  static std::unique_ptr<Tensor> CreateInferTensorForTest(
+      const std::string &name, PlaceType place, void *p_scope) {
+    auto var = static_cast<paddle::framework::Scope *>(p_scope)->Var(name);
+    var->GetMutable<phi::DenseTensor>();
+    phi::DeviceContextPool &pool = phi::DeviceContextPool::Instance();
+    const auto &dev_ctxs = pool.device_contexts();
+    std::unique_ptr<Tensor> res(new Tensor(p_scope, &dev_ctxs));
+    res->input_or_output_ = true;
+    res->SetName(name);
+    res->SetPlace(place, 0 /*device id*/);
+    return res;
+  }
+};
+
+TEST(Tensor, copy_to_cpu_async_stream) {
+  LOG(INFO) << GetVersion();
+  UpdateDllFlag("conv_workspace_size_limit", "4000");
+  std::string model_dir = FLAGS_infer_model + "/model";
+  Config config;
+  config.EnableNewIR(false);
+  config.SetModel(model_dir + "/model", model_dir + "/params");
+  config.EnableUseGpu(100, 0);
+
+  auto predictor = CreatePredictor(config);
+  auto pred_clone = predictor->Clone();
+
+  std::vector<int> in_shape = {1, 3, 318, 318};
+  int in_num = std::accumulate(
+      in_shape.begin(), in_shape.end(), 1, std::multiplies<int>());
+
+  std::vector<float> input(in_num, 1.0);
+
+  const auto &input_names = predictor->GetInputNames();
+  auto input_tensor = predictor->GetInputHandle(input_names[0]);
+
+  input_tensor->Reshape(in_shape);
+  input_tensor->CopyFromCpu(input.data());
+
+  predictor->Run();
+
+  const auto &output_names = predictor->GetOutputNames();
+  auto output_tensor = predictor->GetOutputHandle(output_names[0]);
+  std::vector<int> output_shape = output_tensor->shape();
+  int out_num = std::accumulate(
+      output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
+
+  float *out_data = static_cast<float *>(
+      contrib::TensorUtils::CudaMallocPinnedMemory(sizeof(float) * out_num));
+  memset(out_data, 0, sizeof(float) * out_num);
+  std::vector<float> correct_out_data = {
+      127.78,
+      1.07353,
+      -229.42,
+      1127.28,
+      -177.365,
+      -292.412,
+      -271.614,
+      466.054,
+      540.436,
+      -214.223,
+  };
+
+  for (int i = 0; i < 100; i++) {
+    predictor->Run();
+  }
+
+  cudaStream_t stream;
+  output_tensor->CopyToCpuAsync(out_data, static_cast<void *>(&stream));
+
+  // sync
+  cudaStreamSynchronize(stream);
+
+  for (int i = 0; i < 10; i++) {
+    EXPECT_NEAR(out_data[i] / correct_out_data[i], 1.0, 1e-3);
+  }
+  contrib::TensorUtils::CudaFreePinnedMemory(static_cast<void *>(out_data));
+}
+
+TEST(Tensor, copy_to_cpu_async_callback) {
+  LOG(INFO) << GetVersion();
+  UpdateDllFlag("conv_workspace_size_limit", "4000");
+  std::string model_dir = FLAGS_infer_model + "/model";
+  Config config;
+  config.SwitchIrOptim(false);
+  config.EnableNewIR(false);
+  config.SetModel(model_dir + "/model", model_dir + "/params");
+  config.EnableUseGpu(100, 0);
+
+  auto predictor = CreatePredictor(config);
+  auto pred_clone = predictor->Clone();
+
+  std::vector<int> in_shape = {1, 3, 318, 318};
+  int in_num = std::accumulate(
+      in_shape.begin(), in_shape.end(), 1, std::multiplies<int>());
+
+  std::vector<float> input(in_num, 1.0);
+
+  const auto &input_names = predictor->GetInputNames();
+  auto input_tensor = predictor->GetInputHandle(input_names[0]);
+
+  input_tensor->Reshape(in_shape);
+  input_tensor->CopyFromCpu(input.data());
+
+  predictor->Run();
+
+  const auto &output_names = predictor->GetOutputNames();
+  auto output_tensor = predictor->GetOutputHandle(output_names[0]);
+  std::vector<int> output_shape = output_tensor->shape();
+  int out_num = std::accumulate(
+      output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
+
+  float *out_data = static_cast<float *>(
+      contrib::TensorUtils::CudaMallocPinnedMemory(sizeof(float) * out_num));
+  memset(out_data, 0, sizeof(float) * out_num);
+
+  for (int i = 0; i < 100; i++) {
+    predictor->Run();
+  }
+  cudaDeviceSynchronize();
+
+  output_tensor->CopyToCpuAsync(
+      out_data,
+      [](void *cb_params) {
+        float *data = static_cast<float *>(cb_params);
+        std::vector<float> correct_out_data = {
+            127.78,
+            1.07353,
+            -229.42,
+            1127.28,
+            -177.365,
+            -292.412,
+            -271.614,
+            466.054,
+            540.436,
+            -214.223,
+        };
+        for (int i = 0; i < 10; i++) {
+          EXPECT_NEAR(data[i] / correct_out_data[i], 1.0, 1e-3);
+        }
+      },
+      static_cast<void *>(out_data));
+
+  cudaDeviceSynchronize();
+  contrib::TensorUtils::CudaFreePinnedMemory(static_cast<void *>(out_data));
+}
+
+template <class DTYPE>
+static void test_copy_tensor(PlaceType src_place, PlaceType dst_place) {
+  paddle::framework::Scope scope;
+  auto tensor_src = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
+      "tensor_src", src_place, static_cast<void *>(&scope));
+  auto tensor_dst = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
+      "tensor_dst", dst_place, static_cast<void *>(&scope));
+  std::vector<DTYPE> data_src(6, 1);
+  tensor_src->Reshape({2, 3});
+  tensor_src->CopyFromCpu(data_src.data());
+
+  std::vector<DTYPE> data_dst(4, 2);
+  tensor_dst->Reshape({2, 2});
+  tensor_dst->CopyFromCpu(data_dst.data());
+
+  paddle_infer::contrib::TensorUtils::CopyTensor(tensor_dst.get(), *tensor_src);
+
+  EXPECT_EQ(tensor_dst->shape().size(), (size_t)2);
+  EXPECT_EQ(tensor_dst->shape()[0], 2);
+  EXPECT_EQ(tensor_dst->shape()[1], 3);
+
+  std::vector<DTYPE> data_check(6, 3);
+  tensor_dst->CopyToCpu<DTYPE>(static_cast<DTYPE *>(data_check.data()));
+
+  for (int i = 0; i < 6; i++) {
+    EXPECT_NEAR(data_check[i], 1, 1e-5);
+  }
+}
+
+TEST(CopyTensor, float64) {
+  test_copy_tensor<double>(PlaceType::kCPU, PlaceType::kCPU);
+  test_copy_tensor<double>(PlaceType::kCPU, PlaceType::kGPU);
+  test_copy_tensor<double>(PlaceType::kGPU, PlaceType::kCPU);
+  test_copy_tensor<double>(PlaceType::kGPU, PlaceType::kGPU);
+}
+
+TEST(CopyTensor, float32) {
+  test_copy_tensor<float>(PlaceType::kCPU, PlaceType::kCPU);
+  test_copy_tensor<float>(PlaceType::kCPU, PlaceType::kGPU);
+  test_copy_tensor<float>(PlaceType::kGPU, PlaceType::kCPU);
+  test_copy_tensor<float>(PlaceType::kGPU, PlaceType::kGPU);
+}
+
+TEST(CopyTensor, int32) {
+  test_copy_tensor<int32_t>(PlaceType::kCPU, PlaceType::kCPU);
+  test_copy_tensor<int32_t>(PlaceType::kCPU, PlaceType::kGPU);
+  test_copy_tensor<int32_t>(PlaceType::kGPU, PlaceType::kCPU);
+  test_copy_tensor<int32_t>(PlaceType::kGPU, PlaceType::kGPU);
+}
+
+TEST(CopyTensor, int64) {
+  test_copy_tensor<int64_t>(PlaceType::kCPU, PlaceType::kCPU);
+  test_copy_tensor<int64_t>(PlaceType::kCPU, PlaceType::kGPU);
+  test_copy_tensor<int64_t>(PlaceType::kGPU, PlaceType::kCPU);
+  test_copy_tensor<int64_t>(PlaceType::kGPU, PlaceType::kGPU);
+}
+
+TEST(CopyTensor, int8) {
+  test_copy_tensor<int8_t>(PlaceType::kCPU, PlaceType::kCPU);
+  test_copy_tensor<int8_t>(PlaceType::kCPU, PlaceType::kGPU);
+  test_copy_tensor<int8_t>(PlaceType::kGPU, PlaceType::kCPU);
+  test_copy_tensor<int8_t>(PlaceType::kGPU, PlaceType::kGPU);
+}
+
+TEST(CopyTensor, uint8) {
+  test_copy_tensor<uint8_t>(PlaceType::kCPU, PlaceType::kCPU);
+  test_copy_tensor<uint8_t>(PlaceType::kCPU, PlaceType::kGPU);
+  test_copy_tensor<uint8_t>(PlaceType::kGPU, PlaceType::kCPU);
+  test_copy_tensor<uint8_t>(PlaceType::kGPU, PlaceType::kGPU);
+}
+
+TEST(CopyTensor, bool_cpu_to_cpu) {
+  paddle::framework::Scope scope;
+  auto tensor_src = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
+      "tensor_src", PlaceType::kCPU, static_cast<void *>(&scope));
+  auto tensor_dst = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
+      "tensor_dst", PlaceType::kCPU, static_cast<void *>(&scope));
+
+  std::array<bool, 6> data_src;
+  data_src.fill(true);
+  tensor_src->Reshape({2, 3});
+  tensor_src->CopyFromCpu(data_src.data());
+
+  std::array<bool, 4> data_dst;
+  data_dst.fill(false);
+  tensor_dst->Reshape({2, 2});
+  tensor_dst->CopyFromCpu(data_dst.data());
+
+  paddle_infer::contrib::TensorUtils::CopyTensor(tensor_dst.get(), *tensor_src);
+
+  EXPECT_EQ(tensor_dst->shape().size(), (size_t)2);
+  EXPECT_EQ(tensor_dst->shape()[0], 2);
+  EXPECT_EQ(tensor_dst->shape()[1], 3);
+
+  std::array<bool, 6> data_check;
+  data_check.fill(false);
+  tensor_dst->CopyToCpu<bool>(data_check.data());
+
+  for (int i = 0; i < 6; i++) {
+    EXPECT_TRUE(data_check[i] == true);
+  }
+}
+
+TEST(CopyTensor, bool_gpu_to_gpu) {
+  paddle::framework::Scope scope;
+  auto tensor_src = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
+      "tensor_src", PlaceType::kGPU, static_cast<void *>(&scope));
+  auto tensor_dst = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
+      "tensor_dst", PlaceType::kGPU, static_cast<void *>(&scope));
+
+  std::array<bool, 6> data_src;
+  data_src.fill(true);
+  tensor_src->Reshape({2, 3});
+  tensor_src->CopyFromCpu(data_src.data());
+
+  std::array<bool, 4> data_dst;
+  data_dst.fill(false);
+  tensor_dst->Reshape({2, 2});
+  tensor_dst->CopyFromCpu(data_dst.data());
+
+  paddle_infer::contrib::TensorUtils::CopyTensor(tensor_dst.get(), *tensor_src);
+
+  EXPECT_EQ(tensor_dst->shape().size(), (size_t)2);
+  EXPECT_EQ(tensor_dst->shape()[0], 2);
+  EXPECT_EQ(tensor_dst->shape()[1], 3);
+
+  std::array<bool, 6> data_check;
+  data_check.fill(false);
+  tensor_dst->CopyToCpu<bool>(data_check.data());
+
+  for (int i = 0; i < 6; i++) {
+    EXPECT_TRUE(data_check[i] == true);
+  }
+}
+
+TEST(CopyTensor, bool_gpu_to_cpu) {
+  paddle::framework::Scope scope;
+  auto tensor_src = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
+      "tensor_src", PlaceType::kGPU, static_cast<void *>(&scope));
+  auto tensor_dst = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
+      "tensor_dst", PlaceType::kCPU, static_cast<void *>(&scope));
+
+  std::array<bool, 6> data_src;
+  data_src.fill(true);
+  tensor_src->Reshape({2, 3});
+  tensor_src->CopyFromCpu(data_src.data());
+
+  std::array<bool, 4> data_dst;
+  data_dst.fill(false);
+  tensor_dst->Reshape({2, 2});
+  tensor_dst->CopyFromCpu(data_dst.data());
+
+  paddle_infer::contrib::TensorUtils::CopyTensor(tensor_dst.get(), *tensor_src);
+
+  EXPECT_EQ(tensor_dst->shape().size(), (size_t)2);
+  EXPECT_EQ(tensor_dst->shape()[0], 2);
+  EXPECT_EQ(tensor_dst->shape()[1], 3);
+
+  std::array<bool, 6> data_check;
+  data_check.fill(false);
+  tensor_dst->CopyToCpu<bool>(data_check.data());
+
+  for (int i = 0; i < 6; i++) {
+    EXPECT_TRUE(data_check[i] == true);
+  }
+}
+
+TEST(CopyTensor, bool_cpu_to_gpu) {
+  paddle::framework::Scope scope;
+  auto tensor_src = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
+      "tensor_src", PlaceType::kCPU, static_cast<void *>(&scope));
+  auto tensor_dst = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
+      "tensor_dst", PlaceType::kGPU, static_cast<void *>(&scope));
+
+  std::array<bool, 6> data_src;
+  data_src.fill(true);
+  tensor_src->Reshape({2, 3});
+  tensor_src->CopyFromCpu(data_src.data());
+
+  std::array<bool, 4> data_dst;
+  data_dst.fill(false);
+  tensor_dst->Reshape({2, 2});
+  tensor_dst->CopyFromCpu(data_dst.data());
+
+  paddle_infer::contrib::TensorUtils::CopyTensor(tensor_dst.get(), *tensor_src);
+
+  EXPECT_EQ(tensor_dst->shape().size(), (size_t)2);
+  EXPECT_EQ(tensor_dst->shape()[0], 2);
+  EXPECT_EQ(tensor_dst->shape()[1], 3);
+
+  std::array<bool, 6> data_check{false};
+  data_check.fill(false);
+  tensor_dst->CopyToCpu<bool>(data_check.data());
+
+  for (int i = 0; i < 6; i++) {
+    EXPECT_TRUE(data_check[i] == true);
+  }
+}
+
+TEST(CopyTensor, float16_cpu_to_cpu) {
+  paddle::framework::Scope scope;
+  auto tensor_src = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
+      "tensor_src", PlaceType::kCPU, static_cast<void *>(&scope));
+  auto tensor_dst = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
+      "tensor_dst", PlaceType::kCPU, static_cast<void *>(&scope));
+
+  using phi::dtype::float16;
+  std::vector<float16> data_src(6, float16(1.0));
+  tensor_src->Reshape({2, 3});
+  tensor_src->CopyFromCpu(data_src.data());
+
+  std::vector<float16> data_dst(4, float16(2.0));
+  tensor_dst->Reshape({2, 2});
+  tensor_dst->CopyFromCpu(data_dst.data());
+
+  paddle_infer::contrib::TensorUtils::CopyTensor(tensor_dst.get(), *tensor_src);
+
+  EXPECT_EQ(tensor_dst->shape().size(), (size_t)2);
+  EXPECT_EQ(tensor_dst->shape()[0], 2);
+  EXPECT_EQ(tensor_dst->shape()[1], 3);
+
+  std::vector<float16> data_check(6, float16(2.0));
+  tensor_dst->CopyToCpu<float16>(data_check.data());
+
+  for (int i = 0; i < 6; i++) {
+    EXPECT_TRUE(data_check[i] == float16(1.0));
+  }
+}
+
+TEST(CopyTensor, float16_gpu_to_gpu) {
+  paddle::framework::Scope scope;
+  auto tensor_src = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
+      "tensor_src", PlaceType::kGPU, static_cast<void *>(&scope));
+  auto tensor_dst = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
+      "tensor_dst", PlaceType::kGPU, static_cast<void *>(&scope));
+
+  using phi::dtype::float16;
+  std::vector<float16> data_src(6, float16(1.0));
+  tensor_src->Reshape({2, 3});
+  tensor_src->CopyFromCpu(data_src.data());
+
+  std::vector<float16> data_dst(4, float16(2.0));
+  tensor_dst->Reshape({2, 2});
+  tensor_dst->CopyFromCpu(data_dst.data());
+
+  paddle_infer::contrib::TensorUtils::CopyTensor(tensor_dst.get(), *tensor_src);
+
+  EXPECT_EQ(tensor_dst->shape().size(), (size_t)2);
+  EXPECT_EQ(tensor_dst->shape()[0], 2);
+  EXPECT_EQ(tensor_dst->shape()[1], 3);
+
+  std::vector<float16> data_check(6, float16(2.0));
+  tensor_dst->CopyToCpu<float16>(data_check.data());
+
+  for (int i = 0; i < 6; i++) {
+    EXPECT_TRUE(data_check[i] == float16(1.0));
+  }
+}
+
+TEST(CopyTensor, float16_cpu_to_gpu) {
+  paddle::framework::Scope scope;
+  auto tensor_src = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
+      "tensor_src", PlaceType::kCPU, static_cast<void *>(&scope));
+  auto tensor_dst = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
+      "tensor_dst", PlaceType::kGPU, static_cast<void *>(&scope));
+
+  using phi::dtype::float16;
+  std::vector<float16> data_src(6, float16(1.0));
+  tensor_src->Reshape({2, 3});
+  tensor_src->CopyFromCpu(data_src.data());
+
+  std::vector<float16> data_dst(4, float16(2.0));
+  tensor_dst->Reshape({2, 2});
+  tensor_dst->CopyFromCpu(data_dst.data());
+
+  paddle_infer::contrib::TensorUtils::CopyTensor(tensor_dst.get(), *tensor_src);
+
+  EXPECT_EQ(tensor_dst->shape().size(), (size_t)2);
+  EXPECT_EQ(tensor_dst->shape()[0], 2);
+  EXPECT_EQ(tensor_dst->shape()[1], 3);
+
+  std::vector<float16> data_check(6, float16(2.0));
+  tensor_dst->CopyToCpu<float16>(data_check.data());
+
+  for (int i = 0; i < 6; i++) {
+    EXPECT_TRUE(data_check[i] == float16(1.0));
+  }
+}
+
+TEST(CopyTensor, float16_gpu_to_cpu) {
+  paddle::framework::Scope scope;
+  auto tensor_src = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
+      "tensor_src", PlaceType::kGPU, static_cast<void *>(&scope));
+  auto tensor_dst = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
+      "tensor_dst", PlaceType::kCPU, static_cast<void *>(&scope));
+
+  using phi::dtype::float16;
+  std::vector<float16> data_src(6, float16(1.0));
+  tensor_src->Reshape({2, 3});
+  tensor_src->CopyFromCpu(data_src.data());
+
+  std::vector<float16> data_dst(4, float16(2.0));
+  tensor_dst->Reshape({2, 2});
+  tensor_dst->CopyFromCpu(data_dst.data());
+
+  paddle_infer::contrib::TensorUtils::CopyTensor(tensor_dst.get(), *tensor_src);
+
+  EXPECT_EQ(tensor_dst->shape().size(), (size_t)2);
+  EXPECT_EQ(tensor_dst->shape()[0], 2);
+  EXPECT_EQ(tensor_dst->shape()[1], 3);
+
+  std::vector<float16> data_check(6, float16(2.0));
+  tensor_dst->CopyToCpu<float16>(data_check.data());
+
+  for (int i = 0; i < 6; i++) {
+    EXPECT_TRUE(data_check[i] == float16(1.0));
+  }
+}
+
+TEST(CopyTensor, async_stream) {
+  paddle::framework::Scope scope;
+  auto tensor_src = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
+      "tensor_src", PlaceType::kGPU, static_cast<void *>(&scope));
+  auto tensor_dst = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
+      "tensor_dst", PlaceType::kGPU, static_cast<void *>(&scope));
+
+  std::vector<float> data_src(6, 1.0);
+  tensor_src->Reshape({2, 3});
+  tensor_src->CopyFromCpu(data_src.data());
+
+  std::vector<float> data_dst(4, 2.0);
+  tensor_dst->Reshape({2, 2});
+  tensor_dst->CopyFromCpu(data_dst.data());
+
+  cudaStream_t stream;
+  paddle_infer::contrib::TensorUtils::CopyTensorAsync(
+      tensor_dst.get(), *tensor_src, static_cast<void *>(&stream));
+
+  EXPECT_EQ(tensor_dst->shape().size(), (size_t)2);
+  EXPECT_EQ(tensor_dst->shape()[0], 2);
+  EXPECT_EQ(tensor_dst->shape()[1], 3);
+
+  cudaStreamSynchronize(stream);
+
+  std::vector<float> data_check(6, 1.0);
+  tensor_dst->CopyToCpu<float>(data_check.data());
+
+  for (int i = 0; i < 6; i++) {
+    EXPECT_NEAR(data_check[i], static_cast<float>(1.0), 1e-5);
+  }
+}
+
+TEST(CopyTensor, async_callback) {
+  paddle::framework::Scope scope;
+  auto tensor_src = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
+      "tensor_src", PlaceType::kCPU, static_cast<void *>(&scope));
+  auto tensor_dst = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
+      "tensor_dst", PlaceType::kGPU, static_cast<void *>(&scope));
+
+  std::vector<float> data_src(6, 1.0);
+  tensor_src->Reshape({2, 3});
+  tensor_src->CopyFromCpu(data_src.data());
+
+  std::vector<float> data_dst(4, 2.0);
+  tensor_dst->Reshape({2, 2});
+  tensor_dst->CopyFromCpu(data_dst.data());
+
+  paddle_infer::contrib::TensorUtils::CopyTensorAsync(
+      tensor_dst.get(),
+      *tensor_src,
+      [](void *cb_params) {
+        Tensor *tensor = static_cast<Tensor *>(cb_params);
+        EXPECT_EQ(tensor->shape().size(), (size_t)2);
+        EXPECT_EQ(tensor->shape()[0], 2);
+        EXPECT_EQ(tensor->shape()[1], 3);
+      },
+      static_cast<void *>(&(*tensor_dst)));
+
+  cudaDeviceSynchronize();
+}
+
+}  // namespace paddle_infer

test/cpp/inference/api/paddle_infer_api_errors_tester.cc

@@ -0,0 +1,90 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/common/flags.h"
+#include "paddle/fluid/inference/api/paddle_infer_contrib.h"
+#include "paddle/fluid/platform/enforce.h"
+
+namespace paddle_infer {
+namespace contrib {
+
+TEST(Status, ctor) { CHECK(Status::OK().ok()); }
+
+struct FakeException {
+  void pd_exception(int a) const {
+    PADDLE_ENFORCE_NE(a,
+                      a,
+                      common::errors::InvalidArgument(
+                          "This is a preset error message used to verify "
+                          "whether the exception meets expectations: %d, %d.",
+                          a,
+                          a));
+  }
+  [[noreturn]] void base_exception() const { throw std::exception(); }
+  void no_exception() const noexcept {}
+};
+
+TEST(Status, pd_exception) {
+  FakeException e;
+  Status status = get_status([&]() { e.pd_exception(1); });
+  PADDLE_ENFORCE_EQ(
+      status.ok(),
+      false,
+      common::errors::PreconditionNotMet("Status should not be OK."));
+  PADDLE_ENFORCE_EQ(
+      status == status,
+      true,
+      common::errors::PreconditionNotMet("Status should be equal to itself."));
+  PADDLE_ENFORCE_EQ(status != status,
+                    false,
+                    common::errors::PreconditionNotMet(
+                        "Status should not be different from itself."));
+  PADDLE_ENFORCE_EQ(
+      status.code(),
+      common::ErrorCode::INVALID_ARGUMENT + 1,
+      common::errors::InvalidArgument(
+          "Required status.code() should be equal to INVALID_ARGUMENT + 1. "));
+  LOG(INFO) << status.error_message();
+}
+
+TEST(Status, basic_exception) {
+  FakeException e;
+  Status status;
+  status = get_status([&]() { e.base_exception(); });
+  PADDLE_ENFORCE_EQ(
+      status.ok(),
+      false,
+      common::errors::PreconditionNotMet("Status should not be OK."));
+  LOG(INFO) << status.error_message();
+}
+
+TEST(Status, no_exception) {
+  FakeException e;
+  Status status;
+  status = get_status([&]() { e.no_exception(); });
+  PADDLE_ENFORCE_EQ(status.ok(),
+                    true,
+                    common::errors::PreconditionNotMet("Status should be OK."));
+}
+
+TEST(Status, copy) {
+  Status status;
+  Status status_1(status);
+  status_1 = status;
+}
+
+}  // namespace contrib
+}  // namespace paddle_infer

test/cpp/inference/api/paddle_infer_api_test.cc

@@ -0,0 +1,97 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include <functional>
+#include <numeric>
+
+#include "paddle/common/flags.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle_infer {
+
+TEST(Predictor, use_gpu) {
+  LOG(INFO) << GetVersion();
+  UpdateDllFlag("conv_workspace_size_limit", "4000");
+  std::string model_dir = FLAGS_infer_model + "/model";
+  Config config;
+  config.EnableNewIR(false);
+  config.SetModel(model_dir + "/model", model_dir + "/params");
+  config.EnableUseGpu(100, 0);
+
+  auto predictor = CreatePredictor(config);
+  auto pred_clone = predictor->Clone();
+
+  std::vector<int> in_shape = {1, 3, 318, 318};
+  int in_num = std::accumulate(
+      in_shape.begin(), in_shape.end(), 1, std::multiplies<int>());
+
+  std::vector<float> input(in_num, 0);
+
+  auto input_names = predictor->GetInputNames();
+  auto input_t = predictor->GetInputHandle(input_names[0]);
+
+  input_t->Reshape(in_shape);
+  input_t->CopyFromCpu(input.data());
+  predictor->Run();
+
+  auto output_names = predictor->GetOutputNames();
+  auto output_t = predictor->GetOutputHandle(output_names[0]);
+  std::vector<int> output_shape = output_t->shape();
+  int out_num = std::accumulate(
+      output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
+
+  std::vector<float> out_data;
+  out_data.resize(out_num);
+  output_t->CopyToCpu(out_data.data());
+  predictor->ClearIntermediateTensor();
+}
+
+TEST(PredictorPool, basic) {
+  LOG(INFO) << GetVersion();
+  UpdateDllFlag("conv_workspace_size_limit", "4000");
+  std::string model_dir = FLAGS_infer_model + "/model";
+  Config config;
+  config.EnableNewIR(false);
+  config.SetModel(model_dir + "/model", model_dir + "/params");
+  config.EnableUseGpu(100, 0);
+
+  services::PredictorPool pred_pool(config, 4);
+  auto pred = pred_pool.Retrieve(2);
+
+  std::vector<int> in_shape = {1, 3, 318, 318};
+  int in_num = std::accumulate(
+      in_shape.begin(), in_shape.end(), 1, std::multiplies<int>());
+  std::vector<float> input(in_num, 0);
+
+  auto in_names = pred->GetInputNames();
+  auto input_t = pred->GetInputHandle(in_names[0]);
+  input_t->name();
+  input_t->Reshape(in_shape);
+  input_t->CopyFromCpu(input.data());
+  pred->Run();
+  auto out_names = pred->GetOutputNames();
+  auto output_t = pred->GetOutputHandle(out_names[0]);
+  auto out_type = output_t->type();
+  LOG(INFO) << GetNumBytesOfDataType(out_type);
+  if (out_type == DataType::FLOAT32) {
+    PlaceType place;
+    int size;
+    output_t->data<float>(&place, &size);
+  }
+}
+
+}  // namespace paddle_infer

test/cpp/inference/api/test_detection_dataset_preprocess.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import unittest
+
+
+class Test_Preprocess(unittest.TestCase):
+    def test_local_convert(self):
+        os.system("python full_pascalvoc_test_preprocess.py --choice=local")
+
+    def test_online_convert(self):
+        os.system(
+            "python full_pascalvoc_test_preprocess.py --choice=VOC_test_2007"
+        )
+
+
+if __name__ == '__main__':
+    unittest.main()

test/cpp/inference/api/trt_cascade_rcnn_test.cc

@@ -0,0 +1,62 @@
+/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include "paddle/common/flags.h"
+#include "test/cpp/inference/api/trt_test_helper.h"
+
+namespace paddle {
+namespace inference {
+
+TEST(TensorRT, cascade_rcnn) {
+  std::string model_dir = FLAGS_infer_model + "/cascade_rcnn";
+  AnalysisConfig config;
+  int batch_size = 1;
+  config.EnableNewIR(false);
+  config.EnableUseGpu(100, 0);
+  config.SetModel(model_dir + "/model", model_dir + "/params");
+  config.EnableTensorRtEngine(
+      1 << 30, batch_size, 40, AnalysisConfig::Precision::kFloat32, false);
+
+  auto predictor = CreatePaddlePredictor(config);
+
+  int channels = 3;
+  int height = 640;
+  int width = 640;
+  int input_num = batch_size * channels * height * width;
+  float *input = new float[input_num];
+  memset(input, 1.0, input_num * sizeof(float));
+
+  float *im_shape = new float[3];
+  im_shape[0] = 3.0;
+  im_shape[1] = 640.0;
+  im_shape[2] = 640.0;
+
+  auto input_names = predictor->GetInputNames();
+
+  auto input_t = predictor->GetInputTensor(input_names[0]);
+  input_t->Reshape({batch_size, channels, height, width});
+  input_t->copy_from_cpu(input);
+
+  auto input_t1 = predictor->GetInputTensor(input_names[1]);
+  input_t1->Reshape({batch_size, 3});
+  input_t1->copy_from_cpu(im_shape);
+
+  ASSERT_TRUE(predictor->ZeroCopyRun());
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/trt_disable_tensorrt_half_ops_test.cc

@@ -0,0 +1,43 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include "test/cpp/inference/api/trt_test_helper.h"
+
+namespace paddle {
+namespace inference {
+
+TEST(TensorRT, disable_tensorrt_half_ops) {
+  std::string model_dir = FLAGS_infer_model + "/resnet50";
+  AnalysisConfig config;
+  config.SetModel(model_dir);
+  config.EnableUseGpu(100, 0);
+  config.EnableTensorRtEngine(
+      1 << 30, 1, 5, AnalysisConfig::Precision::kHalf, false, false);
+
+  paddle_infer::experimental::InternalUtils::DisableTensorRtHalfOps(&config,
+                                                                    {"conv2d"});
+
+  std::vector<std::vector<PaddleTensor>> inputs_all;
+  auto predictor = CreatePaddlePredictor(config);
+  SetFakeImageInput(&inputs_all, model_dir, false, "__model__", "");
+
+  std::vector<PaddleTensor> outputs;
+  for (auto &input : inputs_all) {
+    ASSERT_TRUE(predictor->Run(input, &outputs));
+    predictor->ClearIntermediateTensor();
+  }
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/trt_dynamic_shape_ernie_fp16_serialize_deserialize_test.cc

@@ -0,0 +1,38 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <dirent.h>
+#ifndef _WIN32
+#include <unistd.h>
+#else  // headers below are substitute of unistd.h in windows
+#include <io.h>
+#include <process.h>
+#endif
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include "paddle/common/flags.h"
+#include "test/cpp/inference/api/trt_dynamic_shape_ernie_serialize_deserialize_test.h"
+
+namespace paddle {
+namespace inference {
+
+TEST(AnalysisPredictor, fp16) {
+  std::vector<float> result = {0.59923654, 0.21923761, 0.18152587};
+  trt_ernie(true, result);
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/trt_dynamic_shape_ernie_serialize_deserialize_test.cc

@@ -0,0 +1,40 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <dirent.h>
+#ifndef _WIN32
+#include <unistd.h>
+#else  // headers below are substitute of unistd.h in windows
+#include <io.h>
+#include <process.h>
+#endif
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include "paddle/common/flags.h"
+#include "test/cpp/inference/api/trt_dynamic_shape_ernie_serialize_deserialize_test.h"
+
+namespace paddle {
+namespace inference {
+#if defined _WIN32
+#else
+TEST(AnalysisPredictor, no_fp16) {
+  std::vector<float> result = {0.597841, 0.219972, 0.182187};
+  trt_ernie(false, result);
+}
+#endif
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/trt_dynamic_shape_ernie_serialize_deserialize_test.h

@@ -0,0 +1,153 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+#pragma once
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+#ifndef _WIN32
+#include <unistd.h>
+#else  // headers below are substitute of unistd.h in windows
+#include <io.h>
+#include <process.h>
+#endif
+#include <functional>
+#include <map>
+#include <string>
+#include <vector>
+
+#include "paddle/common/flags.h"
+#include "test/cpp/inference/api/trt_test_helper.h"
+
+namespace paddle {
+namespace inference {
+
+static void run(const AnalysisConfig& config, std::vector<float>* out_data) {
+  auto predictor = CreatePaddlePredictor(config);
+  auto input_names = predictor->GetInputNames();
+
+  int run_batch = 1;
+  const int run_seq_len = 128;
+
+  std::vector<int32_t> tmp_input;
+  std::vector<float> tmp_four_input;
+  tmp_input.reserve(run_batch * run_seq_len);
+  tmp_four_input.reserve(run_batch * run_seq_len);
+
+  int32_t i0[run_seq_len] = {
+      1,    3558, 4,   75,  491, 89, 340, 313, 93,   4,   255,   10, 75,    321,
+      4095, 1902, 4,   134, 49,  75, 311, 14,  44,   178, 543,   15, 12043, 2,
+      75,   201,  340, 9,   14,  44, 486, 218, 1140, 279, 12043, 2};
+  int32_t i1[run_seq_len] = {
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+  int32_t i2[run_seq_len] = {0,  1,  2,  3,  4,  5,  6,  7,  8,  9,
+                             10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
+                             20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
+                             30, 31, 32, 33, 34, 35, 36, 37, 38, 39};
+  float i3[run_seq_len] = {1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
+                           1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
+                           1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
+                           1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0};
+
+  // first input
+  auto input_t = predictor->GetInputTensor(input_names[0]);
+  input_t->Reshape({run_batch, run_seq_len, 1});
+  input_t->copy_from_cpu(i0);
+
+  // second input
+  auto input_t2 = predictor->GetInputTensor(input_names[1]);
+  input_t2->Reshape({run_batch, run_seq_len, 1});
+  input_t2->copy_from_cpu(i1);
+
+  // third input.
+  auto input_t3 = predictor->GetInputTensor(input_names[2]);
+  input_t3->Reshape({run_batch, run_seq_len, 1});
+  input_t3->copy_from_cpu(i2);
+
+  auto input_t4 = predictor->GetInputTensor(input_names[3]);
+  input_t4->Reshape({run_batch, run_seq_len, 1});
+  input_t4->copy_from_cpu(i3);
+
+  ASSERT_TRUE(predictor->ZeroCopyRun());
+
+  auto output_names = predictor->GetOutputNames();
+  auto output_t = predictor->GetOutputTensor(output_names[0]);
+  std::vector<int> output_shape = output_t->shape();
+  int out_num = std::accumulate(
+      output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
+  out_data->resize(out_num);
+  output_t->copy_to_cpu(out_data->data());
+}
+
+static void trt_ernie(bool with_fp16, std::vector<float> result) {
+  AnalysisConfig config;
+  std::string model_dir = FLAGS_infer_model;
+  // Delete serialization cache to perform serialization first rather than
+  // deserialization.
+  std::string opt_cache_dir = FLAGS_infer_model + "/opt_cache";
+  delete_cache_files(opt_cache_dir);
+  config.SetOptimCacheDir(opt_cache_dir);
+
+  SetConfig(&config, model_dir, true /* use_gpu */);
+
+  int batch = 1;
+  int min_seq_len = 1;
+  int max_seq_len = 128;
+  int opt_seq_len = 128;
+
+  std::vector<int> min_shape = {batch, min_seq_len, 1};
+  std::vector<int> max_shape = {batch, max_seq_len, 1};
+  std::vector<int> opt_shape = {batch, opt_seq_len, 1};
+  // Set the input's min, max, opt shape
+  std::map<std::string, std::vector<int>> min_input_shape = {
+      {"read_file_0.tmp_0", min_shape},
+      {"read_file_0.tmp_1", min_shape},
+      {"read_file_0.tmp_2", min_shape},
+      {"read_file_0.tmp_4", min_shape}};
+  std::map<std::string, std::vector<int>> max_input_shape = {
+      {"read_file_0.tmp_0", max_shape},
+      {"read_file_0.tmp_1", max_shape},
+      {"read_file_0.tmp_2", max_shape},
+      {"read_file_0.tmp_4", max_shape}};
+  std::map<std::string, std::vector<int>> opt_input_shape = {
+      {"read_file_0.tmp_0", opt_shape},
+      {"read_file_0.tmp_1", opt_shape},
+      {"read_file_0.tmp_2", opt_shape},
+      {"read_file_0.tmp_4", opt_shape}};
+
+  auto precision = AnalysisConfig::Precision::kFloat32;
+  if (with_fp16) {
+    precision = AnalysisConfig::Precision::kHalf;
+  }
+
+  config.EnableTensorRtEngine(1 << 30, 1, 5, precision, true, false);
+  config.SetTRTDynamicShapeInfo(
+      min_input_shape, max_input_shape, opt_input_shape);
+  paddle_infer::experimental::InternalUtils::SetTransformerMaskid(
+      &config, "read_file_0.tmp_4");
+  AnalysisConfig* config_deser = new AnalysisConfig(config);
+
+  std::vector<float> out_data;
+  run(config, &out_data);         // serialize
+  run(*config_deser, &out_data);  // deserialize
+  for (size_t i = 0; i < out_data.size(); i++) {
+    EXPECT_NEAR(result[i], out_data[i], 1e-2);
+  }
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/trt_dynamic_shape_ernie_test.cc

@@ -0,0 +1,443 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+#include "paddle/common/flags.h"
+
+#include "paddle/common/enforce.h"
+#include "paddle/fluid/inference/tensorrt/helper.h"
+#include "test/cpp/inference/api/trt_test_helper.h"
+
+namespace paddle {
+namespace inference {
+
+void run(const AnalysisConfig& config, std::vector<float>* out_data, int bs) {
+#if !defined(_WIN32)
+  setenv("NVIDIA_TF32_OVERRIDE", "0", 1);
+#endif
+  auto predictor = CreatePaddlePredictor(config);
+  auto input_names = predictor->GetInputNames();
+
+  int run_batch = bs;
+  const int run_seq_len = 128;
+  size_t len = run_batch * run_seq_len;
+
+  std::array<int32_t, 128> i0_bs1 = {
+      1,    3558, 4,   75,  491, 89, 340, 313, 93,   4,   255,   10, 75,    321,
+      4095, 1902, 4,   134, 49,  75, 311, 14,  44,   178, 543,   15, 12043, 2,
+      75,   201,  340, 9,   14,  44, 486, 218, 1140, 279, 12043, 2};
+  std::array<int32_t, 128> i1_bs1 = {
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+  std::array<int32_t, 128> i2_bs1 = {0,  1,  2,  3,  4,  5,  6,  7,  8,  9,
+                                     10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
+                                     20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
+                                     30, 31, 32, 33, 34, 35, 36, 37, 38, 39};
+  std::array<float, 128> i3_bs1 = {
+      1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
+      1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
+      1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0};
+  std::vector<int32_t> i0_data(len), i1_data(len), i2_data(len);
+  std::vector<float> i3_data(len);
+
+  for (size_t i = 0; i < len; i++) {
+    i0_data[i] = i0_bs1[i % run_seq_len];
+    i1_data[i] = i1_bs1[i % run_seq_len];
+    i2_data[i] = i2_bs1[i % run_seq_len];
+    i3_data[i] = i3_bs1[i % run_seq_len];
+  }
+  // first input
+  auto input_t = predictor->GetInputTensor(input_names[0]);
+  input_t->Reshape({run_batch, run_seq_len, 1});
+  input_t->copy_from_cpu(i0_data.data());
+
+  // second input
+  auto input_t2 = predictor->GetInputTensor(input_names[1]);
+  input_t2->Reshape({run_batch, run_seq_len, 1});
+  input_t2->copy_from_cpu(i1_data.data());
+
+  // third input.
+  auto input_t3 = predictor->GetInputTensor(input_names[2]);
+  input_t3->Reshape({run_batch, run_seq_len, 1});
+  input_t3->copy_from_cpu(i2_data.data());
+
+  auto input_t4 = predictor->GetInputTensor(input_names[3]);
+  input_t4->Reshape({run_batch, run_seq_len, 1});
+  input_t4->copy_from_cpu(i3_data.data());
+
+  ASSERT_TRUE(predictor->ZeroCopyRun());
+
+  auto output_names = predictor->GetOutputNames();
+  auto output_t = predictor->GetOutputTensor(output_names[0]);
+  std::vector<int> output_shape = output_t->shape();
+  int out_num = std::accumulate(
+      output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
+  out_data->resize(out_num);
+  output_t->copy_to_cpu(out_data->data());
+}
+
+void trt_ernie(bool with_fp16,
+               std::vector<float> result,
+               float near_tolerance,
+               int batch_size = 1) {
+  AnalysisConfig config;
+  std::string model_dir = FLAGS_infer_model;
+  SetConfig(&config, model_dir, true);
+
+  int batch = 32;
+  int min_seq_len = 1;
+  int max_seq_len = 128;
+  int opt_seq_len = 128;
+
+  std::vector<int> min_shape = {1, min_seq_len, 1};
+  std::vector<int> max_shape = {batch, max_seq_len, 1};
+  std::vector<int> opt_shape = {batch, opt_seq_len, 1};
+  // Set the input's min, max, opt shape
+  std::map<std::string, std::vector<int>> min_input_shape = {
+      {"read_file_0.tmp_0", min_shape},
+      {"read_file_0.tmp_1", min_shape},
+      {"read_file_0.tmp_2", min_shape},
+      {"read_file_0.tmp_4", min_shape}};
+  std::map<std::string, std::vector<int>> max_input_shape = {
+      {"read_file_0.tmp_0", max_shape},
+      {"read_file_0.tmp_1", max_shape},
+      {"read_file_0.tmp_2", max_shape},
+      {"read_file_0.tmp_4", max_shape}};
+  std::map<std::string, std::vector<int>> opt_input_shape = {
+      {"read_file_0.tmp_0", opt_shape},
+      {"read_file_0.tmp_1", opt_shape},
+      {"read_file_0.tmp_2", opt_shape},
+      {"read_file_0.tmp_4", opt_shape}};
+
+  auto precision = AnalysisConfig::Precision::kFloat32;
+  if (with_fp16) {
+    precision = AnalysisConfig::Precision::kHalf;
+  }
+  config.EnableTensorRtEngine(1 << 30, 1, 5, precision, false, false);
+  config.SetTRTDynamicShapeInfo(
+      min_input_shape, max_input_shape, opt_input_shape);
+  paddle_infer::experimental::InternalUtils::SetTransformerMaskid(
+      &config, "read_file_0.tmp_4");
+  std::vector<float> out_data;
+  run(config, &out_data, batch_size);
+
+  for (size_t i = 0; i < out_data.size(); i++) {
+    EXPECT_NEAR(result[i], out_data[i], near_tolerance);
+  }
+}
+
+TEST(AnalysisPredictor, no_fp16) {
+  std::vector<float> result = {0.597841, 0.219972, 0.182187};
+  trt_ernie(false, result, 1e-4);
+}
+
+TEST(AnalysisPredictor, fp16) {
+#ifdef PADDLE_WITH_CUDA
+  std::vector<float> result = {0.598, 0.219, 0.182};
+  trt_ernie(true, result, 4e-3);
+#endif
+}
+
+TEST(AnalysisPredictor, no_fp16_bs2) {
+  std::vector<float> result = {
+      0.597841, 0.219972, 0.182187, 0.597841, 0.219972, 0.182187};
+  trt_ernie(false, result, 1e-4, 2);
+}
+
+TEST(AnalysisPredictor, fp16_bs2) {
+#ifdef PADDLE_WITH_CUDA
+  std::vector<float> result = {0.598, 0.219, 0.182, 0.598, 0.219, 0.182};
+  trt_ernie(true, result, 4e-3, 2);
+#endif
+}
+
+// ernie_varlen
+std::shared_ptr<paddle_infer::Predictor> InitPredictor() {
+  paddle_infer::Config config;
+  config.SetModel(FLAGS_infer_model);
+
+  config.EnableUseGpu(100, 0);
+
+  // Open the memory optim.
+  config.EnableMemoryOptim();
+
+  int max_batch = 32;
+  int max_single_seq_len = 128;
+  int opt_single_seq_len = 64;
+  int min_batch_seq_len = 1;
+  int max_batch_seq_len = 512;
+  int opt_batch_seq_len = 256;
+
+  std::string input_name0 = "read_file_0.tmp_0";
+  std::string input_name1 = "read_file_0.tmp_1";
+  std::string input_name2 = "read_file_0.tmp_2";
+  std::string input_name3 = "read_file_0.tmp_4";
+
+  std::vector<int> min_shape = {min_batch_seq_len};
+  std::vector<int> max_shape = {max_batch_seq_len};
+  std::vector<int> opt_shape = {opt_batch_seq_len};
+  // Set the input's min, max, opt shape
+  std::map<std::string, std::vector<int>> min_input_shape = {
+      {input_name0, min_shape},
+      {input_name1, min_shape},
+      {input_name2, {1}},
+      {input_name3, {1, 1, 1}}};
+  std::map<std::string, std::vector<int>> max_input_shape = {
+      {input_name0, max_shape},
+      {input_name1, max_shape},
+      {input_name2, {max_batch + 1}},
+      {input_name3, {1, max_single_seq_len, 1}}};
+  std::map<std::string, std::vector<int>> opt_input_shape = {
+      {input_name0, opt_shape},
+      {input_name1, opt_shape},
+      {input_name2, {max_batch + 1}},
+      {input_name3, {1, opt_single_seq_len, 1}}};
+
+  // only kHalf supported
+  config.EnableTensorRtEngine(
+      1 << 30, 1, 5, paddle_infer::Config::Precision::kHalf, false, false);
+  // erinie varlen must be used with dynamic shape
+  config.SetTRTDynamicShapeInfo(
+      min_input_shape, max_input_shape, opt_input_shape);
+  // erinie varlen must be used with oss
+  config.EnableVarseqlen();
+  paddle_infer::experimental::InternalUtils::SetTransformerPosid(&config,
+                                                                 input_name2);
+  paddle_infer::experimental::InternalUtils::SetTransformerMaskid(&config,
+                                                                  input_name3);
+
+  return paddle_infer::CreatePredictor(config);
+}
+
+void run(paddle_infer::Predictor* predictor, std::vector<float>* out_data) {
+#if !defined(_WIN32)
+  setenv("NVIDIA_TF32_OVERRIDE", "0", 1);
+#endif
+  const int run_batch = 2;
+  const int run_seq_len = 71;
+  const int max_seq_len = 128;
+  std::vector<int32_t> i1 = {
+      // sentence 1
+      1,
+      3558,
+      4,
+      75,
+      491,
+      89,
+      340,
+      313,
+      93,
+      4,
+      255,
+      10,
+      75,
+      321,
+      4095,
+      1902,
+      4,
+      134,
+      49,
+      75,
+      311,
+      14,
+      44,
+      178,
+      543,
+      15,
+      12043,
+      2,
+      75,
+      201,
+      340,
+      9,
+      14,
+      44,
+      486,
+      218,
+      1140,
+      279,
+      12043,
+      2,
+      // sentence 2
+      101,
+      2054,
+      2234,
+      2046,
+      2486,
+      2044,
+      1996,
+      2047,
+      4552,
+      2001,
+      9536,
+      1029,
+      102,
+      2004,
+      1997,
+      2008,
+      2154,
+      1010,
+      1996,
+      2047,
+      4552,
+      9536,
+      2075,
+      1996,
+      2117,
+      3072,
+      2234,
+      2046,
+      2486,
+      1012,
+      102,
+  };
+  std::vector<int32_t> i2 = {// sentence 1
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             1,
+                             1,
+                             1,
+                             1,
+                             1,
+                             1,
+                             1,
+                             1,
+                             1,
+                             1,
+                             1,
+                             1,
+                             // sentence 2
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             0,
+                             1,
+                             1,
+                             1,
+                             1,
+                             1,
+                             1,
+                             1,
+                             1,
+                             1,
+                             1,
+                             1,
+                             1,
+                             1,
+                             1,
+                             1,
+                             1,
+                             1,
+                             1};
+  // shape info of this batch
+  std::vector<int32_t> i3 = {0, 40, 71};
+  // max_seq_len represents the max sentence length of all the sentences, only
+  // length of
+  // input i4 is useful, data means nothing.
+  std::vector<float> i4(max_seq_len, 0);
+
+  auto input_names = predictor->GetInputNames();
+  // first input
+  auto input_t1 = predictor->GetInputHandle(input_names[0]);
+  input_t1->Reshape({run_seq_len});
+  input_t1->CopyFromCpu(i1.data());
+
+  // second input
+  auto input_t2 = predictor->GetInputHandle(input_names[1]);
+  input_t2->Reshape({run_seq_len});
+  input_t2->CopyFromCpu(i2.data());
+
+  // third input
+  auto input_t3 = predictor->GetInputHandle(input_names[2]);
+  input_t3->Reshape({run_batch + 1});
+  input_t3->CopyFromCpu(i3.data());
+
+  // fourth input
+  auto input_t4 = predictor->GetInputHandle(input_names[3]);
+  input_t4->Reshape({1, max_seq_len, 1});
+  input_t4->CopyFromCpu(i4.data());
+
+  PADDLE_ENFORCE(
+      predictor->Run(),
+      common::errors::PreconditionNotMet("Predictor is not runnable"));
+
+  auto output_names = predictor->GetOutputNames();
+  auto output_t = predictor->GetOutputHandle(output_names[0]);
+  std::vector<int> output_shape = output_t->shape();
+  int out_num = std::accumulate(
+      output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
+  out_data->resize(out_num);
+  output_t->CopyToCpu(out_data->data());
+
+  return;
+}
+
+TEST(AnalysisPredictor, ernie_varlen) {
+#if IS_TRT_VERSION_GE(7234)
+  if (platform::GetGPUComputeCapability(platform::GetCurrentDeviceId()) >= 75) {
+    auto predictor = InitPredictor();
+    std::vector<float> out_data;
+    run(predictor.get(), &out_data);
+    std::vector<float> ref_data{
+        0.59814, 0.219882, 0.181978, 0.359796, 0.577414, 0.0627908};
+    float near_tolerance = 4e-3;
+    for (size_t i = 0; i < out_data.size(); i++) {
+      EXPECT_NEAR(ref_data[i], out_data[i], near_tolerance);
+    }
+  }
+#endif
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/trt_dynamic_shape_test.cc

@@ -0,0 +1,307 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include "paddle/common/flags.h"
+#include "test/cpp/inference/api/trt_test_helper.h"
+
+namespace paddle {
+namespace inference {
+
+void TestDynamic(bool with_dynamic = true,
+                 bool delete_cache = true,
+                 bool delete_conv_bn = false) {
+  std::string model_dir =
+      FLAGS_infer_model + "/conv_bn_swish_split_gelu/conv_bn_swish_split_gelu";
+
+  std::string opt_cache_dir = model_dir + "/my_cache";
+  if (delete_cache) {
+    delete_cache_files(opt_cache_dir);
+  }
+
+  AnalysisConfig config;
+  config.EnableNewIR(false);
+  config.EnableUseGpu(100, 0);
+  std::string buffer_prog, buffer_param;
+  ReadBinaryFile(model_dir + "/model", &buffer_prog);
+  ReadBinaryFile(model_dir + "/params", &buffer_param);
+  config.SetModelBuffer(&buffer_prog[0],
+                        buffer_prog.size(),
+                        &buffer_param[0],
+                        buffer_param.size());
+  config.SetOptimCacheDir(opt_cache_dir);
+
+  // Set the input's min, max, opt shape
+  config.EnableTensorRtEngine(
+      1 << 30, 1, 1, AnalysisConfig::Precision::kFloat32, true, true);
+  if (delete_conv_bn) {
+    config.pass_builder()->DeletePass("conv_bn_fuse_pass");
+  }
+  if (with_dynamic) {
+    std::map<std::string, std::vector<int>> min_input_shape = {
+        {"image", {1, 1, 3, 3}}};
+    std::map<std::string, std::vector<int>> max_input_shape = {
+        {"image", {1, 1, 10, 10}}};
+    std::map<std::string, std::vector<int>> opt_input_shape = {
+        {"image", {1, 1, 3, 3}}};
+
+    config.SetTRTDynamicShapeInfo(
+        min_input_shape, max_input_shape, opt_input_shape);
+  }
+  auto predictor = CreatePaddlePredictor(config);
+  auto input_names = predictor->GetInputNames();
+  int channels = 1;
+  int height = 3;
+  int width = 3;
+  int input_num = channels * height * width * 1;
+
+  float *input = new float[input_num];
+  memset(input, 0, input_num * sizeof(float));
+  auto input_t = predictor->GetInputTensor(input_names[0]);
+  input_t->Reshape({1, channels, height, width});
+  input_t->copy_from_cpu(input);
+
+  ASSERT_TRUE(predictor->ZeroCopyRun());
+
+  std::vector<float> out_data;
+  auto output_names = predictor->GetOutputNames();
+  auto output_t = predictor->GetOutputTensor(output_names[0]);
+  std::vector<int> output_shape = output_t->shape();
+  int out_num = std::accumulate(
+      output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
+  out_data.resize(out_num);
+  output_t->copy_to_cpu(out_data.data());
+}
+
+void TestDynamic2() {
+  std::string model_dir =
+      FLAGS_infer_model + "/complex_model_dynamic/complex_model_dynamic2";
+  AnalysisConfig config;
+  config.EnableUseGpu(100, 0);
+  config.SetModel(model_dir + "/model", model_dir + "/params");
+  // Set the input's min, max, opt shape
+  int batch_size = 1;
+  std::map<std::string, std::vector<int>> min_input_shape = {
+      {"image", {1, 3, 3, 3}}, {"in1", {1, 2, 1, 1}}, {"in2", {1, 2, 1, 1}}};
+  std::map<std::string, std::vector<int>> max_input_shape = {
+      {"image", {1, 3, 10, 10}}, {"in1", {1, 2, 1, 1}}, {"in2", {1, 2, 1, 1}}};
+  std::map<std::string, std::vector<int>> opt_input_shape = {
+      {"image", {1, 3, 5, 5}}, {"in1", {1, 2, 1, 1}}, {"in2", {1, 2, 1, 1}}};
+  config.EnableTensorRtEngine(
+      1 << 30, batch_size, 0, AnalysisConfig::Precision::kFloat32, false, true);
+
+  config.SetTRTDynamicShapeInfo(
+      min_input_shape, max_input_shape, opt_input_shape);
+
+  auto predictor = CreatePaddlePredictor(config);
+  int channels = 3;
+  int height = 5;
+  int width = 5;
+  int input_num = channels * height * width * 1;
+
+  float *input = new float[input_num];
+  memset(input, 0, input_num * sizeof(float));
+  auto input_names = predictor->GetInputNames();
+  auto input_t = predictor->GetInputTensor(input_names[0]);
+  input_t->Reshape({batch_size, channels, height, width});
+  input_t->copy_from_cpu(input);
+
+  auto input_t1 = predictor->GetInputTensor(input_names[1]);
+  input_t1->Reshape({batch_size, 2, 1, 1});
+  std::vector<float> first;
+  for (int i = 0; i < batch_size * 2; i++) first.push_back(1.0);
+  input_t1->copy_from_cpu(first.data());
+
+  auto input_t2 = predictor->GetInputTensor(input_names[2]);
+  input_t2->Reshape({batch_size, 2, 1, 1});
+  input_t2->copy_from_cpu(first.data());
+
+  ASSERT_TRUE(predictor->ZeroCopyRun());
+
+  std::vector<float> out_data;
+  auto output_names = predictor->GetOutputNames();
+  auto output_t = predictor->GetOutputTensor(output_names[0]);
+  std::vector<int> output_shape = output_t->shape();
+  int out_num = std::accumulate(
+      output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
+  out_data.resize(out_num);
+  output_t->copy_to_cpu(out_data.data());
+  std::vector<float> result = {0.617728, 1.63504, 2.15771, 0.535556};
+  for (size_t i = 0; i < out_data.size(); i++) {
+    EXPECT_NEAR(result[i], out_data[i], 1e-5);
+  }
+}
+
+void TestTunedDynamic() {
+  std::string model_dir =
+      FLAGS_infer_model + "/complex_model_dynamic/complex_model_dynamic2";
+  AnalysisConfig config_tuned;
+  const std::string shape_range = "shape_range.pbtxt";
+  config_tuned.EnableUseGpu(100, 0);
+  config_tuned.SetModel(model_dir + "/model", model_dir + "/params");
+  config_tuned.CollectShapeRangeInfo(shape_range);
+
+  int batch_size = 1;
+  auto predictor_tuned = CreatePaddlePredictor(config_tuned);
+
+  auto check_func = [batch_size](PaddlePredictor *predictor) {
+    int channels = 3;
+    int height = 5;
+    int width = 5;
+    int input_num = channels * height * width * 1;
+
+    float *input = new float[input_num];
+    memset(input, 0, input_num * sizeof(float));
+    auto input_names = predictor->GetInputNames();
+    auto input_t = predictor->GetInputTensor(input_names[0]);
+    input_t->Reshape({batch_size, channels, height, width});
+    input_t->copy_from_cpu(input);
+
+    auto input_t1 = predictor->GetInputTensor(input_names[1]);
+    input_t1->Reshape({batch_size, 2, 1, 1});
+    std::vector<float> first;
+    for (int i = 0; i < batch_size * 2; i++) first.push_back(1.0);
+    input_t1->copy_from_cpu(first.data());
+
+    auto input_t2 = predictor->GetInputTensor(input_names[2]);
+    input_t2->Reshape({batch_size, 2, 1, 1});
+    input_t2->copy_from_cpu(first.data());
+
+    ASSERT_TRUE(predictor->ZeroCopyRun());
+
+    std::vector<float> out_data;
+    auto output_names = predictor->GetOutputNames();
+    auto output_t = predictor->GetOutputTensor(output_names[0]);
+    std::vector<int> output_shape = output_t->shape();
+    int out_num = std::accumulate(
+        output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
+    out_data.resize(out_num);
+    output_t->copy_to_cpu(out_data.data());
+  };
+  check_func(predictor_tuned.get());
+  predictor_tuned.reset(nullptr);
+
+  // check tuned_dynamic_shape
+  AnalysisConfig config;
+  config.EnableUseGpu(100, 0);
+  std::string cache_dir = "tuned_cache";
+  config.SetOptimCacheDir(cache_dir);
+  delete_cache_files(cache_dir);
+  config.SetModel(model_dir + "/model", model_dir + "/params");
+  config.EnableTunedTensorRtDynamicShape(shape_range, true);
+  config.EnableTensorRtEngine(
+      1 << 30, batch_size, 0, AnalysisConfig::Precision::kFloat32, true, false);
+  auto test_predictor = CreatePaddlePredictor(config);
+  check_func(test_predictor.get());
+}
+
+void TestDynamicClone(bool with_dynamic = true,
+                      bool delete_cache = true,
+                      bool delete_conv_bn = false) {
+  std::string model_dir =
+      FLAGS_infer_model + "/conv_bn_swish_split_gelu/conv_bn_swish_split_gelu";
+
+  std::string opt_cache_dir = model_dir + "/my_cache";
+  if (delete_cache) {
+    delete_cache_files(opt_cache_dir);
+  }
+
+  AnalysisConfig config;
+  config.EnableUseGpu(100, 0);
+  std::string buffer_prog, buffer_param;
+  ReadBinaryFile(model_dir + "/model", &buffer_prog);
+  ReadBinaryFile(model_dir + "/params", &buffer_param);
+  config.SetModelBuffer(&buffer_prog[0],
+                        buffer_prog.size(),
+                        &buffer_param[0],
+                        buffer_param.size());
+  config.SetOptimCacheDir(opt_cache_dir);
+
+  // Set the input's min, max, opt shape
+  config.EnableTensorRtEngine(
+      1 << 30, 1, 1, AnalysisConfig::Precision::kFloat32, false, false);
+  if (delete_conv_bn) {
+    config.pass_builder()->DeletePass("conv_bn_fuse_pass");
+  }
+  if (with_dynamic) {
+    std::map<std::string, std::vector<int>> min_input_shape = {
+        {"image", {1, 1, 3, 3}}};
+    std::map<std::string, std::vector<int>> max_input_shape = {
+        {"image", {1, 1, 10, 10}}};
+    std::map<std::string, std::vector<int>> opt_input_shape = {
+        {"image", {1, 1, 3, 3}}};
+
+    config.SetTRTDynamicShapeInfo(
+        min_input_shape, max_input_shape, opt_input_shape);
+  }
+  auto predictor = CreatePaddlePredictor(config);
+  auto input_names = predictor->GetInputNames();
+  int channels = 1;
+  int height = 3;
+  int width = 3;
+  int input_num = channels * height * width * 1;
+
+  float *input = new float[input_num];
+  memset(input, 0, input_num * sizeof(float));
+  auto input_t = predictor->GetInputTensor(input_names[0]);
+  input_t->Reshape({1, channels, height, width});
+  input_t->copy_from_cpu(input);
+
+  ASSERT_TRUE(predictor->ZeroCopyRun());
+
+  std::vector<float> out_data;
+  auto output_names = predictor->GetOutputNames();
+  auto output_t = predictor->GetOutputTensor(output_names[0]);
+  std::vector<int> output_shape = output_t->shape();
+  int out_num = std::accumulate(
+      output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
+  out_data.resize(out_num);
+  output_t->copy_to_cpu(out_data.data());
+
+  auto predictor2 = predictor->Clone();
+  auto input_t2 = predictor2->GetInputTensor(input_names[0]);
+  input_t2->Reshape({1, channels, height, width});
+  input_t2->copy_from_cpu(input);
+
+  ASSERT_TRUE(predictor2->ZeroCopyRun());
+
+  std::vector<float> out_data2;
+  auto output_t2 = predictor2->GetOutputTensor(output_names[0]);
+  std::vector<int> output_shape2 = output_t2->shape();
+  int out_num2 = std::accumulate(
+      output_shape2.begin(), output_shape2.end(), 1, std::multiplies<int>());
+  out_data2.resize(out_num2);
+  output_t2->copy_to_cpu(out_data2.data());
+  ASSERT_TRUE(out_data2.size() == out_data.size());
+  for (size_t i = 0; i < out_data.size(); i++) {
+    EXPECT_NEAR(out_data2[i], out_data[i], 1e-5);
+  }
+}
+
+TEST(AnalysisPredictor, trt_dynamic) { TestDynamic(true); }
+TEST(AnalysisPredictor, trt_memory_serialize) {
+  // serialize
+  TestDynamic(true, true, true);
+  // deserialize
+  TestDynamic(true, false, true);
+}
+TEST(AnalysisPredictor, trt_dynamic2) { TestDynamic2(); }
+
+TEST(AnalysisPredictor, trt_tuned_dynamic) { TestTunedDynamic(); }
+TEST(AnalysisPredictor, trt_dynamic_clone) { TestDynamicClone(); }
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/trt_instance_norm_converter_test.cc

@@ -0,0 +1,49 @@
+/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include "paddle/common/flags.h"
+#include "test/cpp/inference/api/trt_test_helper.h"
+
+namespace paddle {
+namespace inference {
+
+TEST(TensorRT, instance_norm) {
+  std::string model_dir = FLAGS_infer_model + "/instance_norm";
+  AnalysisConfig config;
+  int batch_size = 4;
+  config.EnableUseGpu(100, 0);
+  config.SetModel(model_dir);
+  config.EnableTensorRtEngine(
+      1 << 20, batch_size, 0, AnalysisConfig::Precision::kFloat32, false);
+
+  auto predictor = CreatePaddlePredictor(config);
+
+  int length = 4;
+  int input_num = batch_size * length;
+  float *input = new float[input_num];
+  memset(input, 1.0, input_num * sizeof(float));
+
+  auto input_names = predictor->GetInputNames();
+  auto input_t = predictor->GetInputTensor(input_names[0]);
+  input_t->Reshape({batch_size, length});
+  input_t->copy_from_cpu(input);
+
+  ASSERT_TRUE(predictor->ZeroCopyRun());
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/trt_mark_trt_engine_outputs_test.cc

@@ -0,0 +1,43 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include "test/cpp/inference/api/trt_test_helper.h"
+
+namespace paddle {
+namespace inference {
+
+TEST(TensorRT, mark_trt_engine_outputs) {
+  std::string model_dir = FLAGS_infer_model + "/resnet50";
+  AnalysisConfig config;
+  config.SetModel(model_dir);
+  config.EnableUseGpu(100, 0);
+  config.EnableTensorRtEngine(
+      1 << 30, 1, 5, AnalysisConfig::Precision::kFloat32, false, false);
+  // The name of the tensor that needs to be marked
+  std::vector<std::string> markOutput = {"pool2d_0.tmp_0"};
+  config.MarkTrtEngineOutputs(markOutput);
+
+  std::vector<std::vector<PaddleTensor>> inputs_all;
+  auto predictor = CreatePaddlePredictor(config);
+  SetFakeImageInput(&inputs_all, model_dir, false, "__model__", "");
+
+  std::vector<PaddleTensor> outputs;
+  for (auto &input : inputs_all) {
+    ASSERT_TRUE(predictor->Run(input, &outputs));
+    predictor->ClearIntermediateTensor();
+  }
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/trt_mobilenet_test.cc

@@ -0,0 +1,72 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include <functional>
+#include <numeric>
+
+#include "paddle/common/flags.h"
+#include "test/cpp/inference/api/trt_test_helper.h"
+
+namespace paddle_infer {
+TEST(PredictorPool, use_gpu) {
+  std::string model_dir = FLAGS_infer_model + "/" + "mobilenet";
+  Config config;
+  config.EnableUseGpu(100, 0);
+  config.SetModel(model_dir);
+  config.EnableTensorRtEngine();
+  config.Exp_DisableTensorRtOPs({"fc"});
+  config.EnableTensorRtDLA(0);
+  services::PredictorPool pred_pool(config, 1);
+
+  auto predictor = pred_pool.Retrieve(0);
+  auto input_names = predictor->GetInputNames();
+  auto input_t = predictor->GetInputHandle(input_names[0]);
+  std::vector<int> in_shape = {1, 3, 224, 224};
+  int in_num = std::accumulate(
+      in_shape.begin(), in_shape.end(), 1, std::multiplies<int>());
+
+  std::vector<float> input(in_num, 0);
+  input_t->Reshape(in_shape);
+  input_t->CopyFromCpu(input.data());
+  predictor->Run();
+}
+
+TEST(PredictorPool, use_trt_cuda_graph) {
+  std::string model_dir = FLAGS_infer_model + "/" + "mobilenet";
+  Config config;
+  config.EnableUseGpu(100, 0);
+  config.SetModel(model_dir);
+  config.EnableTensorRtEngine(
+      1 << 20, 1, 3, PrecisionType::kFloat32, false, false, true);
+  config.Exp_DisableTensorRtOPs({"fc"});
+  config.EnableTensorRtDLA(0);
+  services::PredictorPool pred_pool(config, 1);
+
+  auto predictor = pred_pool.Retrieve(0);
+  auto input_names = predictor->GetInputNames();
+  auto input_t = predictor->GetInputHandle(input_names[0]);
+  std::vector<int> in_shape = {1, 3, 224, 224};
+  int in_num = std::accumulate(
+      in_shape.begin(), in_shape.end(), 1, std::multiplies<int>());
+
+  std::vector<float> input(in_num, 0);
+  input_t->Reshape(in_shape);
+  input_t->CopyFromCpu(input.data());
+  predictor->Run();
+}
+
+}  // namespace paddle_infer

test/cpp/inference/api/trt_quant_int8_test.cc

@@ -0,0 +1,68 @@
+/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include <numeric>
+
+#include "paddle/common/flags.h"
+#include "test/cpp/inference/api/trt_test_helper.h"
+
+namespace paddle {
+namespace inference {
+
+TEST(quant_int8, resnet50) {
+  std::string model_dir = FLAGS_infer_model;
+  AnalysisConfig config;
+  config.EnableUseGpu(1000, 0);
+  config.SetModel(model_dir);
+  config.EnableTensorRtEngine(
+      1 << 30, 1, 1, AnalysisConfig::Precision::kInt8, false, false);
+  std::map<std::string, std::vector<int>> min_input_shape = {
+      {"image", {1, 1, 3, 3}}};
+  std::map<std::string, std::vector<int>> max_input_shape = {
+      {"image", {1, 1, 10, 10}}};
+  std::map<std::string, std::vector<int>> opt_input_shape = {
+      {"image", {1, 1, 3, 3}}};
+
+  config.SetTRTDynamicShapeInfo(
+      min_input_shape, max_input_shape, opt_input_shape);
+  auto predictor = CreatePaddlePredictor(config);
+  auto input_names = predictor->GetInputNames();
+  int channels = 1;
+  int height = 3;
+  int width = 3;
+  int input_num = channels * height * width * 1;
+
+  float *input = new float[input_num];
+  memset(input, 0, input_num * sizeof(float));
+  auto input_t = predictor->GetInputTensor(input_names[0]);
+  input_t->Reshape({1, channels, height, width});
+  input_t->copy_from_cpu(input);
+
+  ASSERT_TRUE(predictor->ZeroCopyRun());
+
+  std::vector<float> out_data;
+  auto output_names = predictor->GetOutputNames();
+  auto output_t = predictor->GetOutputTensor(output_names[0]);
+  std::vector<int> output_shape = output_t->shape();
+  int out_num = std::accumulate(
+      output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
+  out_data.resize(out_num);
+  output_t->copy_to_cpu(out_data.data());
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/trt_quant_int8_yolov3_r50_test.cc

@@ -0,0 +1,64 @@
+/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include <numeric>
+
+#include "paddle/common/flags.h"
+#include "test/cpp/inference/api/trt_test_helper.h"
+
+namespace paddle {
+namespace inference {
+
+TEST(quant_int8, yolov3_resnet50) {
+  AnalysisConfig config;
+  config.EnableNewIR(false);
+  config.EnableUseGpu(100, 0);
+  config.SetModel(FLAGS_infer_model + "/model", FLAGS_infer_model + "/params");
+  config.EnableTensorRtEngine(
+      1 << 30, 1, 3, AnalysisConfig::Precision::kInt8, false, false);
+
+  auto predictor = CreatePaddlePredictor(config);
+  auto input_names = predictor->GetInputNames();
+  int channels = 3;
+  int height = 608;
+  int width = 608;
+  int input_num = channels * height * width * 1;
+
+  float *input = new float[input_num];
+  int32_t *im_shape = new int32_t[2];
+  im_shape[0] = 608;
+  im_shape[1] = 608;
+  memset(input, 1.0, input_num * sizeof(float));
+  auto input_t = predictor->GetInputTensor(input_names[0]);
+  input_t->Reshape({1, channels, height, width});
+  input_t->copy_from_cpu(input);
+
+  auto input_t1 = predictor->GetInputTensor(input_names[1]);
+  input_t1->Reshape({1, 2});
+  input_t1->copy_from_cpu(im_shape);
+
+  ASSERT_TRUE(predictor->ZeroCopyRun());
+
+  std::vector<float> out_data;
+  auto output_names = predictor->GetOutputNames();
+  auto output_t = predictor->GetOutputTensor(output_names[0]);
+  std::vector<int> output_shape = output_t->shape();
+  int out_num = std::accumulate(
+      output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
+  out_data.resize(out_num);
+  output_t->copy_to_cpu(out_data.data());
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/trt_rebind_stream_test.cc

@@ -0,0 +1,183 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+#include <thread>
+
+#include "paddle/common/flags.h"
+#include "paddle/fluid/inference/api/paddle_inference_api.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+
+// TODO(inference): This case failed in windows with a SEH error, we need to fix
+// it.
+TEST(ReBindStream_single, use_gpu) {
+  std::string model_dir = FLAGS_infer_model + "/mobilenet";
+  AnalysisConfig config;
+  config.EnableUseGpu(100, 0);
+  config.SetModel(model_dir);
+  config.EnableTensorRtEngine();
+
+  cudaStream_t stream1, stream2, stream3;
+  cudaStreamCreate(&stream1);
+  cudaStreamCreate(&stream2);
+  cudaStreamCreate(&stream3);
+
+  config.SetExecStream(stream1);
+  auto predictor = paddle_infer::CreatePredictor(config);
+  auto x_t = predictor->GetInputHandle("x");
+  x_t->Reshape({1, 3, 224, 224});
+  std::array<float, 3 * 224 * 224> x_data = {0};
+  x_t->CopyFromCpu(x_data.data());
+  ASSERT_TRUE(predictor->Run());
+  cudaDeviceSynchronize();
+  ASSERT_TRUE(paddle_infer::experimental::InternalUtils::RunWithExternalStream(
+      predictor.get(), stream2));
+  cudaDeviceSynchronize();
+  ASSERT_TRUE(paddle_infer::experimental::InternalUtils::RunWithExternalStream(
+      predictor.get(), stream3));
+  cudaDeviceSynchronize();
+}
+
+TEST(ReBindStream_multi, use_gpu) {
+  std::string model_dir = FLAGS_infer_model + "/mobilenet";
+  AnalysisConfig config1;
+  config1.EnableUseGpu(100, 0);
+  config1.SetModel(model_dir);
+  config1.EnableTensorRtEngine();
+  AnalysisConfig config2;
+  config2.EnableUseGpu(100, 0);
+  config2.EnableTensorRtEngine();
+  config2.SetModel(model_dir);
+
+  cudaStream_t stream1, stream2, stream3;
+  cudaStreamCreate(&stream1);
+  cudaStreamCreate(&stream2);
+  cudaStreamCreate(&stream3);
+
+  config1.SetExecStream(stream1);
+  config2.SetExecStream(stream1);
+  auto predictor1 = paddle_infer::CreatePredictor(config1);
+  auto predictor2 = paddle_infer::CreatePredictor(config2);
+
+  std::vector<float> x1(3 * 224 * 224, 1.0);
+  auto x_t1 = predictor1->GetInputHandle("x");
+  x_t1->Reshape({1, 3, 224, 224});
+  x_t1->CopyFromCpu(x1.data());
+  std::vector<float> x2(3 * 224 * 224, 2.0);
+  auto x_t2 = predictor2->GetInputHandle("x");
+  x_t2->Reshape({1, 3, 224, 224});
+  x_t2->CopyFromCpu(x2.data());
+
+  ASSERT_TRUE(predictor1->Run());
+  cudaStreamSynchronize(stream1);
+  ASSERT_TRUE(predictor2->Run());
+  cudaStreamSynchronize(stream1);
+
+  ASSERT_TRUE(paddle_infer::experimental::InternalUtils::RunWithExternalStream(
+      predictor1.get(), stream2));
+  cudaDeviceSynchronize();
+  ASSERT_TRUE(paddle_infer::experimental::InternalUtils::RunWithExternalStream(
+      predictor2.get(), stream2));
+  cudaDeviceSynchronize();
+
+  ASSERT_TRUE(paddle_infer::experimental::InternalUtils::RunWithExternalStream(
+      predictor1.get(), stream3));
+  cudaStreamSynchronize(stream3);
+  ASSERT_TRUE(paddle_infer::experimental::InternalUtils::RunWithExternalStream(
+      predictor2.get(), stream3));
+  cudaStreamSynchronize(stream3);
+}
+
+TEST(SwitchStream_multi, use_gpu) {
+  std::string model_dir = FLAGS_infer_model + "/mobilenet";
+  AnalysisConfig config1;
+  config1.EnableUseGpu(100, 0);
+  config1.SetModel(model_dir);
+  AnalysisConfig config2;
+  config2.EnableUseGpu(100, 0);
+  config2.SetModel(model_dir);
+  AnalysisConfig config3;
+  config3.EnableUseGpu(100, 0);
+  config3.SetModel(model_dir);
+
+  // config1.EnableTensorRtEngine();
+  // config2.EnableTensorRtEngine();
+  // config3.EnableTensorRtEngine();
+
+  cudaStream_t stream1, stream2, stream3;
+  cudaStreamCreate(&stream1);
+  cudaStreamCreate(&stream2);
+  cudaStreamCreate(&stream3);
+
+  config1.SetExecStream(stream1);
+  config2.SetExecStream(stream1);
+  config3.SetExecStream(stream1);
+  auto predictor1 = paddle_infer::CreatePredictor(config1);
+  auto predictor2 = paddle_infer::CreatePredictor(config2);
+  auto predictor3 = paddle_infer::CreatePredictor(config3);
+
+  std::vector<float> x1(3 * 224 * 224, 1.0);
+  auto x_t1 = predictor1->GetInputHandle("x");
+  x_t1->Reshape({1, 3, 224, 224});
+  x_t1->CopyFromCpu(x1.data());
+  std::vector<float> x2(3 * 224 * 224, 2.0);
+  auto x_t2 = predictor2->GetInputHandle("x");
+  x_t2->Reshape({1, 3, 224, 224});
+  x_t2->CopyFromCpu(x2.data());
+  std::vector<float> x3(3 * 224 * 224, 2.5);
+  auto x_t3 = predictor3->GetInputHandle("x");
+  x_t3->Reshape({1, 3, 224, 224});
+  x_t3->CopyFromCpu(x3.data());
+
+  // TODO(wilber): fix.
+  // NOTE: Must run once on master thread, but why?
+  // if remove the code, the unit test fail.
+  ASSERT_TRUE(predictor1->Run());
+  cudaStreamSynchronize(stream1);
+  ASSERT_TRUE(predictor2->Run());
+  cudaStreamSynchronize(stream1);
+  ASSERT_TRUE(predictor3->Run());
+  cudaStreamSynchronize(stream1);
+
+  auto Run = [&](paddle_infer::Predictor* p,
+                 std::vector<cudaStream_t> streams) {
+    for (auto s : streams) {
+      paddle_infer::experimental::InternalUtils::RunWithExternalStream(p, s);
+    }
+  };
+
+  std::thread p1(Run,
+                 predictor1.get(),
+                 std::vector<cudaStream_t>{
+                     stream1, stream2, stream3, stream3, stream2, stream2});
+  std::thread p2(Run,
+                 predictor2.get(),
+                 std::vector<cudaStream_t>{
+                     stream1, stream3, stream1, stream2, stream1, stream3});
+  std::thread p3(Run,
+                 predictor3.get(),
+                 std::vector<cudaStream_t>{
+                     stream1, stream1, stream2, stream3, stream3, stream2});
+  p1.join();
+  p2.join();
+  p3.join();
+  cudaDeviceSynchronize();
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/trt_split_converter_test.cc

@@ -0,0 +1,46 @@
+/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include "paddle/common/flags.h"
+#include "test/cpp/inference/api/trt_test_helper.h"
+
+namespace paddle {
+namespace inference {
+
+TEST(TensorRT, split_converter) {
+  std::string model_dir = FLAGS_infer_model + "/split_converter";
+  std::string opt_cache_dir = model_dir + "/_opt_cache";
+  delete_cache_files(opt_cache_dir);
+
+  AnalysisConfig config;
+  int batch_size = 4;
+  int channels = 4;
+  int height = 4;
+  int width = 4;
+  config.EnableUseGpu(100, 0);
+  config.SetModel(model_dir);
+  config.EnableTensorRtEngine(
+      1 << 20, batch_size, 1, AnalysisConfig::Precision::kInt8, false, true);
+
+  std::map<std::string, std::vector<int>> input_shape;
+  input_shape["x"] = {batch_size, channels, height, width};
+  config.SetTRTDynamicShapeInfo(input_shape, input_shape, input_shape, false);
+  auto predictor = CreatePaddlePredictor(config);
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/trt_test_helper.h

@@ -0,0 +1,178 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+#pragma once
+#include <dirent.h>
+
+#include <string>
+#include <vector>
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/common/flags.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle {
+namespace inference {
+
+PD_DEFINE_bool(use_tensorrt, true, "Test the performance of TensorRT engine.");
+PD_DEFINE_string(prog_filename, "", "Name of model file.");
+PD_DEFINE_string(param_filename, "", "Name of parameters file.");
+
+template <typename ConfigType>
+void SetConfig(ConfigType* config,
+               std::string model_dir,
+               bool use_gpu,
+               bool use_tensorrt = false,
+               int batch_size = -1) {
+  if (!FLAGS_prog_filename.empty() && !FLAGS_param_filename.empty()) {
+    config->prog_file = model_dir + "/" + FLAGS_prog_filename;
+    config->param_file = model_dir + "/" + FLAGS_param_filename;
+  } else {
+    config->model_dir = model_dir;
+  }
+  if (use_gpu) {
+    config->use_gpu = true;
+    config->device = 0;
+    config->fraction_of_gpu_memory = 0.15;
+  }
+}
+
+template <>
+void SetConfig<AnalysisConfig>(AnalysisConfig* config,
+                               std::string model_dir,
+                               bool use_gpu,
+                               bool use_tensorrt,
+                               int batch_size) {
+  if (!FLAGS_prog_filename.empty() && !FLAGS_param_filename.empty()) {
+    config->SetModel(model_dir + "/" + FLAGS_prog_filename,
+                     model_dir + "/" + FLAGS_param_filename);
+  } else {
+    config->SetModel(model_dir);
+  }
+  if (use_gpu) {
+    config->EnableUseGpu(100, 0);
+    if (use_tensorrt) {
+      config->EnableTensorRtEngine(
+          1 << 10, batch_size, 3, AnalysisConfig::Precision::kFloat32, false);
+      config->pass_builder()->DeletePass("conv_bn_fuse_pass");
+      config->pass_builder()->DeletePass("fc_fuse_pass");
+      config->pass_builder()->TurnOnDebug();
+    } else {
+      config->EnableCUDNN();
+      config->SwitchIrOptim();
+    }
+  }
+}
+
+void profile(std::string model_dir, bool use_analysis, bool use_tensorrt) {
+  std::vector<std::vector<PaddleTensor>> inputs_all;
+  if (!FLAGS_prog_filename.empty() && !FLAGS_param_filename.empty()) {
+    SetFakeImageInput(&inputs_all,
+                      model_dir,
+                      true,
+                      FLAGS_prog_filename,
+                      FLAGS_param_filename);
+  } else {
+    SetFakeImageInput(&inputs_all, model_dir, false, "__model__", "");
+  }
+
+  std::vector<std::vector<PaddleTensor>> outputs;
+  if (use_analysis || use_tensorrt) {
+    AnalysisConfig config;
+    config.EnableUseGpu(100, 0);
+    config.pass_builder()->TurnOnDebug();
+    SetConfig<AnalysisConfig>(
+        &config, model_dir, true, use_tensorrt, FLAGS_batch_size);
+    TestPrediction(reinterpret_cast<PaddlePredictor::Config*>(&config),
+                   inputs_all,
+                   &outputs,
+                   FLAGS_num_threads,
+                   true);
+  } else {
+    NativeConfig config;
+    SetConfig<NativeConfig>(&config, model_dir, true, false);
+    TestPrediction(reinterpret_cast<PaddlePredictor::Config*>(&config),
+                   inputs_all,
+                   &outputs,
+                   FLAGS_num_threads,
+                   false);
+  }
+}
+
+void compare(std::string model_dir, bool use_tensorrt) {
+  std::vector<std::vector<PaddleTensor>> inputs_all;
+  if (!FLAGS_prog_filename.empty() && !FLAGS_param_filename.empty()) {
+    SetFakeImageInput(&inputs_all,
+                      model_dir,
+                      true,
+                      FLAGS_prog_filename,
+                      FLAGS_param_filename);
+  } else {
+    SetFakeImageInput(&inputs_all, model_dir, false, "__model__", "");
+  }
+
+  AnalysisConfig analysis_config;
+  SetConfig<AnalysisConfig>(
+      &analysis_config, model_dir, true, use_tensorrt, FLAGS_batch_size);
+  CompareNativeAndAnalysis(
+      reinterpret_cast<const PaddlePredictor::Config*>(&analysis_config),
+      inputs_all);
+}
+
+void compare_continuous_input(std::string model_dir, bool use_tensorrt) {
+  AnalysisConfig analysis_config;
+  SetConfig<AnalysisConfig>(
+      &analysis_config, model_dir, true, use_tensorrt, FLAGS_batch_size);
+  auto config =
+      reinterpret_cast<const PaddlePredictor::Config*>(&analysis_config);
+  auto native_pred = CreateTestPredictor(config, false);
+  auto analysis_pred = CreateTestPredictor(config, true);
+  for (int i = 0; i < 20; i++) {
+    std::vector<std::vector<PaddleTensor>> inputs_all;
+    if (!FLAGS_prog_filename.empty() && !FLAGS_param_filename.empty()) {
+      SetFakeImageInput(&inputs_all,
+                        model_dir,
+                        true,
+                        FLAGS_prog_filename,
+                        FLAGS_param_filename,
+                        nullptr,
+                        i);
+    } else {
+      SetFakeImageInput(
+          &inputs_all, model_dir, false, "__model__", "", nullptr, i);
+    }
+    CompareNativeAndAnalysis(
+        native_pred.get(), analysis_pred.get(), inputs_all);
+  }
+}
+
+void delete_cache_files(std::string path) {
+  DIR* dir = opendir(path.c_str());
+  if (dir == NULL) return;
+  struct dirent* ptr;
+  while ((ptr = readdir(dir)) != NULL) {
+    if (std::strcmp(ptr->d_name, ".") == 0 ||
+        std::strcmp(ptr->d_name, "..") == 0) {
+      continue;
+    } else if (ptr->d_type == 8) {
+      std::string file_rm = path + "/" + ptr->d_name;
+      remove(file_rm.c_str());
+    }
+  }
+  closedir(dir);
+  remove(path.c_str());
+}
+
+}  // namespace inference
+}  // namespace paddle

test/cpp/inference/api/xpu_config_resnet50_test.cc

@@ -0,0 +1,89 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+#include <cmath>
+#include "paddle/common/flags.h"
+#include "test/cpp/inference/api/tester_helper.h"
+
+namespace paddle_infer {
+
+static const std::vector<float> TRUTH_VALUES = {
+    127.779f,  738.165f,  1013.22f,  -438.17f,  366.401f,  927.659f,  736.222f,
+    -633.684f, -329.927f, -430.155f, -633.062f, -146.548f, -1324.28f, -1349.36f,
+    -242.675f, 117.448f,  -801.723f, -391.514f, -404.818f, 454.16f,   515.48f,
+    -133.031f, 69.293f,   590.096f,  -1434.69f, -1070.89f, 307.074f,  400.525f,
+    -316.12f,  -587.125f, -161.056f, 800.363f,  -96.4708f, 748.706f,  868.174f,
+    -447.938f, 112.737f,  1127.2f,   47.4355f,  677.72f,   593.186f,  -336.4f,
+    551.362f,  397.823f,  78.3979f,  -715.398f, 405.969f,  404.256f,  246.019f,
+    -8.42969f, 131.365f,  -648.051f};
+
+void PrepareInput(std::shared_ptr<Predictor> predictor) {
+  const int batch = 1;
+  const int channel = 3;
+  const int height = 318;
+  const int width = 318;
+  const int input_num = batch * channel * height * width;
+  std::vector<float> input(input_num, 1);
+  auto input_names = predictor->GetInputNames();
+  auto input_t = predictor->GetInputHandle(input_names[0]);
+  input_t->Reshape({batch, channel, height, width});
+  input_t->CopyFromCpu(input.data());
+}
+
+void CompareOutput(std::shared_ptr<Predictor> predictor) {
+  auto output_names = predictor->GetOutputNames();
+  auto output_t = predictor->GetOutputHandle(output_names[0]);
+  std::vector<int> output_shape = output_t->shape();
+  size_t out_num = std::accumulate(
+      output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
+
+  std::vector<float> out_data;
+  out_data.resize(out_num);
+  output_t->CopyToCpu(out_data.data());
+
+  float* data_o = out_data.data();
+  for (size_t j = 0; j < out_num; j += 10) {
+    EXPECT_NEAR(
+        (data_o[j] - TRUTH_VALUES[j / 10]) / TRUTH_VALUES[j / 10], 0., 10e-3);
+  }
+}
+
+TEST(xpu_config, inference) {
+  size_t l3_size = 10 * 1024 * 1024;
+  XpuConfig xpu_config;
+  xpu_config.l3_size = l3_size;
+  std::string model_dir = FLAGS_infer_model + "/" + "model";
+  Config config;
+  config.SetModel(model_dir + "/model", model_dir + "/params");
+  config.EnableXpu();
+  config.SetXpuConfig(xpu_config);
+
+  XpuConfig xpu_config_test = config.xpu_config();
+  PADDLE_ENFORCE_EQ(xpu_config_test.l3_size,
+                    l3_size,
+                    common::errors::InvalidArgument(
+                        "xpu_config_test.l3_size %d is different from our "
+                        "expected value l3_size %d.",
+                        xpu_config_test.l3_size,
+                        l3_size));
+
+  auto predictor = CreatePredictor(config);
+  PrepareInput(predictor);
+  predictor->Run();
+  CompareOutput(predictor);
+}
+
+}  // namespace paddle_infer

test/cpp/inference/api/xpu_runtime_config_resnet50_test.cc

@@ -0,0 +1,269 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+#include <cmath>
+#include "paddle/common/enforce.h"
+#include "paddle/common/flags.h"
+#include "test/cpp/inference/api/tester_helper.h"
+#include "xpu/runtime.h"
+#include "xpu/xdnn.h"
+
+namespace paddle_infer {
+
+static const std::vector<float> TRUTH_VALUES = {
+    127.779f,  738.165f,  1013.22f,  -438.17f,  366.401f,  927.659f,  736.222f,
+    -633.684f, -329.927f, -430.155f, -633.062f, -146.548f, -1324.28f, -1349.36f,
+    -242.675f, 117.448f,  -801.723f, -391.514f, -404.818f, 454.16f,   515.48f,
+    -133.031f, 69.293f,   590.096f,  -1434.69f, -1070.89f, 307.074f,  400.525f,
+    -316.12f,  -587.125f, -161.056f, 800.363f,  -96.4708f, 748.706f,  868.174f,
+    -447.938f, 112.737f,  1127.2f,   47.4355f,  677.72f,   593.186f,  -336.4f,
+    551.362f,  397.823f,  78.3979f,  -715.398f, 405.969f,  404.256f,  246.019f,
+    -8.42969f, 131.365f,  -648.051f};
+
+void PrepareInput(std::shared_ptr<Predictor> predictor) {
+  const int batch = 1;
+  const int channel = 3;
+  const int height = 318;
+  const int width = 318;
+  const int input_num = batch * channel * height * width;
+  std::vector<float> input(input_num, 1);
+  auto input_names = predictor->GetInputNames();
+  auto input_t = predictor->GetInputHandle(input_names[0]);
+  input_t->Reshape({batch, channel, height, width});
+  input_t->CopyFromCpu(input.data());
+}
+
+void CompareOutput(std::shared_ptr<Predictor> predictor) {
+  auto output_names = predictor->GetOutputNames();
+  auto output_t = predictor->GetOutputHandle(output_names[0]);
+  std::vector<int> output_shape = output_t->shape();
+  size_t out_num = std::accumulate(
+      output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
+
+  std::vector<float> out_data;
+  out_data.resize(out_num);
+  output_t->CopyToCpu(out_data.data());
+
+  float* data_o = out_data.data();
+  for (size_t j = 0; j < out_num; j += 10) {
+    EXPECT_NEAR(
+        (data_o[j] - TRUTH_VALUES[j / 10]) / TRUTH_VALUES[j / 10], 0., 10e-3);
+  }
+}
+
+Config InferXpuConfig() {
+  std::string model_dir = FLAGS_infer_model + "/" + "model";
+  Config config;
+  config.SetModel(model_dir + "/model", model_dir + "/params");
+  config.EnableXpu();
+  return config;
+}
+
+TEST(resnet50_xpu, basic) {
+  Config config = InferXpuConfig();
+  auto predictor = CreatePredictor(config);
+  PrepareInput(predictor);
+  predictor->Run();
+  CompareOutput(predictor);
+}
+
+#define RUN_WITH_RUNTIME_CONFIG(idx_, config_)                             \
+  Config config##idx_ = InferXpuConfig();                                  \
+  auto predictor##idx_ = CreatePredictor(config##idx_);                    \
+  PrepareInput(predictor##idx_);                                           \
+  experimental::InternalUtils::RunWithRuntimeConfig(predictor##idx_.get(), \
+                                                    &config_);             \
+  CompareOutput(predictor##idx_);                                          \
+  PADDLE_ENFORCE_EQ(                                                       \
+      predictor##idx_->GetExecStream(),                                    \
+      config_.stream,                                                      \
+      common::errors::InvalidArgument(                                     \
+          "predictor##idx_->GetExecStream() is not equal with "            \
+          "config_.stream while predictor##idx_->GetExecStream() "         \
+          "is %d and config_.stream is %d",                                \
+          predictor##idx_->GetExecStream(),                                \
+          config_.stream));
+
+TEST(runtime_stream, null_stream) {
+  experimental::XpuRuntimeConfig xpu_runtime_config;
+  xpu_runtime_config.context = nullptr;
+  xpu_runtime_config.stream = nullptr;
+  xpu_runtime_config.l3_size = 0;
+  xpu_runtime_config.l3_ptr = nullptr;
+  xpu_runtime_config.l3_autotune_size = 0;
+  RUN_WITH_RUNTIME_CONFIG(0, xpu_runtime_config);
+}
+
+TEST(runtime_stream, new_stream) {
+  void* stream = nullptr;
+  xpu_stream_create(&stream);
+  CHECK_NOTNULL(stream);
+  {
+    experimental::XpuRuntimeConfig xpu_runtime_config;
+    xpu_runtime_config.context = nullptr;
+    xpu_runtime_config.stream = stream;
+    xpu_runtime_config.l3_size = 0;
+    xpu_runtime_config.l3_ptr = nullptr;
+    xpu_runtime_config.l3_autotune_size = 0;
+    RUN_WITH_RUNTIME_CONFIG(0, xpu_runtime_config);
+  }
+  xpu_stream_destroy(stream);
+}
+
+TEST(runtime_stream, 2_null_stream) {
+  experimental::XpuRuntimeConfig xpu_runtime_config;
+  xpu_runtime_config.context = nullptr;
+  xpu_runtime_config.stream = nullptr;
+  xpu_runtime_config.l3_size = 0;
+  xpu_runtime_config.l3_ptr = nullptr;
+  xpu_runtime_config.l3_autotune_size = 0;
+  RUN_WITH_RUNTIME_CONFIG(0, xpu_runtime_config);
+  RUN_WITH_RUNTIME_CONFIG(1, xpu_runtime_config);
+}
+
+TEST(runtime_stream, null_and_new_stream) {
+  experimental::XpuRuntimeConfig xpu_runtime_config0;
+  xpu_runtime_config0.context = nullptr;
+  xpu_runtime_config0.stream = nullptr;
+  xpu_runtime_config0.l3_size = 0;
+  xpu_runtime_config0.l3_ptr = nullptr;
+  xpu_runtime_config0.l3_autotune_size = 0;
+  void* stream = nullptr;
+  xpu_stream_create(&stream);
+  CHECK_NOTNULL(stream);
+  {
+    experimental::XpuRuntimeConfig xpu_runtime_config1;
+    xpu_runtime_config1.context = nullptr;
+    xpu_runtime_config1.stream = stream;
+    xpu_runtime_config1.l3_size = 0;
+    xpu_runtime_config1.l3_ptr = nullptr;
+    xpu_runtime_config1.l3_autotune_size = 0;
+    RUN_WITH_RUNTIME_CONFIG(0, xpu_runtime_config0);
+    RUN_WITH_RUNTIME_CONFIG(1, xpu_runtime_config1);
+  }
+  xpu_stream_destroy(stream);
+}
+
+TEST(runtime_stream, 2_new_same_stream) {
+  void* stream = nullptr;
+  xpu_stream_create(&stream);
+  CHECK_NOTNULL(stream);
+  experimental::XpuRuntimeConfig xpu_runtime_config;
+  xpu_runtime_config.context = nullptr;
+  xpu_runtime_config.stream = stream;
+  xpu_runtime_config.l3_size = 0;
+  xpu_runtime_config.l3_ptr = nullptr;
+  xpu_runtime_config.l3_autotune_size = 0;
+  {
+    RUN_WITH_RUNTIME_CONFIG(0, xpu_runtime_config);
+    RUN_WITH_RUNTIME_CONFIG(1, xpu_runtime_config);
+  }
+  xpu_stream_destroy(stream);
+}
+
+TEST(runtime_stream, 2_new_different_stream) {
+  void* stream0 = nullptr;
+  xpu_stream_create(&stream0);
+  CHECK_NOTNULL(stream0);
+  experimental::XpuRuntimeConfig xpu_runtime_config0;
+  xpu_runtime_config0.context = nullptr;
+  xpu_runtime_config0.stream = stream0;
+  xpu_runtime_config0.l3_size = 0;
+  xpu_runtime_config0.l3_ptr = nullptr;
+  xpu_runtime_config0.l3_autotune_size = 0;
+  void* stream1 = nullptr;
+  xpu_stream_create(&stream1);
+  CHECK_NOTNULL(stream1);
+  experimental::XpuRuntimeConfig xpu_runtime_config1;
+  xpu_runtime_config1.context = nullptr;
+  xpu_runtime_config1.stream = stream1;
+  xpu_runtime_config1.l3_size = 0;
+  xpu_runtime_config1.l3_ptr = nullptr;
+  xpu_runtime_config1.l3_autotune_size = 0;
+  {
+    RUN_WITH_RUNTIME_CONFIG(0, xpu_runtime_config0);
+    RUN_WITH_RUNTIME_CONFIG(1, xpu_runtime_config1);
+  }
+  xpu_stream_destroy(stream0);
+  xpu_stream_destroy(stream1);
+}
+
+void RunPredictorWithRuntimeConfig(
+    std::shared_ptr<Predictor> predictor,
+    experimental::XpuRuntimeConfig runtime_config) {
+  PrepareInput(predictor);
+  experimental::InternalUtils::RunWithRuntimeConfig(predictor.get(),
+                                                    &runtime_config);
+  CompareOutput(predictor);
+  PADDLE_ENFORCE_EQ(predictor->GetExecStream(),
+                    runtime_config.stream,
+                    common::errors::InvalidArgument(
+                        "predictor->GetExecStream() is not equal with "
+                        "runtime_config.stream"));
+}
+
+TEST(runtime_stream, 2_thread) {
+  void* stream0 = nullptr;
+  xpu_stream_create(&stream0);
+  CHECK_NOTNULL(stream0);
+  experimental::XpuRuntimeConfig xpu_runtime_config0;
+  xpu_runtime_config0.context = nullptr;
+  xpu_runtime_config0.stream = stream0;
+  xpu_runtime_config0.l3_size = 0;
+  xpu_runtime_config0.l3_ptr = nullptr;
+  xpu_runtime_config0.l3_autotune_size = 0;
+
+  void* stream1 = nullptr;
+  xpu_stream_create(&stream1);
+  CHECK_NOTNULL(stream1);
+  experimental::XpuRuntimeConfig xpu_runtime_config1;
+  xpu_runtime_config1.context = nullptr;
+  xpu_runtime_config1.stream = stream1;
+  xpu_runtime_config1.l3_size = 0;
+  xpu_runtime_config1.l3_ptr = nullptr;
+  xpu_runtime_config1.l3_autotune_size = 0;
+
+  {
+    RUN_WITH_RUNTIME_CONFIG(0, xpu_runtime_config0);
+    RUN_WITH_RUNTIME_CONFIG(1, xpu_runtime_config1);
+    std::thread t0(
+        RunPredictorWithRuntimeConfig, predictor0, xpu_runtime_config0);
+    std::thread t1(
+        RunPredictorWithRuntimeConfig, predictor1, xpu_runtime_config1);
+    t0.join();
+    t1.join();
+  }
+
+  xpu_stream_destroy(stream0);
+  xpu_stream_destroy(stream1);
+}
+
+TEST(runtime_context, new_context) {
+  auto* context = baidu::xpu::api::create_context();
+  CHECK_NOTNULL(context);
+  {
+    experimental::XpuRuntimeConfig xpu_runtime_config;
+    xpu_runtime_config.context = context;
+    xpu_runtime_config.stream = nullptr;
+    xpu_runtime_config.l3_size = 0;
+    xpu_runtime_config.l3_ptr = nullptr;
+    xpu_runtime_config.l3_autotune_size = 0;
+    RUN_WITH_RUNTIME_CONFIG(0, xpu_runtime_config);
+  }
+  baidu::xpu::api::destroy_context(context);
+}
+
+}  // namespace paddle_infer

test/cpp/inference/infer_ut/CMakeLists.txt

@@ -0,0 +1,353 @@
+cmake_minimum_required(VERSION 3.5)
+project(cpp_inference_demo CXX C)
+option(WITH_MKL "Compile demo with MKL/OpenBlas support, default use MKL." ON)
+option(WITH_GPU "Compile demo with GPU/CPU, default use CPU." OFF)
+option(WITH_STATIC_LIB
+       "Compile demo with static/shared library, default use static." OFF)
+option(USE_TENSORRT "Compile demo with TensorRT." OFF)
+option(WITH_GTEST "Compile demo with GTEST" OFF)
+option(WITH_ONNXRUNTIME "Compile demo with ONNXRuntime" OFF)
+
+if(NOT WITH_STATIC_LIB)
+  add_definitions("-DPADDLE_WITH_SHARED_LIB")
+else()
+  # PD_INFER_DECL is mainly used to set the dllimport/dllexport attribute in dynamic library mode.
+  # Set it to empty in static library mode to avoid compilation issues.
+  add_definitions("/DPD_INFER_DECL=")
+endif()
+
+macro(safe_set_static_flag)
+  foreach(flag_var
+          CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
+          CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
+    if(${flag_var} MATCHES "/MD")
+      string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
+    endif()
+  endforeach()
+endmacro()
+
+if(NOT DEFINED PADDLE_LIB)
+  message(
+    FATAL_ERROR "please set PADDLE_LIB with -DPADDLE_LIB=/path/paddle/lib")
+endif()
+if(NOT DEFINED DEMO_NAME)
+  message(FATAL_ERROR "please set DEMO_NAME with -DDEMO_NAME=demo_name")
+endif()
+
+include_directories("${PADDLE_LIB}/")
+set(PADDLE_LIB_THIRD_PARTY_PATH "${PADDLE_LIB}/third_party/install/")
+include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}protobuf/include")
+include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}glog/include")
+include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}gflags/include")
+include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}xxhash/include")
+include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}cryptopp/include")
+
+link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}protobuf/lib")
+link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}glog/lib")
+link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}gflags/lib")
+link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}xxhash/lib")
+link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}cryptopp/lib")
+link_directories("${PADDLE_LIB}/paddle/lib")
+if(WITH_ONNXRUNTIME)
+  include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}onnxruntime/include")
+  include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}paddle2onnx/include")
+
+  link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}onnxruntime/lib")
+  link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}paddle2onnx/lib")
+endif()
+
+if(WIN32)
+  add_definitions("/DGOOGLE_GLOG_DLL_DECL=")
+  option(MSVC_STATIC_CRT "use static C Runtime library by default" ON)
+  if(MSVC_STATIC_CRT)
+    if(WITH_MKL)
+      set(FLAG_OPENMP "/openmp")
+    endif()
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4244 /wd4530")
+    set(CMAKE_C_FLAGS_DEBUG
+        "${CMAKE_C_FLAGS_DEBUG} /bigobj /MTd ${FLAG_OPENMP}")
+    set(CMAKE_C_FLAGS_RELEASE
+        "${CMAKE_C_FLAGS_RELEASE} /bigobj /MT ${FLAG_OPENMP}")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4530")
+    set(CMAKE_CXX_FLAGS_DEBUG
+        "${CMAKE_CXX_FLAGS_DEBUG} /bigobj /MTd ${FLAG_OPENMP}")
+    set(CMAKE_CXX_FLAGS_RELEASE
+        "${CMAKE_CXX_FLAGS_RELEASE} /bigobj /MT ${FLAG_OPENMP}")
+    safe_set_static_flag()
+    if(WITH_STATIC_LIB)
+      add_definitions(-DSTATIC_LIB)
+    endif()
+  endif()
+else()
+  if(WITH_MKL)
+    set(FLAG_OPENMP "-fopenmp")
+  endif()
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 ${FLAG_OPENMP}")
+endif()
+
+if(WITH_GPU)
+  if(NOT WIN32)
+    add_definitions("-DPADDLE_WITH_GPU")
+    set(CUDA_LIB
+        "/usr/local/cuda/lib64/"
+        CACHE STRING "CUDA Library")
+    include_directories("${CUDA_LIB}/../include")
+  else()
+    set(CUDA_LIB
+        ""
+        CACHE STRING "CUDA_LIB")
+    if("${CUDA_LIB}" STREQUAL "")
+      set(CUDA_LIB "$ENV{CUDA_PATH}\\lib\\x64")
+    endif()
+    message(STATUS "Current CUDA lib path: ${CUDA_LIB}")
+  endif()
+endif()
+
+if(USE_TENSORRT AND WITH_GPU)
+  set(TENSORRT_ROOT
+      ""
+      CACHE STRING "The root directory of TensorRT library")
+  if("${TENSORRT_ROOT}" STREQUAL "")
+    message(
+      FATAL_ERROR
+        "The TENSORRT_ROOT is empty, you must assign it a value with CMake command. Such as: -DTENSORRT_ROOT=TENSORRT_ROOT_PATH "
+    )
+  endif()
+  set(TENSORRT_INCLUDE_DIR ${TENSORRT_ROOT}/include)
+  set(TENSORRT_LIB_DIR ${TENSORRT_ROOT}/lib)
+  file(READ ${TENSORRT_INCLUDE_DIR}/NvInfer.h TENSORRT_VERSION_FILE_CONTENTS)
+  string(REGEX MATCH "define NV_TENSORRT_MAJOR +([0-9]+)"
+               TENSORRT_MAJOR_VERSION "${TENSORRT_VERSION_FILE_CONTENTS}")
+  string(REGEX MATCH "define NV_TENSORRT_MINOR +([0-9]+)"
+               TENSORRT_MINOR_VERSION "${TENSORRT_VERSION_FILE_CONTENTS}")
+  string(REGEX MATCH "define NV_TENSORRT_PATCH +([0-9]+)"
+               TENSORRT_PATCH_VERSION "${TENSORRT_VERSION_FILE_CONTENTS}")
+  string(REGEX MATCH "define NV_TENSORRT_BUILD +([0-9]+)"
+               TENSORRT_BUILD_VERSION "${TENSORRT_VERSION_FILE_CONTENTS}")
+  if("${TENSORRT_MAJOR_VERSION}" STREQUAL "")
+    file(READ ${TENSORRT_INCLUDE_DIR}/NvInferVersion.h
+         TENSORRT_VERSION_FILE_CONTENTS)
+    string(REGEX MATCH "define NV_TENSORRT_MAJOR +([0-9]+)"
+                 TENSORRT_MAJOR_VERSION "${TENSORRT_VERSION_FILE_CONTENTS}")
+    string(REGEX MATCH "define NV_TENSORRT_MINOR +([0-9]+)"
+                 TENSORRT_MINOR_VERSION "${TENSORRT_VERSION_FILE_CONTENTS}")
+    string(REGEX MATCH "define NV_TENSORRT_PATCH +([0-9]+)"
+                 TENSORRT_PATCH_VERSION "${TENSORRT_VERSION_FILE_CONTENTS}")
+    string(REGEX MATCH "define NV_TENSORRT_BUILD +([0-9]+)"
+                 TENSORRT_BUILD_VERSION "${TENSORRT_VERSION_FILE_CONTENTS}")
+  endif()
+  if("${TENSORRT_MAJOR_VERSION}" STREQUAL "")
+    message(SEND_ERROR "Failed to detect TensorRT version.")
+  endif()
+  string(REGEX REPLACE "define NV_TENSORRT_MAJOR +([0-9]+)" "\\1"
+                       TENSORRT_MAJOR_VERSION "${TENSORRT_MAJOR_VERSION}")
+  string(REGEX REPLACE "define NV_TENSORRT_MINOR +([0-9]+)" "\\1"
+                       TENSORRT_MINOR_VERSION "${TENSORRT_MINOR_VERSION}")
+  string(REGEX REPLACE "define NV_TENSORRT_PATCH +([0-9]+)" "\\1"
+                       TENSORRT_PATCH_VERSION "${TENSORRT_PATCH_VERSION}")
+  string(REGEX REPLACE "define NV_TENSORRT_BUILD +([0-9]+)" "\\1"
+                       TENSORRT_BUILD_VERSION "${TENSORRT_BUILD_VERSION}")
+  message(
+    STATUS
+      "Current TensorRT header is ${TENSORRT_INCLUDE_DIR}/NvInfer.h. "
+      "Current TensorRT version is v${TENSORRT_MAJOR_VERSION}.${TENSORRT_MINOR_VERSION}.${TENSORRT_PATCH_VERSION}.${TENSORRT_BUILD_VERSION} "
+  )
+  include_directories("${TENSORRT_INCLUDE_DIR}")
+  link_directories("${TENSORRT_LIB_DIR}")
+  add_compile_definitions(NV_TENSORRT_MAJOR=${TENSORRT_MAJOR_VERSION})
+  add_compile_definitions(NV_TENSORRT_MINOR=${TENSORRT_MINOR_VERSION})
+  add_compile_definitions(NV_TENSORRT_PATCH=${TENSORRT_PATCH_VERSION})
+  add_compile_definitions(NV_TENSORRT_BUILD=${TENSORRT_BUILD_VERSION})
+endif()
+
+if(WITH_MKL)
+  set(MATH_LIB_PATH "${PADDLE_LIB_THIRD_PARTY_PATH}mklml")
+  include_directories("${MATH_LIB_PATH}/include")
+  if(WIN32)
+    set(MATH_LIB ${MATH_LIB_PATH}/lib/mklml${CMAKE_STATIC_LIBRARY_SUFFIX}
+                 ${MATH_LIB_PATH}/lib/libiomp5md${CMAKE_STATIC_LIBRARY_SUFFIX})
+  else()
+    set(MATH_LIB
+        ${MATH_LIB_PATH}/lib/libmklml_intel${CMAKE_SHARED_LIBRARY_SUFFIX}
+        ${MATH_LIB_PATH}/lib/libiomp5${CMAKE_SHARED_LIBRARY_SUFFIX})
+  endif()
+  set(ONEDNN_PATH "${PADDLE_LIB_THIRD_PARTY_PATH}onednn")
+  if(EXISTS ${ONEDNN_PATH})
+    include_directories("${ONEDNN_PATH}/include")
+    if(WIN32)
+      set(ONEDNN_LIB ${ONEDNN_PATH}/lib/mkldnn.lib)
+    else()
+      set(ONEDNN_LIB ${ONEDNN_PATH}/lib/libdnnl.so.3)
+    endif()
+  endif()
+else()
+  set(OPENBLAS_LIB_PATH "${PADDLE_LIB_THIRD_PARTY_PATH}openblas")
+  include_directories("${OPENBLAS_LIB_PATH}/include/openblas")
+  if(WIN32)
+    set(MATH_LIB
+        ${OPENBLAS_LIB_PATH}/lib/openblas${CMAKE_STATIC_LIBRARY_SUFFIX})
+  else()
+    set(MATH_LIB
+        ${OPENBLAS_LIB_PATH}/lib/libopenblas${CMAKE_STATIC_LIBRARY_SUFFIX})
+  endif()
+endif()
+
+if(WITH_STATIC_LIB)
+  set(DEPS
+      ${PADDLE_LIB}/paddle/lib/libpaddle_inference${CMAKE_STATIC_LIBRARY_SUFFIX}
+  )
+else()
+  if(WIN32)
+    set(DEPS
+        ${PADDLE_LIB}/paddle/lib/paddle_inference${CMAKE_STATIC_LIBRARY_SUFFIX})
+  else()
+    set(DEPS
+        ${PADDLE_LIB}/paddle/lib/libpaddle_inference${CMAKE_SHARED_LIBRARY_SUFFIX}
+    )
+  endif()
+endif()
+
+if(WITH_ONNXRUNTIME)
+  set(DEPS ${DEPS} onnxruntime paddle2onnx)
+endif()
+
+if(NOT WIN32)
+  set(EXTERNAL_LIB "-lrt -ldl -lpthread")
+  set(DEPS
+      ${DEPS}
+      ${MATH_LIB}
+      ${ONEDNN_LIB}
+      glog
+      gflags
+      protobuf
+      xxhash
+      cryptopp
+      ${EXTERNAL_LIB})
+else()
+  set(DEPS
+      ${DEPS}
+      ${MATH_LIB}
+      ${ONEDNN_LIB}
+      glog
+      gflags_static
+      libprotobuf
+      xxhash
+      cryptopp-static
+      ${EXTERNAL_LIB})
+  set(DEPS ${DEPS} shlwapi.lib)
+endif()
+
+if(WITH_GPU)
+  if(NOT WIN32)
+    if(USE_TENSORRT)
+      set(DEPS ${DEPS}
+               ${TENSORRT_LIB_DIR}/libnvinfer${CMAKE_SHARED_LIBRARY_SUFFIX})
+      set(DEPS
+          ${DEPS}
+          ${TENSORRT_LIB_DIR}/libnvinfer_plugin${CMAKE_SHARED_LIBRARY_SUFFIX})
+    endif()
+    set(DEPS ${DEPS} ${CUDA_LIB}/libcudart${CMAKE_SHARED_LIBRARY_SUFFIX})
+  else()
+    if(USE_TENSORRT)
+      set(DEPS ${DEPS}
+               ${TENSORRT_LIB_DIR}/nvinfer${CMAKE_STATIC_LIBRARY_SUFFIX})
+      set(DEPS ${DEPS}
+               ${TENSORRT_LIB_DIR}/nvinfer_plugin${CMAKE_STATIC_LIBRARY_SUFFIX})
+      if(${TENSORRT_MAJOR_VERSION} EQUAL 7)
+        set(DEPS ${DEPS}
+                 ${TENSORRT_LIB_DIR}/myelin64_1${CMAKE_STATIC_LIBRARY_SUFFIX})
+      endif()
+    endif()
+    set(DEPS ${DEPS} ${CUDA_LIB}/cudart${CMAKE_STATIC_LIBRARY_SUFFIX})
+    set(DEPS ${DEPS} ${CUDA_LIB}/cublas${CMAKE_STATIC_LIBRARY_SUFFIX})
+    set(DEPS ${DEPS} ${CUDA_LIB}/cudnn${CMAKE_STATIC_LIBRARY_SUFFIX})
+  endif()
+endif()
+
+if(WITH_GTEST)
+  include(ExternalProject)
+  include(external-cmake/gtest-cpp.cmake)
+endif()
+
+add_executable(${DEMO_NAME} ${DEMO_NAME}.cc)
+target_link_libraries(${DEMO_NAME} ${DEPS})
+if(WITH_GTEST)
+  include(GNUInstallDirs)
+  include_directories(${GTEST_INSTALL_DIR}/include)
+  add_dependencies(${DEMO_NAME} thirdparty_gtest)
+  if(WIN32)
+    target_link_libraries(${DEMO_NAME} ${GTEST_LIBRARIES})
+  else()
+    target_link_libraries(
+      ${DEMO_NAME}
+      ${GTEST_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libgtest${CMAKE_STATIC_LIBRARY_SUFFIX}
+    )
+  endif()
+endif()
+if(WIN32)
+  if("${CMAKE_GENERATOR}" MATCHES "Ninja")
+    set(LIB_PATH ${CMAKE_BINARY_DIR})
+  else()
+    set(LIB_PATH ${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE})
+  endif()
+
+  if(USE_TENSORRT)
+    add_custom_command(
+      TARGET ${DEMO_NAME}
+      POST_BUILD
+      COMMAND
+        ${CMAKE_COMMAND} -E copy
+        ${TENSORRT_LIB_DIR}/nvinfer${CMAKE_SHARED_LIBRARY_SUFFIX} ${LIB_PATH}
+      COMMAND
+        ${CMAKE_COMMAND} -E copy
+        ${TENSORRT_LIB_DIR}/nvinfer_plugin${CMAKE_SHARED_LIBRARY_SUFFIX}
+        ${LIB_PATH})
+    if(${TENSORRT_MAJOR_VERSION} EQUAL 7)
+      add_custom_command(
+        TARGET ${DEMO_NAME}
+        POST_BUILD
+        COMMAND
+          ${CMAKE_COMMAND} -E copy
+          ${TENSORRT_LIB_DIR}/myelin64_1${CMAKE_SHARED_LIBRARY_SUFFIX}
+          ${LIB_PATH})
+    endif()
+  endif()
+  if(WITH_MKL)
+    message("LIB_PATH IS ${LIB_PATH}")
+    add_custom_command(
+      TARGET ${DEMO_NAME}
+      POST_BUILD
+      COMMAND ${CMAKE_COMMAND} -E copy ${MATH_LIB_PATH}/lib/mklml.dll
+              ${LIB_PATH}
+      COMMAND ${CMAKE_COMMAND} -E copy ${MATH_LIB_PATH}/lib/libiomp5md.dll
+              ${LIB_PATH}
+      COMMAND ${CMAKE_COMMAND} -E copy ${ONEDNN_PATH}/lib/mkldnn.dll
+              ${LIB_PATH})
+  else()
+    add_custom_command(
+      TARGET ${DEMO_NAME}
+      POST_BUILD
+      COMMAND ${CMAKE_COMMAND} -E copy ${OPENBLAS_LIB_PATH}/lib/openblas.dll
+              ${LIB_PATH})
+  endif()
+  if(WITH_ONNXRUNTIME)
+    add_custom_command(
+      TARGET ${DEMO_NAME}
+      POST_BUILD
+      COMMAND
+        ${CMAKE_COMMAND} -E copy
+        ${PADDLE_LIB_THIRD_PARTY_PATH}onnxruntime/lib/onnxruntime.dll
+        ${LIB_PATH}
+      COMMAND
+        ${CMAKE_COMMAND} -E copy
+        ${PADDLE_LIB_THIRD_PARTY_PATH}paddle2onnx/lib/paddle2onnx.dll
+        ${LIB_PATH})
+  endif()
+  if(NOT WITH_STATIC_LIB)
+    add_custom_command(
+      TARGET ${DEMO_NAME}
+      POST_BUILD
+      COMMAND ${CMAKE_COMMAND} -E copy
+              "${PADDLE_LIB}/paddle/lib/paddle_inference.dll" ${LIB_PATH})
+  endif()
+endif()

test/cpp/inference/infer_ut/README.md

@@ -0,0 +1,37 @@
+# Inference Model UT
+
+There are several model tests currently:
+- test_ernie_text_cls.cc
+- test_LeViT.cc
+- test_ppyolo_mbv3.cc
+- test_ppyolov2_r50vd.cc
+- test_resnet50.cc
+- test_resnet50_quant.cc
+- test_yolov3.cc
+
+To build and execute tests on Linux, simply run
+```
+./run.sh $PADDLE_ROOT $TURN_ON_MKL $TEST_GPU_CPU $DATA_DIR
+```
+To build on windows, run command with busybox
+```
+busybox bash ./run.sh $PADDLE_ROOT $TURN_ON_MKL $TEST_GPU_CPU $DATA_DIR
+```
+
+- After run command, it will build and execute tests and download to ${DATA_DIR} automatically.
+- `$PADDLE_ROOT`: paddle library path
+- `$TURN_ON_MKL`: use MKL or Openblas
+- `$TEST_GPU_CPU`: test both GPU/CPU mode or only CPU mode
+- `$DATA_DIR`: download data path
+
+now only support 4 kinds of tests which controlled by `--gtest_filter` argument, test suite name should be same as following.
+- `TEST(gpu_tester_*, test_name)`
+- `TEST(cpu_tester_*, test_name)`
+- `TEST(onednn_tester_*, test_name)`
+- `TEST(tensorrt_tester_*, test_name)`
+
+skpied test suite name.
+- `TEST(DISABLED_gpu_tester_*, test_name)`
+- `TEST(DISABLED_cpu_tester_*, test_name)`
+- `TEST(DISABLED_onednn_tester_*, test_name)`
+- `TEST(DISABLED_tensorrt_tester_*, test_name)`

test/cpp/inference/infer_ut/external-cmake/gtest-cpp.cmake

@@ -0,0 +1,59 @@
+find_package(Git REQUIRED)
+message("${CMAKE_BUILD_TYPE}")
+set(GTEST_PREFIX_DIR ${CMAKE_CURRENT_BINARY_DIR}/gtest)
+set(PADDLE_SOURCE_DIR $ENV{PADDLE_SOURCE_DIR})
+set(GTEST_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/gtest)
+set(GTEST_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/install/gtest)
+set(GTEST_INCLUDE_DIR
+    "${GTEST_INSTALL_DIR}/include"
+    CACHE PATH "gtest include directory." FORCE)
+set(GTEST_TAG release-1.8.1)
+
+if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.0.0")
+  message(
+    WARNING
+      "gtest-cpp: forcing CMake policy compatibility for CMake >= 4.0 (CMAKE_POLICY_VERSION_MINIMUM=3.5)"
+  )
+  set(GTEST_POLICY_ARGS -DCMAKE_POLICY_VERSION_MINIMUM=3.5)
+endif()
+
+include_directories(${GTEST_INCLUDE_DIR})
+if(WIN32)
+  # if use CMAKE_INSTALL_LIBDIR, the path of lib actually is \
+  # install/gtest/lib/gtest.lib but GTEST_LIBRARIES
+  # is install/gtest/gtest.lib
+  set(GTEST_LIBRARIES
+      "${GTEST_INSTALL_DIR}/lib/gtest.lib"
+      CACHE FILEPATH "gtest libraries." FORCE)
+  set(GTEST_MAIN_LIBRARIES
+      "${GTEST_INSTALL_DIR}/lib/gtest_main.lib"
+      CACHE FILEPATH "gtest main libraries." FORCE)
+else()
+  set(GTEST_LIBRARIES
+      "${GTEST_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libgtest.a"
+      CACHE FILEPATH "gtest libraries." FORCE)
+  set(GTEST_MAIN_LIBRARIES
+      "${GTEST_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libgtest_main.a"
+      CACHE FILEPATH "gtest main libraries." FORCE)
+endif()
+ExternalProject_Add(
+  extern_gtest
+  PREFIX gtest
+  SOURCE_DIR ${GTEST_SOURCE_DIR}
+  DOWNLOAD_DIR "${DOWNLOAD_LOCATION}"
+  UPDATE_COMMAND ""
+  CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GTEST_INSTALL_DIR}
+             -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
+             -DCMAKE_BUILD_TYPE:STRING=Release ${GTEST_POLICY_ARGS}
+  BUILD_BYPRODUCTS ${GTEST_LIBRARIES}
+  BUILD_BYPRODUCTS ${GTEST_MAIN_LIBRARIES})
+
+add_library(thirdparty_gtest STATIC IMPORTED GLOBAL)
+set_property(TARGET thirdparty_gtest PROPERTY IMPORTED_LOCATION
+                                              ${GTEST_LIBRARIES})
+add_dependencies(thirdparty_gtest extern_gtest)
+
+add_library(thirdparty_gtest_main STATIC IMPORTED GLOBAL)
+set_property(TARGET thirdparty_gtest_main PROPERTY IMPORTED_LOCATION
+                                                   ${GTEST_MAIN_LIBRARIES})
+add_dependencies(thirdparty_gtest_main extern_gtest)

test/cpp/inference/infer_ut/run.sh

@@ -0,0 +1,326 @@
+#!/bin/bash
+
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+PADDLE_ROOT=$1
+export PADDLE_SOURCE_DIR=$1
+TURN_ON_MKL=$2 # use MKL or Openblas
+TEST_GPU_CPU=$3 # test both GPU/CPU mode or only CPU mode
+DATA_DIR=$4 # dataset
+TENSORRT_ROOT_DIR=$5 # TensorRT ROOT dir, default to /usr/local/TensorRT
+WITH_ONNXRUNTIME=$6
+MSVC_STATIC_CRT=$7
+CUDA_LIB=$8/lib/x64
+inference_install_dir=${PADDLE_ROOT}/build/paddle_inference_install_dir
+EXIT_CODE=0 # init default exit code
+WIN_DETECT=$(echo `uname` | grep "Win") # detect current platform
+test_suite_list="cpu_tester*" # init test suite list, pass to --gtest_filter
+
+export RED='\033[0;31m' # red color
+export NC='\033[0m' # no color
+export YELLOW='\033[33m' # yellow color
+
+cd `dirname $0`
+current_dir=`pwd`
+build_dir=${current_dir}/build
+log_dir=${current_dir}/log
+
+# check onednn installation
+if [ $2 == ON ]; then
+  # You can export yourself if move the install path
+  MKL_LIB=${inference_install_dir}/third_party/install/mklml/lib
+  export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${MKL_LIB}
+  test_suite_list="${test_suite_list}:onednn_tester*"
+fi
+
+if [ $3 == ON ]; then
+  use_gpu_list='true false'
+  test_suite_list="${test_suite_list}:gpu_tester*"
+else
+  use_gpu_list='false'
+fi
+
+# check tensorrt installation
+TENSORRT_COMPILED=$(cat "${inference_install_dir}/version.txt" | grep "WITH_TENSORRT")
+USE_TENSORRT=OFF
+if [ -d "$TENSORRT_ROOT_DIR" ] && [ ! -z "$TENSORRT_COMPILED" ]  ; then
+  USE_TENSORRT=ON
+  test_suite_list="${test_suite_list}:tensorrt_tester*"
+fi
+
+function download() {
+  url_prefix=$1
+  model_name=$2
+  mkdir -p $model_name
+  cd $model_name
+  if [[ -e "${model_name}.tgz" ]]; then
+    echo "${model_name}.tgz has been downloaded."
+  else
+      if [ "$WIN_DETECT" != "" ]; then
+        wget -q -Y off ${url_prefix}/${model_name}.tgz
+        tar xzf *.tgz
+      else
+        wget -q --no-proxy ${url_prefix}/${model_name}.tgz
+        tar xzf *.tgz
+      fi
+  fi
+  cd ..
+}
+
+mkdir -p $DATA_DIR
+cd $DATA_DIR
+download_list='resnet50'
+for model_name in $download_list; do
+    url_prefix="https://paddle-inference-dist.bj.bcebos.com/Paddle-Inference-Demo"
+    download $url_prefix $model_name
+done
+
+ocr_download_list='ocr_det_mv3_db'
+for model_name in $ocr_download_list; do
+    url_prefix="https://paddle-qa.bj.bcebos.com/inference_model/2.1.1/ocr"
+    download $url_prefix $model_name
+done
+
+clas_download_list='LeViT'
+for model_name in $clas_download_list; do
+    url_prefix="https://paddle-qa.bj.bcebos.com/inference_model/2.1.1/class"
+    download $url_prefix $model_name
+done
+
+nlp_download_list='ernie_text_cls'
+for model_name in $nlp_download_list; do
+    url_prefix="https://paddle-qa.bj.bcebos.com/inference_model/2.1.1/nlp"
+    download $url_prefix $model_name
+done
+
+det_download_list='yolov3 ppyolo_mbv3 ppyolov2_r50vd'
+for model_name in $det_download_list; do
+    url_prefix="https://paddle-qa.bj.bcebos.com/inference_model/2.1.1/detection"
+    download $url_prefix $model_name
+done
+
+unknown_download_list='resnet50_quant'
+for model_name in $unknown_download_list; do
+    url_prefix="https://paddle-qa.bj.bcebos.com/inference_model/unknown"
+    download $url_prefix $model_name
+done
+
+# ernie int8 quant with matmul
+unknown_nlp_download_list='quant_post_model_xnli_predict_matmul'
+for model_name in $unknown_nlp_download_list; do
+    url_prefix="https://paddle-qa.bj.bcebos.com/inference_model/unknown/nlp"
+    download $url_prefix $model_name
+done
+
+# mobilnetv1 with prune op attribute
+dev_class_download_list='MobileNetV1'
+for model_name in $dev_class_download_list; do
+    url_prefix="https://paddle-qa.bj.bcebos.com/inference_model/2021-09-16/class"
+    download $url_prefix $model_name
+done
+
+function compile_test() {
+    mkdir -p ${build_dir}
+    cd ${build_dir}
+    TEST_NAME=$1
+    if [ "$WIN_DETECT" != "" ]; then
+        cmake .. -GNinja -DPADDLE_LIB=${inference_install_dir} \
+             -DWITH_MKL=$TURN_ON_MKL \
+             -DDEMO_NAME=${TEST_NAME} \
+             -DWITH_GPU=$TEST_GPU_CPU \
+             -DWITH_STATIC_LIB=OFF \
+             -DUSE_TENSORRT=$USE_TENSORRT \
+             -DTENSORRT_ROOT=$TENSORRT_ROOT_DIR \
+             -DMSVC_STATIC_CRT=$MSVC_STATIC_CRT \
+             -DWITH_GTEST=ON \
+             -DCMAKE_CXX_FLAGS='/std:c++17' \
+             -DCMAKE_BUILD_TYPE=Release \
+             -DWITH_ONNXRUNTIME=$WITH_ONNXRUNTIME \
+             -DCUDA_LIB="$CUDA_LIB"
+        ninja
+    else
+        cmake .. -DPADDLE_LIB=${inference_install_dir} \
+                 -DWITH_MKL=$TURN_ON_MKL \
+                 -DDEMO_NAME=${TEST_NAME} \
+                 -DWITH_GPU=$TEST_GPU_CPU \
+                 -DWITH_STATIC_LIB=OFF \
+                 -DUSE_TENSORRT=$USE_TENSORRT \
+                 -DTENSORRT_ROOT=$TENSORRT_ROOT_DIR \
+                 -DWITH_GTEST=ON \
+                 -DWITH_ONNXRUNTIME=$WITH_ONNXRUNTIME
+        make -j$(nproc)
+    fi
+    cd -
+}
+
+
+# compile and run test
+cd $current_dir
+mkdir -p ${build_dir}
+mkdir -p ${log_dir}
+cd ${build_dir}
+rm -rf *
+
+exe_dir=${build_dir}
+
+# printf "${YELLOW} start test_resnet50 ${NC} \n";
+# compile_test "test_resnet50"
+# ${exe_dir}/test_resnet50 \
+#     --modeldir=$DATA_DIR/resnet50/resnet50 \
+#     --gtest_filter=${test_suite_list} \
+#     --gtest_output=xml:${log_dir}/test_resnet50.xml
+# if [ $? -ne 0 ]; then
+#     echo "${RED} test_resnet50 runs failed ${NC}" >> ${exe_dir}/test_summary.txt
+#     EXIT_CODE=8
+# fi
+
+# printf "${YELLOW} start test_det_mv3_db ${NC} \n";
+# compile_test "test_det_mv3_db"
+# ${exe_dir}/test_det_mv3_db \
+#     --modeldir=$DATA_DIR/ocr_det_mv3_db/ocr_det_mv3_db \
+#     --gtest_filter=${test_suite_list} \
+#     --gtest_output=xml:${log_dir}/test_det_mv3_db.xml
+# if [ $? -ne 0 ]; then
+#     echo "${RED} test_det_mv3_db runs failed ${NC}" >> ${exe_dir}/test_summary.txt
+#     EXIT_CODE=8
+# fi
+
+# printf "${YELLOW} start test_LeViT ${NC} \n";
+# compile_test "test_LeViT"
+# ${exe_dir}/test_LeViT \
+#     --modeldir=$DATA_DIR/LeViT/LeViT \
+#     --gtest_filter=${test_suite_list} \
+#     --gtest_output=xml:${log_dir}/test_LeViT.xml
+# if [ $? -ne 0 ]; then
+#     echo "${RED} test_LeViT runs failed ${NC}" >> ${exe_dir}/test_summary.txt
+#     EXIT_CODE=8
+# fi
+
+if [ "$WIN_DETECT" != "" ]; then
+    #TODO(OliverLPH): enable test_ernie_text_cls on windows after fix compile issue
+    echo "  skip test_ernie_text_cls  "
+else
+    printf "${YELLOW} start test_ernie_text_cls ${NC} \n";
+    compile_test "test_ernie_text_cls"
+    ${exe_dir}/test_ernie_text_cls \
+        --modeldir=$DATA_DIR/ernie_text_cls/ernie_text_cls \
+        --gtest_filter=${test_suite_list} \
+        --gtest_output=xml:${log_dir}/test_ernie_text_cls.xml
+    if [ $? -ne 0 ]; then
+        echo "${RED} test_ernie_text_cls runs failed ${NC}" >> ${exe_dir}/test_summary.txt
+        EXIT_CODE=8
+    fi
+fi
+
+printf "${YELLOW} start test_yolov3 ${NC} \n";
+compile_test "test_yolov3"
+${exe_dir}/test_yolov3 \
+    --modeldir=$DATA_DIR/yolov3/yolov3 \
+    --gtest_filter=${test_suite_list} \
+    --gtest_output=xml:${log_dir}/test_yolov3.xml
+if [ $? -ne 0 ]; then
+    echo "${RED} test_yolov3 runs failed ${NC}" >> ${exe_dir}/test_summary.txt
+    EXIT_CODE=8
+fi
+
+# printf "${YELLOW} start test_ppyolo_mbv3 ${NC} \n";
+# compile_test "test_ppyolo_mbv3"
+# ${exe_dir}/test_ppyolo_mbv3 \
+#     --modeldir=$DATA_DIR/ppyolo_mbv3/ppyolo_mbv3 \
+#     --gtest_filter=${test_suite_list} \
+#     --gtest_output=xml:${log_dir}/test_ppyolo_mbv3.xml
+# if [ $? -ne 0 ]; then
+#     echo "${RED} test_ppyolo_mbv3 runs failed ${NC}" >> ${exe_dir}/test_summary.txt
+#     EXIT_CODE=8
+# fi
+
+# printf "${YELLOW} start test_ppyolov2_r50vd ${NC} \n";
+# compile_test "test_ppyolov2_r50vd"
+# ${exe_dir}/test_ppyolov2_r50vd \
+#     --modeldir=$DATA_DIR/ppyolov2_r50vd/ppyolov2_r50vd \
+#     --gtest_filter=${test_suite_list} \
+#     --gtest_output=xml:${log_dir}/test_ppyolov2_r50vd.xml
+# if [ $? -ne 0 ]; then
+#     echo "${RED} test_ppyolov2_r50vd runs failed ${NC}" >> ${exe_dir}/test_summary.txt
+#     EXIT_CODE=8
+# fi
+
+printf "${YELLOW} start test_resnet50_quant ${NC} \n";
+compile_test "test_resnet50_quant"
+${exe_dir}/test_resnet50_quant \
+    --int8dir=$DATA_DIR/resnet50_quant/resnet50_quant/resnet50_quant \
+    --modeldir=$DATA_DIR/resnet50/resnet50 \
+    --datadir=$DATA_DIR/resnet50_quant/resnet50_quant/imagenet-eval-binary/9.data \
+    --gtest_filter=${test_suite_list} \
+    --gtest_output=xml:${log_dir}/test_resnet50_quant.xml
+if [ $? -ne 0 ]; then
+    echo "${RED} test_resnet50_quant runs failed ${NC}" >> ${exe_dir}/test_summary.txt
+    EXIT_CODE=8
+fi
+
+# printf "${YELLOW} start test_ernie_xnli_int8 ${NC} \n";
+# compile_test "test_ernie_xnli_int8"
+# ernie_qat_model="quant_post_model_xnli_predict_matmul"
+# ${exe_dir}/test_ernie_xnli_int8 \
+#     --modeldir=$DATA_DIR/$ernie_qat_model/$ernie_qat_model \
+#     --datadir=$DATA_DIR/$ernie_qat_model/$ernie_qat_model/xnli_var_len \
+#     --truth_data=$DATA_DIR/$ernie_qat_model/$ernie_qat_model/truth_data \
+#     --gtest_filter=${test_suite_list} \
+#     --gtest_output=xml:${log_dir}/test_ernie_xnli_int8.xml
+# if [ $? -ne 0 ]; then
+#     echo "${RED} test_ernie_xnli_int8 runs failed ${NC}" >> ${exe_dir}/test_summary.txt
+#     EXIT_CODE=8
+# fi
+
+printf "${YELLOW} start test_mobilnetv1 ${NC} \n";
+compile_test "test_mobilnetv1"
+${exe_dir}/test_mobilnetv1 \
+    --modeldir=$DATA_DIR/MobileNetV1/MobileNetV1 \
+    --gtest_filter=${test_suite_list} \
+    --gtest_output=xml:${log_dir}/test_mobilnetv1.xml
+if [ $? -ne 0 ]; then
+    echo "${RED} test_mobilnetv1 runs failed ${NC}" >> ${exe_dir}/test_summary.txt
+    EXIT_CODE=8
+fi
+
+set +x
+
+test_suites=$(echo ${test_suite_list} | sed 's/:/ /g')
+echo " "
+echo "CI Tested Following Patterns: "
+echo "=====================test patterns======================"
+for test_suite in ${test_suites}; do
+  echo "  ${test_suite}"
+done
+echo "========================================================"
+echo " "
+
+if [[ -f ${exe_dir}/test_summary.txt ]];then
+  echo " "
+  echo "Summary infer_ut Failed Tests ..."
+  echo "=====================test summary======================"
+  echo "The following tests Failed: "
+  cat ${exe_dir}/test_summary.txt
+  echo "========================================================"
+  echo " "
+fi
+set -x
+
+# tar Gtest output report
+tar -zcvf infer_ut_log.tgz ${log_dir}
+
+echo "infer_ut script finished"
+exit ${EXIT_CODE}

test/cpp/inference/infer_ut/test_LeViT.cc

@@ -0,0 +1,253 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test_suite.h"  // NOLINT
+#ifdef PADDLE_WITH_GPU
+#include <cuda_runtime.h>
+#endif
+
+DEFINE_string(modeldir, "", "Directory of the inference model.");
+
+namespace paddle_infer {
+
+paddle::test::Record PrepareInput(int batch_size) {
+  // init input data
+  int channel = 3;
+  int width = 224;
+  int height = 224;
+  paddle::test::Record image_Record;
+  int input_num = batch_size * channel * width * height;
+  std::vector<float> input_data(input_num, 1);
+  image_Record.data = input_data;
+  image_Record.shape = std::vector<int>{batch_size, channel, width, height};
+  image_Record.type = paddle::PaddleDType::FLOAT32;
+  return image_Record;
+}
+
+TEST(gpu_tester_LeViT, analysis_gpu_bz1) {
+  // init input data
+  std::map<std::string, paddle::test::Record> my_input_data_map;
+  my_input_data_map["x"] = PrepareInput(1);
+  // init output data
+  std::map<std::string, paddle::test::Record> infer_output_data,
+      truth_output_data;
+  // prepare ground truth config
+  paddle_infer::Config config, config_no_ir;
+  config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                        FLAGS_modeldir + "/inference.pdiparams");
+  config_no_ir.SwitchIrOptim(false);
+  // prepare inference config
+  config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                  FLAGS_modeldir + "/inference.pdiparams");
+  // get ground truth by disable ir
+  paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
+  SingleThreadPrediction(
+      pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
+  // get infer results
+  paddle_infer::services::PredictorPool pred_pool(config, 1);
+  SingleThreadPrediction(
+      pred_pool.Retrieve(0), &my_input_data_map, &infer_output_data);
+  // check outputs
+  CompareRecord(&truth_output_data, &infer_output_data);
+  std::cout << "finish test" << std::endl;
+}
+
+TEST(tensorrt_tester_LeViT, trt_fp32_bz2) {
+  // init input data
+  std::map<std::string, paddle::test::Record> my_input_data_map;
+  my_input_data_map["x"] = PrepareInput(2);
+  // init output data
+  std::map<std::string, paddle::test::Record> infer_output_data,
+      truth_output_data;
+  // prepare ground truth config
+  paddle_infer::Config config, config_no_ir;
+  config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                        FLAGS_modeldir + "/inference.pdiparams");
+  config_no_ir.SwitchIrOptim(false);
+  // prepare inference config
+  config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                  FLAGS_modeldir + "/inference.pdiparams");
+  config.EnableUseGpu(100, 0);
+  config.EnableTensorRtEngine(
+      1 << 20, 2, 50, paddle_infer::PrecisionType::kFloat32, false, false);
+  // get ground truth by disable ir
+  paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
+  SingleThreadPrediction(
+      pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
+  // get infer results
+  paddle_infer::services::PredictorPool pred_pool(config, 1);
+  SingleThreadPrediction(
+      pred_pool.Retrieve(0), &my_input_data_map, &infer_output_data);
+  // check outputs
+  CompareRecord(&truth_output_data, &infer_output_data);
+  std::cout << "finish test" << std::endl;
+}
+
+TEST(tensorrt_tester_LeViT, serial_diff_batch_trt_fp32) {
+  int max_batch_size = 5;
+  // prepare ground truth config
+  paddle_infer::Config config, config_no_ir;
+  config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                        FLAGS_modeldir + "/inference.pdiparams");
+  config_no_ir.SwitchIrOptim(false);
+  paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
+  // prepare inference config
+  config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                  FLAGS_modeldir + "/inference.pdiparams");
+  config.EnableUseGpu(100, 0);
+  config.EnableTensorRtEngine(1 << 20,
+                              max_batch_size,
+                              50,
+                              paddle_infer::PrecisionType::kFloat32,
+                              false,
+                              false);
+  paddle_infer::services::PredictorPool pred_pool(config, 1);
+
+  for (int i = 1; i < max_batch_size; i++) {
+    // init input data
+    std::map<std::string, paddle::test::Record> my_input_data_map;
+    my_input_data_map["x"] = PrepareInput(i);
+    // init output data
+    std::map<std::string, paddle::test::Record> infer_output_data,
+        truth_output_data;
+    // get ground truth by disable ir
+    SingleThreadPrediction(
+        pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
+    // get infer results
+    SingleThreadPrediction(
+        pred_pool.Retrieve(0), &my_input_data_map, &infer_output_data);
+    // check outputs
+    CompareRecord(&truth_output_data, &infer_output_data);
+  }
+  std::cout << "finish test" << std::endl;
+}
+
+TEST(tensorrt_tester_LeViT, multi_thread4_trt_fp32_bz2) {
+  int thread_num = 4;
+  // init input data
+  std::map<std::string, paddle::test::Record> my_input_data_map;
+  my_input_data_map["x"] = PrepareInput(2);
+  // init output data
+  std::map<std::string, paddle::test::Record> infer_output_data,
+      truth_output_data;
+  // prepare ground truth config
+  paddle_infer::Config config, config_no_ir;
+  config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                        FLAGS_modeldir + "/inference.pdiparams");
+  config_no_ir.SwitchIrOptim(false);
+  // prepare inference config
+  config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                  FLAGS_modeldir + "/inference.pdiparams");
+  config.EnableUseGpu(100, 0);
+  config.EnableTensorRtEngine(
+      1 << 20, 2, 50, paddle_infer::PrecisionType::kFloat32, false, false);
+  // get ground truth by disable ir
+  paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
+  SingleThreadPrediction(
+      pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
+
+  // get infer results from multi threads
+  std::vector<std::thread> threads;
+  services::PredictorPool pred_pool(config, thread_num);
+  for (int i = 0; i < thread_num; ++i) {
+    threads.emplace_back(paddle::test::SingleThreadPrediction,
+                         pred_pool.Retrieve(i),
+                         &my_input_data_map,
+                         &infer_output_data,
+                         10);
+  }
+
+  // thread join & check outputs
+  for (int i = 0; i < thread_num; ++i) {
+    LOG(INFO) << "join tid : " << i;
+    threads[i].join();
+    CompareRecord(&truth_output_data, &infer_output_data);
+  }
+
+  std::cout << "finish multi-thread test" << std::endl;
+}
+
+#ifdef PADDLE_WITH_GPU
+TEST(tensorrt_tester_LeViT, multi_stream_thread4_trt_fp32_bz2) {
+  int thread_num = 4;
+
+  // init stream
+  std::vector<cudaStream_t> streams(thread_num);
+  for (size_t i = 0; i < thread_num; ++i) {
+    cudaStreamCreate(&streams[i]);
+  }
+
+  // init input data
+  std::map<std::string, paddle::test::Record> my_input_data_map;
+  my_input_data_map["x"] = PrepareInput(2);
+  // init output data
+  std::map<std::string, paddle::test::Record> infer_output_data,
+      truth_output_data;
+  // prepare ground truth config
+  paddle_infer::Config config, config_no_ir;
+  config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                        FLAGS_modeldir + "/inference.pdiparams");
+  config_no_ir.SwitchIrOptim(false);
+  // prepare inference config
+  config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                  FLAGS_modeldir + "/inference.pdiparams");
+  config.EnableUseGpu(100, 0);
+  config.EnableTensorRtEngine(
+      1 << 20, 2, 50, paddle_infer::PrecisionType::kFloat32, false, false);
+  // get ground truth by disable ir
+
+  paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
+  SingleThreadPrediction(
+      pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
+
+  // get infer results from multi threads
+  std::vector<std::thread> threads;
+  config.SetExecStream(streams[0]);
+  config.pass_builder()->DeletePass("add_support_int8_pass");
+  auto main_predictor = CreatePredictor(config);
+  std::vector<decltype(main_predictor)> predictors;
+  for (size_t i = 0; i < thread_num - 1; ++i) {
+    predictors.push_back(std::move(main_predictor->Clone(streams[i + 1])));
+    LOG(INFO) << "predictors[" << i << "] stream is "
+              << predictors[i]->GetExecStream();
+  }
+  predictors.push_back(std::move(main_predictor));
+  LOG(INFO) << "predictors[" << thread_num - 1 << "] stream is "
+            << predictors[thread_num - 1]->GetExecStream();
+  for (int i = 0; i < thread_num; ++i) {
+    threads.emplace_back(paddle::test::SingleThreadPrediction,
+                         predictors[i].get(),
+                         &my_input_data_map,
+                         &infer_output_data,
+                         10);
+  }
+
+  // thread join & check outputs
+  for (int i = 0; i < thread_num; ++i) {
+    LOG(INFO) << "join tid : " << i;
+    threads[i].join();
+    CompareRecord(&truth_output_data, &infer_output_data);
+  }
+
+  std::cout << "finish multi-thread test" << std::endl;
+}
+#endif
+
+}  // namespace paddle_infer
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  return RUN_ALL_TESTS();
+}

test/cpp/inference/infer_ut/test_det_mv3_db.cc

@@ -0,0 +1,188 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test_suite.h"  // NOLINT
+
+DEFINE_string(modeldir, "", "Directory of the inference model.");
+
+namespace paddle_infer {
+
+paddle::test::Record PrepareInput(int batch_size, int image_shape = 640) {
+  // init input data
+  int channel = 3;
+  int width = image_shape;
+  int height = image_shape;
+  paddle::test::Record image_Record;
+  int input_num = batch_size * channel * width * height;
+  std::vector<float> input_data(input_num, 1);
+  image_Record.data = input_data;
+  image_Record.shape = std::vector<int>{batch_size, channel, width, height};
+  image_Record.type = paddle::PaddleDType::FLOAT32;
+  return image_Record;
+}
+
+void PrepareDynamicShape(paddle_infer::Config* config, int max_batch_size = 4) {
+  // set dynamic shape range
+  std::map<std::string, std::vector<int>> min_input_shape = {
+      {"x", {1, 3, 224, 224}},
+      {"conv2d_124.tmp_0", {1, 256, 56, 56}},
+      {"nearest_interp_v2_2.tmp_0", {1, 256, 56, 56}},
+      {"nearest_interp_v2_3.tmp_0", {1, 64, 56, 56}},
+      {"nearest_interp_v2_4.tmp_0", {1, 64, 56, 56}},
+      {"nearest_interp_v2_5.tmp_0", {1, 64, 56, 56}}};
+  std::map<std::string, std::vector<int>> max_input_shape = {
+      {"x", {max_batch_size, 3, 448, 448}},
+      {"conv2d_124.tmp_0", {max_batch_size, 256, 112, 112}},
+      {"nearest_interp_v2_2.tmp_0", {max_batch_size, 256, 112, 112}},
+      {"nearest_interp_v2_3.tmp_0", {max_batch_size, 64, 112, 112}},
+      {"nearest_interp_v2_4.tmp_0", {max_batch_size, 64, 112, 112}},
+      {"nearest_interp_v2_5.tmp_0", {max_batch_size, 64, 112, 112}}};
+  std::map<std::string, std::vector<int>> opt_input_shape = {
+      {"x", {1, 3, 256, 256}},
+      {"conv2d_124.tmp_0", {1, 256, 64, 64}},
+      {"nearest_interp_v2_2.tmp_0", {1, 256, 64, 64}},
+      {"nearest_interp_v2_3.tmp_0", {1, 64, 64, 64}},
+      {"nearest_interp_v2_4.tmp_0", {1, 64, 64, 64}},
+      {"nearest_interp_v2_5.tmp_0", {1, 64, 64, 64}}};
+  config->SetTRTDynamicShapeInfo(
+      min_input_shape, max_input_shape, opt_input_shape);
+}
+
+TEST(gpu_tester_det_mv3_db, analysis_gpu_bz4) {
+  // init input data
+  std::map<std::string, paddle::test::Record> my_input_data_map;
+  my_input_data_map["x"] = PrepareInput(4, 640);
+  // init output data
+  std::map<std::string, paddle::test::Record> infer_output_data,
+      truth_output_data;
+  // prepare ground truth config
+  paddle_infer::Config config, config_no_ir;
+  config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                        FLAGS_modeldir + "/inference.pdiparams");
+  config_no_ir.SwitchIrOptim(false);
+  // prepare inference config
+  config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                  FLAGS_modeldir + "/inference.pdiparams");
+  // get ground truth by disable ir
+  paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
+  SingleThreadPrediction(
+      pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
+  // get infer results
+  paddle_infer::services::PredictorPool pred_pool(config, 1);
+  SingleThreadPrediction(
+      pred_pool.Retrieve(0), &my_input_data_map, &infer_output_data);
+  // check outputs
+  CompareRecord(&truth_output_data, &infer_output_data, 1e-4);
+  std::cout << "finish test" << std::endl;
+}
+
+TEST(tensorrt_tester_det_mv3_db, multi_thread2_trt_fp32_dynamic_shape_bz2) {
+  int thread_num = 2;  // thread > 2 may OOM
+  // init input data
+  std::map<std::string, paddle::test::Record> my_input_data_map;
+  my_input_data_map["x"] = PrepareInput(2, 256);
+  // init output data
+  std::map<std::string, paddle::test::Record> infer_output_data,
+      truth_output_data;
+  // prepare ground truth config
+  paddle_infer::Config config, config_no_ir;
+  config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                        FLAGS_modeldir + "/inference.pdiparams");
+  config_no_ir.SwitchIrOptim(false);
+  // prepare inference config
+  config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                  FLAGS_modeldir + "/inference.pdiparams");
+  config.EnableUseGpu(100, 0);
+  config.EnableTensorRtEngine(
+      1 << 20, 4, 3, paddle_infer::PrecisionType::kFloat32, false, false);
+  PrepareDynamicShape(&config, 4);
+  // get ground truth by disable ir
+  paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
+  SingleThreadPrediction(
+      pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
+
+  // get infer results from multi threads
+  std::vector<std::thread> threads;
+  services::PredictorPool pred_pool(config, thread_num);
+  for (int i = 0; i < thread_num; ++i) {
+    threads.emplace_back(paddle::test::SingleThreadPrediction,
+                         pred_pool.Retrieve(i),
+                         &my_input_data_map,
+                         &infer_output_data,
+                         2);
+  }
+
+  // thread join & check outputs
+  for (int i = 0; i < thread_num; ++i) {
+    LOG(INFO) << "join tid : " << i;
+    threads[i].join();
+    CompareRecord(&truth_output_data, &infer_output_data, 1e-4);
+  }
+
+  std::cout << "finish multi-thread test" << std::endl;
+}
+
+TEST(onednn_tester_det_mv3_db, multi_thread2_mkl_fp32_bz2) {
+  int thread_num = 2;  // thread > 2 may OOM
+  // init input data
+  std::map<std::string, paddle::test::Record> my_input_data_map;
+  my_input_data_map["x"] = PrepareInput(2, 640);
+  // init output data
+  std::map<std::string, paddle::test::Record> infer_output_data,
+      truth_output_data;
+  // prepare ground truth config
+  paddle_infer::Config config, config_no_ir;
+  config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                        FLAGS_modeldir + "/inference.pdiparams");
+  config_no_ir.SwitchIrOptim(false);
+  // prepare inference config
+  config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                  FLAGS_modeldir + "/inference.pdiparams");
+  config.DisableGpu();
+  config.EnableONEDNN();
+  config.SetOnednnCacheCapacity(10);
+  config.SetCpuMathLibraryNumThreads(10);
+  // get ground truth by disable ir
+  paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
+  SingleThreadPrediction(
+      pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
+
+  // get infer results from multi threads
+  std::vector<std::thread> threads;
+  services::PredictorPool pred_pool(config, thread_num);
+  for (int i = 0; i < thread_num; ++i) {
+    threads.emplace_back(paddle::test::SingleThreadPrediction,
+                         pred_pool.Retrieve(i),
+                         &my_input_data_map,
+                         &infer_output_data,
+                         2);
+  }
+
+  // thread join & check outputs
+  for (int i = 0; i < thread_num; ++i) {
+    LOG(INFO) << "join tid : " << i;
+    threads[i].join();
+    CompareRecord(&truth_output_data, &infer_output_data, 1e-4);
+  }
+
+  std::cout << "finish multi-thread test" << std::endl;
+}
+
+}  // namespace paddle_infer
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  return RUN_ALL_TESTS();
+}

test/cpp/inference/infer_ut/test_ernie_text_cls.cc

@@ -0,0 +1,139 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test_suite.h"  // NOLINT
+
+DEFINE_string(modeldir, "", "Directory of the inference model.");
+
+namespace paddle_infer {
+
+template <typename T>
+T cRandom(int min, int max) {
+  unsigned int seed = 100;
+  return (min +
+          static_cast<T>(max * rand_r(&seed) / static_cast<T>(RAND_MAX + 1)));
+}
+
+std::map<std::string, paddle::test::Record> PrepareInput(int batch_size) {
+  // init input data
+  int digit_length = 115;
+  paddle::test::Record input_ids, segment_ids;
+  int input_num = batch_size * digit_length;
+  std::vector<int64_t> input_data(input_num, 1);
+  std::vector<int64_t> segment_data(input_num, 0);
+  srand((unsigned)time(NULL));
+  for (int x = 0; x < input_data.size(); x++) {
+    input_data[x] = cRandom<int>(1, 100);
+  }
+  input_ids.data = std::vector<float>(input_data.begin(), input_data.end());
+  input_ids.shape = std::vector<int>{batch_size, digit_length};
+  input_ids.type = paddle::PaddleDType::INT64;
+
+  segment_ids.data =
+      std::vector<float>(segment_data.begin(), segment_data.end());
+  segment_ids.shape = std::vector<int>{batch_size, digit_length};
+  segment_ids.type = paddle::PaddleDType::INT64;
+
+  std::map<std::string, paddle::test::Record> my_input_data_map;
+  my_input_data_map.insert({"input_ids", input_ids});
+  my_input_data_map.insert({"token_type_ids", segment_ids});
+
+  return my_input_data_map;
+}
+
+TEST(gpu_tester_ernie_text_cls, analysis_gpu_bz2_buffer) {
+  // init input data
+  auto my_input_data_map = PrepareInput(2);
+  // init output data
+  std::map<std::string, paddle::test::Record> infer_output_data,
+      truth_output_data;
+  // prepare ground truth config
+  paddle_infer::Config config, config_no_ir;
+  config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                        FLAGS_modeldir + "/inference.pdiparams");
+  config_no_ir.SwitchIrOptim(false);
+
+  // prepare inference config from buffer
+  std::string prog_file = FLAGS_modeldir + "/inference.pdmodel";
+  std::string params_file = FLAGS_modeldir + "/inference.pdiparams";
+  std::string prog_str = paddle::test::read_file(prog_file);
+  std::string params_str = paddle::test::read_file(params_file);
+  config.SetModelBuffer(
+      prog_str.c_str(), prog_str.size(), params_str.c_str(), params_str.size());
+  // get ground truth by disable ir
+  paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
+  SingleThreadPrediction(
+      pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
+  // get infer results
+  paddle_infer::services::PredictorPool pred_pool(config, 1);
+  SingleThreadPrediction(
+      pred_pool.Retrieve(0), &my_input_data_map, &infer_output_data);
+  // check outputs
+  CompareRecord(&truth_output_data, &infer_output_data);
+  std::cout << "finish test" << std::endl;
+}
+
+TEST(onednn_tester_ernie_text_cls, multi_thread4_mkl_fp32_bz2) {
+  int thread_num = 4;
+  // init input data
+  auto my_input_data_map = PrepareInput(2);
+  // init output data
+  std::map<std::string, paddle::test::Record> infer_output_data,
+      truth_output_data;
+  // prepare ground truth config
+  paddle_infer::Config config, config_no_ir;
+  config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                        FLAGS_modeldir + "/inference.pdiparams");
+  config.DisableGpu();
+  config_no_ir.SwitchIrOptim(false);
+  // prepare inference config
+  config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                  FLAGS_modeldir + "/inference.pdiparams");
+  config.DisableGpu();
+  config.EnableONEDNN();
+  config.SetOnednnCacheCapacity(10);
+  config.SetCpuMathLibraryNumThreads(10);
+  // get ground truth by disable ir
+  paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
+  SingleThreadPrediction(
+      pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
+
+  // get infer results from multi threads
+  std::vector<std::thread> threads;
+  services::PredictorPool pred_pool(config, thread_num);
+  for (int i = 0; i < thread_num; ++i) {
+    threads.emplace_back(paddle::test::SingleThreadPrediction,
+                         pred_pool.Retrieve(i),
+                         &my_input_data_map,
+                         &infer_output_data,
+                         2);
+  }
+
+  // thread join & check outputs
+  for (int i = 0; i < thread_num; ++i) {
+    LOG(INFO) << "join tid : " << i;
+    threads[i].join();
+    CompareRecord(&truth_output_data, &infer_output_data);
+  }
+
+  std::cout << "finish multi-thread test" << std::endl;
+}
+
+}  // namespace paddle_infer
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  return RUN_ALL_TESTS();
+}

test/cpp/inference/infer_ut/test_ernie_xnli_int8.cc

@@ -0,0 +1,201 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test_helper.h"  // NOLINT
+#include "test_suite.h"   // NOLINT
+
+DEFINE_string(modeldir, "", "Directory of the inference model.");
+DEFINE_string(datadir, "", "dataset.");
+DEFINE_string(truth_data, "", "Directory of the inference data truth result");
+
+namespace paddle_infer {
+
+std::shared_ptr<Predictor> InitPredictor() {
+  Config config;
+  config.SetModel(FLAGS_modeldir + "/__model__",
+                  FLAGS_modeldir + "/__params__");
+  config.EnableUseGpu(1000, 0);
+  // Open the memory optim.
+  config.EnableMemoryOptim();
+
+  int max_batch = 32;
+  int max_single_seq_len = 128;
+  int opt_single_seq_len = 64;
+  int min_batch_seq_len = 1;
+  int max_batch_seq_len = 512;
+  int opt_batch_seq_len = 256;
+
+  std::string input_name0 = "eval_placeholder_0";
+  std::string input_name1 = "eval_placeholder_1";
+  std::string input_name2 = "eval_placeholder_2";
+  std::string input_name3 = "eval_placeholder_3";
+
+  std::vector<int> min_shape = {min_batch_seq_len};
+  std::vector<int> max_shape = {max_batch_seq_len};
+  std::vector<int> opt_shape = {opt_batch_seq_len};
+  // Set the input's min, max, opt shape
+  std::map<std::string, std::vector<int>> min_input_shape = {
+      {input_name0, min_shape},
+      {input_name1, min_shape},
+      {input_name2, {1}},
+      {input_name3, {1, min_batch_seq_len, 1}}};
+  std::map<std::string, std::vector<int>> max_input_shape = {
+      {input_name0, max_shape},
+      {input_name1, max_shape},
+      {input_name2, {max_batch + 1}},
+      {input_name3, {1, max_single_seq_len, 1}}};
+  std::map<std::string, std::vector<int>> opt_input_shape = {
+      {input_name0, opt_shape},
+      {input_name1, opt_shape},
+      {input_name2, {max_batch + 1}},
+      {input_name3, {1, opt_single_seq_len, 1}}};
+
+  // only kHalf supported
+  config.EnableTensorRtEngine(
+      1 << 30, 1, 5, Config::Precision::kInt8, false, false);
+  // ernie varlen must be used with dynamic shape
+  config.SetTRTDynamicShapeInfo(
+      min_input_shape, max_input_shape, opt_input_shape);
+  // ernie varlen must be used with oss
+  config.EnableVarseqlen();
+  paddle_infer::experimental::InternalUtils::SetTransformerPosid(&config,
+                                                                 input_name2);
+  paddle_infer::experimental::InternalUtils::SetTransformerMaskid(&config,
+                                                                  input_name3);
+
+  return CreatePredictor(config);
+}
+
+// Parse tensor from string
+template <typename T>
+std::vector<T> ParseTensor(const std::string &field) {
+  std::string mat_str = field;
+
+  std::vector<T> mat;
+  paddle::test::Split(mat_str, ' ', &mat);
+
+  return mat;
+}
+
+void run(Predictor *predictor, std::vector<float> *out_data) {
+  clock_t start, end;
+  start = clock();
+  CHECK(predictor->Run());
+  end = clock();
+
+  auto output_names = predictor->GetOutputNames();
+  auto output_t = predictor->GetOutputHandle(output_names[0]);
+  std::vector<int> output_shape = output_t->shape();
+  int out_num = std::accumulate(
+      output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
+  out_data->resize(out_num);
+  output_t->CopyToCpu(out_data->data());
+  return;
+}
+
+auto PrepareOutput(std::string input_file) -> std::deque<float> {
+  std::ifstream fin(input_file);
+  std::string line;
+  std::vector<std::string> buffer;
+  while (std::getline(fin, line)) {
+    buffer.emplace_back(line);
+  }
+  std::deque<float> resDeque(buffer.size());
+  std::transform(buffer.begin(),
+                 buffer.end(),
+                 resDeque.begin(),
+                 [](const std::string &val) { return std::stof(val); });
+
+  return resDeque;
+}  // PrepareOutput
+
+TEST(tensorrt_tester_ernie_xnli, oss_varlen_truth_data_int8) {
+  auto resDeque = PrepareOutput(FLAGS_truth_data);
+  auto predictor = InitPredictor();
+
+  ASSERT_FALSE(FLAGS_datadir.empty());
+  std::ifstream fin(FLAGS_datadir);
+  std::string line;
+
+  int lineno = 0;
+  const int max_seq_len = 128;
+  const int run_batch = 1;
+  int correct_num = 0;
+  while (std::getline(fin, line)) {
+    std::vector<std::string> fields;
+    paddle::test::Split(line, ';', &fields);
+
+    auto src_ids = ParseTensor<int32_t>(fields[0]);
+    auto sent_ids = ParseTensor<int32_t>(fields[1]);
+    auto pos_ids = ParseTensor<int64_t>(fields[2]);
+
+    int run_seq_len = src_ids.size();
+    int32_t i3[2] = {0, run_seq_len};
+    int32_t i4[max_seq_len] = {0};
+
+    auto input_names = predictor->GetInputNames();
+
+    // first input
+    auto input_t1 = predictor->GetInputHandle(input_names[0]);
+    input_t1->Reshape({run_seq_len});
+    input_t1->CopyFromCpu(src_ids.data());
+
+    // second input
+    auto input_t2 = predictor->GetInputHandle(input_names[1]);
+    input_t2->Reshape({run_seq_len});
+    input_t2->CopyFromCpu(sent_ids.data());
+
+    // third input
+    auto input_t3 = predictor->GetInputHandle(input_names[2]);
+    input_t3->Reshape({run_batch + 1});
+    input_t3->CopyFromCpu(i3);
+
+    // fourth input
+    auto input_t4 = predictor->GetInputHandle(input_names[3]);
+    input_t4->Reshape({1, max_seq_len, 1});
+    input_t4->CopyFromCpu(i4);
+
+    std::vector<float> out_data;
+    run(predictor.get(), &out_data);
+
+    lineno++;
+    int maxPosition =
+        max_element(out_data.begin(), out_data.end()) - out_data.begin();
+
+    if (maxPosition == resDeque[0]) {
+      correct_num += 1;
+    }
+    resDeque.pop_front();
+
+    VLOG(2) << "predict result: " << maxPosition;
+    for (auto r : out_data) {
+      VLOG(2) << r;
+    }
+  }
+  ASSERT_GT(correct_num,
+            3855);  // total input 5010, int8 res should greater than 3855
+  LOG(INFO) << "=== finish oss test ===";
+}
+
+}  // namespace paddle_infer
+
+int main(int argc, char **argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+
+#if IS_TRT_VERSION_GE(7200)
+  return RUN_ALL_TESTS();
+#endif
+  return 0;
+}

test/cpp/inference/infer_ut/test_helper.h

@@ -0,0 +1,81 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+#include <sstream>
+#include <string>
+#include <vector>
+
+namespace paddle {
+namespace test {
+
+// split string to vector<string> by sep
+static void split(const std::string &str,
+                  char sep,
+                  std::vector<std::string> *pieces,
+                  bool ignore_null = true) {
+  pieces->clear();
+  if (str.empty()) {
+    if (!ignore_null) {
+      pieces->push_back(str);
+    }
+    return;
+  }
+  size_t pos = 0;
+  size_t next = str.find(sep, pos);
+  while (next != std::string::npos) {
+    pieces->push_back(str.substr(pos, next - pos));
+    pos = next + 1;
+    next = str.find(sep, pos);
+  }
+  if (!str.substr(pos).empty()) {
+    pieces->push_back(str.substr(pos));
+  }
+}
+
+template <typename T>
+void GetValueFromStream(std::stringstream *ss, T *t) {
+  (*ss) >> (*t);
+}
+
+template <>
+void GetValueFromStream<std::string>(std::stringstream *ss, std::string *t) {
+  *t = ss->str();
+}
+
+// Split string to multiple vector
+template <typename T>
+void Split(const std::string &line, char sep, std::vector<T> *v) {
+  std::stringstream ss;
+  T t;
+  for (auto c : line) {
+    if (c != sep) {
+      ss << c;
+    } else {
+      GetValueFromStream<T>(&ss, &t);
+      v->push_back(std::move(t));
+      ss.str({});
+      ss.clear();
+    }
+  }
+
+  if (!ss.str().empty()) {
+    GetValueFromStream<T>(&ss, &t);
+    v->push_back(std::move(t));
+    ss.str({});
+    ss.clear();
+  }
+}
+
+}  // namespace test
+}  // namespace paddle

test/cpp/inference/infer_ut/test_mobilnetv1.cc

@@ -0,0 +1,86 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test_helper.h"  // NOLINT
+#include "test_suite.h"   // NOLINT
+
+DEFINE_string(modeldir, "", "Directory of the inference model.");
+
+namespace paddle_infer {
+
+paddle::test::Record PrepareInput(int batch_size, int shape_size = 224) {
+  // init input data
+  int channel = 3;
+  int width = shape_size;   // w = 224
+  int height = shape_size;  // h = 224
+  paddle::test::Record image_Record;
+  int input_num = batch_size * channel * width * height;
+  std::vector<float> input_data(input_num, 1);
+  image_Record.data = input_data;
+  image_Record.shape = std::vector<int>{batch_size, channel, width, height};
+  image_Record.type = paddle::PaddleDType::FLOAT32;
+  return image_Record;
+}
+
+TEST(tensorrt_tester_mobilenetv1, tuned_dynamic_trt_fp32_bz2) {
+  bool tuned_shape = true;
+  std::string shape_range_info = FLAGS_modeldir + "/shape_range_info.pbtxt";
+  LOG(INFO) << "tensorrt tuned info saved to " << shape_range_info;
+
+  // init input data
+  std::map<std::string, paddle::test::Record> my_input_data_map;
+  my_input_data_map["x"] = PrepareInput(2, 448);
+  // init output data
+  std::map<std::string, paddle::test::Record> infer_output_data,
+      truth_output_data;
+  if (tuned_shape) {
+    // NOTE: shape_range_info will be saved after destructor of predictor
+    // function
+    // prepare ground truth config
+    paddle_infer::Config tune_config;
+    tune_config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                         FLAGS_modeldir + "/inference.pdiparams");
+    tune_config.SwitchIrOptim(false);
+    tune_config.EnableUseGpu(1000, 0);
+    tune_config.CollectShapeRangeInfo(shape_range_info);
+
+    auto predictor_tune = paddle_infer::CreatePredictor(tune_config);
+    SingleThreadPrediction(
+        predictor_tune.get(), &my_input_data_map, &truth_output_data, 1);
+  }
+
+  // prepare inference config
+  paddle_infer::Config config;
+  config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                  FLAGS_modeldir + "/inference.pdiparams");
+  config.EnableUseGpu(1000, 0);
+  config.EnableTensorRtEngine(
+      1 << 20, 2, 5, paddle_infer::PrecisionType::kFloat32, false, false);
+  config.EnableTunedTensorRtDynamicShape(shape_range_info, true);
+  LOG(INFO) << config.Summary();
+  paddle_infer::services::PredictorPool pred_pool(config, 1);
+  SingleThreadPrediction(
+      pred_pool.Retrieve(0), &my_input_data_map, &infer_output_data);
+  // check outputs
+  CompareRecord(&truth_output_data, &infer_output_data);
+  VLOG(1) << "finish test";
+}
+
+}  // namespace paddle_infer
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  return RUN_ALL_TESTS();
+}

test/cpp/inference/infer_ut/test_ppyolo_mbv3.cc

@@ -0,0 +1,160 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test_suite.h"  // NOLINT
+
+DEFINE_string(modeldir, "", "Directory of the inference model.");
+
+namespace paddle_infer {
+
+std::map<std::string, paddle::test::Record> PrepareInput(int batch_size) {
+  // init input data
+  int channel = 3;
+  int width = 320;
+  int height = 320;
+  paddle::test::Record image, im_shape, scale_factor;
+  int input_num = batch_size * channel * width * height;
+  int shape_num = batch_size * 2;
+  std::vector<float> image_data(input_num, 1);
+  for (int i = 1; i < input_num + 1; ++i) {
+    image_data[i] = i % 10 * 0.5;
+  }
+  std::vector<float> im_shape_data(shape_num, 1);
+  std::vector<float> scale_factor_data(shape_num, 1);
+
+  image.data = std::vector<float>(image_data.begin(), image_data.end());
+  image.shape = std::vector<int>{batch_size, channel, width, height};
+  image.type = paddle::PaddleDType::FLOAT32;
+
+  im_shape.data =
+      std::vector<float>(im_shape_data.begin(), im_shape_data.end());
+  im_shape.shape = std::vector<int>{batch_size, 2};
+  im_shape.type = paddle::PaddleDType::FLOAT32;
+
+  scale_factor.data =
+      std::vector<float>(scale_factor_data.begin(), scale_factor_data.end());
+  scale_factor.shape = std::vector<int>{batch_size, 2};
+  scale_factor.type = paddle::PaddleDType::FLOAT32;
+
+  std::map<std::string, paddle::test::Record> input_data_map;
+  input_data_map.insert({"image", image});
+  input_data_map.insert({"im_shape", im_shape});
+  input_data_map.insert({"scale_factor", scale_factor});
+
+  return input_data_map;
+}
+
+TEST(tensorrt_tester_ppyolo_mbv3, multi_thread4_trt_fp32_bz2) {
+  int thread_num = 4;
+  // init input data
+  auto input_data_map = PrepareInput(2);
+  // init output data
+  std::map<std::string, paddle::test::Record> infer_output_data,
+      truth_output_data;
+  // prepare ground truth config
+  paddle_infer::Config config, config_no_ir;
+  config_no_ir.SetModel(FLAGS_modeldir + "/model.pdmodel",
+                        FLAGS_modeldir + "/model.pdiparams");
+  config_no_ir.EnableUseGpu(100, 0);
+  config_no_ir.SwitchIrOptim(false);
+  // prepare inference config
+  config.SetModel(FLAGS_modeldir + "/model.pdmodel",
+                  FLAGS_modeldir + "/model.pdiparams");
+  config.EnableUseGpu(100, 0);
+  config.EnableTensorRtEngine(
+      1 << 25, 2, 3, paddle_infer::PrecisionType::kFloat32, false, false);
+  LOG(INFO) << config.Summary();
+  // get ground truth by disable ir
+  paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
+  SingleThreadPrediction(
+      pred_pool_no_ir.Retrieve(0), &input_data_map, &truth_output_data, 1);
+
+  // get infer results from multi threads
+  std::vector<std::thread> threads;
+  services::PredictorPool pred_pool(config, thread_num);
+  for (int i = 0; i < thread_num; ++i) {
+    threads.emplace_back(paddle::test::SingleThreadPrediction,
+                         pred_pool.Retrieve(i),
+                         &input_data_map,
+                         &infer_output_data,
+                         2);
+  }
+
+  // thread join & check outputs
+  for (int i = 0; i < thread_num; ++i) {
+    LOG(INFO) << "join tid : " << i;
+    threads[i].join();
+    CompareRecord(&truth_output_data, &infer_output_data, 0.18);
+    // TODO(OliverLPH): precision set to 1e-2 since input is fake, change to
+    // real input later
+  }
+
+  std::cout << "finish multi-thread test" << std::endl;
+}
+
+TEST(DISABLED_onednn_tester_ppyolo_mbv3, multi_thread4_mkl_bz2) {
+  // TODO(OliverLPH): onednn multi thread will fail
+  int thread_num = 4;
+  // init input data
+  auto input_data_map = PrepareInput(2);
+  // init output data
+  std::map<std::string, paddle::test::Record> infer_output_data,
+      truth_output_data;
+  // prepare ground truth config
+  paddle_infer::Config config, config_no_ir;
+  config_no_ir.SetModel(FLAGS_modeldir + "/model.pdmodel",
+                        FLAGS_modeldir + "/model.pdiparams");
+  config_no_ir.DisableGpu();
+  config_no_ir.SwitchIrOptim(false);
+  // prepare inference config
+  config.SetModel(FLAGS_modeldir + "/model.pdmodel",
+                  FLAGS_modeldir + "/model.pdiparams");
+  config.DisableGpu();
+  config.EnableONEDNN();
+  config.SetOnednnCacheCapacity(10);
+  config.SetCpuMathLibraryNumThreads(10);
+  LOG(INFO) << config.Summary();
+  // get ground truth by disable ir
+  paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
+  SingleThreadPrediction(
+      pred_pool_no_ir.Retrieve(0), &input_data_map, &truth_output_data, 1);
+
+  // get infer results from multi threads
+  std::vector<std::thread> threads;
+  services::PredictorPool pred_pool(config, thread_num);
+  for (int i = 0; i < thread_num; ++i) {
+    threads.emplace_back(paddle::test::SingleThreadPrediction,
+                         pred_pool.Retrieve(i),
+                         &input_data_map,
+                         &infer_output_data,
+                         2);
+  }
+
+  // thread join & check outputs
+  for (int i = 0; i < thread_num; ++i) {
+    LOG(INFO) << "join tid : " << i;
+    threads[i].join();
+    CompareRecord(&truth_output_data, &infer_output_data, 1e-4);
+  }
+
+  std::cout << "finish multi-thread test" << std::endl;
+}
+
+}  // namespace paddle_infer
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  return RUN_ALL_TESTS();
+}

test/cpp/inference/infer_ut/test_ppyolov2_r50vd.cc

@@ -0,0 +1,168 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test_suite.h"  // NOLINT
+
+DEFINE_string(modeldir, "", "Directory of the inference model.");
+
+namespace paddle_infer {
+
+std::map<std::string, paddle::test::Record> PrepareInput(int batch_size) {
+  // init input data
+  int channel = 3;
+  int width = 640;
+  int height = 640;
+  paddle::test::Record image, im_shape, scale_factor;
+  int input_num = batch_size * channel * width * height;
+  int shape_num = batch_size * 2;
+  std::vector<float> image_data(input_num, 1);
+  for (int i = 1; i < input_num + 1; ++i) {
+    image_data[i] = i % 10 * 0.5;
+  }
+  std::vector<float> im_shape_data(shape_num, 1);
+  std::vector<float> scale_factor_data(shape_num, 1);
+
+  image.data = std::vector<float>(image_data.begin(), image_data.end());
+  image.shape = std::vector<int>{batch_size, channel, width, height};
+  image.type = paddle::PaddleDType::FLOAT32;
+
+  im_shape.data =
+      std::vector<float>(im_shape_data.begin(), im_shape_data.end());
+  im_shape.shape = std::vector<int>{batch_size, 2};
+  im_shape.type = paddle::PaddleDType::FLOAT32;
+
+  scale_factor.data =
+      std::vector<float>(scale_factor_data.begin(), scale_factor_data.end());
+  scale_factor.shape = std::vector<int>{batch_size, 2};
+  scale_factor.type = paddle::PaddleDType::FLOAT32;
+
+  std::map<std::string, paddle::test::Record> input_data_map;
+  input_data_map.insert({"image", image});
+  input_data_map.insert({"im_shape", im_shape});
+  input_data_map.insert({"scale_factor", scale_factor});
+
+  return input_data_map;
+}
+
+TEST(tensorrt_tester_ppyolov2_r50vd, multi_thread2_trt_fp32_bz1) {
+  int thread_num = 2;  // thread > 2 may OOM
+  // init input data
+  auto input_data_map = PrepareInput(1);
+  // init output data
+  std::map<std::string, paddle::test::Record> infer_output_data,
+      truth_output_data;
+  // prepare ground truth config
+  paddle_infer::Config config, config_no_ir;
+  config_no_ir.SetModel(FLAGS_modeldir + "/model.pdmodel",
+                        FLAGS_modeldir + "/model.pdiparams");
+  config_no_ir.EnableUseGpu(100, 0);
+  config_no_ir.SwitchIrOptim(false);
+  // prepare inference config
+  config.SetModel(FLAGS_modeldir + "/model.pdmodel",
+                  FLAGS_modeldir + "/model.pdiparams");
+  config.EnableUseGpu(100, 0);
+  config.EnableTensorRtEngine(
+      1 << 28, 2, 10, paddle_infer::PrecisionType::kFloat32, false, false);
+  std::map<std::string, std::vector<int>> input_shape;
+  input_shape["image"] = {1, 3, 640, 640};
+  input_shape["im_shape"] = {1, 2};
+  input_shape["scale_factor"] = {1, 2};
+  config.SetTRTDynamicShapeInfo(input_shape, input_shape, input_shape, false);
+  LOG(INFO) << config.Summary();
+  // get ground truth by disable ir
+  paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
+  SingleThreadPrediction(
+      pred_pool_no_ir.Retrieve(0), &input_data_map, &truth_output_data, 1);
+
+  // get infer results from multi threads
+  std::vector<std::thread> threads;
+  services::PredictorPool pred_pool(config, thread_num);
+  for (int i = 0; i < thread_num; ++i) {
+    threads.emplace_back(paddle::test::SingleThreadPrediction,
+                         pred_pool.Retrieve(i),
+                         &input_data_map,
+                         &infer_output_data,
+                         2);
+  }
+
+  // thread join & check outputs
+  for (int i = 0; i < thread_num; ++i) {
+    LOG(INFO) << "join tid : " << i;
+    threads[i].join();
+    // CompareRecord(&truth_output_data, &infer_output_data, 1e-2);
+    // TODO(OliverLPH): disable comparison since precision is low
+  }
+
+  std::cout << "finish multi-thread test" << std::endl;
+}
+
+// fused_softplus is about to be removed, the test uses fused_softplus and is
+// disabled
+/*
+TEST(onednn_tester_ppyolov2_r50vd, multi_thread2_mkl_bz2) {
+  int thread_num = 2;
+  // init input data
+  auto input_data_map = PrepareInput(2);
+  // init output data
+  std::map<std::string, paddle::test::Record> infer_output_data,
+      truth_output_data;
+  // prepare ground truth config
+  paddle_infer::Config config, config_no_ir;
+  config_no_ir.SetModel(FLAGS_modeldir + "/model.pdmodel",
+                        FLAGS_modeldir + "/model.pdiparams");
+  config_no_ir.DisableGpu();
+  config_no_ir.SwitchIrOptim(false);
+  // prepare inference config
+  config.SetModel(FLAGS_modeldir + "/model.pdmodel",
+                  FLAGS_modeldir + "/model.pdiparams");
+  config.DisableGpu();
+  config.EnableONEDNN();
+  config.SetOnednnCacheCapacity(10);
+  config.SetCpuMathLibraryNumThreads(10);
+  LOG(INFO) << config.Summary();
+  // get ground truth by disable ir
+  paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
+  SingleThreadPrediction(
+      pred_pool_no_ir.Retrieve(0), &input_data_map, &truth_output_data, 1);
+
+  // get infer results from multi threads
+  std::vector<std::thread> threads;
+  services::PredictorPool pred_pool(config, thread_num);
+  for (int i = 0; i < thread_num; ++i) {
+    threads.emplace_back(paddle::test::SingleThreadPrediction,
+                         pred_pool.Retrieve(i),
+                         &input_data_map,
+                         &infer_output_data,
+                         2);
+  }
+
+  // thread join & check outputs
+  for (int i = 0; i < thread_num; ++i) {
+    LOG(INFO) << "join tid : " << i;
+    threads[i].join();
+    // CompareRecord(&truth_output_data, &infer_output_data, 1e-4);
+    // TODO(OliverLPH): disable comparison since precision is low
+  }
+
+  std::cout << "finish multi-thread test" << std::endl;
+}
+*/
+
+}  // namespace paddle_infer
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  return RUN_ALL_TESTS();
+}

test/cpp/inference/infer_ut/test_resnet50.cc

@@ -0,0 +1,247 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test_suite.h"  // NOLINT
+
+DEFINE_string(modeldir, "", "Directory of the inference model.");
+
+namespace paddle_infer {
+
+paddle::test::Record PrepareInput(int batch_size) {
+  // init input data
+  int channel = 3;
+  int width = 224;
+  int height = 224;
+  paddle::test::Record image_Record;
+  int input_num = batch_size * channel * width * height;
+  std::vector<float> input_data(input_num, 1);
+  image_Record.data = input_data;
+  image_Record.shape = std::vector<int>{batch_size, channel, width, height};
+  image_Record.type = paddle::PaddleDType::FLOAT32;
+  return image_Record;
+}
+
+TEST(gpu_tester_resnet50, analysis_gpu_bz1) {
+  // init input data
+  std::map<std::string, paddle::test::Record> my_input_data_map;
+  my_input_data_map["inputs"] = PrepareInput(1);
+  // init output data
+  std::map<std::string, paddle::test::Record> infer_output_data,
+      truth_output_data;
+  // prepare ground truth config
+  paddle_infer::Config config, config_no_ir;
+  config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                        FLAGS_modeldir + "/inference.pdiparams");
+  config_no_ir.SwitchIrOptim(false);
+  // prepare inference config
+  config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                  FLAGS_modeldir + "/inference.pdiparams");
+  // get ground truth by disable ir
+  paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
+  SingleThreadPrediction(
+      pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
+  // get infer results
+  paddle_infer::services::PredictorPool pred_pool(config, 1);
+  SingleThreadPrediction(
+      pred_pool.Retrieve(0), &my_input_data_map, &infer_output_data);
+  // check outputs
+  CompareRecord(&truth_output_data, &infer_output_data);
+  std::cout << "finish test" << std::endl;
+}
+
+TEST(tensorrt_tester_resnet50, trt_fp32_bz2) {
+  // init input data
+  std::map<std::string, paddle::test::Record> my_input_data_map;
+  my_input_data_map["inputs"] = PrepareInput(2);
+  // init output data
+  std::map<std::string, paddle::test::Record> infer_output_data,
+      truth_output_data;
+  // prepare ground truth config
+  paddle_infer::Config config, config_no_ir;
+  config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                        FLAGS_modeldir + "/inference.pdiparams");
+  config_no_ir.SwitchIrOptim(false);
+  // prepare inference config
+  config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                  FLAGS_modeldir + "/inference.pdiparams");
+  config.EnableUseGpu(100, 0);
+  config.EnableTensorRtEngine(
+      1 << 20, 2, 3, paddle_infer::PrecisionType::kFloat32, false, false);
+  // get ground truth by disable ir
+  paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
+  SingleThreadPrediction(
+      pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
+  // get infer results
+  paddle_infer::services::PredictorPool pred_pool(config, 1);
+  SingleThreadPrediction(
+      pred_pool.Retrieve(0), &my_input_data_map, &infer_output_data);
+  // check outputs
+  CompareRecord(&truth_output_data, &infer_output_data, 2e-4);
+  std::cout << "finish test" << std::endl;
+}
+
+TEST(tensorrt_tester_resnet50, serial_diff_batch_trt_fp32) {
+  int max_batch_size = 5;
+  // prepare ground truth config
+  paddle_infer::Config config, config_no_ir;
+  config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                        FLAGS_modeldir + "/inference.pdiparams");
+  config_no_ir.SwitchIrOptim(false);
+  paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
+  // prepare inference config
+  config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                  FLAGS_modeldir + "/inference.pdiparams");
+  config.EnableUseGpu(100, 0);
+  config.EnableTensorRtEngine(1 << 20,
+                              max_batch_size,
+                              3,
+                              paddle_infer::PrecisionType::kFloat32,
+                              false,
+                              false);
+  paddle_infer::services::PredictorPool pred_pool(config, 1);
+
+  for (int i = 1; i < max_batch_size; i++) {
+    // init input data
+    std::map<std::string, paddle::test::Record> my_input_data_map;
+    my_input_data_map["inputs"] = PrepareInput(i);
+    // init output data
+    std::map<std::string, paddle::test::Record> infer_output_data,
+        truth_output_data;
+    // get ground truth by disable ir
+    SingleThreadPrediction(
+        pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
+    // get infer results
+    SingleThreadPrediction(
+        pred_pool.Retrieve(0), &my_input_data_map, &infer_output_data);
+    // check outputs
+    CompareRecord(&truth_output_data, &infer_output_data, 1e-4);
+  }
+  std::cout << "finish test" << std::endl;
+}
+
+TEST(tensorrt_tester_resnet50, multi_thread4_trt_fp32_bz2) {
+  int thread_num = 4;
+  // init input data
+  std::map<std::string, paddle::test::Record> my_input_data_map;
+  my_input_data_map["inputs"] = PrepareInput(2);
+  // init output data
+  std::map<std::string, paddle::test::Record> infer_output_data,
+      truth_output_data;
+  // prepare ground truth config
+  paddle_infer::Config config, config_no_ir;
+  config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                        FLAGS_modeldir + "/inference.pdiparams");
+  config_no_ir.SwitchIrOptim(false);
+  // prepare inference config
+  config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                  FLAGS_modeldir + "/inference.pdiparams");
+  config.EnableUseGpu(100, 0);
+  config.EnableTensorRtEngine(
+      1 << 20, 2, 3, paddle_infer::PrecisionType::kFloat32, false, false);
+  // get ground truth by disable ir
+  paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
+  SingleThreadPrediction(
+      pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
+
+  // get infer results from multi threads
+  std::vector<std::thread> threads;
+  services::PredictorPool pred_pool(config, thread_num);
+  for (int i = 0; i < thread_num; ++i) {
+    threads.emplace_back(paddle::test::SingleThreadPrediction,
+                         pred_pool.Retrieve(i),
+                         &my_input_data_map,
+                         &infer_output_data,
+                         2);
+  }
+
+  // thread join & check outputs
+  for (int i = 0; i < thread_num; ++i) {
+    LOG(INFO) << "join tid : " << i;
+    threads[i].join();
+    CompareRecord(&truth_output_data, &infer_output_data, 2e-4);
+  }
+
+  std::cout << "finish multi-thread test" << std::endl;
+}
+
+TEST(tensorrt_tester_resnet50, trt_int8_bz2) {
+  // init input data
+  std::map<std::string, paddle::test::Record> my_input_data_map;
+  my_input_data_map["inputs"] = PrepareInput(2);
+  // init output data
+  std::map<std::string, paddle::test::Record> infer_output_data,
+      truth_output_data;
+  // prepare inference config
+  paddle_infer::Config config;
+  config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                  FLAGS_modeldir + "/inference.pdiparams");
+  config.EnableUseGpu(100, 0);
+  config.EnableTensorRtEngine(
+      1 << 20, 2, 3, paddle_infer::PrecisionType::kInt8, true, true);
+
+  // get first time prediction int8 results
+  paddle_infer::services::PredictorPool pred_pool(config, 1);
+  SingleThreadPrediction(
+      pred_pool.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
+
+  // get repeat 5 times prediction int8 results
+  SingleThreadPrediction(
+      pred_pool.Retrieve(0), &my_input_data_map, &infer_output_data, 5);
+
+  // check outputs
+  CompareRecord(&truth_output_data, &infer_output_data);
+  std::cout << "finish test" << std::endl;
+}
+
+TEST(DISABLED_tensorrt_tester_resnet50, profile_multi_thread_trt_fp32) {
+  int batch_size = 2;
+  int thread_num = 4;
+  int repeat_time = 1000;
+  // init input data
+  std::map<std::string, paddle::test::Record> my_input_data_map;
+  my_input_data_map["inputs"] = PrepareInput(batch_size);
+  // init output data
+  std::map<std::string, paddle::test::Record> infer_output_data;
+  // prepare inference config
+  paddle_infer::Config config;
+  config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                  FLAGS_modeldir + "/inference.pdiparams");
+  config.EnableUseGpu(100, 0);
+  config.EnableTensorRtEngine(
+      1 << 20, 2, 3, paddle_infer::PrecisionType::kFloat32, false, false);
+  // get infer results from multi threads
+  services::PredictorPool pred_pool(config, thread_num);
+  std::vector<std::future<double>> calcs;
+  for (int i = 0; i < thread_num; ++i) {
+    calcs.push_back(std::async(&paddle::test::SingleThreadProfile,
+                               pred_pool.Retrieve(i),
+                               &my_input_data_map,
+                               repeat_time));
+  }
+  double total_time_ = 0.0;
+  for (auto&& fut : calcs) {
+    total_time_ += fut.get();
+  }
+  std::cout << total_time_ << std::endl;
+
+  std::cout << "finish multi-thread profile" << std::endl;
+}
+
+}  // namespace paddle_infer
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  return RUN_ALL_TESTS();
+}

test/cpp/inference/infer_ut/test_resnet50_quant.cc

@@ -0,0 +1,176 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test_suite.h"  // NOLINT
+
+DEFINE_string(modeldir, "", "Directory of the inference model.");
+DEFINE_string(int8dir, "", "Directory of the quant inference model.");
+DEFINE_string(datadir, "", "Directory of the infer data.");
+
+namespace paddle_infer {
+
+paddle::test::Record PrepareInput(int batch_size) {
+  // init input data
+  int channel = 3;
+  int width = 224;
+  int height = 224;
+  paddle::test::Record image_Record;
+  int input_num = batch_size * channel * width * height;
+
+  // load from binary data
+  std::ifstream fs(FLAGS_datadir, std::ifstream::binary);
+  EXPECT_TRUE(fs.is_open());
+  CHECK(fs.is_open());
+
+  float* input = new float[input_num];
+  memset(input, 0, input_num * sizeof(float));
+  auto input_data_tmp = input;
+  for (int i = 0; i < input_num; ++i) {
+    fs.read(reinterpret_cast<char*>(input_data_tmp), sizeof(*input_data_tmp));
+    input_data_tmp++;
+  }
+  int label = 0;
+  fs.read(reinterpret_cast<char*>(&label), sizeof(label));
+  fs.close();
+
+  std::vector<float> input_data{input, input + input_num};
+  image_Record.data = input_data;
+  image_Record.shape = std::vector<int>{batch_size, channel, width, height};
+  image_Record.type = paddle::PaddleDType::FLOAT32;
+  image_Record.label = label;
+  return image_Record;
+}
+
+TEST(DISABLED_tensorrt_tester_resnet50_quant, multi_thread4_trt_int8_bz1) {
+  int thread_num = 4;
+  // init input data
+  std::map<std::string, paddle::test::Record> input_data_map;
+  input_data_map["image"] = PrepareInput(1);
+  // init output data
+  std::map<std::string, paddle::test::Record> infer_output_data;
+  // prepare inference config
+  paddle_infer::Config config;
+  config.SetModel(FLAGS_int8dir);
+  config.EnableUseGpu(1000, 0);
+  config.EnableTensorRtEngine(
+      1 << 20, 10, 3, paddle_infer::PrecisionType::kInt8, false, false);
+  // get infer results from multi threads
+  std::vector<std::thread> threads;
+  services::PredictorPool pred_pool(config, thread_num);
+  for (int i = 0; i < thread_num; ++i) {
+    threads.emplace_back(paddle::test::SingleThreadPrediction,
+                         pred_pool.Retrieve(i),
+                         &input_data_map,
+                         &infer_output_data,
+                         5);
+  }
+
+  // thread join & check outputs
+  for (int i = 0; i < thread_num; ++i) {
+    LOG(INFO) << "join tid : " << i;
+    threads[i].join();
+
+    // check outputs
+    std::vector<int> index(1000);
+    std::iota(index.begin(), index.end(), 0);
+    auto out_data =
+        infer_output_data["save_infer_model/scale_0.tmp_0"].data.data();
+    std::sort(index.begin(), index.end(), [out_data](size_t i1, size_t i2) {
+      return out_data[i1] > out_data[i2];
+    });
+    // compare inference & ground truth label
+    ASSERT_EQ(index[0], input_data_map["image"].label);
+  }
+
+  std::cout << "finish test" << std::endl;
+}
+
+TEST(DISABLED_tensorrt_tester_resnet50_quant, multi_thread_multi_instance) {
+  int thread_num = 4;
+  // init input data
+  std::map<std::string, paddle::test::Record> input_data_fp32, input_data_quant;
+  input_data_quant["image"] = PrepareInput(1);
+  input_data_fp32["inputs"] = PrepareInput(1);
+
+  // init output data
+  std::map<std::string, paddle::test::Record> infer_output_data;
+  // prepare inference config
+  paddle_infer::Config config_fp32, config_quant;
+  config_fp32.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                       FLAGS_modeldir + "/inference.pdiparams");
+  config_fp32.EnableUseGpu(1000, 0);
+  config_fp32.EnableTensorRtEngine(
+      1 << 20, 10, 3, paddle_infer::PrecisionType::kFloat32, false, false);
+
+  config_quant.SetModel(FLAGS_int8dir);
+  config_quant.EnableUseGpu(1000, 0);
+  config_quant.EnableTensorRtEngine(
+      1 << 20, 10, 3, paddle_infer::PrecisionType::kInt8, false, false);
+
+  // get infer results from multi threads
+  std::vector<std::thread> threads;
+  services::PredictorPool pred_pool_fp32(config_fp32, thread_num);
+  services::PredictorPool pred_pool_quant(config_quant, thread_num);
+  for (int i = 0; i < thread_num; ++i) {
+    if (i % 2 == 0) {
+      threads.emplace_back(paddle::test::SingleThreadPrediction,
+                           pred_pool_fp32.Retrieve(i),
+                           &input_data_fp32,
+                           &infer_output_data,
+                           5);
+    } else {
+      threads.emplace_back(paddle::test::SingleThreadPrediction,
+                           pred_pool_quant.Retrieve(i),
+                           &input_data_quant,
+                           &infer_output_data,
+                           5);
+    }
+  }
+
+  // thread join & check outputs
+  for (int i = 0; i < thread_num; ++i) {
+    LOG(INFO) << "join tid : " << i;
+    std::vector<int> index(1000);
+    threads[i].join();
+    if (i % 2 == 0) {
+      // check outputs
+      std::iota(index.begin(), index.end(), 0);
+      auto out_data =
+          infer_output_data["save_infer_model/scale_0.tmp_0"].data.data();
+      std::sort(index.begin(), index.end(), [out_data](size_t i1, size_t i2) {
+        return out_data[i1] > out_data[i2];
+      });
+      // compare inference & ground truth label
+      ASSERT_EQ(index[0], input_data_fp32["inputs"].label);
+    } else {
+      // check outputs
+      std::iota(index.begin(), index.end(), 0);
+      auto out_data =
+          infer_output_data["save_infer_model/scale_0.tmp_0"].data.data();
+      std::sort(index.begin(), index.end(), [out_data](size_t i1, size_t i2) {
+        return out_data[i1] > out_data[i2];
+      });
+      // compare inference & ground truth label
+      ASSERT_EQ(index[0], input_data_quant["image"].label);
+    }
+  }
+}
+
+}  // namespace paddle_infer
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  return RUN_ALL_TESTS();
+}

test/cpp/inference/infer_ut/test_suite.h

@@ -0,0 +1,260 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+#include <math.h>
+
+#include <algorithm>
+#include <deque>
+#include <fstream>
+#include <future>
+#include <iostream>
+#include <numeric>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include "gflags/gflags.h"
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "paddle/include/paddle_inference_api.h"
+
+namespace paddle {
+namespace test {
+
+#define IS_TRT_VERSION_GE(version)                       \
+  ((NV_TENSORRT_MAJOR * 1000 + NV_TENSORRT_MINOR * 100 + \
+    NV_TENSORRT_PATCH * 10 + NV_TENSORRT_BUILD) >= version)
+
+#define IS_TRT_VERSION_LT(version)                       \
+  ((NV_TENSORRT_MAJOR * 1000 + NV_TENSORRT_MINOR * 100 + \
+    NV_TENSORRT_PATCH * 10 + NV_TENSORRT_BUILD) < version)
+
+#define TRT_VERSION                                    \
+  NV_TENSORRT_MAJOR * 1000 + NV_TENSORRT_MINOR * 100 + \
+      NV_TENSORRT_PATCH * 10 + NV_TENSORRT_BUILD
+
+class Record {
+ public:
+  std::vector<float> data;
+  std::vector<int32_t> shape;
+  paddle::PaddleDType type;
+  int label;
+};
+
+std::string read_file(std::string filename) {
+  std::ifstream file(filename);
+  return std::string((std::istreambuf_iterator<char>(file)),
+                     std::istreambuf_iterator<char>());
+}
+
+void SingleThreadPrediction(paddle_infer::Predictor *predictor,
+                            std::map<std::string, Record> *input_data_map,
+                            std::map<std::string, Record> *output_data_map,
+                            int repeat_times = 2) {
+  // prepare input tensor
+  auto input_names = predictor->GetInputNames();
+  for (const auto &[key, value] : *input_data_map) {
+    switch (value.type) {
+      case paddle::PaddleDType::INT64: {
+        std::vector<int64_t> input_value =
+            std::vector<int64_t>(value.data.begin(), value.data.end());
+        auto input_tensor = predictor->GetInputHandle(key);
+        input_tensor->Reshape(value.shape);
+        input_tensor->CopyFromCpu(input_value.data());
+        break;
+      }
+      case paddle::PaddleDType::INT32: {
+        std::vector<int32_t> input_value =
+            std::vector<int32_t>(value.data.begin(), value.data.end());
+        auto input_tensor = predictor->GetInputHandle(key);
+        input_tensor->Reshape(value.shape);
+        input_tensor->CopyFromCpu(input_value.data());
+        break;
+      }
+      case paddle::PaddleDType::FLOAT32: {
+        std::vector<float> input_value =
+            std::vector<float>(value.data.begin(), value.data.end());
+        auto input_tensor = predictor->GetInputHandle(key);
+        input_tensor->Reshape(value.shape);
+        input_tensor->CopyFromCpu(input_value.data());
+        break;
+      }
+    }
+  }
+
+  // inference
+  for (size_t i = 0; i < repeat_times; ++i) {
+    ASSERT_TRUE(predictor->Run());
+  }
+
+  // get output data to Record
+  auto output_names = predictor->GetOutputNames();
+  for (auto &output_name : output_names) {
+    Record output_Record;
+    auto output_tensor = predictor->GetOutputHandle(output_name);
+    std::vector<int> output_shape = output_tensor->shape();
+    int out_num = std::accumulate(
+        output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
+
+    switch (output_tensor->type()) {
+      case paddle::PaddleDType::INT64: {
+        VLOG(1) << "output_tensor dtype: int64";
+        std::vector<int64_t> out_data;
+        output_Record.type = paddle::PaddleDType::INT64;
+        out_data.resize(out_num);
+        output_tensor->CopyToCpu(out_data.data());
+        output_Record.shape = output_shape;
+        std::vector<float> floatVec(out_data.begin(), out_data.end());
+        output_Record.data = floatVec;
+        (*output_data_map)[output_name] = output_Record;
+        break;
+      }
+      case paddle::PaddleDType::FLOAT32: {
+        VLOG(1) << "output_tensor dtype: float32";
+        std::vector<float> out_data;
+        output_Record.type = paddle::PaddleDType::FLOAT32;
+        out_data.resize(out_num);
+        output_tensor->CopyToCpu(out_data.data());
+        output_Record.shape = output_shape;
+        output_Record.data = out_data;
+        (*output_data_map)[output_name] = output_Record;
+        break;
+      }
+      case paddle::PaddleDType::INT32: {
+        VLOG(1) << "output_tensor dtype: int32";
+        std::vector<int32_t> out_data;
+        output_Record.type = paddle::PaddleDType::INT32;
+        out_data.resize(out_num);
+        output_tensor->CopyToCpu(out_data.data());
+        output_Record.shape = output_shape;
+        std::vector<float> floatVec(out_data.begin(), out_data.end());
+        output_Record.data = floatVec;
+        (*output_data_map)[output_name] = output_Record;
+        break;
+      }
+    }
+  }
+}
+
+void CompareRecord(std::map<std::string, Record> *truth_output_data,
+                   std::map<std::string, Record> *infer_output_data,
+                   float epsilon = 1e-5) {
+  for (const auto &[key, value] : *infer_output_data) {
+    auto truth_record = (*truth_output_data)[key];
+    VLOG(1) << "output name: " << key;
+    size_t numel = value.data.size() / sizeof(float);
+    EXPECT_EQ(value.data.size(), truth_record.data.size());
+    for (size_t i = 0; i < numel; ++i) {
+      VLOG(1) << "compare: " << value.data.data()[i] << ",\t"
+              << truth_record.data.data()[i];
+      ASSERT_LT(fabs(value.data.data()[i] - truth_record.data.data()[i]),
+                epsilon);
+    }
+  }
+}
+
+// Timer, count in ms
+class Timer {
+ public:
+  Timer() { reset(); }
+  void start() { start_t = std::chrono::high_resolution_clock::now(); }
+  void stop() {
+    auto end_t = std::chrono::high_resolution_clock::now();
+    typedef std::chrono::microseconds ms;
+    auto diff = end_t - start_t;
+    ms counter = std::chrono::duration_cast<ms>(diff);
+    total_time += counter.count();
+  }
+  void reset() { total_time = 0.; }
+  double report() { return total_time / 1000.0; }
+
+ private:
+  double total_time;
+  std::chrono::high_resolution_clock::time_point start_t;
+};
+
+// single thread inference benchmark, return double time in ms
+double SingleThreadProfile(paddle_infer::Predictor *predictor,
+                           std::map<std::string, Record> *input_data_map,
+                           int repeat_times = 2) {
+  // prepare input tensor
+  auto input_names = predictor->GetInputNames();
+  for (const auto &[key, value] : *input_data_map) {
+    switch (value.type) {
+      case paddle::PaddleDType::INT64: {
+        std::vector<int64_t> input_value =
+            std::vector<int64_t>(value.data.begin(), value.data.end());
+        auto input_tensor = predictor->GetInputHandle(key);
+        input_tensor->Reshape(value.shape);
+        input_tensor->CopyFromCpu(input_value.data());
+        break;
+      }
+      case paddle::PaddleDType::INT32: {
+        std::vector<int32_t> input_value =
+            std::vector<int32_t>(value.data.begin(), value.data.end());
+        auto input_tensor = predictor->GetInputHandle(key);
+        input_tensor->Reshape(value.shape);
+        input_tensor->CopyFromCpu(input_value.data());
+        break;
+      }
+      case paddle::PaddleDType::FLOAT32: {
+        std::vector<float> input_value =
+            std::vector<float>(value.data.begin(), value.data.end());
+        auto input_tensor = predictor->GetInputHandle(key);
+        input_tensor->Reshape(value.shape);
+        input_tensor->CopyFromCpu(input_value.data());
+        break;
+      }
+    }
+  }
+
+  Timer timer;  // init prediction timer
+  timer.start();
+  // inference
+  for (size_t i = 0; i < repeat_times; ++i) {
+    CHECK(predictor->Run());
+    auto output_names = predictor->GetOutputNames();
+    for (auto &output_name : output_names) {
+      auto output_tensor = predictor->GetOutputHandle(output_name);
+      std::vector<int> output_shape = output_tensor->shape();
+      int out_num = std::accumulate(
+          output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
+      switch (output_tensor->type()) {
+        case paddle::PaddleDType::INT64: {
+          std::vector<int64_t> out_data;
+          out_data.resize(out_num);
+          output_tensor->CopyToCpu(out_data.data());
+          break;
+        }
+        case paddle::PaddleDType::FLOAT32: {
+          std::vector<float> out_data;
+          out_data.resize(out_num);
+          output_tensor->CopyToCpu(out_data.data());
+          break;
+        }
+        case paddle::PaddleDType::INT32: {
+          std::vector<int32_t> out_data;
+          out_data.resize(out_num);
+          output_tensor->CopyToCpu(out_data.data());
+          break;
+        }
+      }
+    }
+  }
+  timer.stop();
+  return timer.report();
+}
+
+}  // namespace test
+}  // namespace paddle

test/cpp/inference/infer_ut/test_yolov3.cc

@@ -0,0 +1,159 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test_suite.h"  // NOLINT
+
+DEFINE_string(modeldir, "", "Directory of the inference model.");
+
+namespace paddle_infer {
+
+std::map<std::string, paddle::test::Record> PrepareInput(int batch_size) {
+  // init input data
+  int channel = 3;
+  int width = 608;
+  int height = 608;
+  paddle::test::Record image, im_shape, scale_factor;
+  int input_num = batch_size * channel * width * height;
+  int shape_num = batch_size * 2;
+  std::vector<float> image_data(input_num, 1);
+  for (int i = 1; i < input_num + 1; ++i) {
+    image_data[i] = i % 10 * 0.5;
+  }
+  std::vector<float> im_shape_data(shape_num, 1);
+  std::vector<float> scale_factor_data(shape_num, 1);
+
+  image.data = std::vector<float>(image_data.begin(), image_data.end());
+  image.shape = std::vector<int>{batch_size, channel, width, height};
+  image.type = paddle::PaddleDType::FLOAT32;
+
+  im_shape.data =
+      std::vector<float>(im_shape_data.begin(), im_shape_data.end());
+  im_shape.shape = std::vector<int>{batch_size, 2};
+  im_shape.type = paddle::PaddleDType::FLOAT32;
+
+  scale_factor.data =
+      std::vector<float>(scale_factor_data.begin(), scale_factor_data.end());
+  scale_factor.shape = std::vector<int>{batch_size, 2};
+  scale_factor.type = paddle::PaddleDType::FLOAT32;
+
+  std::map<std::string, paddle::test::Record> input_data_map;
+  input_data_map.insert({"image", image});
+  input_data_map.insert({"im_shape", im_shape});
+  input_data_map.insert({"scale_factor", scale_factor});
+
+  return input_data_map;
+}
+
+TEST(test_yolov3, multi_thread3_trt_fp32_bz2) {
+  int thread_num = 3;
+  // init input data
+  auto input_data_map = PrepareInput(2);
+  // init output data
+  std::map<std::string, paddle::test::Record> infer_output_data,
+      truth_output_data;
+  // prepare ground truth config
+  paddle_infer::Config config, config_no_ir;
+  config_no_ir.SetModel(FLAGS_modeldir + "/model.pdmodel",
+                        FLAGS_modeldir + "/model.pdiparams");
+  config_no_ir.EnableUseGpu(100, 0);
+  config_no_ir.SwitchIrOptim(false);
+  // prepare inference config
+  config.SetModel(FLAGS_modeldir + "/model.pdmodel",
+                  FLAGS_modeldir + "/model.pdiparams");
+  config.EnableUseGpu(100, 0);
+  config.EnableTensorRtEngine(
+      1 << 20, 2, 3, paddle_infer::PrecisionType::kFloat32, false, false);
+  LOG(INFO) << config.Summary();
+  // get ground truth by disable ir
+  paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
+  SingleThreadPrediction(
+      pred_pool_no_ir.Retrieve(0), &input_data_map, &truth_output_data, 1);
+
+  // get infer results from multi threads
+  std::vector<std::thread> threads;
+  services::PredictorPool pred_pool(config, thread_num);
+  for (int i = 0; i < thread_num; ++i) {
+    threads.emplace_back(paddle::test::SingleThreadPrediction,
+                         pred_pool.Retrieve(i),
+                         &input_data_map,
+                         &infer_output_data,
+                         2);
+  }
+
+  // thread join & check outputs
+  for (int i = 0; i < thread_num; ++i) {
+    LOG(INFO) << "join tid : " << i;
+    threads[i].join();
+    CompareRecord(&truth_output_data, &infer_output_data, 1e-2);
+    // TODO(OliverLPH): precision set to 1e-2 since input is fake, change to
+    // real input later
+  }
+
+  std::cout << "finish multi-thread test" << std::endl;
+}
+
+TEST(test_yolov3, multi_thread4_mkl_bz2) {
+  int thread_num = 4;
+  // init input data
+  auto input_data_map = PrepareInput(2);
+  // init output data
+  std::map<std::string, paddle::test::Record> infer_output_data,
+      truth_output_data;
+  // prepare ground truth config
+  paddle_infer::Config config, config_no_ir;
+  config_no_ir.SetModel(FLAGS_modeldir + "/model.pdmodel",
+                        FLAGS_modeldir + "/model.pdiparams");
+  config_no_ir.DisableGpu();
+  config_no_ir.SwitchIrOptim(false);
+  // prepare inference config
+  config.SetModel(FLAGS_modeldir + "/model.pdmodel",
+                  FLAGS_modeldir + "/model.pdiparams");
+  config.DisableGpu();
+  config.EnableONEDNN();
+  config.SetOnednnCacheCapacity(10);
+  config.SetCpuMathLibraryNumThreads(10);
+  LOG(INFO) << config.Summary();
+  // get ground truth by disable ir
+  paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
+  SingleThreadPrediction(
+      pred_pool_no_ir.Retrieve(0), &input_data_map, &truth_output_data, 1);
+
+  // get infer results from multi threads
+  std::vector<std::thread> threads;
+  services::PredictorPool pred_pool(config, thread_num);
+  for (int i = 0; i < thread_num; ++i) {
+    threads.emplace_back(paddle::test::SingleThreadPrediction,
+                         pred_pool.Retrieve(i),
+                         &input_data_map,
+                         &infer_output_data,
+                         2);
+  }
+
+  // thread join & check outputs
+  for (int i = 0; i < thread_num; ++i) {
+    LOG(INFO) << "join tid : " << i;
+    threads[i].join();
+    CompareRecord(&truth_output_data, &infer_output_data, 1e-4);
+  }
+
+  std::cout << "finish multi-thread test" << std::endl;
+}
+
+}  // namespace paddle_infer
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  return RUN_ALL_TESTS();
+}

test/cpp/inference/tensorrt/CMakeLists.txt

@@ -0,0 +1,21 @@
+set(TENSORRT_VERSION_NUMBER
+    "${TENSORRT_MAJOR_VERSION}${TENSORRT_MINOR_VERSION}")
+if(${TENSORRT_VERSION_NUMBER} GREATER_EQUAL 85)
+  nv_test(
+    test_tensorrt_engine_instruction
+    SRCS test_tensorrt_engine_instruction.cc
+    DEPS pir
+         trt_engine
+         naive_executor
+         phi
+         common
+         pir_save_load
+         pir_transforms
+         pir_tensorrt_plugin)
+  set_tests_properties(test_tensorrt_engine_instruction PROPERTIES TIMEOUT 120)
+  if(WITH_ONNXRUNTIME AND WIN32)
+    # Copy onnxruntime for some c++ test in Windows, since the test will
+    # be build only in CI, so suppose the generator in Windows is Ninja.
+    copy_onnx(test_tensorrt_engine_instruction)
+  endif()
+endif()

test/cpp/inference/tensorrt/test_tensorrt_engine_instruction.cc

@@ -0,0 +1,543 @@
+/* Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+#include <memory>
+
+#include "paddle/common/ddim.h"
+#include "paddle/fluid/framework/naive_executor.h"
+#include "paddle/fluid/framework/new_executor/instruction/tensorrt_engine_instruction.h"
+#include "paddle/fluid/framework/new_executor/interpreter/execution_config.h"
+#include "paddle/fluid/framework/new_executor/standalone_executor.h"
+#include "paddle/fluid/framework/tensor.h"
+#include "paddle/fluid/inference/analysis/helper.h"
+#include "paddle/fluid/inference/tensorrt/pir/declare_plugin.h"
+#include "paddle/fluid/pir/dialect/operator/ir/manual_api.h"
+#include "paddle/fluid/pir/dialect/operator/ir/pd_op.h"
+#include "paddle/fluid/pir/dialect/operator/ir/tensorrt_op.h"
+#include "paddle/fluid/pir/dialect/operator/utils/utils.h"
+#include "paddle/fluid/pir/serialize_deserialize/include/ir_serialize.h"
+#include "paddle/fluid/pir/transforms/pd_op_to_kernel_pass.h"
+#include "paddle/fluid/platform/enforce.h"
+#include "paddle/fluid/platform/init.h"
+#include "paddle/fluid/platform/tensorrt/engine.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/tensor_utils.h"
+#include "paddle/pir/include/core/builtin_dialect.h"
+
+PD_DECLARE_KERNEL(full, GPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(assign, GPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(memcpy_h2d, GPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(arange, GPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(argsort, GPU, ALL_LAYOUT);
+
+TEST(TensorRTEngineInstructionTest, test_tensorrt_engine_instruction) {
+  // 1. Init env
+  const int size = 1;
+  float raw_weight[1] = {2.};  // Weight in CPU memory.
+  float raw_bias[1] = {0.};
+  paddle::framework::InitMemoryMethod();
+  paddle::framework::InitDevices();
+  paddle::framework::InitDefaultKernelSignatureMap();
+  std::unique_ptr<paddle::framework::Scope> scope =
+      std::make_unique<paddle::framework::Scope>();
+  auto dev_ctx =
+      paddle::platform::DeviceContextPool::Instance().Get(phi::GPUPlace());
+  auto weight_tensor = scope->Var("weight")->GetMutable<phi::DenseTensor>();
+  weight_tensor->Resize({1});
+  dev_ctx->Alloc<float>(weight_tensor);
+  auto y_tensor = scope->Var("y")->GetMutable<phi::DenseTensor>();
+  y_tensor->Resize({1});
+  dev_ctx->Alloc<float>(y_tensor);
+
+  // 2. construct trt engine
+  std::map<std::string, std::vector<int>> min_input_shape = {
+      {"x", {1, 1, 1, 1}}};
+  std::map<std::string, std::vector<int>> max_input_shape = {
+      {"x", {10, 1, 1, 1}}};
+  std::map<std::string, std::vector<int>> optim_input_shape = {
+      {"x", {5, 1, 1, 1}}};
+
+  paddle::platform::EngineParams params;
+  params.max_workspace_size = 1 << 10;
+  params.min_input_shape = min_input_shape;
+  params.max_input_shape = max_input_shape;
+  params.optim_input_shape = optim_input_shape;
+  auto engine = std::make_unique<paddle::platform::TensorRTEngine>(params);
+  engine->InitNetwork();
+
+  LOG(INFO) << "create weights";
+  paddle::platform::TensorRTEngine::Weight weight(
+      nvinfer1::DataType::kFLOAT, raw_weight, size);
+  paddle::platform::TensorRTEngine::Weight bias(
+      nvinfer1::DataType::kFLOAT, raw_bias, size);
+  auto *x = engine->DeclareInput(
+      "x", nvinfer1::DataType::kFLOAT, nvinfer1::Dims4{-1, 1, 1, 1});
+  auto *flatten_layer = engine->network()->addShuffle(*x);
+  PADDLE_ENFORCE_NOT_NULL(
+      flatten_layer,
+      common::errors::InvalidArgument(
+          "Unable to build the TensorRT shuffle layer for the input tensor "
+          "'x'. "
+          "This usually indicates the TensorRT network failed to allocate the "
+          "intermediate reshape layer."));
+  flatten_layer->setReshapeDimensions(nvinfer1::Dims2{-1, 1});
+
+  auto *weight_layer = TRT_ENGINE_ADD_LAYER(
+      engine, Constant, nvinfer1::Dims2{1, 1}, weight.get());
+  PADDLE_ENFORCE_NOT_NULL(
+      weight_layer,
+      common::errors::InvalidArgument("TensorRT failed to create the constant "
+                                      "layer for parameter 'weight'. "
+                                      "Please confirm the TensorRT builder "
+                                      "supports constant initialisation "
+                                      "for the provided weight shape."));
+
+  auto *bias_layer =
+      TRT_ENGINE_ADD_LAYER(engine, Constant, nvinfer1::Dims2{1, 1}, bias.get());
+  PADDLE_ENFORCE_NOT_NULL(
+      bias_layer,
+      common::errors::InvalidArgument(
+          "TensorRT failed to create the constant layer for parameter 'bias'. "
+          "Check whether the provided bias data matches the expected shape."));
+
+  auto *matmul_layer = TRT_ENGINE_ADD_LAYER(engine,
+                                            MatrixMultiply,
+                                            *flatten_layer->getOutput(0),
+                                            nvinfer1::MatrixOperation::kNONE,
+                                            *weight_layer->getOutput(0),
+                                            nvinfer1::MatrixOperation::kNONE);
+  PADDLE_ENFORCE_NOT_NULL(
+      matmul_layer,
+      common::errors::InvalidArgument(
+          "TensorRT returned a null matrix-multiply layer while fusing the "
+          "fully-connected op. Verify the network input ranks and TensorRT "
+          "version."));
+
+  auto *add_layer = TRT_ENGINE_ADD_LAYER(engine,
+                                         ElementWise,
+                                         *matmul_layer->getOutput(0),
+                                         *bias_layer->getOutput(0),
+                                         nvinfer1::ElementWiseOperation::kSUM);
+  PADDLE_ENFORCE_NOT_NULL(
+      add_layer,
+      common::errors::InvalidArgument(
+          "TensorRT could not construct the elementwise-add layer for bias "
+          "fusion. Ensure the bias tensor uses broadcastable dimensions."));
+
+  auto *reshape_layer = engine->network()->addShuffle(*add_layer->getOutput(0));
+  PADDLE_ENFORCE_NOT_NULL(
+      reshape_layer,
+      common::errors::InvalidArgument(
+          "TensorRT could not emit the final shuffle layer to restore the "
+          "output shape. Confirm the shape tensor and inferred dimensions are "
+          "valid."));
+  reshape_layer->setReshapeDimensions(nvinfer1::Dims4{-1, 1, 1, 1});
+
+  engine->DeclareOutput(reshape_layer, 0, "y");
+  std::vector<std::string> input_names = {"x", ""};
+  std::vector<std::string> output_names = {"y"};
+  std::vector<std::vector<int64_t>> outputs_shape = {{1}};
+  std::vector<phi::DataType> outputs_dtype = {phi::DataType::FLOAT32};
+  LOG(INFO) << "freeze network";
+  engine->FreezeNetwork();
+#if IS_TRT_VERSION_GE(8600)
+  ASSERT_EQ(engine->engine()->getNbIOTensors(), 2);
+#else
+  ASSERT_EQ(engine->engine()->getNbBindings(), 2);
+#endif
+  nvinfer1::IHostMemory *serialized_engine_data = engine->Serialize();
+
+  std::ofstream outFile("engine_serialized_data.bin", std::ios::binary);
+  outFile.write(static_cast<const char *>(serialized_engine_data->data()),
+                serialized_engine_data->size());
+  outFile.close();
+
+  auto trt_engine_serialized_path = "engine_serialized_data.bin";
+  params.engine_serialized_data = trt_engine_serialized_path;
+
+  // 3. Build PIR Program
+  // x --------
+  //           |------> trt_op(matmul) -> pd_op.assign -> output value
+  // weight ---
+  pir::IrContext *ctx = pir::IrContext::Instance();
+  ctx->GetOrRegisterDialect<pir::BuiltinDialect>();
+  ctx->GetOrRegisterDialect<paddle::dialect::OperatorDialect>();
+
+  pir::Program program(ctx);
+  pir::Builder builder(ctx, program.block());
+  auto x_value = builder
+                     .Build<paddle::dialect::FullOp>(
+                         std::vector<int64_t>{1, 1, 1, 1}, 100.0f)
+                     .out();
+  auto weight_value =
+      builder.Build<pir::ParameterOp>("weight", x_value.type()).result(0);
+  auto y_value =
+      builder.Build<pir::ParameterOp>("y", x_value.type())
+          .result(0);  // Use for load y, although y is not a parameter
+  std::vector<pir::Value> combine_input = {x_value, weight_value};
+  auto tensorrt_input = builder.Build<pir::CombineOp>(combine_input).out();
+  auto tensorrt_result =
+      builder
+          .Build<paddle::dialect::TensorRTEngineOp>(tensorrt_input,
+                                                    params,
+                                                    input_names,
+                                                    output_names,
+                                                    outputs_shape,
+                                                    outputs_dtype,
+                                                    "NO DEBUG INFO")
+          .out();
+  auto assign_input = builder.Build<pir::SplitOp>(tensorrt_result).outputs()[0];
+  builder.Build<paddle::dialect::AssignOut_Op>(assign_input, y_value);
+  y_value.set_attribute(
+      "persistable", pir::BoolAttribute::get(pir::IrContext::Instance(), true));
+
+  // 4. Run Program
+  auto kernel_program = pir::PdOpLowerToKernelPass(&program, phi::GPUPlace());
+  std::unique_ptr<paddle::framework::NaiveExecutor> executor =
+      std::make_unique<paddle::framework::NaiveExecutor>(phi::GPUPlace());
+  paddle::framework::interpreter::ExecutionConfig execution_config;
+  execution_config.create_local_scope = false;
+  execution_config.used_for_inference = true;
+  executor->PrepareInterpreterCore(
+      scope.get(), *(kernel_program.get()), execution_config);
+  executor->RunInterpreterCore();
+
+  // check
+  auto y = scope->Var("y")->Get<phi::DenseTensor>();
+  phi::DenseTensor result;
+  phi::Copy(*(static_cast<phi::CPUContext *>(dev_ctx)),
+            y,
+            phi::CPUPlace(),
+            true,
+            &result);
+  auto *result_data = result.data<float>();
+  ASSERT_EQ(result_data[0], 200);
+}
+
+TEST(TensorRTEngineInstructionTest, test_tensorrt_engine_instruction_dynamic) {
+  // 1. Init env
+  paddle::framework::InitMemoryMethod();
+  paddle::framework::InitDevices();
+  paddle::framework::InitDefaultKernelSignatureMap();
+  std::unique_ptr<paddle::framework::Scope> scope =
+      std::make_unique<paddle::framework::Scope>();
+  auto dev_ctx =
+      paddle::platform::DeviceContextPool::Instance().Get(phi::GPUPlace());
+  auto y_tensor = scope->Var("y")->GetMutable<phi::DenseTensor>();
+  y_tensor->Resize({8, 8, 4});
+  dev_ctx->Alloc<float>(y_tensor);
+
+  // 2. construct trt engine
+  std::map<std::string, std::vector<int>> min_input_shape = {
+      {"input", {1, 32}}};
+  std::map<std::string, std::vector<int>> max_input_shape = {
+      {"input", {18, 32}}};
+  std::map<std::string, std::vector<int>> optim_input_shape = {
+      {"input", {18, 32}}};
+  std::map<std::string, std::vector<int>> min_input_value = {
+      {"shape", {1, 8, 4}}};
+  std::map<std::string, std::vector<int>> max_input_value = {
+      {"shape", {18, 8, 4}}};
+  std::map<std::string, std::vector<int>> optim_input_value = {
+      {"shape", {18, 8, 4}}};
+
+  paddle::platform::EngineParams params;
+  params.max_workspace_size = 1 << 10;
+  params.min_input_shape = min_input_shape;
+  params.max_input_shape = max_input_shape;
+  params.optim_input_shape = optim_input_shape;
+  params.min_shape_tensor = min_input_value;
+  params.max_shape_tensor = max_input_value;
+  params.optim_shape_tensor = optim_input_value;
+
+  auto engine = std::make_unique<paddle::platform::TensorRTEngine>(
+      params, paddle::platform::NaiveLogger::Global());
+  engine->InitNetwork();
+
+  auto *x = engine->DeclareInput(
+      "input", nvinfer1::DataType::kFLOAT, nvinfer1::Dims2{-1, 32});
+  nvinfer1::Dims shape_dim;
+  shape_dim.nbDims = 1;
+  shape_dim.d[0] = 3;
+  auto *shape =
+      engine->DeclareInput("shape", nvinfer1::DataType::kINT32, shape_dim);
+  auto layer = engine->network()->addShuffle(*x);
+  layer->setInput(1, *shape);
+  PADDLE_ENFORCE_NOT_NULL(
+      layer,
+      common::errors::InvalidArgument(
+          "TensorRT failed to construct the dynamic shuffle layer that "
+          "consumes the runtime shape tensor. Please check the provided "
+          "shape binding."));
+  engine->DeclareOutput(layer, 0, "y");
+  engine->FreezeNetwork();
+
+  nvinfer1::IHostMemory *serialized_engine_data = engine->Serialize();
+  std::ofstream outFile("engine_serialized_data.bin", std::ios::binary);
+  outFile.write(static_cast<const char *>(serialized_engine_data->data()),
+                serialized_engine_data->size());
+  outFile.close();
+
+  auto trt_engine_serialized_path = "engine_serialized_data.bin";
+  params.engine_serialized_data = trt_engine_serialized_path;
+
+  LOG(INFO) << "freeze network";
+
+  // 3. Build PIR Program
+  // x --------
+  //           |------> trt_op(matmul) -> pd_op.assign -> output value
+  // weight ---
+  pir::IrContext *ctx = pir::IrContext::Instance();
+  ctx->GetOrRegisterDialect<pir::BuiltinDialect>();
+  ctx->GetOrRegisterDialect<paddle::dialect::OperatorDialect>();
+
+  pir::Program program(ctx);
+  pir::Builder builder(ctx, program.block());
+  auto x_value =
+      builder.Build<paddle::dialect::FullOp>(std::vector<int64_t>{8, 32}, 1.0f)
+          .out();
+  auto shape_value = builder
+                         .Build<paddle::dialect::FullIntArrayOp>(
+                             std::vector<int64_t>({8, 8, 4}),
+                             phi::DataType::INT64,
+                             phi::CPUPlace())
+                         .out();
+  auto y_value =
+      builder.Build<pir::ParameterOp>("y", x_value.type())
+          .result(0);  // Use for load y, although y is not a parameter
+  std::vector<pir::Value> combine_input = {x_value, shape_value};
+  auto tensorrt_input = builder.Build<pir::CombineOp>(combine_input).out();
+
+  auto vec_shape = paddle::dialect::GetInt64Vector(
+      shape_value.defining_op()
+          ->dyn_cast<paddle::dialect::FullIntArrayOp>()
+          .attribute("value"));
+
+  std::vector<std::string> input_names = {"input", "shape"};
+  std::vector<std::string> output_names = {"y"};
+  std::vector<std::vector<int64_t>> outputs_shape = {vec_shape};
+  std::vector<phi::DataType> outputs_dtype = {phi::DataType::FLOAT32};
+
+  auto tensorrt_result =
+      builder
+          .Build<paddle::dialect::TensorRTEngineOp>(tensorrt_input,
+                                                    params,
+                                                    input_names,
+                                                    output_names,
+                                                    outputs_shape,
+                                                    outputs_dtype,
+                                                    "NO DEBUG INFO")
+          .out();
+  auto assign_input = builder.Build<pir::SplitOp>(tensorrt_result).outputs()[0];
+  builder.Build<paddle::dialect::AssignOut_Op>(assign_input, y_value);
+  y_value.set_attribute(
+      "persistable", pir::BoolAttribute::get(pir::IrContext::Instance(), true));
+
+  // 4. Run Program
+  auto kernel_program = pir::PdOpLowerToKernelPass(&program, phi::GPUPlace());
+  std::unique_ptr<paddle::framework::NaiveExecutor> executor =
+      std::make_unique<paddle::framework::NaiveExecutor>(phi::GPUPlace());
+  paddle::framework::interpreter::ExecutionConfig execution_config;
+  execution_config.create_local_scope = false;
+  execution_config.used_for_inference = true;
+  executor->PrepareInterpreterCore(
+      scope.get(), *(kernel_program.get()), execution_config);
+  executor->RunInterpreterCore();
+
+  // check
+  auto y = scope->Var("y")->Get<phi::DenseTensor>();
+  phi::DenseTensor result;
+  phi::Copy(*(static_cast<phi::CPUContext *>(dev_ctx)),
+            y,
+            phi::CPUPlace(),
+            true,
+            &result);
+  ASSERT_EQ(result.dims()[0], 8);
+  ASSERT_EQ(result.dims()[1], 8);
+  ASSERT_EQ(result.dims()[2], 4);
+  auto *result_data = result.data<float>();
+  ASSERT_EQ(result_data[0], 1);
+}
+
+TEST(PluginTest, test_generic_plugin) {
+  // 1. Init env
+  paddle::framework::InitMemoryMethod();
+  paddle::framework::InitDevices();
+  paddle::framework::InitDefaultKernelSignatureMap();
+  std::unique_ptr<paddle::framework::Scope> scope =
+      std::make_unique<paddle::framework::Scope>();
+  pir::IrContext *ctx = pir::IrContext::Instance();
+  ctx->GetOrRegisterDialect<pir::BuiltinDialect>();
+  ctx->GetOrRegisterDialect<paddle::dialect::OperatorDialect>();
+
+  pir::Program program(ctx);
+  pir::Builder builder(ctx, program.block());
+  auto x_value = builder.Build<paddle::dialect::ArangeOp>(0, 10, 1).out();
+  std::vector<int64_t> x_shape{1, 10};
+  auto reshape_value =
+      builder.Build<paddle::dialect::ReshapeOp>(x_value, x_shape).out();
+  auto argsort_out =
+      builder.Build<paddle::dialect::ArgsortOp>(reshape_value, -1, true, false)
+          .out();
+  auto dev_ctx =
+      paddle::platform::DeviceContextPool::Instance().Get(phi::GPUPlace());
+  auto y_tensor = scope->Var("y")->GetMutable<phi::DenseTensor>();
+  y_tensor->Resize({1, 10});
+  dev_ctx->Alloc<float>(y_tensor);
+
+  // 2. construct trt engine
+  std::map<std::string, std::vector<int>> min_input_shape = {{"x", {1, 10}}};
+  std::map<std::string, std::vector<int>> max_input_shape = {{"x", {10, 10}}};
+  std::map<std::string, std::vector<int>> optim_input_shape = {{"x", {5, 10}}};
+
+  paddle::platform::EngineParams params;
+  params.max_workspace_size = 1 << 10;
+  params.min_input_shape = min_input_shape;
+  params.max_input_shape = max_input_shape;
+  params.optim_input_shape = optim_input_shape;
+  auto engine = std::make_unique<paddle::platform::TensorRTEngine>(params);
+  engine->InitNetwork();
+
+  auto *x = engine->DeclareInput(
+      "x", nvinfer1::DataType::kFLOAT, nvinfer1::Dims2{-1, 10});
+
+  auto creator = paddle::platform::GetPluginRegistry()->getPluginCreator(
+      "pir_generic_plugin", "1");
+  assert(creator != nullptr);
+  auto op = argsort_out.defining_op();
+  ::pir::ProgramWriter writer(1, false);
+
+  std::string op_name = op->name();
+  auto attrs_map_info = writer.GetAttributesMapJson(op->attributes()).dump();
+  std::stringstream inputs_type_info_ss;
+  for (auto operand : op->operands_source()) {
+    inputs_type_info_ss << (writer.GetTypeJson(operand.type()).dump())
+                        << '\n';  // use '\n' as separator
+  }
+  std::stringstream outputs_type_info_ss;
+  for (auto result : op->results()) {
+    outputs_type_info_ss << (writer.GetTypeJson(result.type()).dump())
+                         << '\n';  // use '\n' as separator
+  }
+  std::string inputs_type_info = inputs_type_info_ss.str();
+
+  std::string outputs_type_info = outputs_type_info_ss.str();
+  std::vector<nvinfer1::PluginField> fields{
+      {"op_name",
+       op_name.c_str(),
+       nvinfer1::PluginFieldType::kCHAR,
+       static_cast<int>(op_name.size())},
+      {"attrs_map_info",
+       attrs_map_info.c_str(),
+       nvinfer1::PluginFieldType::kCHAR,
+       static_cast<int>(attrs_map_info.size())},
+      {"inputs_type_info",
+       inputs_type_info.c_str(),
+       nvinfer1::PluginFieldType::kCHAR,
+       static_cast<int>(inputs_type_info.size())},
+      {"outputs_type_info",
+       outputs_type_info.c_str(),
+       nvinfer1::PluginFieldType::kCHAR,
+       static_cast<int>(outputs_type_info.size())}};
+  std::unique_ptr<nvinfer1::PluginFieldCollection> plugin_collection(
+      new nvinfer1::PluginFieldCollection);
+
+  plugin_collection->nbFields = static_cast<int>(fields.size());
+  plugin_collection->fields = fields.data();
+  auto generic_plugin =
+      creator->createPlugin("pir_generic_plugin", plugin_collection.get());
+  PADDLE_ENFORCE_NOT_NULL(
+      generic_plugin,
+      common::errors::InvalidArgument(
+          "TensorRT plugin registry returned nullptr while creating "
+          "'pir_generic_plugin'. Verify the plugin has been registered before "
+          "building the engine."));
+  std::vector<nvinfer1::ITensor *> plugin_inputs;
+  plugin_inputs.emplace_back(x);
+  auto plugin_layer = engine->network()->addPluginV2(
+      plugin_inputs.data(), plugin_inputs.size(), *generic_plugin);
+  PADDLE_ENFORCE_NOT_NULL(
+      plugin_layer,
+      common::errors::InvalidArgument(
+          "TensorRT failed to add the generic plugin layer to the network. "
+          "Ensure the plugin inputs match the expected TensorRT types."));
+
+  engine->DeclareOutput(plugin_layer, 0, "y");
+  std::vector<std::string> input_names = {"x"};
+  std::vector<std::string> output_names = {"y"};
+  std::vector<std::vector<int64_t>> outputs_shape = {{1}};
+  std::vector<phi::DataType> outputs_dtype = {phi::DataType::FLOAT32};
+  LOG(INFO) << "freeze network";
+  engine->FreezeNetwork();
+#if IS_TRT_VERSION_GE(8600)
+  ASSERT_EQ(engine->engine()->getNbIOTensors(), 2);
+#else
+  ASSERT_EQ(engine->engine()->getNbBindings(), 2);
+#endif
+  nvinfer1::IHostMemory *serialized_engine_data = engine->Serialize();
+  std::ofstream outFile("engine_serialized_data.bin", std::ios::binary);
+  outFile.write(static_cast<const char *>(serialized_engine_data->data()),
+                serialized_engine_data->size());
+  outFile.close();
+
+  auto trt_engine_serialized_path = "engine_serialized_data.bin";
+  params.engine_serialized_data = trt_engine_serialized_path;
+
+  // 3. Build PIR Program
+  // x ------> trt_op(argsort) -> pd_op.assign -> output value
+
+  auto y_value =
+      builder.Build<pir::ParameterOp>("y", reshape_value.type())
+          .result(0);  // Use for load y, although y is not a parameter
+  std::vector<pir::Value> combine_input = {reshape_value};
+  auto tensorrt_input = builder.Build<pir::CombineOp>(combine_input).out();
+  auto tensorrt_result =
+      builder
+          .Build<paddle::dialect::TensorRTEngineOp>(tensorrt_input,
+                                                    params,
+                                                    input_names,
+                                                    output_names,
+                                                    outputs_shape,
+                                                    outputs_dtype,
+                                                    "NO DEBUG INFO")
+          .out();
+  auto assign_input = builder.Build<pir::SplitOp>(tensorrt_result).outputs()[0];
+  builder.Build<paddle::dialect::AssignOut_Op>(assign_input, y_value);
+  y_value.set_attribute(
+      "persistable", pir::BoolAttribute::get(pir::IrContext::Instance(), true));
+
+  // 4. Run Program
+  auto kernel_program = pir::PdOpLowerToKernelPass(&program, phi::GPUPlace());
+  std::unique_ptr<paddle::framework::NaiveExecutor> executor =
+      std::make_unique<paddle::framework::NaiveExecutor>(phi::GPUPlace());
+  paddle::framework::interpreter::ExecutionConfig execution_config;
+  execution_config.create_local_scope = false;
+  execution_config.used_for_inference = true;
+  executor->PrepareInterpreterCore(
+      scope.get(), *(kernel_program.get()), execution_config);
+  executor->RunInterpreterCore();
+
+  // check
+  auto y = scope->Var("y")->Get<phi::DenseTensor>();
+  phi::DenseTensor result;
+  phi::Copy(*(static_cast<phi::CPUContext *>(dev_ctx)),
+            y,
+            phi::CPUPlace(),
+            true,
+            &result);
+  auto *result_data = result.data<float>();
+  ASSERT_EQ(result_data[0], 9);
+}

test/cpp/inference/test.cmake

@@ -0,0 +1,167 @@
+include(ExternalProject)
+set(INFERENCE_URL
+    "http://paddle-inference-dist.bj.bcebos.com"
+    CACHE STRING "inference download url")
+set(INFERENCE_DEMO_INSTALL_DIR
+    "${THIRD_PARTY_PATH}/inference_demo"
+    CACHE STRING "A path setting inference demo download directories.")
+set(CPU_NUM_THREADS_ON_CI
+    4
+    CACHE STRING "Run multi-threads on CI to reduce CI time.")
+set(WARMUP_BATCH_SIZE
+    100
+    CACHE STRING "Default warmup_batch_size.")
+function(inference_download INSTALL_DIR URL FILENAME)
+  message(STATUS "Download inference test stuff from ${URL}/${FILENAME}")
+  string(REGEX REPLACE "[-%.]" "_" FILENAME_EX ${FILENAME})
+  ExternalProject_Add(
+    extern_inference_download_${FILENAME_EX}
+    ${EXTERNAL_PROJECT_LOG_ARGS}
+    PREFIX ${INSTALL_DIR}
+    URL ${URL}/${FILENAME}
+    DOWNLOAD_COMMAND wget --no-check-certificate -q -O
+                     ${INSTALL_DIR}/${FILENAME} ${URL}/${FILENAME}
+    DOWNLOAD_DIR ${INSTALL_DIR}
+    DOWNLOAD_NO_PROGRESS 1
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+    UPDATE_COMMAND ""
+    INSTALL_COMMAND "")
+endfunction()
+
+function(inference_download_and_uncompress INSTALL_DIR URL FILENAME CHECK_SUM)
+  message(STATUS "Download inference test stuff from ${URL}/${FILENAME}")
+  string(REGEX REPLACE "[-%./\\]" "_" FILENAME_EX ${FILENAME})
+  string(REGEX MATCH "[^/\\]+$" DOWNLOAD_NAME ${FILENAME})
+  set(EXTERNAL_PROJECT_NAME "extern_download_${FILENAME_EX}")
+  set(UNPACK_DIR "${INSTALL_DIR}/src/${EXTERNAL_PROJECT_NAME}")
+  ExternalProject_Add(
+    ${EXTERNAL_PROJECT_NAME}
+    ${EXTERNAL_PROJECT_LOG_ARGS}
+    PREFIX ${INSTALL_DIR}
+    URL ${URL}/${FILENAME}
+    URL_HASH MD5=${CHECK_SUM}
+    DOWNLOAD_DIR ${INSTALL_DIR}
+    DOWNLOAD_NO_EXTRACT 1
+    DOWNLOAD_NO_PROGRESS 1
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ${CMAKE_COMMAND} -E chdir ${INSTALL_DIR} ${CMAKE_COMMAND} -E
+                  tar xzf ${DOWNLOAD_NAME}
+    UPDATE_COMMAND ""
+    INSTALL_COMMAND "")
+endfunction()
+
+function(inference_download_and_uncompress_without_verify INSTALL_DIR URL
+         FILENAME)
+  message(STATUS "Download inference test stuff from ${URL}/${FILENAME}")
+  string(REGEX REPLACE "[-%./\\]" "_" FILENAME_EX ${FILENAME})
+  string(REGEX MATCH "[^/\\]+$" DOWNLOAD_NAME ${FILENAME})
+  set(EXTERNAL_PROJECT_NAME "extern_download_${FILENAME_EX}")
+  set(UNPACK_DIR "${INSTALL_DIR}/src/${EXTERNAL_PROJECT_NAME}")
+
+  get_property(TARGET_EXIST GLOBAL PROPERTY ${EXTERNAL_PROJECT_NAME})
+  if(NOT "${TARGET_EXIST}" STREQUAL EXIST)
+    ExternalProject_Add(
+      ${EXTERNAL_PROJECT_NAME}
+      ${EXTERNAL_PROJECT_LOG_ARGS}
+      PREFIX ${INSTALL_DIR}
+      URL ${URL}/${FILENAME}
+      DOWNLOAD_DIR ${INSTALL_DIR}
+      DOWNLOAD_NO_EXTRACT 1
+      DOWNLOAD_NO_PROGRESS 1
+      CONFIGURE_COMMAND ""
+      BUILD_COMMAND ${CMAKE_COMMAND} -E chdir ${INSTALL_DIR} ${CMAKE_COMMAND} -E
+                    tar xzf ${DOWNLOAD_NAME}
+      UPDATE_COMMAND ""
+      INSTALL_COMMAND "")
+    set_property(GLOBAL PROPERTY ${EXTERNAL_PROJECT_NAME} "EXIST")
+  endif()
+endfunction()
+
+function(inference_base_test_build TARGET)
+  set(options "")
+  set(oneValueArgs "")
+  set(multiValueArgs SRCS DEPS)
+  cmake_parse_arguments(base_test "${options}" "${oneValueArgs}"
+                        "${multiValueArgs}" ${ARGN})
+  add_executable(${TARGET} ${base_test_SRCS})
+  if(WIN32)
+    target_compile_definitions(${TARGET} PUBLIC STATIC_PADDLE)
+  endif()
+  if("${base_test_DEPS};" MATCHES "paddle_inference_shared;")
+    list(REMOVE_ITEM base_test_DEPS paddle_inference_shared)
+
+    target_link_libraries(${TARGET}
+                          $<TARGET_LINKER_FILE:paddle_inference_shared>)
+    add_dependencies(${TARGET} paddle_inference_shared)
+
+  elseif("${base_test_DEPS};" MATCHES "paddle_inference_c_shared;")
+    list(REMOVE_ITEM base_test_DEPS paddle_inference_c_shared)
+    target_link_libraries(
+      ${TARGET} $<TARGET_LINKER_FILE:paddle_inference_c_shared> common)
+    add_dependencies(${TARGET} paddle_inference_c_shared)
+  else()
+    message(
+      FATAL_ERROR
+        "inference_base_test_build must link either paddle_inference_shared or paddle_inference_c_shared"
+    )
+  endif()
+  if(NOT ((NOT WITH_PYTHON) AND ON_INFER))
+    target_link_libraries(${TARGET} ${PYTHON_LIBRARIES})
+  endif()
+  if(WITH_SHARED_PHI)
+    target_link_libraries(${TARGET} phi)
+    add_dependencies(${TARGET} phi)
+  endif()
+  if(WITH_CINN)
+    target_link_libraries(${TARGET} $<TARGET_LINKER_FILE:cinnapi>)
+    add_dependencies(${TARGET} cinnapi)
+  endif()
+  if(WITH_GPU)
+    target_link_libraries(${TARGET} ${CUDA_CUDART_LIBRARY})
+  endif()
+  if(WITH_XPU)
+    target_link_libraries(${TARGET} xpulib)
+  endif()
+  if(WITH_ROCM)
+    target_link_libraries(${TARGET} ${ROCM_HIPRTC_LIB})
+  endif()
+  if(WITH_ONNXRUNTIME)
+    target_link_libraries(${TARGET} onnxruntime)
+  endif()
+  if(APPLE)
+    target_link_libraries(
+      ${TARGET}
+      "-Wl,-rpath,$<TARGET_FILE_DIR:${paddle_lib}> -Wl,-rpath,$<TARGET_FILE_DIR:phi> -Wl,-rpath,$<TARGET_FILE_DIR:pir>"
+    )
+  endif()
+  target_link_libraries(${TARGET} ${base_test_DEPS} paddle_gtest_main_new gtest
+                        glog)
+  add_dependencies(${TARGET} ${base_test_DEPS} paddle_gtest_main_new)
+  common_link(${TARGET})
+  check_coverage_opt(${TARGET} ${base_test_SRCS})
+endfunction()
+
+function(inference_base_test_run TARGET)
+  set(options "")
+  set(oneValueArgs "")
+  set(multiValueArgs COMMAND ARGS)
+  cmake_parse_arguments(base_test "${options}" "${oneValueArgs}"
+                        "${multiValueArgs}" ${ARGN})
+  if(WITH_GPU)
+    set(mem_opt "--fraction_of_gpu_memory_to_use=0.5")
+  endif()
+  cc_test_run(${TARGET} COMMAND ${base_test_COMMAND} ARGS ${mem_opt}
+              ${base_test_ARGS})
+endfunction()
+
+function(inference_base_test TARGET)
+  set(options "")
+  set(oneValueArgs "")
+  set(multiValueArgs SRCS ARGS DEPS)
+  cmake_parse_arguments(base_test "${options}" "${oneValueArgs}"
+                        "${multiValueArgs}" ${ARGN})
+  inference_base_test_build(${TARGET} SRCS ${base_test_SRCS} DEPS
+                            ${base_test_DEPS})
+  inference_base_test_run(${TARGET} COMMAND ${TARGET} ARGS ${base_test_ARGS})
+endfunction()

test/cpp/inference/test_cases.cmake

@@ -0,0 +1,34 @@
+include(test.cmake) # some generic cmake function for inference
+
+set(WORD2VEC_INSTALL_DIR "${INFERENCE_DEMO_INSTALL_DIR}/word2vec")
+
+if(NOT EXISTS ${WORD2VEC_INSTALL_DIR}/word2vec.inference.model.tar.gz)
+  inference_download_and_uncompress_without_verify(
+    ${WORD2VEC_INSTALL_DIR} ${INFERENCE_URL} "word2vec.inference.model.tar.gz")
+endif()
+
+set(WORD2VEC_MODEL_DIR "${WORD2VEC_INSTALL_DIR}/word2vec.inference.model")
+
+set(IMG_CLS_RESNET_INSTALL_DIR
+    "${INFERENCE_DEMO_INSTALL_DIR}/image_classification_resnet")
+
+if(NOT EXISTS
+   ${IMG_CLS_RESNET_INSTALL_DIR}/image_classification_resnet.inference.model.tgz
+)
+  inference_download_and_uncompress_without_verify(
+    ${IMG_CLS_RESNET_INSTALL_DIR} ${INFERENCE_URL}
+    "image_classification_resnet.inference.model.tgz")
+endif()
+
+set(IMG_CLS_RESNET_MODEL_DIR
+    "${IMG_CLS_RESNET_INSTALL_DIR}/image_classification_resnet.inference.model")
+
+if(WITH_ONNXRUNTIME)
+  set(MOBILENETV2_INSTALL_DIR "${INFERENCE_DEMO_INSTALL_DIR}/MobileNetV2")
+  if(NOT EXISTS ${MOBILENETV2_INSTALL_DIR}/MobileNetV2.inference.model.tar.gz)
+    inference_download_and_uncompress_without_verify(
+      ${MOBILENETV2_INSTALL_DIR} ${INFERENCE_URL}
+      "MobileNetV2.inference.model.tar.gz")
+  endif()
+  set(MOBILENETV2_MODEL_DIR "${MOBILENETV2_INSTALL_DIR}/MobileNetV2")
+endif()

test/cpp/inference/test_helper.h

@@ -0,0 +1,303 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+#pragma once
+
+#include <map>
+#include <memory>
+#include <random>
+#include <string>
+#include <vector>
+
+#include "paddle/common/errors.h"
+#include "paddle/common/flags.h"
+#include "paddle/fluid/framework/lod_tensor.h"
+#include "paddle/fluid/inference/io.h"
+#include "paddle/phi/common/port.h"
+#include "paddle/phi/core/platform/profiler.h"
+
+COMMON_DECLARE_bool(use_onednn);
+
+namespace paddle {
+bool gpu_place_used(const paddle::PaddlePlace& place) {
+  return place == paddle::PaddlePlace::kGPU;
+}
+bool xpu_place_used(const paddle::PaddlePlace& place) {
+  return place == paddle::PaddlePlace::kXPU;
+}
+bool cpu_place_used(const paddle::PaddlePlace& place) {
+  return place == paddle::PaddlePlace::kCPU;
+}
+}  // namespace paddle
+
+template <typename T>
+void SetupTensor(phi::DenseTensor* input, phi::DDim dims, T lower, T upper) {
+  static unsigned int seed = 100;
+  std::mt19937 rng(seed++);
+  std::uniform_real_distribution<double> uniform_dist(0, 1);
+
+  T* input_ptr = input->mutable_data<T>(dims, phi::CPUPlace());
+  for (int i = 0; i < input->numel(); ++i) {
+    input_ptr[i] = static_cast<T>(uniform_dist(rng) * (upper - lower) + lower);
+  }
+}
+
+template <typename T>
+void SetupTensor(phi::DenseTensor* input,
+                 phi::DDim dims,
+                 const std::vector<T>& data) {
+  PADDLE_ENFORCE_EQ(common::product(dims),
+                    static_cast<int64_t>(data.size()),
+                    common::errors::InvalidArgument(
+                        "common::product(dims) and data.size() are not equal, "
+                        "common::product(dims) is %d and data.size() is %d",
+                        common::product(dims),
+                        static_cast<int64_t>(data.size())));
+  T* input_ptr = input->mutable_data<T>(dims, phi::CPUPlace());
+  memcpy(input_ptr, data.data(), input->numel() * sizeof(T));
+}
+
+template <typename T>
+void SetupDenseTensor(phi::DenseTensor* input,
+                      const phi::LegacyLoD& lod,
+                      T lower,
+                      T upper) {
+  input->set_lod(lod);
+  int dim = lod[0][lod[0].size() - 1];
+  SetupTensor<T>(input, {dim, 1}, lower, upper);
+}
+
+template <typename T>
+void SetupDenseTensor(phi::DenseTensor* input,
+                      phi::DDim dims,
+                      const phi::LegacyLoD lod,
+                      const std::vector<T>& data) {
+  const size_t level = lod.size() - 1;
+  PADDLE_ENFORCE_EQ(dims[0],
+                    static_cast<int64_t>((lod[level]).back()),
+                    common::errors::InvalidArgument(
+                        "dims[0] is not equal with (lod[level]).back()"
+                        "while dims[0] is %d and (lod[level]).back() is %d",
+                        dims[0],
+                        static_cast<int64_t>((lod[level]).back())));
+  input->set_lod(lod);
+  SetupTensor<T>(input, dims, data);
+}
+
+template <typename T>
+void CheckError(const phi::DenseTensor& output1,
+                const phi::DenseTensor& output2) {
+  // Check lod information
+  EXPECT_EQ(output1.lod(), output2.lod());
+
+  EXPECT_EQ(output1.dims(), output2.dims());
+  EXPECT_EQ(output1.numel(), output2.numel());
+
+  T err = static_cast<T>(0);
+  if (typeid(T) == typeid(float)) {
+    err = 1E-3;
+  } else if (typeid(T) == typeid(double)) {
+    err = 1E-6;
+  } else {
+    err = 0;
+  }
+
+  size_t count = 0;
+  for (int64_t i = 0; i < output1.numel(); ++i) {
+    if (fabs(output1.data<T>()[i] - output2.data<T>()[i]) > err) {
+      count++;
+    }
+  }
+  EXPECT_EQ(count, 0U) << "There are " << count << " different elements.";
+}
+
+std::unique_ptr<paddle::framework::ProgramDesc> InitProgram(
+    paddle::framework::Executor* executor,
+    paddle::framework::Scope* scope,
+    const std::string& dirname,
+    const bool is_combined = false,
+    const std::string& prog_filename = "__model_combined__",
+    const std::string& param_filename = "__params_combined__") {
+  std::unique_ptr<paddle::framework::ProgramDesc> inference_program;
+  if (is_combined) {
+    // All parameters are saved in a single file.
+    // Hard-coding the file names of program and parameters in unittest.
+    // The file names should be consistent with that used in Python API
+    //  `fluid.io.save_inference_model`.
+    inference_program = paddle::inference::Load(executor,
+                                                scope,
+                                                dirname + "/" + prog_filename,
+                                                dirname + "/" + param_filename);
+  } else {
+    // Parameters are saved in separate files sited in the specified
+    // `dirname`.
+    inference_program = paddle::inference::Load(executor, scope, dirname);
+  }
+  return inference_program;
+}
+
+std::vector<std::vector<int64_t>> GetFeedTargetShapes(
+    const std::string& dirname,
+    const bool is_combined = false,
+    const std::string& prog_filename = "__model_combined__",
+    const std::string& param_filename = "__params_combined__") {
+  auto place = phi::CPUPlace();
+  auto executor = paddle::framework::Executor(place);
+  auto* scope = new paddle::framework::Scope();
+
+  auto inference_program = InitProgram(
+      &executor, scope, dirname, is_combined, prog_filename, param_filename);
+  auto& global_block = inference_program->Block(0);
+
+  const std::vector<std::string>& feed_target_names =
+      inference_program->GetFeedTargetNames();
+  std::vector<std::vector<int64_t>> feed_target_shapes;
+  for (size_t i = 0; i < feed_target_names.size(); ++i) {
+    auto* var = global_block.FindVar(feed_target_names[i]);
+    std::vector<int64_t> var_shape = var->GetShape();
+    feed_target_shapes.push_back(var_shape);
+  }
+
+  delete scope;
+  return feed_target_shapes;
+}
+
+template <typename Place, bool CreateVars = true, bool PrepareContext = false>
+void TestInference(
+    const std::string& dirname,
+    const std::vector<phi::DenseTensor*>& cpu_feeds,
+    const std::vector<paddle::framework::FetchType*>& cpu_fetches,
+    const int repeat = 1,
+    const bool is_combined = false) {
+  // 1. Define place, executor, scope
+  auto place = Place();
+  auto executor = paddle::framework::Executor(place);
+  auto* scope = new paddle::framework::Scope();
+
+  // Profile the performance
+  paddle::platform::ProfilerState state;
+  if (phi::is_cpu_place(place)) {
+    state = paddle::platform::ProfilerState::kCPU;
+  } else {
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+    state = paddle::platform::ProfilerState::kAll;
+    // The default device_id of phi::GPUPlace is 0.
+    // Users can get the device_id using:
+    //   int device_id = place.GetDeviceId();
+    paddle::platform::SetDeviceId(0);
+#else
+    PADDLE_THROW(common::errors::Unavailable(
+        "'CUDAPlace' is not supported in CPU only device."));
+#endif
+  }
+
+  // 2. Initialize the inference_program and load parameters
+  std::unique_ptr<paddle::framework::ProgramDesc> inference_program;
+
+  // Enable the profiler
+  paddle::platform::EnableProfiler(state);
+  {
+    phi::RecordEvent record_event("init_program");
+    inference_program = InitProgram(&executor, scope, dirname, is_combined);
+  }
+
+  // Disable the profiler and print the timing information
+  paddle::platform::DisableProfiler(paddle::platform::EventSortingKey::kDefault,
+                                    "load_program_profiler");
+  paddle::platform::ResetProfiler();
+
+  // 3. Get the feed_target_names and fetch_target_names
+  const std::vector<std::string>& feed_target_names =
+      inference_program->GetFeedTargetNames();
+  const std::vector<std::string>& fetch_target_names =
+      inference_program->GetFetchTargetNames();
+
+  // 4. Prepare inputs: set up maps for feed targets
+  std::map<std::string, const phi::DenseTensor*> feed_targets;
+  for (size_t i = 0; i < feed_target_names.size(); ++i) {
+    // Please make sure that cpu_feeds[i] is right for feed_target_names[i]
+    feed_targets[feed_target_names[i]] = cpu_feeds[i];
+  }
+
+  // 5. Define Tensor to get the outputs: set up maps for fetch targets
+  std::map<std::string, paddle::framework::FetchType*> fetch_targets;
+  for (size_t i = 0; i < fetch_target_names.size(); ++i) {
+    fetch_targets[fetch_target_names[i]] = cpu_fetches[i];
+  }
+
+  // 6. If export Flags_use_onednn=True, use onednn related ops.
+  if (FLAGS_use_onednn) executor.EnableONEDNN(*inference_program);
+
+  // 7. Run the inference program
+  {
+    if (!CreateVars) {
+      // If users don't want to create and destroy variables every time they
+      // run, they need to set `create_vars` to false and manually call
+      // `CreateVariables` before running.
+      executor.CreateVariables(*inference_program, scope, 0);
+    }
+
+    // Ignore the profiling results of the first run
+    std::unique_ptr<paddle::framework::ExecutorPrepareContext> ctx;
+    bool CreateLocalScope = CreateVars;
+    if (PrepareContext) {
+      ctx = executor.Prepare(*inference_program, 0);
+      executor.RunPreparedContext(ctx.get(),
+                                  scope,
+                                  &feed_targets,
+                                  &fetch_targets,
+                                  CreateLocalScope,
+                                  CreateVars);
+    } else {
+      executor.Run(*inference_program,
+                   scope,
+                   &feed_targets,
+                   &fetch_targets,
+                   CreateLocalScope,
+                   CreateVars);
+    }
+
+    // Enable the profiler
+    paddle::platform::EnableProfiler(state);
+
+    // Run repeat times to profile the performance
+    for (int i = 0; i < repeat; ++i) {
+      phi::RecordEvent record_event("run_inference");
+
+      if (PrepareContext) {
+        // Note: if you change the inference_program, you need to call
+        // executor.Prepare() again to get a new ExecutorPrepareContext.
+        executor.RunPreparedContext(ctx.get(),
+                                    scope,
+                                    &feed_targets,
+                                    &fetch_targets,
+                                    CreateLocalScope,
+                                    CreateVars);
+      } else {
+        executor.Run(*inference_program,
+                     scope,
+                     &feed_targets,
+                     &fetch_targets,
+                     CreateLocalScope,
+                     CreateVars);
+      }
+    }
+
+    // Disable the profiler and print the timing information
+    paddle::platform::DisableProfiler(
+        paddle::platform::EventSortingKey::kDefault, "run_inference_profiler");
+    paddle::platform::ResetProfiler();
+  }
+
+  delete scope;
+}