chore: import upstream snapshot with attribution

paddle/fluid/inference/api/CMakeLists.txt

@@ -0,0 +1,94 @@
+if(APPLE)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=pessimizing-move")
+endif()
+
+add_subdirectory(details)
+
+cc_library(
+  paddle_infer_contrib
+  SRCS paddle_infer_contrib.cc
+  DEPS zero_copy_tensor)
+cc_library(
+  paddle_pass_builder
+  SRCS paddle_pass_builder.cc
+  DEPS framework_proto)
+
+set(paddle_inference_api_deps
+    reset_tensor_array
+    paddle_infer_contrib
+    paddle_pass_builder
+    zero_copy_tensor
+    trainer_desc_proto
+    custom_operator
+    lod_tensor
+    scope
+    drr)
+
+if(WITH_CRYPTO)
+  list(APPEND paddle_inference_api_deps framework_io)
+endif()
+if(WITH_CUSTOM_DEVICE)
+  set(paddle_inference_api_deps ${paddle_inference_api_deps} phi common)
+endif()
+
+if(WIN32)
+  cc_library(
+    paddle_inference_api
+    SRCS api.cc api_impl.cc helper.cc
+    DEPS executor ${paddle_inference_api_deps})
+else()
+  cc_library(
+    paddle_inference_api
+    SRCS api.cc api_impl.cc helper.cc
+    DEPS executor paddle_inference_io ${paddle_inference_api_deps})
+endif()
+
+cc_library(
+  analysis_config
+  SRCS analysis_config.cc
+  DEPS paddle_inference_api paddle_pass_builder table_printer utf8proc)
+
+if(WIN32)
+  target_link_libraries(paddle_inference_api phi common)
+endif()
+
+set(inference_deps
+    ${analysis_deps}
+    paddle_inference_api
+    analysis
+    analysis_config
+    naive_executor
+    ${GLOB_PASS_LIB}
+    pir_transforms)
+
+if(WITH_GPU AND TENSORRT_FOUND)
+  set(inference_deps ${inference_deps} tensorrt_engine tensorrt_converter)
+endif()
+
+if(WITH_OPENVINO)
+  set(inference_deps ${inference_deps} openvino_engine)
+endif()
+
+set(ANALYSIS_PREDICTOR_SRCS analysis_predictor.cc resource_manager.cc
+                            infer_context.cc)
+set(ANALYSIS_PREDICTOR_DEPS ${inference_deps} zero_copy_tensor ir_pass_manager
+                            infer_io_utils model_utils)
+
+if(WITH_ONNXRUNTIME)
+  set(ANALYSIS_PREDICTOR_SRCS ${ANALYSIS_PREDICTOR_SRCS}
+                              onnxruntime_predictor.cc)
+  set(ANALYSIS_PREDICTOR_DEPS ${ANALYSIS_PREDICTOR_DEPS} onnxruntime
+                              paddle2onnx)
+elseif(WITH_CINN)
+  set(ANALYSIS_PREDICTOR_DEPS ${ANALYSIS_PREDICTOR_DEPS} add_cinn_pass)
+endif()
+cc_library(
+  analysis_predictor
+  SRCS ${ANALYSIS_PREDICTOR_SRCS}
+  DEPS ${ANALYSIS_PREDICTOR_DEPS})
+
+if(WITH_ONNXRUNTIME AND WIN32)
+  # Copy onnxruntime for some c++ test in Windows, since the test will
+  # be build only in CI, so suppose the generator in Windows is Ninja.
+  copy_onnx(test_paddle_inference_api)
+endif()

paddle/fluid/inference/api/analysis_predictor.h

@@ -0,0 +1,569 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <algorithm>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "paddle/fluid/framework/naive_executor.h"
+#include "paddle/fluid/inference/analysis/analyzer.h"
+#include "paddle/fluid/inference/api/api_impl.h"
+#include "paddle/fluid/inference/api/details/reset_tensor_array.h"
+#include "paddle/fluid/inference/api/helper.h"
+#include "paddle/fluid/inference/api/paddle_inference_api.h"
+#include "paddle/fluid/inference/api/resource_manager.h"
+#include "paddle/phi/common/bfloat16.h"
+#include "paddle/phi/common/float16.h"
+#include "paddle/phi/core/platform/device/gpu/gpu_types.h"
+#include "paddle/utils/string/printf.h"
+
+#ifdef PADDLE_WITH_TESTING
+#include <gtest/gtest.h>
+#include <gtest/gtest_prod.h>
+#endif
+
+#include "paddle/phi/common/data_type.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/pir/include/core/operation.h"
+#include "paddle/pir/include/core/program.h"
+
+namespace paddle_infer {
+namespace experimental {
+class InternalUtils;
+};
+}  // namespace paddle_infer
+///
+/// \file analysis_predictor.h
+///
+/// \brief Compared to NativePredictor, AnalysisPredictor is a high-performance
+/// predictor that includes many optimizations
+///
+/// \author paddle-infer@baidu.com
+/// \date 2020-01-01
+/// \since 1.7.0
+///
+
+namespace paddle {
+
+using framework::NaiveExecutor;
+using framework::proto::ProgramDesc;
+using inference::analysis::Analyzer;
+using inference::analysis::Argument;
+
+///
+/// \class AnalysisPredictor
+///
+/// \brief The analysis predictor is based on the original native predictor with
+/// IR and Analysis support. It will optimize IR and Parameters in the runtime.
+///
+/// The predictor has the following typical uses:
+///
+/// Get predictor
+/// \code{cpp}
+///   auto predictor = CreatePaddlePredictor(config);
+/// \endcode
+///
+/// Get input or output names
+/// \code{cpp}
+///   auto input_names = predictor->GetInputNames();
+///   auto output_names = predictor->GetOutputNames();
+/// \endcode
+///
+/// Get input or output tensors
+/// \code{cpp}
+///   auto input_t = predictor->GetInputTensor(input_names[0]);
+///   auto output_t = predictor->GetOutputTensor(output_names[0]);
+/// \endcode
+///
+/// Run predictor
+/// \code{cpp}
+///   predictor->ZeroCopyRun();
+/// \endcode
+///
+class AnalysisPredictor : public PaddlePredictor {
+ public:
+  ///
+  /// \brief Construct a new Analysis Predictor object
+  ///
+  /// \param[in] AnalysisConfig config
+  ///
+  explicit AnalysisPredictor(const AnalysisConfig &config);
+  ///
+  /// \brief Destroy the Analysis Predictor object
+  ///
+  ~AnalysisPredictor();
+
+  ///
+  /// \brief Initialize predictor
+  ///
+  /// Initializing predictor mainly includes the following tasks:
+  /// preparing scope, creating executor, preparing program, initializing the
+  /// variables required by the executor, getting the feed_target_names and
+  /// fetch_target_names, etc.
+  ///
+  /// \param[in] parent_scope parent scope
+  /// \param[in] program program
+  /// \return Whether the init function executed successfully
+  ///
+  bool Init(const std::shared_ptr<framework::Scope> &parent_scope,
+            const std::shared_ptr<framework::ProgramDesc> &program = nullptr);
+
+  ///
+  /// \brief Run the prediction engine. Deprecated. Please refer to ZeroCopyRun
+  ///
+  /// \param[in] inputs input tensors
+  /// \param[out] output_data output tensors
+  /// \param[in] batch_size data's batch size
+  /// \return Whether the function executed successfully
+  ///
+  bool Run(const std::vector<PaddleTensor> &inputs,
+           std::vector<PaddleTensor> *output_data,
+           int batch_size = -1) override;
+
+  ///
+  /// \brief Run the prediction engine (Recommended).
+  ///
+  /// \param[in] inputs input tensors
+  /// \param[out] outputs output tensors
+  /// \return Whether the function executed successfully
+  ///
+  bool Run(const std::vector<paddle::Tensor> &inputs,
+           std::vector<paddle::Tensor> *outputs) override;
+
+  ///
+  /// \brief Get the input names
+  ///
+  /// \return input names
+  ///
+  std::vector<std::string> GetInputNames() override;
+  ///
+  /// \brief Get the output names
+  ///
+  /// \return output names
+  ///
+  std::vector<std::string> GetOutputNames() override;
+
+  ///
+  /// \brief Get the value really need place, only for pir
+  ///
+  /// \return phi::place
+  ///
+  phi::Place GetTensorPlace(const pir::Value &value);
+  ///
+  /// \brief Get the Input Tensor object
+  ///
+  /// \param[in] name input name
+  /// \return input tensor
+  ///
+  std::unique_ptr<ZeroCopyTensor> GetInputTensor(
+      const std::string &name) override;
+  ///
+  /// \brief Get the Output Tensor object
+  ///
+  /// \param[in] name output name
+  /// \return output tensor
+  ///
+  std::unique_ptr<ZeroCopyTensor> GetOutputTensor(
+      const std::string &name) override;
+  ///
+  /// \brief Get all input names and their corresponding shapes
+  ///
+  /// \return the map of input names and shapes
+  ///
+  std::map<std::string, std::vector<int64_t>> GetInputTensorShape() override;
+  ///
+  /// \brief Get all input names and their corresponding type
+  ///
+  /// \return the map of input names and type
+  ///
+  std::map<std::string, paddle_infer::DataType> GetInputTypes() override;
+  ///
+  /// \brief Get all output names and their corresponding shapes
+  ///
+  /// \return the map of output names and shapes
+  ///
+  std::map<std::string, std::vector<int64_t>> GetOutputTensorShape() override;
+  ///
+  /// \brief Get all output names and their corresponding type
+  ///
+  /// \return the map of output names and type
+  ///
+  std::map<std::string, paddle_infer::DataType> GetOutputTypes() override;
+
+  ///
+  /// \brief Run the prediction engine
+  ///
+  /// \param switch_stream Whether the stream is switched
+  /// \return Whether the function executed successfully
+  ///
+  bool ZeroCopyRun(bool switch_stream = false) override;
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  // Note: Can only be used under thread_local semantics.
+  bool ExpRunWithExternalStream(const gpuStream_t stream);
+#endif
+
+  // Note: Can only be used under thread_local semantics.
+  bool ExpRunWithExternalStream(void *stream);
+
+  // Note: Can only be used under thread_local semantics.
+  bool ExpRunWithRuntimeConfig(void *config);
+
+  ///
+  /// \brief Get the execution stream on devices with a concept of stream,
+  /// otherwise returns nullptr.
+  ///
+  /// \return The execution stream or nullptr (CPU).
+  ///
+  void *GetExecStream() const override;
+
+  ///
+  /// \brief Create feed fetch variables
+  ///
+  /// \param[in] scope Scope needed to create variables
+  ///
+  void CreateFeedFetchVar(framework::Scope *scope);
+  ///
+  /// \brief Determine the model's inputs and outputs based on the program's
+  /// feed fetch op
+  ///
+  void PrepareFeedFetch();
+
+  ///
+  /// \brief Set predictor's argument according to config, which mainly includes
+  /// execution information and graph optimization related pass information
+  ///
+  void PrepareArgument();
+  ///
+  /// \brief According to argument information, execute the relevant pass
+  /// to get the optimized model program
+  ///
+  void OptimizeInferenceProgram();
+  ///
+  /// \brief According to argument information, execute the relevant pass
+  /// to get the optimized model program
+  ///
+  void OptimizeInferencePirProgram();
+  ///
+  /// \brief Clear the intermediate tensors of the predictor
+  ///
+  ///
+  void ClearIntermediateTensor() override;
+
+  ///
+  /// \brief Release all tmp tensor to compress the size of the memory pool.
+  /// The memory pool is considered to be composed of a list of chunks, if
+  /// the chunk is not occupied, it can be released.
+  ///
+  /// \return Number of bytes released. It may be smaller than the actual
+  /// released memory, because part of the memory is not managed by the
+  /// MemoryPool.
+  ///
+  uint64_t TryShrinkMemory() override;
+
+  ///
+  /// \brief Get the argument used by predictor
+  ///
+  /// \return the argument obtained by config
+  ///
+  Argument &analysis_argument() { return *argument_; }
+  ///
+  /// \brief Clone to get the new predictor. thread safe.
+  ///
+  /// \return get a new predictor
+  ///
+  std::unique_ptr<PaddlePredictor> Clone(void *stream = nullptr) override;
+  ///
+  /// \brief Get the scope used by predictor
+  ///
+  /// \return scope
+  ///
+  framework::Scope *scope() { return scope_.get(); }
+  ///
+  /// \brief Get the inference program
+  ///
+  /// \return the inference program
+  ///
+  framework::ProgramDesc &program() { return *inference_program_; }
+
+  ///
+  /// \brief Get the serialized program
+  ///
+  /// \return the serialized program
+  ///
+  std::string GetSerializedProgram() const override;
+
+  ///
+  /// \brief Get the fusion_statis_t
+  ///
+  /// \return the fusion_statis_t
+  ///
+  Argument::fusion_statis_t fusion_statis() { return fusion_statis_; }
+
+  ///
+  /// \brief Register a output hook function to operate the intermediate tensor
+  /// of op output. when using this function, memory reuse should be turned off.
+  /// The hook function signature is void(const std::string&, const
+  /// std::string&, const paddle::Tensor&>). Here, the first parameter is op's
+  /// type, the second param is output var name of the op, and the third
+  /// parameter is output tensor with the var name.
+  ///
+  void RegisterOutputHook(const OutputTensorHookFunc &hookfunc) override;
+
+  /// \brief Same as RegisterOutputHook
+  void RegisterInputHook(const InputTensorHookFunc &hookfunc) override;
+
+  ///
+  /// \brief Initialize onednn quantizer and execute onednn quantization pass
+  ///
+  /// \return Whether the function executed successfully
+  ///
+  bool MkldnnQuantize();
+
+ protected:
+  ///
+  /// \brief Prepare predictor's required programs, including loading model
+  /// information, graph optimization, and executor creation variables, etc.
+  ///
+  /// \param[in] program paddle program
+  /// \return Whether the function executed successfully
+  ///
+  bool PrepareProgram(const std::shared_ptr<framework::ProgramDesc> &program);
+  ///
+  /// \brief Prepare predictor's required programs, including loading model
+  /// information, graph optimization, and executor creation variables, etc.
+  ///
+  /// \return Whether the function executed successfully
+  ///
+  bool PreparePirProgram();
+  ///
+  /// \brief Prepare scope environment, each predictor has its own scope
+  ///
+  /// \param[in] parent_scope The scope of the predictor to be cloned, or null
+  /// \return Whether the function executed successfully
+  ///
+  bool PrepareScope(const std::shared_ptr<framework::Scope> &parent_scope);
+  ///
+  /// \brief Create an Executor object
+  ///
+  /// \return Whether the function executed successfully
+  ///
+  bool CreateExecutor();
+  ///
+  /// \brief According to the model's program, the executor creates ops
+  ///
+  /// \return Whether the function executed successfully
+  ///
+  bool PrepareExecutor();
+
+  ///
+  /// \brief Load model program.
+  ///
+  /// \return Whether the function executed successfully
+  ///
+  bool LoadProgramDesc();
+  ///
+  /// \brief Load model parameters.
+  ///
+  /// \return Whether the function executed successfully
+  ///
+  bool LoadParameters();
+
+  ///
+  /// \brief Save or Load pir model parameters.
+  ///
+  /// \return Whether the function executed successfully
+  ///
+  bool SaveOrLoadPirParameters(bool for_save);
+
+  ///
+  /// \brief Prepare input data, only used in Run()
+  ///
+  /// \param[in] input_datas input tensors
+  /// \param[in] scope the scope used by predictor
+  /// \return Whether the function executed successfully
+  ///
+  bool SetFeed(const std::vector<PaddleTensor> &input_datas,
+               framework::Scope *scope);
+
+  ///
+  /// \brief Prepare input data, only used in Run()
+  ///
+  /// \param[in] inputs input tensors
+  /// \param[in] scope the scope used by predictor
+  /// \return Whether the function executed successfully
+  ///
+  bool SetFeed(const std::vector<paddle::Tensor> &inputs,
+               framework::Scope *scope);
+
+  ///
+  /// \brief Get the output data, only used in Run()
+  ///
+  /// \param[out] output_data output tensors
+  /// \param[in] scope the scope used by predictor
+  /// \return Whether the function executed successfully
+  ///
+  bool GetFetch(std::vector<PaddleTensor> *output_data,
+                framework::Scope *scope);
+
+  ///
+  /// \brief Get the output data, only used in Run()
+  ///
+  /// \param[out] outputs output tensors
+  /// \param[in] scope the scope used by predictor
+  /// \return Whether the function executed successfully
+  ///
+  bool GetFetch(std::vector<paddle::Tensor> *outputs, framework::Scope *scope);
+
+  ///
+  /// \brief Get the output data, only used in GetFetch()
+  ///
+  /// \param[in] tensor for fetch op
+  /// \param[out] output_data output tensor
+  ///
+  template <typename T>
+  void GetFetchOne(const phi::DenseTensor &fetches, PaddleTensor *output_data);
+  ///
+  /// \brief PreSet for Mkldnn multi-thread and dynamic shape input.
+  ///
+  /// Used in AnalysisPredictor::Run(), do not support
+  /// AnalysisPredictor::ZeroCopyRun() now.
+  ///
+  /// \param[in] inputs tensors
+  ///
+  void MkldnnPreSet(const std::vector<PaddleTensor> &inputs);
+  ///
+  /// \brief PreSet for Mkldnn multi-thread and dynamic shape input.
+  ///
+  /// Used in AnalysisPredictor::Run().
+  ///
+  /// \param[in] inputs tensors
+  ///
+  void MkldnnPreSet(const std::vector<paddle::Tensor> &inputs);
+
+  ///
+  /// \brief PreSet for Mkldnn multi-thread and dynamic shape input.
+  ///
+  /// Used in AnalysisPredictor::Run(), do not support
+  /// AnalysisPredictor::ZeroCopyRun() now.
+  ///
+  /// \param[in] inputs tensor shape
+  ///
+  void MkldnnPreSet(const std::vector<std::vector<int>> &inputs_shape);
+
+  ///
+  /// \brief PostReset for Mkldnn multi-thread and dynamic shape input.
+  ///
+  /// Used in AnalysisPredictor::Run(), do not support
+  /// AnalysisPredictor::ZeroCopyRun() now.
+  ///
+  void MkldnnPostReset();
+
+#ifdef PADDLE_WITH_TENSORRT
+  ///
+  /// \brief save calibration table
+  ///
+  /// When we use Paddle-TRT INT8 engine, we need to generate calibration table
+  /// data first,
+  /// the calibration table contains the range for each op's input and output,
+  /// this whole process can be divided into several steps:
+  /// 1. Builds a 32-bit engine, runs it on the calibration set, and records a
+  ///  histogram for each tensor of the distribution of activation values.
+  /// 2. Builds a calibration table from the histograms.
+  /// After step 2, we need to store the calibration table on disk.
+  ///
+  /// \return Whether the function executed successfully
+  ///
+  bool SaveTrtCalibToDisk();
+#endif
+
+// Some more detailed tests, they are made the friends of the predictor, so that
+// the all the details can be tested.
+#if PADDLE_WITH_TESTING
+  FRIEND_TEST(AnalysisPredictor, analysis_off);
+  FRIEND_TEST(AnalysisPredictor, analysis_on);
+  FRIEND_TEST(AnalysisPredictor, with_gpu);
+#endif
+
+ protected:
+  const void *GetDeviceContexts() const override;
+
+ private:
+  void StatisticShapeRangeInfo();
+  void HookCollectShapeRangeInfo();
+  void InitPlace();
+  void InitDeviceContexts();
+  void InitResourceManager(void *stream);
+  std::string GetOptimizedModelPath();
+  void ClearExtraParams();
+
+ private:
+  AnalysisConfig config_;
+  std::unique_ptr<Argument> argument_ = nullptr;
+  Argument::fusion_statis_t fusion_statis_;
+  std::unique_ptr<NaiveExecutor> executor_;
+  phi::Place place_;
+  std::shared_ptr<framework::Scope> scope_;
+  framework::Scope *sub_scope_{nullptr};
+  std::shared_ptr<framework::ProgramDesc> inference_program_;
+  std::shared_ptr<pir::Program> pir_program_;
+  bool load_pir_model_{false};
+  std::vector<framework::OpDesc *> feeds_;
+  std::vector<pir::Operation *> pir_feeds_;
+  std::map<std::string, size_t> feed_names_;
+  // Sorted according to the idx.
+  std::map<size_t, std::string> idx2feeds_;
+  std::map<std::string, std::vector<int64_t>> feed_name2shapes_;
+  std::vector<framework::OpDesc *> fetches_;
+  std::vector<pir::Operation *> pir_fetches_;
+  std::map<size_t, std::string> idx2fetches_;
+  std::map<std::string, std::vector<int64_t>> fetch_name2shapes_;
+
+  phi::DataType model_precision_{phi::DataType::FLOAT32};
+
+  // Memory buffer for feed inputs. The temporary DenseTensor will cause serious
+  // concurrency problems, wrong results and memory leak, so cache them.
+  std::vector<phi::DenseTensor> feed_tensors_;
+  details::TensorArrayBatchCleaner tensor_array_batch_cleaner_;
+  // A mutex help to make Clone thread safe.
+  std::mutex clone_mutex_;
+  static int clone_num_;
+
+  int predictor_id_;
+  int root_predictor_id_{-1};
+
+ private:
+  std::once_flag register_input_hook_flag_;
+  std::once_flag register_output_hook_flag_;
+  std::vector<OutputTensorHookFunc> output_hookfuncs_;
+  std::vector<InputTensorHookFunc> input_hookfuncs_;
+  // Some status here that help to determine the status inside the predictor.
+  bool status_is_cloned_{false};
+
+  std::map<std::string, std::vector<std::vector<int32_t>>> shape_info_;
+  std::map<std::string, std::vector<std::vector<int32_t>>> shape_tensor_value_;
+
+  bool private_context_{false};
+  void *predictor_stream_{nullptr};
+  std::map<phi::Place, std::shared_future<std::unique_ptr<phi::DeviceContext>>>
+      device_contexts_;
+
+  friend class paddle_infer::experimental::InternalUtils;
+};
+
+}  // namespace paddle

paddle/fluid/inference/api/api.cc

@@ -0,0 +1,163 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <sstream>
+
+#include "paddle/common/flags.h"
+#include "paddle/fluid/framework/commit.h"
+#include "paddle/fluid/framework/lod_tensor.h"
+#include "paddle/fluid/framework/scope.h"
+#include "paddle/fluid/inference/api/paddle_inference_api.h"
+#include "paddle/fluid/inference/api/paddle_pass_builder.h"
+#include "paddle/fluid/platform/enforce.h"
+
+namespace paddle {
+
+int PaddleDtypeSize(PaddleDType dtype) {
+  switch (dtype) {
+    case PaddleDType::FLOAT32:
+      return sizeof(float);
+    case PaddleDType::BFLOAT16:
+      return sizeof(uint16_t);
+    case PaddleDType::INT64:
+      return sizeof(int64_t);
+    case PaddleDType::INT32:
+      return sizeof(int32_t);
+    case PaddleDType::UINT8:
+      return sizeof(uint8_t);
+    default:
+      assert(false);
+      return -1;
+  }
+}
+
+PaddleBuf::PaddleBuf(PaddleBuf &&other) noexcept
+    : data_(other.data_),
+      length_(other.length_),
+      memory_owned_(other.memory_owned_) {
+  other.memory_owned_ = false;
+  other.data_ = nullptr;
+  other.length_ = 0;
+}
+
+PaddleBuf::PaddleBuf(const PaddleBuf &other) { *this = other; }
+
+PaddleBuf &PaddleBuf::operator=(const PaddleBuf &other) {
+  if (this == &other) return *this;
+  if (!other.memory_owned_) {
+    data_ = other.data_;
+    length_ = other.length_;
+    memory_owned_ = other.memory_owned_;
+  } else {
+    Resize(other.length());
+    // if other.length() == 0 or other.data() == nullptr, then the memcpy
+    // behavior is undefined
+    if (other.length() && other.data())
+      memcpy(data_, other.data(), other.length());
+    else if (other.length())
+      PADDLE_THROW(common::errors::InvalidArgument(
+          "Invalid argument, null pointer data with length %u is passed",
+          other.length()));
+
+    length_ = other.length();
+    memory_owned_ = true;
+  }
+  return *this;
+}
+
+PaddleBuf &PaddleBuf::operator=(PaddleBuf &&other) noexcept {
+  // only the buffer with external memory can be copied
+  data_ = other.data_;
+  length_ = other.length_;
+  memory_owned_ = other.memory_owned_;
+  other.data_ = nullptr;
+  other.length_ = 0;
+  other.memory_owned_ = false;
+  return *this;
+}
+
+void PaddleBuf::Resize(size_t length) {
+  // Only the owned memory can be reset, the external memory can't be changed.
+  if (length_ >= length) return;
+  if (memory_owned_) {
+    Free();
+    data_ = new char[length];
+    length_ = length;
+    memory_owned_ = true;
+  } else {
+    PADDLE_THROW(common::errors::PreconditionNotMet(
+        "The memory is allocated externally, can not Resized"));
+  }
+}
+
+void PaddleBuf::Reset(void *data, size_t length) {
+  Free();
+  memory_owned_ = false;
+  data_ = data;
+  length_ = length;
+}
+
+void PaddleBuf::Free() {
+  if (memory_owned_ && data_) {
+    PADDLE_ENFORCE_GT(
+        length_,
+        0UL,
+        common::errors::PreconditionNotMet(
+            "The memory used in PaddleBuf %d should be greater than 0",
+            length_));
+    delete[] static_cast<char *>(data_);
+    data_ = nullptr;
+    length_ = 0;
+  }
+}
+
+NativeConfig::NativeConfig() {
+  LOG(WARNING) << "The paddle::NativeConfig interface is going to be "
+                  "deprecated in the next release, please use the latest "
+                  "paddle_infer::Config instead.";
+}
+
+std::string get_version() {
+  std::stringstream ss;
+  ss << "version: " << framework::paddle_version() << "\n";
+  ss << "commit: " << framework::paddle_commit() << "\n";
+  ss << "branch: " << framework::paddle_compile_branch() << "\n";
+  return ss.str();
+}
+
+void UpdateDllFlag(const char *name, const char *value) {
+  std::string ret;
+  LOG(WARNING)
+      << "The function \"UpdateDllFlag\" is only used to update the flag "
+         "on the Windows shared library";
+  bool success = paddle::flags::SetFlagValue(name, value);
+
+  PADDLE_ENFORCE_EQ(
+      success,
+      true,
+      common::errors::InvalidArgument(
+          "Fail to update flag: %s, please make sure the flag exists.", name));
+}
+
+#ifdef PADDLE_WITH_CRYPTO
+std::shared_ptr<framework::Cipher> MakeCipher(const std::string &config_file) {
+  return framework::CipherFactory::CreateCipher(config_file);
+}
+#endif
+
+}  // namespace paddle
+
+#ifdef PADDLE_WITH_CUSTOM_DEVICE
+#include "paddle/phi/capi/capi.h"
+#endif

paddle/fluid/inference/api/api_impl.cc

@@ -0,0 +1,403 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/fluid/inference/api/api_impl.h"
+
+#include <glog/logging.h>
+
+#include <memory>
+#include <sstream>
+#include <string>
+
+#include "paddle/fluid/framework/feed_fetch_method.h"
+#include "paddle/fluid/inference/api/helper.h"
+#include "paddle/phi/common/place.h"
+#include "paddle/phi/core/platform/cpu_helper.h"
+#include "paddle/phi/core/platform/profiler.h"
+
+PD_DEFINE_bool(profile, false, "Turn on profiler for fluid");  // NOLINT
+
+namespace paddle {
+namespace {
+using paddle::inference::Timer;
+
+template <class T>
+std::string num2str(T a) {
+  std::stringstream istr;
+  istr << a;
+  return istr.str();
+}
+}  // namespace
+
+void NativePaddlePredictor::PrepareFeedFetch() {
+  for (auto *op : inference_program_->Block(0).AllOps()) {
+    if (op->Type() == "feed") {
+      int idx = PADDLE_GET_CONST(int, op->GetAttr("col"));
+      if (feeds_.size() <= static_cast<size_t>(idx)) {
+        feeds_.resize(idx + 1);
+      }
+      feeds_[idx] = op;
+      feed_names_[op->Output("Out")[0]] = idx;
+    } else if (op->Type() == "fetch") {
+      int idx = PADDLE_GET_CONST(int, op->GetAttr("col"));
+      if (fetches_.size() <= static_cast<size_t>(idx)) {
+        fetches_.resize(idx + 1);
+      }
+      fetches_[idx] = op;
+    }
+  }
+}
+
+bool NativePaddlePredictor::Init(
+    std::shared_ptr<framework::Scope> parent_scope) {
+  VLOG(3) << "Predictor::init()";
+  if (FLAGS_profile) {
+    LOG(WARNING) << "Profiler is activated, might affect the performance";
+    LOG(INFO) << "You can turn off by set gflags '-profile false'";
+
+    auto tracking_device = config_.use_gpu ? platform::ProfilerState::kAll
+                                           : platform::ProfilerState::kCPU;
+    platform::EnableProfiler(tracking_device);
+  }
+
+  // no matter with or without OneDNN
+  paddle::platform::SetNumThreads(config_.cpu_math_library_num_threads());
+
+  if (config_.use_gpu) {
+    PADDLE_ENFORCE_EQ(config_.use_xpu,
+                      false,
+                      common::errors::InvalidArgument(
+                          "Only one choice can be made between CPU and XPU."));
+    place_ = phi::GPUPlace(config_.device);
+  } else if (config_.use_xpu) {
+    place_ = phi::XPUPlace(config_.device);
+  } else {
+    place_ = phi::CPUPlace();
+  }
+  if (parent_scope) {
+    scope_ = parent_scope;
+    sub_scope_ = &(parent_scope->NewScope());
+    PADDLE_ENFORCE_NOT_NULL(sub_scope_,
+                            common::errors::PreconditionNotMet(
+                                "The sub_scope should not be nullptr."));
+  } else {
+    paddle::framework::InitMemoryMethod();
+    paddle::framework::InitDevices();
+    paddle::framework::InitDefaultKernelSignatureMap();
+    scope_ = std::make_unique<paddle::framework::Scope>();
+  }
+
+  executor_ = std::make_unique<paddle::framework::Executor>(place_);
+
+  // Initialize the inference program
+  if (!config_.model_dir.empty()) {  // NOLINT
+    // Parameters are saved in separate files sited in
+    // the specified `dirname`.
+    inference_program_ = paddle::inference::Load(
+        executor_.get(), scope_.get(), config_.model_dir);
+  } else if (!config_.prog_file.empty() && !config_.param_file.empty()) {
+    // All parameters are saved in a single file.
+    // The file names should be consistent with that used
+    // in Python API `fluid.io.save_inference_model`.
+    inference_program_ = paddle::inference::Load(
+        executor_.get(), scope_.get(), config_.prog_file, config_.param_file);
+  } else {
+    LOG(ERROR) << "fail to load inference model from " << config_.model_dir;
+    return false;
+  }
+
+  ctx_ = executor_->Prepare(*inference_program_, 0);
+  executor_->CreateVariables(
+      *inference_program_, sub_scope_ ? sub_scope_ : scope_.get(), 0);
+
+  // Get the feed_target_names and fetch_target_names
+  PrepareFeedFetch();
+  return true;
+}
+
+NativePaddlePredictor::~NativePaddlePredictor() {
+  if (FLAGS_profile) {
+    platform::DisableProfiler(platform::EventSortingKey::kTotal,
+                              "./profile.log");
+  }
+  if (sub_scope_) {
+    scope_->DeleteScope(sub_scope_);
+  }
+}
+
+bool NativePaddlePredictor::Run(const std::vector<PaddleTensor> &inputs,
+                                std::vector<PaddleTensor> *output_data,
+                                int batch_size) {
+#ifndef PADDLE_ON_INFERENCE
+  LOG_FIRST_N(WARNING, 5) << "The NaiveExecutor can not work properly if the "
+                             "cmake flag ON_INFER is not set.";
+  LOG_FIRST_N(WARNING, 5) << "Unlike the training phase, all the scopes and "
+                             "variables will be reused to save the allocation "
+                             "overhead.";
+  LOG_FIRST_N(WARNING, 5) << "Please re-compile the inference library by "
+                             "setting the cmake flag ON_INFER=ON if you are "
+                             "running Paddle Inference";
+#endif  // PADDLE_ON_INFERENCE
+  if (UNLIKELY(config_.cpu_math_library_num_threads() > 1)) {
+    paddle::platform::SetNumThreads(config_.cpu_math_library_num_threads());
+  }
+  VLOG(3) << "Predictor::predict";
+  Timer timer;
+  timer.tic();
+  // set feed variable
+  framework::Scope *scope = sub_scope_ != nullptr ? sub_scope_ : scope_.get();
+  if (!SetFeed(inputs, scope)) {
+    LOG(ERROR) << "fail to set feed";
+    return false;
+  }
+  // Run the inference program
+  // if share variables, we need not create variables
+  VLOG(4) << "Run prepared context";
+  executor_->RunPreparedContext(ctx_.get(),
+                                scope,
+                                false, /* don't create local scope each time*/
+                                false /* don't create variable each time */);
+  VLOG(4) << "Finish prepared context";
+  // get fetch variable
+  if (!GetFetch(output_data, scope)) {
+    LOG(ERROR) << "fail to get fetches";
+    return false;
+  }
+  VLOG(3) << "predict cost: " << timer.toc() << "ms";
+
+  // For some other vector like containers not cleaned after each batch.
+  tensor_array_batch_cleaner_.CollectNoTensorVars(scope_.get());
+  tensor_array_batch_cleaner_.ResetNoTensorVars();
+  return true;
+}
+
+std::unique_ptr<PaddlePredictor> NativePaddlePredictor::Clone(void *stream) {
+  std::lock_guard<std::mutex> lk(clone_mutex_);
+  VLOG(3) << "Predictor::clone";
+  std::unique_ptr<PaddlePredictor> cls(new NativePaddlePredictor(config_));
+  // Hot fix the bug that result diff in multi-thread.
+  // TODO(Superjomn) re-implement a real clone here.
+  PADDLE_ENFORCE_NOT_NULL(
+      dynamic_cast<NativePaddlePredictor *>(cls.get()),
+      common::errors::PreconditionNotMet(
+          "Dynamic_cast from PaddlePredictor to NativePaddlePredictor failed"));
+  if (!dynamic_cast<NativePaddlePredictor *>(cls.get())->Init(nullptr)) {
+    LOG(ERROR) << "fail to call Init";
+    return nullptr;
+  }
+  return cls;
+}
+
+bool NativePaddlePredictor::SetFeed(const std::vector<PaddleTensor> &inputs,
+                                    framework::Scope *scope) {
+  VLOG(3) << "Predictor::set_feed";
+  if (inputs.size() != feeds_.size()) {
+    LOG(ERROR) << "wrong feed input size, need " << feeds_.size() << " but get "
+               << inputs.size();
+    return false;
+  }
+
+  // Cache the inputs memory for better concurrency performance.
+  feed_tensors_.resize(inputs.size());
+
+  for (size_t i = 0; i < inputs.size(); ++i) {
+    auto &input = feed_tensors_[i];
+    phi::DDim ddim = common::make_ddim(inputs[i].shape);
+    void *input_ptr = nullptr;
+    if (inputs[i].dtype == PaddleDType::INT64) {
+      input_ptr = input.mutable_data<int64_t>(ddim, place_);
+    } else if (inputs[i].dtype == PaddleDType::FLOAT32) {
+      input_ptr = input.mutable_data<float>(ddim, place_);
+    } else if (inputs[i].dtype == PaddleDType::INT32) {
+      input_ptr = input.mutable_data<int32_t>(ddim, place_);
+    } else if (inputs[i].dtype == PaddleDType::BFLOAT16) {
+      input_ptr = input.mutable_data<bfloat16>(ddim, place_);
+    } else {
+      LOG(ERROR) << "unsupported feed type " << inputs[i].dtype;
+      return false;
+    }
+
+    PADDLE_ENFORCE_NOT_NULL(input_ptr,
+                            common::errors::InvalidArgument(
+                                "The input_ptr should not be nullptr."));
+    PADDLE_ENFORCE_NOT_NULL(
+        inputs[i].data.data(),
+        common::errors::InvalidArgument(
+            "The data of input tensor should not be null."));
+    PADDLE_ENFORCE_EQ(
+        inputs[i].data.length(),
+        input.numel() * phi::SizeOf(input.dtype()),
+        common::errors::InvalidArgument(
+            "The data contained in the input PaddleTensor had wrong length."));
+
+    if (phi::is_cpu_place(place_)) {
+      // TODO(panyx0718): Init DenseTensor from existing memcpy to save a copy.
+      std::memcpy(static_cast<void *>(input_ptr),
+                  inputs[i].data.data(),
+                  inputs[i].data.length());
+    } else if (phi::is_gpu_place(place_)) {
+      PADDLE_ENFORCE_EQ(
+          phi::is_xpu_place(place_),
+          false,
+          common::errors::InvalidArgument(
+              "Only one choice can be made between CPU and XPU."));
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+      phi::DeviceContextPool &pool = phi::DeviceContextPool::Instance();
+      auto *dev_ctx = static_cast<const phi::GPUContext *>(pool.Get(place_));
+      auto dst_gpu_place = place_;
+      memory::Copy(dst_gpu_place,
+                   static_cast<void *>(input_ptr),
+                   phi::CPUPlace(),
+                   inputs[i].data.data(),
+                   inputs[i].data.length(),
+                   dev_ctx->stream());
+#else
+      PADDLE_THROW(common::errors::Unavailable(
+          "Not compile with CUDA, should not reach here."));
+#endif
+    } else if (phi::is_xpu_place(place_)) {
+#ifdef PADDLE_WITH_XPU
+      auto dst_xpu_place = place_;
+      memory::Copy(dst_xpu_place,
+                   static_cast<void *>(input_ptr),
+                   phi::CPUPlace(),
+                   inputs[i].data.data(),
+                   inputs[i].data.length());
+#else
+      PADDLE_THROW(common::errors::Unavailable(
+          "Not compile with XPU, should not reach here."));
+#endif
+    }
+
+    // TODO(Superjomn) Low performance, need optimization for heavy LoD copy.
+    phi::LegacyLoD lod;
+    for (auto &level : inputs[i].lod) {
+      lod.emplace_back(level);
+    }
+    input.set_lod(lod);
+    int idx = -1;
+    if (config_.specify_input_name) {  // NOLINT
+      idx = static_cast<int>(feed_names_[inputs[i].name]);
+    } else {
+      idx = PADDLE_GET_CONST(int, feeds_[i]->GetAttr("col"));
+    }
+    framework::SetFeedVariable(scope, input, "feed", idx);
+  }
+  return true;
+}
+template <typename T>
+void NativePaddlePredictor::GetFetchOne(const phi::DenseTensor &fetch,
+                                        PaddleTensor *output) {
+  // set shape.
+  auto shape = common::vectorize(fetch.dims());
+  output->shape.assign(shape.begin(), shape.end());
+  // set data.
+  const T *data = fetch.data<T>();
+  int num_elems = inference::VecReduceToInt(shape);
+  output->data.Resize(num_elems * sizeof(T));
+  // The fetched tensor output by fetch op, should always in CPU memory, so just
+  // copy.
+  memcpy(output->data.data(), data, num_elems * sizeof(T));
+  // set lod
+  output->lod.clear();
+  for (auto &level : fetch.lod()) {
+    output->lod.emplace_back(level.begin(), level.end());
+  }
+}
+
+bool NativePaddlePredictor::GetFetch(std::vector<PaddleTensor> *outputs,
+                                     framework::Scope *scope) {
+  VLOG(3) << "Predictor::get_fetch";
+  outputs->resize(fetches_.size());
+  for (size_t i = 0; i < fetches_.size(); ++i) {
+    int idx = PADDLE_GET_CONST(int, fetches_[i]->GetAttr("col"));
+    PADDLE_ENFORCE_EQ(
+        static_cast<size_t>(idx),
+        i,
+        common::errors::InvalidArgument(
+            "Fetch op's col attr(%d) should be equal to the index(%d)",
+            idx,
+            i));
+    framework::FetchType &fetch_var =
+        framework::GetFetchVariable(*scope, "fetch", idx);
+    auto fetch = PADDLE_GET_CONST(phi::DenseTensor, fetch_var);
+    auto type = framework::TransToProtoVarType(fetch.dtype());
+    auto output = &(outputs->at(i));
+    output->name = fetches_[idx]->Input("X")[0];
+    if (type == framework::DataTypeTrait<float>::DataType()) {
+      GetFetchOne<float>(fetch, output);
+      output->dtype = PaddleDType::FLOAT32;
+    } else if (type == framework::DataTypeTrait<int64_t>::DataType()) {
+      GetFetchOne<int64_t>(fetch, output);
+      output->dtype = PaddleDType::INT64;
+    } else if (type == framework::DataTypeTrait<int32_t>::DataType()) {
+      GetFetchOne<int32_t>(fetch, output);
+      output->dtype = PaddleDType::INT32;
+    } else {
+      LOG(ERROR) << "unknown type, only support float32, int64 and int32 now.";
+    }
+  }
+  return true;
+}
+
+template <>
+std::unique_ptr<PaddlePredictor>
+CreatePaddlePredictor<NativeConfig, PaddleEngineKind::kNative>(
+    const NativeConfig &config) {
+  // TODO(NHZlX): Should add the link to the doc of
+  // paddle_infer::CreatePredictor<paddle_infer::Config>
+  VLOG(3) << "create NativePaddlePredictor";
+  if (config.use_gpu) {
+    // 1. GPU memory
+    PADDLE_ENFORCE_GE(config.fraction_of_gpu_memory,
+                      0.f,
+                      common::errors::InvalidArgument(
+                          "fraction_of_gpu_memory in the config should be set "
+                          "to range (0., 1.]"));
+    PADDLE_ENFORCE_GE(config.device,
+                      0,
+                      common::errors::PreconditionNotMet(
+                          "Invalid device id %d, the device id should be "
+                          "greater than or equal to 0.",
+                          config.device));
+    std::vector<std::string> flags;
+    if (config.fraction_of_gpu_memory >= 0.0f ||
+        config.fraction_of_gpu_memory <= 0.95f) {
+      std::string flag = "--fraction_of_gpu_memory_to_use=" +
+                         num2str<float>(config.fraction_of_gpu_memory);
+      flags.push_back(flag);
+      VLOG(3) << "set flag: " << flag;
+      framework::InitGflags(flags);
+    }
+  }
+
+  std::unique_ptr<PaddlePredictor> predictor(new NativePaddlePredictor(config));
+  PADDLE_ENFORCE_NOT_NULL(
+      dynamic_cast<NativePaddlePredictor *>(predictor.get()),
+      common::errors::PreconditionNotMet(
+          "Dynamic_cast from PaddlePredictor to NativePaddlePredictor failed"));
+  if (!dynamic_cast<NativePaddlePredictor *>(predictor.get())->Init(nullptr)) {
+    return nullptr;
+  }
+  return predictor;
+}
+
+template <>
+std::unique_ptr<PaddlePredictor> CreatePaddlePredictor<NativeConfig>(
+    const NativeConfig &config) {
+  LOG(WARNING) << "Deprecated. Please use CreatePredictor instead.";
+  return CreatePaddlePredictor<NativeConfig, PaddleEngineKind::kNative>(config);
+}
+
+}  // namespace paddle

paddle/fluid/inference/api/api_impl.h

@@ -0,0 +1,88 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <glog/logging.h>
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "paddle/common/ddim.h"
+#include "paddle/fluid/framework/dense_tensor_array.h"
+#include "paddle/fluid/framework/lod_tensor.h"
+#include "paddle/fluid/framework/naive_executor.h"
+#include "paddle/fluid/inference/api/details/reset_tensor_array.h"
+#include "paddle/fluid/inference/api/paddle_api.h"
+#include "paddle/fluid/inference/api/paddle_inference_api.h"
+#include "paddle/fluid/inference/io.h"
+#include "paddle/fluid/platform/init.h"
+#include "paddle/phi/common/place.h"
+#include "paddle/phi/core/platform/profiler.h"
+
+namespace paddle {
+
+namespace framework {
+class Scope;
+}  // namespace framework
+
+class NativePaddlePredictor : public PaddlePredictor {
+ public:
+  explicit NativePaddlePredictor(const NativeConfig &config)
+      : config_(config) {}
+
+  // will only create sub scope if have global scope
+  bool Init(std::shared_ptr<framework::Scope> parent_scope);
+
+  bool Run(const std::vector<PaddleTensor> &inputs,
+           std::vector<PaddleTensor> *output_data,
+           int batch_size = -1) override;
+
+  std::unique_ptr<PaddlePredictor> Clone(void *stream = nullptr) override;
+
+  ~NativePaddlePredictor() override;
+
+  framework::Scope *scope() { return sub_scope_ ? sub_scope_ : scope_.get(); }
+
+ protected:
+  bool SetFeed(const std::vector<PaddleTensor> &input_datas,
+               framework::Scope *scope);
+  bool GetFetch(std::vector<PaddleTensor> *output_data,
+                framework::Scope *scope);
+  template <typename T>
+  void GetFetchOne(const phi::DenseTensor &fetches, PaddleTensor *output_data);
+  void PrepareFeedFetch();
+
+  NativeConfig config_;
+  phi::Place place_;
+  std::unique_ptr<framework::Executor> executor_;
+  std::shared_ptr<framework::Scope> scope_;
+  std::unique_ptr<framework::ExecutorPrepareContext> ctx_;
+  std::unique_ptr<framework::ProgramDesc> inference_program_;
+  std::vector<framework::OpDesc *> feeds_;
+  std::map<std::string, size_t> feed_names_;
+  std::vector<framework::OpDesc *> fetches_;
+  // Memory buffer for feed inputs. The temporary DenseTensor will cause serious
+  // concurrency problems, wrong results and memory leak, so cache them.
+  std::vector<phi::DenseTensor> feed_tensors_;
+  // Do not use unique_ptr, use parent scope to delete
+  framework::Scope *sub_scope_{nullptr};
+  details::TensorArrayBatchCleaner tensor_array_batch_cleaner_;
+  // A mutex to make Clone thread safe.
+  std::mutex clone_mutex_;
+};
+
+}  // namespace paddle

paddle/fluid/inference/api/demo_ci/.gitignore

@@ -0,0 +1 @@
+data

paddle/fluid/inference/api/demo_ci/CMakeLists.txt

@@ -0,0 +1,356 @@
+cmake_minimum_required(VERSION 3.5)
+project(cpp_inference_demo CXX C)
+option(WITH_MKL "Compile demo with MKL/OpenBlas support, default use MKL." ON)
+option(WITH_GPU "Compile demo with GPU/CPU, default use CPU." OFF)
+option(WITH_STATIC_LIB
+       "Compile demo with static/shared library, default use static." ON)
+option(USE_TENSORRT "Compile demo with TensorRT." OFF)
+option(WITH_ONNXRUNTIME "Compile demo with ONNXRuntime" OFF)
+option(WITH_SHARED_PHI "Compile demo with phi shared lib" ON)
+option(CUSTOM_OPERATOR_FILES "List of file names for custom operators" "")
+option(CUSTOM_PASS_FILES "List of file names for custom passes" "")
+
+if(NOT WITH_STATIC_LIB)
+  add_definitions("-DPADDLE_WITH_SHARED_LIB")
+else()
+  # PD_INFER_DECL is mainly used to set the dllimport/dllexport attribute in dynamic library mode.
+  # Set it to empty in static library mode to avoid compilation issues.
+  add_definitions("/DPD_INFER_DECL=")
+endif()
+
+macro(safe_set_static_flag)
+  foreach(flag_var
+          CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
+          CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
+    if(${flag_var} MATCHES "/MD")
+      string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
+    endif()
+  endforeach()
+endmacro()
+
+if(NOT DEFINED PADDLE_LIB)
+  message(
+    FATAL_ERROR "please set PADDLE_LIB with -DPADDLE_LIB=/path/paddle/lib")
+endif()
+if(NOT DEFINED DEMO_NAME)
+  message(FATAL_ERROR "please set DEMO_NAME with -DDEMO_NAME=demo_name")
+endif()
+
+include_directories("${PADDLE_LIB}/paddle/include")
+set(PADDLE_LIB_THIRD_PARTY_PATH "${PADDLE_LIB}/third_party/install/")
+include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}protobuf/include")
+include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}glog/include")
+include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}utf8proc/include")
+include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}gflags/include")
+include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}xxhash/include")
+include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}cryptopp/include")
+include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}yaml-cpp/include")
+
+link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}protobuf/lib")
+link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}glog/lib")
+link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}utf8proc/lib")
+link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}gflags/lib")
+link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}xxhash/lib")
+link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}cryptopp/lib")
+link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}yaml-cpp/lib")
+link_directories("${PADDLE_LIB}/paddle/lib")
+
+if(WITH_ONNXRUNTIME)
+  include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}onnxruntime/include")
+  include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}paddle2onnx/include")
+
+  link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}onnxruntime/lib")
+  link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}paddle2onnx/lib")
+endif()
+
+if(WIN32)
+  add_definitions("/DGOOGLE_GLOG_DLL_DECL=")
+  option(MSVC_STATIC_CRT "use static C Runtime library by default" ON)
+  if(MSVC_STATIC_CRT)
+    if(WITH_MKL)
+      set(FLAG_OPENMP "/openmp")
+    endif()
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4244 /wd4251 /wd4267 /wd4305")
+    set(CMAKE_C_FLAGS_DEBUG
+        "${CMAKE_C_FLAGS_DEBUG} /bigobj /MTd ${FLAG_OPENMP}")
+    set(CMAKE_C_FLAGS_RELEASE
+        "${CMAKE_C_FLAGS_RELEASE} /bigobj /MT ${FLAG_OPENMP}")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4251 /wd4267 /wd4305")
+    set(CMAKE_CXX_FLAGS_DEBUG
+        "${CMAKE_CXX_FLAGS_DEBUG} /bigobj /MTd ${FLAG_OPENMP}")
+    set(CMAKE_CXX_FLAGS_RELEASE
+        "${CMAKE_CXX_FLAGS_RELEASE} /bigobj /MT ${FLAG_OPENMP}")
+    safe_set_static_flag()
+    if(WITH_STATIC_LIB)
+      add_definitions(-DSTATIC_LIB)
+    endif()
+  endif()
+else()
+  if(WITH_MKL)
+    set(FLAG_OPENMP "-fopenmp")
+  endif()
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 ${FLAG_OPENMP}")
+endif()
+
+if(WITH_GPU)
+  if(NOT WIN32)
+    set(CUDA_LIB
+        "/usr/local/cuda/lib64/"
+        CACHE STRING "CUDA Library")
+  else()
+    set(CUDA_LIB
+        ""
+        CACHE STRING "CUDA_LIB")
+    if("${CUDA_LIB}" STREQUAL "")
+      if(DEFINED ENV{CUDA_PATH})
+        set(CUDA_LIB "$ENV{CUDA_PATH}\\lib\\x64")
+      else()
+        set(CUDA_LIB
+            "C:\\Program\ Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v10.2\\lib\\x64"
+        )
+      endif()
+    endif()
+    message(STATUS "Current CUDA lib path: ${CUDA_LIB}")
+  endif()
+endif()
+
+if(USE_TENSORRT AND WITH_GPU)
+  set(TENSORRT_ROOT
+      ""
+      CACHE STRING "The root directory of TensorRT library")
+  if("${TENSORRT_ROOT}" STREQUAL "")
+    message(
+      FATAL_ERROR
+        "The TENSORRT_ROOT is empty, you must assign it a value with CMake command. Such as: -DTENSORRT_ROOT=TENSORRT_ROOT_PATH "
+    )
+  endif()
+  set(TENSORRT_INCLUDE_DIR ${TENSORRT_ROOT}/include)
+  set(TENSORRT_LIB_DIR ${TENSORRT_ROOT}/lib)
+  file(READ ${TENSORRT_INCLUDE_DIR}/NvInfer.h TENSORRT_VERSION_FILE_CONTENTS)
+  string(REGEX MATCH "define NV_TENSORRT_MAJOR +([0-9]+)"
+               TENSORRT_MAJOR_VERSION "${TENSORRT_VERSION_FILE_CONTENTS}")
+  if("${TENSORRT_MAJOR_VERSION}" STREQUAL "")
+    file(READ ${TENSORRT_INCLUDE_DIR}/NvInferVersion.h
+         TENSORRT_VERSION_FILE_CONTENTS)
+    string(REGEX MATCH "define NV_TENSORRT_MAJOR +([0-9]+)"
+                 TENSORRT_MAJOR_VERSION "${TENSORRT_VERSION_FILE_CONTENTS}")
+  endif()
+  if("${TENSORRT_MAJOR_VERSION}" STREQUAL "")
+    message(SEND_ERROR "Failed to detect TensorRT version.")
+  endif()
+  string(REGEX REPLACE "define NV_TENSORRT_MAJOR +([0-9]+)" "\\1"
+                       TENSORRT_MAJOR_VERSION "${TENSORRT_MAJOR_VERSION}")
+  message(
+    STATUS "Current TensorRT header is ${TENSORRT_INCLUDE_DIR}/NvInfer.h. "
+           "Current TensorRT version is v${TENSORRT_MAJOR_VERSION}. ")
+  include_directories("${TENSORRT_INCLUDE_DIR}")
+  link_directories("${TENSORRT_LIB_DIR}")
+endif()
+
+if(WITH_MKL)
+  set(MATH_LIB_PATH "${PADDLE_LIB_THIRD_PARTY_PATH}mklml")
+  include_directories("${MATH_LIB_PATH}/include")
+  if(WIN32)
+    set(MATH_LIB ${MATH_LIB_PATH}/lib/mklml${CMAKE_STATIC_LIBRARY_SUFFIX}
+                 ${MATH_LIB_PATH}/lib/libiomp5md${CMAKE_STATIC_LIBRARY_SUFFIX})
+  else()
+    set(MATH_LIB
+        ${MATH_LIB_PATH}/lib/libmklml_intel${CMAKE_SHARED_LIBRARY_SUFFIX}
+        ${MATH_LIB_PATH}/lib/libiomp5${CMAKE_SHARED_LIBRARY_SUFFIX})
+  endif()
+  set(ONEDNN_PATH "${PADDLE_LIB_THIRD_PARTY_PATH}onednn")
+  if(EXISTS ${ONEDNN_PATH})
+    include_directories("${ONEDNN_PATH}/include")
+    if(WIN32)
+      set(ONEDNN_LIB ${ONEDNN_PATH}/lib/mkldnn.lib)
+    else()
+      set(ONEDNN_LIB ${ONEDNN_PATH}/lib/libdnnl.so.3)
+    endif()
+  endif()
+else()
+  set(OPENBLAS_LIB_PATH "${PADDLE_LIB_THIRD_PARTY_PATH}openblas")
+  include_directories("${OPENBLAS_LIB_PATH}/include/openblas")
+  if(WIN32)
+    set(MATH_LIB
+        ${OPENBLAS_LIB_PATH}/lib/openblas${CMAKE_STATIC_LIBRARY_SUFFIX})
+  else()
+    set(MATH_LIB
+        ${OPENBLAS_LIB_PATH}/lib/libopenblas${CMAKE_STATIC_LIBRARY_SUFFIX})
+  endif()
+endif()
+
+if(WITH_STATIC_LIB)
+  set(DEPS
+      ${PADDLE_LIB}/paddle/lib/libpaddle_inference${CMAKE_STATIC_LIBRARY_SUFFIX}
+  )
+else()
+  if(WIN32)
+    set(DEPS
+        ${PADDLE_LIB}/paddle/lib/paddle_inference${CMAKE_STATIC_LIBRARY_SUFFIX})
+  else()
+    set(DEPS
+        ${PADDLE_LIB}/paddle/lib/libpaddle_inference${CMAKE_SHARED_LIBRARY_SUFFIX}
+    )
+  endif()
+endif()
+
+if(WITH_ONNXRUNTIME)
+  set(DEPS ${DEPS} onnxruntime paddle2onnx)
+endif()
+
+if(NOT WIN32)
+  set(EXTERNAL_LIB "-lrt -ldl -lpthread")
+  set(DEPS
+      ${DEPS}
+      ${MATH_LIB}
+      ${ONEDNN_LIB}
+      glog
+      gflags
+      protobuf
+      xxhash
+      cryptopp
+      utf8proc
+      ${PADDLE_LIB_THIRD_PARTY_PATH}yaml-cpp/lib/libyaml-cpp.a
+      ${EXTERNAL_LIB})
+  if(WITH_SHARED_PHI)
+    set(DEPS
+        ${DEPS} ${PADDLE_LIB}/paddle/lib/libphi${CMAKE_SHARED_LIBRARY_SUFFIX}
+        ${PADDLE_LIB}/paddle/lib/libphi_core${CMAKE_SHARED_LIBRARY_SUFFIX}
+        ${PADDLE_LIB}/paddle/lib/libcommon${CMAKE_SHARED_LIBRARY_SUFFIX})
+    if(WITH_GPU OR WITH_ROCM)
+      set(DEPS
+          ${DEPS}
+          ${PADDLE_LIB}/paddle/lib/libphi_gpu${CMAKE_SHARED_LIBRARY_SUFFIX})
+    endif()
+  endif()
+else()
+  set(DEPS
+      ${DEPS}
+      ${MATH_LIB}
+      ${ONEDNN_LIB}
+      phi
+      glog
+      gflags_static
+      libprotobuf
+      xxhash
+      cryptopp-static
+      utf8proc_static
+      ${EXTERNAL_LIB})
+  set(DEPS ${DEPS} shlwapi.lib
+           ${PADDLE_LIB_THIRD_PARTY_PATH}yaml-cpp/lib/yaml-cpp.lib)
+endif()
+
+if(NOT WIN32 AND NOT APPLE)
+  set(DEPS ${DEPS} stdc++fs)
+endif()
+
+if(WITH_GPU)
+  if(NOT WIN32)
+    if(USE_TENSORRT)
+      set(DEPS ${DEPS}
+               ${TENSORRT_LIB_DIR}/libnvinfer${CMAKE_SHARED_LIBRARY_SUFFIX})
+      set(DEPS
+          ${DEPS}
+          ${TENSORRT_LIB_DIR}/libnvinfer_plugin${CMAKE_SHARED_LIBRARY_SUFFIX})
+    endif()
+    set(DEPS ${DEPS} ${CUDA_LIB}/libcudart${CMAKE_SHARED_LIBRARY_SUFFIX})
+  else()
+    if(USE_TENSORRT)
+      set(DEPS ${DEPS}
+               ${TENSORRT_LIB_DIR}/nvinfer${CMAKE_STATIC_LIBRARY_SUFFIX})
+      set(DEPS ${DEPS}
+               ${TENSORRT_LIB_DIR}/nvinfer_plugin${CMAKE_STATIC_LIBRARY_SUFFIX})
+    endif()
+    set(DEPS ${DEPS} ${CUDA_LIB}/cudart${CMAKE_STATIC_LIBRARY_SUFFIX})
+    set(DEPS ${DEPS} ${CUDA_LIB}/cublas${CMAKE_STATIC_LIBRARY_SUFFIX})
+    set(DEPS ${DEPS} ${CUDA_LIB}/cudnn${CMAKE_STATIC_LIBRARY_SUFFIX})
+  endif()
+endif()
+
+if(CUSTOM_OPERATOR_FILES)
+  if(WITH_GPU AND NOT APPLE)
+    add_definitions("-DPADDLE_WITH_CUDA")
+    enable_language(CUDA)
+    find_package(CUDA REQUIRED)
+    include_directories("${CUDA_INCLUDE_DIRS}")
+  endif()
+  add_library(pd_infer_custom_op SHARED ${CUSTOM_OPERATOR_FILES})
+  set(DEPS ${DEPS} pd_infer_custom_op)
+endif()
+
+if(CUSTOM_PASS_FILES)
+  add_library(pd_infer_custom_pass SHARED ${CUSTOM_PASS_FILES})
+  set(DEPS ${DEPS} pd_infer_custom_pass)
+endif()
+
+add_executable(${DEMO_NAME} ${DEMO_NAME}.cc)
+target_link_libraries(${DEMO_NAME} ${DEPS})
+if(WIN32)
+  if("${CMAKE_GENERATOR}" MATCHES "Ninja")
+    set(LIB_PATH ${CMAKE_BINARY_DIR})
+  else()
+    set(LIB_PATH ${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE})
+  endif()
+
+  if(USE_TENSORRT)
+    add_custom_command(
+      TARGET ${DEMO_NAME}
+      POST_BUILD
+      COMMAND
+        ${CMAKE_COMMAND} -E copy
+        ${TENSORRT_LIB_DIR}/nvinfer${CMAKE_SHARED_LIBRARY_SUFFIX} ${LIB_PATH}
+      COMMAND
+        ${CMAKE_COMMAND} -E copy
+        ${TENSORRT_LIB_DIR}/nvinfer_plugin${CMAKE_SHARED_LIBRARY_SUFFIX}
+        ${LIB_PATH})
+  endif()
+  if(WITH_SHARED_PHI)
+    add_custom_command(
+      TARGET ${DEMO_NAME}
+      POST_BUILD
+      COMMAND ${CMAKE_COMMAND} -E copy ${PADDLE_LIB}/paddle/lib/common.dll
+              ${LIB_PATH})
+    add_custom_command(
+      TARGET ${DEMO_NAME}
+      POST_BUILD
+      COMMAND ${CMAKE_COMMAND} -E copy ${PADDLE_LIB}/paddle/lib/phi.dll
+              ${LIB_PATH})
+  endif()
+  if(WITH_MKL)
+    add_custom_command(
+      TARGET ${DEMO_NAME}
+      POST_BUILD
+      COMMAND ${CMAKE_COMMAND} -E copy ${MATH_LIB_PATH}/lib/mklml.dll
+              ${LIB_PATH}
+      COMMAND ${CMAKE_COMMAND} -E copy ${MATH_LIB_PATH}/lib/libiomp5md.dll
+              ${LIB_PATH}
+      COMMAND ${CMAKE_COMMAND} -E copy ${ONEDNN_PATH}/lib/mkldnn.dll
+              ${LIB_PATH})
+  else()
+    add_custom_command(
+      TARGET ${DEMO_NAME}
+      POST_BUILD
+      COMMAND ${CMAKE_COMMAND} -E copy ${OPENBLAS_LIB_PATH}/lib/openblas.dll
+              ${LIB_PATH})
+  endif()
+  if(WITH_ONNXRUNTIME)
+    add_custom_command(
+      TARGET ${DEMO_NAME}
+      POST_BUILD
+      COMMAND
+        ${CMAKE_COMMAND} -E copy
+        ${PADDLE_LIB_THIRD_PARTY_PATH}onnxruntime/lib/onnxruntime.dll
+        ${LIB_PATH}
+      COMMAND
+        ${CMAKE_COMMAND} -E copy
+        ${PADDLE_LIB_THIRD_PARTY_PATH}paddle2onnx/lib/paddle2onnx.dll
+        ${LIB_PATH})
+  endif()
+  if(NOT WITH_STATIC_LIB)
+    add_custom_command(
+      TARGET ${DEMO_NAME}
+      POST_BUILD
+      COMMAND ${CMAKE_COMMAND} -E copy
+              "${PADDLE_LIB}/paddle/lib/paddle_inference.dll" ${LIB_PATH})
+  endif()
+endif()

paddle/fluid/inference/api/demo_ci/README.md

@@ -0,0 +1,26 @@
+# Inference Demos
+
+There are several demos:
+
+- simple_on_word2vec:
+  - Follow the C++ codes is in `simple_on_word2vec.cc`.
+  - It is suitable for word2vec model.
+- vis_demo:
+  - Follow the C++ codes is in `vis_demo.cc`.
+  - It is suitable for mobilenet, se_resnext50 and ocr three models.
+  - Input data format:
+    - Each line contains a single record
+    - Each record's format is
+    ```
+    <space split floats as data>\t<space split ints as shape>
+    ```
+
+To build and execute the demos, simply run
+```
+./run.sh $PADDLE_ROOT $TURN_ON_MKL $TEST_GPU_CPU
+```
+- It will build and execute the demos in both static and shared library.
+- `$PADDLE_ROOT`: paddle library path
+- `$TURN_ON_MKL`: use MKL or Openblas
+- `$TEST_GPU_CPU`: test both GPU/CPU mode or only CPU mode
+- NOTE: for simple_on_word2vec, must run `ctest -R test_word2vec -R` to obtain word2vec model at first.

paddle/fluid/inference/api/demo_ci/clean.sh

@@ -0,0 +1,18 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+cd `dirname $0`
+rm -rf build/ data/
+set +x

paddle/fluid/inference/api/demo_ci/custom_op_demo.cc

@@ -0,0 +1,65 @@
+/* Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gflags/gflags.h>
+#include <glog/logging.h>
+#include <numeric>
+
+#include "paddle_inference_api.h"  //NOLINT
+
+DEFINE_string(modeldir, "", "Directory of the inference model.");
+
+using paddle_infer::Config;
+using paddle_infer::CreatePredictor;
+using paddle_infer::Predictor;
+
+void run(Predictor *predictor,
+         const std::vector<float> &input,
+         const std::vector<int> &input_shape,
+         std::vector<float> *out_data) {
+  auto input_names = predictor->GetInputNames();
+  auto input_t = predictor->GetInputHandle(input_names[0]);
+  input_t->Reshape(input_shape);
+  input_t->CopyFromCpu(input.data());
+
+  CHECK(predictor->Run());
+
+  auto output_names = predictor->GetOutputNames();
+  auto output_t = predictor->GetOutputHandle(output_names[0]);
+  std::vector<int> output_shape = output_t->shape();
+  int out_num = std::accumulate(
+      output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
+
+  out_data->resize(out_num);
+  output_t->CopyToCpu(out_data->data());
+}
+
+int main(int argc, char **argv) {
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  Config config;
+  config.EnableUseGpu(100, 0);
+  config.SetModel(FLAGS_modeldir + "/custom_relu.pdmodel",
+                  FLAGS_modeldir + "/custom_relu.pdiparams");
+  config.EnableNewExecutor(true);
+  config.EnableNewIR(true);
+  auto predictor = CreatePredictor(config);
+  std::vector<int> input_shape = {1, 1, 28, 28};
+  std::vector<float> input_data(1 * 1 * 28 * 28, 1);
+  std::vector<float> out_data;
+  run(predictor.get(), input_data, input_shape, &out_data);
+  for (auto e : out_data) {
+    LOG(INFO) << e << '\n';
+  }
+  return 0;
+}

paddle/fluid/inference/api/demo_ci/custom_pass_demo.cc

@@ -0,0 +1,108 @@
+/* Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <gflags/gflags.h>
+#include <glog/logging.h>
+#include <cmath>
+#include <memory>
+#include <numeric>
+
+#include "paddle/extension.h"
+#include "paddle_inference_api.h"  //NOLINT
+
+DEFINE_string(modeldir, "", "Directory of the inference model.");
+
+using paddle_infer::Config;
+using paddle_infer::CreatePredictor;
+using paddle_infer::Predictor;
+
+std::shared_ptr<Predictor> InitPredictor(bool use_custom_pass) {
+  Config config;
+  config.EnableUseGpu(100, 0);
+  config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                  FLAGS_modeldir + "/inference.pdiparams");
+  config.EnableNewExecutor(true);
+  config.EnableNewIR(true);
+  // config.SwitchIrDebug(true);
+  if (use_custom_pass) {
+    config.EnableCustomPasses({"relu_replace_pass"});
+  }
+
+  return CreatePredictor(config);
+}
+
+std::vector<float> GetOutputData(const std::shared_ptr<Predictor> &predictor) {
+  auto input_names = predictor->GetInputNames();
+  auto input_shapes = predictor->GetInputTensorShape();
+
+  for (const auto &input_name : input_names) {
+    // update input shape's batch size
+    input_shapes[input_name][0] = 1;
+  }
+
+  std::vector<paddle::Tensor> inputs, outputs;
+  for (const auto &input_name : input_names) {
+    auto input_tensor = paddle::full(input_shapes[input_name],
+                                     0.5,
+                                     paddle::DataType::FLOAT32,
+                                     paddle::GPUPlace{});
+    input_tensor.set_name(input_name);
+    inputs.emplace_back(std::move(input_tensor));
+  }
+  PADDLE_ENFORCE_EQ(
+      predictor->Run(inputs, &outputs),
+      true,
+      common::errors::ExecutionTimeout("Sorry, predictor run failed"));
+  PADDLE_ENFORCE_EQ(outputs[0].place(),
+                    paddle::GPUPlace{},
+                    common::errors::InvalidArgument(
+                        "Sorry, output tensor place is not GPUPlace"));
+  PADDLE_ENFORCE_EQ(outputs[0].dtype(),
+                    paddle::DataType::FLOAT32,
+                    common::errors::InvalidArgument(
+                        "Sorry, output tensor dtype is not FLOAT32"));
+  auto output = outputs[0].copy_to(paddle::CPUPlace{}, true);
+
+  std::vector<float> output_data;
+  for (int64_t i = 0; i < output.numel(); i++) {
+    output_data.push_back(output.data<float>()[i]);
+  }
+  return output_data;
+}
+
+bool AreEqual(const std::vector<float> &vec1,
+              const std::vector<float> &vec2,
+              float epsilon) {
+  if (vec1.size() != vec2.size()) {
+    return false;
+  }
+  for (size_t i = 0; i < vec1.size(); ++i) {
+    if (std::fabs(vec1[i] - vec2[i]) > epsilon) {
+      return false;
+    }
+  }
+  return true;
+}
+
+int main(int argc, char **argv) {
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  auto base_data = GetOutputData(InitPredictor(false));
+  auto custom_data = GetOutputData(InitPredictor(true));
+
+  PADDLE_ENFORCE_EQ(AreEqual(base_data, custom_data, 1e-3),
+                    true,
+                    common::errors::InvalidArgument(
+                        "Sorry, base_data and custom_data are not equal"));
+  return 0;
+}

paddle/fluid/inference/api/demo_ci/custom_relu_op.cc

@@ -0,0 +1,104 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <iostream>
+#include <vector>
+
+#include "paddle/extension.h"
+
+template <typename data_t>
+void relu_cpu_forward_kernel(const data_t* x_data,
+                             data_t* out_data,
+                             int64_t x_numel) {
+  for (int i = 0; i < x_numel; ++i) {
+    out_data[i] = std::max(static_cast<data_t>(0.), x_data[i]);
+  }
+}
+
+template <typename data_t>
+void relu_cpu_backward_kernel(const data_t* grad_out_data,
+                              const data_t* out_data,
+                              data_t* grad_x_data,
+                              int64_t out_numel) {
+  for (int i = 0; i < out_numel; ++i) {
+    grad_x_data[i] =
+        grad_out_data[i] * (out_data[i] > static_cast<data_t>(0) ? 1. : 0.);
+  }
+}
+
+std::vector<paddle::Tensor> relu_cpu_forward(const paddle::Tensor& x) {
+  auto out = paddle::empty_like(x);
+  PD_DISPATCH_FLOATING_TYPES(
+      x.type(), "relu_cpu_forward", ([&] {
+        relu_cpu_forward_kernel<data_t>(
+            x.data<data_t>(), out.data<data_t>(), x.size());
+      }));
+
+  return {out};
+}
+
+std::vector<paddle::Tensor> relu_cpu_backward(const paddle::Tensor& x,
+                                              const paddle::Tensor& out,
+                                              const paddle::Tensor& grad_out) {
+  auto grad_x = paddle::empty_like(x);
+
+  PD_DISPATCH_FLOATING_TYPES(out.type(), "relu_cpu_backward", ([&] {
+                               relu_cpu_backward_kernel<data_t>(
+                                   grad_out.data<data_t>(),
+                                   out.data<data_t>(),
+                                   grad_x.data<data_t>(),
+                                   out.size());
+                             }));
+
+  return {grad_x};
+}
+
+std::vector<paddle::Tensor> relu_cuda_forward(const paddle::Tensor& x);
+std::vector<paddle::Tensor> relu_cuda_backward(const paddle::Tensor& x,
+                                               const paddle::Tensor& out,
+                                               const paddle::Tensor& grad_out);
+
+std::vector<paddle::Tensor> ReluForward(const paddle::Tensor& x) {
+  // TODO(chenweihang): Check Input
+  if (x.is_cpu()) {
+    return relu_cpu_forward(x);
+  } else if (x.is_gpu()) {
+    return relu_cuda_forward(x);
+  } else {
+    throw std::runtime_error("Not implemented.");
+  }
+}
+
+std::vector<paddle::Tensor> ReluBackward(const paddle::Tensor& x,
+                                         const paddle::Tensor& out,
+                                         const paddle::Tensor& grad_out) {
+  // TODO(chenweihang): Check Input
+  if (x.is_cpu()) {
+    return relu_cpu_backward(x, out, grad_out);
+  } else if (x.is_gpu()) {
+    return relu_cuda_backward(x, out, grad_out);
+  } else {
+    throw std::runtime_error("Not implemented.");
+  }
+}
+
+PD_BUILD_OP(custom_relu)
+    .Inputs({"X"})
+    .Outputs({"Out"})
+    .SetKernelFn(PD_KERNEL(ReluForward));
+
+PD_BUILD_GRAD_OP(custom_relu)
+    .Inputs({"X", "Out", paddle::Grad("Out")})
+    .Outputs({paddle::Grad("X")})
+    .SetKernelFn(PD_KERNEL(ReluBackward));

paddle/fluid/inference/api/demo_ci/custom_relu_op.cu

@@ -0,0 +1,71 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/extension.h"
+
+template <typename data_t>
+__global__ void relu_cuda_forward_kernel(const data_t* x,
+                                         data_t* y,
+                                         const int num) {
+  int gid = blockIdx.x * blockDim.x + threadIdx.x;
+  for (int i = gid; i < num; i += blockDim.x * gridDim.x) {
+    y[i] = max(x[i], static_cast<data_t>(0.));
+  }
+}
+
+template <typename data_t>
+__global__ void relu_cuda_backward_kernel(const data_t* dy,
+                                          const data_t* y,
+                                          data_t* dx,
+                                          const int num) {
+  int gid = blockIdx.x * blockDim.x + threadIdx.x;
+  for (int i = gid; i < num; i += blockDim.x * gridDim.x) {
+    dx[i] = dy[i] * (y[i] > 0 ? 1. : 0.);
+  }
+}
+
+std::vector<paddle::Tensor> relu_cuda_forward(const paddle::Tensor& x) {
+  auto out = paddle::empty_like(x);
+
+  int numel = x.size();
+  int block = 512;
+  int grid = (numel + block - 1) / block;
+  PD_DISPATCH_FLOATING_TYPES(
+      x.type(), "relu_cuda_forward_kernel", ([&] {
+        relu_cuda_forward_kernel<data_t><<<grid, block, 0, x.stream()>>>(
+            x.data<data_t>(), out.data<data_t>(), numel);
+      }));
+
+  return {out};
+}
+
+std::vector<paddle::Tensor> relu_cuda_backward(const paddle::Tensor& x,
+                                               const paddle::Tensor& out,
+                                               const paddle::Tensor& grad_out) {
+  auto grad_x = paddle::empty_like(x);
+
+  int numel = out.size();
+  int block = 512;
+  int grid = (numel + block - 1) / block;
+  PD_DISPATCH_FLOATING_TYPES(out.type(), "relu_cuda_backward_kernel", ([&] {
+                               relu_cuda_backward_kernel<data_t>
+                                   <<<grid, block, 0, x.stream()>>>(
+                                       grad_out.data<data_t>(),
+                                       out.data<data_t>(),
+                                       grad_x.data<data_t>(),
+                                       numel);
+                             }));
+
+  return {grad_x};
+}

paddle/fluid/inference/api/demo_ci/custom_relu_pass.cc

@@ -0,0 +1,47 @@
+/* Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/extension.h"
+
+namespace {
+
+class ReluReplacePattern : public paddle::drr::DrrPatternBase {
+ public:
+  std::string name() const override { return "ReluReplacePattern"; }
+
+  void operator()(paddle::drr::DrrPatternContext *ctx) const override {
+    paddle::drr::SourcePattern pat = ctx->SourcePattern();
+    const auto &relu = pat.Op("pd_op.relu");
+    relu({&pat.Tensor("in")}, {&pat.Tensor("out")});
+
+    paddle::drr::ResultPattern res = pat.ResultPattern();
+    const auto &custom_relu = res.Op("custom_op.custom_relu");
+    custom_relu({&res.Tensor("in")}, {&res.Tensor("out")});
+  }
+};
+
+class ReluReplacePass : public pir::PatternRewritePass {
+ public:
+  ReluReplacePass() : pir::PatternRewritePass("relu_replace_pass", 2) {}
+
+  pir::RewritePatternSet InitializePatterns(pir::IrContext *context) override {
+    pir::RewritePatternSet ps(context);
+    ps.Add(paddle::drr::Create<ReluReplacePattern>(context));
+    return ps;
+  }
+};
+
+}  // namespace
+
+REGISTER_IR_PASS(relu_replace_pass, ReluReplacePass);

paddle/fluid/inference/api/demo_ci/onnxruntime_mobilenet_demo.cc

@@ -0,0 +1,103 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+/*
+ * This file contains demo of mobilenet for onnxruntime backend.
+ */
+#include <glog/logging.h>  // use glog instead of CHECK to avoid importing other paddle header files.
+
+#include <algorithm>
+#include <numeric>
+#include <vector>
+
+#include "gflags/gflags.h"
+#include "utils.h"  // NOLINT
+
+DEFINE_string(modeldir, "", "Directory of the inference model.");
+DEFINE_string(data, "", "path of data");
+
+namespace paddle {
+namespace demo {
+
+/*
+ * Use the onnxruntime engine to inference the demo.
+ */
+void Main() {
+  paddle::AnalysisConfig config;
+  config.EnableONNXRuntime();
+  config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
+                  FLAGS_modeldir + "/inference.pdiparams");
+  auto predictor = paddle_infer::CreatePredictor(config);
+
+  // Inference.
+  LOG(INFO) << "--- prepare input data ----";
+  std::vector<int> input_shape = {1, 3, 224, 224};
+  std::vector<float> input_data;
+  std::string line;
+  std::ifstream file(FLAGS_data);
+  std::getline(file, line);
+  file.close();
+  std::vector<std::string> data_strs;
+  split(line, ' ', &data_strs);
+  int input_num = 0;
+  for (auto& d : data_strs) {
+    input_num += 1;
+    input_data.push_back(std::stof(d));
+  }
+
+  std::vector<float> out_data;
+  out_data.resize(1000);
+  auto input_names = predictor->GetInputNames();
+  auto output_names = predictor->GetOutputNames();
+  auto input_tensor = predictor->GetInputHandle(input_names[0]);
+  input_tensor->Reshape(input_shape);
+  auto output_tensor = predictor->GetOutputHandle(output_names[0]);
+
+  input_tensor->CopyFromCpu(input_data.data());
+  predictor->Run();
+  output_tensor->CopyToCpu(out_data.data());
+
+  std::vector<int> out_index(out_data.size());
+  std::iota(out_index.begin(), out_index.end(), 0);
+  std::sort(
+      out_index.begin(), out_index.end(), [&out_data](int index1, int index2) {
+        return out_data[index1] > out_data[index2];
+      });
+  LOG(INFO) << "output.size " << out_data.size()
+            << "  max_index:" << out_index[0];
+  PADDLE_ENFORCE_EQ(out_data.size(),
+                    1000,
+                    common::errors::InvalidArgument(
+                        "Required out_data.size() should be equal to 1000. "));
+  int max_index = out_index[0];
+  PADDLE_ENFORCE_EQ(max_index,
+                    13,
+                    common::errors::InvalidArgument(
+                        "Required max_index should be equal to 13. "));
+  float max_score = out_data[max_index];
+  PADDLE_ENFORCE_LE(fabs(max_score - 0.99981),
+                    1e-4,
+                    common::errors::InvalidArgument(
+                        "Required fabs(max_score - 0.99981) should "
+                        "be less than or equal to 1e-4. "));
+}
+
+}  // namespace demo
+}  // namespace paddle
+
+int main(int argc, char** argv) {
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  paddle::demo::Main();
+  return 0;
+}

paddle/fluid/inference/api/demo_ci/run.sh

@@ -0,0 +1,272 @@
+#!/bin/bash
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+PADDLE_ROOT=$1
+TURN_ON_MKL=$2 # use MKL or Openblas
+TEST_GPU_CPU=$3 # test both GPU/CPU mode or only CPU mode
+DATA_DIR=$4 # dataset
+USE_TENSORRT=$5
+TENSORRT_ROOT_DIR=$6 # TensorRT root dir, default to /usr
+WITH_ONNXRUNTIME=$7
+MSVC_STATIC_CRT=$8
+CUDA_LIB=$9/lib/x64
+inference_install_dir=${PADDLE_ROOT}/build/paddle_inference_install_dir
+WIN_DETECT=$(echo `uname` | grep "Win") # detect current platform
+
+cd `dirname $0`
+current_dir=`pwd`
+
+if [ $2 == ON ]; then
+  # You can export yourself if move the install path
+  MKL_LIB=${inference_install_dir}/third_party/install/mklml/lib
+  export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${MKL_LIB}
+fi
+if [ $3 == ON ]; then
+  use_gpu_list='true false'
+else
+  use_gpu_list='false'
+fi
+
+mkdir -p $DATA_DIR
+cd $DATA_DIR
+
+if [ $7 == ON ]; then
+  ONNXRUNTIME_LIB=${inference_install_dir}/third_party/install/onnxruntime/lib
+  export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${ONNXRUNTIME_LIB}
+  PADDLE2ONNX_LIB=${inference_install_dir}/third_party/install/paddle2onnx/lib
+  export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${PADDLE2ONNX_LIB}
+  #download model
+  mkdir -p MobileNetV2
+  cd MobileNetV2
+  if [[ -e "MobileNetV2.inference.model.tar.gz" ]]; then
+    rm -rf MobileNetV2.inference.model.tar.gz
+  fi
+    # echo "MobileNetV2.inference.model.tar.gz has been downloaded."
+  # else
+    if [ $WIN_DETECT != "" ]; then
+      wget -q -Y off http://paddle-inference-dist.bj.bcebos.com/MobileNetV2.inference.model.tar.gz
+    else
+      wget -q --no-proxy http://paddle-inference-dist.bj.bcebos.com/MobileNetV2.inference.model.tar.gz
+    fi
+    tar xzf *.tar.gz
+  # fi
+  cd ..
+fi
+
+PREFIX=inference-vis-demos%2F
+URL_ROOT=http://paddlemodels.bj.bcebos.com/${PREFIX}
+
+# download vis_demo data
+function download() {
+  dir_name=$1
+  mkdir -p $dir_name
+  cd $dir_name
+  if [[ -e "${PREFIX}${dir_name}.tar.gz" ]]; then
+    echo "${PREFIX}${dir_name}.tar.gz has been downloaded."
+  else
+      if [ $WIN_DETECT != "" ]; then
+        wget -q -Y off ${URL_ROOT}$dir_name.tar.gz
+      else
+        wget -q --no-proxy ${URL_ROOT}$dir_name.tar.gz
+      fi
+      tar xzf *.tar.gz
+  fi
+  cd ..
+}
+
+vis_demo_list='se_resnext50 ocr mobilenet'
+for vis_demo_name in $vis_demo_list; do
+  download $vis_demo_name
+done
+
+# download word2vec data
+mkdir -p word2vec
+cd word2vec
+if [[ -e "word2vec.inference.model.tar.gz" ]]; then
+  echo "word2vec.inference.model.tar.gz has been downloaded."
+else
+    wget -q http://paddle-inference-dist.bj.bcebos.com/word2vec.inference.model.tar.gz
+    tar xzf *.tar.gz
+fi
+cd ..
+
+#download custom_op_demo data
+mkdir -p custom_op
+cd custom_op
+if [[ -e "custom_relu_infer_model.tgz" ]]; then
+  echo "custom_relu_infer_model.tgz has been downloaded."
+else
+    wget -q https://paddle-inference-dist.bj.bcebos.com/inference_demo/custom_operator/custom_relu_infer_model.tgz
+    tar xzf *.tgz
+fi
+cd ..
+
+#download custom_pass_demo data
+mkdir -p custom_pass
+cd custom_pass
+if [ ! -d resnet50 ]; then
+    wget https://paddle-inference-dist.bj.bcebos.com/Paddle-Inference-Demo/resnet50.tgz
+    tar xzf resnet50.tgz
+fi
+
+# compile and test the demo
+cd $current_dir
+mkdir -p build
+cd build
+rm -rf *
+
+# run all test cases before exit
+EXIT_CODE=0
+
+for WITH_STATIC_LIB in ON OFF; do
+  if [ $(echo `uname` | grep "Win") != "" ]; then
+    # TODO(wilber, T8T9): Do we still need to support windows gpu static library
+    if [ $TEST_GPU_CPU == ON ] && [ $WITH_STATIC_LIB == ON ]; then
+      continue
+    fi
+    # -----simple_on_word2vec on windows-----
+    cmake .. -GNinja -DPADDLE_LIB=${inference_install_dir} \
+      -DWITH_MKL=$TURN_ON_MKL \
+      -DDEMO_NAME=simple_on_word2vec \
+      -DWITH_GPU=$TEST_GPU_CPU \
+      -DWITH_STATIC_LIB=$WITH_STATIC_LIB \
+      -DMSVC_STATIC_CRT=$MSVC_STATIC_CRT \
+      -DWITH_ONNXRUNTIME=$WITH_ONNXRUNTIME \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DCUDA_LIB="$CUDA_LIB"
+    ninja
+    for use_gpu in $use_gpu_list; do
+      ./simple_on_word2vec.exe \
+        --dirname=$DATA_DIR/word2vec/word2vec.inference.model \
+        --use_gpu=$use_gpu
+      if [ $? -ne 0 ]; then
+        echo "simple_on_word2vec use_gpu:${use_gpu} runs failed " > ${current_dir}/test_summary.txt
+        EXIT_CODE=1
+      fi
+    done
+
+    # --------tensorrt mobilenet on windows------
+    if [ $USE_TENSORRT == ON -a $TEST_GPU_CPU == ON ]; then
+      rm -rf *
+    fi
+  else
+    # -----simple_on_word2vec on linux/mac-----
+    rm -rf *
+    cmake .. -DPADDLE_LIB=${inference_install_dir} \
+      -DWITH_MKL=$TURN_ON_MKL \
+      -DDEMO_NAME=simple_on_word2vec \
+      -DWITH_GPU=$TEST_GPU_CPU \
+      -DWITH_STATIC_LIB=$WITH_STATIC_LIB \
+      -DWITH_ONNXRUNTIME=$WITH_ONNXRUNTIME
+    make -j$(nproc)
+    word2vec_model=$DATA_DIR'/word2vec/word2vec.inference.model'
+    if [ -d $word2vec_model ]; then
+      for use_gpu in $use_gpu_list; do
+        ./simple_on_word2vec \
+          --dirname=$DATA_DIR/word2vec/word2vec.inference.model \
+          --use_gpu=$use_gpu
+        if [ $? -ne 0 ]; then
+          echo "simple_on_word2vec use_gpu:${use_gpu} runs failed " >> ${current_dir}/test_summary.txt
+          EXIT_CODE=1
+        fi
+      done
+    fi
+
+    # --------onnxruntime mobilenetv2 on linux/mac------
+    if [ $WITH_ONNXRUNTIME == ON ]; then
+      rm -rf *
+      cmake .. -DPADDLE_LIB=${inference_install_dir} \
+        -DWITH_MKL=$TURN_ON_MKL \
+        -DDEMO_NAME=onnxruntime_mobilenet_demo \
+        -DWITH_GPU=$TEST_GPU_CPU \
+        -DWITH_STATIC_LIB=$WITH_STATIC_LIB \
+        -DUSE_TENSORRT=$USE_TENSORRT \
+        -DTENSORRT_ROOT=$TENSORRT_ROOT_DIR \
+        -DWITH_ONNXRUNTIME=$WITH_ONNXRUNTIME
+      make -j$(nproc)
+      ./onnxruntime_mobilenet_demo \
+        --modeldir=$DATA_DIR/MobileNetV2/MobileNetV2 \
+        --data=$DATA_DIR/MobileNetV2/MobileNetV2/data.txt
+      if [ $? -ne 0 ]; then
+        echo "onnxruntime_mobilenet_demo runs failed " >> ${current_dir}/test_summary.txt
+        EXIT_CODE=1
+      fi
+    fi
+
+    # --------custom op demo on linux/mac------
+    if [ $TEST_GPU_CPU == ON -a $WITH_STATIC_LIB == OFF ]; then
+      rm -rf *
+      CUSTOM_OPERATOR_FILES="custom_relu_op.cc;custom_relu_op.cu"
+      cmake .. -DPADDLE_LIB=${inference_install_dir} \
+        -DWITH_MKL=$TURN_ON_MKL \
+        -DDEMO_NAME=custom_op_demo \
+        -DWITH_GPU=$TEST_GPU_CPU \
+        -DWITH_STATIC_LIB=OFF \
+        -DUSE_TENSORRT=$USE_TENSORRT \
+        -DTENSORRT_ROOT=$TENSORRT_ROOT_DIR \
+        -DCUSTOM_OPERATOR_FILES=$CUSTOM_OPERATOR_FILES \
+        -DWITH_ONNXRUNTIME=$WITH_ONNXRUNTIME
+      make -j$(nproc)
+      ./custom_op_demo \
+        --modeldir=$DATA_DIR/custom_op/custom_relu_infer_model
+      if [ $? -ne 0 ]; then
+        echo "custom_op_demo runs failed " >> ${current_dir}/test_summary.txt
+        EXIT_CODE=1
+      fi
+    fi
+
+    # --------custom pass demo on linux/mac------
+    if [ $TEST_GPU_CPU == ON -a $WITH_STATIC_LIB == OFF ]; then
+      rm -rf *
+      CUSTOM_OPERATOR_FILES="custom_relu_op.cc;custom_relu_op.cu"
+      CUSTOM_PASS_FILES="custom_relu_pass.cc"
+      cmake .. -DPADDLE_LIB=${inference_install_dir} \
+        -DWITH_MKL=$TURN_ON_MKL \
+        -DDEMO_NAME=custom_pass_demo \
+        -DWITH_GPU=$TEST_GPU_CPU \
+        -DWITH_STATIC_LIB=OFF \
+        -DUSE_TENSORRT=$USE_TENSORRT \
+        -DTENSORRT_ROOT=$TENSORRT_ROOT_DIR \
+        -DCUSTOM_OPERATOR_FILES=$CUSTOM_OPERATOR_FILES \
+        -DCUSTOM_PASS_FILES=${CUSTOM_PASS_FILES} \
+        -DWITH_ONNXRUNTIME=$WITH_ONNXRUNTIME
+      make -j$(nproc)
+      ./custom_pass_demo \
+        --modeldir=$DATA_DIR/custom_pass/resnet50
+      if [ $? -ne 0 ]; then
+        echo "custom_pass_demo runs failed " >> ${current_dir}/test_summary.txt
+        EXIT_CODE=1
+      fi
+    fi
+  fi
+done
+
+set +x
+
+if [[ -f ${current_dir}/test_summary.txt ]];then
+  echo " "
+  echo "Summary demo_ci Failed Tests ..."
+  echo "=====================test summary======================"
+  echo "The following tests Failed: "
+  cat ${current_dir}/test_summary.txt
+  echo "========================================================"
+  echo " "
+fi
+
+set -x
+
+exit ${EXIT_CODE}

paddle/fluid/inference/api/demo_ci/run_windows_demo.bat

@@ -0,0 +1,248 @@
+@echo off
+setlocal
+set source_path=%~dp0
+set build_path=%~dp0\build
+
+setlocal enabledelayedexpansion
+
+rem set gpu_inference
+SET /P gpu_inference="Use GPU_inference_lib or not(Y/N), default: N   =======>"
+IF /i "%gpu_inference%"=="y" (
+  SET gpu_inference=Y
+) else (
+  SET gpu_inference=N
+)
+
+SET /P use_mkl="Use MKL or not (Y/N), default: Y   =======>"
+if /i "%use_mkl%"=="N" (
+  set use_mkl=N
+) else (
+  set use_mkl=Y
+)
+
+:set_paddle_inference_lib
+SET /P paddle_inference_lib="Please input the path of paddle inference library, such as D:\paddle_inference_install_dir   =======>"
+set tmp_var=!paddle_inference_lib!
+call:remove_space
+set paddle_inference_lib=!tmp_var!
+IF NOT EXIST "%paddle_inference_lib%" (
+echo "------------%paddle_inference_lib% not exist------------"
+goto set_paddle_inference_lib
+)
+
+IF "%use_mkl%"=="N" (
+  IF NOT EXIST "%paddle_inference_lib%\third_party\install\openblas" (
+    echo "------------It's not a OpenBlas inference library------------"
+    goto:eof
+  )
+) else (
+  IF NOT EXIST "%paddle_inference_lib%\third_party\install\mklml" (
+    echo "------------It's not a MKL inference library------------"
+    goto:eof
+  )
+)
+
+:set_path_cuda
+if /i "!gpu_inference!"=="Y" (
+    SET /P cuda_lib_dir="Please input the path of cuda libraries, such as D:\cuda\lib\x64   =======>"
+    set tmp_var=!cuda_lib_dir!
+    call:remove_space
+    set cuda_lib_dir=!tmp_var!
+    IF NOT EXIST "!cuda_lib_dir!" (
+        echo "------------!cuda_lib_dir!not exist------------"
+        goto set_path_cuda
+    )
+)
+
+rem set_use_gpu
+if /i "!gpu_inference!"=="Y" (
+    SET /P use_gpu="Use GPU or not(Y/N), default: N   =======>"
+)
+
+if /i "%use_gpu%"=="Y" (
+  set use_gpu=Y
+) else (
+  set use_gpu=N
+)
+
+rem set_path_vs_command_prompt
+:set_vcvarsall_dir
+SET /P vcvarsall_dir="Please input the path of visual studio command Prompt, such as C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat   =======>"
+set tmp_var=!vcvarsall_dir!
+call:remove_space
+set vcvarsall_dir=!tmp_var!
+IF NOT EXIST "%vcvarsall_dir%" (
+    echo "------------%vcvarsall_dir% not exist------------"
+    goto set_vcvarsall_dir
+)
+
+rem set_demo_name
+:set_demo_name
+SET /P demo_name="Please input the demo name, default: windows_mobilenet  =======>"
+if   "%demo_name%"==""  set demo_name=windows_mobilenet
+IF NOT EXIST "%source_path%\%demo_name%.cc" (
+    echo "------------%source_path%\%demo_name%.cc not exist------------"
+    goto set_demo_name
+)
+if "%demo_name%"=="windows_mobilenet" set model_name=mobilenet
+if "%demo_name%"=="vis_demo" set model_name=mobilenet
+if "%demo_name%"=="simple_on_word2vec" set model_name=word2vec.inference.model
+if "%demo_name%"=="trt_mobilenet_demo" set model_name=mobilenet
+
+rem download model
+if NOT EXIST "%source_path%\%model_name%.tar.gz" (
+  if "%model_name%"=="mobilenet" (
+     call:download_model_mobilenet
+  )
+  if "%model_name%"=="word2vec.inference.model" (
+     call:download_model_word2vec
+  )
+)
+
+if EXIST "%source_path%\%model_name%.tar.gz" (
+  if NOT EXIST "%source_path%\%model_name%" (
+    SET /P python_path="Please input the path of python.exe, such as C:\Python37\python.exe =======>"
+    set tmp_var=!python_path!
+    call:remove_space
+    set python_path=!tmp_var!
+    if "!python_path!"=="" (
+      set python_path=python.exe
+    ) else (
+      if NOT exist "!python_path!" (
+        echo "------------!python_path! not exist------------"
+        goto:eof
+      )
+    )
+    md %source_path%\%model_name%
+    !python_path! %source_path%\untar_model.py %source_path%\%model_name%.tar.gz %source_path%\%model_name%
+
+    SET error_code=N
+    if "%model_name%"=="mobilenet" (
+      if NOT EXIST "%source_path%\%model_name%\model" set error_code=Y
+    ) else (
+      if NOT EXIST "%source_path%\%model_name%\%model_name%" set error_code=Y
+    )
+    if  "!error_code!"=="Y"  (
+       echo "========= Unzip %model_name%.tar.gz failed ======="
+       del /f /s /q "%source_path%\%model_name%\*.*" >nul 2>&1
+       rd /s /q  "%source_path%\%model_name%" >nul 2>&1
+       goto:eof
+    )
+  )
+)
+
+echo "=================================================================="
+echo.
+echo "use_gpu_inference=%gpu_inference%"
+echo.
+echo "use_mkl=%use_mkl%"
+echo.
+echo "use_gpu=%use_gpu%"
+echo.
+echo "paddle_inference_lib=%paddle_inference_lib%"
+echo.
+IF /i "%gpu_inference%"=="y" (
+  echo "cuda_lib_dir=%cuda_lib_dir%"
+  echo.
+)
+echo "vs_vcvarsall_dir=%vcvarsall_dir%"
+echo.
+echo "demo_name=%demo_name%"
+echo.
+if NOT "!python_path!"=="" (
+  echo "python_path=!python_path!"
+  echo.
+)
+echo "===================================================================="
+pause
+
+
+rem compile and run demo
+
+if NOT EXIST "%build_path%" (
+    md %build_path%
+    cd %build_path%
+) else (
+    del /f /s /q "%build_path%\*.*" >nul 2>&1
+    rd /s /q  "%build_path%" >nul 2>&1
+    md %build_path%
+    cd %build_path%
+)
+
+if /i "%use_mkl%"=="N" (
+  set use_mkl=OFF
+) else (
+  set use_mkl=ON
+)
+
+if /i "%gpu_inference%"=="Y" (
+    if  "%demo_name%"=="trt_mobilenet_demo" (
+      cmake .. -G "Visual Studio 15 2017 Win64"  -T host=x64 -DWITH_GPU=ON ^
+      -DWITH_MKL=%use_mkl% -DWITH_STATIC_LIB=ON -DCMAKE_BUILD_TYPE=Release -DDEMO_NAME=%demo_name% ^
+      -DPADDLE_LIB="%paddle_inference_lib%" -DMSVC_STATIC_CRT=ON -DCUDA_LIB="%cuda_lib_dir%" -DUSE_TENSORRT=ON
+    ) else (
+      cmake .. -G "Visual Studio 15 2017 Win64"  -T host=x64 -DWITH_GPU=ON ^
+      -DWITH_MKL=%use_mkl% -DWITH_STATIC_LIB=ON -DCMAKE_BUILD_TYPE=Release -DDEMO_NAME=%demo_name% ^
+      -DPADDLE_LIB="%paddle_inference_lib%" -DMSVC_STATIC_CRT=ON -DCUDA_LIB="%cuda_lib_dir%"
+    )
+) else (
+    cmake .. -G "Visual Studio 15 2017 Win64"  -T host=x64 -DWITH_GPU=OFF ^
+    -DWITH_MKL=%use_mkl% -DWITH_STATIC_LIB=ON -DCMAKE_BUILD_TYPE=Release -DDEMO_NAME=%demo_name% ^
+    -DPADDLE_LIB="%paddle_inference_lib%" -DMSVC_STATIC_CRT=ON
+)
+
+call "%vcvarsall_dir%" amd64
+msbuild /m /p:Configuration=Release %demo_name%.vcxproj
+
+if /i "%use_gpu%"=="Y" (
+  SET use_gpu=true
+) else (
+  SET use_gpu=false
+)
+
+if exist "%build_path%\Release\%demo_name%.exe" (
+  cd %build_path%\Release
+  set GLOG_v=4
+  if "%demo_name%"=="simple_on_word2vec" (
+      %demo_name%.exe --dirname="%source_path%\%model_name%\%model_name%" --use_gpu="%use_gpu%"
+  ) else (
+    if "%demo_name%"=="windows_mobilenet" (
+        %demo_name%.exe --modeldir="%source_path%\%model_name%\model" --use_gpu="%use_gpu%"
+    ) else (
+      if "%demo_name%"=="trt_mobilenet_demo" (
+        %demo_name%.exe --modeldir="%source_path%\%model_name%\model" --data=%source_path%\%model_name%\data.txt ^
+        --refer=%source_path%\%model_name%\result.txt
+      ) else (
+        %demo_name%.exe --modeldir="%source_path%\%model_name%\model" --data=%source_path%\%model_name%\data.txt ^
+        --refer=%source_path%\%model_name%\result.txt --use_gpu="%use_gpu%"
+      )
+    )
+  )
+) else (
+  echo "=========compilation fails!!=========="
+)
+echo.&pause&goto:eof
+
+:download_model_mobilenet
+powershell.exe (new-object System.Net.WebClient).DownloadFile('http://paddlemodels.bj.bcebos.com//inference-vis-demos/mobilenet.tar.gz', ^
+'%source_path%\mobilenet.tar.gz')
+goto:eof
+
+:download_model_word2vec
+powershell.exe (new-object System.Net.WebClient).DownloadFile('http://paddle-inference-dist.bj.bcebos.com/word2vec.inference.model.tar.gz', ^
+'%source_path%\word2vec.inference.model.tar.gz')
+goto:eof
+
+:remove_space
+:remove_left_space
+if "%tmp_var:~0,1%"==" " (
+    set "tmp_var=%tmp_var:~1%"
+    goto remove_left_space
+)
+
+:remove_right_space
+if "%tmp_var:~-1%"==" " (
+    set "tmp_var=%tmp_var:~0,-1%"
+    goto remove_left_space
+)
+goto:eof

paddle/fluid/inference/api/demo_ci/simple_on_word2vec.cc

@@ -0,0 +1,146 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+/*
+ * This file contains a simple demo for how to take a model for inference.
+ */
+
+#include <glog/logging.h>
+
+#include <algorithm>
+#include <memory>
+#include <thread>  //NOLINT
+
+#include "gflags/gflags.h"
+#include "utils.h"  // NOLINT
+
+DEFINE_string(dirname, "", "Directory of the inference model.");
+DEFINE_bool(use_gpu, false, "Whether use gpu.");
+
+namespace paddle {
+namespace demo {
+
+void Main(bool use_gpu) {
+  //# 1. Create PaddlePredictor with a config.
+  NativeConfig config;
+  if (FLAGS_dirname.empty()) {
+    LOG(INFO) << "Usage: ./simple_on_word2vec --dirname=path/to/your/model";
+    exit(1);
+  }
+  config.model_dir = FLAGS_dirname;
+  config.use_gpu = use_gpu;
+  config.fraction_of_gpu_memory = 0.15;
+  config.device = 0;
+  auto predictor = CreatePaddlePredictor<NativeConfig>(config);
+
+  for (int batch_id = 0; batch_id < 3; batch_id++) {
+    //# 2. Prepare input.
+    int64_t data[4] = {1, 2, 3, 4};
+
+    PaddleTensor tensor;
+    tensor.shape = std::vector<int>({4, 1});
+    tensor.data = PaddleBuf(data, sizeof(data));
+    tensor.dtype = PaddleDType::INT64;
+
+    // For simplicity, we set all the slots with the same data.
+    std::vector<PaddleTensor> slots(4, tensor);
+
+    //# 3. Run
+    std::vector<PaddleTensor> outputs;
+    CHECK(predictor->Run(slots, &outputs));
+
+    //# 4. Get output.
+    CHECK_EQ(outputs.size(), 1UL);
+    // Check the output buffer size and result of each tid.
+    CHECK_EQ(outputs.front().data.length(), 33168UL);
+    float result[5] = {
+        0.00129761, 0.00151112, 0.000423564, 0.00108815, 0.000932706};
+    const size_t num_elements = outputs.front().data.length() / sizeof(float);
+    // The outputs' buffers are in CPU memory.
+    for (size_t i = 0; i < std::min(static_cast<size_t>(5), num_elements);
+         i++) {
+      CHECK_NEAR(static_cast<float*>(outputs.front().data.data())[i],
+                 result[i],
+                 0.001);
+    }
+  }
+}
+
+void MainThreads(int num_threads, bool use_gpu) {
+  // Multi-threads only support on CPU
+  // 0. Create PaddlePredictor with a config.
+  NativeConfig config;
+  config.model_dir = FLAGS_dirname;
+  config.use_gpu = use_gpu;
+  config.fraction_of_gpu_memory = 0.15;
+  config.device = 0;
+  auto main_predictor = CreatePaddlePredictor<NativeConfig>(config);
+
+  std::vector<std::thread> threads;
+  for (int tid = 0; tid < num_threads; ++tid) {
+    threads.emplace_back([&, tid]() {
+      // 1. clone a predictor which shares the same parameters
+      auto predictor = main_predictor->Clone();
+      constexpr int num_batches = 3;
+      for (int batch_id = 0; batch_id < num_batches; ++batch_id) {
+        // 2. Dummy Input Data
+        int64_t data[4] = {1, 2, 3, 4};
+        PaddleTensor tensor;
+        tensor.shape = std::vector<int>({4, 1});
+        tensor.data = PaddleBuf(data, sizeof(data));
+        tensor.dtype = PaddleDType::INT64;
+
+        std::vector<PaddleTensor> inputs(4, tensor);
+        std::vector<PaddleTensor> outputs;
+        // 3. Run
+        CHECK(predictor->Run(inputs, &outputs));
+
+        // 4. Get output.
+        CHECK_EQ(outputs.size(), 1UL);
+        // Check the output buffer size and result of each tid.
+        CHECK_EQ(outputs.front().data.length(), 33168UL);
+        float result[5] = {
+            0.00129761, 0.00151112, 0.000423564, 0.00108815, 0.000932706};
+        const size_t num_elements =
+            outputs.front().data.length() / sizeof(float);
+        // The outputs' buffers are in CPU memory.
+        for (size_t i = 0; i < std::min(static_cast<size_t>(5), num_elements);
+             i++) {
+          CHECK_NEAR(static_cast<float*>(outputs.front().data.data())[i],
+                     result[i],
+                     0.001);
+        }
+      }
+    });
+  }
+  for (int i = 0; i < num_threads; ++i) {
+    threads[i].join();
+  }
+}
+
+}  // namespace demo
+}  // namespace paddle
+
+int main(int argc, char** argv) {
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  paddle::demo::Main(false /* use_gpu*/);
+  paddle::demo::MainThreads(1, false /* use_gpu*/);
+  paddle::demo::MainThreads(4, false /* use_gpu*/);
+  if (FLAGS_use_gpu) {
+    paddle::demo::Main(true /*use_gpu*/);
+    paddle::demo::MainThreads(1, true /*use_gpu*/);
+    paddle::demo::MainThreads(4, true /*use_gpu*/);
+  }
+  return 0;
+}

paddle/fluid/inference/api/demo_ci/trt_mobilenet_demo.cc

@@ -0,0 +1,79 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+/*
+ * This file contains demo of mobilenet for tensorrt.
+ */
+
+#include <glog/logging.h>  // use glog instead of CHECK to avoid importing other paddle header files.
+#include "gflags/gflags.h"
+#include "utils.h"  // NOLINT
+
+DEFINE_string(modeldir, "", "Directory of the inference model.");
+DEFINE_string(refer, "", "path to reference result for comparison.");
+DEFINE_string(data,
+              "",
+              "path of data; each line is a record, format is "
+              "'<space split floats as data>\t<space split ints as shape'");
+
+namespace paddle {
+namespace demo {
+
+/*
+ * Use the tensorrt fluid engine to inference the demo.
+ */
+void Main() {
+  std::unique_ptr<PaddlePredictor> predictor;
+  paddle::AnalysisConfig config;
+  config.EnableUseGpu(100, 0);
+  config.SetModel(FLAGS_modeldir + "/__model__",
+                  FLAGS_modeldir + "/__params__");
+  config.EnableTensorRtEngine();
+  predictor = CreatePaddlePredictor(config);
+
+  VLOG(3) << "begin to process data";
+  // Just a single batch of data.
+  std::string line;
+  std::ifstream file(FLAGS_data);
+  std::getline(file, line);
+  auto record = ProcessALine(line);
+  file.close();
+
+  // Inference.
+  PaddleTensor input;
+  input.shape = record.shape;
+  input.data =
+      PaddleBuf(record.data.data(), record.data.size() * sizeof(float));
+  input.dtype = PaddleDType::FLOAT32;
+
+  VLOG(3) << "run executor";
+  std::vector<PaddleTensor> output;
+  predictor->Run({input}, &output, 1);
+
+  VLOG(3) << "output.size " << output.size();
+  auto& tensor = output.front();
+  VLOG(3) << "output: " << SummaryTensor(tensor);
+
+  // compare with reference result
+  CheckOutput(FLAGS_refer, tensor);
+}
+
+}  // namespace demo
+}  // namespace paddle
+
+int main(int argc, char** argv) {
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  paddle::demo::Main();
+  return 0;
+}

paddle/fluid/inference/api/demo_ci/untar_model.py

@@ -0,0 +1,35 @@
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import tarfile
+
+
+def untar(fname, dirs):
+    """
+    extract the tar.gz file
+    :param fname: the name of tar.gz file
+    :param dirs: the path of decompressed file
+    :return: bool
+    """
+    try:
+        t = tarfile.open(name=fname, mode='r:gz')
+        t.extractall(path=dirs)
+        return True
+    except Exception as e:
+        print(e)
+        return False
+
+
+untar(sys.argv[1], sys.argv[2])

paddle/fluid/inference/api/demo_ci/utils.h

@@ -0,0 +1,149 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include <math.h>
+
+#include <algorithm>
+#include <fstream>
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include "paddle/extension.h"      // For pr test!
+#include "paddle_inference_api.h"  // NOLINT
+
+namespace paddle {
+namespace demo {
+
+struct Record {
+  std::vector<float> data;
+  std::vector<int32_t> shape;
+};
+
+static void split(const std::string& str,
+                  char sep,
+                  std::vector<std::string>* pieces) {
+  pieces->clear();
+  if (str.empty()) {
+    return;
+  }
+  size_t pos = 0;
+  size_t next = str.find(sep, pos);
+  while (next != std::string::npos) {
+    pieces->push_back(str.substr(pos, next - pos));
+    pos = next + 1;
+    next = str.find(sep, pos);
+  }
+  if (!str.substr(pos).empty()) {
+    pieces->push_back(str.substr(pos));
+  }
+}
+
+Record ProcessALine(const std::string& line) {
+  VLOG(3) << "process a line";
+  std::vector<std::string> columns;
+  split(line, '\t', &columns);
+  CHECK_EQ(columns.size(), 2UL)
+      << "data format error, should be <data>\t<shape>";
+
+  Record record;
+  std::vector<std::string> data_strs;
+  split(columns[0], ' ', &data_strs);
+  for (auto& d : data_strs) {
+    record.data.push_back(std::stof(d));
+  }
+
+  std::vector<std::string> shape_strs;
+  split(columns[1], ' ', &shape_strs);
+  for (auto& s : shape_strs) {
+    record.shape.push_back(std::stoi(s));
+  }
+  VLOG(3) << "data size " << record.data.size();
+  VLOG(3) << "data shape size " << record.shape.size();
+  return record;
+}
+
+void CheckOutput(const std::string& referfile,
+                 const PaddleTensor& output,
+                 float threshold = 1e-5) {
+  std::string line;
+  std::ifstream file(referfile);
+  std::getline(file, line);
+  auto refer = ProcessALine(line);
+  file.close();
+
+  paddle::Tensor dummy;
+
+  size_t numel = output.data.length() / PaddleDtypeSize(output.dtype);
+  VLOG(3) << "predictor output numel " << numel;
+  VLOG(3) << "reference output numel " << refer.data.size();
+  CHECK_EQ(numel, refer.data.size());
+  switch (output.dtype) {
+    case PaddleDType::INT64: {
+      for (size_t i = 0; i < numel; ++i) {
+        CHECK_EQ(static_cast<int64_t*>(output.data.data())[i], refer.data[i]);
+      }
+      break;
+    }
+    case PaddleDType::FLOAT32: {
+      for (size_t i = 0; i < numel; ++i) {
+        CHECK_LT(
+            fabs(static_cast<float*>(output.data.data())[i] - refer.data[i]),
+            threshold);
+      }
+      break;
+    }
+    case PaddleDType::INT32: {
+      for (size_t i = 0; i < numel; ++i) {
+        CHECK_EQ(static_cast<int32_t*>(output.data.data())[i], refer.data[i]);
+      }
+      break;
+    }
+  }
+}
+
+/*
+ * Get a summary of a PaddleTensor content.
+ */
+static std::string SummaryTensor(const PaddleTensor& tensor) {
+  std::stringstream ss;
+  int num_elems = tensor.data.length() / PaddleDtypeSize(tensor.dtype);
+
+  ss << "data[:10]\t";
+  switch (tensor.dtype) {
+    case PaddleDType::INT64: {
+      for (int i = 0; i < std::min(num_elems, 10); i++) {
+        ss << static_cast<int64_t*>(tensor.data.data())[i] << " ";
+      }
+      break;
+    }
+    case PaddleDType::FLOAT32: {
+      for (int i = 0; i < std::min(num_elems, 10); i++) {
+        ss << static_cast<float*>(tensor.data.data())[i] << " ";
+      }
+      break;
+    }
+    case PaddleDType::INT32: {
+      for (int i = 0; i < std::min(num_elems, 10); i++) {
+        ss << static_cast<int32_t*>(tensor.data.data())[i] << " ";
+      }
+      break;
+    }
+  }
+  return ss.str();
+}
+
+}  // namespace demo
+}  // namespace paddle

paddle/fluid/inference/api/demo_ci/vis_demo.cc

@@ -0,0 +1,88 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+/*
+ * This file contains demo for mobilenet, se-resnext50 and ocr.
+ */
+
+#include <glog/logging.h>
+
+#include "gflags/gflags.h"
+#include "utils.h"  // NOLINT
+
+DEFINE_string(modeldir, "", "Directory of the inference model.");
+DEFINE_string(refer, "", "path to reference result for comparison.");
+DEFINE_string(data,
+              "",
+              "path of data; each line is a record, format is "
+              "'<space split floats as data>\t<space split ints as shape'");
+DEFINE_bool(use_gpu, false, "Whether use gpu.");
+
+namespace paddle {
+namespace demo {
+
+/*
+ * Use the native and analysis fluid engine to inference the demo.
+ */
+void Main(bool use_gpu) {
+  std::unique_ptr<PaddlePredictor> predictor, analysis_predictor;
+  AnalysisConfig config;
+  if (use_gpu) {
+    config.EnableUseGpu(100, 0);
+  }
+  config.SetModel(FLAGS_modeldir + "/__model__",
+                  FLAGS_modeldir + "/__params__");
+
+  predictor = CreatePaddlePredictor<NativeConfig>(config.ToNativeConfig());
+  analysis_predictor = CreatePaddlePredictor(config);
+
+  // Just a single batch of data.
+  std::string line;
+  std::ifstream file(FLAGS_data);
+  std::getline(file, line);
+  auto record = ProcessALine(line);
+  file.close();
+
+  // Inference.
+  PaddleTensor input;
+  input.shape = record.shape;
+  input.data =
+      PaddleBuf(record.data.data(), record.data.size() * sizeof(float));
+  input.dtype = PaddleDType::FLOAT32;
+
+  std::vector<PaddleTensor> output, analysis_output;
+  predictor->Run({input}, &output, 1);
+
+  auto& tensor = output.front();
+
+  // compare with reference result
+  CheckOutput(FLAGS_refer, tensor, 1e-4);
+
+  // the analysis_output has some diff with native_output,
+  // TODO(luotao): add CheckOutput for analysis_output later.
+  analysis_predictor->Run({input}, &analysis_output, 1);
+}
+
+}  // namespace demo
+}  // namespace paddle
+
+int main(int argc, char** argv) {
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  if (FLAGS_use_gpu) {
+    paddle::demo::Main(true /*use_gpu*/);
+  } else {
+    paddle::demo::Main(false /*use_gpu*/);
+  }
+  return 0;
+}

paddle/fluid/inference/api/demo_ci/windows_inference.md

@@ -0,0 +1,19 @@
+# windows inference
+本文介绍windows inference，目前只提供了静态编译，编译出paddle_inference.lib，包含了除openblas.dll之外的所有第三方依赖库。
+
+1. 下载最新的paddle_inference.lib和openblas.dll，并把它们放在同一个目录下。
+
+2. 准备预训练好的模型文件，例如models中的模型，可以将模型用safe_inference_model接口保存下来。将模型文件放到该目录下
+
+3. 进入Paddle/paddle/fluid/inference/api/demo_ci目录，新建build目录，然后使用cmake生成vs2015的solution文件。
+其中PADDLE_LIB是前面的paddle_inference.lib对应文件夹, CUDA_LIB指定为x64格式下的cuda系统库目录文件夹。
+```shell
+ cmake .. -G "Visual Studio 15 2017 Win64" -T host=x64 -DWITH_GPU=ON -DWITH_MKL=OFF -DWITH_STATIC_LIB=ON -DCMAKE_BUILD_TYPE=Release -DDEMO_NAME=inference_icnet -DPADDLE_LIB=D:\to_the_paddle_inference.lib -DCUDA_LIB=D:\tools\v8.0\lib\x64
+```
+然后用vs2015打开对应的项目文件，注意使用静态链接 "/MT"，生成对应的exe。将openblas.dll放到exe所在目录。
+
+4. 该exe即为项目生成文件，可绑定运行。
+
+## FAQ
+1. cmake需要您手动下载，并添加到系统路径里
+2. 路径中的不要包含空格，例如发现CUDA_LIB路径是Program Files(x86)可能会出错。可以将CUDA拷贝到一个新位置。

paddle/fluid/inference/api/demo_ci/windows_mobilenet.cc

@@ -0,0 +1,90 @@
+// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <glog/logging.h>
+
+#include <algorithm>
+#include <fstream>
+#include <iostream>
+#include <numeric>
+#include <string>
+#include <vector>
+
+#include "gflags/gflags.h"
+#include "paddle_inference_api.h"  // NOLINT
+
+DEFINE_string(modeldir, "", "Directory of the inference model.");
+DEFINE_bool(use_gpu, false, "Whether use gpu.");
+
+namespace paddle {
+namespace demo {
+
+void RunAnalysis() {
+  // 1. create AnalysisConfig
+  AnalysisConfig config;
+  if (FLAGS_modeldir.empty()) {
+    LOG(INFO) << "Usage: path\\mobilenet --modeldir=path/to/your/model";
+    exit(1);
+  }
+
+  // CreateConfig(&config);
+  if (FLAGS_use_gpu) {
+    config.EnableUseGpu(100, 0);
+  }
+  config.SetModel(FLAGS_modeldir + "/__model__",
+                  FLAGS_modeldir + "/__params__");
+
+  // 2. create predictor, prepare input data
+  std::unique_ptr<PaddlePredictor> predictor = CreatePaddlePredictor(config);
+  int batch_size = 1;
+  int channels = 3;
+  int height = 300;
+  int width = 300;
+  int nums = batch_size * channels * height * width;
+
+  float* input = new float[nums];
+  for (int i = 0; i < nums; ++i) input[i] = 0;
+
+  // 3. create input tensor, use ZeroCopyTensor
+  auto input_names = predictor->GetInputNames();
+  auto input_t = predictor->GetInputTensor(input_names[0]);
+  input_t->Reshape({batch_size, channels, height, width});
+  input_t->copy_from_cpu(input);
+
+  // 4. run predictor
+  predictor->ZeroCopyRun();
+
+  // 5. get out put
+  std::vector<float> out_data;
+  auto output_names = predictor->GetOutputNames();
+  auto output_t = predictor->GetOutputTensor(output_names[0]);
+  std::vector<int> output_shape = output_t->shape();
+  int out_num = std::accumulate(
+      output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
+
+  out_data.resize(out_num);
+  output_t->copy_to_cpu(out_data.data());
+  delete[] input;
+}
+
+}  // namespace demo
+}  // namespace paddle
+
+int main(int argc, char** argv) {
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  paddle::demo::RunAnalysis();
+  std::cout << "=========================Runs successfully===================="
+            << std::endl;
+  return 0;
+}

paddle/fluid/inference/api/details/CMakeLists.txt

@@ -0,0 +1,51 @@
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+cc_library(
+  reset_tensor_array
+  SRCS reset_tensor_array.cc
+  DEPS lod_tensor scope)
+if(WITH_ONNXRUNTIME)
+  cc_library(
+    zero_copy_tensor
+    SRCS zero_copy_tensor.cc
+    DEPS scope lod_tensor phi onnxruntime common)
+  cc_library(
+    zero_copy_tensor_dummy
+    SRCS zero_copy_tensor_dummy.cc
+    DEPS onnxruntime phi common)
+else()
+  cc_library(
+    zero_copy_tensor
+    SRCS zero_copy_tensor.cc
+    DEPS scope lod_tensor phi common)
+  cc_library(
+    zero_copy_tensor_dummy
+    SRCS zero_copy_tensor_dummy.cc
+    DEPS phi common)
+endif()
+
+if(NOT WIN32)
+  cc_test(
+    zero_copy_tensor_test
+    SRCS zero_copy_tensor_test.cc
+    DEPS paddle_inference_api)
+endif()
+
+if(WITH_ONNXRUNTIME AND WIN32)
+  # Copy onnxruntime for some c++ test in Windows, since the test will
+  # be build only in CI, so suppose the generator in Windows is Ninja.
+  copy_onnx(zero_copy_tensor_test)
+endif()

paddle/fluid/inference/api/details/reset_tensor_array.cc

@@ -0,0 +1,77 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/inference/api/details/reset_tensor_array.h"
+
+#include "glog/logging.h"
+
+namespace paddle::framework {
+class Scope;
+}  // namespace paddle::framework
+
+namespace paddle::details {
+
+// Should be called after the parameters are loaded.
+void TensorArrayBatchCleaner::CollectTensorArrays(framework::Scope *scope) {
+  if (flag_) {
+    for (auto &var_name : scope->LocalVarNames()) {
+      auto *var = scope->FindVar(var_name);
+      // TODO(Superjomn) should avoid the case when a TensorArray is a
+      // parameter.
+      if (var_name == "feed" || var_name == "fetch") continue;
+      if (var->IsType<phi::TensorArray>()) {
+        VLOG(4) << "collect " << var_name;
+        arrays_.push_back(var->GetMutable<phi::TensorArray>());
+      }
+    }
+    for (auto *kid : scope->kids()) {
+      CollectTensorArrays(kid);
+    }
+
+    VLOG(3) << "Collect " << arrays_.size() << " arrays";
+    flag_ = false;
+  }
+}
+
+// Should be called when `Run` finished.
+void TensorArrayBatchCleaner::ResetTensorArray() {
+  for (auto *arr : arrays_) {
+    arr->clear();
+  }
+}
+
+void TensorArrayBatchCleaner::CollectNoTensorVars(framework::Scope *scope) {
+  if (no_tensor_flag_) {
+    for (auto &var_name : scope->LocalVarNames()) {
+      auto *var = scope->FindVar(var_name);
+      if (!var->IsInitialized()) continue;
+      if (!valid_types_.count(var->Type())) {
+        no_tensor_vars_.insert(var);
+      }
+    }
+
+    for (auto *kid : scope->kids()) {
+      CollectTensorArrays(kid);
+    }
+    no_tensor_flag_ = false;  // Only collect one time.
+  }
+}
+
+void TensorArrayBatchCleaner::ResetNoTensorVars() {
+  for (auto *var : no_tensor_vars_) {
+    var->Clear();
+  }
+}
+
+}  // namespace paddle::details

paddle/fluid/inference/api/details/reset_tensor_array.h

@@ -0,0 +1,75 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <unordered_set>
+#include <vector>
+
+#include "paddle/fluid/framework/dense_tensor_array.h"
+#include "paddle/fluid/framework/scope.h"
+#include "paddle/fluid/framework/variable.h"
+
+namespace phi {
+class DenseTensor;
+}  // namespace phi
+
+namespace paddle {
+namespace framework {
+class Scope;
+class SelectedRows;
+}  // namespace framework
+}  // namespace paddle
+
+namespace paddle {
+namespace details {
+
+// Clean the TensorArray each batch to make the behavior the same with the
+// training phase.
+struct TensorArrayBatchCleaner {
+  TensorArrayBatchCleaner() {
+    constexpr auto kTensorId = framework::VarTypeTrait<phi::DenseTensor>::kId;
+    constexpr auto kDenseTensorId =
+        framework::VarTypeTrait<phi::DenseTensor>::kId;
+    constexpr auto kSelectedRowsId =
+        framework::VarTypeTrait<phi::SelectedRows>::kId;
+    constexpr auto kFetchListId =
+        framework::VarTypeTrait<framework::FetchList>::kId;
+    valid_types_.insert(kTensorId);
+    valid_types_.insert(kDenseTensorId);
+    valid_types_.insert(kSelectedRowsId);
+    valid_types_.insert(kFetchListId);
+  }
+  // Collect the variables that are not Tensor or DenseTensor, and reset them to
+  // a bool(trick), because some of them are containers, and some operators just
+  // keep inserting new items without clearing the containers first; So the
+  // memory grow larger and larger in inference service deployed online.
+  void CollectNoTensorVars(framework::Scope *scope);
+  void ResetNoTensorVars();
+
+  // Fix the tensor array not clear in the inference scenarios.
+  void CollectTensorArrays(framework::Scope *scope);
+  void ResetTensorArray();
+
+ private:
+  bool flag_{true};
+  bool no_tensor_flag_{true};
+  std::vector<phi::TensorArray *> arrays_;
+
+  std::unordered_set<int> valid_types_;
+  std::unordered_set<framework::Variable *> no_tensor_vars_;
+};
+
+}  // namespace details
+}  // namespace paddle

paddle/fluid/inference/api/details/zero_copy_tensor.cc

@@ -0,0 +1,984 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/framework/convert_utils.h"
+#include "paddle/fluid/framework/data_layout_transform.h"
+#include "paddle/fluid/framework/lod_tensor.h"
+#include "paddle/fluid/framework/scope.h"
+#include "paddle/fluid/inference/api/paddle_inference_api.h"
+#include "paddle/fluid/inference/api/paddle_tensor.h"
+#include "paddle/fluid/platform/enforce.h"
+#include "paddle/phi/common/bfloat16.h"
+#include "paddle/phi/common/float16.h"
+#include "paddle/phi/core/allocator.h"
+#include "paddle/phi/core/memory/memcpy.h"
+#include "paddle/phi/core/vocab/string_array.h"
+#ifdef PADDLE_WITH_ONNXRUNTIME
+#include "onnxruntime_c_api.h"    // NOLINT
+#include "onnxruntime_cxx_api.h"  // NOLINT
+#endif
+
+namespace paddle_infer {
+
+using float16 = phi::dtype::float16;
+using bfloat16 = phi::dtype::bfloat16;
+
+void Tensor::Reshape(const std::vector<int> &shape) {
+#ifdef PADDLE_WITH_ONNXRUNTIME
+  if (is_ort_tensor_) {
+    shape_.assign(shape.begin(), shape.end());
+    return;
+  }
+#endif
+
+  PADDLE_ENFORCE_EQ(
+      name_.empty(),
+      false,
+      common::errors::PreconditionNotMet(
+          "Need to SetName first, so that the corresponding tensor can "
+          "be retrieved."));
+  PADDLE_ENFORCE_EQ(input_or_output_,
+                    true,
+                    common::errors::PermissionDenied(
+                        "Can't reshape the output tensor, it is readonly"));
+  auto *scope = static_cast<paddle::framework::Scope *>(scope_);
+  auto *var = scope->FindVar(name_);
+  PADDLE_ENFORCE_NOT_NULL(
+      var,
+      common::errors::PreconditionNotMet(
+          "No tensor called [%s] in the runtime scope", name_));
+  auto *tensor = var->GetMutable<phi::DenseTensor>();
+  tensor->Resize(common::make_ddim(shape));
+}
+
+void Tensor::ReshapeStrings(const size_t &shape) {
+  PADDLE_ENFORCE_EQ(
+      name_.empty(),
+      false,
+      common::errors::PreconditionNotMet(
+          "Need to SetName first, so that the corresponding tensor can "
+          "be retrieved."));
+  PADDLE_ENFORCE_EQ(input_or_output_,
+                    true,
+                    common::errors::PermissionDenied(
+                        "Can't reshape the output tensor, it is readonly"));
+  auto *scope = static_cast<paddle::framework::Scope *>(scope_);
+  auto *var = scope->FindVar(name_);
+  PADDLE_ENFORCE_NOT_NULL(
+      var,
+      common::errors::PreconditionNotMet(
+          "No tensor called [%s] in the runtime scope", name_));
+  phi::Strings *tensor = var->GetMutable<phi::Strings>();
+  tensor->resize(shape);
+}
+
+#define EAGER_GET_TENSOR(tensor_type)    \
+  if (!tensor_) {                        \
+    tensor_ = FindTensor<tensor_type>(); \
+  }                                      \
+  auto *tensor = static_cast<tensor_type *>(tensor_);
+
+template <typename T>
+T *Tensor::mutable_data(PlaceType place) {
+#ifdef PADDLE_WITH_ONNXRUNTIME
+  if (is_ort_tensor_) {
+    return ORTGetMutableData<T>();
+  }
+#endif
+  EAGER_GET_TENSOR(phi::DenseTensor);
+  PADDLE_ENFORCE_GT(
+      tensor->numel(),
+      0,
+      common::errors::PreconditionNotMet(
+          "You should call Tensor::Reshape(const std::vector<int> "
+          "&shape)"
+          "function before retrieving mutable_data from input tensor."));
+  switch (static_cast<int>(place)) {
+    case static_cast<int>(PlaceType::kCPU): {
+      return tensor->mutable_data<T>(phi::CPUPlace());
+    }
+    case static_cast<int>(PlaceType::kGPU): {
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+      phi::GPUPlace gpu_place(device_);
+      auto *dev_ctxs = reinterpret_cast<const std::map<
+          phi::Place,
+          std::shared_future<std::unique_ptr<phi::DeviceContext>>> *>(
+          device_contexts_);
+      auto *dev_ctx =
+          static_cast<phi::GPUContext *>(dev_ctxs->at(gpu_place).get().get());
+      return dev_ctx->Alloc<T>(tensor, tensor->numel() * sizeof(T));
+#else
+      return tensor->mutable_data<T>(phi::GPUPlace(device_));
+#endif
+    }
+    case static_cast<int>(PlaceType::kXPU): {
+      return tensor->mutable_data<T>(phi::XPUPlace(device_));
+    }
+    case static_cast<int>(PlaceType::kCUSTOM): {
+      return tensor->mutable_data<T>(phi::CustomPlace(device_type_, device_));
+    }
+    default:
+      PADDLE_THROW(common::errors::Unavailable(
+          "Only CPU / CUDA / XPU places is supported. The place `%d` is "
+          "not supported.",
+          static_cast<int>(place)));
+      break;
+  }
+  return nullptr;
+}
+
+template <typename T>
+T *Tensor::data(PlaceType *place, int *size) const {
+  EAGER_GET_TENSOR(phi::DenseTensor);
+  auto *res = tensor->data<T>();
+
+  if (phi::is_cpu_place(tensor->place())) {
+    *place = PlaceType::kCPU;
+  } else if (phi::is_gpu_place(tensor->place())) {
+    *place = PlaceType::kGPU;
+  } else if (phi::is_xpu_place(tensor->place())) {
+    *place = PlaceType::kXPU;
+  } else if (phi::is_custom_place(tensor->place())) {
+    *place = PlaceType::kCUSTOM;
+  } else {
+    *place = PlaceType::kUNK;
+  }
+
+  *size = static_cast<int>(tensor->numel());
+  return res;
+}
+
+DataType Tensor::type() const {
+#ifdef PADDLE_WITH_ONNXRUNTIME
+  if (is_ort_tensor_) {
+    return dtype_;
+  }
+#endif
+  EAGER_GET_TENSOR(phi::DenseTensor);
+  auto type = paddle::framework::TransToProtoVarType(tensor->dtype());
+  if (type == paddle::framework::proto::VarType::FP64) {
+    return DataType::FLOAT64;
+  } else if (type == paddle::framework::proto::VarType::FP32) {
+    return DataType::FLOAT32;
+  } else if (type == paddle::framework::proto::VarType::FP16) {
+    return DataType::FLOAT16;
+  } else if (type == paddle::framework::proto::VarType::BF16) {
+    return DataType::BFLOAT16;
+  } else if (type == paddle::framework::proto::VarType::INT64) {
+    return DataType::INT64;
+  } else if (type == paddle::framework::proto::VarType::INT32) {
+    return DataType::INT32;
+  } else if (type == paddle::framework::proto::VarType::UINT8) {
+    return DataType::UINT8;
+  } else if (type == paddle::framework::proto::VarType::INT8) {
+    return DataType::INT8;
+  } else if (type == paddle::framework::proto::VarType::BOOL) {
+    return DataType::BOOL;
+  }
+  return DataType::FLOAT32;
+}
+
+PlaceType Tensor::place() const { return place_; }
+
+template <typename T>
+void Tensor::CopyFromCpu(const T *data) {
+  EAGER_GET_TENSOR(phi::DenseTensor);
+  PADDLE_ENFORCE_GE(tensor->numel(),
+                    0,
+                    common::errors::PreconditionNotMet(
+                        "You should call Tensor::Reshape(const "
+                        "std::vector<int> &shape)"
+                        "function before copying data from cpu."));
+  size_t ele_size = tensor->numel() * sizeof(T);
+
+  if (place_ == PlaceType::kCPU) {
+    auto *t_data = tensor->mutable_data<T>(phi::CPUPlace());
+    std::memcpy(static_cast<void *>(t_data), data, ele_size);
+  } else if (place_ == PlaceType::kGPU) {
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+
+    phi::GPUPlace gpu_place(device_);
+    auto *dev_ctxs = reinterpret_cast<const std::map<
+        phi::Place,
+        std::shared_future<std::unique_ptr<phi::DeviceContext>>> *>(
+        device_contexts_);
+    auto *dev_ctx =
+        static_cast<phi::GPUContext *>(dev_ctxs->at(gpu_place).get().get());
+    auto *t_data = dev_ctx->Alloc<T>(tensor, tensor->numel() * sizeof(T));
+
+    paddle::memory::Copy(gpu_place,
+                         static_cast<void *>(t_data),
+                         phi::CPUPlace(),
+                         data,
+                         ele_size,
+                         dev_ctx->stream());
+#else
+    PADDLE_THROW(common::errors::Unavailable(
+        "Can not create tensor with CUDA place because paddle is not compiled "
+        "with CUDA."));
+#endif
+  } else if (place_ == PlaceType::kXPU) {
+#ifdef PADDLE_WITH_XPU
+    phi::XPUPlace xpu_place(device_);
+    auto *t_data = tensor->mutable_data<T>(xpu_place);
+    paddle::memory::Copy(xpu_place,
+                         static_cast<void *>(t_data),
+                         phi::CPUPlace(),
+                         data,
+                         ele_size);
+#else
+    PADDLE_THROW(common::errors::Unavailable(
+        "Can not create tensor with XPU place because paddle is not compiled "
+        "with XPU."));
+#endif
+  } else if (place_ == PlaceType::kCUSTOM) {
+#ifdef PADDLE_WITH_CUSTOM_DEVICE
+    phi::DeviceContextPool &pool = phi::DeviceContextPool::Instance();
+    phi::CustomPlace custom_place(device_type_, device_);
+    auto *t_data = tensor->mutable_data<T>(custom_place);
+    auto *dev_ctx =
+        static_cast<const phi::CustomContext *>(pool.Get(custom_place));
+    paddle::memory::Copy(custom_place,
+                         static_cast<void *>(t_data),
+                         phi::CPUPlace(),
+                         data,
+                         ele_size,
+                         dev_ctx->stream());
+#else
+    PADDLE_THROW(common::errors::Unavailable(
+        "Can not create tensor with Custom place because paddle is not "
+        "compiled "
+        "with XPU."));
+#endif
+  } else {
+    PADDLE_THROW(common::errors::InvalidArgument(
+        "The analysis predictor supports CPU, GPU, XPU and CUSTOM_DEVICE "
+        "now."));
+  }
+}
+
+template <typename T>
+struct DataTypeInfo;
+
+template <>
+struct DataTypeInfo<double> {
+  phi::DataType TYPE = phi::DataType::FLOAT64;
+};
+
+template <>
+struct DataTypeInfo<float> {
+  phi::DataType TYPE = phi::DataType::FLOAT32;
+};
+
+template <>
+struct DataTypeInfo<float16> {
+  phi::DataType TYPE = phi::DataType::FLOAT16;
+};
+
+template <>
+struct DataTypeInfo<bfloat16> {
+  phi::DataType TYPE = phi::DataType::BFLOAT16;
+};
+
+template <>
+struct DataTypeInfo<int64_t> {
+  phi::DataType TYPE = phi::DataType::INT64;
+};
+
+template <>
+struct DataTypeInfo<int8_t> {
+  phi::DataType TYPE = phi::DataType::INT8;
+};
+
+template <>
+struct DataTypeInfo<uint8_t> {
+  phi::DataType TYPE = phi::DataType::UINT8;
+};
+
+template <>
+struct DataTypeInfo<int32_t> {
+  phi::DataType TYPE = phi::DataType::INT32;
+};
+
+template <>
+struct DataTypeInfo<bool> {
+  phi::DataType TYPE = phi::DataType::BOOL;
+};
+
+phi::DataLayout LayoutConvert(DataLayout layout) {
+  PADDLE_ENFORCE_EQ(
+      layout,
+      DataLayout::kNCHW,
+      common::errors::InvalidArgument("Only NCHW is supported now."));
+  return phi::DataLayout::NCHW;
+}
+
+template <typename T>
+void Tensor::ShareExternalData(const T *data,
+                               const std::vector<int> &shape,
+                               PlaceType place,
+                               DataLayout layout) {
+  EAGER_GET_TENSOR(phi::DenseTensor)
+  size_t size =
+      std::accumulate(shape.begin(), shape.end(), 1, std::multiplies<int>()) *
+      sizeof(T);
+  phi::DenseTensorMeta meta(
+      DataTypeInfo<T>().TYPE, common::make_ddim(shape), LayoutConvert(layout));
+  if (place == PlaceType::kCPU) {
+    phi::DenseTensor dtensor(std::make_shared<phi::Allocation>(
+                                 const_cast<T *>(data), size, phi::CPUPlace()),
+                             meta);
+    *tensor = std::move(dtensor);
+  } else if (place == PlaceType::kGPU) {
+    phi::DenseTensor dtensor(
+        std::make_shared<phi::Allocation>(
+            const_cast<T *>(data), size, phi::GPUPlace(device_)),
+        meta);
+    *tensor = std::move(dtensor);
+  } else if (place == PlaceType::kXPU) {
+    phi::DenseTensor dtensor(
+        std::make_shared<phi::Allocation>(
+            const_cast<T *>(data), size, phi::XPUPlace(device_)),
+        meta);
+    *tensor = std::move(dtensor);
+  } else if (place == PlaceType::kCUSTOM) {
+    phi::DenseTensor dtensor(std::make_shared<phi::Allocation>(
+                                 const_cast<T *>(data),
+                                 size,
+                                 phi::CustomPlace(device_type_, device_)),
+                             meta);
+    *tensor = std::move(dtensor);
+  } else {
+    PADDLE_THROW(common::errors::InvalidArgument(
+        "PlaceType must be one of [PlaceType::kCPU, PlaceType::kGPU, "
+        "PlaceType::kXPU]."));
+  }
+}
+
+void Tensor::CopyStringsFromCpu(const paddle_infer::Strings *data) {
+  EAGER_GET_TENSOR(phi::Strings);
+  PADDLE_ENFORCE_GE(tensor->size(),
+                    0,
+                    common::errors::PreconditionNotMet(
+                        "You should call Tensor::Reshape(const "
+                        "std::size_t &shape) function before copying "
+                        "the string data from cpu."));
+  *tensor = *data;
+}
+
+template <typename T>
+void Tensor::CopyToCpuImpl(T *data,
+                           void *exec_stream,
+                           CallbackFunc cb,
+                           void *cb_params) const {
+  EAGER_GET_TENSOR(phi::DenseTensor);
+  auto ele_num = tensor->numel();
+  auto *t_data = tensor->data<T>();
+  auto t_place = tensor->place();
+
+  if (phi::is_cpu_place(t_place)) {
+#ifdef PADDLE_WITH_DNNL
+    if (tensor->layout() == phi::DataLayout::ONEDNN) {
+      phi::DenseTensor out;
+      auto mem_allocation =
+          std::make_shared<paddle::memory::allocation::Allocation>(
+              static_cast<void *>(data), ele_num * sizeof(T), phi::CPUPlace());
+      out.ResetHolder(mem_allocation);
+      phi::funcs::TransDataLayoutFromOneDNN(
+          tensor->layout(),
+          phi::OneDNNContext::tls().get_cur_paddle_data_layout(),
+          *tensor,
+          &out,
+          phi::CPUPlace(),
+          true);
+    } else {
+      std::memcpy(static_cast<void *>(data), t_data, ele_num * sizeof(T));
+    }
+#else
+    std::memcpy(static_cast<void *>(data), t_data, ele_num * sizeof(T));
+#endif
+  } else if (phi::is_ipu_place(t_place)) {
+#ifdef PADDLE_WITH_IPU
+    std::memcpy(static_cast<void *>(data), t_data, ele_num * sizeof(T));
+#else
+    PADDLE_THROW(common::errors::Unavailable(
+        "Can not create tensor with IPU place because paddle is not compiled "
+        "with IPU."));
+#endif
+  } else if (place_ == PlaceType::kGPU) {
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+    auto gpu_place = t_place;
+    auto *dev_ctxs = reinterpret_cast<const std::map<
+        phi::Place,
+        std::shared_future<std::unique_ptr<phi::DeviceContext>>> *>(
+        device_contexts_);
+    auto *dev_ctx =
+        static_cast<phi::GPUContext *>(dev_ctxs->at(gpu_place).get().get());
+    paddle::memory::Copy(phi::CPUPlace(),
+                         static_cast<void *>(data),
+                         gpu_place,
+                         t_data,
+                         ele_num * sizeof(T),
+                         dev_ctx->stream());
+#ifdef PADDLE_WITH_HIP
+    hipStreamSynchronize(dev_ctx->stream());
+#else
+    // async, return stream
+    if (nullptr != exec_stream) {
+      *(static_cast<cudaStream_t *>(exec_stream)) = dev_ctx->stream();
+      // async with callback
+    } else if (cb) {
+      cudaLaunchHostFunc(dev_ctx->stream(), cb, cb_params);
+      // sync
+    } else {
+      cudaStreamSynchronize(dev_ctx->stream());
+    }
+#endif
+#else
+    PADDLE_THROW(common::errors::Unavailable(
+        "Can not create tensor with CUDA place because paddle is not compiled "
+        "with CUDA."));
+#endif
+  } else if (place_ == PlaceType::kXPU) {
+#ifdef PADDLE_WITH_XPU
+    auto xpu_place = t_place;
+    paddle::memory::Copy(phi::CPUPlace(),
+                         static_cast<void *>(data),
+                         xpu_place,
+                         t_data,
+                         ele_num * sizeof(T));
+#else
+    PADDLE_THROW(common::errors::Unavailable(
+        "Can not create tensor with XPU place because paddle is not compiled "
+        "with XPU."));
+#endif
+  } else {
+#ifdef PADDLE_WITH_CUSTOM_DEVICE
+    phi::DeviceContextPool &pool = phi::DeviceContextPool::Instance();
+    auto custom_place = t_place;
+    auto *dev_ctx =
+        static_cast<const phi::CustomContext *>(pool.Get(custom_place));
+    paddle::memory::Copy(phi::CPUPlace(),
+                         static_cast<void *>(data),
+                         custom_place,
+                         t_data,
+                         ele_num * sizeof(T),
+                         dev_ctx->stream());
+    dev_ctx->GetStream()->Synchronize();
+#else
+    PADDLE_THROW(common::errors::InvalidArgument(
+        "The analysis predictor supports CPU, GPU and XPU now."));
+#endif
+  }
+}
+
+template <typename T>
+void Tensor::CopyToCpu(T *data) const {
+#ifdef PADDLE_WITH_ONNXRUNTIME
+  if (is_ort_tensor_) {
+    ORTCopyToCpu<T>(data);
+    return;
+  }
+#endif
+
+  CopyToCpuImpl<T>(data, nullptr, nullptr, nullptr);
+}
+
+template <typename T>
+void Tensor::CopyToCpuAsync(T *data, void *exec_stream) const {
+  CopyToCpuImpl<T>(data, exec_stream, nullptr, nullptr);
+}
+
+template <typename T>
+void Tensor::CopyToCpuAsync(T *data, CallbackFunc cb, void *cb_params) const {
+  CopyToCpuImpl<T>(data, nullptr, cb, cb_params);
+}
+
+template PD_INFER_DECL void Tensor::CopyFromCpu<double>(const double *data);
+template PD_INFER_DECL void Tensor::CopyFromCpu<float>(const float *data);
+template PD_INFER_DECL void Tensor::CopyFromCpu<int64_t>(const int64_t *data);
+template PD_INFER_DECL void Tensor::CopyFromCpu<int32_t>(const int32_t *data);
+template PD_INFER_DECL void Tensor::CopyFromCpu<uint8_t>(const uint8_t *data);
+template PD_INFER_DECL void Tensor::CopyFromCpu<int8_t>(const int8_t *data);
+template PD_INFER_DECL void Tensor::CopyFromCpu<float16>(const float16 *data);
+template PD_INFER_DECL void Tensor::CopyFromCpu<bfloat16>(const bfloat16 *data);
+template PD_INFER_DECL void Tensor::CopyFromCpu<bool>(const bool *data);
+
+template PD_INFER_DECL void Tensor::ShareExternalData<double>(
+    const double *data,
+    const std::vector<int> &shape,
+    PlaceType place,
+    DataLayout layout);
+template PD_INFER_DECL void Tensor::ShareExternalData<float>(
+    const float *data,
+    const std::vector<int> &shape,
+    PlaceType place,
+    DataLayout layout);
+template PD_INFER_DECL void Tensor::ShareExternalData<int64_t>(
+    const int64_t *data,
+    const std::vector<int> &shape,
+    PlaceType place,
+    DataLayout layout);
+template PD_INFER_DECL void Tensor::ShareExternalData<int32_t>(
+    const int32_t *data,
+    const std::vector<int> &shape,
+    PlaceType place,
+    DataLayout layout);
+template PD_INFER_DECL void Tensor::ShareExternalData<uint8_t>(
+    const uint8_t *data,
+    const std::vector<int> &shape,
+    PlaceType place,
+    DataLayout layout);
+template PD_INFER_DECL void Tensor::ShareExternalData<int8_t>(
+    const int8_t *data,
+    const std::vector<int> &shape,
+    PlaceType place,
+    DataLayout layout);
+template PD_INFER_DECL void Tensor::ShareExternalData<float16>(
+    const float16 *data,
+    const std::vector<int> &shape,
+    PlaceType place,
+    DataLayout layout);
+template PD_INFER_DECL void Tensor::ShareExternalData<bfloat16>(
+    const bfloat16 *data,
+    const std::vector<int> &shape,
+    PlaceType place,
+    DataLayout layout);
+template PD_INFER_DECL void Tensor::ShareExternalData<bool>(
+    const bool *data,
+    const std::vector<int> &shape,
+    PlaceType place,
+    DataLayout layout);
+
+template PD_INFER_DECL void Tensor::CopyToCpu<double>(double *data) const;
+template PD_INFER_DECL void Tensor::CopyToCpu<float>(float *data) const;
+template PD_INFER_DECL void Tensor::CopyToCpu<int64_t>(int64_t *data) const;
+template PD_INFER_DECL void Tensor::CopyToCpu<int32_t>(int32_t *data) const;
+template PD_INFER_DECL void Tensor::CopyToCpu<uint8_t>(uint8_t *data) const;
+template PD_INFER_DECL void Tensor::CopyToCpu<int8_t>(int8_t *data) const;
+template PD_INFER_DECL void Tensor::CopyToCpu<float16>(float16 *data) const;
+template PD_INFER_DECL void Tensor::CopyToCpu<bfloat16>(bfloat16 *data) const;
+template PD_INFER_DECL void Tensor::CopyToCpu<bool>(bool *data) const;
+
+template PD_INFER_DECL void Tensor::CopyToCpuImpl<double>(
+    double *data, void *exec_stream, CallbackFunc cb, void *cb_params) const;
+template PD_INFER_DECL void Tensor::CopyToCpuImpl<float>(float *data,
+                                                         void *exec_stream,
+                                                         CallbackFunc cb,
+                                                         void *cb_params) const;
+template PD_INFER_DECL void Tensor::CopyToCpuImpl<int64_t>(
+    int64_t *data, void *exec_stream, CallbackFunc cb, void *cb_params) const;
+template PD_INFER_DECL void Tensor::CopyToCpuImpl<int32_t>(
+    int32_t *data, void *exec_stream, CallbackFunc cb, void *cb_params) const;
+template PD_INFER_DECL void Tensor::CopyToCpuImpl<uint8_t>(
+    uint8_t *data, void *exec_stream, CallbackFunc cb, void *cb_params) const;
+template PD_INFER_DECL void Tensor::CopyToCpuImpl<int8_t>(
+    int8_t *data, void *exec_stream, CallbackFunc cb, void *cb_params) const;
+template PD_INFER_DECL void Tensor::CopyToCpuImpl<float16>(
+    float16 *data, void *exec_stream, CallbackFunc cb, void *cb_params) const;
+template PD_INFER_DECL void Tensor::CopyToCpuImpl<bfloat16>(
+    bfloat16 *data, void *exec_stream, CallbackFunc cb, void *cb_params) const;
+template PD_INFER_DECL void Tensor::CopyToCpuImpl<bool>(bool *data,
+                                                        void *exec_stream,
+                                                        CallbackFunc cb,
+                                                        void *cb_params) const;
+
+template PD_INFER_DECL void Tensor::CopyToCpuAsync<double>(
+    double *data, void *exec_stream) const;
+template PD_INFER_DECL void Tensor::CopyToCpuAsync<float>(
+    float *data, void *exec_stream) const;
+template PD_INFER_DECL void Tensor::CopyToCpuAsync<int64_t>(
+    int64_t *data, void *exec_stream) const;
+template PD_INFER_DECL void Tensor::CopyToCpuAsync<int32_t>(
+    int32_t *data, void *exec_stream) const;
+template PD_INFER_DECL void Tensor::CopyToCpuAsync<uint8_t>(
+    uint8_t *data, void *exec_stream) const;
+template PD_INFER_DECL void Tensor::CopyToCpuAsync<int8_t>(
+    int8_t *data, void *exec_stream) const;
+template PD_INFER_DECL void Tensor::CopyToCpuAsync<float16>(
+    float16 *data, void *exec_stream) const;
+template PD_INFER_DECL void Tensor::CopyToCpuAsync<bfloat16>(
+    bfloat16 *data, void *exec_stream) const;
+template PD_INFER_DECL void Tensor::CopyToCpuAsync<bool>(
+    bool *data, void *exec_stream) const;
+
+template PD_INFER_DECL void Tensor::CopyToCpuAsync<double>(
+    double *data, CallbackFunc cb, void *cb_params) const;
+template PD_INFER_DECL void Tensor::CopyToCpuAsync<float>(
+    float *data, CallbackFunc cb, void *cb_params) const;
+template PD_INFER_DECL void Tensor::CopyToCpuAsync<int64_t>(
+    int64_t *data, CallbackFunc cb, void *cb_params) const;
+template PD_INFER_DECL void Tensor::CopyToCpuAsync<int32_t>(
+    int32_t *data, CallbackFunc cb, void *cb_params) const;
+template PD_INFER_DECL void Tensor::CopyToCpuAsync<uint8_t>(
+    uint8_t *data, CallbackFunc cb, void *cb_params) const;
+template PD_INFER_DECL void Tensor::CopyToCpuAsync<int8_t>(
+    int8_t *data, CallbackFunc cb, void *cb_params) const;
+template PD_INFER_DECL void Tensor::CopyToCpuAsync<float16>(
+    float16 *data, CallbackFunc cb, void *cb_params) const;
+template PD_INFER_DECL void Tensor::CopyToCpuAsync<bfloat16>(
+    bfloat16 *data, CallbackFunc cb, void *cb_params) const;
+template PD_INFER_DECL void Tensor::CopyToCpuAsync<bool>(bool *data,
+                                                         CallbackFunc cb,
+                                                         void *cb_params) const;
+
+template PD_INFER_DECL double *Tensor::data<double>(PlaceType *place,
+                                                    int *size) const;
+template PD_INFER_DECL float *Tensor::data<float>(PlaceType *place,
+                                                  int *size) const;
+template PD_INFER_DECL int64_t *Tensor::data<int64_t>(PlaceType *place,
+                                                      int *size) const;
+template PD_INFER_DECL int32_t *Tensor::data<int32_t>(PlaceType *place,
+                                                      int *size) const;
+template PD_INFER_DECL uint8_t *Tensor::data<uint8_t>(PlaceType *place,
+                                                      int *size) const;
+template PD_INFER_DECL int8_t *Tensor::data<int8_t>(PlaceType *place,
+                                                    int *size) const;
+template PD_INFER_DECL float16 *Tensor::data<float16>(PlaceType *place,
+                                                      int *size) const;
+template PD_INFER_DECL bfloat16 *Tensor::data<bfloat16>(PlaceType *place,
+                                                        int *size) const;
+template PD_INFER_DECL bool *Tensor::data<bool>(PlaceType *place,
+                                                int *size) const;
+
+template PD_INFER_DECL double *Tensor::mutable_data<double>(PlaceType place);
+template PD_INFER_DECL float *Tensor::mutable_data<float>(PlaceType place);
+template PD_INFER_DECL int64_t *Tensor::mutable_data<int64_t>(PlaceType place);
+template PD_INFER_DECL int32_t *Tensor::mutable_data<int32_t>(PlaceType place);
+template PD_INFER_DECL uint8_t *Tensor::mutable_data<uint8_t>(PlaceType place);
+template PD_INFER_DECL int8_t *Tensor::mutable_data<int8_t>(PlaceType place);
+template PD_INFER_DECL float16 *Tensor::mutable_data<float16>(PlaceType place);
+template PD_INFER_DECL bfloat16 *Tensor::mutable_data<bfloat16>(
+    PlaceType place);
+template PD_INFER_DECL bool *Tensor::mutable_data<bool>(PlaceType place);
+
+Tensor::Tensor(void *scope, const void *device_contexts)
+    : dtype_(DataType::FLOAT16),
+      input_or_output_(false),
+      scope_{scope},
+      device_contexts_(device_contexts),
+      place_(PlaceType::kCPU),
+      device_(0) {}
+
+template <typename T>
+void *Tensor::FindTensor() const {
+  PADDLE_ENFORCE_EQ(
+      name_.empty(),
+      false,
+      common::errors::PreconditionNotMet(
+          "Need to SetName first, so that the corresponding tensor can "
+          "be retrieved."));
+  auto *scope = static_cast<paddle::framework::Scope *>(scope_);
+  auto *var = scope->FindVar(name_);
+  PADDLE_ENFORCE_NOT_NULL(
+      var,
+      common::errors::PreconditionNotMet(
+          "No tensor called [%s] in the runtime scope", name_));
+  auto *tensor = var->GetMutable<T>();
+  return tensor;
+}
+
+std::vector<int> Tensor::shape() const {
+#ifdef PADDLE_WITH_ONNXRUNTIME
+  if (is_ort_tensor_) {
+    std::vector<int> shape;
+    // input handle
+    if (idx_ < 0) {
+      shape.assign(shape_.begin(), shape_.end());
+    } else {  // output handle
+      auto binding = binding_.lock();
+      PADDLE_ENFORCE_NOT_NULL(binding,
+                              common::errors::PreconditionNotMet(
+                                  "output tensor [%s] no binding ptr", name_));
+      std::vector<Ort::Value> outputs = binding->GetOutputValues();
+      Ort::Value &value = outputs[idx_];
+      auto info = value.GetTensorTypeAndShapeInfo();
+      auto ort_shape = info.GetShape();
+      shape.assign(ort_shape.begin(), ort_shape.end());
+    }
+    return shape;
+  }
+#endif
+  EAGER_GET_TENSOR(phi::DenseTensor);
+  PADDLE_ENFORCE_NOT_NULL(
+      tensor_,
+      common::errors::PreconditionNotMet(
+          "Not found tensor called %s in the scope", name_));
+// oneDNN may does layout transform internally, so need to reorder before
+// return
+#ifdef PADDLE_WITH_DNNL
+  if (tensor->layout() == phi::DataLayout::ONEDNN) {
+    phi::DataLayout out_layout =
+        phi::OneDNNContext::tls().get_cur_paddle_data_layout();
+    // Set default as NCHW in case not specified
+    out_layout = out_layout == phi::DataLayout::kAnyLayout
+                     ? phi::DataLayout::kNCHW
+                     : out_layout;
+    // In these data layouts, channel dimension is either on 2nd position: nChw
+    // or
+    // at last nhwC, so for dim==2 these layouts are the same and nothing should
+    // be done. Similarly for dim==1 when you have just one possible
+    // combination.
+    if (tensor->dims().size() < 3)
+      return common::vectorize<int>(tensor->dims());
+    if (out_layout == phi::DataLayout::NHWC ||
+        out_layout == phi::DataLayout::NDHWC) {
+      auto dims = common::vectorize<int>(tensor->dims());
+      std::rotate(dims.begin() + 1, dims.begin() + 2, dims.end());
+      return dims;
+    } else {
+      return common::vectorize<int>(tensor->dims());
+    }
+  }
+#endif
+  return common::vectorize<int>(tensor->dims());
+}
+
+void Tensor::SetLoD(const std::vector<std::vector<size_t>> &x) {
+  EAGER_GET_TENSOR(phi::DenseTensor);
+  phi::LegacyLoD lod;
+  for (auto &level : x) {
+    lod.emplace_back(level);
+  }
+  tensor->set_lod(lod);
+}
+
+std::vector<std::vector<size_t>> Tensor::lod() const {
+  EAGER_GET_TENSOR(phi::DenseTensor);
+  std::vector<std::vector<size_t>> res;
+  for (auto &level : tensor->lod()) {
+    res.emplace_back(level);
+  }
+  return res;
+}
+
+void Tensor::SetName(const std::string &name) { name_ = name; }
+
+const std::string &Tensor::name() const { return name_; }
+
+void Tensor::SetPlace(PlaceType place,
+                      int device,
+                      const std::string device_type) {
+  place_ = place;
+  device_ = device;
+  device_type_ = device_type;
+}
+
+#ifdef PADDLE_WITH_ONNXRUNTIME
+void Tensor::SetOrtMark(bool is_ort_tensor) { is_ort_tensor_ = is_ort_tensor; }
+
+void Tensor::SetOrtBinding(const std::shared_ptr<Ort::IoBinding> binding) {
+  binding_ = binding;
+}
+
+template <typename T>
+T *Tensor::ORTGetMutableData() {
+  auto binding = binding_.lock();
+  PADDLE_ENFORCE_NOT_NULL(binding,
+                          common::errors::PreconditionNotMet(
+                              "output tensor [%s] no binding ptr", name_));
+  std::vector<Ort::Value> outputs = binding->GetOutputValues();
+  Ort::Value &value = outputs[idx_];
+  return value.GetTensorMutableData<T>();
+}
+
+template <typename T>
+void Tensor::ORTCopyToCpu(T *data) const {
+  auto binding = binding_.lock();
+  PADDLE_ENFORCE_NOT_NULL(binding,
+                          common::errors::PreconditionNotMet(
+                              "output tensor [%s] no binding ptr", name_));
+  std::vector<Ort::Value> outputs = binding->GetOutputValues();
+  Ort::Value &value = outputs[idx_];
+  auto info = value.GetTensorTypeAndShapeInfo();
+  size_t size = info.GetElementCount() * sizeof(T);
+
+  if (place_ == PlaceType::kCPU) {
+    std::memcpy(static_cast<void *>(data), value.GetTensorData<void *>(), size);
+  } else {
+    PADDLE_THROW(common::errors::Unavailable(
+        "CopyToCpu error.The current ONNXRuntime backend doesn't support "
+        "GPU."));
+  }
+}
+
+template void Tensor::ORTCopyToCpu<float>(float *data) const;
+template void Tensor::ORTCopyToCpu<int32_t>(int32_t *data) const;
+template void Tensor::ORTCopyToCpu<uint8_t>(uint8_t *data) const;
+template void Tensor::ORTCopyToCpu<int8_t>(int8_t *data) const;
+template void Tensor::ORTCopyToCpu<float16>(float16 *data) const;
+template void Tensor::ORTCopyToCpu<bfloat16>(bfloat16 *data) const;
+#endif
+
+namespace experimental {
+template <typename T>
+void InternalUtils::CopyFromCpuWithIoStream(paddle_infer::Tensor *t,
+                                            const T *data,
+                                            cudaStream_t stream) {
+  if (t->tensor_ == nullptr) {
+    PADDLE_ENFORCE_EQ(
+        t->name_.empty(),
+        false,
+        common::errors::PreconditionNotMet(
+            "Need to SetName first, so that the corresponding tensor can "
+            "be retrieved."));
+    auto *scope = static_cast<paddle::framework::Scope *>(t->scope_);
+    auto *var = scope->FindVar(t->name_);
+    PADDLE_ENFORCE_NOT_NULL(
+        var,
+        common::errors::PreconditionNotMet(
+            "No tensor called [%s] in the runtime scope", t->name_));
+    auto *tensor = var->GetMutable<phi::DenseTensor>();
+    t->tensor_ = tensor;
+  }
+
+  auto *tensor = static_cast<phi::DenseTensor *>(t->tensor_);
+  PADDLE_ENFORCE_GE(tensor->numel(),
+                    0,
+                    common::errors::PreconditionNotMet(
+                        "You should call Tensor::Reshape(const "
+                        "std::vector<int> &shape)"
+                        "function before copying data from cpu."));
+  size_t ele_size = tensor->numel() * sizeof(T);
+  if (t->place_ == PlaceType::kCPU) {
+    auto *t_data = tensor->mutable_data<T>(phi::CPUPlace());
+    std::memcpy(static_cast<void *>(t_data), data, ele_size);
+  } else if (t->place_ == PlaceType::kGPU) {
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+    phi::GPUPlace gpu_place(t->device_);
+    auto *t_data = tensor->mutable_data<T>(gpu_place);
+    paddle::memory::Copy(gpu_place,
+                         static_cast<void *>(t_data),
+                         phi::CPUPlace(),
+                         data,
+                         ele_size,
+                         stream);
+#else
+    PADDLE_THROW(common::errors::Unavailable(
+        "Can not create tensor with CUDA place because paddle is not compiled "
+        "with CUDA."));
+#endif
+  } else {
+    PADDLE_THROW(common::errors::InvalidArgument(
+        "CopyFromCpuWithIoStream only supports CPU and GPU now."));
+  }
+}
+
+template <typename T>
+void InternalUtils::CopyToCpuWithIoStream(paddle_infer::Tensor *t,
+                                          T *data,
+                                          cudaStream_t stream) {
+  if (t->tensor_ == nullptr) {
+    PADDLE_ENFORCE_EQ(
+        t->name_.empty(),
+        false,
+        common::errors::PreconditionNotMet(
+            "Need to SetName first, so that the corresponding tensor can "
+            "be retrieved."));
+    auto *scope = static_cast<paddle::framework::Scope *>(t->scope_);
+    auto *var = scope->FindVar(t->name_);
+    PADDLE_ENFORCE_NOT_NULL(
+        var,
+        common::errors::PreconditionNotMet(
+            "No tensor called [%s] in the runtime scope", t->name_));
+    auto *tensor = var->GetMutable<phi::DenseTensor>();
+    t->tensor_ = tensor;
+  }
+
+  auto *tensor = static_cast<phi::DenseTensor *>(t->tensor_);
+  auto ele_num = tensor->numel();
+  auto *t_data = tensor->data<T>();
+  auto t_place = tensor->place();
+
+  if (phi::is_cpu_place(t_place)) {
+#ifdef PADDLE_WITH_DNNL
+    if (tensor->layout() == phi::DataLayout::ONEDNN) {
+      phi::DenseTensor out;
+      auto mem_allocation =
+          std::make_shared<paddle::memory::allocation::Allocation>(
+              static_cast<void *>(data), ele_num * sizeof(T), phi::CPUPlace());
+      out.ResetHolder(mem_allocation);
+      phi::funcs::TransDataLayoutFromOneDNN(
+          tensor->layout(),
+          phi::OneDNNContext::tls().get_cur_paddle_data_layout(),
+          *tensor,
+          &out,
+          phi::CPUPlace(),
+          true);
+    } else {
+      std::memcpy(static_cast<void *>(data), t_data, ele_num * sizeof(T));
+    }
+#else
+    std::memcpy(static_cast<void *>(data), t_data, ele_num * sizeof(T));
+#endif
+  } else if (t->place_ == PlaceType::kGPU) {
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+    paddle::memory::Copy(phi::CPUPlace(),
+                         static_cast<void *>(data),
+                         t_place,
+                         t_data,
+                         ele_num * sizeof(T),
+                         stream);
+#else
+    PADDLE_THROW(common::errors::Unavailable(
+        "Can not create tensor with CUDA place because paddle is not compiled "
+        "with CUDA."));
+#endif
+  } else {
+    PADDLE_THROW(common::errors::InvalidArgument(
+        "CopyToCpuWithIoStream only supports CPU and GPU now."));
+  }
+}
+
+template void InternalUtils::CopyFromCpuWithIoStream<double>(
+    paddle_infer::Tensor *t, const double *data, cudaStream_t stream);
+template void InternalUtils::CopyFromCpuWithIoStream<float>(
+    paddle_infer::Tensor *t, const float *data, cudaStream_t stream);
+template void InternalUtils::CopyFromCpuWithIoStream<int64_t>(
+    paddle_infer::Tensor *t, const int64_t *data, cudaStream_t stream);
+template void InternalUtils::CopyFromCpuWithIoStream<int32_t>(
+    paddle_infer::Tensor *t, const int32_t *data, cudaStream_t stream);
+template void InternalUtils::CopyFromCpuWithIoStream<uint8_t>(
+    paddle_infer::Tensor *t, const uint8_t *data, cudaStream_t stream);
+template void InternalUtils::CopyFromCpuWithIoStream<int8_t>(
+    paddle_infer::Tensor *t, const int8_t *data, cudaStream_t stream);
+template void InternalUtils::CopyFromCpuWithIoStream<float16>(
+    paddle_infer::Tensor *t, const float16 *data, cudaStream_t stream);
+template void InternalUtils::CopyFromCpuWithIoStream<bfloat16>(
+    paddle_infer::Tensor *t, const bfloat16 *data, cudaStream_t stream);
+template void InternalUtils::CopyFromCpuWithIoStream<bool>(
+    paddle_infer::Tensor *t, const bool *data, cudaStream_t stream);
+
+template void InternalUtils::CopyToCpuWithIoStream<double>(
+    paddle_infer::Tensor *t, double *data, cudaStream_t stream);
+template void InternalUtils::CopyToCpuWithIoStream<float>(
+    paddle_infer::Tensor *t, float *data, cudaStream_t stream);
+template void InternalUtils::CopyToCpuWithIoStream<int64_t>(
+    paddle_infer::Tensor *t, int64_t *data, cudaStream_t stream);
+template void InternalUtils::CopyToCpuWithIoStream<int32_t>(
+    paddle_infer::Tensor *t, int32_t *data, cudaStream_t stream);
+template void InternalUtils::CopyToCpuWithIoStream<uint8_t>(
+    paddle_infer::Tensor *t, uint8_t *data, cudaStream_t stream);
+template void InternalUtils::CopyToCpuWithIoStream<int8_t>(
+    paddle_infer::Tensor *t, int8_t *data, cudaStream_t stream);
+template void InternalUtils::CopyToCpuWithIoStream<float16>(
+    paddle_infer::Tensor *t, float16 *data, cudaStream_t stream);
+template void InternalUtils::CopyToCpuWithIoStream<bfloat16>(
+    paddle_infer::Tensor *t, bfloat16 *data, cudaStream_t stream);
+template void InternalUtils::CopyToCpuWithIoStream<bool>(
+    paddle_infer::Tensor *t, bool *data, cudaStream_t stream);
+
+}  // namespace experimental
+
+}  // namespace paddle_infer

paddle/fluid/inference/api/details/zero_copy_tensor_dummy.cc

@@ -0,0 +1,52 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/inference/api/paddle_api.h"
+#include "paddle/fluid/inference/api/paddle_infer_declare.h"
+
+namespace paddle_infer {
+
+void Tensor::Reshape(const std::vector<int> &shape) {}
+
+template <typename T>
+T *Tensor::mutable_data(PlaceType place) {
+  return nullptr;
+}
+
+template <typename T>
+T *Tensor::data(PlaceType *place, int *size) const {
+  return nullptr;
+}
+
+template PD_INFER_DECL float *Tensor::data<float>(PlaceType *place,
+                                                  int *size) const;
+template PD_INFER_DECL int64_t *Tensor::data<int64_t>(PlaceType *place,
+                                                      int *size) const;
+template float *Tensor::mutable_data(PlaceType place);
+template int64_t *Tensor::mutable_data(PlaceType place);
+
+template <typename T>
+void *Tensor::FindTensor() const {
+  return nullptr;
+}
+
+std::vector<int> Tensor::shape() const { return {}; }
+
+void Tensor::SetLoD(const std::vector<std::vector<size_t>> &x) {}
+
+std::vector<std::vector<size_t>> Tensor::lod() const {
+  return std::vector<std::vector<size_t>>();
+}
+
+}  // namespace paddle_infer

paddle/fluid/inference/api/details/zero_copy_tensor_test.cc

@@ -0,0 +1,155 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include <algorithm>
+#include <functional>
+#include <limits>
+#include <memory>
+#include <random>
+
+#include "paddle/fluid/framework/data_type.h"
+#include "paddle/fluid/framework/scope.h"
+#include "paddle/fluid/inference/api/helper.h"
+#include "paddle/fluid/inference/api/paddle_tensor.h"
+#include "paddle/phi/common/place.h"
+#include "paddle/phi/core/platform/device_context.h"
+
+namespace paddle_infer {
+
+struct TensorWrapper : public Tensor {
+  TensorWrapper(
+      paddle_infer::PlaceType place,
+      paddle::framework::Scope* scope,
+      const std::map<phi::Place,
+                     std::shared_future<std::unique_ptr<phi::DeviceContext>>>*
+          dev_ctxs,
+      const std::string& name)
+      : Tensor{static_cast<void*>(scope), dev_ctxs} {
+    SetPlace(place, 0 /*device_id*/);
+    SetName(name);
+    input_or_output_ = true;
+  }
+};
+
+std::unique_ptr<Tensor> CreateTensor(paddle_infer::PlaceType place,
+                                     paddle::framework::Scope* scope,
+                                     const std::string& name) {
+  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+  const auto& dev_ctxs = pool.device_contexts();
+  return std::unique_ptr<Tensor>(
+      new TensorWrapper{place, scope, &dev_ctxs, name});
+}
+
+template <typename T>
+struct RandomGenerator {
+  RandomGenerator(
+      double min = static_cast<double>((std::numeric_limits<T>::min)()),
+      double max = static_cast<double>((std::numeric_limits<T>::max)()))
+      : dist_{min, max} {}
+  T operator()() { return static_cast<T>(dist_(random_engine_)); }
+
+ private:
+  std::mt19937_64 random_engine_{std::random_device()()};
+  std::uniform_real_distribution<double> dist_;
+};
+
+template <typename T, template <typename> class G>
+bool FillRandomDataAndCheck(PlaceType place,
+                            size_t length,
+                            G<T>&& generator,
+                            float threshold = 10e-5) {
+  std::vector<T> data_in(length);
+  std::generate(data_in.begin(), data_in.end(), std::forward<G<T>>(generator));
+  paddle::framework::Scope scope;
+  const std::string name{"name"};
+  scope.Var(name);
+  auto tensor = CreateTensor(place, &scope, name);
+  tensor->CopyFromCpu<T>(data_in.data());
+  if (tensor->type() != paddle::inference::ConvertToPaddleDType(
+                            paddle::framework::DataTypeTrait<T>::DataType())) {
+    return false;
+  }
+  std::vector<T> data_out(length);
+  tensor->CopyToCpu<T>(data_out.data());
+  for (size_t i = 0; i < length; ++i) {
+    if (std::abs(data_out[i] - data_out[i]) > threshold) {
+      return false;
+    }
+  }
+  return true;
+}
+
+template <typename T>
+bool SetPlaceAndCheck(PlaceType place, size_t length) {
+  paddle::framework::Scope scope;
+  const std::string name{"name"};
+  const std::vector<std::vector<size_t>> lod{{0, length}};
+  scope.Var(name);
+  auto tensor = CreateTensor(place, &scope, name);
+  std::vector<int> shape{static_cast<int>(length)};
+  tensor->Reshape(shape);
+  tensor->mutable_data<T>(place);
+  tensor->SetLoD(lod);
+
+  PlaceType place_out{PlaceType::kUNK};
+  int length_out{-1};
+  tensor->data<T>(&place_out, &length_out);
+  if (length_out != static_cast<int>(length) || place_out != place) {
+    return false;
+  }
+  if (tensor->name() != name || tensor->lod() != lod) {
+    return false;
+  }
+  return true;
+}
+
+bool FillRandomDataAndCheck(PlaceType place) {
+  const size_t length{RandomGenerator<size_t>{1, 1000}()};
+  VLOG(3) << "FillRandomDataAndCheck: length = " << length;
+  return FillRandomDataAndCheck<float>(
+             place, length, RandomGenerator<float>{}) &&
+         FillRandomDataAndCheck<int64_t>(
+             place, length, RandomGenerator<int64_t>{}) &&
+         FillRandomDataAndCheck<int32_t>(
+             place, length, RandomGenerator<int32_t>{}) &&
+         FillRandomDataAndCheck<uint8_t>(
+             place, length, RandomGenerator<uint8_t>{});
+}
+
+bool SetPlaceAndCheck(PlaceType place) {
+  const size_t length{RandomGenerator<size_t>{1, 1000}()};
+  VLOG(3) << "SetPlaceAndCheck: length = " << length;
+  return SetPlaceAndCheck<float>(place, length) &&
+         SetPlaceAndCheck<int64_t>(place, length) &&
+         SetPlaceAndCheck<int32_t>(place, length) &&
+         SetPlaceAndCheck<uint8_t>(place, length);
+}
+
+TEST(Tensor, FillRandomDataAndCheck) {
+  ASSERT_TRUE(FillRandomDataAndCheck(PlaceType::kCPU));
+  ASSERT_TRUE(SetPlaceAndCheck(PlaceType::kCPU));
+#ifdef PADDLE_WITH_CUDA
+  ASSERT_TRUE(FillRandomDataAndCheck(PlaceType::kGPU));
+  ASSERT_TRUE(SetPlaceAndCheck(PlaceType::kGPU));
+#endif
+#ifdef PADDLE_WITH_XPU
+  ASSERT_TRUE(FillRandomDataAndCheck(PlaceType::kXPU));
+  ASSERT_TRUE(SetPlaceAndCheck(PlaceType::kXPU));
+#endif
+}
+
+}  // namespace paddle_infer

paddle/fluid/inference/api/helper.cc

@@ -0,0 +1,448 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/inference/api/helper.h"
+#include <cstdint>
+
+#include "paddle/common/enforce.h"
+#include "paddle/common/errors.h"
+#include "paddle/common/flags.h"
+#include "paddle/fluid/framework/custom_operator.h"
+#include "paddle/fluid/framework/custom_operator_utils.h"
+#include "paddle/fluid/framework/operator.h"
+#include "paddle/fluid/pir/dialect/operator/ir/op_dialect.h"
+#include "paddle/fluid/pir/dialect/operator/ir/op_type.h"
+#include "paddle/fluid/pir/dialect/operator/utils/utils.h"
+#include "paddle/fluid/pir/drr/src/ir_operation_factory.h"
+#include "paddle/fluid/platform/init.h"
+#include "paddle/phi/api/ext/op_meta_info.h"
+#include "paddle/phi/core/enforce.h"
+#include "paddle/pir/include/core/builtin_attribute.h"
+#include "paddle/pir/include/core/builtin_type.h"
+#include "paddle/pir/include/core/ir_context.h"
+#include "paddle/pir/include/core/operation.h"
+#include "paddle/pir/include/core/value.h"
+
+namespace paddle::inference {
+
+template <>
+std::string to_string<std::vector<float>>(
+    const std::vector<std::vector<float>> &vec) {
+  std::stringstream ss;
+  for (const auto &piece : vec) {
+    ss << to_string(piece) << "\n";
+  }
+  return ss.str();
+}
+
+template <>
+std::string to_string<std::vector<std::vector<float>>>(
+    const std::vector<std::vector<std::vector<float>>> &vec) {
+  std::stringstream ss;
+  for (const auto &line : vec) {
+    for (const auto &rcd : line) {
+      ss << to_string(rcd) << ";\t";
+    }
+    ss << '\n';
+  }
+  return ss.str();
+}
+
+void RegisterAllCustomOperator(bool use_pir) {
+  const auto &meta_info_map = OpMetaInfoMap::Instance().GetMap();
+  for (auto &pair : meta_info_map) {
+    if (use_pir) {
+      auto *custom_dialect =
+          ::pir::IrContext::Instance()
+              ->GetOrRegisterDialect<paddle::dialect::CustomOpDialect>();
+      if (custom_dialect->HasRegistered(pair.first)) {
+        VLOG(3) << "The operator `" << pair.first
+                << "` has been registered. "
+                   "Therefore, we will not repeat the registration here.";
+        continue;
+      }
+      for (const auto &meta_info : pair.second) {
+        VLOG(3) << "register pir custom op: " << pair.first;
+        custom_dialect->RegisterCustomOp(meta_info);
+      }
+
+      std::string pir_op_name =
+          paddle::framework::kCustomDialectPrefix + pair.first;
+      paddle::drr::OperationFactory::Instance().RegisterOperationCreator(
+          pir_op_name,
+          [pair, pir_op_name](
+              const std::vector<::pir::Value> &inputs,
+              const ::pir::AttributeMap &attrs,
+              ::pir::PatternRewriter &rewriter) mutable -> ::pir::Operation * {
+            const auto &meta_inputs =
+                paddle::OpMetaInfoHelper::GetInputs(pair.second[0]);
+            const auto &meta_attrs =
+                paddle::OpMetaInfoHelper::GetAttrs(pair.second[0]);
+            const auto &meta_outputs =
+                paddle::OpMetaInfoHelper::GetOutputs(pair.second[0]);
+            const auto &inplace_map =
+                paddle::OpMetaInfoHelper::GetInplaceMap(pair.second[0]);
+            const auto &inplace_reverse_map =
+                paddle::OpMetaInfoHelper::GetInplaceReverseMap(pair.second[0]);
+            auto infershape_func =
+                OpMetaInfoHelper::GetInferShapeFn(pair.second[0]);
+            auto inferdtype_func =
+                OpMetaInfoHelper::GetInferDtypeFn(pair.second[0]);
+
+            PADDLE_ENFORCE_EQ(
+                meta_inputs.size(),
+                inputs.size(),
+                common::errors::InvalidArgument(
+                    "The number of inputs for the custom operator [%s] given "
+                    "in the Pattern needs to be consistent with the number at "
+                    "implementation time.",
+                    pir_op_name));
+            PADDLE_ENFORCE_EQ(
+                meta_attrs.size(),
+                attrs.size(),
+                common::errors::InvalidArgument(
+                    "The number of attrs for the custom operator [%s] given "
+                    "in the Pattern needs to be consistent with the number at "
+                    "implementation time.",
+                    pir_op_name));
+
+            if (!inplace_map.empty()) {
+              pir_op_name += "_";
+            }
+            ::pir::OperationArgument argument(
+                rewriter.ir_context()->GetRegisteredOpInfo(pir_op_name));
+            argument.attributes = attrs;
+            argument.inputs = inputs;
+
+            std::vector<pir::Type> argument_outputs;
+            std::vector<std::vector<int64_t>> input_shapes;
+            std::vector<DataType> input_dtypes;
+            std::unordered_map<std::string, int> input_name2id_map;
+            std::vector<std::vector<std::vector<int64_t>>> vec_input_shapes;
+            std::vector<std::vector<DataType>> vec_input_dtypes;
+            std::unordered_map<std::string, int> vec_input_name2id_map;
+            std::vector<paddle::any> custom_attrs;
+            int input_index = 0;
+            int vec_input_index = 0;
+
+            for (size_t i = 0; i < meta_inputs.size(); ++i) {
+              const auto &meta_input = meta_inputs.at(i);
+              if (!inputs[i]) {
+                VLOG(6) << "Add un-initialized tensor because the optional "
+                           "input is None.";
+                if (paddle::framework::detail::IsDuplicableVar(meta_input)) {
+                  std::vector<std::vector<int64_t>> vec_input_shape;
+                  std::vector<DataType> vec_input_dtype;
+                  vec_input_shapes.emplace_back(vec_input_shape);
+                  vec_input_dtypes.emplace_back(vec_input_dtype);
+                  vec_input_name2id_map[meta_inputs[i]] = vec_input_index;
+                  vec_input_index++;
+                } else {
+                  std::vector<int64_t> input_shape;
+                  DataType input_dtype = DataType::UNDEFINED;
+                  input_shapes.emplace_back(input_shape);
+                  input_dtypes.emplace_back(input_dtype);
+                  input_name2id_map[meta_inputs[i]] = input_index;
+                  input_index++;
+                }
+                continue;
+              }
+              if (paddle::framework::detail::IsDuplicableVar(meta_input)) {
+                PADDLE_ENFORCE_EQ(
+                    inputs[i].type().isa<::pir::VectorType>(),
+                    true,
+                    common::errors::InvalidArgument(
+                        "The [%d] input of the custom operator [%s] "
+                        "should be a pir::VectorType.",
+                        i,
+                        pir_op_name));
+                std::vector<std::vector<int64_t>> tmp_input_shapes;
+                std::vector<phi::DataType> tmp_input_dtypes;
+                vec_input_name2id_map[meta_inputs[i]] = vec_input_index;
+                vec_input_index++;
+                auto input_value_types =
+                    inputs[i].type().dyn_cast<::pir::VectorType>().data();
+                for (auto &input_value_type : input_value_types) {
+                  auto input_tensor =
+                      input_value_type
+                          .dyn_cast<paddle::dialect::DenseTensorType>();
+                  tmp_input_shapes.push_back(
+                      phi::vectorize(input_tensor.dims()));
+                  tmp_input_dtypes.push_back(
+                      paddle::dialect::TransToPhiDataType(
+                          input_tensor.dtype()));
+                }
+                vec_input_shapes.push_back(tmp_input_shapes);
+                vec_input_dtypes.push_back(tmp_input_dtypes);
+              } else {
+                input_name2id_map[meta_inputs[i]] = input_index;
+                input_index++;
+                auto input_tensor =
+                    inputs[i]
+                        .type()
+                        .dyn_cast<paddle::dialect::DenseTensorType>();
+                input_shapes.push_back(phi::vectorize(input_tensor.dims()));
+                input_dtypes.push_back(
+                    paddle::dialect::TransToPhiDataType(input_tensor.dtype()));
+              }
+            }
+
+            for (const auto &meta_attr : meta_attrs) {
+              auto attr_name_and_type = paddle::ParseAttrStr(meta_attr);
+              auto attr_name = attr_name_and_type[0];
+              auto attr_type = attr_name_and_type[1];
+              PADDLE_ENFORCE_EQ(attrs.count(attr_name),
+                                true,
+                                common::errors::InvalidArgument(
+                                    "The attr [%s] in the custom operator [%s] "
+                                    "specified in the Pattern needs to be "
+                                    "consistent with the implementation",
+                                    attr_name,
+                                    pir_op_name));
+              VLOG(6) << "Custom operator add attrs " << attr_name
+                      << " to CustomOpKernelContext. Attribute type = "
+                      << attr_type;
+              if (attr_type == "bool") {
+                auto bool_attr =
+                    attrs.at(attr_name).dyn_cast<::pir::BoolAttribute>().data();
+                custom_attrs.emplace_back(bool_attr);
+              } else if (attr_type == "int") {
+                int int_attr = attrs.at(attr_name)
+                                   .dyn_cast<::pir::Int32Attribute>()
+                                   .data();
+                custom_attrs.emplace_back(int_attr);
+              } else if (attr_type == "float") {
+                float float_attr = attrs.at(attr_name)
+                                       .dyn_cast<::pir::FloatAttribute>()
+                                       .data();
+                custom_attrs.emplace_back(float_attr);
+              } else if (attr_type == "int64_t") {
+                int64_t long_attr = attrs.at(attr_name)
+                                        .dyn_cast<::pir::Int64Attribute>()
+                                        .data();
+                custom_attrs.emplace_back(long_attr);
+              } else if (attr_type == "std::string") {
+                std::string str_attr = attrs.at(attr_name)
+                                           .dyn_cast<::pir::StrAttribute>()
+                                           .AsString();
+                custom_attrs.emplace_back(str_attr);
+              } else if (attr_type == "std::vector<int>") {
+                auto vec_attr = attrs.at(attr_name)
+                                    .dyn_cast<::pir::ArrayAttribute>()
+                                    .AsVector();
+                std::vector<int> vec_int_attr;
+                for (const auto &int_attr : vec_attr) {
+                  vec_int_attr.push_back(
+                      int_attr.dyn_cast<::pir::Int32Attribute>().data());
+                }
+                custom_attrs.emplace_back(vec_int_attr);
+              } else if (attr_type == "std::vector<float>") {
+                auto vec_attr = attrs.at(attr_name)
+                                    .dyn_cast<::pir::ArrayAttribute>()
+                                    .AsVector();
+                std::vector<float> vec_float_attr;
+                for (const auto &float_attr : vec_attr) {
+                  vec_float_attr.push_back(
+                      float_attr.dyn_cast<::pir::FloatAttribute>().data());
+                }
+                custom_attrs.emplace_back(vec_float_attr);
+              } else if (attr_type == "std::vector<int64_t>") {
+                auto vec_attr = attrs.at(attr_name)
+                                    .dyn_cast<::pir::ArrayAttribute>()
+                                    .AsVector();
+                std::vector<int64_t> vec_long_attr;
+                for (const auto &long_attr : vec_attr) {
+                  vec_long_attr.push_back(
+                      long_attr.dyn_cast<::pir::Int64Attribute>().data());
+                }
+                custom_attrs.emplace_back(vec_long_attr);
+              } else if (attr_type == "std::vector<std::string>") {
+                auto vec_attr = attrs.at(attr_name)
+                                    .dyn_cast<::pir::ArrayAttribute>()
+                                    .AsVector();
+                std::vector<std::string> vec_string_attr;
+                for (const auto &string_attr : vec_attr) {
+                  vec_string_attr.push_back(
+                      string_attr.dyn_cast<::pir::StrAttribute>().AsString());
+                }
+                custom_attrs.emplace_back(vec_string_attr);
+              } else {
+                PADDLE_THROW(common::errors::Unimplemented(
+                    "Unsupported `%s` type value as custom attribute now. "
+                    "Supported data types include `bool`, `int`, `float`, "
+                    "`int64_t`, `std::string`, `std::vector<int>`, "
+                    "`std::vector<float>`, `std::vector<int64_t>`, "
+                    "`std::vector<std::string>`, Please check whether "
+                    "the attribute data type and data type string are matched.",
+                    attr_type));
+              }
+            }
+
+            paddle::framework::CheckDefaultInferShapeDtype(
+                infershape_func, inferdtype_func, pair.second[0]);
+            std::vector<std::vector<int64_t>> output_shapes =
+                paddle::framework::RunInferShape(infershape_func,
+                                                 pair.second[0],
+                                                 input_shapes,
+                                                 input_name2id_map,
+                                                 vec_input_shapes,
+                                                 vec_input_name2id_map,
+                                                 custom_attrs);
+            std::vector<phi::DataType> output_dtypes =
+                paddle::framework::RunInferDtype(inferdtype_func,
+                                                 pair.second[0],
+                                                 input_dtypes,
+                                                 input_name2id_map,
+                                                 vec_input_dtypes,
+                                                 vec_input_name2id_map,
+                                                 custom_attrs);
+
+            size_t all_values_num = 0;
+            // output name -> value num (that output should hold)
+            std::unordered_map<std::string, size_t> output_name2value_num;
+            for (const auto &output : meta_outputs) {
+              if (paddle::framework::detail::IsDuplicableVar(output)) {
+                PADDLE_ENFORCE_NE(inplace_reverse_map.find(output),
+                                  inplace_reverse_map.end(),
+                                  common::errors::InvalidArgument(
+                                      "Only support vector output that is set "
+                                      "for inplace, Please use "
+                                      "`SetInplaceMap` in your output when "
+                                      "registry custom operator."));
+                const auto &input = inplace_reverse_map.at(output);
+                auto index = vec_input_name2id_map[input];
+                auto &vec_input_shape = vec_input_shapes[index];
+                output_name2value_num[output] = vec_input_shape.size();
+              } else {
+                if (inplace_reverse_map.find(output) !=
+                    inplace_reverse_map.end()) {
+                  const auto &input = inplace_reverse_map.at(output);
+                  auto index = input_name2id_map[input];
+                  // input_shapes[index] is dim of tensor, if the dim doesn't
+                  // have element, it must be a optional tensor that is None in
+                  // custom operator
+                  output_name2value_num[output] =
+                      input_shapes[index].empty() ? 0 : 1;
+                } else {
+                  output_name2value_num[output]++;
+                }
+              }
+              all_values_num += output_name2value_num[output];
+            }
+
+            PADDLE_ENFORCE_EQ(output_shapes.size(),
+                              all_values_num,
+                              common::errors::InvalidArgument(
+                                  "The number of output shapes "
+                                  "after running custom operator's "
+                                  "InferShapeFunc is wrong, "
+                                  "expected contains %d Tensors' "
+                                  "shape, but actually contains %d "
+                                  "Tensors' shape",
+                                  all_values_num,
+                                  output_shapes.size()));
+
+            PADDLE_ENFORCE_EQ(output_dtypes.size(),
+                              all_values_num,
+                              common::errors::InvalidArgument(
+                                  "The number of output dtypes "
+                                  "after running custom operator's "
+                                  "InferDtypeFunc is wrong, "
+                                  "expected contains %d Tensors' "
+                                  "dtype, but actually contains %d "
+                                  "Tensors' dtype",
+                                  all_values_num,
+                                  output_dtypes.size()));
+
+            size_t value_index = 0;
+            for (const auto &output : meta_outputs) {
+              auto value_num = output_name2value_num[output];
+              if (value_num == 0) {
+                // Optional value condition
+                pir::Type out_type;
+                argument_outputs.push_back(out_type);
+                continue;
+              }
+              if (paddle::framework::detail::IsDuplicableVar(output)) {
+                auto value_num = output_name2value_num[output];
+                std::vector<pir::Type> out_types;
+                for (size_t j = 0; j < value_num; ++j) {
+                  auto ddims = phi::make_ddim(output_shapes[value_index]);
+                  auto dtype = output_dtypes[value_index];
+                  phi::DataLayout layout{DataLayout::NCHW};
+                  phi::LegacyLoD lod;
+                  out_types.push_back(paddle::dialect::DenseTensorType::get(
+                      pir::IrContext::Instance(),
+                      paddle::dialect::TransToIrDataType(dtype),
+                      ddims,
+                      layout,
+                      lod,
+                      0));
+                  value_index++;
+                }
+                pir::Type out_vector_type =
+                    pir::VectorType::get(pir::IrContext::Instance(), out_types);
+                argument_outputs.push_back(out_vector_type);
+              } else {
+                auto ddims = phi::make_ddim(output_shapes[value_index]);
+                auto dtype = output_dtypes[value_index];
+                phi::DataLayout layout{DataLayout::NCHW};
+                phi::LegacyLoD lod;
+                auto out_type = paddle::dialect::DenseTensorType::get(
+                    pir::IrContext::Instance(),
+                    paddle::dialect::TransToIrDataType(dtype),
+                    ddims,
+                    layout,
+                    lod,
+                    0);
+                argument_outputs.push_back(out_type);
+                value_index++;
+              }
+            }
+
+            argument.AddOutputs(argument_outputs.begin(),
+                                argument_outputs.end());
+            ::pir::PassStopGradientsDefaultly(argument);
+            return rewriter.Build(std::move(argument));
+          });
+    }
+    const auto &all_op_kernels{framework::OperatorWithKernel::AllOpKernels()};
+    if (all_op_kernels.find(pair.first) == all_op_kernels.end()) {
+      framework::RegisterOperatorWithMetaInfo(pair.second);
+    } else {
+      VLOG(3) << "The operator `" << pair.first
+              << "` has been registered. Therefore, we will not repeat the "
+                 "registration here.";
+    }
+  }
+}
+
+void InitGflagsFromEnv() {
+  // support set gflags from environment.
+  std::vector<std::string> gflags;
+  const phi::ExportedFlagInfoMap &env_map = phi::GetExportedFlagInfoMap();
+  std::ostringstream os;
+  for (auto &pair : env_map) {
+    os << pair.second.name << ",";
+  }
+  std::string tryfromenv_str = os.str();
+  if (!tryfromenv_str.empty()) {
+    tryfromenv_str.pop_back();
+    tryfromenv_str = "--tryfromenv=" + tryfromenv_str;
+    gflags.push_back(tryfromenv_str);
+  }
+  framework::InitGflags(gflags);
+}
+
+}  // namespace paddle::inference

paddle/fluid/inference/api/helper.h

@@ -0,0 +1,518 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include <glog/logging.h>
+#include <sys/stat.h>
+#ifdef _WIN32
+#include <windows.h>
+#include <codecvt>
+#endif
+#include <fstream>
+#if !defined(_WIN32)
+#include <sys/time.h>
+#endif
+#include <algorithm>
+#include <chrono>  // NOLINT
+#include <functional>
+#include <iterator>
+#include <numeric>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "paddle/fluid/framework/data_type.h"
+#include "paddle/fluid/inference/api/paddle_analysis_config.h"
+#include "paddle/fluid/inference/api/paddle_inference_api.h"
+#include "paddle/fluid/platform/enforce.h"
+#include "paddle/phi/common/place.h"
+#include "paddle/phi/common/port.h"
+#include "paddle/phi/core/memory/stats.h"
+#include "paddle/utils/string/printf.h"
+
+extern std::string paddle::framework::DataTypeToString(
+    const framework::proto::VarType::Type type);
+
+namespace paddle {
+namespace inference {
+
+template <typename T>
+constexpr PaddleDType PaddleTensorGetDType();
+
+template <>
+constexpr PaddleDType PaddleTensorGetDType<int32_t>() {
+  return PaddleDType::INT32;
+}
+
+template <>
+constexpr PaddleDType PaddleTensorGetDType<int64_t>() {
+  return PaddleDType::INT64;
+}
+
+template <>
+constexpr PaddleDType PaddleTensorGetDType<float>() {
+  return PaddleDType::FLOAT32;
+}
+
+inline PaddleDType ConvertToPaddleDType(
+    paddle::framework::proto::VarType::Type type) {
+  if (type == paddle::framework::proto::VarType::FP32) {
+    return PaddleDType::FLOAT32;
+  } else if (type == paddle::framework::proto::VarType::INT64) {
+    return PaddleDType::INT64;
+  } else if (type == paddle::framework::proto::VarType::INT32) {
+    return PaddleDType::INT32;
+  } else if (type == paddle::framework::proto::VarType::UINT8) {
+    return PaddleDType::UINT8;
+  } else {
+    PADDLE_THROW(common::errors::Unimplemented(
+        "The paddle dtype convert function only supports FLOAT32, INT64, INT32 "
+        "and UINT8 now. But "
+        "we get %d here.",
+        static_cast<int>(type)));
+    return PaddleDType::FLOAT32;
+  }
+}
+
+inline bool IsFloatVar(framework::proto::VarType::Type t) {
+  if (t == framework::proto::VarType::FP16 ||
+      t == framework::proto::VarType::FP32 ||
+      t == framework::proto::VarType::FP64 ||
+      t == framework::proto::VarType::BF16)
+    return true;
+  return false;
+}
+
+using paddle::framework::DataTypeToString;
+
+// Timer for timer
+class Timer {
+ public:
+  std::chrono::high_resolution_clock::time_point start;
+  std::chrono::high_resolution_clock::time_point startu;
+
+  void tic() { start = std::chrono::high_resolution_clock::now(); }
+  double toc() {
+    startu = std::chrono::high_resolution_clock::now();
+    std::chrono::duration<double> time_span =
+        std::chrono::duration_cast<std::chrono::duration<double>>(startu -
+                                                                  start);
+    double used_time_ms = static_cast<double>(time_span.count()) * 1000.0;
+    return used_time_ms;
+  }
+};
+
+static int GetUniqueId() {
+  static int id = 0;
+  return id++;
+}
+
+static void split(const std::string &str,
+                  char sep,
+                  std::vector<std::string> *pieces,
+                  bool ignore_null = true) {
+  pieces->clear();
+  if (str.empty()) {
+    if (!ignore_null) {
+      pieces->push_back(str);
+    }
+    return;
+  }
+  size_t pos = 0;
+  size_t next = str.find(sep, pos);
+  while (next != std::string::npos) {
+    pieces->push_back(str.substr(pos, next - pos));
+    pos = next + 1;
+    next = str.find(sep, pos);
+  }
+  if (!str.substr(pos).empty()) {
+    pieces->push_back(str.substr(pos));
+  }
+}
+
+template <typename T>
+static T convert(const std::string &item,
+                 std::function<T(const std::string &item)> func) {
+  T res;
+  try {
+    res = func(item);
+  } catch (std::invalid_argument &e) {
+    std::string message =
+        "invalid_argument exception when try to convert : " + item;
+    LOG(ERROR) << message;
+    PADDLE_THROW(common::errors::InvalidArgument(
+        "invalid_argument exception when try to convert %s.", item));
+  } catch (std::out_of_range &e) {
+    std::string message =
+        "out_of_range exception when try to convert : " + item;
+    LOG(ERROR) << message;
+    PADDLE_THROW(common::errors::InvalidArgument(
+        "out_of_range exception when try to convert %s.", item));
+  } catch (...) {
+    std::string message = "unexpected exception when try to convert " + item;
+    LOG(ERROR) << message;
+    PADDLE_THROW(common::errors::InvalidArgument(
+        "unexpected exception when try to convert %s.", item));
+  }
+  return res;
+}
+
+static void split_to_float(const std::string &str,
+                           char sep,
+                           std::vector<float> *fs) {
+  std::vector<std::string> pieces;
+  split(str, sep, &pieces);
+  std::transform(pieces.begin(),
+                 pieces.end(),
+                 std::back_inserter(*fs),
+                 [](const std::string &v) {
+                   return convert<float>(v, [](const std::string &item) {
+                     return std::stof(item);
+                   });
+                 });
+}
+static void split_to_int64(const std::string &str,
+                           char sep,
+                           std::vector<int64_t> *is) {
+  std::vector<std::string> pieces;
+  split(str, sep, &pieces);
+  std::transform(pieces.begin(),
+                 pieces.end(),
+                 std::back_inserter(*is),
+                 [](const std::string &v) {
+                   return convert<int64_t>(v, [](const std::string &item) {
+                     return std::stoll(item);
+                   });
+                 });
+}
+static void split_to_int(const std::string &str,
+                         char sep,
+                         std::vector<int> *is) {
+  std::vector<std::string> pieces;
+  split(str, sep, &pieces);
+  std::transform(pieces.begin(),
+                 pieces.end(),
+                 std::back_inserter(*is),
+                 [](const std::string &v) {
+                   return convert<int>(v, [](const std::string &item) {
+                     return std::stoi(item);
+                   });
+                 });
+}
+template <typename T>
+std::string to_string(const std::vector<T> &vec) {
+  std::stringstream ss;
+  for (const auto &c : vec) {
+    ss << c << " ";
+  }
+  return ss.str();
+}
+template <>
+std::string to_string<std::vector<float>>(
+    const std::vector<std::vector<float>> &vec);
+
+template <>
+std::string to_string<std::vector<std::vector<float>>>(
+    const std::vector<std::vector<std::vector<float>>> &vec);
+
+template <typename T>
+int VecReduceToInt(const std::vector<T> &v) {
+  return std::accumulate(v.begin(), v.end(), 1, [](T a, T b) { return a * b; });
+}
+
+template <typename T>
+void CheckAssignedData(const std::vector<std::vector<T>> &data,
+                       const int num_elems) {
+  int num = 0;
+  for (auto it = data.begin(); it != data.end(); ++it) {
+    num += (*it).size();
+  }
+  PADDLE_ENFORCE_EQ(
+      num,
+      num_elems,
+      common::errors::OutOfRange(
+          "The number of elements out of bounds. "
+          "Expected number of elements = %d. But received %d. Suggested Fix: "
+          "If the tensor is expected to assign %d elements, check the number "
+          "of elements of your 'infer_data'.",
+          num_elems,
+          num,
+          num_elems));
+}
+
+template <typename T>
+static void TensorAssignData(PaddleTensor *tensor,
+                             const std::vector<std::vector<T>> &data) {
+  // Assign buffer
+  int num_elems = VecReduceToInt(tensor->shape);
+  CheckAssignedData(data, num_elems);
+  tensor->data.Resize(sizeof(T) * num_elems);
+  int c = 0;
+  for (const auto &f : data) {
+    for (T v : f) {
+      static_cast<T *>(tensor->data.data())[c++] = v;
+    }
+  }
+}
+
+template <typename T>
+static void TensorAssignData(PaddleTensor *tensor,
+                             const std::vector<std::vector<T>> &data,
+                             const std::vector<size_t> &lod) {
+  int size = lod[lod.size() - 1];
+  tensor->shape.assign({size, 1});
+  tensor->lod.assign({lod});
+  TensorAssignData(tensor, data);
+}
+
+template <typename T>
+static void ZeroCopyTensorAssignData(ZeroCopyTensor *tensor,
+                                     const std::vector<std::vector<T>> &data) {
+  auto *ptr = tensor->mutable_data<T>(PaddlePlace::kCPU);
+  int c = 0;
+  for (const auto &f : data) {
+    for (T v : f) {
+      ptr[c++] = v;
+    }
+  }
+}
+
+template <typename T>
+static void ZeroCopyTensorAssignData(ZeroCopyTensor *tensor,
+                                     const PaddleBuf &data) {
+  auto *ptr = tensor->mutable_data<T>(PaddlePlace::kCPU);
+  for (size_t i = 0; i < data.length() / sizeof(T); i++) {
+    ptr[i] = *(reinterpret_cast<T *>(data.data()) + i);
+  }
+}
+
+static bool CompareTensor(const PaddleTensor &a, const PaddleTensor &b) {
+  if (a.dtype != b.dtype) {
+    LOG(ERROR) << "dtype not match";
+    return false;
+  }
+
+  if (a.lod.size() != b.lod.size()) {
+    LOG(ERROR) << "lod not match";
+    return false;
+  }
+  for (size_t i = 0; i < a.lod.size(); i++) {
+    if (a.lod[i].size() != b.lod[i].size()) {
+      LOG(ERROR) << "lod not match";
+      return false;
+    }
+    for (size_t j = 0; j < a.lod[i].size(); j++) {
+      if (a.lod[i][j] != b.lod[i][j]) {
+        LOG(ERROR) << "lod not match";
+        return false;
+      }
+    }
+  }
+
+  if (a.shape.size() != b.shape.size()) {
+    LOG(INFO) << "shape not match";
+    return false;
+  }
+  for (size_t i = 0; i < a.shape.size(); i++) {
+    if (a.shape[i] != b.shape[i]) {
+      LOG(ERROR) << "shape not match";
+      return false;
+    }
+  }
+
+  auto *adata = static_cast<float *>(a.data.data());
+  auto *bdata = static_cast<float *>(b.data.data());
+  for (int i = 0; i < VecReduceToInt(a.shape); i++) {
+    if (adata[i] != bdata[i]) {
+      LOG(ERROR) << "data not match";
+      return false;
+    }
+  }
+  return true;
+}
+
+static std::string DescribeTensor(const PaddleTensor &tensor,
+                                  int max_num_of_data UNUSED = 15) {
+  std::stringstream os;
+  os << "Tensor [" << tensor.name << "]\n";
+  os << " - type: ";
+  switch (tensor.dtype) {
+    case PaddleDType::FLOAT32:
+      os << "float32";
+      break;
+    case PaddleDType::INT64:
+      os << "int64";
+      break;
+    case PaddleDType::INT32:
+      os << "int32";
+      break;
+    default:
+      os << "unset";
+  }
+  os << '\n';
+
+  os << " - shape: " << to_string(tensor.shape) << '\n';
+  os << " - lod: ";
+  for (auto &l : tensor.lod) {
+    os << to_string(l) << "; ";
+  }
+  os << "\n";
+  os << " - memory length: " << tensor.data.length();
+  os << "\n";
+
+  os << " - data: ";
+  int dim = VecReduceToInt(tensor.shape);
+  float *pdata = static_cast<float *>(tensor.data.data());
+  for (int i = 0; i < dim; i++) {
+    os << pdata[i] << " ";
+  }
+  os << '\n';
+  return os.str();
+}
+
+static std::string DescribeZeroCopyTensor(const ZeroCopyTensor &tensor) {
+  std::stringstream os;
+  os << "Tensor [" << tensor.name() << "]\n";
+
+  os << " - shape: " << to_string(tensor.shape()) << '\n';
+  os << " - lod: ";
+  for (auto &l : tensor.lod()) {
+    os << to_string(l) << "; ";
+  }
+  os << "\n";
+  PaddlePlace place;
+  int size;
+  const auto *data = tensor.data<float>(&place, &size);
+  os << " - numel: " << size;
+  os << "\n";
+  os << " - data: ";
+  for (int i = 0; i < size; i++) {
+    os << data[i] << " ";
+  }
+  return os.str();
+}
+
+static void PrintTime(int batch_size,
+                      int repeat,
+                      int num_threads,
+                      int tid,
+                      double batch_latency,
+                      int epoch = 1,
+                      const framework::proto::VarType::Type data_type =
+                          framework::proto::VarType::FP32) {
+  PADDLE_ENFORCE_GT(
+      batch_size,
+      0,
+      common::errors::InvalidArgument("Non-positive batch size."));
+  double sample_latency = batch_latency / batch_size;
+  LOG(INFO) << "====== threads: " << num_threads << ", thread id: " << tid
+            << " ======";
+  LOG(INFO) << "====== batch size: " << batch_size << ", iterations: " << epoch
+            << ", repetitions: " << repeat << " ======";
+  LOG(INFO) << "====== batch latency: " << batch_latency
+            << "ms, number of samples: " << batch_size * epoch
+            << ", sample latency: " << sample_latency
+            << "ms, fps: " << 1000.f / sample_latency
+            << ", data type: " << DataTypeToString(data_type) << " ======";
+}
+
+static bool IsFileExists(const std::string &path) {
+#ifdef _WIN32
+  std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
+  std::wstring wpath = converter.from_bytes(path);
+  std::ifstream file(wpath, std::ios::binary);
+#else
+  std::ifstream file(path, std::ios::binary);
+#endif
+  bool exists = file.is_open();
+  file.close();
+  return exists;
+}
+
+static bool IsDirectory(const std::string &path) {
+#ifdef _WIN32
+  std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
+  std::wstring wpath = converter.from_bytes(path);
+  DWORD attr = GetFileAttributesW(wpath.c_str());
+  if (attr == INVALID_FILE_ATTRIBUTES) return false;
+  return (attr & FILE_ATTRIBUTE_DIRECTORY);
+#else
+  struct stat info;
+  if (stat(path.c_str(), &info) != 0) {
+    return false;
+  } else if (info.st_mode & S_IFDIR) {
+    return true;
+  }
+  return false;
+#endif
+}
+
+void RegisterAllCustomOperator(bool use_pir);
+
+void InitGflagsFromEnv();
+
+static inline double ToMegaBytes(size_t bytes) {
+  return static_cast<double>(bytes) / (1 << 20);
+}
+
+static inline void DisplayMemoryInfo(phi::Place place,
+                                     const std::string &hint) {
+#ifdef PADDLE_WITH_CUDA
+  // size_t free, total;
+  // cudaSetDevice(place.GetDeviceId());
+  // cudaMemGetInfo(&free, &total);
+  // VLOG(1) << "[" << ToMegaBytes(total - free) << "MB/" << ToMegaBytes(total)
+  // << "MB]";
+
+  VLOG(1) << hint << " : [gpu current allocated memory: "
+          << ToMegaBytes(paddle::memory::DeviceMemoryStatCurrentValue(
+                 "Allocated", place.GetDeviceId()))
+          << "MB], [gpu current reserved memory: "
+          << ToMegaBytes(paddle::memory::DeviceMemoryStatCurrentValue(
+                 "Reserved", place.GetDeviceId()))
+          << "MB], [gpu peak allocated memory: "
+          << ToMegaBytes(paddle::memory::DeviceMemoryStatPeakValue(
+                 "Allocated", place.GetDeviceId()))
+          << "MB], [gpu peak reserved memory: "
+          << ToMegaBytes(paddle::memory::DeviceMemoryStatPeakValue(
+                 "Reserved", place.GetDeviceId()))
+          << "MB]";
+#endif
+  VLOG(1)
+      << hint << " : [cpu current allocated memory: "
+      << ToMegaBytes(paddle::memory::HostMemoryStatCurrentValue("Allocated", 0))
+      << "MB], [cpu current reserved memory: "
+      << ToMegaBytes(paddle::memory::HostMemoryStatCurrentValue("Reserved", 0))
+      << "MB], [cpu peak allocated memory: "
+      << ToMegaBytes(paddle::memory::HostMemoryStatPeakValue("Allocated", 0))
+      << "MB], [cpu peak reserved memory: "
+      << ToMegaBytes(paddle::memory::HostMemoryStatPeakValue("Reserved", 0))
+      << "MB]";
+}
+
+static std::string Precision2String(AnalysisConfig::Precision precision) {
+  if (precision == AnalysisConfig::Precision::kFloat32)
+    return "fp32";
+  else if (precision == AnalysisConfig::Precision::kHalf)
+    return "fp16";
+  else if (precision == AnalysisConfig::Precision::kInt8)
+    return "int8";
+  else if (precision == AnalysisConfig::Precision::kBf16)
+    return "bf16";
+  else
+    return "none";
+}
+
+}  // namespace inference
+}  // namespace paddle

paddle/fluid/inference/api/infer_context.cc

@@ -0,0 +1,290 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/inference/api/infer_context.h"
+#include "paddle/fluid/platform/enforce.h"
+#include "paddle/phi/core/dense_tensor.h"
+#ifdef PADDLE_WITH_XPU
+#include "xpu/runtime.h"
+#endif
+#include "glog/logging.h"
+
+namespace paddle {
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+InferGPUContext::InferGPUContext(const phi::Place& place)
+    : phi::GPUContext(place, false) {}
+#endif
+
+#ifdef PADDLE_WITH_XPU
+InferXPUContext::InferXPUContext(const phi::Place& place, int context_gm_size)
+    : phi::XPUContext(place) {
+  if (context_gm_size >= 0) {
+    x_context()->set_option("XPUAPI_DEFAULT_SIZE",
+                            std::to_string(context_gm_size).c_str());
+  } else {
+    x_context()->set_option("XPUAPI_DEFAULT_SIZE", "");
+  }
+}
+
+void* InferXPUContext::Alloc(phi::TensorBase* tensor,
+                             phi::DataType dtype,
+                             size_t requested_size,
+                             bool pinned,
+                             bool fake_alloc) const {
+  size_t size = tensor->numel() * phi::SizeOf(tensor->dtype());
+  if (l3_autotune_size_ > 0 && holder_map_.empty()) {
+    void* data_ptr =
+        DeviceContext::Alloc(tensor, dtype, requested_size, pinned, fake_alloc);
+    phi::XPUL3CacheBlock* l3_block = nullptr;
+    phi::Allocation* holder =
+        reinterpret_cast<phi::DenseTensor*>(tensor)->Holder().get();
+    if (holder_l3_blocks_.count(holder) == 0) {
+      l3_block = new phi::XPUL3CacheBlock();
+      holder_l3_blocks_[holder] = l3_block;
+      l3_blocks_.push_back(l3_block);
+    } else {
+      l3_block = holder_l3_blocks_[holder];
+    }
+    l3_block->Record(size);
+    return data_ptr;
+  } else if (l3_autotune_size_ > 0 && !holder_map_.empty()) {
+    phi::Allocation* holder =
+        reinterpret_cast<phi::DenseTensor*>(tensor)->Holder().get();
+    auto holder_iter = holder_map_.find(holder);
+    if (holder_iter != holder_map_.end()) {
+      auto& holder_pair = holder_iter->second;
+      auto* swap_holder = holder_pair.first;
+      bool& swap_holder_is_l3 = holder_pair.second;
+      if (swap_holder_is_l3 && swap_holder->size() >= size) {
+        swap(*holder, *swap_holder);
+        swap_holder_is_l3 = false;
+      } else if (!swap_holder_is_l3 && holder->size() < size) {
+        swap(*holder, *swap_holder);
+        swap_holder_is_l3 = true;
+      }
+    }
+    return DeviceContext::Alloc(
+        tensor, dtype, requested_size, pinned, fake_alloc);
+  } else {
+    return DeviceContext::Alloc(
+        tensor, dtype, requested_size, pinned, fake_alloc);
+  }
+}
+
+void InferXPUContext::SetXContext(xpu::Context* x_context) {
+  auto* old_x_context = this->x_context();
+  if (old_x_context != x_context) {
+    if (l3_owned_ && l3_size_ > 0 &&
+        (x_context->_l3_mgr.get_size() != l3_size_ ||
+         x_context->_l3_mgr.get_ptr() != l3_ptr_)) {
+      xpu_free(l3_ptr_);
+    }
+    old_x_context->_l3_mgr.set(nullptr, 0);
+    l3_size_ = x_context->_l3_mgr.get_size();
+    l3_ptr_ = x_context->_l3_mgr.get_ptr();
+    l3_owned_ = false;
+    phi::XPUContext::SetXContext(x_context);
+  }
+}
+
+void InferXPUContext::SetL3Info(size_t l3_size,
+                                void* l3_ptr,
+                                size_t l3_autotune_size,
+                                const phi::Place& place) {
+  phi::backends::xpu::XPUDeviceGuard guard(place.GetDeviceId());
+  if (l3_ptr == nullptr) {
+    if (l3_size_ != l3_size) {
+      if (l3_owned_) {
+        xpu_free(l3_ptr_);
+      }
+      if (l3_size > 0) {
+        xpu_malloc(&l3_ptr_, l3_size, XPU_MEM_L3);
+        if (l3_ptr_ != nullptr) {
+          VLOG(3) << "remalloc l3(" << l3_size << ") success.";
+          l3_size_ = l3_size;
+          l3_owned_ = true;
+          l3_autotune_size_ = l3_autotune_size;
+        } else {
+          VLOG(3) << "malloc l3(" << l3_size << ") failed. No l3 will be used.";
+          l3_size_ = 0;
+          l3_owned_ = false;
+          l3_autotune_size_ = 0;
+        }
+      }
+    }
+  } else {
+    if (l3_owned_) {
+      xpu_free(l3_ptr_);
+    }
+    l3_ptr_ = l3_ptr;
+    l3_size_ = l3_size;
+    l3_autotune_size_ = l3_autotune_size;
+  }
+  if (l3_autotune_size_ == 0) {
+    x_context()->_l3_mgr.set(l3_ptr_, l3_size_);
+  }
+}
+
+void InferXPUContext::SetConvAutotuneInfo(std::string conv_autotune_file,
+                                          int conv_autotune_level,
+                                          bool conv_autotune_file_writeback,
+                                          const phi::Place& place) {
+  phi::backends::xpu::XPUDeviceGuard guard(place.GetDeviceId());
+
+  VLOG(5) << "XPU conv autotune level:" << conv_autotune_level;
+  VLOG(5) << "XPU conv autotune file:" << conv_autotune_file;
+  VLOG(5) << "XPU conv autotune file writeback:"
+          << conv_autotune_file_writeback;
+
+  if (!conv_autotune_file.empty()) {
+    int ret;
+    ret = x_context()->set_option("XPU_CONV_AUTOTUNE_FILE",
+                                  conv_autotune_file.c_str());
+    PADDLE_ENFORCE_EQ(
+        ret,
+        0,
+        common::errors::Unavailable("Failed to set XPU conv autotune file %s.",
+                                    conv_autotune_file));
+  }
+  if (conv_autotune_level > 0) {
+    int ret;
+    ret = x_context()->set_option(
+        "XPU_CONV_AUTOTUNE", (std::to_string(conv_autotune_level)).c_str());
+    PADDLE_ENFORCE_EQ(
+        ret,
+        0,
+        common::errors::Unavailable("Failed to set XPU conv autotune  %d.",
+                                    conv_autotune_level));
+  }
+  if (conv_autotune_file_writeback) {
+    int ret;
+    ret = x_context()->set_option(
+        "XPU_AUTOTUNE_WRITEBACK",
+        (std::to_string(conv_autotune_file_writeback)).c_str());
+    PADDLE_ENFORCE_EQ(ret,
+                      0,
+                      common::errors::Unavailable(
+                          "Failed to set XPU conv autotune writeback %d.",
+                          conv_autotune_file_writeback));
+  }
+}
+void InferXPUContext::SetContextOption(const char* name, const char* value) {
+  phi::backends::xpu::XPUDeviceGuard guard(GetPlace().GetDeviceId());
+  VLOG(5) << "XPU Set Option name:" << name << " value:" << value;
+  int ret;
+  ret = x_context()->set_option(name, value);
+  PADDLE_ENFORCE_EQ(
+      ret,
+      0,
+      common::errors::Unavailable("Failed to set XPU option %s.", name));
+}
+
+void InferXPUContext::SetFcAutotuneInfo(std::string fc_autotune_file,
+                                        int fc_autotune_level,
+                                        bool fc_autotune_file_writeback,
+                                        const phi::Place& place) {
+  phi::backends::xpu::XPUDeviceGuard guard(place.GetDeviceId());
+
+  VLOG(5) << "XPU fc autotune level:" << fc_autotune_level;
+  VLOG(5) << "XPU fc autotune file:" << fc_autotune_file;
+  VLOG(5) << "XPU fc autotune file writeback:" << fc_autotune_file_writeback;
+
+  if (!fc_autotune_file.empty()) {
+    int ret;
+    ret = x_context()->set_option("XPU_FC_AUTOTUNE_FILE",
+                                  fc_autotune_file.c_str());
+    PADDLE_ENFORCE_EQ(
+        ret,
+        0,
+        common::errors::Unavailable("Failed to set XPU fc autotune file %s.",
+                                    fc_autotune_file));
+  }
+  if (fc_autotune_level > 0) {
+    int ret;
+    ret = x_context()->set_option("XPU_FC_AUTOTUNE",
+                                  (std::to_string(fc_autotune_level)).c_str());
+    PADDLE_ENFORCE_EQ(
+        ret,
+        0,
+        common::errors::Unavailable("Failed to set XPU fc autotune  %d.",
+                                    fc_autotune_level));
+  }
+  if (fc_autotune_file_writeback) {
+    int ret;
+    ret = x_context()->set_option(
+        "XPU_FC_AUTOTUNE_WRITEBACK",
+        (std::to_string(fc_autotune_file_writeback)).c_str());
+    PADDLE_ENFORCE_EQ(ret,
+                      0,
+                      common::errors::Unavailable(
+                          "Failed to set XPU fc autotune writeback %d.",
+                          fc_autotune_file_writeback));
+  }
+}
+
+void InferXPUContext::L3CacheAutotune() {
+  if (l3_autotune_size_ == 0) return;
+  if (holder_map_.empty()) {
+    bool ret = l3_plan_.RunAutotune(l3_blocks_, l3_size_);
+    if (!ret) {
+      return;
+    }
+    auto* plan = l3_plan_.plan();
+    int8_t* cur_l3_ptr = reinterpret_cast<int8_t*>(l3_ptr_);
+    for (size_t i = 0; i < l3_blocks_.size(); i++) {
+      size_t block_size = plan->at(i);
+      if (block_size > 0) {
+        l3_blocks_[i]->Set(cur_l3_ptr, block_size);
+        cur_l3_ptr += block_size;
+      }
+    }
+    x_context()->_l3_mgr.set(
+        reinterpret_cast<int8_t*>(l3_ptr_) + l3_size_ - plan->back(),
+        plan->back());
+
+    for (auto holder_l3_block : holder_l3_blocks_) {
+      auto* l3_block = holder_l3_block.second;
+      if (l3_block->size() > 0) {
+        auto* holder = holder_l3_block.first;
+        auto place = holder->place();
+        phi::Allocation* l3_holder =
+            new phi::Allocation(l3_block->data(), l3_block->size(), place);
+        holder_map_[holder] = std::make_pair(l3_holder, true);
+
+        if (output_holder_set_.find(holder) != output_holder_set_.end()) {
+          VLOG(4) << "Insert output tensor's l3 holder:" << l3_holder->ptr();
+          SetOutHolder(l3_holder);
+        }
+      }
+    }
+  } else {
+    for (auto& holders : holder_map_) {
+      auto* holder = holders.first;
+      auto& holder_pair = holders.second;
+      if (!holder_pair.second &&
+          output_holder_set_.find(holder) == output_holder_set_.end()) {
+        swap(*holder, *(holder_pair.first));
+        holder_pair.second = true;
+      }
+    }
+  }
+}
+
+void InferXPUContext::SetOutHolder(phi::Allocation* holder) {
+  output_holder_set_.insert(holder);
+}
+#endif
+
+}  // namespace paddle

paddle/fluid/inference/api/infer_context.h

@@ -0,0 +1,102 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+#include "paddle/phi/backends/all_context.h"
+#include "paddle/phi/common/place.h"
+#ifdef PADDLE_WITH_XPU
+#include "paddle/phi/backends/xpu/xpu_l3_strategy.h"
+#endif
+#include <unordered_set>
+
+namespace paddle {
+
+class InferCPUContext : public phi::CPUContext {
+ public:
+  using phi::CPUContext::SetEigenDevice;
+};
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+class InferGPUContext : public phi::GPUContext {
+ public:
+  explicit InferGPUContext(const phi::Place& place);
+  using phi::GPUContext::SetBlasHandle;
+  using phi::GPUContext::SetBlasTensorCoreHandle;
+  using phi::GPUContext::SetBlasTF32Handle;
+  using phi::GPUContext::SetDnnHandle;
+  using phi::GPUContext::SetEigenDevice;
+  using phi::GPUContext::SetSolverHandle;
+  using phi::GPUContext::SetSparseHandle;
+  using phi::GPUContext::SetStream;
+  // using phi::GPUContext::SetDnnWorkspaceHandle;
+  using phi::GPUContext::SetComputeCapability;
+  using phi::GPUContext::SetDriverVersion;
+  using phi::GPUContext::SetMaxGridDimSize;
+  using phi::GPUContext::SetMaxThreadsPerBlock;
+  using phi::GPUContext::SetMaxThreadsPerMultiProcessor;
+  using phi::GPUContext::SetMultiProcessors;
+  using phi::GPUContext::SetRuntimeVersion;
+};
+#endif
+
+#ifdef PADDLE_WITH_XPU
+class InferXPUContext : public phi::XPUContext {
+ public:
+  explicit InferXPUContext(const phi::Place& place, int context_gm_size = -1);
+
+  void* Alloc(phi::TensorBase* tensor,
+              phi::DataType dtype,
+              size_t requested_size = 0,
+              bool pinned = false,
+              bool fake_alloc = false) const override;
+
+  void SetXContext(xpu::Context* x_context);
+
+  void SetL3Info(size_t l3_size,
+                 void* l3_ptr,
+                 size_t l3_autotune_size,
+                 const phi::Place& place);
+
+  void L3CacheAutotune();
+
+  void SetConvAutotuneInfo(std::string conv_autotune_file,
+                           int conv_autotune_level,
+                           bool conv_autotune_file_writeback,
+                           const phi::Place& place);
+
+  void SetFcAutotuneInfo(std::string fc_autotune_file,
+                         int fc_autotune_level,
+                         bool fc_autotune_file_writeback,
+                         const phi::Place& place);
+  void SetContextOption(const char* name, const char* value);
+
+  void SetOutHolder(phi::Allocation* holder);
+
+ private:
+  size_t l3_size_{0};
+  void* l3_ptr_{nullptr};
+  bool l3_owned_{false};
+  size_t l3_autotune_size_{0};
+  mutable std::vector<phi::XPUL3CacheBlock*> l3_blocks_;
+  mutable std::unordered_map<phi::Allocation*, phi::XPUL3CacheBlock*>
+      holder_l3_blocks_;
+  mutable std::unordered_map<phi::Allocation*,
+                             std::pair<phi::Allocation*, bool>>
+      holder_map_;
+
+  mutable std::unordered_set<phi::Allocation*> output_holder_set_;
+  phi::XPUL3Planner l3_plan_;
+};
+#endif
+}  // namespace paddle

paddle/fluid/inference/api/onnxruntime_predictor.cc

@@ -0,0 +1,389 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/inference/api/onnxruntime_predictor.h"
+
+#include <glog/logging.h>
+
+#include <algorithm>
+#include <fstream>
+#include <memory>
+#include <set>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "paddle/fluid/framework/scope.h"
+#include "paddle/fluid/framework/var_type_traits.h"
+#include "paddle/fluid/framework/variable_helper.h"
+#include "paddle/fluid/inference/analysis/helper.h"
+#include "paddle/fluid/inference/api/helper.h"
+#include "paddle/fluid/inference/api/paddle_inference_api.h"
+#include "paddle/fluid/inference/api/paddle_inference_pass.h"
+#include "paddle/fluid/inference/utils/io_utils.h"
+#include "paddle/phi/common/place.h"
+#include "paddle/phi/core/memory/memcpy.h"
+#include "paddle/phi/core/platform/cpu_helper.h"
+#include "paddle/phi/core/platform/device/gpu/gpu_info.h"
+#include "paddle/phi/core/platform/profiler.h"
+
+namespace paddle {
+
+paddle_infer::DataType ConvertONNXType(ONNXTensorElementDataType type) {
+  switch (type) {
+    case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT:
+      return paddle_infer::DataType::FLOAT32;
+    case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16:
+      return paddle_infer::DataType::FLOAT16;
+    case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8:
+      return paddle_infer::DataType::INT8;
+    case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32:
+      return paddle_infer::DataType::INT32;
+    case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64:
+      return paddle_infer::DataType::INT64;
+    case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8:
+      return paddle_infer::DataType::UINT8;
+    default:
+      LOG(ERROR) << "unsupported ONNX Tensor Type: " << static_cast<int>(type);
+      return paddle_infer::DataType::FLOAT32;
+  }
+}
+
+bool CheckConvertToONNX(const AnalysisConfig &config) {
+  if (!config.model_dir().empty()) {
+    LOG(ERROR) << "Paddle2ONNX not support model_dir config";
+    // TODO(heliqi jiangjiajun): Paddle2ONNX not support
+    // config.model_dir() + "/__model__"
+    // config.model_dir() + var_name
+    return false;
+  } else if (config.prog_file().empty() || config.params_file().empty()) {
+    LOG(ERROR) << string::Sprintf(
+        "not valid model path '%s' or program path '%s' or params path '%s'.",
+        config.model_dir(),
+        config.prog_file(),
+        config.params_file());
+    return false;
+  }
+  if (config.model_from_memory()) {
+    return paddle2onnx::IsExportable(config.prog_file().data(),
+                                     config.prog_file().size(),
+                                     config.params_file().data(),
+                                     config.params_file().size());
+  } else {
+    return paddle2onnx::IsExportable(config.prog_file().c_str(),
+                                     config.params_file().c_str());
+  }
+}
+
+bool ONNXRuntimePredictor::InitBinding() {
+  // Now ONNXRuntime only support CPU
+  const char *device_name = config_.use_gpu() ? "Cuda" : "Cpu";
+  if (config_.use_gpu()) {
+    place_ = phi::GPUPlace(config_.gpu_device_id());
+  } else {
+    place_ = phi::CPUPlace();
+  }
+  scope_.reset(new paddle::framework::Scope());
+
+  binding_ = std::make_shared<Ort::IoBinding>(*session_);
+  Ort::MemoryInfo memory_info(
+      device_name, OrtDeviceAllocator, place_.GetDeviceId(), OrtMemTypeDefault);
+  Ort::Allocator allocator(*session_, memory_info);
+
+  size_t n_inputs = session_->GetInputCount();
+  framework::proto::VarType::Type proto_type =
+      framework::proto::VarType::DENSE_TENSOR;
+  for (size_t i = 0; i < n_inputs; ++i) {
+    auto input_name = session_->GetInputName(i, allocator);
+    auto type_info = session_->GetInputTypeInfo(i);
+    std::vector<int64_t> shape =
+        type_info.GetTensorTypeAndShapeInfo().GetShape();
+    ONNXTensorElementDataType data_type =
+        type_info.GetTensorTypeAndShapeInfo().GetElementType();
+    input_desc_.emplace_back(ONNXDesc{input_name, shape, data_type});
+
+    auto *ptr = scope_->Var(input_name);
+    framework::InitializeVariable(ptr, proto_type);
+
+    allocator.Free(input_name);
+  }
+
+  size_t n_outputs = session_->GetOutputCount();
+  for (size_t i = 0; i < n_outputs; ++i) {
+    auto output_name = session_->GetOutputName(i, allocator);
+    auto type_info = session_->GetOutputTypeInfo(i);
+    std::vector<int64_t> shape =
+        type_info.GetTensorTypeAndShapeInfo().GetShape();
+    ONNXTensorElementDataType data_type =
+        type_info.GetTensorTypeAndShapeInfo().GetElementType();
+    output_desc_.emplace_back(ONNXDesc{output_name, shape, data_type});
+
+    Ort::MemoryInfo out_memory_info(device_name,
+                                    OrtDeviceAllocator,
+                                    place_.GetDeviceId(),
+                                    OrtMemTypeDefault);
+    binding_->BindOutput(output_name, out_memory_info);
+
+    allocator.Free(output_name);
+  }
+  return true;
+}
+
+bool ONNXRuntimePredictor::Init() {
+  VLOG(3) << "ONNXRuntime Predictor::init()";
+
+  char *onnx_proto = nullptr;
+  int out_size;
+  if (config_.model_from_memory()) {
+    paddle2onnx::Export(config_.prog_file().data(),
+                        config_.prog_file().size(),
+                        config_.params_file().data(),
+                        config_.params_file().size(),
+                        &onnx_proto,
+                        &out_size);
+  } else {
+    paddle2onnx::Export(config_.prog_file().c_str(),
+                        config_.params_file().c_str(),
+                        &onnx_proto,
+                        &out_size);
+  }
+
+  Ort::SessionOptions session_options;
+  if (config_.ort_optimization_enabled()) {
+    session_options.SetGraphOptimizationLevel(
+        GraphOptimizationLevel::ORT_ENABLE_ALL);
+  }
+  // Turn optimization off first, and then turn it on when it's stable
+  // session_options.SetExecutionMode(ExecutionMode::ORT_SEQUENTIAL);
+  // session_options.EnableCpuMemArena();
+  // session_options.EnableMemPattern();
+  // session_options.SetInterOpNumThreads(config_.cpu_math_library_num_threads());
+  session_options.SetIntraOpNumThreads(config_.cpu_math_library_num_threads());
+  VLOG(2) << "ONNXRuntime threads " << config_.cpu_math_library_num_threads();
+  if (config_.profile_enabled()) {
+    LOG(WARNING) << "ONNXRuntime Profiler is activated, which might affect the "
+                    "performance";
+#if defined(_WIN32)
+    session_options.EnableProfiling(L"ONNX");
+#else
+    session_options.EnableProfiling("ONNX");
+#endif
+  } else {
+    VLOG(2) << "ONNXRuntime Profiler is deactivated, and no profiling report "
+               "will be "
+               "generated.";
+  }
+  session_ = std::make_shared<Ort::Session>(
+      *env_, onnx_proto, static_cast<size_t>(out_size), session_options);
+  InitBinding();
+  paddle::framework::InitMemoryMethod();
+  delete onnx_proto;
+  onnx_proto = nullptr;
+  return true;
+}
+
+template <>
+std::unique_ptr<PaddlePredictor>
+CreatePaddlePredictor<AnalysisConfig, PaddleEngineKind::kONNXRuntime>(
+    const AnalysisConfig &config) {
+  if (config.glog_info_disabled()) {
+    FLAGS_logtostderr = true;
+    FLAGS_minloglevel = 2;  // GLOG_ERROR
+  }
+
+  PADDLE_ENFORCE_EQ(
+      config.is_valid(),
+      true,
+      common::errors::InvalidArgument(
+          "Note: Each config can only be used for one predictor."));
+
+  VLOG(3) << "create ONNXRuntimePredictor";
+
+  std::unique_ptr<PaddlePredictor> predictor(new ONNXRuntimePredictor(config));
+  // Each config can only be used for one predictor.
+  config.SetInValid();
+  auto predictor_p = dynamic_cast<ONNXRuntimePredictor *>(predictor.get());
+
+  if (!predictor_p->Init()) {
+    return nullptr;
+  }
+
+  return predictor;
+}
+
+std::vector<std::string> ONNXRuntimePredictor::GetInputNames() {
+  std::vector<std::string> input_names;
+  for (auto input_desc : input_desc_) {
+    input_names.push_back(input_desc.name);
+  }
+  return input_names;
+}
+
+std::map<std::string, std::vector<int64_t>>
+ONNXRuntimePredictor::GetInputTensorShape() {
+  std::map<std::string, std::vector<int64_t>> input_shapes;
+  for (auto input_desc : input_desc_) {
+    input_shapes[input_desc.name] = input_desc.shape;
+  }
+  return input_shapes;
+}
+
+std::vector<std::string> ONNXRuntimePredictor::GetOutputNames() {
+  std::vector<std::string> output_names;
+  for (auto output_desc : output_desc_) {
+    output_names.push_back(output_desc.name);
+  }
+  return output_names;
+}
+
+bool ONNXRuntimePredictor::FindONNXDesc(const std::string &name,
+                                        bool is_input) {
+  if (is_input) {
+    for (auto i : input_desc_)
+      if (i.name == name) return true;
+  } else {
+    for (auto i : output_desc_)
+      if (i.name == name) return true;
+  }
+  return false;
+}
+
+std::unique_ptr<ZeroCopyTensor> ONNXRuntimePredictor::GetInputTensor(
+    const std::string &name) {
+  PADDLE_ENFORCE_NOT_NULL(scope_->FindVar(name),
+                          common::errors::PreconditionNotMet(
+                              "The in variable named %s is not found in the "
+                              "ONNXPredictor.",
+                              name));
+  std::unique_ptr<ZeroCopyTensor> res(
+      new ZeroCopyTensor(static_cast<void *>(scope_.get()), this));
+  res->input_or_output_ = true;
+  res->SetName(name);
+  if (phi::is_cpu_place(place_)) {
+    res->SetPlace(PaddlePlace::kCPU);
+  } else {
+    auto gpu_place = place_;
+    res->SetPlace(PaddlePlace::kGPU, gpu_place.GetDeviceId());
+  }
+  return res;
+}
+
+std::unique_ptr<ZeroCopyTensor> ONNXRuntimePredictor::GetOutputTensor(
+    const std::string &name) {
+  PADDLE_ENFORCE_EQ(FindONNXDesc(name, false),
+                    true,
+                    common::errors::PreconditionNotMet(
+                        "The out variable named %s is not found in the "
+                        "ONNXPredictor.",
+                        name));
+  std::unique_ptr<ZeroCopyTensor> res(new ZeroCopyTensor(nullptr, this));
+  res->input_or_output_ = false;
+  res->SetName(name);
+  if (phi::is_cpu_place(place_)) {
+    res->SetPlace(PaddlePlace::kCPU);
+  } else {
+    auto gpu_place = place_;
+    res->SetPlace(PaddlePlace::kGPU, gpu_place.GetDeviceId());
+  }
+  res->SetOrtMark(true);
+  res->SetOrtBinding(binding_);
+  int size = output_desc_.size();
+  for (int i = 0; i < size; ++i)
+    if (output_desc_[i].name == name) {
+      res->idx_ = i;
+      res->dtype_ = ConvertONNXType(output_desc_[i].dtype);
+      break;
+    }
+  return res;
+}
+
+Ort::Value ONNXRuntimePredictor::GetOrtValue(const ONNXDesc &desc,
+                                             const char *device_name) {
+  Ort::MemoryInfo memory_info(
+      device_name, OrtDeviceAllocator, place_.GetDeviceId(), OrtMemTypeDefault);
+  auto *var = scope_->FindVar(desc.name);
+  auto *tensor = var->GetMutable<phi::DenseTensor>();
+  size_t size =
+      tensor->numel() *
+      framework::SizeOfType(framework::TransToProtoVarType(tensor->dtype()));
+  std::vector<int64_t> shape = common::vectorize<int64_t>(tensor->dims());
+  return Ort::Value::CreateTensor(memory_info,
+                                  static_cast<void *>(tensor->data()),
+                                  size,
+                                  shape.data(),
+                                  shape.size(),
+                                  desc.dtype);
+}
+
+bool ONNXRuntimePredictor::Run(const std::vector<PaddleTensor> &inputs,
+                               std::vector<PaddleTensor> *output_data,
+                               int batch_size) {
+  LOG(ERROR) << "Not support Run";
+  return false;
+}
+
+bool ONNXRuntimePredictor::ZeroCopyRun(bool switch_stream) {
+  try {
+    const char *device_name = phi::is_cpu_place(place_) ? "Cpu" : "Cuda";
+    std::vector<Ort::Value> inputs;
+    inputs.reserve(input_desc_.size());
+    for (auto desc : input_desc_) {
+      inputs.push_back(GetOrtValue(desc, device_name));
+      binding_->BindInput(desc.name.c_str(), inputs.back());
+    }
+    for (auto output : output_desc_) {
+      Ort::MemoryInfo out_memory_info(device_name,
+                                      OrtDeviceAllocator,
+                                      place_.GetDeviceId(),
+                                      OrtMemTypeDefault);
+      binding_->BindOutput(output.name.c_str(), out_memory_info);
+    }
+    session_->Run({}, *(binding_.get()));
+  } catch (const std::exception &e) {
+    LOG(ERROR) << e.what();
+    return false;
+  }
+
+  return true;
+}
+
+std::unique_ptr<PaddlePredictor> ONNXRuntimePredictor::Clone(void *stream) {
+  std::lock_guard<std::mutex> lk(clone_mutex_);
+  auto *x = new ONNXRuntimePredictor(config_, env_, session_);
+  x->InitBinding();
+  return std::unique_ptr<PaddlePredictor>(x);
+}
+
+uint64_t ONNXRuntimePredictor::TryShrinkMemory() {
+  return paddle::memory::Release(place_);
+}
+
+ONNXRuntimePredictor::~ONNXRuntimePredictor() {
+  binding_->ClearBoundInputs();
+  binding_->ClearBoundOutputs();
+
+  memory::Release(place_);
+}
+
+const void *ONNXRuntimePredictor::GetDeviceContexts() const {
+  // TODO(inference): Support private device contexts.
+  phi::DeviceContextPool &pool = phi::DeviceContextPool::Instance();
+  const auto &dev_ctxs = pool.device_contexts();
+  return &const_cast<
+      std::map<phi::Place,
+               std::shared_future<std::unique_ptr<phi::DeviceContext>>> &>(
+      dev_ctxs);
+}
+
+}  // namespace paddle

paddle/fluid/inference/api/onnxruntime_predictor.h

@@ -0,0 +1,247 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include <algorithm>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "onnxruntime_c_api.h"    // NOLINT
+#include "onnxruntime_cxx_api.h"  // NOLINT
+#include "paddle/fluid/inference/analysis/analyzer.h"
+#include "paddle/fluid/inference/api/api_impl.h"
+#include "paddle/fluid/inference/api/details/reset_tensor_array.h"
+#include "paddle/fluid/inference/api/helper.h"
+#include "paddle/fluid/inference/api/paddle_inference_api.h"
+#include "paddle/phi/core/platform/device/gpu/gpu_types.h"
+#include "paddle/utils/string/printf.h"
+#include "paddle2onnx/converter.h"
+
+#ifdef PADDLE_WITH_TESTING
+#include <gtest/gtest.h>
+#include <gtest/gtest_prod.h>
+#endif
+
+///
+/// \file onnxruntime_predictor.h
+///
+/// \brief A predictor using ONNXRuntime
+///
+/// \author heliqi@baidu.com
+/// \date 2022-02-14
+/// \since 2.3.0
+///
+
+namespace paddle {
+
+bool CheckConvertToONNX(const AnalysisConfig &config);
+
+struct ONNXDesc {
+  std::string name;
+  std::vector<int64_t> shape;
+  ONNXTensorElementDataType dtype;
+};
+
+///
+/// \class ONNXRuntimePredictor
+///
+/// \brief The ONNXRuntimePredictor using ONNXRuntime for inference
+///
+/// The predictor has the following typical uses:
+///
+/// Get predictor
+/// \code{cpp}
+///   auto predictor = CreatePaddlePredictor(config);
+/// \endcode
+///
+/// Get input or output names
+/// \code{cpp}
+///   auto input_names = predictor->GetInputNames();
+///   auto output_names = predictor->GetOutputNames();
+/// \endcode
+///
+/// Get input or output tensors
+/// \code{cpp}
+///   auto input_t = predictor->GetInputTensor(input_names[0]);
+///   auto output_t = predictor->GetOutputTensor(output_names[0]);
+/// \endcode
+///
+/// Run predictor
+/// \code{cpp}
+///   predictor->ZeroCopyRun();
+/// \endcode
+///
+class ONNXRuntimePredictor : public PaddlePredictor {
+ public:
+  ///
+  /// \brief Construct a new ONNXRuntime Predictor object
+  ///
+  /// \param[in] AnalysisConfig config
+  ///
+  explicit ONNXRuntimePredictor(const AnalysisConfig &config)
+      : env_(std::make_shared<Ort::Env>(ORT_LOGGING_LEVEL_WARNING,
+                                        "paddle-ort")),
+        session_(nullptr),
+        binding_(nullptr),
+        config_(config) {
+    predictor_id_ = inference::GetUniqueId();
+  }
+  ///
+  /// \brief Clone a ONNXRuntime Predictor object
+  ///
+  /// \param[in] AnalysisConfig config
+  ///
+  explicit ONNXRuntimePredictor(const AnalysisConfig &config,
+                                std::shared_ptr<Ort::Env> env,
+                                std::shared_ptr<Ort::Session> session)
+      : env_(env), session_(session), binding_(nullptr), config_(config) {
+    predictor_id_ = inference::GetUniqueId();
+  }
+  ///
+  /// \brief Destroy the ONNXRuntime Predictor object
+  ///
+  ~ONNXRuntimePredictor();
+
+  ///
+  /// \brief Initialize ORT Binding
+  ///
+  /// \return Whether the init function executed successfully
+  ///
+  bool InitBinding();
+
+  ///
+  /// \brief Initialize predictor
+  ///
+  /// \return Whether the init function executed successfully
+  ///
+  bool Init();
+
+  ///
+  /// \brief Get the input names
+  ///
+  /// \return input names
+  ///
+  std::vector<std::string> GetInputNames();
+
+  ///
+  /// \brief Get the output names
+  ///
+  /// \return output names
+  ///
+  std::vector<std::string> GetOutputNames();
+
+  ///
+  /// \brief Get the Input Tensor object
+  ///
+  /// \param[in] name input name
+  /// \return input tensor
+  ///
+  std::unique_ptr<ZeroCopyTensor> GetInputTensor(
+      const std::string &name) override;
+
+  ///
+  /// \brief Get the Output Tensor object
+  ///
+  /// \param[in] name output name
+  /// \return output tensor
+  ///
+  std::unique_ptr<ZeroCopyTensor> GetOutputTensor(
+      const std::string &name) override;
+  ///
+  /// \brief Get all input names and their corresponding shapes
+  ///
+  /// \return the map of input names and shapes
+  ///
+  std::map<std::string, std::vector<int64_t>> GetInputTensorShape() override;
+
+  /// Not support
+  bool Run(const std::vector<PaddleTensor> &inputs,
+           std::vector<PaddleTensor> *output_data,
+           int batch_size = -1) override;
+
+  ///
+  /// \brief Run the prediction engine
+  ///
+  /// \param switch_stream Whether the stream is switched
+  /// \return Whether the function executed successfully
+  ///
+  bool ZeroCopyRun(bool switch_stream = false) override;
+
+  ///
+  /// \brief Release all tmp tensor to compress the size of the memory pool.
+  /// The memory pool is considered to be composed of a list of chunks, if
+  /// the chunk is not occupied, it can be released.
+  ///
+  /// \return Number of bytes released. It may be smaller than the actual
+  /// released memory, because part of the memory is not managed by the
+  /// MemoryPool.
+  ///
+  uint64_t TryShrinkMemory() override;
+  ///
+  /// \brief Clone to get the new predictor. thread safe.
+  ///
+  /// \return get a new predictor
+  ///
+  std::unique_ptr<PaddlePredictor> Clone(void *stream = nullptr) override;
+
+  std::shared_ptr<framework::Scope> scope_;
+
+ protected:
+  const void *GetDeviceContexts() const override;
+
+ private:
+  ///
+  /// \brief Whether to find in/out by name.
+  ///
+  /// \param[in] name input or output name
+  ///
+  /// \param[in] is_input input(true) or output(false)
+  ///
+  /// \return Whether to find by name
+  ///
+  bool FindONNXDesc(const std::string &name, bool is_input);
+
+  /// \brief get the Ort Value(input Tensor).
+  ///
+  /// \param[in] desc ONNXDesc(name、shape、dtype)
+  ///
+  /// \param[in] device_name "cpu" or "gpu" of device
+  ///
+  /// \return get a Ort::Value
+  ///
+  Ort::Value GetOrtValue(const ONNXDesc &desc, const char *device_name);
+
+ private:
+  // ONNXRuntime
+  std::shared_ptr<Ort::Env> env_;
+  std::shared_ptr<Ort::Session> session_{nullptr};
+  std::shared_ptr<Ort::IoBinding> binding_;
+
+  AnalysisConfig config_;
+  std::mutex clone_mutex_;
+  phi::Place place_;
+  std::vector<ONNXDesc> input_desc_;
+  std::vector<ONNXDesc> output_desc_;
+  int predictor_id_;
+
+// Some more detailed tests, they are made the friends of the predictor, so that
+// the all the details can be tested.
+#if PADDLE_WITH_TESTING
+  FRIEND_TEST(ONNXRuntimePredictor, onnxruntime_on);
+#endif
+};
+
+}  // namespace paddle

paddle/fluid/inference/api/paddle_api.h

@@ -0,0 +1,541 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+/*! \file paddle_api.h
+ */
+
+/*! \mainpage Paddle Inference APIs
+ * \section intro_sec Introduction
+ * The Paddle inference library aims to offer an high performance inference SDK
+ * for Paddle users.
+ */
+
+#include <cassert>
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <string>
+#include <unordered_set>
+#include <vector>
+
+#include "crypto/cipher.h"
+#include "paddle_infer_declare.h"  // NOLINT
+#include "paddle_tensor.h"         // NOLINT
+                                   /*! \namespace paddle
+                                    */
+#define ONEDNN_UPDATE_WARNING(api)                                       \
+  "Warning: The api is deprecated since version 3.x, please use onednn " \
+  "api " #api "."
+namespace paddle {
+
+using PaddleDType = paddle_infer::DataType;
+using PaddlePlace = paddle_infer::PlaceType;
+using PaddleDataLayout = paddle_infer::DataLayout;
+using paddle_infer::InputTensorHookFunc;
+using paddle_infer::OutputTensorHookFunc;
+
+/// \brief Memory manager for PaddleTensor.
+///
+/// The PaddleBuf holds a buffer for data input or output. The memory can be
+/// allocated by user or by PaddleBuf itself, but in any case, the PaddleBuf
+/// should be reused for better performance.
+///
+/// For user allocated memory, the following API can be used:
+/// - PaddleBuf(void* data, size_t length) to set an external memory by
+/// specifying the memory address and length.
+/// - Reset(void* data, size_t length) to reset the PaddleBuf with an external
+/// memory.
+/// ATTENTION, for user allocated memory, deallocation should be done by users
+/// externally after the program finished. The PaddleBuf won't do any allocation
+/// or deallocation.
+///
+/// To have the PaddleBuf allocate and manage the memory:
+/// - PaddleBuf(size_t length) will allocate a memory of size `length`.
+/// - Resize(size_t length) resize the memory to no less than `length`,
+/// ATTENTION
+///  if the allocated memory is larger than `length`, nothing will done.
+///
+/// Usage:
+///
+/// Let PaddleBuf manage the memory internally.
+/// \code{cpp}
+/// const int num_elements = 128;
+/// PaddleBuf buf(num_elements/// sizeof(float));
+/// \endcode
+///
+/// Or
+/// \code{cpp}
+/// PaddleBuf buf;
+/// buf.Resize(num_elements/// sizeof(float));
+/// \endcode
+/// Works the exactly the same.
+///
+/// One can also make the `PaddleBuf` use the external memory.
+/// \code{cpp}
+/// PaddleBuf buf;
+/// void* external_memory = new float[num_elements];
+/// buf.Reset(external_memory, num_elements*sizeof(float));
+/// ...
+/// delete[] external_memory; // manage the memory lifetime outside.
+/// \endcode
+///
+class PD_INFER_DECL PaddleBuf {
+ public:
+  ///
+  /// \brief PaddleBuf allocate memory internally, and manage it.
+  ///
+  /// \param[in] length The length of data.
+  ///
+  explicit PaddleBuf(size_t length)
+      : data_(new char[length]), length_(length), memory_owned_(true) {}
+  ///
+  /// \brief Set external memory, the PaddleBuf won't manage it.
+  ///
+  /// \param[in] data The start address of the external memory.
+  /// \param[in] length The length of data.
+  ///
+  PaddleBuf(void* data, size_t length)
+      : data_(data), length_(length), memory_owned_{false} {}
+  ///
+  /// \brief Copy only available when memory is managed externally.
+  ///
+  /// \param[in] other another `PaddleBuf`
+  ///
+  explicit PaddleBuf(const PaddleBuf& other);
+  ///
+  /// \brief Resize the memory.
+  ///
+  /// \param[in] length The length of data.
+  ///
+  void Resize(size_t length);
+  ///
+  /// \brief Reset to external memory, with address and length set.
+  ///
+  /// \param[in] data The start address of the external memory.
+  /// \param[in] length The length of data.
+  ///
+  void Reset(void* data, size_t length);
+  ///
+  /// \brief Tell whether the buffer is empty.
+  ///
+  bool empty() const { return length_ == 0; }
+  ///
+  /// \brief Get the data's memory address.
+  ///
+  void* data() const { return data_; }
+  ///
+  /// \brief Get the memory length.
+  ///
+  size_t length() const { return length_; }
+
+  ~PaddleBuf() { Free(); }
+  PaddleBuf& operator=(const PaddleBuf&);
+  PaddleBuf& operator=(PaddleBuf&&) noexcept;
+  PaddleBuf() = default;
+  PaddleBuf(PaddleBuf&& other) noexcept;
+
+ private:
+  void Free();
+  void* data_{nullptr};  ///< pointer to the data memory.
+  size_t length_{0};     ///< number of memory bytes.
+  bool memory_owned_{true};
+};
+
+///
+/// \brief Basic input and output data structure for PaddlePredictor.
+///
+struct PD_INFER_DECL PaddleTensor {
+  PaddleTensor() = default;
+  std::string name;  ///<  variable name.
+  std::vector<int> shape;
+  PaddleBuf data;  ///<  blob of data.
+  PaddleDType dtype;
+  std::vector<std::vector<size_t>> lod;  ///<  Tensor+LoD equals DenseTensor
+};
+
+/// \brief Represents an n-dimensional array of values.
+/// The ZeroCopyTensor is used to store the input or output of the network.
+/// Zero copy means that the tensor supports direct copy of host or device data
+/// to device,
+/// eliminating additional CPU copy. ZeroCopyTensor is only used in the
+/// AnalysisPredictor.
+/// It is obtained through PaddlePredictor::GetInputTensor()
+/// and PaddlePredictor::GetOutputTensor() interface.
+
+class PD_INFER_DECL ZeroCopyTensor : public paddle_infer::Tensor {
+ public:
+  /// \brief Copy the host memory to tensor data.
+  /// It's usually used to set the input tensor data.
+  /// \param data The pointer of the data, from which the tensor will copy.
+  template <typename T>
+  void copy_from_cpu(const T* data) {
+    return CopyFromCpu(data);
+  }
+
+  /// \brief Experimental interface.
+  /// It's usually used to set the input tensor data with Strings data type.
+  /// \param data The pointer of the data, from which the tensor will copy.
+  void copy_strings_from_cpu(const paddle_infer::Strings* data) {
+    return CopyStringsFromCpu(data);
+  }
+
+  /// \brief Copy the tensor data to the host memory.
+  /// It's usually used to get the output tensor data.
+  /// \param[out] data The tensor will copy the data to the address.
+  template <typename T>
+  void copy_to_cpu(T* data) {
+    return CopyToCpu(data);
+  }
+
+ private:
+  friend class AnalysisPredictor;
+  friend class ONNXRuntimePredictor;
+  explicit ZeroCopyTensor(void* scope, const void* device_contexts)
+      : paddle_infer::Tensor{scope, device_contexts} {}
+};
+
+/// \brief A Predictor for executing inference on a model.
+/// Base class for AnalysisPredictor and NativePaddlePredictor.
+class PD_INFER_DECL PaddlePredictor {
+ public:
+  struct Config;
+  PaddlePredictor() = default;
+  PaddlePredictor(const PaddlePredictor&) = delete;
+  PaddlePredictor& operator=(const PaddlePredictor&) = delete;
+
+  /// \brief This interface takes input and runs the network.
+  /// There are redundant copies of data between hosts in this operation,
+  /// so it is more recommended to use the zecopyrun interface
+  /// \param[in] inputs An list of PaddleTensor as the input to the network.
+  /// \param[out] output_data Pointer to the tensor list, which holds the output
+  /// paddletensor
+  /// \param[in] batch_size This setting has been discarded and can be ignored.
+  /// \return Whether the run is successful
+  virtual bool Run(const std::vector<PaddleTensor>& inputs,
+                   std::vector<PaddleTensor>* output_data,
+                   int batch_size = -1) = 0;
+
+  /// \brief This interface takes input and runs the network (Recommended).
+  /// \param[in] inputs An list of Tensor as the input to the network.
+  /// \param[out] output_data Pointer to the tensor list, which holds the output
+  /// Tensor
+  /// \return Whether the run is successful
+  virtual bool Run(const std::vector<paddle::Tensor>& inputs,
+                   std::vector<paddle::Tensor>* outputs) {
+    return false;
+  }
+
+  /// \brief  Used to get the name of the network input.
+  /// Be inherited by AnalysisPredictor, Only used in ZeroCopy scenarios.
+  /// \return Input tensor names.
+  virtual std::vector<std::string> GetInputNames() { return {}; }
+
+  /// \brief Get the input shape of the model.
+  /// \return A map contains all the input names and shape defined in the model.
+  virtual std::map<std::string, std::vector<int64_t>> GetInputTensorShape() {
+    return {};
+  }
+
+  /// \brief Get the input type of the model.
+  /// \return A map contains all the input names and type defined in the model.
+  virtual std::map<std::string, paddle_infer::DataType> GetInputTypes() {
+    return {};
+  }
+
+  /// \brief Used to get the name of the network output.
+  /// Be inherited by AnalysisPredictor, Only used in ZeroCopy scenarios.
+  /// \return Output tensor names.
+  virtual std::vector<std::string> GetOutputNames() { return {}; }
+
+  /// \brief Get the output shape of the model.
+  /// \return A map contains all the output names and shape defined in the
+  /// model.
+  virtual std::map<std::string, std::vector<int64_t>> GetOutputTensorShape() {
+    return {};
+  }
+
+  /// \brief Get the output type of the model.
+  /// \return A map contains all the output names and type defined in the model.
+  virtual std::map<std::string, paddle_infer::DataType> GetOutputTypes() {
+    return {};
+  }
+
+  /// \brief Get the input ZeroCopyTensor by name.
+  /// Be inherited by AnalysisPredictor, Only used in ZeroCopy scenarios.
+  /// The name is obtained from the GetInputNames() interface.
+  /// \param name The input tensor name.
+  /// \return Return the corresponding input ZeroCopyTensor.
+  virtual std::unique_ptr<ZeroCopyTensor> GetInputTensor(
+      const std::string& name) {
+    return nullptr;
+  }
+
+  /// \brief Get the output ZeroCopyTensor by name.
+  /// Be inherited by AnalysisPredictor, Only used in ZeroCopy scenarios.
+  /// The name is obtained from the GetOutputNames() interface.
+  /// \param name The output tensor name.
+  /// \return Return the corresponding output ZeroCopyTensor.
+  virtual std::unique_ptr<ZeroCopyTensor> GetOutputTensor(
+      const std::string& name) {
+    return nullptr;
+  }
+  /// \brief Run the network with zero-copied inputs and outputs.
+  /// Be inherited by AnalysisPredictor and only used in ZeroCopy scenarios.
+  /// This will save the IO copy for transferring inputs and outputs to
+  /// predictor workspace and get some performance improvement. To use it, one
+  /// should call the AnalysisConfig.SwitchUseFeedFetchOp(false) and then use
+  /// the `GetInputTensor` and `GetOutputTensor` to directly write or read the
+  /// input/output tensors.
+  /// \param switch_stream Whether the stream is switched.
+  /// \return Whether the run is successful
+  virtual bool ZeroCopyRun(bool switch_stream = false) { return false; }
+
+  ///
+  /// \brief Clear the intermediate tensors of the predictor
+  ///
+  ///
+  virtual void ClearIntermediateTensor() {}
+
+  ///
+  /// \brief Release all tmp tensor to compress the size of the memory pool.
+  /// The memory pool is considered to be composed of a list of chunks, if
+  /// the chunk is not occupied, it can be released.
+  ///
+  /// \return Number of bytes released. It may be smaller than the actual
+  /// released memory, because part of the memory is not managed by the
+  /// MemoryPool.
+  ///
+  virtual uint64_t TryShrinkMemory() { return 0; }
+
+  ///
+  /// \brief Register a output hook function to operate the intermediate tensor
+  /// of op output. when using this function, memory reuse should be turned off.
+  /// The hook function signature is void(const std::string&, const
+  /// std::string&, const paddle::Tensor&>). Here, the first parameter is op's
+  /// type, the second param is output var name of the op, and the third
+  /// parameter is output tensor with the var name.
+  ///
+  virtual void RegisterOutputHook(const OutputTensorHookFunc& hookfunc) {}
+
+  /// \brief Same as RegisterOutputHook
+  virtual void RegisterInputHook(const InputTensorHookFunc& hookfunc) {}
+
+  /// \brief Clone an existing predictor
+  /// When using clone, the same network will be created,
+  /// and the parameters between them are shared.
+  /// \return unique_ptr which contains the pointer of predictor
+  virtual std::unique_ptr<PaddlePredictor> Clone(void* stream = nullptr) = 0;
+
+  /// \brief Destroy the Predictor.
+  virtual ~PaddlePredictor() = default;
+
+  virtual std::string GetSerializedProgram() const {
+    assert(false);  // Force raise error.
+    return "NotImplemented";
+  }
+
+  /// \brief Base class for NativeConfig and AnalysisConfig.
+  struct Config {
+    std::string model_dir; /*!< path to the model directory. */
+  };
+
+  virtual void* GetExecStream() const { return nullptr; }
+
+ protected:
+  virtual const void* GetDeviceContexts() const { return nullptr; }
+};
+
+///
+/// \brief configuration manager for `NativePredictor`.
+///
+/// `AnalysisConfig` manages configurations of `NativePredictor`.
+/// During inference procedure, there are many parameters(model/params path,
+/// place of inference, etc.)
+///
+struct PD_INFER_DECL NativeConfig : public PaddlePredictor::Config {
+  NativeConfig();
+  /// GPU related fields.
+  bool use_xpu{false};
+  bool use_gpu{false};
+  int device{0};
+  float fraction_of_gpu_memory{
+      -1.f};  ///< Change to a float in (0,1] if needed.
+
+  std::string prog_file;
+  std::string
+      param_file;  ///< Specify the exact path of program and parameter files.
+
+  bool specify_input_name{false};  ///< Specify the variable's name of each
+                                   ///< input if input tensors don't follow the
+                                   ///< `feeds` and `fetches` of the phase
+                                   ///< `save_inference_model`.
+
+  /// Set and get the number of cpu math library threads.
+  void SetCpuMathLibraryNumThreads(int cpu_math_library_num_threads) {
+    cpu_math_library_num_threads_ = cpu_math_library_num_threads;
+  }
+  int cpu_math_library_num_threads() const {
+    return cpu_math_library_num_threads_;
+  }
+
+ protected:
+  int cpu_math_library_num_threads_{1};  ///< number of cpu math library (such
+                                         ///< as MKL, OpenBlas) threads for each
+                                         ///< instance.
+};
+
+///
+/// \brief A factory to help create different predictors.
+///
+/// Usage:
+///
+/// \code{.cpp}
+/// NativeConfig config;
+/// ... // change the configs.
+/// auto native_predictor = CreatePaddlePredictor(config);
+/// \endcode
+///
+/// FOR EXTENSION DEVELOPER:
+/// Different predictors are designated by config type. Similar configs can be
+/// merged, but there shouldn't be a huge config containing different fields for
+/// more than one kind of predictors.
+////
+template <typename ConfigT>
+std::unique_ptr<PaddlePredictor> CreatePaddlePredictor(const ConfigT& config);
+
+struct AnalysisConfig;
+struct NativeConfig;
+struct DemoConfig;
+
+template <>
+PD_INFER_DECL std::unique_ptr<PaddlePredictor>
+CreatePaddlePredictor<AnalysisConfig>(const AnalysisConfig& config);
+
+template <>
+PD_INFER_DECL std::unique_ptr<PaddlePredictor>
+CreatePaddlePredictor<NativeConfig>(const NativeConfig& config);
+
+template <>
+PD_INFER_DECL std::unique_ptr<PaddlePredictor>
+CreatePaddlePredictor<DemoConfig>(const DemoConfig& config);
+
+/// NOTE The following APIs are too trivial, we will discard it in the following
+/// versions.
+///
+enum class PaddleEngineKind {
+  kNative = 0,         ///< Use the native Fluid facility.
+  kAutoMixedTensorRT,  ///< Automatically mix Fluid with TensorRT.
+  kAnalysis,           ///< More optimization.
+  kONNXRuntime,        ///< Use ONNXRuntime
+};
+
+template <typename ConfigT, PaddleEngineKind engine>
+PD_INFER_DECL std::unique_ptr<PaddlePredictor> CreatePaddlePredictor(
+    const ConfigT& config);
+
+template <>
+PD_INFER_DECL std::unique_ptr<PaddlePredictor>
+CreatePaddlePredictor<NativeConfig, PaddleEngineKind::kNative>(
+    const NativeConfig& config);
+
+template <>
+PD_INFER_DECL std::unique_ptr<PaddlePredictor>
+CreatePaddlePredictor<AnalysisConfig, PaddleEngineKind::kAnalysis>(
+    const AnalysisConfig& config);
+
+template <>
+PD_INFER_DECL std::unique_ptr<PaddlePredictor>
+CreatePaddlePredictor<AnalysisConfig, PaddleEngineKind::kONNXRuntime>(
+    const AnalysisConfig& config);
+
+PD_INFER_DECL int PaddleDtypeSize(PaddleDType dtype);
+
+PD_INFER_DECL std::string get_version();
+
+PD_INFER_DECL void UpdateDllFlag(const char* name, const char* value);
+
+PD_INFER_DECL std::shared_ptr<framework::Cipher> MakeCipher(
+    const std::string& config_file);
+
+}  // namespace paddle
+
+// forward declaration
+using cudaStream_t = struct CUstream_st*;
+using hipStream_t = struct ihipStream_t*;
+
+namespace paddle_infer {
+class Predictor;
+class Tensor;
+using Config = paddle::AnalysisConfig;
+namespace experimental {
+struct XpuRuntimeConfig {
+  // xpu_context(from baidu::xpu::api::create_context) for execution.
+  // If context is nullptr, default context is used.
+  void* context{nullptr};
+  // Stream for execution.
+  // Note: It has a higher priority than stream in "context"
+  void* stream{nullptr};
+  // Available l3 size (Byte)
+  // For kunlun1, max l3_size is 16773120 Byte
+  // For kunlun2, max l3_size is 67104768 Byte
+  // Note: If it is difference from l3_size in "context", new l3 buffer is
+  // malloced.
+  size_t l3_size{16773120};
+  // If l3_ptr is not nullptr, it is used as l3 buffer.
+  // If l3_ptr is nullptr, new l3 buffer will be created.
+  void* l3_ptr{nullptr};
+  // Available l3 size for autotune.
+  // If l3_autotune_size is 0, autotune is closed.
+  // Note: The remaining l3 size (l3_size - l3_autotune_size) is for
+  // kernels (both paddle/xdnn kernels)
+  size_t l3_autotune_size{0};
+};
+
+// Unstable interface, may be modified or deleted in the future.
+class PD_INFER_DECL InternalUtils {
+ public:
+  // Note: Can only be used under thread_local semantics.
+  static bool RunWithExternalStream(paddle_infer::Predictor* pred,
+                                    cudaStream_t stream);
+  static bool RunWithExternalStream(paddle_infer::Predictor* pred,
+                                    hipStream_t stream);
+  static bool RunWithRuntimeConfig(paddle_infer::Predictor* pred, void* config);
+
+  static void UpdateConfigInterleaved(paddle_infer::Config* c,
+                                      bool with_interleaved);
+
+  static void SetTransformerPosid(
+      paddle_infer::Config* c, const std::string& tensorrt_transformer_posid);
+
+  static void SetTransformerMaskid(
+      paddle_infer::Config* c, const std::string& tensorrt_transformer_maskid);
+
+  static void DisableTensorRtHalfOps(
+      paddle_infer::Config* c, const std::unordered_set<std::string>& ops);
+
+  static void SyncStream(paddle_infer::Predictor* pred);
+  static void SyncStream(cudaStream_t stream);
+  static void SyncStream(hipStream_t stream);
+  template <typename T>
+  static void CopyFromCpuWithIoStream(paddle_infer::Tensor* t,
+                                      const T* data,
+                                      cudaStream_t stream);
+  template <typename T>
+  static void CopyToCpuWithIoStream(paddle_infer::Tensor* t,
+                                    T* data,
+                                    cudaStream_t stream);
+};
+}  // namespace experimental
+}  // namespace paddle_infer

paddle/fluid/inference/api/paddle_infer_contrib.cc

@@ -0,0 +1,291 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/inference/api/paddle_infer_contrib.h"
+
+#include "paddle/fluid/framework/scope.h"
+#include "paddle/fluid/platform/enforce.h"
+#include "paddle/phi/common/float16.h"
+#include "paddle/phi/core/memory/memcpy.h"
+#include "paddle/phi/core/platform/device_context.h"
+
+namespace paddle_infer::contrib {
+
+using paddle::PaddleDType;
+
+void* TensorUtils::CudaMallocPinnedMemory(size_t size) {
+#if defined(PADDLE_WITH_CUDA)
+  void* ptr = nullptr;
+  PADDLE_ENFORCE_GPU_SUCCESS(cudaMallocHost(&ptr, size));
+  return ptr;
+#else
+  return nullptr;
+#endif
+}
+
+void TensorUtils::CudaFreePinnedMemory(void* ptr) {
+#if defined(PADDLE_WITH_CUDA)
+  PADDLE_ENFORCE_GPU_SUCCESS(cudaFreeHost(ptr));
+#endif
+}
+
+void TensorUtils::CopyTensorImpl(Tensor* p_dst,
+                                 const Tensor& src,
+                                 void* exec_stream,
+                                 CallbackFunc cb,
+                                 void* cb_params) {
+  Tensor& dst = *p_dst;
+  dst.Reshape(src.shape());
+  PADDLE_ENFORCE(
+      src.place() == PlaceType::kCPU || src.place() == PlaceType::kGPU,
+      common::errors::InvalidArgument(
+          "CopyTensor only support PlaceType kCPU/kGPU now."));
+  PADDLE_ENFORCE(
+      dst.place() == PlaceType::kCPU || dst.place() == PlaceType::kGPU,
+      common::errors::InvalidArgument(
+          "CopyTensor only support PlaceType kCPU/kGPU now."));
+  // copy to cpu, gpu => cpu or cpu => cpu
+  if (dst.place() == PlaceType::kCPU) {
+    switch (src.type()) {
+      case PaddleDType::INT32:
+        src.CopyToCpuImpl(dst.mutable_data<int32_t>(PlaceType::kCPU),
+                          exec_stream,
+                          cb,
+                          cb_params);
+        break;
+      case PaddleDType::INT64:
+        src.CopyToCpuImpl(dst.mutable_data<int64_t>(PlaceType::kCPU),
+                          exec_stream,
+                          cb,
+                          cb_params);
+        break;
+      case PaddleDType::FLOAT64:
+        src.CopyToCpuImpl(dst.mutable_data<double>(PlaceType::kCPU),
+                          exec_stream,
+                          cb,
+                          cb_params);
+        break;
+      case PaddleDType::FLOAT32:
+        src.CopyToCpuImpl(dst.mutable_data<float>(PlaceType::kCPU),
+                          exec_stream,
+                          cb,
+                          cb_params);
+        break;
+      case PaddleDType::UINT8:
+        src.CopyToCpuImpl(dst.mutable_data<uint8_t>(PlaceType::kCPU),
+                          exec_stream,
+                          cb,
+                          cb_params);
+        break;
+      case PaddleDType::INT8:
+        src.CopyToCpuImpl(dst.mutable_data<int8_t>(PlaceType::kCPU),
+                          exec_stream,
+                          cb,
+                          cb_params);
+        break;
+      case PaddleDType::BOOL:
+        src.CopyToCpuImpl(dst.mutable_data<bool>(PlaceType::kCPU),
+                          exec_stream,
+                          cb,
+                          cb_params);
+        break;
+      case PaddleDType::FLOAT16:
+        src.CopyToCpuImpl(
+            dst.mutable_data<phi::dtype::float16>(PlaceType::kCPU),
+            exec_stream,
+            cb,
+            cb_params);
+        break;
+      case PaddleDType::BFLOAT16:
+        src.CopyToCpuImpl(
+            dst.mutable_data<phi::dtype::bfloat16>(PlaceType::kCPU),
+            exec_stream,
+            cb,
+            cb_params);
+        break;
+      default:
+        PADDLE_THROW(common::errors::Unimplemented(
+            "Only INT32, INT64, UINT8, INT8, BOOL, FLOAT16, BFLOAT16, FLOAT32 "
+            "and "
+            "FLOAT64 is supported in Tensor. Others not implements"));
+    }
+    // gpu => gpu or cpu => gpu
+  } else {
+#if defined(PADDLE_WITH_CUDA)
+    void* dst_data = nullptr;
+    void* src_data = nullptr;
+    size_t data_len = 0;
+    int data_size = 0;
+    PlaceType src_place;
+    switch (src.type()) {
+      case PaddleDType::INT32:
+        dst_data =
+            static_cast<void*>(dst.mutable_data<int32_t>(PlaceType::kGPU));
+        src_data =
+            static_cast<void*>(src.data<int32_t>(&src_place, &data_size));
+        data_len = data_size * sizeof(int32_t);
+        break;
+      case PaddleDType::INT64:
+        dst_data =
+            static_cast<void*>(dst.mutable_data<int64_t>(PlaceType::kGPU));
+        src_data =
+            static_cast<void*>(src.data<int64_t>(&src_place, &data_size));
+        data_len = data_size * sizeof(int64_t);
+        break;
+      case PaddleDType::FLOAT64:
+        dst_data =
+            static_cast<void*>(dst.mutable_data<double>(PlaceType::kGPU));
+        src_data = static_cast<void*>(src.data<double>(&src_place, &data_size));
+        data_len = data_size * sizeof(double);
+        break;
+      case PaddleDType::FLOAT32:
+        dst_data = static_cast<void*>(dst.mutable_data<float>(PlaceType::kGPU));
+        src_data = static_cast<void*>(src.data<float>(&src_place, &data_size));
+        data_len = data_size * sizeof(float);
+        break;
+      case PaddleDType::UINT8:
+        dst_data =
+            static_cast<void*>(dst.mutable_data<uint8_t>(PlaceType::kGPU));
+        src_data =
+            static_cast<void*>(src.data<uint8_t>(&src_place, &data_size));
+        data_len = data_size * sizeof(uint8_t);
+        break;
+      case PaddleDType::INT8:
+        dst_data =
+            static_cast<void*>(dst.mutable_data<int8_t>(PlaceType::kGPU));
+        src_data = static_cast<void*>(src.data<int8_t>(&src_place, &data_size));
+        data_len = data_size * sizeof(int8_t);
+        break;
+      case PaddleDType::BOOL:
+        dst_data = static_cast<void*>(dst.mutable_data<bool>(PlaceType::kGPU));
+        src_data = static_cast<void*>(src.data<bool>(&src_place, &data_size));
+        data_len = data_size * sizeof(bool);
+        break;
+      case PaddleDType::FLOAT16:
+        dst_data = static_cast<void*>(
+            dst.mutable_data<phi::dtype::float16>(PlaceType::kGPU));
+        src_data = static_cast<void*>(
+            src.data<phi::dtype::float16>(&src_place, &data_size));
+        data_len = data_size * 2;
+        break;
+      case PaddleDType::BFLOAT16:
+        dst_data = static_cast<void*>(
+            dst.mutable_data<phi::dtype::bfloat16>(PlaceType::kGPU));
+        src_data = static_cast<void*>(
+            src.data<phi::dtype::bfloat16>(&src_place, &data_size));
+        data_len = data_size * 2;
+        break;
+      default:
+        PADDLE_THROW(common::errors::Unimplemented(
+            "Only INT32, INT64, UINT8, INT8, BOOL, FLOAT16, BFLOAT16, FLOAT32 "
+            "and "
+            "FLOAT64 is supported in Tensor. Others not implements"));
+    }
+
+    phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
+    phi::GPUPlace gpu_place(dst.device_);
+    auto* dev_ctx = static_cast<const phi::GPUContext*>(pool.Get(gpu_place));
+
+    if (src.place() == PlaceType::kCPU) {
+      paddle::memory::Copy(gpu_place,
+                           static_cast<void*>(dst_data),
+                           phi::CPUPlace(),
+                           src_data,
+                           data_len,
+                           dev_ctx->stream());
+    } else {
+      paddle::memory::Copy(gpu_place,
+                           static_cast<void*>(dst_data),
+                           phi::GPUPlace(),
+                           src_data,
+                           data_len,
+                           dev_ctx->stream());
+    }
+
+    if (nullptr != exec_stream) {
+      *(static_cast<cudaStream_t*>(exec_stream)) = dev_ctx->stream();
+    } else if (cb) {
+      cudaLaunchHostFunc(dev_ctx->stream(), cb, cb_params);
+    } else {
+      cudaStreamSynchronize(dev_ctx->stream());
+    }
+#else
+    PADDLE_THROW(common::errors::Unavailable(
+        "Can not copy tensor to GPU CUDA place because paddle is not compiled "
+        "with CUDA."));
+#endif
+  }
+  return;
+}
+
+void TensorUtils::CopyTensor(Tensor* p_dst, const Tensor& src) {
+  CopyTensorImpl(p_dst, src, nullptr, nullptr, nullptr);
+}
+
+void TensorUtils::CopyTensorAsync(Tensor* p_dst,
+                                  const Tensor& src,
+                                  void* exec_stream) {
+  CopyTensorImpl(p_dst, src, exec_stream, nullptr, nullptr);
+}
+
+void TensorUtils::CopyTensorAsync(Tensor* p_dst,
+                                  const Tensor& src,
+                                  CallbackFunc cb,
+                                  void* cb_params) {
+  CopyTensorImpl(p_dst, src, nullptr, cb, cb_params);
+}
+
+struct Status::Impl {
+  int ec{0};
+  std::string msg;
+};
+
+Status::Status() : impl_(std::make_shared<Impl>()) {}
+Status::Status(const Status& status) : impl_(std::make_shared<Impl>()) {
+  *impl_ = *status.impl_;
+}
+
+Status& Status::operator=(const Status& status) noexcept {
+  if (this == &status) {
+    return *this;
+  }
+  *impl_ = *status.impl_;
+  return *this;
+}
+Status::Status(std::exception_ptr e) : impl_(std::make_shared<Impl>()) {
+  constexpr int kDefaultError{-1};
+  impl_->ec = kDefaultError;
+  try {
+    std::rethrow_exception(e);
+  } catch (paddle::platform::EnforceNotMet& e) {
+    // Add one to the error code to make the number zero a non-error
+    // status code.
+    impl_->ec = e.code() + 1;
+    impl_->msg = e.what();
+  } catch (const std::exception& e) {
+    impl_->msg = e.what();
+  }
+}
+Status Status::OK() { return Status(); }
+bool Status::ok() const noexcept { return impl_->ec == 0; }
+Status::Code Status::code() const noexcept { return impl_->ec; }
+const std::string& Status::error_message() const noexcept { return impl_->msg; }
+bool Status::operator==(const Status& x) const noexcept {
+  return code() == x.code() && error_message() == x.error_message();
+}
+bool Status::operator!=(const Status& x) const noexcept {
+  return !(*this == x);
+}
+
+}  // namespace paddle_infer::contrib

paddle/fluid/inference/api/paddle_infer_contrib.h

@@ -0,0 +1,132 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/common/macros.h"
+#include "paddle_inference_api.h"  // NOLINT
+
+namespace paddle_infer {
+namespace contrib {
+
+class TensorUtils {
+ public:
+  static void* CudaMallocPinnedMemory(size_t size);
+  static void CudaFreePinnedMemory(void* mem);
+
+  static void CopyTensor(Tensor* p_dst, const Tensor& src);
+  static void CopyTensorAsync(Tensor* p_dst,
+                              const Tensor& src,
+                              void* exec_stream);
+  static void CopyTensorAsync(Tensor* p_dst,
+                              const Tensor& src,
+                              CallbackFunc cb,
+                              void* cb_params);
+
+ private:
+  static void CopyTensorImpl(Tensor* p_dst,
+                             const Tensor& src,
+                             void* exec_stream,
+                             CallbackFunc cb,
+                             void* cb_params);
+};
+
+/// \brief A status class, used to intercept exceptions and convert
+/// them into a status number.
+class PADDLE_API Status {
+ public:
+  using Code = int;
+  struct Impl;
+
+  Status();
+  explicit Status(std::exception_ptr e);
+
+  Status(const Status&);
+  Status& operator=(const Status&) noexcept;
+  Status& operator=(Status&&) = default;
+  Status(Status&&) = default;
+
+  ///
+  /// \brief Construct a status which indicate ok.
+  ///
+  /// \return A status which indicate ok.
+  ///
+  static Status OK();
+
+  ///
+  /// \brief Determine whether the status is ok.
+  ///
+  /// \return Whether the status is ok.
+  ///
+  bool ok() const noexcept;
+
+  ///
+  /// \brief Return the error code.
+  /// The meaning corresponds to the following.
+  ///
+  /// CODE    IMPLICATION
+  ///  -1      UNKNOWN
+  ///  0        NORMAL
+  ///  1        LEGACY
+  ///  2    INVALID_ARGUMENT
+  ///  3       NOT_FOUND
+  ///  4     OUT_OF_RANGE
+  ///  5    ALREADY_EXISTS
+  ///  6   RESOURCE_EXHAUSTED
+  ///  7  PRECONDITION_NOT_MET
+  ///  8   PERMISSION_DENIED
+  ///  9   EXECUTION_TIMEOUT
+  ///  10    UNIMPLEMENTED
+  ///  11     UNAVAILABLE
+  ///  12        FATAL
+  ///  13       EXTERNAL
+  ///
+  /// \return The error code.
+  ///
+  Code code() const noexcept;
+
+  ///
+  /// \brief Return the error message.
+  ///
+  /// \return The error message.
+  ///
+  const std::string& error_message() const noexcept;
+
+  bool operator==(const Status& x) const noexcept;
+  bool operator!=(const Status& x) const noexcept;
+
+ private:
+  std::shared_ptr<Impl> impl_;
+};
+
+///
+/// \brief A wrapper used to provide exception safety.
+///
+/// \param func Wrapped function.
+/// \param args Parameters of the wrapped function.
+/// \return State result of calling function.
+///
+template <typename Func, typename... Args>
+Status get_status(Func func, Args&&... args) noexcept(
+    noexcept(Status(std::declval<Status>()))) {
+  try {
+    func(std::forward<Args>(args)...);
+  } catch (...) {
+    return Status(std::current_exception());
+  }
+  return Status::OK();
+}
+
+}  // namespace contrib
+}  // namespace paddle_infer

paddle/fluid/inference/api/paddle_infer_declare.h

@@ -0,0 +1,29 @@
+// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#if defined(_WIN32)
+#ifndef PD_INFER_DECL
+#ifdef PADDLE_DLL_INFERENCE
+#define PD_INFER_DECL __declspec(dllexport)
+#else
+#define PD_INFER_DECL __declspec(dllimport)
+#endif  // PADDLE_DLL_INFERENCE
+#endif  // PD_INFER_DECL
+#else
+#ifndef PD_INFER_DECL
+#define PD_INFER_DECL __attribute__((visibility("default")))
+#endif  // PD_INFER_DECL
+#endif  // _WIN32

paddle/fluid/inference/api/paddle_inference_api.h

@@ -0,0 +1,278 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+/*
+ * This file contains the definition of a simple Inference API for Paddle.
+ *
+ * ATTENTION: It requires some C++11 features, for lower version C++ or C, we
+ * might release another API.
+ */
+
+#pragma once
+
+#include <cassert>
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <string>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "paddle_analysis_config.h"  // NOLINT
+#include "paddle_api.h"              // NOLINT
+
+///
+/// \file paddle_inference_api.h
+///
+/// \brief Paddle Inference API
+///
+/// \author paddle-infer@baidu.com
+/// \date 2020-09-01
+/// \since 2.0.0-beta
+///
+
+namespace paddle_infer {
+
+using PrecisionType = paddle::AnalysisConfig::Precision;
+using Config = paddle::AnalysisConfig;
+using XpuConfig = paddle::XpuConfig;
+
+///
+/// \class Predictor
+///
+/// \brief Predictor is the interface for model prediction.
+///
+/// The predictor has the following typical uses:
+///
+/// Get predictor
+/// \code{cpp}
+///   auto predictor = CreatePredictor(config);
+/// \endcode
+///
+/// Get input or output names
+/// \code{cpp}
+///   auto input_names = predictor->GetInputNames();
+///   auto output_names = predictor->GetOutputNames();
+/// \endcode
+///
+/// Get input or output handle
+/// \code{cpp}
+///   auto input_t = predictor->GetInputHandle(input_names[0]);
+///   auto output_t = predictor->GetOutputHandle(output_names[0]);
+/// \endcode
+///
+/// Run predictor
+/// \code{cpp}
+///   predictor->Run();
+/// \endcode
+///
+class PD_INFER_DECL Predictor {
+ public:
+  Predictor() = delete;
+  ~Predictor() {}
+  // Use for clone
+  explicit Predictor(std::unique_ptr<paddle::PaddlePredictor>&& pred)
+      : predictor_(std::move(pred)) {}
+
+  ///
+  /// \brief Construct a new Predictor object
+  ///
+  /// \param[in] Config config
+  ///
+  explicit Predictor(const Config& config);
+
+  ///
+  /// \brief Get all input names and their corresponding shapes
+  ///
+  /// \return the map of input names and shape
+  ///
+  std::map<std::string, std::vector<int64_t>> GetInputTensorShape();
+
+  ///
+  /// \brief Get all input names and their corresponding type
+  ///
+  /// \return the map of input names and type
+  ///
+  std::map<std::string, DataType> GetInputTypes();
+
+  ///
+  /// \brief Get the input names
+  ///
+  /// \return input names
+  ///
+  std::vector<std::string> GetInputNames();
+
+  ///
+  /// \brief Get the Input Tensor object
+  ///
+  /// \param[in] name input name
+  /// \return input tensor
+  ///
+  std::unique_ptr<Tensor> GetInputHandle(const std::string& name);
+
+  ///
+  /// \brief Run the prediction engine
+  ///
+  /// \return Whether the function executed successfully
+  ///
+  bool Run();
+
+  ///
+  /// \brief Run the prediction engine (Recommended)
+  ///
+  /// \param[in] inputs An list of Tensor as the input to the network.
+  /// \param[out] outputs Pointer to the tensor list, which holds the output
+  /// Tensor
+  ///
+  /// \return Whether the run is successful
+  bool Run(const std::vector<paddle::Tensor>& inputs,
+           std::vector<paddle::Tensor>* outputs);
+
+  ///
+  /// \brief Get the output names
+  ///
+  /// \return output names
+  ///
+  std::vector<std::string> GetOutputNames();
+
+  ///
+  /// \brief Get the Output Tensor object
+  ///
+  /// \param[in] name output name
+  /// \return output tensor
+  ///
+  std::unique_ptr<Tensor> GetOutputHandle(const std::string& name);
+
+  ///
+  /// \brief Get all output names and their corresponding shapes
+  ///
+  /// \return the map of output names and shape
+  ///
+  std::map<std::string, std::vector<int64_t>> GetOutputTensorShape();
+
+  ///
+  /// \brief Get all output names and their corresponding type
+  ///
+  /// \return the map of output names and type
+  ///
+  std::map<std::string, DataType> GetOutputTypes();
+
+  ///
+  /// \brief Clone to get the new predictor. thread safe.
+  ///
+  /// \return get a new predictor
+  ///
+  std::unique_ptr<Predictor> Clone(void* stream = nullptr);
+
+  /// \brief Clear the intermediate tensors of the predictor
+  void ClearIntermediateTensor();
+
+  ///
+  /// \brief Release all tmp tensor to compress the size of the memory pool.
+  /// The memory pool is considered to be composed of a list of chunks, if
+  /// the chunk is not occupied, it can be released.
+  ///
+  /// \return Number of bytes released. It may be smaller than the actual
+  /// released memory, because part of the memory is not managed by the
+  /// MemoryPool.
+  ///
+  uint64_t TryShrinkMemory();
+
+  ///
+  /// \brief Register a output hook function to operate the intermediate tensor
+  /// of op output. when using this function, memory reuse should be turned off.
+  /// The hook function signature is void(const std::string&, const
+  /// std::string&, const Tensor&>). Here, the first parameter is op's
+  /// type, the second param is output var name of the op, and the third
+  /// parameter is output tensor with the var name.
+  ///
+  void RegisterOutputHook(const OutputTensorHookFunc& hookfunc);
+
+  /// The same as RegisterOutputHook.
+  void RegisterInputHook(const InputTensorHookFunc& hookfunc);
+
+  ///
+  /// \brief Get the execution stream on devices with a concept of stream,
+  /// otherwise returns nullptr.
+  ///
+  /// \return The execution stream or nullptr (CPU).
+  ///
+  void* GetExecStream() const;
+
+ private:
+  std::unique_ptr<paddle::PaddlePredictor> predictor_;
+  friend class paddle_infer::experimental::InternalUtils;
+};
+
+///
+/// \brief A factory to help create predictors.
+///
+/// Usage:
+///
+/// \code{.cpp}
+/// Config config;
+/// ... // change the configs.
+/// auto predictor = CreatePredictor(config);
+/// \endcode
+///
+PD_INFER_DECL std::shared_ptr<Predictor> CreatePredictor(
+    const Config& config);  // NOLINT
+
+PD_INFER_DECL int GetNumBytesOfDataType(DataType dtype);
+
+PD_INFER_DECL std::string GetVersion();
+PD_INFER_DECL std::tuple<int, int, int> GetTrtCompileVersion();
+PD_INFER_DECL std::tuple<int, int, int> GetTrtRuntimeVersion();
+PD_INFER_DECL void UpdateDllFlag(const char* name, const char* value);
+
+PD_INFER_DECL void ConvertToMixedPrecision(
+    const std::string& model_file,
+    const std::string& params_file,
+    const std::string& mixed_model_file,
+    const std::string& mixed_params_file,
+    PrecisionType mixed_precision,
+    PlaceType backend,
+    bool keep_io_types = true,
+    std::unordered_set<std::string> black_list = {},
+    std::unordered_set<std::string> white_list = {});
+
+namespace services {
+///
+/// \class PredictorPool
+///
+/// \brief PredictorPool is a simple encapsulation of Predictor, suitable for
+/// use in multi-threaded situations. According to the thread id, the
+/// corresponding Predictor is taken out from PredictorPool to complete the
+/// prediction.
+///
+class PD_INFER_DECL PredictorPool {
+ public:
+  PredictorPool() = delete;
+  PredictorPool(const PredictorPool&) = delete;
+  PredictorPool& operator=(const PredictorPool&) = delete;
+
+  /// \brief Construct the predictor pool with \param size predictor instances.
+  explicit PredictorPool(const Config& config, size_t size = 1);
+
+  /// \brief Get \param id-th predictor.
+  Predictor* Retrieve(size_t idx);
+
+ private:
+  std::shared_ptr<Predictor> main_pred_;
+  std::vector<std::unique_ptr<Predictor>> preds_;
+};
+}  // namespace services
+
+}  // namespace paddle_infer

paddle/fluid/inference/api/paddle_pass_builder.cc

@@ -0,0 +1,729 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/inference/api/paddle_pass_builder.h"
+#ifdef PADDLE_WITH_CUDA
+#include <cudnn.h>
+#endif
+#ifdef PADDLE_WITH_HIP
+#include <miopen/miopen.h>
+#endif
+#ifdef PADDLE_WITH_TENSORRT
+#include "paddle/fluid/inference/tensorrt/helper.h"
+#endif
+
+#include <glog/logging.h>
+
+#include <algorithm>
+#include <sstream>
+#include "paddle/fluid/inference/api/paddle_api.h"
+namespace paddle {
+
+void PaddlePassBuilder::AppendPass(const std::string &pass_type) {
+  passes_.push_back(pass_type);
+}
+
+void PaddlePassBuilder::TurnOnDebug() {
+  std::vector<std::string> passes;
+  auto it = std::begin(passes_);
+  while (it != std::end(passes_)) {
+    if (*it != "graph_viz_pass") {
+      it = passes_.insert(it + 1, "graph_viz_pass");
+    } else {
+      ++it;
+    }
+  }
+}
+
+std::string PaddlePassBuilder::DebugString() {
+  std::stringstream ss;
+  ss << "Passes to apply:\n";
+  for (auto &pass : passes_) {
+    ss << "  - " << pass << '\n';
+  }
+  return ss.str();
+}
+
+void PaddlePassBuilder::DeletePass(const std::string &pass_type) {
+  deleted_passes_.insert(pass_type);
+  auto it = std::begin(passes_);
+  while (it != std::end(passes_)) {
+    if (*it == pass_type) {
+      it = passes_.erase(it);
+    } else {
+      ++it;
+    }
+  }
+}
+
+size_t PaddlePassBuilder::GetPassIndex(const std::string &pass_type) {
+  auto iter = std::find(std::begin(passes_), std::end(passes_), pass_type);
+  if (iter == std::end(passes_)) return -1;
+  return std::distance(std::begin(passes_), iter);
+}
+
+void PaddlePassBuilder::InsertPass(size_t idx, const std::string &pass_type) {
+  passes_.insert(std::begin(passes_) + idx, pass_type);  // NOLINT
+}
+
+void PaddlePassBuilder::DeletePass(size_t idx) {
+  passes_.erase(std::begin(passes_) + idx);  // NOLINT
+}
+
+void PaddlePassBuilder::AppendAnalysisPass(const std::string &pass) {
+  analysis_passes_.push_back(pass);
+}
+
+void PaddlePassBuilder::ClearPasses() { passes_.clear(); }
+#ifdef PADDLE_WITH_OPENVINO
+const std::vector<std::string> kOVSubgraphPasses({"openvino_subgraph_pass"});
+#endif
+const std::vector<std::string> kTRTSubgraphPasses({
+  "set_subgraph_edge_pass",                                       //
+      "trt_remove_amp_strategy_op_pass",                          //
+      "trt_support_nhwc_pass",                                    //
+      "adaptive_pool2d_convert_global_pass",                      //
+      "trt_map_ops_to_matrix_multiply_pass",                      //
+      "shuffle_channel_detect_pass",                              //
+      "quant_conv2d_dequant_fuse_pass",                           //
+      "delete_quant_dequant_op_pass",                             //
+      "delete_quant_dequant_filter_op_pass",                      //
+      "trt_delete_weight_dequant_linear_op_pass",                 //
+      "delete_quant_dequant_linear_op_pass",                      //
+      "identity_op_clean_pass",                                   //
+      "add_support_int8_pass",                                    //
+      "simplify_with_basic_ops_pass",                             //
+      "trt_prompt_tuning_embedding_eltwise_layernorm_fuse_pass",  //
+      "trt_embedding_eltwise_layernorm_fuse_pass",                //
+      "preln_embedding_eltwise_layernorm_fuse_pass",              //
+      "trt_multihead_matmul_fuse_pass_v2",                        //
+      "trt_multihead_matmul_fuse_pass_v3",                        //
+      "multihead_matmul_roformer_fuse_pass",                      //
+#if defined _WIN32  // Windows does not support sparse_conv3d_implicit_gemm
+#else
+      "sparse_conv_optim_pass",                //
+#endif
+      "constant_folding_pass",  //
+#ifdef PADDLE_WITH_TENSORRT
+#if !IS_TRT_VERSION_GE(8610)
+      "trt_flash_multihead_matmul_fuse_pass",  //
+      "trt_cross_multihead_matmul_fuse_pass",  //
+#endif
+#endif
+      "vit_attention_fuse_pass",              //
+      "trt_qk_multihead_matmul_fuse_pass",    //
+      "layernorm_shift_partition_fuse_pass",  //
+      "merge_layernorm_fuse_pass",            //
+#if !defined _WIN32
+      "split_layernorm_to_math_ops_pass",  //
+#endif
+#if defined _WIN32  // Windows CI is TensorRT7.0. Remove this after upgrading.
+#else
+      "trt_skip_layernorm_fuse_pass",          //
+      "preln_skip_layernorm_fuse_pass",        //
+#endif
+      "preln_residual_bias_fuse_pass",   //
+      "preln_layernorm_x_fuse_pass",     //
+      "reverse_roll_fuse_pass",          //
+      "conv_bn_fuse_pass",               //
+      "conv_elementwise_add_fuse_pass",  //
+#if defined _WIN32  // Windows CI is TensorRT7.0. Remove this after upgrading.
+#else
+      "trans_layernorm_fuse_pass",             //
+#endif
+      "remove_padding_recover_padding_pass",         //
+      "delete_remove_padding_recover_padding_pass",  //
+      // "yolo_box_fuse_pass",      //
+      "dense_fc_to_sparse_pass",                //
+      "dense_multihead_matmul_to_sparse_pass",  //
+#if defined _WIN32  // Windows CI is TensorRT7.0. Remove this after upgrading.
+#else
+      "elementwise_groupnorm_act_pass",        //
+      "preln_elementwise_groupnorm_act_pass",  //
+      "groupnorm_act_pass",                    //
+      "elementwiseadd_transpose_pass",         //
+#endif
+      "tensorrt_subgraph_pass",  //
+      "conv_bn_fuse_pass",       //
+// cudnn8.0 has memory leak problem in conv + eltwise + act, so we
+// disable the pass.
+#if !(CUDNN_VERSION >= 8000 && CUDNN_VERSION < 8100)
+      "conv_elementwise_add_act_fuse_pass",   //
+      "conv_elementwise_add2_act_fuse_pass",  //
+#endif
+      "transpose_flatten_concat_fuse_pass",  //
+      "auto_mixed_precision_pass",
+});
+
+// TODO(inference): Most of the existing pass fusion operators do not
+// support fp16/bf16 precision, temporarily use low precision pass to prevent
+// running errors. After fusion operator supports low precision, delete this.
+const std::vector<std::string> kGpuLowerPrecisionPasses{
+    "map_op_to_another_pass",
+    "identity_op_clean_pass",
+    "simplify_with_basic_ops_pass",
+    "silu_fuse_pass",
+    "delete_quant_dequant_linear_op_pass",
+    "delete_weight_dequant_linear_op_pass",
+    "conv_bn_fuse_pass",
+    "conv_eltwiseadd_bn_fuse_pass",
+    "conv_elementwise_add_act_fuse_pass",
+    "conv_elementwise_add2_act_fuse_pass",
+    "conv_elementwise_add_fuse_pass",
+    "transfer_layout_pass",
+    "multihead_matmul_fuse_pass_v2",
+    "fused_multi_transformer_encoder_pass",
+    "fused_multi_transformer_decoder_pass",
+    "fused_multi_transformer_encoder_fuse_qkv_pass",
+    "fused_multi_transformer_decoder_fuse_qkv_pass",
+    "multi_devices_fused_multi_transformer_encoder_pass",
+    "multi_devices_fused_multi_transformer_encoder_fuse_qkv_pass",
+    "multi_devices_fused_multi_transformer_decoder_fuse_qkv_pass",
+    "fuse_multi_transformer_layer_pass",
+    "gpu_cpu_map_matmul_v2_to_mul_pass",
+    "gpu_cpu_map_matmul_v2_to_matmul_pass",
+    "gpu_cpu_map_matmul_to_mul_pass",
+    "fc_fuse_pass",
+    // "fc_elementwise_layernorm_fuse_pass",
+    "embedding_eltwise_layernorm_fuse_pass",
+    "inplace_op_var_pass"};
+
+const std::vector<std::string> kTrtLowerPrecisionPasses{
+    "trt_remove_amp_strategy_op_pass",
+    "trt_support_nhwc_pass",
+    "trt_map_ops_to_matrix_multiply_pass",
+    "simplify_with_basic_ops_pass",
+    // "conv_bn_fuse_pass",
+    // "conv_eltwiseadd_bn_fuse_pass",
+    "trt_embedding_eltwise_layernorm_fuse_pass",
+    "trt_skip_layernorm_fuse_pass",
+    "tensorrt_subgraph_pass",
+};
+
+const std::vector<std::string> kCINNCompilerPasses{
+    "gpu_cpu_map_matmul_v2_to_mul_pass",
+    "gpu_cpu_map_matmul_v2_to_matmul_pass",
+    "gpu_cpu_map_matmul_to_mul_pass",
+};
+
+const std::vector<std::string> CpuBasicPasses{
+    "simplify_with_basic_ops_pass",  //
+    "layer_norm_fuse_pass",
+    "attention_lstm_fuse_pass",       //
+    "seqconv_eltadd_relu_fuse_pass",  //
+    // "seqpool_concat_fuse_pass",    //
+    "seqpool_cvm_concat_fuse_pass",  //
+    // "embedding_fc_lstm_fuse_pass", //
+    // TODO(wilber): fix correctness problem.
+    // "fc_lstm_fuse_pass",                    //
+    "mul_lstm_fuse_pass",                      //
+    "fc_gru_fuse_pass",                        //
+    "mul_gru_fuse_pass",                       //
+    "seq_concat_fc_fuse_pass",                 //
+    "gpu_cpu_squeeze2_matmul_fuse_pass",       //
+    "gpu_cpu_reshape2_matmul_fuse_pass",       //
+    "gpu_cpu_flatten2_matmul_fuse_pass",       //
+    "matmul_v2_scale_fuse_pass",               //
+    "gpu_cpu_map_matmul_v2_to_mul_pass",       //
+    "gpu_cpu_map_matmul_v2_to_matmul_pass",    //
+    "matmul_scale_fuse_pass",                  //
+    "gpu_cpu_map_matmul_to_mul_pass",          //
+    "fc_fuse_pass",                            //
+    "repeated_fc_relu_fuse_pass",              //
+    "squared_mat_sub_fuse_pass",               //
+    "conv_bn_fuse_pass",                       //
+    "conv_eltwiseadd_bn_fuse_pass",            //
+    "conv_transpose_bn_fuse_pass",             //
+    "conv_transpose_eltwiseadd_bn_fuse_pass",  //
+    "is_test_pass",                            //
+    "constant_folding_pass",
+};
+
+GpuPassStrategy::GpuPassStrategy() : PassStrategy({}) {
+  passes_.assign({
+    "map_op_to_another_pass",                    //
+        "is_test_pass",                          //
+        "simplify_with_basic_ops_pass",          //
+        "delete_quant_dequant_linear_op_pass",   //
+        "delete_weight_dequant_linear_op_pass",  //
+#if defined _WIN32  // Windows does not support sparse_conv3d_implicit_gemm
+#else
+        "sparse_conv_optim_pass",              //
+#endif
+        "constant_folding_pass",                                        //
+        "silu_fuse_pass",                                               //
+        "conv_bn_fuse_pass",                                            //
+        "conv_eltwiseadd_bn_fuse_pass",                                 //
+        "embedding_eltwise_layernorm_fuse_pass",                        //
+        "multihead_matmul_fuse_pass_v2",                                //
+        "vit_attention_fuse_pass",                                      //
+        "fused_multi_transformer_encoder_pass",                         //
+        "fused_multi_transformer_decoder_pass",                         //
+        "fused_multi_transformer_encoder_fuse_qkv_pass",                //
+        "fused_multi_transformer_decoder_fuse_qkv_pass",                //
+        "multi_devices_fused_multi_transformer_encoder_pass",           //
+        "multi_devices_fused_multi_transformer_encoder_fuse_qkv_pass",  //
+        "multi_devices_fused_multi_transformer_decoder_fuse_qkv_pass",  //
+        "fuse_multi_transformer_layer_pass",                            //
+        "gpu_cpu_squeeze2_matmul_fuse_pass",                            //
+        "gpu_cpu_reshape2_matmul_fuse_pass",                            //
+        "gpu_cpu_flatten2_matmul_fuse_pass",                            //
+        "gpu_cpu_map_matmul_v2_to_mul_pass",                            //
+        "gpu_cpu_map_matmul_v2_to_matmul_pass",                         //
+        "matmul_scale_fuse_pass",                                       //
+        "multihead_matmul_fuse_pass_v3",                                //
+        "gpu_cpu_map_matmul_to_mul_pass",                               //
+        "fc_fuse_pass",                                                 //
+        "fc_elementwise_layernorm_fuse_pass",                           //
+// cudnn8.0 has memory leak problem in conv + eltwise + act, so we
+// disable the pass.
+#if !(CUDNN_VERSION >= 8000 && CUDNN_VERSION < 8100)
+        "conv_elementwise_add_act_fuse_pass",   //
+        "conv_elementwise_add2_act_fuse_pass",  //
+#endif
+        "conv_elementwise_add_fuse_pass",      //
+        "transpose_flatten_concat_fuse_pass",  //
+        "transfer_layout_pass",                //
+        "transfer_layout_elim_pass",
+        "auto_mixed_precision_pass",  //
+        "identity_op_clean_pass",  // should be after auto_mixed_precision_pass.
+        "inplace_op_var_pass",     // should be the last pass.
+  });
+
+  use_gpu_ = true;
+}
+
+void GpuPassStrategy::EnableCUDNN() {
+  if (!use_cudnn_) {
+    passes_.insert(passes_.begin(), "cudnn_placement_pass");
+  }
+  use_cudnn_ = true;
+}
+
+void GpuPassStrategy::EnableMKLDNN() {
+  LOG(WARNING) << ONEDNN_UPDATE_WARNING(EnableONEDNN);
+  EnableONEDNN();
+}
+void GpuPassStrategy::EnableONEDNN() {
+  LOG(ERROR) << "GPU not support MKLDNN yet";
+}
+
+void GpuPassStrategy::EnableMkldnnBfloat16() {
+  LOG(WARNING) << ONEDNN_UPDATE_WARNING(EnableOnednnBfloat16);
+  EnableOnednnBfloat16();
+}
+void GpuPassStrategy::EnableOnednnBfloat16() {
+  LOG(ERROR) << "GPU not support MKL-DNN bfloat16";
+}
+
+void GpuPassStrategy::EnableMkldnnInt8() {
+  LOG(WARNING) << ONEDNN_UPDATE_WARNING(EnableOnednnInt8);
+  EnableOnednnInt8();
+}
+void GpuPassStrategy::EnableOnednnInt8() {
+  LOG(ERROR) << "GPU not support MKL-DNN int8";
+}
+
+void GpuPassStrategy::DisableMkldnnFcPasses() {
+  LOG(WARNING) << ONEDNN_UPDATE_WARNING(DisableOnednnFcPasses);
+  DisableOnednnFcPasses();
+}
+void GpuPassStrategy::DisableOnednnFcPasses() {
+  LOG(ERROR) << "GPU not support MKL-DNN fc";
+}
+
+CpuPassStrategy::CpuPassStrategy() : PassStrategy({}) {
+  // NOTE the large fusions should be located in the front, so that they will
+  // not be damaged by smaller ones.
+  passes_.assign(CpuBasicPasses.begin(), CpuBasicPasses.end());
+
+  use_gpu_ = false;
+}
+
+void CpuPassStrategy::EnableCUDNN() { LOG(ERROR) << "CPU not support cuDNN"; }
+
+void CpuPassStrategy::EnableMKLDNN() {
+  LOG(WARNING) << ONEDNN_UPDATE_WARNING(EnableONEDNN);
+  EnableONEDNN();
+}
+void CpuPassStrategy::EnableONEDNN() {
+// TODO(Superjomn) Consider the way to mix CPU with GPU.
+#ifdef PADDLE_WITH_DNNL
+  if (!use_onednn_) {
+    passes_.insert(passes_.begin(), "onednn_placement_pass");
+
+    for (auto &pass : std::vector<std::string>({
+             "squeeze2_transpose2_onednn_fuse_pass",
+             "depthwise_conv_onednn_pass",    //
+             "conv_bn_fuse_pass",             // Execute BN passes again to
+             "conv_eltwiseadd_bn_fuse_pass",  // preserve correct pass order
+             "conv_affine_channel_onednn_fuse_pass",    //
+             "conv_transpose_bn_fuse_pass",             //
+             "conv_transpose_eltwiseadd_bn_fuse_pass",  //
+             "conv_bias_onednn_fuse_pass",              //
+             "conv_transpose_bias_onednn_fuse_pass",
+             // TODO(baoachun): Need to support 5-dimensional input.
+             // "conv3d_bias_onednn_fuse_pass",  //
+             "conv_elementwise_add_onednn_fuse_pass",
+             "conv_activation_onednn_fuse_pass",           //
+             "scale_matmul_fuse_pass",                     //
+             "reshape_transpose_matmul_onednn_fuse_pass",  //
+             "matmul_transpose_reshape_onednn_fuse_pass",  //
+             "matmul_elementwise_add_onednn_fuse_pass",    //
+             "matmul_activation_onednn_fuse_pass",         //
+             // Disabled due to topology-dependent speed-up
+             "fc_onednn_pass",
+             "fc_act_onednn_fuse_pass",
+             "self_attention_fuse_pass",              //
+             "batch_norm_act_fuse_pass",              //
+             "softplus_activation_onednn_fuse_pass",  //
+             "shuffle_channel_onednn_detect_pass",    //
+             "elementwise_act_onednn_fuse_pass",      //
+             "operator_scale_onednn_fuse_pass",       //
+             "operator_unsqueeze2_onednn_fuse_pass",  //
+             "operator_reshape2_onednn_fuse_pass",    //
+         })) {
+      passes_.push_back(pass);
+    }
+  }
+  use_onednn_ = true;
+#else
+  use_onednn_ = false;
+#endif
+}
+
+void CpuPassStrategy::DisableMKLDNN() {
+  LOG(WARNING) << ONEDNN_UPDATE_WARNING(DisableONEDNN);
+  DisableONEDNN();
+}
+void CpuPassStrategy::DisableONEDNN() {
+  ClearPasses();
+  passes_.assign(CpuBasicPasses.begin(), CpuBasicPasses.end());
+}
+
+void CpuPassStrategy::EnableMkldnnBfloat16() {
+  LOG(WARNING) << ONEDNN_UPDATE_WARNING(EnableOnednnBfloat16);
+  EnableOnednnBfloat16();
+}
+void CpuPassStrategy::EnableOnednnBfloat16() {
+#ifdef PADDLE_WITH_DNNL
+  if (!use_onednn_bfloat16_) {
+    passes_.emplace_back("fc_onednn_pass");
+    passes_.emplace_back("fc_act_onednn_fuse_pass");
+
+    passes_.emplace_back("cpu_bfloat16_placement_pass");
+    passes_.emplace_back("cpu_bfloat16_pass");
+    passes_.emplace_back("cpu_quantize_squash_pass");
+  }
+  use_onednn_bfloat16_ = true;
+#else
+  use_onednn_bfloat16_ = false;
+#endif
+}
+
+void CpuPassStrategy::EnableMkldnnInt8() {
+  LOG(WARNING) << ONEDNN_UPDATE_WARNING(EnableOnednnInt8);
+  EnableOnednnInt8();
+}
+void CpuPassStrategy::EnableOnednnInt8() {
+#ifdef PADDLE_WITH_DNNL
+  if (!use_onednn_int8_) {
+    passes_.clear();
+    passes_.emplace_back("simplify_with_basic_ops_pass");
+    passes_.emplace_back("quant_dequant_onednn_pass");
+    passes_.emplace_back("onednn_placement_pass");
+    passes_.emplace_back("constant_folding_pass");
+    passes_.emplace_back("squeeze2_transpose2_onednn_fuse_pass");
+    passes_.emplace_back("layer_norm_fuse_pass");
+    passes_.emplace_back("attention_lstm_fuse_pass");
+    passes_.emplace_back("seqconv_eltadd_relu_fuse_pass");
+    passes_.emplace_back("fc_lstm_fuse_pass");
+    passes_.emplace_back("mul_lstm_fuse_pass");
+    passes_.emplace_back("fc_gru_fuse_pass");
+    passes_.emplace_back("mul_gru_fuse_pass");
+    passes_.emplace_back("multi_gru_fuse_pass");
+    passes_.emplace_back("multi_gru_seq_fuse_pass");
+    passes_.emplace_back("seq_concat_fc_fuse_pass");
+    passes_.emplace_back("gpu_cpu_squeeze2_matmul_fuse_pass");
+    passes_.emplace_back("gpu_cpu_reshape2_matmul_fuse_pass");
+    passes_.emplace_back("gpu_cpu_flatten2_matmul_fuse_pass");
+    passes_.emplace_back("matmul_v2_scale_fuse_pass");
+    passes_.emplace_back("squared_mat_sub_fuse_pass");
+    passes_.emplace_back("is_test_pass");
+    passes_.emplace_back("gpu_cpu_map_matmul_v2_to_mul_pass");
+    passes_.emplace_back("gpu_cpu_map_matmul_v2_to_matmul_pass");
+    passes_.emplace_back("matmul_scale_fuse_pass");
+    passes_.emplace_back("gpu_cpu_map_matmul_to_mul_pass");
+    passes_.emplace_back("repeated_fc_relu_fuse_pass");
+    passes_.emplace_back("depthwise_conv_onednn_pass");
+    passes_.emplace_back("conv_bn_fuse_pass");
+    passes_.emplace_back("conv_eltwiseadd_bn_fuse_pass");
+    passes_.emplace_back("conv_affine_channel_onednn_fuse_pass");
+    passes_.emplace_back("conv_transpose_bn_fuse_pass");
+    passes_.emplace_back("conv_transpose_eltwiseadd_bn_fuse_pass");
+    passes_.emplace_back("conv_bias_onednn_fuse_pass");
+    passes_.emplace_back("conv_transpose_bias_onednn_fuse_pass");
+    passes_.emplace_back("conv_elementwise_add_onednn_fuse_pass");
+    passes_.emplace_back("conv_activation_onednn_fuse_pass");
+    passes_.emplace_back("fc_fuse_pass");
+    passes_.emplace_back("repeated_fc_relu_fuse_pass");
+    passes_.emplace_back("fc_onednn_pass");
+    passes_.emplace_back("fc_act_onednn_fuse_pass");
+    passes_.emplace_back("matmul_transpose_reshape_onednn_fuse_pass");
+    passes_.emplace_back("batch_norm_act_fuse_pass");
+    passes_.emplace_back("softplus_activation_onednn_fuse_pass");
+    passes_.emplace_back("compute_propagate_scales_onednn_pass");
+    passes_.emplace_back("scale_matmul_fuse_pass");
+    passes_.emplace_back("reshape_transpose_matmul_onednn_fuse_pass");
+    passes_.emplace_back("matmul_elementwise_add_onednn_fuse_pass");
+    passes_.emplace_back("operator_scale_onednn_fuse_pass");
+    passes_.emplace_back("operator_unsqueeze2_onednn_fuse_pass");
+    passes_.emplace_back("operator_reshape2_onednn_fuse_pass");
+    passes_.emplace_back("cpu_quantize_placement_pass");
+    passes_.emplace_back("cpu_quantize_pass");
+    passes_.emplace_back("cpu_quantize_squash_pass");
+    passes_.emplace_back("quant_transpose2_dequant_onednn_fuse_pass");
+  }
+  use_onednn_int8_ = true;
+#else
+  use_onednn_int8_ = false;
+#endif
+}
+
+void CpuPassStrategy::DisableMkldnnFcPasses() {
+  LOG(WARNING) << ONEDNN_UPDATE_WARNING(DisableOnednnFcPasses);
+  DisableOnednnFcPasses();
+}
+void CpuPassStrategy::DisableOnednnFcPasses() {
+#ifdef PADDLE_WITH_DNNL
+  if (!disable_onednn_fc_passes_) {
+    EraseFcOnednnPasses();
+  }
+  disable_onednn_fc_passes_ = true;
+#else
+  disable_onednn_fc_passes_ = false;
+#endif
+}
+
+void CpuPassStrategy::EraseFcMkldnnPasses() {
+  LOG(WARNING) << ONEDNN_UPDATE_WARNING(EraseFcOnednnPasses);
+  EraseFcOnednnPasses();
+}
+void CpuPassStrategy::EraseFcOnednnPasses() {
+  std::vector<std::string> fc_passes_to_erase(
+      {"fc_onednn_pass", "fc_act_onednn_fuse_pass"});
+  for (const auto &pass : fc_passes_to_erase) {
+    int idx = static_cast<int>(GetPassIndex(pass));
+    if (idx != -1) {
+      passes_.erase(std::begin(passes_) + idx);
+    }
+  }
+}
+
+XpuPassStrategy::XpuPassStrategy() : PassStrategy({}) {
+  passes_.assign({
+      "map_op_to_another_pass",
+      // "quant_dequant_xpu_pass", open this pass when use old int8 model
+      "delete_quant_dequant_linear_op_pass",
+      "delete_weight_dequant_linear_op_pass",
+      "delete_assign_op_pass",
+      "delete_dropout_op_pass",
+      "delete_concat_op_pass",
+      "gather_squeeze_pass",
+      "roformer_relative_pos_fuse_pass",
+      "delete_repeated_ops_pass",
+      "identity_op_clean_pass",
+      "fused_continuous_same_ops_pass",
+      "reshape_unstack_concat_fuse_pass",
+      "delete_op_device_pass",
+      "constant_folding_pass",
+      "cast_embedding_trans_ids_to_int32_pass",
+      "delete_elementwise_mul_op_pass",
+      "generate_sequence_xpu_fuse_pass",
+      "group_norm_silu_xpu_fuse_pass",
+      "layer_norm_relu_xpu_fuse_pass",
+      "embedding_with_eltwise_add_xpu_fuse_pass",
+      "qk_qkv_attention_xpu_fuse_pass",
+      "block_multihead_attention_xpu_pass",
+      "multi_encoder_xpu_fuse_pass",
+      "multi_encoder_xpu_adaptive_seqlen_fuse_pass",
+      "multi_encoder_xpu_slice_fuse_pass",
+      "weight_only_linear_xpu_pass",
+      "fused_multi_transformer_cachekv_layout_trans_pass",
+      "fused_multi_transformer_int8_cachekv_layout_trans_pass",
+      "cross_attention_xpu_fuse_pass",
+      "decoder_attention_xpu_fuse_pass",
+      "one_beam_size_fuse_pass",
+      "fold_interp_outsize_fuse_pass",
+      "fold_two_squeeze2_fuse_pass",
+      // "conv1d_xpu_fuse_pass",
+      "duplicated_transpose_fuse_pass",
+      "conv2d_bias_fuse_pass",
+      "redundant_unsqueeze_squeeze_elimination_pass",
+      "reduce_ops_fuse_pass",
+      "delete_cast_op_pass",
+      "xpu_delete_cast_op_pass",
+      "conv2d_trans_filter_dilations_nxn_to_1x1_pass",
+      "stack_fuse_pass",
+      "fused_multi_transformer_xpu_pass",
+      "fused_multi_transformer_int8_xpu_quant_pass",
+      "relu6_fuse_pass",
+      "sigmoid_elementmul_fuse_pass",
+      "layer_norm_fuse_pass",
+      "matmul_weight_trans_pass",
+      "map_matmulv2_to_matmul_xpu_pass",
+      "reshape2_matmul_xpu_fuse_pass",
+      "squeeze2_matmul_xpu_fuse_pass",
+      "redundant_squeeze_unsqueeze_elimination_pass",
+      "fc_xpu_fuse_pass",
+      "conv2d_xpu_fuse_pass",
+      "conv2d_transpose_xpu_fuse_pass",
+      "squeeze_excitation_fuse_pass",
+      "add_activation_xpu_fuse_pass",
+      "add_layernorm_xpu_fuse_pass",
+      "layer_norm_act_xpu_fuse_pass",
+      "fast_layernorm_xpu_fuse_pass",
+      "bn_act_xpu_fuse_pass",
+      "yolo_box_xpu_fuse_pass",
+      "fast_where_xpu_fuse_pass",
+      "elementwise_mul_add_fuse_pass",
+      "sine_pos_fuse_pass",
+      "pad2d_xpu_fuse_pass",
+      // "auto_mixed_precision_pass",
+      "cast_mixed_precision_op_fuse_pass",
+      "xpu_quantize_op_pass",
+      "xpu_quantize_squash_pass",
+      "link_xpu_op_max_pass",
+      "spatial_transformer_resblock_xpu_fuse_pass",
+      "delete_isolated_node_pass",
+      "inplace_op_var_pass",
+  });
+  use_xpu_ = true;
+}
+
+IpuPassStrategy::IpuPassStrategy() : PassStrategy({}) {
+  passes_.assign({"inference_process_pass"});
+}
+const std::vector<std::string> kPirCustomDevicePasses{
+    // Functional pass
+    "add_shadow_output_after_dead_parameter_pass",
+    "delete_quant_dequant_linear_op_pass",
+    "delete_weight_dequant_linear_op_pass",
+    "map_op_to_another_pass",
+    "identity_op_clean_pass",
+    "matmul_scale_fuse_pass",
+};
+
+const std::vector<std::string> kPirGpuPasses{
+    // Functional pass
+    "add_shadow_output_after_dead_parameter_pass",
+    "delete_quant_dequant_linear_op_pass",
+    "delete_weight_dequant_linear_op_pass",
+    "map_op_to_another_pass",
+    "identity_op_clean_pass",
+    // Operator fusion pass
+    "silu_fuse_pass",
+    "conv2d_bn_fuse_pass",
+    "conv2d_add_act_fuse_pass",
+    "conv2d_add_fuse_pass",
+    "embedding_eltwise_layernorm_fuse_pass",
+    "fused_rotary_position_embedding_pass",
+    "fused_flash_attn_pass",
+    "multihead_matmul_fuse_pass",
+    "fused_weight_only_linear_pass",
+    "matmul_add_act_fuse_pass",
+    "fc_elementwise_layernorm_fuse_pass",
+    "add_norm_fuse_pass",
+    "group_norm_silu_fuse_pass",
+    "matmul_scale_fuse_pass",
+    "matmul_transpose_fuse_pass",
+    "transpose_flatten_concat_fuse_pass",
+    "remove_redundant_transpose_pass",
+    "horizontal_fuse_pass",
+};
+
+const std::vector<std::string> kPirXpuPasses{
+    // Functional pass
+    "add_shadow_output_after_dead_parameter_pass",
+    "delete_quant_dequant_linear_op_pass",
+    "delete_weight_dequant_linear_op_pass",
+    "map_op_to_another_pass",
+    "identity_op_clean_pass",
+    // Operator fusion pass
+    "add_activation_xpu_fuse_pass",
+    "add_layernorm_xpu_fuse_pass",
+    "rms_norm_xpu_fuse_pass",
+    "elementwise_mul_add_xpu_fuse_pass",
+    "conv2d_xpu_fuse_pass",
+    "conv2d_add_xpu_fuse_pass",
+    "group_norm_silu_fuse_pass",
+    "fc_xpu_fuse_pass"};
+
+const std::vector<std::string> kPirOnednnPasses {
+  "add_shadow_output_after_dead_parameter_pass",
+      "delete_quant_dequant_linear_op_pass",      //
+      "delete_weight_dequant_linear_op_pass",     //
+      "depthwise_conv_onednn_pass",               //
+      "squeeze_transpose_onednn_fuse_pass",       //
+      "conv2d_bn_onednn_fuse_pass",               //
+      "conv2d_bias_bn_onednn_fuse_pass",          //
+      "conv2d_bias_fuse_pass",                    //
+      "conv2d_transpose_bn_fuse_pass",            //
+      "conv2d_transpose_bias_bn_fuse_pass",       //
+      "conv2d_transpose_bias_fuse_pass",          //
+      "conv3d_bias_fuse_pass",                    //
+      "conv_elementwise_add_onednn_fuse_pass",    //
+      "conv_activation_onednn_fuse_pass",         //
+      "conv_concat_activation_onednn_fuse_pass",  //
+      "matmul_scale_fuse_pass",                   //
+      "scale_matmul_fuse_pass",                   //
+      "reshape_transpose_matmul_fuse_pass",       //
+      "matmul_transpose_reshape_fuse_pass",       //
+      "matmul_add_act_fuse_pass",                 //
+      "matmul_reshape_add_fuse_pass",             //
+      "fc_onednn_enable_pass",                    //
+      "matmul_elementwise_add_fuse_pass",         //
+      "matmul_activation_fuse_pass",              //
+      "matmul_add_act_fuse_pass",                 //
+      "fc_onednn_enable_pass",                    //
+      "fc_activation_fuse_pass",                  //
+#if defined(PADDLE_WITH_AVX512F) && defined(PADDLE_WITH_MKLML) && \
+    defined(PADDLE_WITH_DNNL)
+      "self_attention_fuse_pass",  //
+#endif
+      "batch_norm_act_fuse_pass",             //
+      "softplus_activation_fuse_pass",        //
+      "shuffle_channel_detect_pass",          //
+      "elementwise_act_onednn_fuse_pass",     //
+      "operator_scale_onednn_fuse_pass",      //
+      "operator_unsqueeze_onednn_fuse_pass",  //
+      "operator_reshape_onednn_fuse_pass",    //
+      "onednn_placement_pass",                //
+};
+
+const std::vector<std::string> kPirOnednnBf16Passes{
+    "add_shadow_output_after_dead_parameter_pass",
+    "cpu_bfloat16_placement_pass",
+    "cpu_bfloat16_pass",
+    "cpu_bfloat16_type_placement_pass",
+    "cpu_special_ops_bf16_pass",
+    "cpu_bf16_quantize_squash_pass",
+};
+
+const std::vector<std::string> kPirCpuPasses{
+    "add_shadow_output_after_dead_parameter_pass",
+    "delete_quant_dequant_linear_op_pass",
+    "delete_weight_dequant_linear_op_pass"};
+
+}  // namespace paddle

paddle/fluid/inference/api/paddle_pass_builder.h

@@ -0,0 +1,398 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <sstream>
+#include <string>
+#include <unordered_set>
+#include <vector>
+
+#include "paddle_infer_declare.h"  // NOLINT
+
+///
+/// \file paddle_pass_builder.h
+///
+/// \brief Class Paddle Pass Builder and its subclasses(pass strategies).
+/// \section sec_intro Introduction
+/// This class aims to build passes for paddle and define passes' strategies.
+///
+/// \author paddle-infer@baidu.com
+/// \date 2020-3-23
+/// \since 1.7
+
+/// \namespace paddle
+namespace paddle {
+
+/// \class PaddlePassBuilder
+/// \brief This class build passes based on vector<string> input. It is part of
+/// inference API. Users can build passes, insert new passes, delete passes
+/// using this class and its functions.
+///
+/// Example Usage:
+///     Build a new pass.
+/// \code{cpp}
+/// const vector<string> passes(1, "conv_relu_onednn_fuse_pass");
+/// PaddlePassBuilder builder(passes);
+/// \endcode
+class PD_INFER_DECL PaddlePassBuilder {
+ public:
+  /// \brief Constructor of the class. It stores the input passes.
+  /// \param[in] passes passes' types.
+  explicit PaddlePassBuilder(const std::vector<std::string> &passes)
+      : passes_(passes) {}
+
+  /// \brief Stores the input passes.
+  /// \param[in] passes passes' types.
+  void SetPasses(std::initializer_list<std::string> passes) {
+    passes_ = passes;
+  }
+
+  /// \brief Append a pass to the end of the passes.
+  /// \param[in] pass_type the type of the new pass.
+  void AppendPass(const std::string &pass_type);
+
+  /// \brief Insert a pass to a specific position.
+  /// \param[in] idx the position to insert.
+  /// \param[in] pass_type the type of insert pass.
+  void InsertPass(size_t idx, const std::string &pass_type);
+
+  /// \brief Delete the pass at certain position 'idx'.
+  /// \param[in] idx the position to delete.
+  void DeletePass(size_t idx);
+
+  /// \brief Get the certain position of a pass.
+  /// \param[in] pass_type the type of insert pass.
+  size_t GetPassIndex(const std::string &pass_type);
+
+  /// \brief Delete all passes that has a certain type 'pass_type'.
+  /// \param[in] pass_type the certain pass type to be deleted.
+  void DeletePass(const std::string &pass_type);
+
+  /// \brief Delete all the passes.
+  void ClearPasses();
+
+  /// \brief Append an analysis pass.
+  /// \param[in] pass the type of the new analysis pass.
+  void AppendAnalysisPass(const std::string &pass);
+
+  /// \brief Visualize the computation graph after each pass by generating a DOT
+  /// language file, one can draw them with the Graphviz toolkit.
+  void TurnOnDebug();
+  /// \brief Human-readable information of the passes.
+  std::string DebugString();
+
+  /// \brief Get information of passes.
+  /// \return Return list of the passes.
+  const std::vector<std::string> &AllPasses() const { return passes_; }
+
+  /// \brief Get information of analysis passes.
+  /// \return Return list of analysis passes.
+  std::vector<std::string> AnalysisPasses() const {
+    auto passes = analysis_passes_;
+    // To make sure the ir_graph_to_program should be the last pass so any
+    // modification of IR will persist to the program.
+    passes.push_back("ir_graph_to_program_pass");
+    return passes;
+  }
+
+  const std::unordered_set<std::string> &GetAllDeletedPasses() const {
+    return deleted_passes_;
+  }
+
+ protected:
+  /// \cond Protected
+  std::vector<std::string> analysis_passes_{{
+      "ir_graph_build_pass",
+      "ir_analysis_pass",
+      "ir_params_sync_among_devices_pass",
+      "adjust_cudnn_workspace_size_pass",
+      "inference_op_replace_pass",
+      "save_optimized_model_pass",
+  }};
+  std::vector<std::string> passes_;
+  std::unordered_set<std::string> deleted_passes_;
+  /// \endcond
+};
+
+/// \class PassStrategy
+/// \brief This class defines the pass strategies like whether to use gpu/cuDNN
+/// kernel/MKLDNN.
+class PD_INFER_DECL PassStrategy : public PaddlePassBuilder {
+ public:
+  /// \brief Constructor of PassStrategy class. It works the same as
+  /// PaddlePassBuilder class. \param[in] passes passes' types.
+  explicit PassStrategy(const std::vector<std::string> &passes)
+      : PaddlePassBuilder(passes) {}
+
+  /// \brief Enable the use of cuDNN kernel.
+  virtual void EnableCUDNN() {}
+
+  /// \brief Enable the use of OneDNN.
+  /// The OneDNN control exists in both CPU and GPU mode, because there can
+  /// still be some CPU kernels running in GPU mode.
+  virtual void EnableMKLDNN() {}  // deprecated
+
+  /// \brief Disable the use of OneDNN.
+  virtual void DisableMKLDNN() {}  // deprecated
+
+  /// \brief Enable OneDNN bfloat16.
+  virtual void EnableMkldnnBfloat16() {}  // deprecated
+
+  /// \brief Enable OneDNN int8.
+  virtual void EnableMkldnnInt8() {}  // deprecated
+
+  /// \brief Disable OneDNN fc passes.
+  virtual void DisableMkldnnFcPasses() {}  // deprecated
+
+  /// \brief Enable the use of OneDNN.
+  /// The OneDNN control exists in both CPU and GPU mode, because there can
+  /// still be some CPU kernels running in GPU mode.
+  virtual void EnableONEDNN() {}
+
+  /// \brief Disable the use of OneDNN.
+  virtual void DisableONEDNN() {}
+
+  /// \brief Enable OneDNN bfloat16.
+  virtual void EnableOnednnBfloat16() {}
+
+  /// \brief Enable OneDNN int8.
+  virtual void EnableOnednnInt8() {}
+
+  /// \brief Disable OneDNN fc passes.
+  virtual void DisableOnednnFcPasses() {}
+
+  /// \brief Check if we are using gpu.
+  /// \return A bool variable implying whether we are in gpu mode.
+  bool use_gpu() const { return use_gpu_; }
+
+  /// \brief Check if we are using xpu.
+  /// \return A bool variable implying whether we are in xpu mode.
+  bool use_xpu() const { return use_xpu_; }
+
+  /// \brief Check if we are using ipu.
+  /// \return A bool variable implying whether we are in ipu mode.
+  bool use_ipu() const { return use_ipu_; }
+
+  /// \brief Check if we are using CustomDevice.
+  /// \return A bool variable implying whether we are in CustomDevice mode.
+  bool use_custom_device() const { return use_custom_device_; }
+
+  /// \brief Default destructor.
+  virtual ~PassStrategy() = default;
+
+ protected:
+  /// \cond Protected
+  bool use_xpu_{false};
+  bool use_gpu_{false};
+  bool use_ipu_{false};
+  bool use_onednn_{false};
+  bool use_custom_device_{false};
+  /// \endcond
+};
+
+/// \class CpuPassStrategy
+/// \brief The CPU passes controller, it is used in AnalysisPredictor with CPU
+/// mode.
+class PD_INFER_DECL CpuPassStrategy : public PassStrategy {
+ public:
+  /// \brief Default constructor of CpuPassStrategy.
+  CpuPassStrategy();
+
+  /// \brief Construct by copying another CpuPassStrategy object.
+  /// \param[in] other The CpuPassStrategy object we want to copy.
+  explicit CpuPassStrategy(const CpuPassStrategy &other)
+      : PassStrategy(other.AllPasses()) {
+    use_gpu_ = other.use_gpu_;
+    use_onednn_ = other.use_onednn_;
+    use_onednn_bfloat16_ = other.use_onednn_bfloat16_;
+    use_onednn_int8_ = other.use_onednn_int8_;
+    disable_onednn_fc_passes_ = other.disable_onednn_fc_passes_;
+    deleted_passes_ = other.deleted_passes_;
+  }
+  /// \brief Default destructor.
+  virtual ~CpuPassStrategy() = default;
+
+  /// \brief Enable the use of cuDNN kernel.
+  void EnableCUDNN() override;
+
+  /// \brief Enable the use of OneDNN.
+  void EnableMKLDNN() override;  // deprecated
+
+  /// \brief Disable the use of OneDNN.
+  void DisableMKLDNN() override;  // deprecated
+
+  /// \brief Enable OneDNN bfloat16.
+  void EnableMkldnnBfloat16() override;  // deprecated
+
+  /// \brief Enable OneDNN int8.
+  void EnableMkldnnInt8() override;  // deprecated
+
+  /// \brief Disable OneDNN fc passes.
+  void DisableMkldnnFcPasses() override;  // deprecated
+
+  /// \brief Enable the use of OneDNN.
+  void EnableONEDNN() override;
+
+  /// \brief Disable the use of OneDNN.
+  void DisableONEDNN() override;
+
+  /// \brief Enable OneDNN bfloat16.
+  void EnableOnednnBfloat16() override;
+
+  /// \brief Enable OneDNN int8.
+  void EnableOnednnInt8() override;
+
+  /// \brief Disable OneDNN fc passes.
+  void DisableOnednnFcPasses() override;
+
+ protected:
+  /// \brief Erase OneDNN fc passes.
+  void EraseFcMkldnnPasses();  // deprecated
+
+  /// \brief Erase OneDNN fc passes.
+  void EraseFcOnednnPasses();
+
+  /// \cond Protected
+  bool use_onednn_bfloat16_{false};
+  bool use_onednn_int8_{false};
+  bool disable_onednn_fc_passes_{false};
+  /// \endcond
+};
+
+/// \class GpuPassStrategy
+/// \brief The GPU passes controller, it is used in AnalysisPredictor with GPU
+/// mode.
+class PD_INFER_DECL GpuPassStrategy : public PassStrategy {
+ public:
+  /// \brief Default constructor of GpuPassStrategy.
+  GpuPassStrategy();
+
+  /// \brief Construct by copying another GpuPassStrategy object.
+  /// \param[in] other The GpuPassStrategy object we want to copy.
+  explicit GpuPassStrategy(const GpuPassStrategy &other)
+      : PassStrategy(other.AllPasses()) {
+    use_gpu_ = true;
+    use_cudnn_ = other.use_cudnn_;
+    deleted_passes_ = other.deleted_passes_;
+  }
+
+  /// \brief Enable the use of cuDNN kernel.
+  void EnableCUDNN() override;
+
+  /// \brief Not supported in GPU mode yet.
+  void EnableMKLDNN() override;  // deprecated
+
+  /// \brief Not supported in GPU mode yet.
+  void EnableMkldnnBfloat16() override;  // deprecated
+
+  /// \brief Not supported in GPU mode yet.
+  void EnableMkldnnInt8() override;  // deprecated
+
+  /// \brief Disable OneDNN fc passes.
+  void DisableMkldnnFcPasses() override;  // deprecated
+
+  /// \brief Not supported in GPU mode yet.
+  void EnableONEDNN() override;
+
+  /// \brief Not supported in GPU mode yet.
+  void EnableOnednnBfloat16() override;
+
+  /// \brief Not supported in GPU mode yet.
+  void EnableOnednnInt8() override;
+
+  /// \brief Disable OneDNN fc passes.
+  void DisableOnednnFcPasses() override;
+
+  /// \brief Default destructor.
+  virtual ~GpuPassStrategy() = default;
+
+ protected:
+  /// \cond Protected
+  bool use_cudnn_{false};
+  /// \endcond
+};
+
+/// \class XpuPassStrategy
+/// \brief The XPU passes controller, it is used in AnalysisPredictor with XPU
+/// mode.
+class PD_INFER_DECL XpuPassStrategy final : public PassStrategy {
+ public:
+  XpuPassStrategy();
+  explicit XpuPassStrategy(const XpuPassStrategy &other)
+      : PassStrategy(other.AllPasses()) {
+    use_xpu_ = true;
+    deleted_passes_ = other.deleted_passes_;
+  }
+};
+
+/// \class CustomDevicePassStrategy
+/// \brief The CustomDevice passes controller, it is used in AnalysisPredictor
+/// with CustomDevice
+/// mode.
+class PD_INFER_DECL CustomDevicePassStrategy final : public PassStrategy {
+ public:
+  CustomDevicePassStrategy() : PassStrategy({}) { use_custom_device_ = true; }
+
+  /// \brief Construct by copying another CustomDevicePassStrategy object.
+  /// \param[in] other The CustomDevicePassStrategy object we want to copy.
+  explicit CustomDevicePassStrategy(const CustomDevicePassStrategy &other)
+      : PassStrategy(other.AllPasses()) {
+    use_custom_device_ = true;
+    deleted_passes_ = other.deleted_passes_;
+  }
+};
+
+/// \class IpuPassStrategy
+/// \brief The IPU passes controller, it is used in AnalysisPredictor with IPU
+/// mode.
+class PD_INFER_DECL IpuPassStrategy final : public PassStrategy {
+ public:
+  /// \brief Default constructor of IpuPassStrategy.
+  IpuPassStrategy();
+
+  /// \brief Construct by copying another IpuPassStrategy object.
+  /// \param[in] other The IpuPassStrategy object we want to copy.
+  explicit IpuPassStrategy(const IpuPassStrategy &other)
+      : PassStrategy(other.AllPasses()) {
+    use_ipu_ = true;
+    deleted_passes_ = other.deleted_passes_;
+  }
+};
+
+#ifdef PADDLE_WITH_OPENVINO
+/// \brief List of OpenVINO subgraph passes.
+PD_INFER_DECL extern const std::vector<std::string> kOVSubgraphPasses;
+#endif
+
+/// \brief List of tensorRT subgraph passes.
+PD_INFER_DECL extern const std::vector<std::string> kTRTSubgraphPasses;
+
+/// \brief List of cinn compiler passes.
+PD_INFER_DECL extern const std::vector<std::string> kCINNCompilerPasses;
+
+/// \brief TODO(inference): Most of the existing pass fusion operators do not
+/// support fp16/bf16 precision, temporarily use low precision pass to prevent
+/// running errors. After fusion operator supports low precision, delete this.
+PD_INFER_DECL extern const std::vector<std::string> kGpuLowerPrecisionPasses;
+PD_INFER_DECL extern const std::vector<std::string> kTrtLowerPrecisionPasses;
+
+PD_INFER_DECL extern const std::vector<std::string> kPirCustomDevicePasses;
+PD_INFER_DECL extern const std::vector<std::string> kPirGpuPasses;
+PD_INFER_DECL extern const std::vector<std::string> kPirCpuPasses;
+PD_INFER_DECL extern const std::vector<std::string> kPirXpuPasses;
+PD_INFER_DECL extern const std::vector<std::string> kPirOnednnPasses;
+PD_INFER_DECL extern const std::vector<std::string> kPirOnednnBf16Passes;
+
+}  // namespace paddle

paddle/fluid/inference/api/paddle_tensor.h

@@ -0,0 +1,237 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "paddle_infer_declare.h"  // NOLINT
+
+#ifdef PADDLE_WITH_ONNXRUNTIME
+#include "onnxruntime_c_api.h"    // NOLINT
+#include "onnxruntime_cxx_api.h"  // NOLINT
+#endif
+
+namespace paddle {
+class Tensor;
+}
+
+namespace paddle_infer {
+
+/// \brief  Experimental.
+/// Strings for text data.
+using Strings = std::vector<std::string>;
+
+using OutputTensorHookFunc = std::function<void(
+    const std::string&, const std::string&, const paddle::Tensor&)>;
+using InputTensorHookFunc = OutputTensorHookFunc;
+
+typedef void (*CallbackFunc)(void*);
+
+#if defined(PADDLE_WITH_TESTING) && defined(PADDLE_WITH_INFERENCE_API_TEST)
+class InferApiTesterUtils;
+#endif
+
+namespace contrib {
+class TensorUtils;
+}
+
+namespace experimental {
+class InternalUtils;
+};
+
+/// \brief Paddle data type.
+enum DataType {
+  FLOAT32,
+  INT64,
+  INT32,
+  UINT8,
+  INT8,
+  FLOAT16,
+  BOOL,
+  FLOAT64,
+  BFLOAT16,
+  FLOAT8E4M3,
+  // TODO(Inference): support more data types if needed.
+};
+
+enum class PlaceType { kUNK = -1, kCPU, kGPU, kXPU, kIPU, kCUSTOM };
+
+enum class DataLayout { kUNK = -1, kAny, kNHWC, kNCHW };
+
+/// \brief Represents an n-dimensional array of values.
+/// The Tensor is used to store the input or output of the network.
+/// Zero copy means that the tensor supports direct copy of host or device data
+/// to device,
+/// eliminating additional CPU copy. Tensor is only used in the
+/// AnalysisPredictor.
+/// It is obtained through PaddlePredictor::GetInputTensor()
+/// and PaddlePredictor::GetOutputTensor() interface.
+class PD_INFER_DECL Tensor {
+ public:
+  /// \brief Reset the shape of the tensor.
+  /// Generally it's only used for the input tensor.
+  /// Reshape must be called before calling mutable_data() or copy_from_cpu()
+  /// \param shape The shape to set.
+  void Reshape(const std::vector<int>& shape);
+
+  /// \brief Experimental interface.
+  /// Reset the shape of the Strings tensor.
+  /// Generally it's only used for the input tensor.
+  /// Reshape must be called before calling
+  /// ZeroCopyStringTensorCreate() or PaddleInferTensorCreate()
+  /// \param shape The shape to set.
+  void ReshapeStrings(const std::size_t& shape);
+
+  /// \brief Get the memory pointer in CPU or GPU with specific data type.
+  /// Please Reshape the tensor first before call this.
+  /// It's usually used to get input data pointer.
+  /// \param place The place of the tensor.
+  template <typename T>
+  T* mutable_data(PlaceType place);
+
+  /// \brief Get the memory pointer directly.
+  /// It's usually used to get the output data pointer.
+  /// \param[out] place To get the device type of the tensor.
+  /// \param[out] size To get the data size of the tensor.
+  /// \return The tensor data buffer pointer.
+  template <typename T>
+  T* data(PlaceType* place, int* size) const;
+
+  /// \brief Copy the host memory to tensor data.
+  /// It's usually used to set the input tensor data.
+  /// \param data The pointer of the data, from which the tensor will copy.
+  template <typename T>
+  void CopyFromCpu(const T* data);
+
+  /// \brief Share the data with tensor data.
+  /// It's usually used to set the tensor data.
+  /// \param data The pointer of the data, from which the tensor will share.
+  /// \param shape The shape of data.
+  /// \param place The place of data.
+  /// \param layout The layout of data. Only NCHW is supported now.
+  template <typename T>
+  void ShareExternalData(const T* data,
+                         const std::vector<int>& shape,
+                         PlaceType place,
+                         DataLayout layout = DataLayout::kNCHW);
+
+  /// \brief Experimental interface.
+  /// It's usually used to set the input tensor data with Strings data type.
+  /// \param data The pointer of the data, from which the tensor will copy.
+  void CopyStringsFromCpu(const paddle_infer::Strings* data);
+
+  /// \brief Copy the tensor data to the host memory.
+  /// It's usually used to get the output tensor data.
+  /// \param[out] data The tensor will copy the data to the address.
+  template <typename T>
+  void CopyToCpu(T* data) const;
+
+  /// \brief Copy the tensor data to the host memory asynchronously.
+  /// \param[out] data The tensor will copy the data to the address.
+  /// \param[out] exec_stream The tensor will execute copy in this stream(Only
+  /// GPU CUDA stream supported now).
+  template <typename T>
+  void CopyToCpuAsync(T* data, void* exec_stream) const;
+
+  /// \brief Copy the tensor data to the host memory asynchronously.
+  /// \param[out] data The tensor will copy the data to the address.
+  /// \param[out] cb Callback function cb(cb_params) will be executed on the
+  /// host after all currently enqueued items in the stream have completed .
+  template <typename T>
+  void CopyToCpuAsync(T* data, CallbackFunc cb, void* cb_params) const;
+
+  /// \brief Return the shape of the Tensor.
+  std::vector<int> shape() const;
+
+  /// \brief Set lod info of the tensor.
+  /// More about LOD can be seen here:
+  ///  https://www.paddlepaddle.org.cn/documentation/docs/zh/beginners_guide/basic_concept/lod_tensor.html#lodtensor
+  /// \param x the lod info.
+  void SetLoD(const std::vector<std::vector<size_t>>& x);
+  /// \brief Return the lod info of the tensor.
+  std::vector<std::vector<size_t>> lod() const;
+  /// \brief Return the name of the tensor.
+  const std::string& name() const;
+
+  /// \brief Return the data type of the tensor.
+  /// It's usually used to get the output tensor data type.
+  /// \return The data type of the tensor.
+  DataType type() const;
+
+  /// \brief Return the place type of the tensor.
+  /// \return The place type of the tensor.
+  PlaceType place() const;
+
+ protected:
+  explicit Tensor(void* scope, const void* device_contexts);
+
+  template <typename T>
+  void* FindTensor() const;
+
+  void SetPlace(PlaceType place,
+                int device = -1,
+                const std::string device_type = "");
+
+  void SetName(const std::string& name);
+
+  template <typename T>
+  void CopyToCpuImpl(T* data,
+                     void* stream = nullptr,
+                     CallbackFunc cb = nullptr,
+                     void* cb_params = nullptr) const;
+
+  std::string name_;
+  // The corresponding tensor pointer inside Paddle workspace is cached for
+  // performance.
+  mutable void* tensor_{nullptr};
+  DataType dtype_;
+  bool input_or_output_;
+  void* scope_{nullptr};
+  const void* device_contexts_{nullptr};
+  PlaceType place_;
+  int device_;
+  std::string device_type_;
+
+#ifdef PADDLE_WITH_ONNXRUNTIME
+  bool is_ort_tensor_{false};
+  std::vector<int64_t> shape_;
+  std::weak_ptr<Ort::IoBinding> binding_;
+  int idx_{-1};
+
+  void SetOrtMark(bool is_ort_tensor);
+
+  void SetOrtBinding(const std::shared_ptr<Ort::IoBinding> binding);
+
+  template <typename T>
+  T* ORTGetMutableData();
+
+  template <typename T>
+  void ORTCopyFromCpu(const T* data);
+
+  template <typename T>
+  void ORTCopyToCpu(T* data) const;
+#endif
+
+  friend class paddle_infer::contrib::TensorUtils;
+  friend class paddle_infer::experimental::InternalUtils;
+#if defined(PADDLE_WITH_TESTING) && defined(PADDLE_WITH_INFERENCE_API_TEST)
+  friend class paddle_infer::InferApiTesterUtils;
+#endif
+};
+
+}  // namespace paddle_infer

paddle/fluid/inference/api/resource_manager.cc

@@ -0,0 +1,477 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/inference/api/resource_manager.h"
+
+#include <functional>
+#include <memory>
+#include <mutex>
+#include <unordered_map>
+#include <utility>
+
+#include "paddle/common/errors.h"
+#include "paddle/phi/backends/gpu/forwards.h"
+#include "paddle/phi/backends/gpu/gpu_decls.h"
+#include "paddle/phi/backends/gpu/gpu_info.h"
+#include "paddle/phi/backends/gpu/gpu_resources.h"
+#include "paddle/phi/common/place.h"
+#include "paddle/phi/core/allocator.h"
+#include "paddle/phi/core/generator.h"
+#include "paddle/phi/core/memory/allocation/allocator_facade.h"
+#include "paddle/phi/core/platform/device/gpu/gpu_types.h"
+#include "unsupported/Eigen/CXX11/Tensor"
+
+#include "paddle/fluid/platform/enforce.h"
+
+#ifdef PADDLE_WITH_CUDA
+#include "paddle/phi/backends/dynload/cublas.h"
+#include "paddle/phi/backends/dynload/cudnn.h"
+#include "paddle/phi/backends/dynload/cusolver.h"
+#include "paddle/phi/backends/dynload/cusparse.h"
+#endif  // PADDLE_WITH_CUDA
+
+namespace paddle {
+namespace internal {
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+class EigenGpuStreamDevice : public Eigen::StreamInterface {
+ public:
+  EigenGpuStreamDevice()
+      : stream_(nullptr),
+        allocator_(nullptr),
+        device_prop_(nullptr),
+        semaphore_(nullptr),
+        allocations_() {
+    Eigen::initializeDeviceProp();
+  }
+  ~EigenGpuStreamDevice() override = default;
+
+  void Reinitialize(gpuStream_t cuda_stream,
+                    phi::Allocator* allocator,
+                    GPUPlace place) {
+    stream_ = cuda_stream;
+    allocator_ = allocator;
+    device_prop_ = &Eigen::m_deviceProperties[place.device];
+  }
+
+  const gpuStream_t& stream() const override { return stream_; }
+
+  const gpuDeviceProp& deviceProperties() const override {
+    return *device_prop_;
+  }
+
+  void* allocate(size_t num_bytes) const override {
+    if (UNLIKELY(num_bytes == 0)) {
+      return nullptr;
+    }
+    auto buf = allocator_->Allocate(num_bytes);
+    VLOG(4) << "Eigen allocated at " << buf->ptr() << " requested "
+            << num_bytes;
+    void* retv = buf->ptr();
+    {
+      std::lock_guard<std::mutex> lock(mtx_);
+      allocations_.emplace(retv, std::move(buf));
+    }
+    return retv;
+  }
+
+  void deallocate(void* buffer) const override {
+    if (LIKELY(buffer)) {
+      std::lock_guard<std::mutex> lock(mtx_);
+      allocations_.erase(buffer);
+    }
+  }
+
+  void* scratchpad() const override {
+    if (scratch_ == nullptr) {
+      scratch_ = allocate(Eigen::kGpuScratchSize + sizeof(unsigned int));
+    }
+    return scratch_;
+  }
+
+  unsigned int* semaphore() const override {
+    if (semaphore_ == nullptr) {
+      char* scratch = static_cast<char*>(scratchpad()) + Eigen::kGpuScratchSize;
+      semaphore_ = reinterpret_cast<unsigned int*>(scratch);
+#ifdef PADDLE_WITH_HIP
+      PADDLE_ENFORCE_GPU_SUCCESS(
+          hipMemsetAsync(semaphore_, 0, sizeof(unsigned int), stream_));
+#else
+      PADDLE_ENFORCE_GPU_SUCCESS(
+          cudaMemsetAsync(semaphore_, 0, sizeof(unsigned int), stream_));
+#endif
+    }
+    return semaphore_;
+  }
+
+ private:
+  gpuStream_t stream_;                // not owned;
+  phi::Allocator* allocator_;         // not owned;
+  const gpuDeviceProp* device_prop_;  // not owned;
+  mutable void* scratch_;
+  mutable unsigned int* semaphore_;
+  mutable std::mutex mtx_;  // to protect allocations_
+  mutable std::unordered_map<void*, phi::Allocator::AllocationPtr> allocations_;
+};
+#endif
+}  // namespace internal
+
+Eigen::DefaultDevice* CPUContextResource::GetCPUEigenDevice() const {
+  return cpu_eigen_device_.get();
+}
+
+void CPUContextResource::InitCPUResource() {
+  cpu_eigen_device_ = std::make_unique<Eigen::DefaultDevice>();
+}
+
+CPUContextResource::CPUContextResource() : cpu_eigen_device_(nullptr) {
+  InitCPUResource();
+}
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+GPUContextResource::GPUContextResource(const phi::Place& place, void* stream)
+    : place_(place),
+      compute_capability_(0),
+      runtime_version_(0),
+      driver_version_(0),
+      multi_process_(0),
+      max_threads_per_mp_(0),
+      max_threads_per_block_(0),
+      stream_(nullptr),
+      gpu_eigen_device_(nullptr),
+      eigen_stream_(nullptr) {
+  InitGPUResource(stream);
+}
+
+GPUContextResource::~GPUContextResource() { DestroyGPUResource(); }  // NOLINT
+
+void GPUContextResource::InitGPUResource(void* stream) {
+  phi::backends::gpu::GPUDeviceGuard guard(place_.device);
+  if (stream == nullptr) {
+    owned_stream_ = true;
+    phi::InitStream(&stream_);
+  } else {
+    owned_stream_ = false;
+    stream_ = reinterpret_cast<gpuStream_t>(stream);
+  }
+
+  InitGpuProperties();
+  InitGpuEigenDevice();
+}
+
+void GPUContextResource::DestroyGPUResource() {
+  if (owned_stream_) {
+#ifdef PADDLE_WITH_HIP
+    PADDLE_ENFORCE_GPU_SUCCESS(hipStreamDestroy(stream_));
+#else
+    PADDLE_ENFORCE_GPU_SUCCESS(cudaStreamDestroy(stream_));
+#endif
+    stream_ = nullptr;
+  }
+
+  DestroyDnnHandle();
+  DestroyBlasHandle();
+  DestroyBlasLtHandle();
+  DestroySolverHandle();
+  DestroySparseHandle();
+}
+
+void GPUContextResource::InitGpuProperties() {
+  phi::InitGpuProperties(place_,
+                         &compute_capability_,
+                         &runtime_version_,
+                         &driver_version_,
+                         &multi_process_,
+                         &max_threads_per_mp_,
+                         &max_threads_per_block_,
+                         &max_grid_dim_size_);
+}
+
+void GPUContextResource::InitGpuEigenDevice() {
+  auto* allocator = paddle::memory::allocation::AllocatorFacade::Instance()
+                        .GetAllocator(place_)
+                        .get();
+  eigen_stream_ = std::make_unique<internal::EigenGpuStreamDevice>();
+  eigen_stream_->Reinitialize(stream_, allocator, place_);
+  gpu_eigen_device_ = std::make_unique<Eigen::GpuDevice>(eigen_stream_.get());
+}
+
+void GPUContextResource::InitDnnHandle() {
+  phi::InitDnnHandle(&dnn_handle_, stream_, place_);
+}
+
+void GPUContextResource::DestroyDnnHandle() {
+  phi::DestroyDnnHandle(dnn_handle_);
+}
+
+void GPUContextResource::DestroyBlasHandle() {
+  phi::DestroyBlasHandle(blas_handle_);
+  phi::DestroyBlasHandle(blas_tensor_core_handle_);
+  phi::DestroyBlasHandle(blas_tf32_tensor_core_handle_);
+}
+
+void GPUContextResource::InitBlasLtHandle() {
+#ifdef PADDLE_WITH_HIP
+  phi::InitBlasLtHandle(reinterpret_cast<void**>(&blaslt_handle_));
+#else  // PADDLE_WITH_CUDA
+  phi::InitBlasLtHandle(&blaslt_handle_);
+#endif
+}
+
+void GPUContextResource::DestroyBlasLtHandle() {
+  phi::DestroyBlasLtHandle(blaslt_handle_);
+}
+
+void GPUContextResource::InitSolverHandle() {
+  phi::InitSolverHandle(&solver_handle_, stream_);
+}
+
+void GPUContextResource::DestroySolverHandle() {
+  phi::DestroySolverHandle(solver_handle_);
+}
+
+void GPUContextResource::InitSparseHandle() {
+  phi::InitSparseHandle(&sparse_handle_, stream_);
+}
+
+void GPUContextResource::DestroySparseHandle() {
+  phi::DestroySparseHandle(sparse_handle_);
+}
+
+phi::Place GPUContextResource::Place() const { return place_; }
+
+gpuStream_t GPUContextResource::GetStream() const { return stream_; }
+
+dnnHandle_t GPUContextResource::GetDnnHandle() const { return dnn_handle_; }
+
+std::function<phi::dnnHandle_t()> GPUContextResource::GetDnnHandleCreator() {
+  return [&]() -> phi::dnnHandle_t {
+    InitDnnHandle();
+    return dnn_handle_;
+  };
+}
+
+blasHandle_t GPUContextResource::GetBlasHandle() const { return blas_handle_; }
+
+std::function<phi::blasHandle_t()> GPUContextResource::GetBlasHandleCreator() {
+  return [&]() -> phi::blasHandle_t {
+    phi::InitBlasHandle(&blas_handle_, stream_);
+    return blas_handle_;
+  };
+}
+
+blasHandle_t GPUContextResource::GetBlasTensorCoreHandle() const {
+  return blas_tensor_core_handle_;
+}
+
+std::function<phi::blasHandle_t()>
+GPUContextResource::GetBlasTensorCoreHandleCreator() {
+  return [&]() -> phi::blasHandle_t {
+#ifdef PADDLE_WITH_CUDA
+#if CUDA_VERSION >= 9000
+    phi::InitBlasHandle(&blas_tensor_core_handle_, stream_);
+    PADDLE_RETRY_CUDA_SUCCESS(phi::dynload::cublasSetMathMode(
+        blas_tensor_core_handle_, CUBLAS_TENSOR_OP_MATH));
+#endif
+#endif
+    return blas_tensor_core_handle_;
+  };
+}
+
+blasHandle_t GPUContextResource::GetBlasTF32Handle() const {
+  return blas_tf32_tensor_core_handle_;
+}
+
+std::function<phi::blasHandle_t()>
+GPUContextResource::GetBlasTF32TensorCoreHandleCreator() {
+  return [&]() -> phi::blasHandle_t {
+#ifdef PADDLE_WITH_CUDA
+#if CUDA_VERSION >= 11000
+    phi::InitBlasHandle(&blas_tf32_tensor_core_handle_, stream_);
+    PADDLE_RETRY_CUDA_SUCCESS(phi::dynload::cublasSetMathMode(
+        blas_tf32_tensor_core_handle_, CUBLAS_TF32_TENSOR_OP_MATH));
+#endif
+#endif
+    return blas_tf32_tensor_core_handle_;
+  };
+}
+
+blasLtHandle_t GPUContextResource::GetBlasLtHandle() const {
+  return blaslt_handle_;
+}
+
+std::function<phi::blasLtHandle_t()>
+GPUContextResource::GetBlasLtHandleCreator() {
+  return [&]() {
+    InitBlasLtHandle();
+    return blaslt_handle_;
+  };
+}
+
+phi::solverHandle_t GPUContextResource::GetSolverDnHandle() const {
+  return solver_handle_;
+}
+
+std::function<phi::solverHandle_t()>
+GPUContextResource::GetSolverDnHandleCreator() {
+  return [&]() {
+    InitSolverHandle();
+    return solver_handle_;
+  };
+}
+
+phi::sparseHandle_t GPUContextResource::GetSparseHandle() const {
+  return sparse_handle_;
+}
+
+std::function<phi::sparseHandle_t()>
+GPUContextResource::GetSparseHandleCreator() {
+  return [&]() {
+    InitSparseHandle();
+    return sparse_handle_;
+  };
+}
+
+Eigen::GpuDevice* GPUContextResource::GetGpuEigenDevice() const {
+  return gpu_eigen_device_.get();
+}
+
+std::function<Eigen::GpuDevice*()>
+GPUContextResource::GetGpuEigenDeviceCreator() {
+  return [&]() {
+    InitGpuEigenDevice();
+    return gpu_eigen_device_.get();
+  };
+}
+
+int GPUContextResource::GetGpuComputeCapability() const {
+  return compute_capability_;
+}
+
+int GPUContextResource::GetGpuRuntimeVersion() const {
+  return runtime_version_;
+}
+
+int GPUContextResource::GetGpuDriverVersion() const { return driver_version_; }
+
+int GPUContextResource::GetGPUMultiProcessors() const { return multi_process_; }
+
+int GPUContextResource::GetGpuMaxThreadsPerMp() const {
+  return max_threads_per_mp_;
+}
+
+int GPUContextResource::GetGpuMaxThreadsPerBlock() const {
+  return max_threads_per_block_;
+}
+
+std::array<unsigned int, 3> GPUContextResource::GetGpuMaxGridDimSize() const {
+  return max_grid_dim_size_;
+}
+
+#endif
+
+ResourceManager& ResourceManager::Instance() {
+  static ResourceManager* resource_manager = new ResourceManager;
+  return *resource_manager;
+}
+
+void ResourceManager::InitCPUResource() {
+  std::lock_guard<std::mutex> lock_guard(cpu_mutex_);
+  if (cpu_resource_ == nullptr) {
+    cpu_resource_ = std::make_unique<CPUContextResource>();
+  }
+}
+
+CPUContextResource* ResourceManager::GetCPUResource() const {
+  PADDLE_ENFORCE_NOT_NULL(
+      cpu_resource_.get(),
+      common::errors::PreconditionNotMet("cpu_resource should be not null!"));
+  return cpu_resource_.get();
+}
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+void* ResourceManager::InitGPUResource(const phi::Place& place, void* stream) {
+  std::lock_guard<std::mutex> lock_guard(gpu_mutex_);
+  if (gpu_resources_.count(stream)) {
+    Increase(stream);
+    return stream;
+  } else {
+    std::unique_ptr<GPUContextResource> resource{
+        new GPUContextResource(place, stream)};
+    gpuStream_t s = resource->GetStream();
+    ref_count_[s] = 1;
+    gpu_resources_.emplace(s, std::move(resource));
+    return s;
+  }
+}
+
+void ResourceManager::DestroyGPUResource(void* stream) {
+  PADDLE_ENFORCE_EQ(gpu_resources_.count(stream),
+                    true,
+                    common::errors::InvalidArgument(
+                        "The stream[%p] not found in gpu_resources.", stream));
+  Decrease(stream);
+}
+
+void ResourceManager::Decrease(void* stream) {
+  if (ref_count_.count(stream) == 0) return;
+  --ref_count_[stream];
+
+  if (ref_count_[stream] == 0) {
+    ref_count_.erase(stream);
+    if (gpu_resources_.count(stream) > 0) gpu_resources_.erase(stream);
+  }
+}
+
+void ResourceManager::Increase(void* stream) { ++ref_count_[stream]; }
+
+GPUContextResource* ResourceManager::GetGPUResource(void* stream) const {
+  PADDLE_ENFORCE_EQ(gpu_resources_.count(stream),
+                    true,
+                    common::errors::InvalidArgument(
+                        "The stream[%p] not found in gpu_resources.", stream));
+  return gpu_resources_.at(stream).get();
+}
+
+void ResourceManager::GpuResourceSwitchStream(void* old_stream,
+                                              void* new_stream) {
+  // NOTE: add lock to support stream rebind in multi-thread
+  std::lock_guard<std::mutex> lock_guard(gpu_mutex_);
+  if (old_stream == new_stream) return;
+  PADDLE_ENFORCE_EQ(
+      gpu_resources_.count(old_stream),
+      true,
+      common::errors::InvalidArgument(
+          "The stream[%p] not found in gpu_resources.", old_stream));
+
+  // NOTE: stream may be used by multiple predictor, skip resource
+  //       operation if resource of new_stream is already exists
+  bool new_stream_existed = gpu_resources_.count(new_stream) > 0;
+  if (!new_stream_existed) {
+    auto place = gpu_resources_.at(old_stream)->Place();
+    std::unique_ptr<GPUContextResource> resource{
+        new GPUContextResource(place, new_stream)};
+    gpu_resources_.emplace(new_stream, std::move(resource));
+  }
+
+  Decrease(old_stream);
+  Increase(new_stream);
+}
+
+int ResourceManager::RefCount(void* stream) const {
+  if (ref_count_.count(stream) == 0) return 0;
+  return ref_count_.at(stream);
+}
+#endif
+}  // namespace paddle

paddle/fluid/inference/api/resource_manager.h

@@ -0,0 +1,168 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+#include <atomic>
+#include <functional>
+#include <map>
+#include <memory>
+#include <mutex>
+
+#include "paddle/common/macros.h"
+#include "paddle/phi/api/include/tensor.h"
+#include "paddle/phi/backends/cpu/forwards.h"
+#include "paddle/phi/common/place.h"
+#include "paddle/phi/core/device_context.h"
+#include "paddle/utils/test_macros.h"
+#include "unsupported/Eigen/CXX11/Tensor"
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+#include "paddle/phi/backends/gpu/forwards.h"
+#include "paddle/phi/backends/gpu/gpu_decls.h"
+#include "paddle/phi/backends/gpu/gpu_resources.h"
+#include "paddle/phi/core/platform/device/gpu/gpu_types.h"
+#endif
+
+namespace paddle {
+namespace internal {
+class EigenGpuStreamDevice;
+}  // namespace internal
+
+class CPUContextResource {
+ public:
+  CPUContextResource();
+  Eigen::DefaultDevice* GetCPUEigenDevice() const;
+
+ private:
+  void InitCPUResource();
+
+ private:
+  std::unique_ptr<Eigen::DefaultDevice> cpu_eigen_device_;
+};
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+class GPUContextResource {
+ public:
+  explicit GPUContextResource(const phi::Place& place, void* stream);
+  TEST_API ~GPUContextResource();
+  phi::Place Place() const;
+
+  std::function<phi::dnnHandle_t()> GetDnnHandleCreator();
+  std::function<phi::blasHandle_t()> GetBlasHandleCreator();
+  std::function<phi::blasHandle_t()> GetBlasTensorCoreHandleCreator();
+  std::function<phi::blasHandle_t()> GetBlasTF32TensorCoreHandleCreator();
+  std::function<phi::blasLtHandle_t()> GetBlasLtHandleCreator();
+  std::function<phi::solverHandle_t()> GetSolverDnHandleCreator();
+  std::function<phi::sparseHandle_t()> GetSparseHandleCreator();
+  std::function<Eigen::GpuDevice*()> GetGpuEigenDeviceCreator();
+
+  gpuStream_t GetStream() const;
+  dnnHandle_t GetDnnHandle() const;
+  blasHandle_t GetBlasHandle() const;
+  blasHandle_t GetBlasTensorCoreHandle() const;
+  blasHandle_t GetBlasTF32Handle() const;
+  blasLtHandle_t GetBlasLtHandle() const;
+  phi::solverHandle_t GetSolverDnHandle() const;
+  phi::sparseHandle_t GetSparseHandle() const;
+  Eigen::GpuDevice* GetGpuEigenDevice() const;
+  int GetGpuComputeCapability() const;
+  int GetGpuRuntimeVersion() const;
+  int GetGpuDriverVersion() const;
+  int GetGPUMultiProcessors() const;
+  int GetGpuMaxThreadsPerMp() const;
+  int GetGpuMaxThreadsPerBlock() const;
+  std::array<unsigned int, 3> GetGpuMaxGridDimSize() const;
+
+ private:
+  void InitGPUResource(void* stream);
+  void DestroyGPUResource();
+  void InitGpuProperties();
+  void InitGpuEigenDevice();
+  void InitDnnHandle();
+  void DestroyDnnHandle();
+  void DestroyBlasHandle();
+  void InitBlasLtHandle();
+  void DestroyBlasLtHandle();
+  void InitSolverHandle();
+  void DestroySolverHandle();
+  void InitSparseHandle();
+  void DestroySparseHandle();
+
+ private:
+  phi::Place place_;
+
+  int compute_capability_;
+  int runtime_version_;
+  int driver_version_;
+  int multi_process_;
+  int max_threads_per_mp_;
+  int max_threads_per_block_;
+  std::array<unsigned int, 3> max_grid_dim_size_;
+
+  bool owned_stream_{true};
+  gpuStream_t stream_;
+  std::unique_ptr<Eigen::GpuDevice> gpu_eigen_device_;
+  std::unique_ptr<internal::EigenGpuStreamDevice> eigen_stream_;
+
+  blasHandle_t blas_handle_{nullptr};
+  blasHandle_t blas_tensor_core_handle_{nullptr};
+  blasHandle_t blas_tf32_tensor_core_handle_{nullptr};
+  blasLtHandle_t blaslt_handle_{nullptr};
+  dnnHandle_t dnn_handle_{nullptr};
+  phi::solverHandle_t solver_handle_{nullptr};
+  phi::sparseHandle_t sparse_handle_{nullptr};
+  // DnnWorkspaceHandle
+};
+#endif
+
+class ResourceManager {
+ public:
+  ResourceManager() = default;
+  TEST_API static ResourceManager& Instance();
+
+  // CPU Resource
+ public:
+  void InitCPUResource();
+  CPUContextResource* GetCPUResource() const;
+
+ private:
+  std::mutex cpu_mutex_;
+  std::unique_ptr<CPUContextResource> cpu_resource_{nullptr};
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+  // GPU Resource
+ public:
+  void* InitGPUResource(const phi::Place& place, void* stream);
+  void DestroyGPUResource(void* stream);
+  TEST_API GPUContextResource* GetGPUResource(void* stream) const;
+  TEST_API int RefCount(void* stream) const;
+  void GpuResourceSwitchStream(void* old_stream, void* new_stream);
+
+ private:
+  void Decrease(void* stream);
+  void Increase(void* stream);
+
+ private:
+  std::mutex gpu_mutex_;
+  // a stream corresponding to a series of resource.
+  std::map<void* /*stream*/, std::atomic<int>> ref_count_;
+  std::map<void* /*stream*/, std::unique_ptr<GPUContextResource>>
+      gpu_resources_;
+#endif
+
+ private:
+  DISABLE_COPY_AND_ASSIGN(ResourceManager);
+};
+
+}  // namespace paddle