chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,353 @@
|
||||
cmake_minimum_required(VERSION 3.5)
|
||||
project(cpp_inference_demo CXX C)
|
||||
option(WITH_MKL "Compile demo with MKL/OpenBlas support, default use MKL." ON)
|
||||
option(WITH_GPU "Compile demo with GPU/CPU, default use CPU." OFF)
|
||||
option(WITH_STATIC_LIB
|
||||
"Compile demo with static/shared library, default use static." OFF)
|
||||
option(USE_TENSORRT "Compile demo with TensorRT." OFF)
|
||||
option(WITH_GTEST "Compile demo with GTEST" OFF)
|
||||
option(WITH_ONNXRUNTIME "Compile demo with ONNXRuntime" OFF)
|
||||
|
||||
if(NOT WITH_STATIC_LIB)
|
||||
add_definitions("-DPADDLE_WITH_SHARED_LIB")
|
||||
else()
|
||||
# PD_INFER_DECL is mainly used to set the dllimport/dllexport attribute in dynamic library mode.
|
||||
# Set it to empty in static library mode to avoid compilation issues.
|
||||
add_definitions("/DPD_INFER_DECL=")
|
||||
endif()
|
||||
|
||||
macro(safe_set_static_flag)
|
||||
foreach(flag_var
|
||||
CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
|
||||
CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
|
||||
if(${flag_var} MATCHES "/MD")
|
||||
string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
|
||||
endif()
|
||||
endforeach()
|
||||
endmacro()
|
||||
|
||||
if(NOT DEFINED PADDLE_LIB)
|
||||
message(
|
||||
FATAL_ERROR "please set PADDLE_LIB with -DPADDLE_LIB=/path/paddle/lib")
|
||||
endif()
|
||||
if(NOT DEFINED DEMO_NAME)
|
||||
message(FATAL_ERROR "please set DEMO_NAME with -DDEMO_NAME=demo_name")
|
||||
endif()
|
||||
|
||||
include_directories("${PADDLE_LIB}/")
|
||||
set(PADDLE_LIB_THIRD_PARTY_PATH "${PADDLE_LIB}/third_party/install/")
|
||||
include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}protobuf/include")
|
||||
include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}glog/include")
|
||||
include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}gflags/include")
|
||||
include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}xxhash/include")
|
||||
include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}cryptopp/include")
|
||||
|
||||
link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}protobuf/lib")
|
||||
link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}glog/lib")
|
||||
link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}gflags/lib")
|
||||
link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}xxhash/lib")
|
||||
link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}cryptopp/lib")
|
||||
link_directories("${PADDLE_LIB}/paddle/lib")
|
||||
if(WITH_ONNXRUNTIME)
|
||||
include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}onnxruntime/include")
|
||||
include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}paddle2onnx/include")
|
||||
|
||||
link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}onnxruntime/lib")
|
||||
link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}paddle2onnx/lib")
|
||||
endif()
|
||||
|
||||
if(WIN32)
|
||||
add_definitions("/DGOOGLE_GLOG_DLL_DECL=")
|
||||
option(MSVC_STATIC_CRT "use static C Runtime library by default" ON)
|
||||
if(MSVC_STATIC_CRT)
|
||||
if(WITH_MKL)
|
||||
set(FLAG_OPENMP "/openmp")
|
||||
endif()
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4244 /wd4530")
|
||||
set(CMAKE_C_FLAGS_DEBUG
|
||||
"${CMAKE_C_FLAGS_DEBUG} /bigobj /MTd ${FLAG_OPENMP}")
|
||||
set(CMAKE_C_FLAGS_RELEASE
|
||||
"${CMAKE_C_FLAGS_RELEASE} /bigobj /MT ${FLAG_OPENMP}")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4530")
|
||||
set(CMAKE_CXX_FLAGS_DEBUG
|
||||
"${CMAKE_CXX_FLAGS_DEBUG} /bigobj /MTd ${FLAG_OPENMP}")
|
||||
set(CMAKE_CXX_FLAGS_RELEASE
|
||||
"${CMAKE_CXX_FLAGS_RELEASE} /bigobj /MT ${FLAG_OPENMP}")
|
||||
safe_set_static_flag()
|
||||
if(WITH_STATIC_LIB)
|
||||
add_definitions(-DSTATIC_LIB)
|
||||
endif()
|
||||
endif()
|
||||
else()
|
||||
if(WITH_MKL)
|
||||
set(FLAG_OPENMP "-fopenmp")
|
||||
endif()
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 ${FLAG_OPENMP}")
|
||||
endif()
|
||||
|
||||
if(WITH_GPU)
|
||||
if(NOT WIN32)
|
||||
add_definitions("-DPADDLE_WITH_GPU")
|
||||
set(CUDA_LIB
|
||||
"/usr/local/cuda/lib64/"
|
||||
CACHE STRING "CUDA Library")
|
||||
include_directories("${CUDA_LIB}/../include")
|
||||
else()
|
||||
set(CUDA_LIB
|
||||
""
|
||||
CACHE STRING "CUDA_LIB")
|
||||
if("${CUDA_LIB}" STREQUAL "")
|
||||
set(CUDA_LIB "$ENV{CUDA_PATH}\\lib\\x64")
|
||||
endif()
|
||||
message(STATUS "Current CUDA lib path: ${CUDA_LIB}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(USE_TENSORRT AND WITH_GPU)
|
||||
set(TENSORRT_ROOT
|
||||
""
|
||||
CACHE STRING "The root directory of TensorRT library")
|
||||
if("${TENSORRT_ROOT}" STREQUAL "")
|
||||
message(
|
||||
FATAL_ERROR
|
||||
"The TENSORRT_ROOT is empty, you must assign it a value with CMake command. Such as: -DTENSORRT_ROOT=TENSORRT_ROOT_PATH "
|
||||
)
|
||||
endif()
|
||||
set(TENSORRT_INCLUDE_DIR ${TENSORRT_ROOT}/include)
|
||||
set(TENSORRT_LIB_DIR ${TENSORRT_ROOT}/lib)
|
||||
file(READ ${TENSORRT_INCLUDE_DIR}/NvInfer.h TENSORRT_VERSION_FILE_CONTENTS)
|
||||
string(REGEX MATCH "define NV_TENSORRT_MAJOR +([0-9]+)"
|
||||
TENSORRT_MAJOR_VERSION "${TENSORRT_VERSION_FILE_CONTENTS}")
|
||||
string(REGEX MATCH "define NV_TENSORRT_MINOR +([0-9]+)"
|
||||
TENSORRT_MINOR_VERSION "${TENSORRT_VERSION_FILE_CONTENTS}")
|
||||
string(REGEX MATCH "define NV_TENSORRT_PATCH +([0-9]+)"
|
||||
TENSORRT_PATCH_VERSION "${TENSORRT_VERSION_FILE_CONTENTS}")
|
||||
string(REGEX MATCH "define NV_TENSORRT_BUILD +([0-9]+)"
|
||||
TENSORRT_BUILD_VERSION "${TENSORRT_VERSION_FILE_CONTENTS}")
|
||||
if("${TENSORRT_MAJOR_VERSION}" STREQUAL "")
|
||||
file(READ ${TENSORRT_INCLUDE_DIR}/NvInferVersion.h
|
||||
TENSORRT_VERSION_FILE_CONTENTS)
|
||||
string(REGEX MATCH "define NV_TENSORRT_MAJOR +([0-9]+)"
|
||||
TENSORRT_MAJOR_VERSION "${TENSORRT_VERSION_FILE_CONTENTS}")
|
||||
string(REGEX MATCH "define NV_TENSORRT_MINOR +([0-9]+)"
|
||||
TENSORRT_MINOR_VERSION "${TENSORRT_VERSION_FILE_CONTENTS}")
|
||||
string(REGEX MATCH "define NV_TENSORRT_PATCH +([0-9]+)"
|
||||
TENSORRT_PATCH_VERSION "${TENSORRT_VERSION_FILE_CONTENTS}")
|
||||
string(REGEX MATCH "define NV_TENSORRT_BUILD +([0-9]+)"
|
||||
TENSORRT_BUILD_VERSION "${TENSORRT_VERSION_FILE_CONTENTS}")
|
||||
endif()
|
||||
if("${TENSORRT_MAJOR_VERSION}" STREQUAL "")
|
||||
message(SEND_ERROR "Failed to detect TensorRT version.")
|
||||
endif()
|
||||
string(REGEX REPLACE "define NV_TENSORRT_MAJOR +([0-9]+)" "\\1"
|
||||
TENSORRT_MAJOR_VERSION "${TENSORRT_MAJOR_VERSION}")
|
||||
string(REGEX REPLACE "define NV_TENSORRT_MINOR +([0-9]+)" "\\1"
|
||||
TENSORRT_MINOR_VERSION "${TENSORRT_MINOR_VERSION}")
|
||||
string(REGEX REPLACE "define NV_TENSORRT_PATCH +([0-9]+)" "\\1"
|
||||
TENSORRT_PATCH_VERSION "${TENSORRT_PATCH_VERSION}")
|
||||
string(REGEX REPLACE "define NV_TENSORRT_BUILD +([0-9]+)" "\\1"
|
||||
TENSORRT_BUILD_VERSION "${TENSORRT_BUILD_VERSION}")
|
||||
message(
|
||||
STATUS
|
||||
"Current TensorRT header is ${TENSORRT_INCLUDE_DIR}/NvInfer.h. "
|
||||
"Current TensorRT version is v${TENSORRT_MAJOR_VERSION}.${TENSORRT_MINOR_VERSION}.${TENSORRT_PATCH_VERSION}.${TENSORRT_BUILD_VERSION} "
|
||||
)
|
||||
include_directories("${TENSORRT_INCLUDE_DIR}")
|
||||
link_directories("${TENSORRT_LIB_DIR}")
|
||||
add_compile_definitions(NV_TENSORRT_MAJOR=${TENSORRT_MAJOR_VERSION})
|
||||
add_compile_definitions(NV_TENSORRT_MINOR=${TENSORRT_MINOR_VERSION})
|
||||
add_compile_definitions(NV_TENSORRT_PATCH=${TENSORRT_PATCH_VERSION})
|
||||
add_compile_definitions(NV_TENSORRT_BUILD=${TENSORRT_BUILD_VERSION})
|
||||
endif()
|
||||
|
||||
if(WITH_MKL)
|
||||
set(MATH_LIB_PATH "${PADDLE_LIB_THIRD_PARTY_PATH}mklml")
|
||||
include_directories("${MATH_LIB_PATH}/include")
|
||||
if(WIN32)
|
||||
set(MATH_LIB ${MATH_LIB_PATH}/lib/mklml${CMAKE_STATIC_LIBRARY_SUFFIX}
|
||||
${MATH_LIB_PATH}/lib/libiomp5md${CMAKE_STATIC_LIBRARY_SUFFIX})
|
||||
else()
|
||||
set(MATH_LIB
|
||||
${MATH_LIB_PATH}/lib/libmklml_intel${CMAKE_SHARED_LIBRARY_SUFFIX}
|
||||
${MATH_LIB_PATH}/lib/libiomp5${CMAKE_SHARED_LIBRARY_SUFFIX})
|
||||
endif()
|
||||
set(ONEDNN_PATH "${PADDLE_LIB_THIRD_PARTY_PATH}onednn")
|
||||
if(EXISTS ${ONEDNN_PATH})
|
||||
include_directories("${ONEDNN_PATH}/include")
|
||||
if(WIN32)
|
||||
set(ONEDNN_LIB ${ONEDNN_PATH}/lib/mkldnn.lib)
|
||||
else()
|
||||
set(ONEDNN_LIB ${ONEDNN_PATH}/lib/libdnnl.so.3)
|
||||
endif()
|
||||
endif()
|
||||
else()
|
||||
set(OPENBLAS_LIB_PATH "${PADDLE_LIB_THIRD_PARTY_PATH}openblas")
|
||||
include_directories("${OPENBLAS_LIB_PATH}/include/openblas")
|
||||
if(WIN32)
|
||||
set(MATH_LIB
|
||||
${OPENBLAS_LIB_PATH}/lib/openblas${CMAKE_STATIC_LIBRARY_SUFFIX})
|
||||
else()
|
||||
set(MATH_LIB
|
||||
${OPENBLAS_LIB_PATH}/lib/libopenblas${CMAKE_STATIC_LIBRARY_SUFFIX})
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(WITH_STATIC_LIB)
|
||||
set(DEPS
|
||||
${PADDLE_LIB}/paddle/lib/libpaddle_inference${CMAKE_STATIC_LIBRARY_SUFFIX}
|
||||
)
|
||||
else()
|
||||
if(WIN32)
|
||||
set(DEPS
|
||||
${PADDLE_LIB}/paddle/lib/paddle_inference${CMAKE_STATIC_LIBRARY_SUFFIX})
|
||||
else()
|
||||
set(DEPS
|
||||
${PADDLE_LIB}/paddle/lib/libpaddle_inference${CMAKE_SHARED_LIBRARY_SUFFIX}
|
||||
)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(WITH_ONNXRUNTIME)
|
||||
set(DEPS ${DEPS} onnxruntime paddle2onnx)
|
||||
endif()
|
||||
|
||||
if(NOT WIN32)
|
||||
set(EXTERNAL_LIB "-lrt -ldl -lpthread")
|
||||
set(DEPS
|
||||
${DEPS}
|
||||
${MATH_LIB}
|
||||
${ONEDNN_LIB}
|
||||
glog
|
||||
gflags
|
||||
protobuf
|
||||
xxhash
|
||||
cryptopp
|
||||
${EXTERNAL_LIB})
|
||||
else()
|
||||
set(DEPS
|
||||
${DEPS}
|
||||
${MATH_LIB}
|
||||
${ONEDNN_LIB}
|
||||
glog
|
||||
gflags_static
|
||||
libprotobuf
|
||||
xxhash
|
||||
cryptopp-static
|
||||
${EXTERNAL_LIB})
|
||||
set(DEPS ${DEPS} shlwapi.lib)
|
||||
endif()
|
||||
|
||||
if(WITH_GPU)
|
||||
if(NOT WIN32)
|
||||
if(USE_TENSORRT)
|
||||
set(DEPS ${DEPS}
|
||||
${TENSORRT_LIB_DIR}/libnvinfer${CMAKE_SHARED_LIBRARY_SUFFIX})
|
||||
set(DEPS
|
||||
${DEPS}
|
||||
${TENSORRT_LIB_DIR}/libnvinfer_plugin${CMAKE_SHARED_LIBRARY_SUFFIX})
|
||||
endif()
|
||||
set(DEPS ${DEPS} ${CUDA_LIB}/libcudart${CMAKE_SHARED_LIBRARY_SUFFIX})
|
||||
else()
|
||||
if(USE_TENSORRT)
|
||||
set(DEPS ${DEPS}
|
||||
${TENSORRT_LIB_DIR}/nvinfer${CMAKE_STATIC_LIBRARY_SUFFIX})
|
||||
set(DEPS ${DEPS}
|
||||
${TENSORRT_LIB_DIR}/nvinfer_plugin${CMAKE_STATIC_LIBRARY_SUFFIX})
|
||||
if(${TENSORRT_MAJOR_VERSION} EQUAL 7)
|
||||
set(DEPS ${DEPS}
|
||||
${TENSORRT_LIB_DIR}/myelin64_1${CMAKE_STATIC_LIBRARY_SUFFIX})
|
||||
endif()
|
||||
endif()
|
||||
set(DEPS ${DEPS} ${CUDA_LIB}/cudart${CMAKE_STATIC_LIBRARY_SUFFIX})
|
||||
set(DEPS ${DEPS} ${CUDA_LIB}/cublas${CMAKE_STATIC_LIBRARY_SUFFIX})
|
||||
set(DEPS ${DEPS} ${CUDA_LIB}/cudnn${CMAKE_STATIC_LIBRARY_SUFFIX})
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(WITH_GTEST)
|
||||
include(ExternalProject)
|
||||
include(external-cmake/gtest-cpp.cmake)
|
||||
endif()
|
||||
|
||||
add_executable(${DEMO_NAME} ${DEMO_NAME}.cc)
|
||||
target_link_libraries(${DEMO_NAME} ${DEPS})
|
||||
if(WITH_GTEST)
|
||||
include(GNUInstallDirs)
|
||||
include_directories(${GTEST_INSTALL_DIR}/include)
|
||||
add_dependencies(${DEMO_NAME} thirdparty_gtest)
|
||||
if(WIN32)
|
||||
target_link_libraries(${DEMO_NAME} ${GTEST_LIBRARIES})
|
||||
else()
|
||||
target_link_libraries(
|
||||
${DEMO_NAME}
|
||||
${GTEST_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libgtest${CMAKE_STATIC_LIBRARY_SUFFIX}
|
||||
)
|
||||
endif()
|
||||
endif()
|
||||
if(WIN32)
|
||||
if("${CMAKE_GENERATOR}" MATCHES "Ninja")
|
||||
set(LIB_PATH ${CMAKE_BINARY_DIR})
|
||||
else()
|
||||
set(LIB_PATH ${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE})
|
||||
endif()
|
||||
|
||||
if(USE_TENSORRT)
|
||||
add_custom_command(
|
||||
TARGET ${DEMO_NAME}
|
||||
POST_BUILD
|
||||
COMMAND
|
||||
${CMAKE_COMMAND} -E copy
|
||||
${TENSORRT_LIB_DIR}/nvinfer${CMAKE_SHARED_LIBRARY_SUFFIX} ${LIB_PATH}
|
||||
COMMAND
|
||||
${CMAKE_COMMAND} -E copy
|
||||
${TENSORRT_LIB_DIR}/nvinfer_plugin${CMAKE_SHARED_LIBRARY_SUFFIX}
|
||||
${LIB_PATH})
|
||||
if(${TENSORRT_MAJOR_VERSION} EQUAL 7)
|
||||
add_custom_command(
|
||||
TARGET ${DEMO_NAME}
|
||||
POST_BUILD
|
||||
COMMAND
|
||||
${CMAKE_COMMAND} -E copy
|
||||
${TENSORRT_LIB_DIR}/myelin64_1${CMAKE_SHARED_LIBRARY_SUFFIX}
|
||||
${LIB_PATH})
|
||||
endif()
|
||||
endif()
|
||||
if(WITH_MKL)
|
||||
message("LIB_PATH IS ${LIB_PATH}")
|
||||
add_custom_command(
|
||||
TARGET ${DEMO_NAME}
|
||||
POST_BUILD
|
||||
COMMAND ${CMAKE_COMMAND} -E copy ${MATH_LIB_PATH}/lib/mklml.dll
|
||||
${LIB_PATH}
|
||||
COMMAND ${CMAKE_COMMAND} -E copy ${MATH_LIB_PATH}/lib/libiomp5md.dll
|
||||
${LIB_PATH}
|
||||
COMMAND ${CMAKE_COMMAND} -E copy ${ONEDNN_PATH}/lib/mkldnn.dll
|
||||
${LIB_PATH})
|
||||
else()
|
||||
add_custom_command(
|
||||
TARGET ${DEMO_NAME}
|
||||
POST_BUILD
|
||||
COMMAND ${CMAKE_COMMAND} -E copy ${OPENBLAS_LIB_PATH}/lib/openblas.dll
|
||||
${LIB_PATH})
|
||||
endif()
|
||||
if(WITH_ONNXRUNTIME)
|
||||
add_custom_command(
|
||||
TARGET ${DEMO_NAME}
|
||||
POST_BUILD
|
||||
COMMAND
|
||||
${CMAKE_COMMAND} -E copy
|
||||
${PADDLE_LIB_THIRD_PARTY_PATH}onnxruntime/lib/onnxruntime.dll
|
||||
${LIB_PATH}
|
||||
COMMAND
|
||||
${CMAKE_COMMAND} -E copy
|
||||
${PADDLE_LIB_THIRD_PARTY_PATH}paddle2onnx/lib/paddle2onnx.dll
|
||||
${LIB_PATH})
|
||||
endif()
|
||||
if(NOT WITH_STATIC_LIB)
|
||||
add_custom_command(
|
||||
TARGET ${DEMO_NAME}
|
||||
POST_BUILD
|
||||
COMMAND ${CMAKE_COMMAND} -E copy
|
||||
"${PADDLE_LIB}/paddle/lib/paddle_inference.dll" ${LIB_PATH})
|
||||
endif()
|
||||
endif()
|
||||
@@ -0,0 +1,37 @@
|
||||
# Inference Model UT
|
||||
|
||||
There are several model tests currently:
|
||||
- test_ernie_text_cls.cc
|
||||
- test_LeViT.cc
|
||||
- test_ppyolo_mbv3.cc
|
||||
- test_ppyolov2_r50vd.cc
|
||||
- test_resnet50.cc
|
||||
- test_resnet50_quant.cc
|
||||
- test_yolov3.cc
|
||||
|
||||
To build and execute tests on Linux, simply run
|
||||
```
|
||||
./run.sh $PADDLE_ROOT $TURN_ON_MKL $TEST_GPU_CPU $DATA_DIR
|
||||
```
|
||||
To build on windows, run command with busybox
|
||||
```
|
||||
busybox bash ./run.sh $PADDLE_ROOT $TURN_ON_MKL $TEST_GPU_CPU $DATA_DIR
|
||||
```
|
||||
|
||||
- After run command, it will build and execute tests and download to ${DATA_DIR} automatically.
|
||||
- `$PADDLE_ROOT`: paddle library path
|
||||
- `$TURN_ON_MKL`: use MKL or Openblas
|
||||
- `$TEST_GPU_CPU`: test both GPU/CPU mode or only CPU mode
|
||||
- `$DATA_DIR`: download data path
|
||||
|
||||
now only support 4 kinds of tests which controlled by `--gtest_filter` argument, test suite name should be same as following.
|
||||
- `TEST(gpu_tester_*, test_name)`
|
||||
- `TEST(cpu_tester_*, test_name)`
|
||||
- `TEST(onednn_tester_*, test_name)`
|
||||
- `TEST(tensorrt_tester_*, test_name)`
|
||||
|
||||
skpied test suite name.
|
||||
- `TEST(DISABLED_gpu_tester_*, test_name)`
|
||||
- `TEST(DISABLED_cpu_tester_*, test_name)`
|
||||
- `TEST(DISABLED_onednn_tester_*, test_name)`
|
||||
- `TEST(DISABLED_tensorrt_tester_*, test_name)`
|
||||
@@ -0,0 +1,59 @@
|
||||
find_package(Git REQUIRED)
|
||||
message("${CMAKE_BUILD_TYPE}")
|
||||
set(GTEST_PREFIX_DIR ${CMAKE_CURRENT_BINARY_DIR}/gtest)
|
||||
set(PADDLE_SOURCE_DIR $ENV{PADDLE_SOURCE_DIR})
|
||||
set(GTEST_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/gtest)
|
||||
set(GTEST_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/install/gtest)
|
||||
set(GTEST_INCLUDE_DIR
|
||||
"${GTEST_INSTALL_DIR}/include"
|
||||
CACHE PATH "gtest include directory." FORCE)
|
||||
set(GTEST_TAG release-1.8.1)
|
||||
|
||||
if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.0.0")
|
||||
message(
|
||||
WARNING
|
||||
"gtest-cpp: forcing CMake policy compatibility for CMake >= 4.0 (CMAKE_POLICY_VERSION_MINIMUM=3.5)"
|
||||
)
|
||||
set(GTEST_POLICY_ARGS -DCMAKE_POLICY_VERSION_MINIMUM=3.5)
|
||||
endif()
|
||||
|
||||
include_directories(${GTEST_INCLUDE_DIR})
|
||||
if(WIN32)
|
||||
# if use CMAKE_INSTALL_LIBDIR, the path of lib actually is \
|
||||
# install/gtest/lib/gtest.lib but GTEST_LIBRARIES
|
||||
# is install/gtest/gtest.lib
|
||||
set(GTEST_LIBRARIES
|
||||
"${GTEST_INSTALL_DIR}/lib/gtest.lib"
|
||||
CACHE FILEPATH "gtest libraries." FORCE)
|
||||
set(GTEST_MAIN_LIBRARIES
|
||||
"${GTEST_INSTALL_DIR}/lib/gtest_main.lib"
|
||||
CACHE FILEPATH "gtest main libraries." FORCE)
|
||||
else()
|
||||
set(GTEST_LIBRARIES
|
||||
"${GTEST_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libgtest.a"
|
||||
CACHE FILEPATH "gtest libraries." FORCE)
|
||||
set(GTEST_MAIN_LIBRARIES
|
||||
"${GTEST_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libgtest_main.a"
|
||||
CACHE FILEPATH "gtest main libraries." FORCE)
|
||||
endif()
|
||||
ExternalProject_Add(
|
||||
extern_gtest
|
||||
PREFIX gtest
|
||||
SOURCE_DIR ${GTEST_SOURCE_DIR}
|
||||
DOWNLOAD_DIR "${DOWNLOAD_LOCATION}"
|
||||
UPDATE_COMMAND ""
|
||||
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GTEST_INSTALL_DIR}
|
||||
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
|
||||
-DCMAKE_BUILD_TYPE:STRING=Release ${GTEST_POLICY_ARGS}
|
||||
BUILD_BYPRODUCTS ${GTEST_LIBRARIES}
|
||||
BUILD_BYPRODUCTS ${GTEST_MAIN_LIBRARIES})
|
||||
|
||||
add_library(thirdparty_gtest STATIC IMPORTED GLOBAL)
|
||||
set_property(TARGET thirdparty_gtest PROPERTY IMPORTED_LOCATION
|
||||
${GTEST_LIBRARIES})
|
||||
add_dependencies(thirdparty_gtest extern_gtest)
|
||||
|
||||
add_library(thirdparty_gtest_main STATIC IMPORTED GLOBAL)
|
||||
set_property(TARGET thirdparty_gtest_main PROPERTY IMPORTED_LOCATION
|
||||
${GTEST_MAIN_LIBRARIES})
|
||||
add_dependencies(thirdparty_gtest_main extern_gtest)
|
||||
Executable
+326
@@ -0,0 +1,326 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -x
|
||||
PADDLE_ROOT=$1
|
||||
export PADDLE_SOURCE_DIR=$1
|
||||
TURN_ON_MKL=$2 # use MKL or Openblas
|
||||
TEST_GPU_CPU=$3 # test both GPU/CPU mode or only CPU mode
|
||||
DATA_DIR=$4 # dataset
|
||||
TENSORRT_ROOT_DIR=$5 # TensorRT ROOT dir, default to /usr/local/TensorRT
|
||||
WITH_ONNXRUNTIME=$6
|
||||
MSVC_STATIC_CRT=$7
|
||||
CUDA_LIB=$8/lib/x64
|
||||
inference_install_dir=${PADDLE_ROOT}/build/paddle_inference_install_dir
|
||||
EXIT_CODE=0 # init default exit code
|
||||
WIN_DETECT=$(echo `uname` | grep "Win") # detect current platform
|
||||
test_suite_list="cpu_tester*" # init test suite list, pass to --gtest_filter
|
||||
|
||||
export RED='\033[0;31m' # red color
|
||||
export NC='\033[0m' # no color
|
||||
export YELLOW='\033[33m' # yellow color
|
||||
|
||||
cd `dirname $0`
|
||||
current_dir=`pwd`
|
||||
build_dir=${current_dir}/build
|
||||
log_dir=${current_dir}/log
|
||||
|
||||
# check onednn installation
|
||||
if [ $2 == ON ]; then
|
||||
# You can export yourself if move the install path
|
||||
MKL_LIB=${inference_install_dir}/third_party/install/mklml/lib
|
||||
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${MKL_LIB}
|
||||
test_suite_list="${test_suite_list}:onednn_tester*"
|
||||
fi
|
||||
|
||||
if [ $3 == ON ]; then
|
||||
use_gpu_list='true false'
|
||||
test_suite_list="${test_suite_list}:gpu_tester*"
|
||||
else
|
||||
use_gpu_list='false'
|
||||
fi
|
||||
|
||||
# check tensorrt installation
|
||||
TENSORRT_COMPILED=$(cat "${inference_install_dir}/version.txt" | grep "WITH_TENSORRT")
|
||||
USE_TENSORRT=OFF
|
||||
if [ -d "$TENSORRT_ROOT_DIR" ] && [ ! -z "$TENSORRT_COMPILED" ] ; then
|
||||
USE_TENSORRT=ON
|
||||
test_suite_list="${test_suite_list}:tensorrt_tester*"
|
||||
fi
|
||||
|
||||
function download() {
|
||||
url_prefix=$1
|
||||
model_name=$2
|
||||
mkdir -p $model_name
|
||||
cd $model_name
|
||||
if [[ -e "${model_name}.tgz" ]]; then
|
||||
echo "${model_name}.tgz has been downloaded."
|
||||
else
|
||||
if [ "$WIN_DETECT" != "" ]; then
|
||||
wget -q -Y off ${url_prefix}/${model_name}.tgz
|
||||
tar xzf *.tgz
|
||||
else
|
||||
wget -q --no-proxy ${url_prefix}/${model_name}.tgz
|
||||
tar xzf *.tgz
|
||||
fi
|
||||
fi
|
||||
cd ..
|
||||
}
|
||||
|
||||
mkdir -p $DATA_DIR
|
||||
cd $DATA_DIR
|
||||
download_list='resnet50'
|
||||
for model_name in $download_list; do
|
||||
url_prefix="https://paddle-inference-dist.bj.bcebos.com/Paddle-Inference-Demo"
|
||||
download $url_prefix $model_name
|
||||
done
|
||||
|
||||
ocr_download_list='ocr_det_mv3_db'
|
||||
for model_name in $ocr_download_list; do
|
||||
url_prefix="https://paddle-qa.bj.bcebos.com/inference_model/2.1.1/ocr"
|
||||
download $url_prefix $model_name
|
||||
done
|
||||
|
||||
clas_download_list='LeViT'
|
||||
for model_name in $clas_download_list; do
|
||||
url_prefix="https://paddle-qa.bj.bcebos.com/inference_model/2.1.1/class"
|
||||
download $url_prefix $model_name
|
||||
done
|
||||
|
||||
nlp_download_list='ernie_text_cls'
|
||||
for model_name in $nlp_download_list; do
|
||||
url_prefix="https://paddle-qa.bj.bcebos.com/inference_model/2.1.1/nlp"
|
||||
download $url_prefix $model_name
|
||||
done
|
||||
|
||||
det_download_list='yolov3 ppyolo_mbv3 ppyolov2_r50vd'
|
||||
for model_name in $det_download_list; do
|
||||
url_prefix="https://paddle-qa.bj.bcebos.com/inference_model/2.1.1/detection"
|
||||
download $url_prefix $model_name
|
||||
done
|
||||
|
||||
unknown_download_list='resnet50_quant'
|
||||
for model_name in $unknown_download_list; do
|
||||
url_prefix="https://paddle-qa.bj.bcebos.com/inference_model/unknown"
|
||||
download $url_prefix $model_name
|
||||
done
|
||||
|
||||
# ernie int8 quant with matmul
|
||||
unknown_nlp_download_list='quant_post_model_xnli_predict_matmul'
|
||||
for model_name in $unknown_nlp_download_list; do
|
||||
url_prefix="https://paddle-qa.bj.bcebos.com/inference_model/unknown/nlp"
|
||||
download $url_prefix $model_name
|
||||
done
|
||||
|
||||
# mobilnetv1 with prune op attribute
|
||||
dev_class_download_list='MobileNetV1'
|
||||
for model_name in $dev_class_download_list; do
|
||||
url_prefix="https://paddle-qa.bj.bcebos.com/inference_model/2021-09-16/class"
|
||||
download $url_prefix $model_name
|
||||
done
|
||||
|
||||
function compile_test() {
|
||||
mkdir -p ${build_dir}
|
||||
cd ${build_dir}
|
||||
TEST_NAME=$1
|
||||
if [ "$WIN_DETECT" != "" ]; then
|
||||
cmake .. -GNinja -DPADDLE_LIB=${inference_install_dir} \
|
||||
-DWITH_MKL=$TURN_ON_MKL \
|
||||
-DDEMO_NAME=${TEST_NAME} \
|
||||
-DWITH_GPU=$TEST_GPU_CPU \
|
||||
-DWITH_STATIC_LIB=OFF \
|
||||
-DUSE_TENSORRT=$USE_TENSORRT \
|
||||
-DTENSORRT_ROOT=$TENSORRT_ROOT_DIR \
|
||||
-DMSVC_STATIC_CRT=$MSVC_STATIC_CRT \
|
||||
-DWITH_GTEST=ON \
|
||||
-DCMAKE_CXX_FLAGS='/std:c++17' \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DWITH_ONNXRUNTIME=$WITH_ONNXRUNTIME \
|
||||
-DCUDA_LIB="$CUDA_LIB"
|
||||
ninja
|
||||
else
|
||||
cmake .. -DPADDLE_LIB=${inference_install_dir} \
|
||||
-DWITH_MKL=$TURN_ON_MKL \
|
||||
-DDEMO_NAME=${TEST_NAME} \
|
||||
-DWITH_GPU=$TEST_GPU_CPU \
|
||||
-DWITH_STATIC_LIB=OFF \
|
||||
-DUSE_TENSORRT=$USE_TENSORRT \
|
||||
-DTENSORRT_ROOT=$TENSORRT_ROOT_DIR \
|
||||
-DWITH_GTEST=ON \
|
||||
-DWITH_ONNXRUNTIME=$WITH_ONNXRUNTIME
|
||||
make -j$(nproc)
|
||||
fi
|
||||
cd -
|
||||
}
|
||||
|
||||
|
||||
# compile and run test
|
||||
cd $current_dir
|
||||
mkdir -p ${build_dir}
|
||||
mkdir -p ${log_dir}
|
||||
cd ${build_dir}
|
||||
rm -rf *
|
||||
|
||||
exe_dir=${build_dir}
|
||||
|
||||
# printf "${YELLOW} start test_resnet50 ${NC} \n";
|
||||
# compile_test "test_resnet50"
|
||||
# ${exe_dir}/test_resnet50 \
|
||||
# --modeldir=$DATA_DIR/resnet50/resnet50 \
|
||||
# --gtest_filter=${test_suite_list} \
|
||||
# --gtest_output=xml:${log_dir}/test_resnet50.xml
|
||||
# if [ $? -ne 0 ]; then
|
||||
# echo "${RED} test_resnet50 runs failed ${NC}" >> ${exe_dir}/test_summary.txt
|
||||
# EXIT_CODE=8
|
||||
# fi
|
||||
|
||||
# printf "${YELLOW} start test_det_mv3_db ${NC} \n";
|
||||
# compile_test "test_det_mv3_db"
|
||||
# ${exe_dir}/test_det_mv3_db \
|
||||
# --modeldir=$DATA_DIR/ocr_det_mv3_db/ocr_det_mv3_db \
|
||||
# --gtest_filter=${test_suite_list} \
|
||||
# --gtest_output=xml:${log_dir}/test_det_mv3_db.xml
|
||||
# if [ $? -ne 0 ]; then
|
||||
# echo "${RED} test_det_mv3_db runs failed ${NC}" >> ${exe_dir}/test_summary.txt
|
||||
# EXIT_CODE=8
|
||||
# fi
|
||||
|
||||
# printf "${YELLOW} start test_LeViT ${NC} \n";
|
||||
# compile_test "test_LeViT"
|
||||
# ${exe_dir}/test_LeViT \
|
||||
# --modeldir=$DATA_DIR/LeViT/LeViT \
|
||||
# --gtest_filter=${test_suite_list} \
|
||||
# --gtest_output=xml:${log_dir}/test_LeViT.xml
|
||||
# if [ $? -ne 0 ]; then
|
||||
# echo "${RED} test_LeViT runs failed ${NC}" >> ${exe_dir}/test_summary.txt
|
||||
# EXIT_CODE=8
|
||||
# fi
|
||||
|
||||
if [ "$WIN_DETECT" != "" ]; then
|
||||
#TODO(OliverLPH): enable test_ernie_text_cls on windows after fix compile issue
|
||||
echo " skip test_ernie_text_cls "
|
||||
else
|
||||
printf "${YELLOW} start test_ernie_text_cls ${NC} \n";
|
||||
compile_test "test_ernie_text_cls"
|
||||
${exe_dir}/test_ernie_text_cls \
|
||||
--modeldir=$DATA_DIR/ernie_text_cls/ernie_text_cls \
|
||||
--gtest_filter=${test_suite_list} \
|
||||
--gtest_output=xml:${log_dir}/test_ernie_text_cls.xml
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "${RED} test_ernie_text_cls runs failed ${NC}" >> ${exe_dir}/test_summary.txt
|
||||
EXIT_CODE=8
|
||||
fi
|
||||
fi
|
||||
|
||||
printf "${YELLOW} start test_yolov3 ${NC} \n";
|
||||
compile_test "test_yolov3"
|
||||
${exe_dir}/test_yolov3 \
|
||||
--modeldir=$DATA_DIR/yolov3/yolov3 \
|
||||
--gtest_filter=${test_suite_list} \
|
||||
--gtest_output=xml:${log_dir}/test_yolov3.xml
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "${RED} test_yolov3 runs failed ${NC}" >> ${exe_dir}/test_summary.txt
|
||||
EXIT_CODE=8
|
||||
fi
|
||||
|
||||
# printf "${YELLOW} start test_ppyolo_mbv3 ${NC} \n";
|
||||
# compile_test "test_ppyolo_mbv3"
|
||||
# ${exe_dir}/test_ppyolo_mbv3 \
|
||||
# --modeldir=$DATA_DIR/ppyolo_mbv3/ppyolo_mbv3 \
|
||||
# --gtest_filter=${test_suite_list} \
|
||||
# --gtest_output=xml:${log_dir}/test_ppyolo_mbv3.xml
|
||||
# if [ $? -ne 0 ]; then
|
||||
# echo "${RED} test_ppyolo_mbv3 runs failed ${NC}" >> ${exe_dir}/test_summary.txt
|
||||
# EXIT_CODE=8
|
||||
# fi
|
||||
|
||||
# printf "${YELLOW} start test_ppyolov2_r50vd ${NC} \n";
|
||||
# compile_test "test_ppyolov2_r50vd"
|
||||
# ${exe_dir}/test_ppyolov2_r50vd \
|
||||
# --modeldir=$DATA_DIR/ppyolov2_r50vd/ppyolov2_r50vd \
|
||||
# --gtest_filter=${test_suite_list} \
|
||||
# --gtest_output=xml:${log_dir}/test_ppyolov2_r50vd.xml
|
||||
# if [ $? -ne 0 ]; then
|
||||
# echo "${RED} test_ppyolov2_r50vd runs failed ${NC}" >> ${exe_dir}/test_summary.txt
|
||||
# EXIT_CODE=8
|
||||
# fi
|
||||
|
||||
printf "${YELLOW} start test_resnet50_quant ${NC} \n";
|
||||
compile_test "test_resnet50_quant"
|
||||
${exe_dir}/test_resnet50_quant \
|
||||
--int8dir=$DATA_DIR/resnet50_quant/resnet50_quant/resnet50_quant \
|
||||
--modeldir=$DATA_DIR/resnet50/resnet50 \
|
||||
--datadir=$DATA_DIR/resnet50_quant/resnet50_quant/imagenet-eval-binary/9.data \
|
||||
--gtest_filter=${test_suite_list} \
|
||||
--gtest_output=xml:${log_dir}/test_resnet50_quant.xml
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "${RED} test_resnet50_quant runs failed ${NC}" >> ${exe_dir}/test_summary.txt
|
||||
EXIT_CODE=8
|
||||
fi
|
||||
|
||||
# printf "${YELLOW} start test_ernie_xnli_int8 ${NC} \n";
|
||||
# compile_test "test_ernie_xnli_int8"
|
||||
# ernie_qat_model="quant_post_model_xnli_predict_matmul"
|
||||
# ${exe_dir}/test_ernie_xnli_int8 \
|
||||
# --modeldir=$DATA_DIR/$ernie_qat_model/$ernie_qat_model \
|
||||
# --datadir=$DATA_DIR/$ernie_qat_model/$ernie_qat_model/xnli_var_len \
|
||||
# --truth_data=$DATA_DIR/$ernie_qat_model/$ernie_qat_model/truth_data \
|
||||
# --gtest_filter=${test_suite_list} \
|
||||
# --gtest_output=xml:${log_dir}/test_ernie_xnli_int8.xml
|
||||
# if [ $? -ne 0 ]; then
|
||||
# echo "${RED} test_ernie_xnli_int8 runs failed ${NC}" >> ${exe_dir}/test_summary.txt
|
||||
# EXIT_CODE=8
|
||||
# fi
|
||||
|
||||
printf "${YELLOW} start test_mobilnetv1 ${NC} \n";
|
||||
compile_test "test_mobilnetv1"
|
||||
${exe_dir}/test_mobilnetv1 \
|
||||
--modeldir=$DATA_DIR/MobileNetV1/MobileNetV1 \
|
||||
--gtest_filter=${test_suite_list} \
|
||||
--gtest_output=xml:${log_dir}/test_mobilnetv1.xml
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "${RED} test_mobilnetv1 runs failed ${NC}" >> ${exe_dir}/test_summary.txt
|
||||
EXIT_CODE=8
|
||||
fi
|
||||
|
||||
set +x
|
||||
|
||||
test_suites=$(echo ${test_suite_list} | sed 's/:/ /g')
|
||||
echo " "
|
||||
echo "CI Tested Following Patterns: "
|
||||
echo "=====================test patterns======================"
|
||||
for test_suite in ${test_suites}; do
|
||||
echo " ${test_suite}"
|
||||
done
|
||||
echo "========================================================"
|
||||
echo " "
|
||||
|
||||
if [[ -f ${exe_dir}/test_summary.txt ]];then
|
||||
echo " "
|
||||
echo "Summary infer_ut Failed Tests ..."
|
||||
echo "=====================test summary======================"
|
||||
echo "The following tests Failed: "
|
||||
cat ${exe_dir}/test_summary.txt
|
||||
echo "========================================================"
|
||||
echo " "
|
||||
fi
|
||||
set -x
|
||||
|
||||
# tar Gtest output report
|
||||
tar -zcvf infer_ut_log.tgz ${log_dir}
|
||||
|
||||
echo "infer_ut script finished"
|
||||
exit ${EXIT_CODE}
|
||||
@@ -0,0 +1,253 @@
|
||||
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "test_suite.h" // NOLINT
|
||||
#ifdef PADDLE_WITH_GPU
|
||||
#include <cuda_runtime.h>
|
||||
#endif
|
||||
|
||||
DEFINE_string(modeldir, "", "Directory of the inference model.");
|
||||
|
||||
namespace paddle_infer {
|
||||
|
||||
paddle::test::Record PrepareInput(int batch_size) {
|
||||
// init input data
|
||||
int channel = 3;
|
||||
int width = 224;
|
||||
int height = 224;
|
||||
paddle::test::Record image_Record;
|
||||
int input_num = batch_size * channel * width * height;
|
||||
std::vector<float> input_data(input_num, 1);
|
||||
image_Record.data = input_data;
|
||||
image_Record.shape = std::vector<int>{batch_size, channel, width, height};
|
||||
image_Record.type = paddle::PaddleDType::FLOAT32;
|
||||
return image_Record;
|
||||
}
|
||||
|
||||
TEST(gpu_tester_LeViT, analysis_gpu_bz1) {
|
||||
// init input data
|
||||
std::map<std::string, paddle::test::Record> my_input_data_map;
|
||||
my_input_data_map["x"] = PrepareInput(1);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data,
|
||||
truth_output_data;
|
||||
// prepare ground truth config
|
||||
paddle_infer::Config config, config_no_ir;
|
||||
config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config_no_ir.SwitchIrOptim(false);
|
||||
// prepare inference config
|
||||
config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
// get ground truth by disable ir
|
||||
paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
|
||||
// get infer results
|
||||
paddle_infer::services::PredictorPool pred_pool(config, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool.Retrieve(0), &my_input_data_map, &infer_output_data);
|
||||
// check outputs
|
||||
CompareRecord(&truth_output_data, &infer_output_data);
|
||||
std::cout << "finish test" << std::endl;
|
||||
}
|
||||
|
||||
TEST(tensorrt_tester_LeViT, trt_fp32_bz2) {
|
||||
// init input data
|
||||
std::map<std::string, paddle::test::Record> my_input_data_map;
|
||||
my_input_data_map["x"] = PrepareInput(2);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data,
|
||||
truth_output_data;
|
||||
// prepare ground truth config
|
||||
paddle_infer::Config config, config_no_ir;
|
||||
config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config_no_ir.SwitchIrOptim(false);
|
||||
// prepare inference config
|
||||
config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config.EnableUseGpu(100, 0);
|
||||
config.EnableTensorRtEngine(
|
||||
1 << 20, 2, 50, paddle_infer::PrecisionType::kFloat32, false, false);
|
||||
// get ground truth by disable ir
|
||||
paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
|
||||
// get infer results
|
||||
paddle_infer::services::PredictorPool pred_pool(config, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool.Retrieve(0), &my_input_data_map, &infer_output_data);
|
||||
// check outputs
|
||||
CompareRecord(&truth_output_data, &infer_output_data);
|
||||
std::cout << "finish test" << std::endl;
|
||||
}
|
||||
|
||||
TEST(tensorrt_tester_LeViT, serial_diff_batch_trt_fp32) {
|
||||
int max_batch_size = 5;
|
||||
// prepare ground truth config
|
||||
paddle_infer::Config config, config_no_ir;
|
||||
config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config_no_ir.SwitchIrOptim(false);
|
||||
paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
|
||||
// prepare inference config
|
||||
config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config.EnableUseGpu(100, 0);
|
||||
config.EnableTensorRtEngine(1 << 20,
|
||||
max_batch_size,
|
||||
50,
|
||||
paddle_infer::PrecisionType::kFloat32,
|
||||
false,
|
||||
false);
|
||||
paddle_infer::services::PredictorPool pred_pool(config, 1);
|
||||
|
||||
for (int i = 1; i < max_batch_size; i++) {
|
||||
// init input data
|
||||
std::map<std::string, paddle::test::Record> my_input_data_map;
|
||||
my_input_data_map["x"] = PrepareInput(i);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data,
|
||||
truth_output_data;
|
||||
// get ground truth by disable ir
|
||||
SingleThreadPrediction(
|
||||
pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
|
||||
// get infer results
|
||||
SingleThreadPrediction(
|
||||
pred_pool.Retrieve(0), &my_input_data_map, &infer_output_data);
|
||||
// check outputs
|
||||
CompareRecord(&truth_output_data, &infer_output_data);
|
||||
}
|
||||
std::cout << "finish test" << std::endl;
|
||||
}
|
||||
|
||||
TEST(tensorrt_tester_LeViT, multi_thread4_trt_fp32_bz2) {
|
||||
int thread_num = 4;
|
||||
// init input data
|
||||
std::map<std::string, paddle::test::Record> my_input_data_map;
|
||||
my_input_data_map["x"] = PrepareInput(2);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data,
|
||||
truth_output_data;
|
||||
// prepare ground truth config
|
||||
paddle_infer::Config config, config_no_ir;
|
||||
config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config_no_ir.SwitchIrOptim(false);
|
||||
// prepare inference config
|
||||
config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config.EnableUseGpu(100, 0);
|
||||
config.EnableTensorRtEngine(
|
||||
1 << 20, 2, 50, paddle_infer::PrecisionType::kFloat32, false, false);
|
||||
// get ground truth by disable ir
|
||||
paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
|
||||
|
||||
// get infer results from multi threads
|
||||
std::vector<std::thread> threads;
|
||||
services::PredictorPool pred_pool(config, thread_num);
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
threads.emplace_back(paddle::test::SingleThreadPrediction,
|
||||
pred_pool.Retrieve(i),
|
||||
&my_input_data_map,
|
||||
&infer_output_data,
|
||||
10);
|
||||
}
|
||||
|
||||
// thread join & check outputs
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
LOG(INFO) << "join tid : " << i;
|
||||
threads[i].join();
|
||||
CompareRecord(&truth_output_data, &infer_output_data);
|
||||
}
|
||||
|
||||
std::cout << "finish multi-thread test" << std::endl;
|
||||
}
|
||||
|
||||
#ifdef PADDLE_WITH_GPU
|
||||
TEST(tensorrt_tester_LeViT, multi_stream_thread4_trt_fp32_bz2) {
|
||||
int thread_num = 4;
|
||||
|
||||
// init stream
|
||||
std::vector<cudaStream_t> streams(thread_num);
|
||||
for (size_t i = 0; i < thread_num; ++i) {
|
||||
cudaStreamCreate(&streams[i]);
|
||||
}
|
||||
|
||||
// init input data
|
||||
std::map<std::string, paddle::test::Record> my_input_data_map;
|
||||
my_input_data_map["x"] = PrepareInput(2);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data,
|
||||
truth_output_data;
|
||||
// prepare ground truth config
|
||||
paddle_infer::Config config, config_no_ir;
|
||||
config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config_no_ir.SwitchIrOptim(false);
|
||||
// prepare inference config
|
||||
config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config.EnableUseGpu(100, 0);
|
||||
config.EnableTensorRtEngine(
|
||||
1 << 20, 2, 50, paddle_infer::PrecisionType::kFloat32, false, false);
|
||||
// get ground truth by disable ir
|
||||
|
||||
paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
|
||||
|
||||
// get infer results from multi threads
|
||||
std::vector<std::thread> threads;
|
||||
config.SetExecStream(streams[0]);
|
||||
config.pass_builder()->DeletePass("add_support_int8_pass");
|
||||
auto main_predictor = CreatePredictor(config);
|
||||
std::vector<decltype(main_predictor)> predictors;
|
||||
for (size_t i = 0; i < thread_num - 1; ++i) {
|
||||
predictors.push_back(std::move(main_predictor->Clone(streams[i + 1])));
|
||||
LOG(INFO) << "predictors[" << i << "] stream is "
|
||||
<< predictors[i]->GetExecStream();
|
||||
}
|
||||
predictors.push_back(std::move(main_predictor));
|
||||
LOG(INFO) << "predictors[" << thread_num - 1 << "] stream is "
|
||||
<< predictors[thread_num - 1]->GetExecStream();
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
threads.emplace_back(paddle::test::SingleThreadPrediction,
|
||||
predictors[i].get(),
|
||||
&my_input_data_map,
|
||||
&infer_output_data,
|
||||
10);
|
||||
}
|
||||
|
||||
// thread join & check outputs
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
LOG(INFO) << "join tid : " << i;
|
||||
threads[i].join();
|
||||
CompareRecord(&truth_output_data, &infer_output_data);
|
||||
}
|
||||
|
||||
std::cout << "finish multi-thread test" << std::endl;
|
||||
}
|
||||
#endif
|
||||
|
||||
} // namespace paddle_infer
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
gflags::ParseCommandLineFlags(&argc, &argv, true);
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "test_suite.h" // NOLINT
|
||||
|
||||
DEFINE_string(modeldir, "", "Directory of the inference model.");
|
||||
|
||||
namespace paddle_infer {
|
||||
|
||||
paddle::test::Record PrepareInput(int batch_size, int image_shape = 640) {
|
||||
// init input data
|
||||
int channel = 3;
|
||||
int width = image_shape;
|
||||
int height = image_shape;
|
||||
paddle::test::Record image_Record;
|
||||
int input_num = batch_size * channel * width * height;
|
||||
std::vector<float> input_data(input_num, 1);
|
||||
image_Record.data = input_data;
|
||||
image_Record.shape = std::vector<int>{batch_size, channel, width, height};
|
||||
image_Record.type = paddle::PaddleDType::FLOAT32;
|
||||
return image_Record;
|
||||
}
|
||||
|
||||
void PrepareDynamicShape(paddle_infer::Config* config, int max_batch_size = 4) {
|
||||
// set dynamic shape range
|
||||
std::map<std::string, std::vector<int>> min_input_shape = {
|
||||
{"x", {1, 3, 224, 224}},
|
||||
{"conv2d_124.tmp_0", {1, 256, 56, 56}},
|
||||
{"nearest_interp_v2_2.tmp_0", {1, 256, 56, 56}},
|
||||
{"nearest_interp_v2_3.tmp_0", {1, 64, 56, 56}},
|
||||
{"nearest_interp_v2_4.tmp_0", {1, 64, 56, 56}},
|
||||
{"nearest_interp_v2_5.tmp_0", {1, 64, 56, 56}}};
|
||||
std::map<std::string, std::vector<int>> max_input_shape = {
|
||||
{"x", {max_batch_size, 3, 448, 448}},
|
||||
{"conv2d_124.tmp_0", {max_batch_size, 256, 112, 112}},
|
||||
{"nearest_interp_v2_2.tmp_0", {max_batch_size, 256, 112, 112}},
|
||||
{"nearest_interp_v2_3.tmp_0", {max_batch_size, 64, 112, 112}},
|
||||
{"nearest_interp_v2_4.tmp_0", {max_batch_size, 64, 112, 112}},
|
||||
{"nearest_interp_v2_5.tmp_0", {max_batch_size, 64, 112, 112}}};
|
||||
std::map<std::string, std::vector<int>> opt_input_shape = {
|
||||
{"x", {1, 3, 256, 256}},
|
||||
{"conv2d_124.tmp_0", {1, 256, 64, 64}},
|
||||
{"nearest_interp_v2_2.tmp_0", {1, 256, 64, 64}},
|
||||
{"nearest_interp_v2_3.tmp_0", {1, 64, 64, 64}},
|
||||
{"nearest_interp_v2_4.tmp_0", {1, 64, 64, 64}},
|
||||
{"nearest_interp_v2_5.tmp_0", {1, 64, 64, 64}}};
|
||||
config->SetTRTDynamicShapeInfo(
|
||||
min_input_shape, max_input_shape, opt_input_shape);
|
||||
}
|
||||
|
||||
TEST(gpu_tester_det_mv3_db, analysis_gpu_bz4) {
|
||||
// init input data
|
||||
std::map<std::string, paddle::test::Record> my_input_data_map;
|
||||
my_input_data_map["x"] = PrepareInput(4, 640);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data,
|
||||
truth_output_data;
|
||||
// prepare ground truth config
|
||||
paddle_infer::Config config, config_no_ir;
|
||||
config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config_no_ir.SwitchIrOptim(false);
|
||||
// prepare inference config
|
||||
config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
// get ground truth by disable ir
|
||||
paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
|
||||
// get infer results
|
||||
paddle_infer::services::PredictorPool pred_pool(config, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool.Retrieve(0), &my_input_data_map, &infer_output_data);
|
||||
// check outputs
|
||||
CompareRecord(&truth_output_data, &infer_output_data, 1e-4);
|
||||
std::cout << "finish test" << std::endl;
|
||||
}
|
||||
|
||||
TEST(tensorrt_tester_det_mv3_db, multi_thread2_trt_fp32_dynamic_shape_bz2) {
|
||||
int thread_num = 2; // thread > 2 may OOM
|
||||
// init input data
|
||||
std::map<std::string, paddle::test::Record> my_input_data_map;
|
||||
my_input_data_map["x"] = PrepareInput(2, 256);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data,
|
||||
truth_output_data;
|
||||
// prepare ground truth config
|
||||
paddle_infer::Config config, config_no_ir;
|
||||
config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config_no_ir.SwitchIrOptim(false);
|
||||
// prepare inference config
|
||||
config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config.EnableUseGpu(100, 0);
|
||||
config.EnableTensorRtEngine(
|
||||
1 << 20, 4, 3, paddle_infer::PrecisionType::kFloat32, false, false);
|
||||
PrepareDynamicShape(&config, 4);
|
||||
// get ground truth by disable ir
|
||||
paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
|
||||
|
||||
// get infer results from multi threads
|
||||
std::vector<std::thread> threads;
|
||||
services::PredictorPool pred_pool(config, thread_num);
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
threads.emplace_back(paddle::test::SingleThreadPrediction,
|
||||
pred_pool.Retrieve(i),
|
||||
&my_input_data_map,
|
||||
&infer_output_data,
|
||||
2);
|
||||
}
|
||||
|
||||
// thread join & check outputs
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
LOG(INFO) << "join tid : " << i;
|
||||
threads[i].join();
|
||||
CompareRecord(&truth_output_data, &infer_output_data, 1e-4);
|
||||
}
|
||||
|
||||
std::cout << "finish multi-thread test" << std::endl;
|
||||
}
|
||||
|
||||
TEST(onednn_tester_det_mv3_db, multi_thread2_mkl_fp32_bz2) {
|
||||
int thread_num = 2; // thread > 2 may OOM
|
||||
// init input data
|
||||
std::map<std::string, paddle::test::Record> my_input_data_map;
|
||||
my_input_data_map["x"] = PrepareInput(2, 640);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data,
|
||||
truth_output_data;
|
||||
// prepare ground truth config
|
||||
paddle_infer::Config config, config_no_ir;
|
||||
config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config_no_ir.SwitchIrOptim(false);
|
||||
// prepare inference config
|
||||
config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config.DisableGpu();
|
||||
config.EnableONEDNN();
|
||||
config.SetOnednnCacheCapacity(10);
|
||||
config.SetCpuMathLibraryNumThreads(10);
|
||||
// get ground truth by disable ir
|
||||
paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
|
||||
|
||||
// get infer results from multi threads
|
||||
std::vector<std::thread> threads;
|
||||
services::PredictorPool pred_pool(config, thread_num);
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
threads.emplace_back(paddle::test::SingleThreadPrediction,
|
||||
pred_pool.Retrieve(i),
|
||||
&my_input_data_map,
|
||||
&infer_output_data,
|
||||
2);
|
||||
}
|
||||
|
||||
// thread join & check outputs
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
LOG(INFO) << "join tid : " << i;
|
||||
threads[i].join();
|
||||
CompareRecord(&truth_output_data, &infer_output_data, 1e-4);
|
||||
}
|
||||
|
||||
std::cout << "finish multi-thread test" << std::endl;
|
||||
}
|
||||
|
||||
} // namespace paddle_infer
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
gflags::ParseCommandLineFlags(&argc, &argv, true);
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "test_suite.h" // NOLINT
|
||||
|
||||
DEFINE_string(modeldir, "", "Directory of the inference model.");
|
||||
|
||||
namespace paddle_infer {
|
||||
|
||||
template <typename T>
|
||||
T cRandom(int min, int max) {
|
||||
unsigned int seed = 100;
|
||||
return (min +
|
||||
static_cast<T>(max * rand_r(&seed) / static_cast<T>(RAND_MAX + 1)));
|
||||
}
|
||||
|
||||
std::map<std::string, paddle::test::Record> PrepareInput(int batch_size) {
|
||||
// init input data
|
||||
int digit_length = 115;
|
||||
paddle::test::Record input_ids, segment_ids;
|
||||
int input_num = batch_size * digit_length;
|
||||
std::vector<int64_t> input_data(input_num, 1);
|
||||
std::vector<int64_t> segment_data(input_num, 0);
|
||||
srand((unsigned)time(NULL));
|
||||
for (int x = 0; x < input_data.size(); x++) {
|
||||
input_data[x] = cRandom<int>(1, 100);
|
||||
}
|
||||
input_ids.data = std::vector<float>(input_data.begin(), input_data.end());
|
||||
input_ids.shape = std::vector<int>{batch_size, digit_length};
|
||||
input_ids.type = paddle::PaddleDType::INT64;
|
||||
|
||||
segment_ids.data =
|
||||
std::vector<float>(segment_data.begin(), segment_data.end());
|
||||
segment_ids.shape = std::vector<int>{batch_size, digit_length};
|
||||
segment_ids.type = paddle::PaddleDType::INT64;
|
||||
|
||||
std::map<std::string, paddle::test::Record> my_input_data_map;
|
||||
my_input_data_map.insert({"input_ids", input_ids});
|
||||
my_input_data_map.insert({"token_type_ids", segment_ids});
|
||||
|
||||
return my_input_data_map;
|
||||
}
|
||||
|
||||
TEST(gpu_tester_ernie_text_cls, analysis_gpu_bz2_buffer) {
|
||||
// init input data
|
||||
auto my_input_data_map = PrepareInput(2);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data,
|
||||
truth_output_data;
|
||||
// prepare ground truth config
|
||||
paddle_infer::Config config, config_no_ir;
|
||||
config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config_no_ir.SwitchIrOptim(false);
|
||||
|
||||
// prepare inference config from buffer
|
||||
std::string prog_file = FLAGS_modeldir + "/inference.pdmodel";
|
||||
std::string params_file = FLAGS_modeldir + "/inference.pdiparams";
|
||||
std::string prog_str = paddle::test::read_file(prog_file);
|
||||
std::string params_str = paddle::test::read_file(params_file);
|
||||
config.SetModelBuffer(
|
||||
prog_str.c_str(), prog_str.size(), params_str.c_str(), params_str.size());
|
||||
// get ground truth by disable ir
|
||||
paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
|
||||
// get infer results
|
||||
paddle_infer::services::PredictorPool pred_pool(config, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool.Retrieve(0), &my_input_data_map, &infer_output_data);
|
||||
// check outputs
|
||||
CompareRecord(&truth_output_data, &infer_output_data);
|
||||
std::cout << "finish test" << std::endl;
|
||||
}
|
||||
|
||||
TEST(onednn_tester_ernie_text_cls, multi_thread4_mkl_fp32_bz2) {
|
||||
int thread_num = 4;
|
||||
// init input data
|
||||
auto my_input_data_map = PrepareInput(2);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data,
|
||||
truth_output_data;
|
||||
// prepare ground truth config
|
||||
paddle_infer::Config config, config_no_ir;
|
||||
config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config.DisableGpu();
|
||||
config_no_ir.SwitchIrOptim(false);
|
||||
// prepare inference config
|
||||
config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config.DisableGpu();
|
||||
config.EnableONEDNN();
|
||||
config.SetOnednnCacheCapacity(10);
|
||||
config.SetCpuMathLibraryNumThreads(10);
|
||||
// get ground truth by disable ir
|
||||
paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
|
||||
|
||||
// get infer results from multi threads
|
||||
std::vector<std::thread> threads;
|
||||
services::PredictorPool pred_pool(config, thread_num);
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
threads.emplace_back(paddle::test::SingleThreadPrediction,
|
||||
pred_pool.Retrieve(i),
|
||||
&my_input_data_map,
|
||||
&infer_output_data,
|
||||
2);
|
||||
}
|
||||
|
||||
// thread join & check outputs
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
LOG(INFO) << "join tid : " << i;
|
||||
threads[i].join();
|
||||
CompareRecord(&truth_output_data, &infer_output_data);
|
||||
}
|
||||
|
||||
std::cout << "finish multi-thread test" << std::endl;
|
||||
}
|
||||
|
||||
} // namespace paddle_infer
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
gflags::ParseCommandLineFlags(&argc, &argv, true);
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "test_helper.h" // NOLINT
|
||||
#include "test_suite.h" // NOLINT
|
||||
|
||||
DEFINE_string(modeldir, "", "Directory of the inference model.");
|
||||
DEFINE_string(datadir, "", "dataset.");
|
||||
DEFINE_string(truth_data, "", "Directory of the inference data truth result");
|
||||
|
||||
namespace paddle_infer {
|
||||
|
||||
std::shared_ptr<Predictor> InitPredictor() {
|
||||
Config config;
|
||||
config.SetModel(FLAGS_modeldir + "/__model__",
|
||||
FLAGS_modeldir + "/__params__");
|
||||
config.EnableUseGpu(1000, 0);
|
||||
// Open the memory optim.
|
||||
config.EnableMemoryOptim();
|
||||
|
||||
int max_batch = 32;
|
||||
int max_single_seq_len = 128;
|
||||
int opt_single_seq_len = 64;
|
||||
int min_batch_seq_len = 1;
|
||||
int max_batch_seq_len = 512;
|
||||
int opt_batch_seq_len = 256;
|
||||
|
||||
std::string input_name0 = "eval_placeholder_0";
|
||||
std::string input_name1 = "eval_placeholder_1";
|
||||
std::string input_name2 = "eval_placeholder_2";
|
||||
std::string input_name3 = "eval_placeholder_3";
|
||||
|
||||
std::vector<int> min_shape = {min_batch_seq_len};
|
||||
std::vector<int> max_shape = {max_batch_seq_len};
|
||||
std::vector<int> opt_shape = {opt_batch_seq_len};
|
||||
// Set the input's min, max, opt shape
|
||||
std::map<std::string, std::vector<int>> min_input_shape = {
|
||||
{input_name0, min_shape},
|
||||
{input_name1, min_shape},
|
||||
{input_name2, {1}},
|
||||
{input_name3, {1, min_batch_seq_len, 1}}};
|
||||
std::map<std::string, std::vector<int>> max_input_shape = {
|
||||
{input_name0, max_shape},
|
||||
{input_name1, max_shape},
|
||||
{input_name2, {max_batch + 1}},
|
||||
{input_name3, {1, max_single_seq_len, 1}}};
|
||||
std::map<std::string, std::vector<int>> opt_input_shape = {
|
||||
{input_name0, opt_shape},
|
||||
{input_name1, opt_shape},
|
||||
{input_name2, {max_batch + 1}},
|
||||
{input_name3, {1, opt_single_seq_len, 1}}};
|
||||
|
||||
// only kHalf supported
|
||||
config.EnableTensorRtEngine(
|
||||
1 << 30, 1, 5, Config::Precision::kInt8, false, false);
|
||||
// ernie varlen must be used with dynamic shape
|
||||
config.SetTRTDynamicShapeInfo(
|
||||
min_input_shape, max_input_shape, opt_input_shape);
|
||||
// ernie varlen must be used with oss
|
||||
config.EnableVarseqlen();
|
||||
paddle_infer::experimental::InternalUtils::SetTransformerPosid(&config,
|
||||
input_name2);
|
||||
paddle_infer::experimental::InternalUtils::SetTransformerMaskid(&config,
|
||||
input_name3);
|
||||
|
||||
return CreatePredictor(config);
|
||||
}
|
||||
|
||||
// Parse tensor from string
|
||||
template <typename T>
|
||||
std::vector<T> ParseTensor(const std::string &field) {
|
||||
std::string mat_str = field;
|
||||
|
||||
std::vector<T> mat;
|
||||
paddle::test::Split(mat_str, ' ', &mat);
|
||||
|
||||
return mat;
|
||||
}
|
||||
|
||||
void run(Predictor *predictor, std::vector<float> *out_data) {
|
||||
clock_t start, end;
|
||||
start = clock();
|
||||
CHECK(predictor->Run());
|
||||
end = clock();
|
||||
|
||||
auto output_names = predictor->GetOutputNames();
|
||||
auto output_t = predictor->GetOutputHandle(output_names[0]);
|
||||
std::vector<int> output_shape = output_t->shape();
|
||||
int out_num = std::accumulate(
|
||||
output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
|
||||
out_data->resize(out_num);
|
||||
output_t->CopyToCpu(out_data->data());
|
||||
return;
|
||||
}
|
||||
|
||||
auto PrepareOutput(std::string input_file) -> std::deque<float> {
|
||||
std::ifstream fin(input_file);
|
||||
std::string line;
|
||||
std::vector<std::string> buffer;
|
||||
while (std::getline(fin, line)) {
|
||||
buffer.emplace_back(line);
|
||||
}
|
||||
std::deque<float> resDeque(buffer.size());
|
||||
std::transform(buffer.begin(),
|
||||
buffer.end(),
|
||||
resDeque.begin(),
|
||||
[](const std::string &val) { return std::stof(val); });
|
||||
|
||||
return resDeque;
|
||||
} // PrepareOutput
|
||||
|
||||
TEST(tensorrt_tester_ernie_xnli, oss_varlen_truth_data_int8) {
|
||||
auto resDeque = PrepareOutput(FLAGS_truth_data);
|
||||
auto predictor = InitPredictor();
|
||||
|
||||
ASSERT_FALSE(FLAGS_datadir.empty());
|
||||
std::ifstream fin(FLAGS_datadir);
|
||||
std::string line;
|
||||
|
||||
int lineno = 0;
|
||||
const int max_seq_len = 128;
|
||||
const int run_batch = 1;
|
||||
int correct_num = 0;
|
||||
while (std::getline(fin, line)) {
|
||||
std::vector<std::string> fields;
|
||||
paddle::test::Split(line, ';', &fields);
|
||||
|
||||
auto src_ids = ParseTensor<int32_t>(fields[0]);
|
||||
auto sent_ids = ParseTensor<int32_t>(fields[1]);
|
||||
auto pos_ids = ParseTensor<int64_t>(fields[2]);
|
||||
|
||||
int run_seq_len = src_ids.size();
|
||||
int32_t i3[2] = {0, run_seq_len};
|
||||
int32_t i4[max_seq_len] = {0};
|
||||
|
||||
auto input_names = predictor->GetInputNames();
|
||||
|
||||
// first input
|
||||
auto input_t1 = predictor->GetInputHandle(input_names[0]);
|
||||
input_t1->Reshape({run_seq_len});
|
||||
input_t1->CopyFromCpu(src_ids.data());
|
||||
|
||||
// second input
|
||||
auto input_t2 = predictor->GetInputHandle(input_names[1]);
|
||||
input_t2->Reshape({run_seq_len});
|
||||
input_t2->CopyFromCpu(sent_ids.data());
|
||||
|
||||
// third input
|
||||
auto input_t3 = predictor->GetInputHandle(input_names[2]);
|
||||
input_t3->Reshape({run_batch + 1});
|
||||
input_t3->CopyFromCpu(i3);
|
||||
|
||||
// fourth input
|
||||
auto input_t4 = predictor->GetInputHandle(input_names[3]);
|
||||
input_t4->Reshape({1, max_seq_len, 1});
|
||||
input_t4->CopyFromCpu(i4);
|
||||
|
||||
std::vector<float> out_data;
|
||||
run(predictor.get(), &out_data);
|
||||
|
||||
lineno++;
|
||||
int maxPosition =
|
||||
max_element(out_data.begin(), out_data.end()) - out_data.begin();
|
||||
|
||||
if (maxPosition == resDeque[0]) {
|
||||
correct_num += 1;
|
||||
}
|
||||
resDeque.pop_front();
|
||||
|
||||
VLOG(2) << "predict result: " << maxPosition;
|
||||
for (auto r : out_data) {
|
||||
VLOG(2) << r;
|
||||
}
|
||||
}
|
||||
ASSERT_GT(correct_num,
|
||||
3855); // total input 5010, int8 res should greater than 3855
|
||||
LOG(INFO) << "=== finish oss test ===";
|
||||
}
|
||||
|
||||
} // namespace paddle_infer
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
gflags::ParseCommandLineFlags(&argc, &argv, true);
|
||||
|
||||
#if IS_TRT_VERSION_GE(7200)
|
||||
return RUN_ALL_TESTS();
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace paddle {
|
||||
namespace test {
|
||||
|
||||
// split string to vector<string> by sep
|
||||
static void split(const std::string &str,
|
||||
char sep,
|
||||
std::vector<std::string> *pieces,
|
||||
bool ignore_null = true) {
|
||||
pieces->clear();
|
||||
if (str.empty()) {
|
||||
if (!ignore_null) {
|
||||
pieces->push_back(str);
|
||||
}
|
||||
return;
|
||||
}
|
||||
size_t pos = 0;
|
||||
size_t next = str.find(sep, pos);
|
||||
while (next != std::string::npos) {
|
||||
pieces->push_back(str.substr(pos, next - pos));
|
||||
pos = next + 1;
|
||||
next = str.find(sep, pos);
|
||||
}
|
||||
if (!str.substr(pos).empty()) {
|
||||
pieces->push_back(str.substr(pos));
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void GetValueFromStream(std::stringstream *ss, T *t) {
|
||||
(*ss) >> (*t);
|
||||
}
|
||||
|
||||
template <>
|
||||
void GetValueFromStream<std::string>(std::stringstream *ss, std::string *t) {
|
||||
*t = ss->str();
|
||||
}
|
||||
|
||||
// Split string to multiple vector
|
||||
template <typename T>
|
||||
void Split(const std::string &line, char sep, std::vector<T> *v) {
|
||||
std::stringstream ss;
|
||||
T t;
|
||||
for (auto c : line) {
|
||||
if (c != sep) {
|
||||
ss << c;
|
||||
} else {
|
||||
GetValueFromStream<T>(&ss, &t);
|
||||
v->push_back(std::move(t));
|
||||
ss.str({});
|
||||
ss.clear();
|
||||
}
|
||||
}
|
||||
|
||||
if (!ss.str().empty()) {
|
||||
GetValueFromStream<T>(&ss, &t);
|
||||
v->push_back(std::move(t));
|
||||
ss.str({});
|
||||
ss.clear();
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace test
|
||||
} // namespace paddle
|
||||
@@ -0,0 +1,86 @@
|
||||
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "test_helper.h" // NOLINT
|
||||
#include "test_suite.h" // NOLINT
|
||||
|
||||
DEFINE_string(modeldir, "", "Directory of the inference model.");
|
||||
|
||||
namespace paddle_infer {
|
||||
|
||||
paddle::test::Record PrepareInput(int batch_size, int shape_size = 224) {
|
||||
// init input data
|
||||
int channel = 3;
|
||||
int width = shape_size; // w = 224
|
||||
int height = shape_size; // h = 224
|
||||
paddle::test::Record image_Record;
|
||||
int input_num = batch_size * channel * width * height;
|
||||
std::vector<float> input_data(input_num, 1);
|
||||
image_Record.data = input_data;
|
||||
image_Record.shape = std::vector<int>{batch_size, channel, width, height};
|
||||
image_Record.type = paddle::PaddleDType::FLOAT32;
|
||||
return image_Record;
|
||||
}
|
||||
|
||||
TEST(tensorrt_tester_mobilenetv1, tuned_dynamic_trt_fp32_bz2) {
|
||||
bool tuned_shape = true;
|
||||
std::string shape_range_info = FLAGS_modeldir + "/shape_range_info.pbtxt";
|
||||
LOG(INFO) << "tensorrt tuned info saved to " << shape_range_info;
|
||||
|
||||
// init input data
|
||||
std::map<std::string, paddle::test::Record> my_input_data_map;
|
||||
my_input_data_map["x"] = PrepareInput(2, 448);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data,
|
||||
truth_output_data;
|
||||
if (tuned_shape) {
|
||||
// NOTE: shape_range_info will be saved after destructor of predictor
|
||||
// function
|
||||
// prepare ground truth config
|
||||
paddle_infer::Config tune_config;
|
||||
tune_config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
tune_config.SwitchIrOptim(false);
|
||||
tune_config.EnableUseGpu(1000, 0);
|
||||
tune_config.CollectShapeRangeInfo(shape_range_info);
|
||||
|
||||
auto predictor_tune = paddle_infer::CreatePredictor(tune_config);
|
||||
SingleThreadPrediction(
|
||||
predictor_tune.get(), &my_input_data_map, &truth_output_data, 1);
|
||||
}
|
||||
|
||||
// prepare inference config
|
||||
paddle_infer::Config config;
|
||||
config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config.EnableUseGpu(1000, 0);
|
||||
config.EnableTensorRtEngine(
|
||||
1 << 20, 2, 5, paddle_infer::PrecisionType::kFloat32, false, false);
|
||||
config.EnableTunedTensorRtDynamicShape(shape_range_info, true);
|
||||
LOG(INFO) << config.Summary();
|
||||
paddle_infer::services::PredictorPool pred_pool(config, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool.Retrieve(0), &my_input_data_map, &infer_output_data);
|
||||
// check outputs
|
||||
CompareRecord(&truth_output_data, &infer_output_data);
|
||||
VLOG(1) << "finish test";
|
||||
}
|
||||
|
||||
} // namespace paddle_infer
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
gflags::ParseCommandLineFlags(&argc, &argv, true);
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "test_suite.h" // NOLINT
|
||||
|
||||
DEFINE_string(modeldir, "", "Directory of the inference model.");
|
||||
|
||||
namespace paddle_infer {
|
||||
|
||||
std::map<std::string, paddle::test::Record> PrepareInput(int batch_size) {
|
||||
// init input data
|
||||
int channel = 3;
|
||||
int width = 320;
|
||||
int height = 320;
|
||||
paddle::test::Record image, im_shape, scale_factor;
|
||||
int input_num = batch_size * channel * width * height;
|
||||
int shape_num = batch_size * 2;
|
||||
std::vector<float> image_data(input_num, 1);
|
||||
for (int i = 1; i < input_num + 1; ++i) {
|
||||
image_data[i] = i % 10 * 0.5;
|
||||
}
|
||||
std::vector<float> im_shape_data(shape_num, 1);
|
||||
std::vector<float> scale_factor_data(shape_num, 1);
|
||||
|
||||
image.data = std::vector<float>(image_data.begin(), image_data.end());
|
||||
image.shape = std::vector<int>{batch_size, channel, width, height};
|
||||
image.type = paddle::PaddleDType::FLOAT32;
|
||||
|
||||
im_shape.data =
|
||||
std::vector<float>(im_shape_data.begin(), im_shape_data.end());
|
||||
im_shape.shape = std::vector<int>{batch_size, 2};
|
||||
im_shape.type = paddle::PaddleDType::FLOAT32;
|
||||
|
||||
scale_factor.data =
|
||||
std::vector<float>(scale_factor_data.begin(), scale_factor_data.end());
|
||||
scale_factor.shape = std::vector<int>{batch_size, 2};
|
||||
scale_factor.type = paddle::PaddleDType::FLOAT32;
|
||||
|
||||
std::map<std::string, paddle::test::Record> input_data_map;
|
||||
input_data_map.insert({"image", image});
|
||||
input_data_map.insert({"im_shape", im_shape});
|
||||
input_data_map.insert({"scale_factor", scale_factor});
|
||||
|
||||
return input_data_map;
|
||||
}
|
||||
|
||||
TEST(tensorrt_tester_ppyolo_mbv3, multi_thread4_trt_fp32_bz2) {
|
||||
int thread_num = 4;
|
||||
// init input data
|
||||
auto input_data_map = PrepareInput(2);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data,
|
||||
truth_output_data;
|
||||
// prepare ground truth config
|
||||
paddle_infer::Config config, config_no_ir;
|
||||
config_no_ir.SetModel(FLAGS_modeldir + "/model.pdmodel",
|
||||
FLAGS_modeldir + "/model.pdiparams");
|
||||
config_no_ir.EnableUseGpu(100, 0);
|
||||
config_no_ir.SwitchIrOptim(false);
|
||||
// prepare inference config
|
||||
config.SetModel(FLAGS_modeldir + "/model.pdmodel",
|
||||
FLAGS_modeldir + "/model.pdiparams");
|
||||
config.EnableUseGpu(100, 0);
|
||||
config.EnableTensorRtEngine(
|
||||
1 << 25, 2, 3, paddle_infer::PrecisionType::kFloat32, false, false);
|
||||
LOG(INFO) << config.Summary();
|
||||
// get ground truth by disable ir
|
||||
paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool_no_ir.Retrieve(0), &input_data_map, &truth_output_data, 1);
|
||||
|
||||
// get infer results from multi threads
|
||||
std::vector<std::thread> threads;
|
||||
services::PredictorPool pred_pool(config, thread_num);
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
threads.emplace_back(paddle::test::SingleThreadPrediction,
|
||||
pred_pool.Retrieve(i),
|
||||
&input_data_map,
|
||||
&infer_output_data,
|
||||
2);
|
||||
}
|
||||
|
||||
// thread join & check outputs
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
LOG(INFO) << "join tid : " << i;
|
||||
threads[i].join();
|
||||
CompareRecord(&truth_output_data, &infer_output_data, 0.18);
|
||||
// TODO(OliverLPH): precision set to 1e-2 since input is fake, change to
|
||||
// real input later
|
||||
}
|
||||
|
||||
std::cout << "finish multi-thread test" << std::endl;
|
||||
}
|
||||
|
||||
TEST(DISABLED_onednn_tester_ppyolo_mbv3, multi_thread4_mkl_bz2) {
|
||||
// TODO(OliverLPH): onednn multi thread will fail
|
||||
int thread_num = 4;
|
||||
// init input data
|
||||
auto input_data_map = PrepareInput(2);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data,
|
||||
truth_output_data;
|
||||
// prepare ground truth config
|
||||
paddle_infer::Config config, config_no_ir;
|
||||
config_no_ir.SetModel(FLAGS_modeldir + "/model.pdmodel",
|
||||
FLAGS_modeldir + "/model.pdiparams");
|
||||
config_no_ir.DisableGpu();
|
||||
config_no_ir.SwitchIrOptim(false);
|
||||
// prepare inference config
|
||||
config.SetModel(FLAGS_modeldir + "/model.pdmodel",
|
||||
FLAGS_modeldir + "/model.pdiparams");
|
||||
config.DisableGpu();
|
||||
config.EnableONEDNN();
|
||||
config.SetOnednnCacheCapacity(10);
|
||||
config.SetCpuMathLibraryNumThreads(10);
|
||||
LOG(INFO) << config.Summary();
|
||||
// get ground truth by disable ir
|
||||
paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool_no_ir.Retrieve(0), &input_data_map, &truth_output_data, 1);
|
||||
|
||||
// get infer results from multi threads
|
||||
std::vector<std::thread> threads;
|
||||
services::PredictorPool pred_pool(config, thread_num);
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
threads.emplace_back(paddle::test::SingleThreadPrediction,
|
||||
pred_pool.Retrieve(i),
|
||||
&input_data_map,
|
||||
&infer_output_data,
|
||||
2);
|
||||
}
|
||||
|
||||
// thread join & check outputs
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
LOG(INFO) << "join tid : " << i;
|
||||
threads[i].join();
|
||||
CompareRecord(&truth_output_data, &infer_output_data, 1e-4);
|
||||
}
|
||||
|
||||
std::cout << "finish multi-thread test" << std::endl;
|
||||
}
|
||||
|
||||
} // namespace paddle_infer
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
gflags::ParseCommandLineFlags(&argc, &argv, true);
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "test_suite.h" // NOLINT
|
||||
|
||||
DEFINE_string(modeldir, "", "Directory of the inference model.");
|
||||
|
||||
namespace paddle_infer {
|
||||
|
||||
std::map<std::string, paddle::test::Record> PrepareInput(int batch_size) {
|
||||
// init input data
|
||||
int channel = 3;
|
||||
int width = 640;
|
||||
int height = 640;
|
||||
paddle::test::Record image, im_shape, scale_factor;
|
||||
int input_num = batch_size * channel * width * height;
|
||||
int shape_num = batch_size * 2;
|
||||
std::vector<float> image_data(input_num, 1);
|
||||
for (int i = 1; i < input_num + 1; ++i) {
|
||||
image_data[i] = i % 10 * 0.5;
|
||||
}
|
||||
std::vector<float> im_shape_data(shape_num, 1);
|
||||
std::vector<float> scale_factor_data(shape_num, 1);
|
||||
|
||||
image.data = std::vector<float>(image_data.begin(), image_data.end());
|
||||
image.shape = std::vector<int>{batch_size, channel, width, height};
|
||||
image.type = paddle::PaddleDType::FLOAT32;
|
||||
|
||||
im_shape.data =
|
||||
std::vector<float>(im_shape_data.begin(), im_shape_data.end());
|
||||
im_shape.shape = std::vector<int>{batch_size, 2};
|
||||
im_shape.type = paddle::PaddleDType::FLOAT32;
|
||||
|
||||
scale_factor.data =
|
||||
std::vector<float>(scale_factor_data.begin(), scale_factor_data.end());
|
||||
scale_factor.shape = std::vector<int>{batch_size, 2};
|
||||
scale_factor.type = paddle::PaddleDType::FLOAT32;
|
||||
|
||||
std::map<std::string, paddle::test::Record> input_data_map;
|
||||
input_data_map.insert({"image", image});
|
||||
input_data_map.insert({"im_shape", im_shape});
|
||||
input_data_map.insert({"scale_factor", scale_factor});
|
||||
|
||||
return input_data_map;
|
||||
}
|
||||
|
||||
TEST(tensorrt_tester_ppyolov2_r50vd, multi_thread2_trt_fp32_bz1) {
|
||||
int thread_num = 2; // thread > 2 may OOM
|
||||
// init input data
|
||||
auto input_data_map = PrepareInput(1);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data,
|
||||
truth_output_data;
|
||||
// prepare ground truth config
|
||||
paddle_infer::Config config, config_no_ir;
|
||||
config_no_ir.SetModel(FLAGS_modeldir + "/model.pdmodel",
|
||||
FLAGS_modeldir + "/model.pdiparams");
|
||||
config_no_ir.EnableUseGpu(100, 0);
|
||||
config_no_ir.SwitchIrOptim(false);
|
||||
// prepare inference config
|
||||
config.SetModel(FLAGS_modeldir + "/model.pdmodel",
|
||||
FLAGS_modeldir + "/model.pdiparams");
|
||||
config.EnableUseGpu(100, 0);
|
||||
config.EnableTensorRtEngine(
|
||||
1 << 28, 2, 10, paddle_infer::PrecisionType::kFloat32, false, false);
|
||||
std::map<std::string, std::vector<int>> input_shape;
|
||||
input_shape["image"] = {1, 3, 640, 640};
|
||||
input_shape["im_shape"] = {1, 2};
|
||||
input_shape["scale_factor"] = {1, 2};
|
||||
config.SetTRTDynamicShapeInfo(input_shape, input_shape, input_shape, false);
|
||||
LOG(INFO) << config.Summary();
|
||||
// get ground truth by disable ir
|
||||
paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool_no_ir.Retrieve(0), &input_data_map, &truth_output_data, 1);
|
||||
|
||||
// get infer results from multi threads
|
||||
std::vector<std::thread> threads;
|
||||
services::PredictorPool pred_pool(config, thread_num);
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
threads.emplace_back(paddle::test::SingleThreadPrediction,
|
||||
pred_pool.Retrieve(i),
|
||||
&input_data_map,
|
||||
&infer_output_data,
|
||||
2);
|
||||
}
|
||||
|
||||
// thread join & check outputs
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
LOG(INFO) << "join tid : " << i;
|
||||
threads[i].join();
|
||||
// CompareRecord(&truth_output_data, &infer_output_data, 1e-2);
|
||||
// TODO(OliverLPH): disable comparison since precision is low
|
||||
}
|
||||
|
||||
std::cout << "finish multi-thread test" << std::endl;
|
||||
}
|
||||
|
||||
// fused_softplus is about to be removed, the test uses fused_softplus and is
|
||||
// disabled
|
||||
/*
|
||||
TEST(onednn_tester_ppyolov2_r50vd, multi_thread2_mkl_bz2) {
|
||||
int thread_num = 2;
|
||||
// init input data
|
||||
auto input_data_map = PrepareInput(2);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data,
|
||||
truth_output_data;
|
||||
// prepare ground truth config
|
||||
paddle_infer::Config config, config_no_ir;
|
||||
config_no_ir.SetModel(FLAGS_modeldir + "/model.pdmodel",
|
||||
FLAGS_modeldir + "/model.pdiparams");
|
||||
config_no_ir.DisableGpu();
|
||||
config_no_ir.SwitchIrOptim(false);
|
||||
// prepare inference config
|
||||
config.SetModel(FLAGS_modeldir + "/model.pdmodel",
|
||||
FLAGS_modeldir + "/model.pdiparams");
|
||||
config.DisableGpu();
|
||||
config.EnableONEDNN();
|
||||
config.SetOnednnCacheCapacity(10);
|
||||
config.SetCpuMathLibraryNumThreads(10);
|
||||
LOG(INFO) << config.Summary();
|
||||
// get ground truth by disable ir
|
||||
paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool_no_ir.Retrieve(0), &input_data_map, &truth_output_data, 1);
|
||||
|
||||
// get infer results from multi threads
|
||||
std::vector<std::thread> threads;
|
||||
services::PredictorPool pred_pool(config, thread_num);
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
threads.emplace_back(paddle::test::SingleThreadPrediction,
|
||||
pred_pool.Retrieve(i),
|
||||
&input_data_map,
|
||||
&infer_output_data,
|
||||
2);
|
||||
}
|
||||
|
||||
// thread join & check outputs
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
LOG(INFO) << "join tid : " << i;
|
||||
threads[i].join();
|
||||
// CompareRecord(&truth_output_data, &infer_output_data, 1e-4);
|
||||
// TODO(OliverLPH): disable comparison since precision is low
|
||||
}
|
||||
|
||||
std::cout << "finish multi-thread test" << std::endl;
|
||||
}
|
||||
*/
|
||||
|
||||
} // namespace paddle_infer
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
gflags::ParseCommandLineFlags(&argc, &argv, true);
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "test_suite.h" // NOLINT
|
||||
|
||||
DEFINE_string(modeldir, "", "Directory of the inference model.");
|
||||
|
||||
namespace paddle_infer {
|
||||
|
||||
paddle::test::Record PrepareInput(int batch_size) {
|
||||
// init input data
|
||||
int channel = 3;
|
||||
int width = 224;
|
||||
int height = 224;
|
||||
paddle::test::Record image_Record;
|
||||
int input_num = batch_size * channel * width * height;
|
||||
std::vector<float> input_data(input_num, 1);
|
||||
image_Record.data = input_data;
|
||||
image_Record.shape = std::vector<int>{batch_size, channel, width, height};
|
||||
image_Record.type = paddle::PaddleDType::FLOAT32;
|
||||
return image_Record;
|
||||
}
|
||||
|
||||
TEST(gpu_tester_resnet50, analysis_gpu_bz1) {
|
||||
// init input data
|
||||
std::map<std::string, paddle::test::Record> my_input_data_map;
|
||||
my_input_data_map["inputs"] = PrepareInput(1);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data,
|
||||
truth_output_data;
|
||||
// prepare ground truth config
|
||||
paddle_infer::Config config, config_no_ir;
|
||||
config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config_no_ir.SwitchIrOptim(false);
|
||||
// prepare inference config
|
||||
config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
// get ground truth by disable ir
|
||||
paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
|
||||
// get infer results
|
||||
paddle_infer::services::PredictorPool pred_pool(config, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool.Retrieve(0), &my_input_data_map, &infer_output_data);
|
||||
// check outputs
|
||||
CompareRecord(&truth_output_data, &infer_output_data);
|
||||
std::cout << "finish test" << std::endl;
|
||||
}
|
||||
|
||||
TEST(tensorrt_tester_resnet50, trt_fp32_bz2) {
|
||||
// init input data
|
||||
std::map<std::string, paddle::test::Record> my_input_data_map;
|
||||
my_input_data_map["inputs"] = PrepareInput(2);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data,
|
||||
truth_output_data;
|
||||
// prepare ground truth config
|
||||
paddle_infer::Config config, config_no_ir;
|
||||
config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config_no_ir.SwitchIrOptim(false);
|
||||
// prepare inference config
|
||||
config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config.EnableUseGpu(100, 0);
|
||||
config.EnableTensorRtEngine(
|
||||
1 << 20, 2, 3, paddle_infer::PrecisionType::kFloat32, false, false);
|
||||
// get ground truth by disable ir
|
||||
paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
|
||||
// get infer results
|
||||
paddle_infer::services::PredictorPool pred_pool(config, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool.Retrieve(0), &my_input_data_map, &infer_output_data);
|
||||
// check outputs
|
||||
CompareRecord(&truth_output_data, &infer_output_data, 2e-4);
|
||||
std::cout << "finish test" << std::endl;
|
||||
}
|
||||
|
||||
TEST(tensorrt_tester_resnet50, serial_diff_batch_trt_fp32) {
|
||||
int max_batch_size = 5;
|
||||
// prepare ground truth config
|
||||
paddle_infer::Config config, config_no_ir;
|
||||
config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config_no_ir.SwitchIrOptim(false);
|
||||
paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
|
||||
// prepare inference config
|
||||
config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config.EnableUseGpu(100, 0);
|
||||
config.EnableTensorRtEngine(1 << 20,
|
||||
max_batch_size,
|
||||
3,
|
||||
paddle_infer::PrecisionType::kFloat32,
|
||||
false,
|
||||
false);
|
||||
paddle_infer::services::PredictorPool pred_pool(config, 1);
|
||||
|
||||
for (int i = 1; i < max_batch_size; i++) {
|
||||
// init input data
|
||||
std::map<std::string, paddle::test::Record> my_input_data_map;
|
||||
my_input_data_map["inputs"] = PrepareInput(i);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data,
|
||||
truth_output_data;
|
||||
// get ground truth by disable ir
|
||||
SingleThreadPrediction(
|
||||
pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
|
||||
// get infer results
|
||||
SingleThreadPrediction(
|
||||
pred_pool.Retrieve(0), &my_input_data_map, &infer_output_data);
|
||||
// check outputs
|
||||
CompareRecord(&truth_output_data, &infer_output_data, 1e-4);
|
||||
}
|
||||
std::cout << "finish test" << std::endl;
|
||||
}
|
||||
|
||||
TEST(tensorrt_tester_resnet50, multi_thread4_trt_fp32_bz2) {
|
||||
int thread_num = 4;
|
||||
// init input data
|
||||
std::map<std::string, paddle::test::Record> my_input_data_map;
|
||||
my_input_data_map["inputs"] = PrepareInput(2);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data,
|
||||
truth_output_data;
|
||||
// prepare ground truth config
|
||||
paddle_infer::Config config, config_no_ir;
|
||||
config_no_ir.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config_no_ir.SwitchIrOptim(false);
|
||||
// prepare inference config
|
||||
config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config.EnableUseGpu(100, 0);
|
||||
config.EnableTensorRtEngine(
|
||||
1 << 20, 2, 3, paddle_infer::PrecisionType::kFloat32, false, false);
|
||||
// get ground truth by disable ir
|
||||
paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool_no_ir.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
|
||||
|
||||
// get infer results from multi threads
|
||||
std::vector<std::thread> threads;
|
||||
services::PredictorPool pred_pool(config, thread_num);
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
threads.emplace_back(paddle::test::SingleThreadPrediction,
|
||||
pred_pool.Retrieve(i),
|
||||
&my_input_data_map,
|
||||
&infer_output_data,
|
||||
2);
|
||||
}
|
||||
|
||||
// thread join & check outputs
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
LOG(INFO) << "join tid : " << i;
|
||||
threads[i].join();
|
||||
CompareRecord(&truth_output_data, &infer_output_data, 2e-4);
|
||||
}
|
||||
|
||||
std::cout << "finish multi-thread test" << std::endl;
|
||||
}
|
||||
|
||||
TEST(tensorrt_tester_resnet50, trt_int8_bz2) {
|
||||
// init input data
|
||||
std::map<std::string, paddle::test::Record> my_input_data_map;
|
||||
my_input_data_map["inputs"] = PrepareInput(2);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data,
|
||||
truth_output_data;
|
||||
// prepare inference config
|
||||
paddle_infer::Config config;
|
||||
config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config.EnableUseGpu(100, 0);
|
||||
config.EnableTensorRtEngine(
|
||||
1 << 20, 2, 3, paddle_infer::PrecisionType::kInt8, true, true);
|
||||
|
||||
// get first time prediction int8 results
|
||||
paddle_infer::services::PredictorPool pred_pool(config, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool.Retrieve(0), &my_input_data_map, &truth_output_data, 1);
|
||||
|
||||
// get repeat 5 times prediction int8 results
|
||||
SingleThreadPrediction(
|
||||
pred_pool.Retrieve(0), &my_input_data_map, &infer_output_data, 5);
|
||||
|
||||
// check outputs
|
||||
CompareRecord(&truth_output_data, &infer_output_data);
|
||||
std::cout << "finish test" << std::endl;
|
||||
}
|
||||
|
||||
TEST(DISABLED_tensorrt_tester_resnet50, profile_multi_thread_trt_fp32) {
|
||||
int batch_size = 2;
|
||||
int thread_num = 4;
|
||||
int repeat_time = 1000;
|
||||
// init input data
|
||||
std::map<std::string, paddle::test::Record> my_input_data_map;
|
||||
my_input_data_map["inputs"] = PrepareInput(batch_size);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data;
|
||||
// prepare inference config
|
||||
paddle_infer::Config config;
|
||||
config.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config.EnableUseGpu(100, 0);
|
||||
config.EnableTensorRtEngine(
|
||||
1 << 20, 2, 3, paddle_infer::PrecisionType::kFloat32, false, false);
|
||||
// get infer results from multi threads
|
||||
services::PredictorPool pred_pool(config, thread_num);
|
||||
std::vector<std::future<double>> calcs;
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
calcs.push_back(std::async(&paddle::test::SingleThreadProfile,
|
||||
pred_pool.Retrieve(i),
|
||||
&my_input_data_map,
|
||||
repeat_time));
|
||||
}
|
||||
double total_time_ = 0.0;
|
||||
for (auto&& fut : calcs) {
|
||||
total_time_ += fut.get();
|
||||
}
|
||||
std::cout << total_time_ << std::endl;
|
||||
|
||||
std::cout << "finish multi-thread profile" << std::endl;
|
||||
}
|
||||
|
||||
} // namespace paddle_infer
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
gflags::ParseCommandLineFlags(&argc, &argv, true);
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "test_suite.h" // NOLINT
|
||||
|
||||
DEFINE_string(modeldir, "", "Directory of the inference model.");
|
||||
DEFINE_string(int8dir, "", "Directory of the quant inference model.");
|
||||
DEFINE_string(datadir, "", "Directory of the infer data.");
|
||||
|
||||
namespace paddle_infer {
|
||||
|
||||
paddle::test::Record PrepareInput(int batch_size) {
|
||||
// init input data
|
||||
int channel = 3;
|
||||
int width = 224;
|
||||
int height = 224;
|
||||
paddle::test::Record image_Record;
|
||||
int input_num = batch_size * channel * width * height;
|
||||
|
||||
// load from binary data
|
||||
std::ifstream fs(FLAGS_datadir, std::ifstream::binary);
|
||||
EXPECT_TRUE(fs.is_open());
|
||||
CHECK(fs.is_open());
|
||||
|
||||
float* input = new float[input_num];
|
||||
memset(input, 0, input_num * sizeof(float));
|
||||
auto input_data_tmp = input;
|
||||
for (int i = 0; i < input_num; ++i) {
|
||||
fs.read(reinterpret_cast<char*>(input_data_tmp), sizeof(*input_data_tmp));
|
||||
input_data_tmp++;
|
||||
}
|
||||
int label = 0;
|
||||
fs.read(reinterpret_cast<char*>(&label), sizeof(label));
|
||||
fs.close();
|
||||
|
||||
std::vector<float> input_data{input, input + input_num};
|
||||
image_Record.data = input_data;
|
||||
image_Record.shape = std::vector<int>{batch_size, channel, width, height};
|
||||
image_Record.type = paddle::PaddleDType::FLOAT32;
|
||||
image_Record.label = label;
|
||||
return image_Record;
|
||||
}
|
||||
|
||||
TEST(DISABLED_tensorrt_tester_resnet50_quant, multi_thread4_trt_int8_bz1) {
|
||||
int thread_num = 4;
|
||||
// init input data
|
||||
std::map<std::string, paddle::test::Record> input_data_map;
|
||||
input_data_map["image"] = PrepareInput(1);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data;
|
||||
// prepare inference config
|
||||
paddle_infer::Config config;
|
||||
config.SetModel(FLAGS_int8dir);
|
||||
config.EnableUseGpu(1000, 0);
|
||||
config.EnableTensorRtEngine(
|
||||
1 << 20, 10, 3, paddle_infer::PrecisionType::kInt8, false, false);
|
||||
// get infer results from multi threads
|
||||
std::vector<std::thread> threads;
|
||||
services::PredictorPool pred_pool(config, thread_num);
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
threads.emplace_back(paddle::test::SingleThreadPrediction,
|
||||
pred_pool.Retrieve(i),
|
||||
&input_data_map,
|
||||
&infer_output_data,
|
||||
5);
|
||||
}
|
||||
|
||||
// thread join & check outputs
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
LOG(INFO) << "join tid : " << i;
|
||||
threads[i].join();
|
||||
|
||||
// check outputs
|
||||
std::vector<int> index(1000);
|
||||
std::iota(index.begin(), index.end(), 0);
|
||||
auto out_data =
|
||||
infer_output_data["save_infer_model/scale_0.tmp_0"].data.data();
|
||||
std::sort(index.begin(), index.end(), [out_data](size_t i1, size_t i2) {
|
||||
return out_data[i1] > out_data[i2];
|
||||
});
|
||||
// compare inference & ground truth label
|
||||
ASSERT_EQ(index[0], input_data_map["image"].label);
|
||||
}
|
||||
|
||||
std::cout << "finish test" << std::endl;
|
||||
}
|
||||
|
||||
TEST(DISABLED_tensorrt_tester_resnet50_quant, multi_thread_multi_instance) {
|
||||
int thread_num = 4;
|
||||
// init input data
|
||||
std::map<std::string, paddle::test::Record> input_data_fp32, input_data_quant;
|
||||
input_data_quant["image"] = PrepareInput(1);
|
||||
input_data_fp32["inputs"] = PrepareInput(1);
|
||||
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data;
|
||||
// prepare inference config
|
||||
paddle_infer::Config config_fp32, config_quant;
|
||||
config_fp32.SetModel(FLAGS_modeldir + "/inference.pdmodel",
|
||||
FLAGS_modeldir + "/inference.pdiparams");
|
||||
config_fp32.EnableUseGpu(1000, 0);
|
||||
config_fp32.EnableTensorRtEngine(
|
||||
1 << 20, 10, 3, paddle_infer::PrecisionType::kFloat32, false, false);
|
||||
|
||||
config_quant.SetModel(FLAGS_int8dir);
|
||||
config_quant.EnableUseGpu(1000, 0);
|
||||
config_quant.EnableTensorRtEngine(
|
||||
1 << 20, 10, 3, paddle_infer::PrecisionType::kInt8, false, false);
|
||||
|
||||
// get infer results from multi threads
|
||||
std::vector<std::thread> threads;
|
||||
services::PredictorPool pred_pool_fp32(config_fp32, thread_num);
|
||||
services::PredictorPool pred_pool_quant(config_quant, thread_num);
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
if (i % 2 == 0) {
|
||||
threads.emplace_back(paddle::test::SingleThreadPrediction,
|
||||
pred_pool_fp32.Retrieve(i),
|
||||
&input_data_fp32,
|
||||
&infer_output_data,
|
||||
5);
|
||||
} else {
|
||||
threads.emplace_back(paddle::test::SingleThreadPrediction,
|
||||
pred_pool_quant.Retrieve(i),
|
||||
&input_data_quant,
|
||||
&infer_output_data,
|
||||
5);
|
||||
}
|
||||
}
|
||||
|
||||
// thread join & check outputs
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
LOG(INFO) << "join tid : " << i;
|
||||
std::vector<int> index(1000);
|
||||
threads[i].join();
|
||||
if (i % 2 == 0) {
|
||||
// check outputs
|
||||
std::iota(index.begin(), index.end(), 0);
|
||||
auto out_data =
|
||||
infer_output_data["save_infer_model/scale_0.tmp_0"].data.data();
|
||||
std::sort(index.begin(), index.end(), [out_data](size_t i1, size_t i2) {
|
||||
return out_data[i1] > out_data[i2];
|
||||
});
|
||||
// compare inference & ground truth label
|
||||
ASSERT_EQ(index[0], input_data_fp32["inputs"].label);
|
||||
} else {
|
||||
// check outputs
|
||||
std::iota(index.begin(), index.end(), 0);
|
||||
auto out_data =
|
||||
infer_output_data["save_infer_model/scale_0.tmp_0"].data.data();
|
||||
std::sort(index.begin(), index.end(), [out_data](size_t i1, size_t i2) {
|
||||
return out_data[i1] > out_data[i2];
|
||||
});
|
||||
// compare inference & ground truth label
|
||||
ASSERT_EQ(index[0], input_data_quant["image"].label);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace paddle_infer
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
gflags::ParseCommandLineFlags(&argc, &argv, true);
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
#include <math.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <deque>
|
||||
#include <fstream>
|
||||
#include <future>
|
||||
#include <iostream>
|
||||
#include <numeric>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#include "gflags/gflags.h"
|
||||
#include "glog/logging.h"
|
||||
#include "gtest/gtest.h"
|
||||
#include "paddle/include/paddle_inference_api.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace test {
|
||||
|
||||
#define IS_TRT_VERSION_GE(version) \
|
||||
((NV_TENSORRT_MAJOR * 1000 + NV_TENSORRT_MINOR * 100 + \
|
||||
NV_TENSORRT_PATCH * 10 + NV_TENSORRT_BUILD) >= version)
|
||||
|
||||
#define IS_TRT_VERSION_LT(version) \
|
||||
((NV_TENSORRT_MAJOR * 1000 + NV_TENSORRT_MINOR * 100 + \
|
||||
NV_TENSORRT_PATCH * 10 + NV_TENSORRT_BUILD) < version)
|
||||
|
||||
#define TRT_VERSION \
|
||||
NV_TENSORRT_MAJOR * 1000 + NV_TENSORRT_MINOR * 100 + \
|
||||
NV_TENSORRT_PATCH * 10 + NV_TENSORRT_BUILD
|
||||
|
||||
class Record {
|
||||
public:
|
||||
std::vector<float> data;
|
||||
std::vector<int32_t> shape;
|
||||
paddle::PaddleDType type;
|
||||
int label;
|
||||
};
|
||||
|
||||
std::string read_file(std::string filename) {
|
||||
std::ifstream file(filename);
|
||||
return std::string((std::istreambuf_iterator<char>(file)),
|
||||
std::istreambuf_iterator<char>());
|
||||
}
|
||||
|
||||
void SingleThreadPrediction(paddle_infer::Predictor *predictor,
|
||||
std::map<std::string, Record> *input_data_map,
|
||||
std::map<std::string, Record> *output_data_map,
|
||||
int repeat_times = 2) {
|
||||
// prepare input tensor
|
||||
auto input_names = predictor->GetInputNames();
|
||||
for (const auto &[key, value] : *input_data_map) {
|
||||
switch (value.type) {
|
||||
case paddle::PaddleDType::INT64: {
|
||||
std::vector<int64_t> input_value =
|
||||
std::vector<int64_t>(value.data.begin(), value.data.end());
|
||||
auto input_tensor = predictor->GetInputHandle(key);
|
||||
input_tensor->Reshape(value.shape);
|
||||
input_tensor->CopyFromCpu(input_value.data());
|
||||
break;
|
||||
}
|
||||
case paddle::PaddleDType::INT32: {
|
||||
std::vector<int32_t> input_value =
|
||||
std::vector<int32_t>(value.data.begin(), value.data.end());
|
||||
auto input_tensor = predictor->GetInputHandle(key);
|
||||
input_tensor->Reshape(value.shape);
|
||||
input_tensor->CopyFromCpu(input_value.data());
|
||||
break;
|
||||
}
|
||||
case paddle::PaddleDType::FLOAT32: {
|
||||
std::vector<float> input_value =
|
||||
std::vector<float>(value.data.begin(), value.data.end());
|
||||
auto input_tensor = predictor->GetInputHandle(key);
|
||||
input_tensor->Reshape(value.shape);
|
||||
input_tensor->CopyFromCpu(input_value.data());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// inference
|
||||
for (size_t i = 0; i < repeat_times; ++i) {
|
||||
ASSERT_TRUE(predictor->Run());
|
||||
}
|
||||
|
||||
// get output data to Record
|
||||
auto output_names = predictor->GetOutputNames();
|
||||
for (auto &output_name : output_names) {
|
||||
Record output_Record;
|
||||
auto output_tensor = predictor->GetOutputHandle(output_name);
|
||||
std::vector<int> output_shape = output_tensor->shape();
|
||||
int out_num = std::accumulate(
|
||||
output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
|
||||
|
||||
switch (output_tensor->type()) {
|
||||
case paddle::PaddleDType::INT64: {
|
||||
VLOG(1) << "output_tensor dtype: int64";
|
||||
std::vector<int64_t> out_data;
|
||||
output_Record.type = paddle::PaddleDType::INT64;
|
||||
out_data.resize(out_num);
|
||||
output_tensor->CopyToCpu(out_data.data());
|
||||
output_Record.shape = output_shape;
|
||||
std::vector<float> floatVec(out_data.begin(), out_data.end());
|
||||
output_Record.data = floatVec;
|
||||
(*output_data_map)[output_name] = output_Record;
|
||||
break;
|
||||
}
|
||||
case paddle::PaddleDType::FLOAT32: {
|
||||
VLOG(1) << "output_tensor dtype: float32";
|
||||
std::vector<float> out_data;
|
||||
output_Record.type = paddle::PaddleDType::FLOAT32;
|
||||
out_data.resize(out_num);
|
||||
output_tensor->CopyToCpu(out_data.data());
|
||||
output_Record.shape = output_shape;
|
||||
output_Record.data = out_data;
|
||||
(*output_data_map)[output_name] = output_Record;
|
||||
break;
|
||||
}
|
||||
case paddle::PaddleDType::INT32: {
|
||||
VLOG(1) << "output_tensor dtype: int32";
|
||||
std::vector<int32_t> out_data;
|
||||
output_Record.type = paddle::PaddleDType::INT32;
|
||||
out_data.resize(out_num);
|
||||
output_tensor->CopyToCpu(out_data.data());
|
||||
output_Record.shape = output_shape;
|
||||
std::vector<float> floatVec(out_data.begin(), out_data.end());
|
||||
output_Record.data = floatVec;
|
||||
(*output_data_map)[output_name] = output_Record;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void CompareRecord(std::map<std::string, Record> *truth_output_data,
|
||||
std::map<std::string, Record> *infer_output_data,
|
||||
float epsilon = 1e-5) {
|
||||
for (const auto &[key, value] : *infer_output_data) {
|
||||
auto truth_record = (*truth_output_data)[key];
|
||||
VLOG(1) << "output name: " << key;
|
||||
size_t numel = value.data.size() / sizeof(float);
|
||||
EXPECT_EQ(value.data.size(), truth_record.data.size());
|
||||
for (size_t i = 0; i < numel; ++i) {
|
||||
VLOG(1) << "compare: " << value.data.data()[i] << ",\t"
|
||||
<< truth_record.data.data()[i];
|
||||
ASSERT_LT(fabs(value.data.data()[i] - truth_record.data.data()[i]),
|
||||
epsilon);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Timer, count in ms
|
||||
class Timer {
|
||||
public:
|
||||
Timer() { reset(); }
|
||||
void start() { start_t = std::chrono::high_resolution_clock::now(); }
|
||||
void stop() {
|
||||
auto end_t = std::chrono::high_resolution_clock::now();
|
||||
typedef std::chrono::microseconds ms;
|
||||
auto diff = end_t - start_t;
|
||||
ms counter = std::chrono::duration_cast<ms>(diff);
|
||||
total_time += counter.count();
|
||||
}
|
||||
void reset() { total_time = 0.; }
|
||||
double report() { return total_time / 1000.0; }
|
||||
|
||||
private:
|
||||
double total_time;
|
||||
std::chrono::high_resolution_clock::time_point start_t;
|
||||
};
|
||||
|
||||
// single thread inference benchmark, return double time in ms
|
||||
double SingleThreadProfile(paddle_infer::Predictor *predictor,
|
||||
std::map<std::string, Record> *input_data_map,
|
||||
int repeat_times = 2) {
|
||||
// prepare input tensor
|
||||
auto input_names = predictor->GetInputNames();
|
||||
for (const auto &[key, value] : *input_data_map) {
|
||||
switch (value.type) {
|
||||
case paddle::PaddleDType::INT64: {
|
||||
std::vector<int64_t> input_value =
|
||||
std::vector<int64_t>(value.data.begin(), value.data.end());
|
||||
auto input_tensor = predictor->GetInputHandle(key);
|
||||
input_tensor->Reshape(value.shape);
|
||||
input_tensor->CopyFromCpu(input_value.data());
|
||||
break;
|
||||
}
|
||||
case paddle::PaddleDType::INT32: {
|
||||
std::vector<int32_t> input_value =
|
||||
std::vector<int32_t>(value.data.begin(), value.data.end());
|
||||
auto input_tensor = predictor->GetInputHandle(key);
|
||||
input_tensor->Reshape(value.shape);
|
||||
input_tensor->CopyFromCpu(input_value.data());
|
||||
break;
|
||||
}
|
||||
case paddle::PaddleDType::FLOAT32: {
|
||||
std::vector<float> input_value =
|
||||
std::vector<float>(value.data.begin(), value.data.end());
|
||||
auto input_tensor = predictor->GetInputHandle(key);
|
||||
input_tensor->Reshape(value.shape);
|
||||
input_tensor->CopyFromCpu(input_value.data());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Timer timer; // init prediction timer
|
||||
timer.start();
|
||||
// inference
|
||||
for (size_t i = 0; i < repeat_times; ++i) {
|
||||
CHECK(predictor->Run());
|
||||
auto output_names = predictor->GetOutputNames();
|
||||
for (auto &output_name : output_names) {
|
||||
auto output_tensor = predictor->GetOutputHandle(output_name);
|
||||
std::vector<int> output_shape = output_tensor->shape();
|
||||
int out_num = std::accumulate(
|
||||
output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
|
||||
switch (output_tensor->type()) {
|
||||
case paddle::PaddleDType::INT64: {
|
||||
std::vector<int64_t> out_data;
|
||||
out_data.resize(out_num);
|
||||
output_tensor->CopyToCpu(out_data.data());
|
||||
break;
|
||||
}
|
||||
case paddle::PaddleDType::FLOAT32: {
|
||||
std::vector<float> out_data;
|
||||
out_data.resize(out_num);
|
||||
output_tensor->CopyToCpu(out_data.data());
|
||||
break;
|
||||
}
|
||||
case paddle::PaddleDType::INT32: {
|
||||
std::vector<int32_t> out_data;
|
||||
out_data.resize(out_num);
|
||||
output_tensor->CopyToCpu(out_data.data());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
timer.stop();
|
||||
return timer.report();
|
||||
}
|
||||
|
||||
} // namespace test
|
||||
} // namespace paddle
|
||||
@@ -0,0 +1,159 @@
|
||||
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "test_suite.h" // NOLINT
|
||||
|
||||
DEFINE_string(modeldir, "", "Directory of the inference model.");
|
||||
|
||||
namespace paddle_infer {
|
||||
|
||||
std::map<std::string, paddle::test::Record> PrepareInput(int batch_size) {
|
||||
// init input data
|
||||
int channel = 3;
|
||||
int width = 608;
|
||||
int height = 608;
|
||||
paddle::test::Record image, im_shape, scale_factor;
|
||||
int input_num = batch_size * channel * width * height;
|
||||
int shape_num = batch_size * 2;
|
||||
std::vector<float> image_data(input_num, 1);
|
||||
for (int i = 1; i < input_num + 1; ++i) {
|
||||
image_data[i] = i % 10 * 0.5;
|
||||
}
|
||||
std::vector<float> im_shape_data(shape_num, 1);
|
||||
std::vector<float> scale_factor_data(shape_num, 1);
|
||||
|
||||
image.data = std::vector<float>(image_data.begin(), image_data.end());
|
||||
image.shape = std::vector<int>{batch_size, channel, width, height};
|
||||
image.type = paddle::PaddleDType::FLOAT32;
|
||||
|
||||
im_shape.data =
|
||||
std::vector<float>(im_shape_data.begin(), im_shape_data.end());
|
||||
im_shape.shape = std::vector<int>{batch_size, 2};
|
||||
im_shape.type = paddle::PaddleDType::FLOAT32;
|
||||
|
||||
scale_factor.data =
|
||||
std::vector<float>(scale_factor_data.begin(), scale_factor_data.end());
|
||||
scale_factor.shape = std::vector<int>{batch_size, 2};
|
||||
scale_factor.type = paddle::PaddleDType::FLOAT32;
|
||||
|
||||
std::map<std::string, paddle::test::Record> input_data_map;
|
||||
input_data_map.insert({"image", image});
|
||||
input_data_map.insert({"im_shape", im_shape});
|
||||
input_data_map.insert({"scale_factor", scale_factor});
|
||||
|
||||
return input_data_map;
|
||||
}
|
||||
|
||||
TEST(test_yolov3, multi_thread3_trt_fp32_bz2) {
|
||||
int thread_num = 3;
|
||||
// init input data
|
||||
auto input_data_map = PrepareInput(2);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data,
|
||||
truth_output_data;
|
||||
// prepare ground truth config
|
||||
paddle_infer::Config config, config_no_ir;
|
||||
config_no_ir.SetModel(FLAGS_modeldir + "/model.pdmodel",
|
||||
FLAGS_modeldir + "/model.pdiparams");
|
||||
config_no_ir.EnableUseGpu(100, 0);
|
||||
config_no_ir.SwitchIrOptim(false);
|
||||
// prepare inference config
|
||||
config.SetModel(FLAGS_modeldir + "/model.pdmodel",
|
||||
FLAGS_modeldir + "/model.pdiparams");
|
||||
config.EnableUseGpu(100, 0);
|
||||
config.EnableTensorRtEngine(
|
||||
1 << 20, 2, 3, paddle_infer::PrecisionType::kFloat32, false, false);
|
||||
LOG(INFO) << config.Summary();
|
||||
// get ground truth by disable ir
|
||||
paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool_no_ir.Retrieve(0), &input_data_map, &truth_output_data, 1);
|
||||
|
||||
// get infer results from multi threads
|
||||
std::vector<std::thread> threads;
|
||||
services::PredictorPool pred_pool(config, thread_num);
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
threads.emplace_back(paddle::test::SingleThreadPrediction,
|
||||
pred_pool.Retrieve(i),
|
||||
&input_data_map,
|
||||
&infer_output_data,
|
||||
2);
|
||||
}
|
||||
|
||||
// thread join & check outputs
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
LOG(INFO) << "join tid : " << i;
|
||||
threads[i].join();
|
||||
CompareRecord(&truth_output_data, &infer_output_data, 1e-2);
|
||||
// TODO(OliverLPH): precision set to 1e-2 since input is fake, change to
|
||||
// real input later
|
||||
}
|
||||
|
||||
std::cout << "finish multi-thread test" << std::endl;
|
||||
}
|
||||
|
||||
TEST(test_yolov3, multi_thread4_mkl_bz2) {
|
||||
int thread_num = 4;
|
||||
// init input data
|
||||
auto input_data_map = PrepareInput(2);
|
||||
// init output data
|
||||
std::map<std::string, paddle::test::Record> infer_output_data,
|
||||
truth_output_data;
|
||||
// prepare ground truth config
|
||||
paddle_infer::Config config, config_no_ir;
|
||||
config_no_ir.SetModel(FLAGS_modeldir + "/model.pdmodel",
|
||||
FLAGS_modeldir + "/model.pdiparams");
|
||||
config_no_ir.DisableGpu();
|
||||
config_no_ir.SwitchIrOptim(false);
|
||||
// prepare inference config
|
||||
config.SetModel(FLAGS_modeldir + "/model.pdmodel",
|
||||
FLAGS_modeldir + "/model.pdiparams");
|
||||
config.DisableGpu();
|
||||
config.EnableONEDNN();
|
||||
config.SetOnednnCacheCapacity(10);
|
||||
config.SetCpuMathLibraryNumThreads(10);
|
||||
LOG(INFO) << config.Summary();
|
||||
// get ground truth by disable ir
|
||||
paddle_infer::services::PredictorPool pred_pool_no_ir(config_no_ir, 1);
|
||||
SingleThreadPrediction(
|
||||
pred_pool_no_ir.Retrieve(0), &input_data_map, &truth_output_data, 1);
|
||||
|
||||
// get infer results from multi threads
|
||||
std::vector<std::thread> threads;
|
||||
services::PredictorPool pred_pool(config, thread_num);
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
threads.emplace_back(paddle::test::SingleThreadPrediction,
|
||||
pred_pool.Retrieve(i),
|
||||
&input_data_map,
|
||||
&infer_output_data,
|
||||
2);
|
||||
}
|
||||
|
||||
// thread join & check outputs
|
||||
for (int i = 0; i < thread_num; ++i) {
|
||||
LOG(INFO) << "join tid : " << i;
|
||||
threads[i].join();
|
||||
CompareRecord(&truth_output_data, &infer_output_data, 1e-4);
|
||||
}
|
||||
|
||||
std::cout << "finish multi-thread test" << std::endl;
|
||||
}
|
||||
|
||||
} // namespace paddle_infer
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
gflags::ParseCommandLineFlags(&argc, &argv, true);
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
Reference in New Issue
Block a user