wehub/paddlepaddle--paddle
1
0
Fork 0
Code Issues Pull Requests Actions 2 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:40:42 +08:00
commit e25996e7db
15472 changed files with 3536181 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/cpp/inference/api/CMakeLists.txt
+1183
View File
File diff suppressed because it is too large Load Diff
test/cpp/inference/api/analyzer_bfloat16_image_classification_tester.cc
+60
View File
@@ -0,0 +1,60 @@
/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <fstream>
#include <iostream>
#include "paddle/fluid/inference/api/paddle_analysis_config.h"
#include "paddle/phi/backends/cpu/cpu_info.h"
#include "test/cpp/inference/api/tester_helper.h"
PD_DEFINE_bool(enable_onednn, true, "Enable ONEDNN");
namespace paddle {
namespace inference {
namespace analysis {
void SetConfig(AnalysisConfig *cfg) {
std::ifstream model_file(FLAGS_infer_model + "/__model__");
if (model_file.good())
cfg->SetModel(FLAGS_infer_model);
else
cfg->SetModel(FLAGS_infer_model + "/inference.pdmodel",
FLAGS_infer_model + "/inference.pdiparams");
cfg->DisableGpu();
cfg->SwitchIrOptim();
cfg->SwitchSpecifyInputNames();
cfg->SetCpuMathLibraryNumThreads(FLAGS_num_threads);
if (!FLAGS_enable_onednn) cfg->DisableONEDNN();
}
TEST(Analyzer_bfloat16_image_classification, bfloat16) {
AnalysisConfig cfg;
SetConfig(&cfg);
AnalysisConfig b_cfg;
SetConfig(&b_cfg);
// read data from file and prepare batches with test data
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInputs(&input_slots_all);
if (FLAGS_enable_onednn && FLAGS_enable_bf16 &&
phi::backends::cpu::MayIUse(phi::backends::cpu::cpu_isa_t::avx512_bf16)) {
b_cfg.EnableOnednnBfloat16();
} else {
FLAGS_enable_bf16 = false;
}
CompareBFloat16AndAnalysis(&cfg, &b_cfg, input_slots_all);
}
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_capi_exp_gpu_tester.cc
+183
View File
@@ -0,0 +1,183 @@
/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <string>
#include <vector>
#if defined(PADDLE_WITH_CUDA)
#include <cuda_runtime.h>
#endif
#include "paddle/common/flags.h"
#include "paddle/fluid/inference/capi_exp/pd_inference_api.h"
PD_DEFINE_string(infer_model, "", "model path");
namespace paddle {
namespace inference {
namespace analysis {
TEST(PD_Config, gpu_interface) {
std::string model_dir = FLAGS_infer_model + "/mobilenet";
std::string prog_file = model_dir + "/__model__";
std::string param_file = model_dir + "/__params__";
std::string opt_cache_dir = FLAGS_infer_model + "/OptimCacheDir";
const char* ops_name = "conv_2d";
PD_Config* config = PD_ConfigCreate();
PD_ConfigSetModel(config, prog_file.c_str(), param_file.c_str());
PD_ConfigSetOptimCacheDir(config, opt_cache_dir.c_str());
PD_ConfigEnableUseGpu(config, 100, 0, 0);
bool use_gpu = PD_ConfigUseGpu(config);
EXPECT_TRUE(use_gpu);
int init_size = PD_ConfigMemoryPoolInitSizeMb(config);
EXPECT_EQ(init_size, 100);
int gpu_device_id = PD_ConfigGpuDeviceId(config);
EXPECT_EQ(gpu_device_id, 0);
float frac = PD_ConfigFractionOfGpuMemoryForPool(config);
LOG(INFO) << frac;
PD_ConfigEnableCudnn(config);
bool cudnn = PD_ConfigCudnnEnabled(config);
EXPECT_TRUE(cudnn);
PD_ConfigEnableTensorRtEngine(
config, 1 << 20, 1, 3, PD_PRECISION_INT8, FALSE, TRUE);
bool trt_enable = PD_ConfigTensorRtEngineEnabled(config);
EXPECT_TRUE(trt_enable);
const char* tensor_name = "image";
std::array<size_t, 1> shapes_num = {4};
std::array<int32_t, 4> min_shape = {1, 3, 36, 36};
std::array<int32_t, 4> max_shape = {1, 3, 224, 224};
std::array<int32_t, 4> opt_shape = {1, 3, 224, 224};
int32_t* min_shape_ptr = min_shape.data();
int32_t* max_shape_ptr = max_shape.data();
int32_t* opt_shape_ptr = opt_shape.data();
PD_ConfigSetTrtDynamicShapeInfo(config,
1,
&tensor_name,
shapes_num.data(),
&min_shape_ptr,
&max_shape_ptr,
&opt_shape_ptr,
FALSE);
PD_ConfigDisableTensorRtOPs(config, 1, &ops_name);
PD_ConfigEnableVarseqlen(config);
bool oss_enabled = PD_ConfigTensorRtOssEnabled(config);
EXPECT_TRUE(oss_enabled);
PD_ConfigEnableTensorRtDla(config, 4);
bool dla_enabled = PD_ConfigTensorRtDlaEnabled(config);
EXPECT_TRUE(dla_enabled);
PD_ConfigEnableGpuMultiStream(config);
bool thread_local_thread = PD_ConfigThreadLocalStreamEnabled(config);
EXPECT_TRUE(thread_local_thread);
#if defined(PADDLE_WITH_CUDA)
{
cudaStream_t external_stream;
cudaStreamCreate(&external_stream);
PD_ConfigSetExecStream(config, external_stream);
}
#endif
PD_ConfigDisableGpu(config);
PD_ConfigDestroy(config);
}
TEST(PD_Config, use_gpu) {
std::string model_dir = FLAGS_infer_model + "/mobilenet";
PD_Config* config = PD_ConfigCreate();
PD_ConfigDisableGpu(config);
PD_ConfigSetCpuMathLibraryNumThreads(config, 10);
int num_thread = PD_ConfigGetCpuMathLibraryNumThreads(config);
EXPECT_EQ(num_thread, 10);
PD_ConfigSwitchIrDebug(config, TRUE);
PD_ConfigSetModelDir(config, model_dir.c_str());
PD_ConfigSetOptimCacheDir(config,
(FLAGS_infer_model + "/OptimCacheDir").c_str());
const char* model_dir_ = PD_ConfigGetModelDir(config);
LOG(INFO) << model_dir_;
PD_ConfigEnableUseGpu(config, 100, 0, 0);
bool use_gpu = PD_ConfigUseGpu(config);
EXPECT_TRUE(use_gpu);
int device_id = PD_ConfigGpuDeviceId(config);
EXPECT_EQ(device_id, 0);
int init_size = PD_ConfigMemoryPoolInitSizeMb(config);
EXPECT_EQ(init_size, 100);
float frac = PD_ConfigFractionOfGpuMemoryForPool(config);
LOG(INFO) << frac;
PD_ConfigEnableCudnn(config);
bool cudnn = PD_ConfigCudnnEnabled(config);
EXPECT_TRUE(cudnn);
PD_ConfigSwitchIrOptim(config, TRUE);
bool ir_optim = PD_ConfigIrOptim(config);
EXPECT_TRUE(ir_optim);
PD_ConfigEnableTensorRtEngine(
config, 1 << 20, 1, 3, PD_PRECISION_FLOAT32, FALSE, FALSE);
bool trt_enable = PD_ConfigTensorRtEngineEnabled(config);
EXPECT_TRUE(trt_enable);
PD_ConfigEnableMemoryOptim(config, true);
bool memory_optim_enable = PD_ConfigMemoryOptimEnabled(config);
EXPECT_TRUE(memory_optim_enable);
PD_ConfigEnableProfile(config);
bool profiler_enable = PD_ConfigProfileEnabled(config);
EXPECT_TRUE(profiler_enable);
PD_ConfigSetInvalid(config);
bool is_valid = PD_ConfigIsValid(config);
EXPECT_FALSE(is_valid);
PD_ConfigDestroy(config);
}
TEST(PD_Config, trt_int8) {
std::string model_dir = FLAGS_infer_model + "/mobilenet";
PD_Config* config = PD_ConfigCreate();
PD_ConfigEnableUseGpu(config, 100, 0, 0);
PD_ConfigEnableTensorRtEngine(
config, 1 << 20, 1, 3, PD_PRECISION_INT8, FALSE, TRUE);
bool trt_enable = PD_ConfigTensorRtEngineEnabled(config);
EXPECT_TRUE(trt_enable);
PD_ConfigDestroy(config);
}
TEST(PD_Config, trt_fp16) {
std::string model_dir = FLAGS_infer_model + "/mobilenet";
PD_Config* config = PD_ConfigCreate();
PD_ConfigEnableUseGpu(config, 100, 0, 0);
PD_ConfigEnableTensorRtEngine(
config, 1 << 20, 1, 3, PD_PRECISION_HALF, FALSE, FALSE);
bool trt_enable = PD_ConfigTensorRtEngineEnabled(config);
EXPECT_TRUE(trt_enable);
PD_Predictor* predictor = PD_PredictorCreate(config);
PD_PredictorDestroy(predictor);
}
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_capi_exp_int_tester.cc
+95
View File
@@ -0,0 +1,95 @@
/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <string>
#include <vector>
#include "paddle/common/flags.h"
#include "paddle/fluid/inference/capi_exp/pd_inference_api.h"
PD_DEFINE_string(infer_model, "", "model path");
namespace paddle {
namespace inference {
namespace analysis {
void predictor_run() {
std::string model_dir = FLAGS_infer_model;
PD_Config* config = PD_ConfigCreate();
PD_ConfigDisableGpu(config);
PD_ConfigSetCpuMathLibraryNumThreads(config, 10);
PD_ConfigSwitchIrDebug(config, TRUE);
PD_ConfigSetModel(config,
(model_dir + "/inference.pdmodel").c_str(),
(model_dir + "/inference.pdiparams").c_str());
PD_Predictor* predictor = PD_PredictorCreate(config);
PD_OneDimArrayCstr* input_names = PD_PredictorGetInputNames(predictor);
LOG(INFO) << "The inputs' size is: " << input_names->size;
EXPECT_EQ(input_names->size, 1u);
PD_IOInfos* in_infos = PD_PredictorGetInputInfos(predictor);
EXPECT_EQ(in_infos->size, 1u);
PD_IOInfos* out_infos = PD_PredictorGetOutputInfos(predictor);
std::array<int32_t, 4> shape_0 = {1, 3, 224, 224};
std::array<float, 1 * 3 * 224 * 224> data_0 = {0};
PD_Tensor* input_0 = PD_PredictorGetInputHandle(predictor, "x");
PD_TensorReshape(input_0, 4, shape_0.data());
PD_TensorCopyFromCpuFloat(input_0, data_0.data());
LOG(INFO) << "Run Inference in CAPI encapsulation. ";
EXPECT_TRUE(PD_PredictorRun(predictor));
PD_OneDimArrayCstr* output_names = PD_PredictorGetOutputNames(predictor);
LOG(INFO) << "output size is: " << output_names->size;
for (size_t index = 0; index < output_names->size; ++index) {
LOG(INFO) << "output[" << index
<< "]'s name is: " << output_names->data[index];
PD_Tensor* output =
PD_PredictorGetOutputHandle(predictor, output_names->data[index]);
PD_OneDimArrayInt32* shape = PD_TensorGetShape(output);
LOG(INFO) << "output[" << index << "]'s shape_size is: " << shape->size;
int32_t out_size = 1;
for (size_t i = 0; i < shape->size; ++i) {
LOG(INFO) << "output[" << index << "]'s shape is: " << shape->data[i];
out_size = out_size * shape->data[i];
}
float* out_data = new float[out_size];
PD_TensorCopyToCpuFloat(output, out_data);
LOG(INFO) << "output[" << index << "]'s DATA is: " << out_data[0];
delete[] out_data;
PD_OneDimArrayInt32Destroy(shape);
PD_TensorDestroy(output);
}
PD_PredictorClearIntermediateTensor(predictor);
PD_PredictorTryShrinkMemory(predictor);
PD_OneDimArrayCstrDestroy(output_names);
PD_TensorDestroy(input_0);
PD_OneDimArrayCstrDestroy(input_names);
PD_IOInfosDestroy(in_infos);
PD_IOInfosDestroy(out_infos);
PD_PredictorDestroy(predictor);
}
#ifdef PADDLE_WITH_DNNL
TEST(PD_PredictorRun, predictor_run) { predictor_run(); }
#endif
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_capi_exp_ner_tester.cc
+113
View File
@@ -0,0 +1,113 @@
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <string>
#include <vector>
#include "paddle/common/flags.h"
#include "paddle/fluid/inference/capi_exp/pd_inference_api.h"
PD_DEFINE_string(infer_model, "", "model path");
namespace paddle {
namespace inference {
namespace analysis {
TEST(PD_PredictorRun, predictor_run) {
auto model_dir = FLAGS_infer_model;
PD_Config *config = PD_ConfigCreate();
PD_ConfigSetModel(config,
(model_dir + "/__model__").c_str(),
(model_dir + "/param").c_str());
PD_ConfigDisableGpu(config);
PD_Predictor *predictor = PD_PredictorCreate(config);
size_t input_num = PD_PredictorGetInputNum(predictor);
LOG(INFO) << "Input num: " << input_num;
size_t output_num = PD_PredictorGetOutputNum(predictor);
LOG(INFO) << "Output num: " << output_num;
PD_OneDimArrayCstr *input_names = PD_PredictorGetInputNames(predictor);
EXPECT_EQ(input_names->size, 2u);
LOG(INFO) << "Predictor start run!";
PD_Tensor *inputs[2]; // NOLINT
inputs[0] = PD_PredictorGetInputHandle(predictor, input_names->data[0]);
inputs[1] = PD_PredictorGetInputHandle(predictor, input_names->data[1]);
LOG(INFO) << "Predictor start run!";
// inputs[0]: word, use lod memory in stack
std::array<int32_t, 2> shape_0 = {11, 1};
std::array<int64_t, 11 * 1> data_0 = {
12673, 9763, 905, 284, 45, 7474, 20, 17, 1, 4, 9};
std::array<size_t, 2> lod_layer_0 = {0, 11};
PD_OneDimArraySize layer_0;
layer_0.size = 2;
layer_0.data = lod_layer_0.data();
PD_OneDimArraySize *layer_0_ptr = &layer_0;
PD_TwoDimArraySize lod_0;
lod_0.size = 1;
lod_0.data = &layer_0_ptr;
PD_TensorReshape(inputs[0], 2, shape_0.data());
PD_TensorCopyFromCpuInt64(inputs[0], data_0.data());
PD_TensorSetLod(inputs[0], &lod_0);
// inputs[1]: mention, use lod memory in heap
std::array<int32_t, 2> shape_1 = {11, 1};
std::array<int64_t, 11 * 1> data_1 = {27, 0, 0, 33, 34, 33, 0, 0, 0, 1, 2};
PD_TwoDimArraySize *lod_1_ptr = new PD_TwoDimArraySize();
lod_1_ptr->size = 1;
lod_1_ptr->data = new PD_OneDimArraySize *[1];
lod_1_ptr->data[0] = new PD_OneDimArraySize();
lod_1_ptr->data[0]->size = 2;
lod_1_ptr->data[0]->data = new size_t[2];
lod_1_ptr->data[0]->data[0] = 0;
lod_1_ptr->data[0]->data[1] = 11;
PD_TensorReshape(inputs[1], 2, shape_1.data());
PD_TensorCopyFromCpuInt64(inputs[1], data_1.data());
PD_TensorSetLod(inputs[1], lod_1_ptr);
// retrieve the lod memory
delete[] lod_1_ptr->data[0]->data;
delete lod_1_ptr->data[0];
delete[] lod_1_ptr->data;
delete lod_1_ptr;
lod_1_ptr = nullptr;
LOG(INFO) << "Predictor start run!";
bool success = PD_PredictorRun(predictor);
EXPECT_TRUE(success);
LOG(INFO) << "Predictor run success!";
PD_OneDimArrayCstr *output_names = PD_PredictorGetOutputNames(predictor);
PD_Tensor *output =
PD_PredictorGetOutputHandle(predictor, output_names->data[0]);
PD_TwoDimArraySize *output_lod = PD_TensorGetLod(output);
PD_TwoDimArraySizeDestroy(output_lod);
PD_TensorDestroy(output);
PD_OneDimArrayCstrDestroy(output_names);
PD_TensorDestroy(inputs[0]);
PD_TensorDestroy(inputs[1]);
PD_OneDimArrayCstrDestroy(input_names);
PD_PredictorDestroy(predictor);
}
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_capi_exp_pd_config_tester.cc
+116
View File
@@ -0,0 +1,116 @@
/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <string>
#include <vector>
#include "paddle/common/flags.h"
#include "paddle/fluid/inference/capi_exp/pd_inference_api.h"
PD_DEFINE_string(infer_model, "", "model path");
namespace paddle {
namespace inference {
namespace analysis {
TEST(PD_Config, interface) {
std::string model_dir = FLAGS_infer_model + "/mobilenet";
std::string prog_file = model_dir + "/__model__";
std::string param_file = model_dir + "/__params__";
std::string opt_cache_dir = FLAGS_infer_model + "/OptimCacheDir";
PD_Config* config = PD_ConfigCreate();
PD_ConfigSetModelDir(config, model_dir.c_str());
std::string model_dir_ = PD_ConfigGetModelDir(config);
EXPECT_EQ(model_dir, model_dir_);
PD_ConfigSetModel(config, prog_file.c_str(), param_file.c_str());
PD_ConfigSetProgFile(config, prog_file.c_str());
PD_ConfigSetParamsFile(config, param_file.c_str());
PD_ConfigSetOptimCacheDir(config, opt_cache_dir.c_str());
std::string prog_file_ = PD_ConfigGetProgFile(config);
std::string param_file_ = PD_ConfigGetParamsFile(config);
EXPECT_EQ(prog_file, prog_file_);
EXPECT_EQ(param_file, param_file_);
PD_ConfigDisableFCPadding(config);
bool fc_padding = PD_ConfigUseFcPadding(config);
EXPECT_FALSE(fc_padding);
PD_ConfigDisableGpu(config);
PD_ConfigSwitchIrOptim(config, TRUE);
bool ir_optim = PD_ConfigIrOptim(config);
EXPECT_TRUE(ir_optim);
PD_ConfigEnableMemoryOptim(config, true);
bool memory_enabled = PD_ConfigMemoryOptimEnabled(config);
EXPECT_TRUE(memory_enabled);
PD_ConfigSwitchIrDebug(config, TRUE);
#ifdef PADDLE_WITH_DNNL
const char* ops_name = "conv_2d";
PD_ConfigEnableONEDNN(config);
PD_ConfigSetOnednnOp(config, 1, &ops_name);
PD_ConfigSetOnednnCacheCapacity(config, 100);
bool onednn_enabled = PD_ConfigOnednnEnabled(config);
EXPECT_TRUE(onednn_enabled);
PD_ConfigSetCpuMathLibraryNumThreads(config, 10);
int32_t cpu_threads = PD_ConfigGetCpuMathLibraryNumThreads(config);
EXPECT_EQ(cpu_threads, 10);
PD_ConfigEnableOnednnBfloat16(config);
PD_ConfigSetBfloat16Op(config, 1, &ops_name);
PD_ConfigEnableOnednnInt8(config);
bool onednn_int8_enabled = PD_ConfigOnednnInt8Enabled(config);
EXPECT_TRUE(onednn_int8_enabled);
#endif
PD_ConfigEnableONNXRuntime(config);
bool onnxruntime_enabled = PD_ConfigONNXRuntimeEnabled(config);
#ifdef PADDLE_WITH_ONNXRUNTIME
EXPECT_TRUE(onnxruntime_enabled);
#else
EXPECT_FALSE(onnxruntime_enabled);
#endif
PD_ConfigDisableONNXRuntime(config);
bool onnxruntime_disabled = PD_ConfigONNXRuntimeEnabled(config);
EXPECT_FALSE(onnxruntime_disabled);
PD_ConfigEnableORTOptimization(config);
PD_ConfigEnableProfile(config);
bool profile_enabled = PD_ConfigProfileEnabled(config);
EXPECT_TRUE(profile_enabled);
PD_ConfigDisableGlogInfo(config);
bool glog_disabled = PD_ConfigGlogInfoDisabled(config);
EXPECT_TRUE(glog_disabled);
PD_ConfigSetInvalid(config);
bool is_valid = PD_ConfigIsValid(config);
EXPECT_FALSE(is_valid);
PD_ConfigDestroy(config);
}
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_capi_exp_pd_tensor_tester.cc
+211
View File
@@ -0,0 +1,211 @@
/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <fstream>
#include <iostream>
#include <numeric>
#include <sstream>
#include <string>
#include <vector>
#include "paddle/common/flags.h"
#include "paddle/fluid/inference/capi_exp/pd_inference_api.h"
PD_DEFINE_string(infer_model, "", "model path");
namespace paddle {
namespace inference {
namespace analysis {
void PD_run() {
auto model_dir = FLAGS_infer_model;
PD_Config* config = PD_ConfigCreate();
PD_ConfigSetModel(config,
(model_dir + "/__model__").c_str(),
(model_dir + "/__params__").c_str());
PD_Predictor* predictor = PD_PredictorCreate(config);
PD_OneDimArrayCstr* input_names = PD_PredictorGetInputNames(predictor);
PD_Tensor* tensor =
PD_PredictorGetInputHandle(predictor, input_names->data[0]);
std::array<int32_t, 4> shapes = {1, 3, 224, 224};
std::vector<float> input(1 * 3 * 224 * 224, 0);
int32_t size;
PD_PlaceType place;
PD_TensorReshape(tensor, 4, shapes.data());
PD_TensorCopyFromCpuFloat(tensor, input.data());
PD_TensorDataFloat(tensor, &place, &size);
PD_TensorMutableDataFloat(tensor, place);
PD_TwoDimArraySize lod;
lod.size = 0;
lod.data = nullptr;
PD_TensorSetLod(tensor, &lod);
PD_PredictorRun(predictor);
std::vector<float> out_data;
PD_OneDimArrayCstr* output_names = PD_PredictorGetOutputNames(predictor);
PD_Tensor* output_tensor =
PD_PredictorGetOutputHandle(predictor, output_names->data[0]);
PD_OneDimArrayInt32* output_shape = PD_TensorGetShape(output_tensor);
int32_t out_num = std::accumulate(output_shape->data,
output_shape->data + output_shape->size,
1,
std::multiplies<>());
out_data.resize(out_num);
PD_TensorCopyToCpuFloat(output_tensor, out_data.data());
LOG(INFO) << "Output tensor name is: " << PD_TensorGetName(output_tensor);
PD_DataType data_type = PD_TensorGetDataType(output_tensor);
EXPECT_EQ(data_type, PD_DATA_FLOAT32);
PD_TwoDimArraySize* out_lod = PD_TensorGetLod(output_tensor);
PD_TwoDimArraySizeDestroy(out_lod);
PD_OneDimArrayInt32Destroy(output_shape);
PD_TensorDestroy(output_tensor);
PD_OneDimArrayCstrDestroy(output_names);
PD_TensorDestroy(tensor);
PD_OneDimArrayCstrDestroy(input_names);
PD_PredictorDestroy(predictor);
}
TEST(PD_Tensor, PD_run) { PD_run(); }
TEST(PD_Tensor, int32) {
auto model_dir = FLAGS_infer_model;
PD_Config* config = PD_ConfigCreate();
PD_ConfigSetModel(config,
(model_dir + "/__model__").c_str(),
(model_dir + "/__params__").c_str());
PD_Predictor* predictor = PD_PredictorCreate(config);
PD_OneDimArrayCstr* input_names = PD_PredictorGetInputNames(predictor);
PD_Tensor* tensor =
PD_PredictorGetInputHandle(predictor, input_names->data[0]);
std::array<int32_t, 4> shapes = {1, 3, 224, 224};
std::vector<int32_t> input(1 * 3 * 224 * 224, 0);
int32_t size;
PD_PlaceType place;
PD_TensorReshape(tensor, 4, shapes.data());
PD_TensorCopyFromCpuInt32(tensor, input.data());
int32_t* data_ptr = PD_TensorDataInt32(tensor, &place, &size);
EXPECT_EQ(place, PD_PLACE_CPU);
EXPECT_EQ(size, 1 * 3 * 224 * 224);
int32_t* mutable_data_ptr = PD_TensorMutableDataInt32(tensor, place);
EXPECT_EQ(data_ptr, mutable_data_ptr);
PD_DataType data_type = PD_TensorGetDataType(tensor);
EXPECT_EQ(data_type, PD_DATA_INT32);
PD_TensorCopyToCpuInt32(tensor, input.data());
PD_TensorDestroy(tensor);
PD_OneDimArrayCstrDestroy(input_names);
PD_PredictorDestroy(predictor);
}
TEST(PD_Tensor, int64) {
auto model_dir = FLAGS_infer_model;
PD_Config* config = PD_ConfigCreate();
PD_ConfigSetModel(config,
(model_dir + "/__model__").c_str(),
(model_dir + "/__params__").c_str());
PD_Predictor* predictor = PD_PredictorCreate(config);
PD_OneDimArrayCstr* input_names = PD_PredictorGetInputNames(predictor);
PD_Tensor* tensor =
PD_PredictorGetInputHandle(predictor, input_names->data[0]);
std::array<int32_t, 4> shapes = {1, 3, 224, 224};
std::vector<int64_t> input(1 * 3 * 224 * 224, 0);
int32_t size;
PD_PlaceType place;
PD_TensorReshape(tensor, 4, shapes.data());
PD_TensorCopyFromCpuInt64(tensor, input.data());
int64_t* data_ptr = PD_TensorDataInt64(tensor, &place, &size);
EXPECT_EQ(place, PD_PLACE_CPU);
EXPECT_EQ(size, 1 * 3 * 224 * 224);
int64_t* mutable_data_ptr = PD_TensorMutableDataInt64(tensor, place);
EXPECT_EQ(data_ptr, mutable_data_ptr);
PD_DataType data_type = PD_TensorGetDataType(tensor);
EXPECT_EQ(data_type, PD_DATA_INT64);
PD_TensorCopyToCpuInt64(tensor, input.data());
PD_TensorDestroy(tensor);
PD_OneDimArrayCstrDestroy(input_names);
PD_PredictorDestroy(predictor);
}
TEST(PD_Tensor, uint8) {
auto model_dir = FLAGS_infer_model;
PD_Config* config = PD_ConfigCreate();
PD_ConfigSetModel(config,
(model_dir + "/__model__").c_str(),
(model_dir + "/__params__").c_str());
PD_Predictor* predictor = PD_PredictorCreate(config);
PD_OneDimArrayCstr* input_names = PD_PredictorGetInputNames(predictor);
PD_Tensor* tensor =
PD_PredictorGetInputHandle(predictor, input_names->data[0]);
std::array<int32_t, 4> shapes = {1, 3, 224, 224};
std::array<uint8_t, 1 * 3 * 224 * 224> input = {0};
int32_t size;
PD_PlaceType place;
PD_TensorReshape(tensor, 4, shapes.data());
PD_TensorCopyFromCpuUint8(tensor, input.data());
uint8_t* data_ptr = PD_TensorDataUint8(tensor, &place, &size);
EXPECT_EQ(place, PD_PLACE_CPU);
EXPECT_EQ(size, 1 * 3 * 224 * 224);
uint8_t* mutable_data_ptr = PD_TensorMutableDataUint8(tensor, place);
EXPECT_EQ(data_ptr, mutable_data_ptr);
PD_DataType data_type = PD_TensorGetDataType(tensor);
EXPECT_EQ(data_type, PD_DATA_UINT8);
PD_TensorCopyToCpuUint8(tensor, input.data());
PD_TensorDestroy(tensor);
PD_OneDimArrayCstrDestroy(input_names);
PD_PredictorDestroy(predictor);
}
std::string read_file(std::string filename) {
std::ifstream file(filename);
return std::string((std::istreambuf_iterator<char>(file)),
std::istreambuf_iterator<char>());
}
TEST(PD_Tensor, from_buffer) {
PD_Config* config = PD_ConfigCreate();
std::string prog_file = FLAGS_infer_model + "/__model__";
std::string params_file = FLAGS_infer_model + "/__params__";
std::string prog_str = read_file(prog_file);
std::string params_str = read_file(params_file);
PD_ConfigSetModelBuffer(config,
prog_str.c_str(),
prog_str.size(),
params_str.c_str(),
params_str.size());
bool model_from_memory = PD_ConfigModelFromMemory(config);
EXPECT_TRUE(model_from_memory);
PD_ConfigDestroy(config);
}
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_capi_exp_pd_threads_tester.cc
+117
View File
@@ -0,0 +1,117 @@
/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <fstream>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#include "paddle/common/flags.h"
#include "paddle/fluid/inference/capi_exp/pd_inference_api.h"
PD_DEFINE_string(infer_model, "", "model path");
namespace paddle {
namespace inference {
namespace analysis {
typedef struct RunParameter {
PD_Predictor* predictor;
int32_t* shapes;
size_t shape_size;
float* input_data;
int32_t out_size;
std::vector<float> out_data;
int32_t thread_index;
} RunParameter;
void* run(void* thread_param) {
struct RunParameter* param = (struct RunParameter*)thread_param;
LOG(INFO) << "Thread " << param->thread_index << " start run!";
PD_OneDimArrayCstr* input_names = PD_PredictorGetInputNames(param->predictor);
PD_Tensor* tensor =
PD_PredictorGetInputHandle(param->predictor, input_names->data[0]);
PD_TensorReshape(tensor, param->shape_size, param->shapes);
PD_TensorCopyFromCpuFloat(tensor, param->input_data);
PD_PredictorRun(param->predictor);
PD_OneDimArrayCstr* output_names =
PD_PredictorGetOutputNames(param->predictor);
PD_Tensor* output_tensor =
PD_PredictorGetOutputHandle(param->predictor, output_names->data[0]);
PD_OneDimArrayInt32* output_shape = PD_TensorGetShape(output_tensor);
param->out_size = 1;
for (size_t index = 0; index < output_shape->size; ++index) {
param->out_size = param->out_size * output_shape->data[index];
}
PD_OneDimArrayInt32Destroy(output_shape);
param->out_data.resize(param->out_size);
PD_TensorCopyToCpuFloat(output_tensor, param->out_data.data());
PD_TensorDestroy(output_tensor);
PD_OneDimArrayCstrDestroy(output_names);
PD_TensorDestroy(tensor);
PD_OneDimArrayCstrDestroy(input_names);
LOG(INFO) << "Thread " << param->thread_index << " end run!";
return nullptr;
}
void threads_run(int thread_num) {
auto model_dir = FLAGS_infer_model;
PD_Config* config = PD_ConfigCreate();
PD_ConfigSetModel(config,
(model_dir + "/__model__").c_str(),
(model_dir + "/__params__").c_str());
PD_Predictor* predictor = PD_PredictorCreate(config);
std::vector<pthread_t> threads(thread_num);
std::vector<RunParameter> params(thread_num);
std::array<int32_t, 4> shapes = {1, 3, 224, 224};
std::vector<float> input(1 * 3 * 224 * 224, 0);
for (int i = 0; i < thread_num; ++i) {
params[i].predictor = PD_PredictorClone(predictor);
params[i].shapes = shapes.data();
params[i].shape_size = 4;
params[i].input_data = input.data();
params[i].out_size = 0;
params[i].thread_index = i;
pthread_create(&(threads[i]), nullptr, run, &(params[i]));
}
for (int i = 0; i < thread_num; ++i) {
pthread_join(threads[i], nullptr);
}
ASSERT_GT(params[0].out_size, 0);
for (int i = 1; i < thread_num; ++i) {
ASSERT_EQ(params[i].out_size, params[0].out_size);
for (int j = 0; j < params[i].out_size; ++j) {
ASSERT_EQ(params[i].out_data[j], params[0].out_data[j]);
}
}
for (int i = 0; i < thread_num; ++i) {
PD_PredictorDestroy(params[i].predictor);
}
PD_PredictorDestroy(predictor);
}
TEST(PD_Predictor, PD_multi_threads_run) { threads_run(10); }
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_capi_exp_tester.cc
+90
View File
@@ -0,0 +1,90 @@
/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <string>
#include <vector>
#include "paddle/common/flags.h"
#include "paddle/fluid/inference/capi_exp/pd_config.h"
#include "paddle/fluid/inference/capi_exp/pd_inference_api.h"
#include "paddle/fluid/inference/capi_exp/pd_utils.h"
PD_DEFINE_string(infer_model, "", "model path");
namespace paddle {
namespace inference {
namespace analysis {
void predictor_run() {
std::string model_dir = FLAGS_infer_model;
std::string prog_file = model_dir + "/model";
std::string params_file = model_dir + "/params";
PD_Config *config = PD_ConfigCreate();
PD_ConfigDisableGpu(config);
PD_ConfigSetCpuMathLibraryNumThreads(config, 10);
PD_ConfigSwitchIrDebug(config, TRUE);
PD_ConfigSetModel(config, prog_file.c_str(), params_file.c_str());
PD_Cstr *config_summary = PD_ConfigSummary(config);
LOG(INFO) << config_summary->data;
PD_Predictor *predictor = PD_PredictorCreate(config);
PD_Tensor *tensor = PD_PredictorGetInputHandle(predictor, "data");
const int batch_size = 1;
const int channels = 3;
const int height = 318;
const int width = 318;
float *input = new float[batch_size * channels * height * width]();
std::array<int32_t, 4> shape = {batch_size, channels, height, width};
PD_TensorReshape(tensor, 4, shape.data());
PD_TensorCopyFromCpuFloat(tensor, input);
EXPECT_TRUE(PD_PredictorRun(predictor));
delete[] input;
PD_TensorDestroy(tensor);
PD_CstrDestroy(config_summary);
PD_PredictorDestroy(predictor);
}
TEST(PD_PredictorRun, predictor_run) { predictor_run(); }
#ifdef PADDLE_WITH_DNNL
TEST(PD_Config, profile_onednn) {
std::string model_dir = FLAGS_infer_model;
std::string prog_file = model_dir + "/model";
std::string params_file = model_dir + "/params";
PD_Config *config = PD_ConfigCreate();
PD_ConfigDisableGpu(config);
PD_ConfigSetCpuMathLibraryNumThreads(config, 10);
PD_ConfigSwitchIrDebug(config, TRUE);
PD_ConfigEnableONEDNN(config);
bool onednn_enable = PD_ConfigOnednnEnabled(config);
EXPECT_TRUE(onednn_enable);
PD_ConfigEnableOnednnBfloat16(config);
PD_ConfigSetOnednnCacheCapacity(config, 0);
PD_ConfigSetModel(config, prog_file.c_str(), params_file.c_str());
PD_ConfigDestroy(config);
}
#endif
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_capi_exp_xpu_tester.cc
+66
View File
@@ -0,0 +1,66 @@
/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <string>
#include <vector>
#include "paddle/common/flags.h"
#include "paddle/fluid/inference/capi_exp/pd_inference_api.h"
PD_DEFINE_string(infer_model, "", "model path");
namespace paddle {
namespace inference {
namespace analysis {
#ifdef PADDLE_WITH_XPU
TEST(PD_Config, use_xpu) {
std::string model_dir = FLAGS_infer_model + "/mobilenet";
PD_Config *config = PD_Config();
PD_ConfigSwitchIrDebug(config, TRUE);
PD_ConfigSetModelDir(config, model_dir.c_str());
PD_ConfigSetOptimCacheDir(config,
(FLAGS_infer_model + "/OptimCacheDir").c_str());
const char *model_dir_ = PD_ConfigGetModelDir(config);
LOG(INFO) << model_dir_;
PD_ConfigEnableXpu(config, 0xfffc00);
bool use_xpu = PD_ConfigUseXpu(config);
EXPECT_TRUE(use_xpu);
int32_t device_id = PD_ConfigXpuDeviceId(config);
EXPECT_EQ(device_id, 0);
PD_ConfigSwitchIrOptim(config, TRUE);
bool ir_optim = PD_IrOptim(config);
EXPECT_TRUE(ir_optim);
PD_ConfigEnableMemoryOptim(config, true);
bool memory_optim_enable = PD_ConfigMemoryOptimEnabled(config);
EXPECT_TRUE(memory_optim_enable);
PD_ConfigEnableProfile(config);
bool profiler_enable = PD_ConfigProfileEnabled(config);
EXPECT_TRUE(profiler_enable);
PD_SetInValid(config);
bool is_valid = PD_ConfigIsValid(config);
EXPECT_FALSE(is_valid);
PD_ConfigDestroy(config);
}
#endif
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_capi_gpu_tester.cc
+181
View File
@@ -0,0 +1,181 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <string>
#include <vector>
#include "paddle/fluid/inference/capi/paddle_c_api.h"
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
namespace analysis {
TEST(PD_AnalysisConfig, use_gpu) {
std::string model_dir = FLAGS_infer_model + "/mobilenet";
PD_AnalysisConfig *config = PD_NewAnalysisConfig();
PD_DisableGpu(config);
PD_SetCpuMathLibraryNumThreads(config, 10);
int num_thread = PD_CpuMathLibraryNumThreads(config);
PADDLE_ENFORCE_EQ(
10,
num_thread,
common::errors::InvalidArgument("The num of thread should be "
"equal to 10, but got %d.",
num_thread));
PD_SwitchSpecifyInputNames(config, true);
PD_SwitchIrDebug(config, true);
PD_SetModel(config, model_dir.c_str(), nullptr);
PD_SetOptimCacheDir(config, (FLAGS_infer_model + "/OptimCacheDir").c_str());
const char *model_dir_ = PD_ModelDir(config);
LOG(INFO) << model_dir_;
PD_EnableUseGpu(config, 100, 0);
bool use_gpu = PD_UseGpu(config);
PADDLE_ENFORCE_EQ(use_gpu,
true,
common::errors::InvalidArgument(
"GPU is not enabled. "
"The configuration indicates that GPU should be used, "
"but it is currently disabled. "
"Please check your configuration settings and ensure "
"that GPU is properly enabled."));
int device = PD_GpuDeviceId(config);
PADDLE_ENFORCE_EQ(device,
0,
common::errors::InvalidArgument(
"The device ID is incorrect. "
"Expected device ID is 0, but received %d. "
"Please check your device configuration and "
"ensure the correct device ID is used.",
device));
int init_size = PD_MemoryPoolInitSizeMb(config);
PADDLE_ENFORCE_EQ(init_size,
100,
common::errors::InvalidArgument(
"The initial size of the memory pool is incorrect. "
"Expected size is 100 MB, but received %d MB. "
"Please check your configuration settings and ensure "
"the correct memory pool size is set.",
init_size));
float frac = PD_FractionOfGpuMemoryForPool(config);
LOG(INFO) << frac;
PD_EnableCUDNN(config);
bool cudnn = PD_CudnnEnabled(config);
PADDLE_ENFORCE_EQ(cudnn,
true,
common::errors::InvalidArgument(
"cuDNN is not enabled. "
"The configuration indicates that cuDNN should be "
"enabled, but it is currently disabled. "
"Please check your configuration settings and ensure "
"that cuDNN is properly enabled."));
PD_SwitchIrOptim(config, true);
bool ir_optim = PD_IrOptim(config);
PADDLE_ENFORCE_EQ(ir_optim,
true,
common::errors::InvalidArgument(
"IR optimization is not enabled. "
"The configuration indicates that IR optimization "
"should be enabled, but it is currently disabled. "
"Please check your configuration settings and ensure "
"that IR optimization is properly enabled."));
PD_EnableTensorRtEngine(
config, 1 << 20, 1, 3, Precision::kFloat32, false, false);
bool trt_enable = PD_TensorrtEngineEnabled(config);
PADDLE_ENFORCE_EQ(trt_enable,
true,
common::errors::InvalidArgument(
"TensorRT engine is not enabled. "
"The configuration indicates that TensorRT engine "
"should be enabled, but it is currently disabled. "
"Please check your configuration settings and ensure "
"that TensorRT engine is properly enabled."));
PD_EnableMemoryOptim(config);
bool memory_optim_enable = PD_MemoryOptimEnabled(config);
PADDLE_ENFORCE_EQ(memory_optim_enable,
true,
common::errors::InvalidArgument(
"Memory optimization is not enabled. "
"The configuration indicates that memory optimization "
"should be enabled, but it is currently disabled. "
"Please check your configuration settings and ensure "
"that memory optimization is properly enabled."));
PD_EnableProfile(config);
bool profiler_enable = PD_ProfileEnabled(config);
PADDLE_ENFORCE_EQ(profiler_enable,
true,
common::errors::InvalidArgument(
"Profiler is not enabled. "
"The configuration indicates that the profiler should "
"be enabled, but it is currently disabled. "
"Please check your configuration settings and ensure "
"that the profiler is properly enabled."));
PD_SetInValid(config);
bool is_valid = PD_IsValid(config);
PADDLE_ENFORCE_EQ(
is_valid,
true,
common::errors::InvalidArgument(
"Configuration is not valid. "
"The configuration should be valid, but it is currently invalid. "
"Please check your configuration settings and ensure they are "
"correct."));
PD_DeleteAnalysisConfig(config);
}
TEST(PD_AnalysisConfig, trt_int8) {
std::string model_dir = FLAGS_infer_model + "/mobilenet";
PD_AnalysisConfig *config = PD_NewAnalysisConfig();
PD_EnableUseGpu(config, 100, 0);
PD_EnableTensorRtEngine(config, 1 << 20, 1, 3, Precision::kInt8, false, true);
bool trt_enable = PD_TensorrtEngineEnabled(config);
PADDLE_ENFORCE_EQ(trt_enable,
true,
common::errors::InvalidArgument(
"TensorRT engine is not enabled. "
"The configuration indicates that TensorRT engine "
"should be enabled, but it is currently disabled. "
"Please check your configuration settings and ensure "
"that TensorRT engine is properly enabled."));
PD_DeleteAnalysisConfig(config);
}
TEST(PD_AnalysisConfig, trt_fp16) {
std::string model_dir = FLAGS_infer_model + "/mobilenet";
PD_AnalysisConfig *config = PD_NewAnalysisConfig();
PD_EnableUseGpu(config, 100, 0);
PD_EnableTensorRtEngine(
config, 1 << 20, 1, 3, Precision::kHalf, false, false);
bool trt_enable = PD_TensorrtEngineEnabled(config);
PADDLE_ENFORCE_EQ(trt_enable,
true,
common::errors::InvalidArgument(
"TensorRT engine is not enabled. "
"The configuration indicates that TensorRT engine "
"should be enabled, but it is currently disabled. "
"Please check your configuration settings and ensure "
"that TensorRT engine is properly enabled."));
PD_Predictor *predictor = PD_NewPredictor(config);
PD_DeletePredictor(predictor);
PD_DeleteAnalysisConfig(config);
}
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_capi_int_tester.cc
+105
View File
@@ -0,0 +1,105 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <string>
#include <vector>
#include "paddle/fluid/inference/capi/paddle_c_api.h"
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
namespace analysis {
void zero_copy_run() {
std::string model_dir = FLAGS_infer_model;
PD_AnalysisConfig *config = PD_NewAnalysisConfig();
PD_DisableGpu(config);
PD_SetCpuMathLibraryNumThreads(config, 10);
PD_SwitchSpecifyInputNames(config, true);
PD_SwitchIrDebug(config, true);
PD_SetModel(config, model_dir.c_str(), nullptr);
bool use_feed_fetch = PD_UseFeedFetchOpsEnabled(config);
PADDLE_ENFORCE_EQ(
use_feed_fetch, false, common::errors::PreconditionNotMet("NO"));
bool specify_input_names = PD_SpecifyInputName(config);
PADDLE_ENFORCE_EQ(
specify_input_names, true, common::errors::PreconditionNotMet("NO"));
const int batch_size = 1;
const int channels = 3;
const int height = 224;
const int width = 224;
float input[batch_size * channels * height * width] = {0};
int shape[4] = {batch_size, channels, height, width};
int shape_size = 4;
int in_size = 2;
int out_size;
PD_ZeroCopyData *inputs = new PD_ZeroCopyData[2];
PD_ZeroCopyData *outputs = nullptr;
inputs[0].data = static_cast<void *>(input);
inputs[0].dtype = PD_FLOAT32;
inputs[0].name = new char[6];
inputs[0].name[0] = 'i';
inputs[0].name[1] = 'm';
inputs[0].name[2] = 'a';
inputs[0].name[3] = 'g';
inputs[0].name[4] = 'e';
inputs[0].name[5] = '\0';
inputs[0].shape = shape;
inputs[0].shape_size = shape_size;
int *label = new int[1];
label[0] = 0;
inputs[1].data = static_cast<void *>(label);
inputs[1].dtype = PD_INT64;
inputs[1].name = new char[6];
inputs[1].name[0] = 'l';
inputs[1].name[1] = 'a';
inputs[1].name[2] = 'b';
inputs[1].name[3] = 'e';
inputs[1].name[4] = 'l';
inputs[1].name[5] = '\0';
int label_shape[2] = {1, 1};
int label_shape_size = 2;
inputs[1].shape = label_shape;
inputs[1].shape_size = label_shape_size;
PD_PredictorZeroCopyRun(config, inputs, in_size, &outputs, &out_size);
LOG(INFO) << "output size is: " << out_size;
LOG(INFO) << outputs[0].name;
for (int j = 0; j < out_size; ++j) {
LOG(INFO) << "output[" << j
<< "]'s shape_size is: " << outputs[j].shape_size;
for (int i = 0; i < outputs[0].shape_size; ++i) {
LOG(INFO) << "output[" << j << "]'s shape is: " << outputs[j].shape[i];
}
LOG(INFO) << "output[" << j
<< "]'s DATA is: " << *(static_cast<float *>(outputs[j].data));
}
delete[] outputs;
delete[] inputs;
}
#ifdef PADDLE_WITH_DNNL
TEST(PD_ZeroCopyRun, zero_copy_run) { zero_copy_run(); }
#endif
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_capi_ner_tester.cc
+125
View File
@@ -0,0 +1,125 @@
// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <string>
#include <vector>
#include "paddle/fluid/inference/capi/paddle_c_api.h"
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
namespace analysis {
void SetConfig(PD_AnalysisConfig *config) {
auto model_dir = FLAGS_infer_model;
PD_SetModel(config,
(model_dir + "/__model__").c_str(),
(model_dir + "/param").c_str());
PD_SwitchSpecifyInputNames(config, true);
PD_DisableGpu(config);
}
TEST(PD_ZeroCopyRun, zero_copy_run) {
PD_AnalysisConfig *config = PD_NewAnalysisConfig();
SetConfig(config);
PD_Predictor *predictor = PD_NewPredictor(config);
int input_num = PD_GetInputNum(predictor);
printf("Input num: %d\n", input_num);
int output_num = PD_GetOutputNum(predictor);
printf("Output num: %d\n", output_num);
PD_ZeroCopyTensor inputs[2];
// inputs[0]: word
PD_InitZeroCopyTensor(&inputs[0]);
inputs[0].name = new char[5];
snprintf(inputs[0].name,
strlen(PD_GetInputName(predictor, 0)) + 1,
"%s",
PD_GetInputName(predictor, 0));
inputs[0].data.capacity = sizeof(int64_t) * 11 * 1;
inputs[0].data.length = inputs[0].data.capacity;
inputs[0].data.data = malloc(inputs[0].data.capacity);
std::vector<int64_t> ref_word(
{12673, 9763, 905, 284, 45, 7474, 20, 17, 1, 4, 9});
inputs[0].data.data = reinterpret_cast<void *>(ref_word.data());
int shape0[] = {11, 1};
inputs[0].shape.data = reinterpret_cast<void *>(shape0);
inputs[0].shape.capacity = sizeof(shape0);
inputs[0].shape.length = sizeof(shape0);
inputs[0].dtype = PD_INT64;
size_t lod0[] = {0, 11};
inputs[0].lod.data = reinterpret_cast<void *>(lod0);
inputs[0].lod.capacity = sizeof(size_t) * 2;
inputs[0].lod.length = sizeof(size_t) * 2;
PD_SetZeroCopyInput(predictor, &inputs[0]);
// inputs[1]: mention
PD_InitZeroCopyTensor(&inputs[1]);
inputs[1].name = new char[8];
snprintf(inputs[1].name,
strlen(PD_GetInputName(predictor, 1)) + 1,
"%s",
PD_GetInputName(predictor, 1));
inputs[1].data.capacity = sizeof(int64_t) * 11 * 1;
inputs[1].data.length = inputs[1].data.capacity;
inputs[1].data.data = malloc(inputs[1].data.capacity);
std::vector<int64_t> ref_mention({27, 0, 0, 33, 34, 33, 0, 0, 0, 1, 2});
inputs[1].data.data = reinterpret_cast<void *>(ref_mention.data());
int shape1[] = {11, 1};
inputs[1].shape.data = reinterpret_cast<void *>(shape1);
inputs[1].shape.capacity = sizeof(shape1);
inputs[1].shape.length = sizeof(shape1);
inputs[1].dtype = PD_INT64;
size_t lod1[] = {0, 11};
inputs[1].lod.data = reinterpret_cast<void *>(lod1);
inputs[1].lod.capacity = sizeof(size_t) * 2;
inputs[1].lod.length = sizeof(size_t) * 2;
PD_SetZeroCopyInput(predictor, &inputs[1]);
PD_ZeroCopyRun(predictor);
PD_ZeroCopyTensor output;
PD_InitZeroCopyTensor(&output);
output.name = new char[21];
snprintf(output.name,
strlen(PD_GetOutputName(predictor, 0)) + 1,
"%s",
PD_GetOutputName(predictor, 0));
// not necessary, just for coverage tests
output.lod.data = std::malloc(sizeof(size_t));
PD_GetZeroCopyOutput(predictor, &output);
PD_DestroyZeroCopyTensor(&output);
PD_DeleteAnalysisConfig(config);
PD_DeletePredictor(predictor);
}
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_capi_pd_tensor_tester.cc
+173
View File
@@ -0,0 +1,173 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <fstream>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#include "paddle/common/enforce.h"
#include "paddle/fluid/inference/capi/c_api_internal.h"
#include "paddle/fluid/inference/capi/paddle_c_api.h"
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
namespace analysis {
void PD_run() {
PD_AnalysisConfig* config = PD_NewAnalysisConfig();
std::string prog_file = FLAGS_infer_model + "/__model__";
std::string params_file = FLAGS_infer_model + "/__params__";
PD_SetModel(config, prog_file.c_str(), params_file.c_str());
PD_SetProgFile(config, prog_file.c_str());
PD_SetParamsFile(config, params_file.c_str());
LOG(INFO) << PD_ProgFile(config);
LOG(INFO) << PD_ParamsFile(config);
PD_Tensor* input = PD_NewPaddleTensor();
PD_PaddleBuf* buf = PD_NewPaddleBuf();
LOG(INFO) << "PaddleBuf empty: " << PD_PaddleBufEmpty(buf);
int batch = 1;
int channel = 3;
int height = 300;
int width = 300;
int shape[4] = {batch, channel, height, width};
int shape_size = 4;
float* data = new float[batch * channel * height * width];
PD_PaddleBufReset(buf,
static_cast<void*>(data),
sizeof(float) * (batch * channel * height * width));
char name[6] = {'i', 'm', 'a', 'g', 'e', '\0'};
PD_SetPaddleTensorName(input, name);
PD_SetPaddleTensorDType(input, PD_FLOAT32);
PD_SetPaddleTensorShape(input, shape, shape_size);
PD_SetPaddleTensorData(input, buf);
PD_Tensor* out_data = PD_NewPaddleTensor();
int out_size;
PD_PredictorRun(config, input, 1, &out_data, &out_size, 1);
LOG(INFO) << out_size;
LOG(INFO) << PD_GetPaddleTensorName(out_data);
LOG(INFO) << PD_GetPaddleTensorDType(out_data);
PD_PaddleBuf* b = PD_GetPaddleTensorData(out_data);
LOG(INFO) << PD_PaddleBufLength(b) / sizeof(float);
float* result = static_cast<float*>(PD_PaddleBufData(b));
LOG(INFO) << *result;
PD_DeletePaddleTensor(input);
int size;
const int* out_shape = PD_GetPaddleTensorShape(out_data, &size);
PADDLE_ENFORCE_EQ(
size,
2,
common::errors::InvalidArgument("The Output shape's size is NOT match."));
std::vector<int> ref_outshape_size({9, 6});
for (int i = 0; i < 2; ++i) {
PADDLE_ENFORCE_EQ(out_shape[i],
ref_outshape_size[i],
common::errors::InvalidArgument(
"The Output shape's size is NOT match."));
}
PD_DeletePaddleBuf(buf);
}
TEST(PD_Tensor, PD_run) { PD_run(); }
TEST(PD_Tensor, int32) {
PD_Tensor* input = PD_NewPaddleTensor();
PD_SetPaddleTensorDType(input, PD_INT32);
LOG(INFO) << PD_GetPaddleTensorDType(input);
}
TEST(PD_Tensor, int64) {
PD_Tensor* input = PD_NewPaddleTensor();
PD_SetPaddleTensorDType(input, PD_INT64);
LOG(INFO) << PD_GetPaddleTensorDType(input);
}
TEST(PD_Tensor, int8) {
PD_Tensor* input = PD_NewPaddleTensor();
PD_SetPaddleTensorDType(input, PD_UINT8);
LOG(INFO) << PD_GetPaddleTensorDType(input);
}
std::string read_file(std::string filename) {
std::ifstream file(filename);
return std::string((std::istreambuf_iterator<char>(file)),
std::istreambuf_iterator<char>());
}
void buffer_run() {
PD_AnalysisConfig* config = PD_NewAnalysisConfig();
std::string prog_file = FLAGS_infer_model + "/__model__";
std::string params_file = FLAGS_infer_model + "/__params__";
std::string prog_str = read_file(prog_file);
std::string params_str = read_file(params_file);
PD_SetModelBuffer(config,
prog_str.c_str(),
prog_str.size(),
params_str.c_str(),
params_str.size());
LOG(INFO) << PD_ProgFile(config);
LOG(INFO) << PD_ParamsFile(config);
PADDLE_ENFORCE(PD_ModelFromMemory(config),
common::errors::PreconditionNotMet(
"PD_ModelFromMemory(config) is failed"));
PD_Tensor* input = PD_NewPaddleTensor();
PD_PaddleBuf* buf = PD_NewPaddleBuf();
LOG(INFO) << "PaddleBuf empty: " << PD_PaddleBufEmpty(buf);
int batch = 1;
int channel = 3;
int height = 300;
int width = 300;
int shape[4] = {batch, channel, height, width};
int shape_size = 4;
float* data = new float[batch * channel * height * width];
PD_PaddleBufReset(buf,
static_cast<void*>(data),
sizeof(float) * (batch * channel * height * width));
char name[6] = {'i', 'm', 'a', 'g', 'e', '\0'};
PD_SetPaddleTensorName(input, name);
PD_SetPaddleTensorDType(input, PD_FLOAT32);
PD_SetPaddleTensorShape(input, shape, shape_size);
PD_SetPaddleTensorData(input, buf);
PD_Tensor* out_data = PD_NewPaddleTensor();
int out_size;
PD_PredictorRun(config, input, 1, &out_data, &out_size, 1);
LOG(INFO) << out_size;
LOG(INFO) << PD_GetPaddleTensorName(out_data);
LOG(INFO) << PD_GetPaddleTensorDType(out_data);
PD_PaddleBuf* b = PD_GetPaddleTensorData(out_data);
LOG(INFO) << PD_PaddleBufLength(b) / sizeof(float);
float* result = static_cast<float*>(PD_PaddleBufData(b));
LOG(INFO) << *result;
PD_DeletePaddleTensor(input);
PD_DeletePaddleBuf(buf);
}
TEST(SetModelBuffer, read) { buffer_run(); }
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_capi_tester.cc
+98
View File
@@ -0,0 +1,98 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <string>
#include <vector>
#include "paddle/fluid/inference/capi/paddle_c_api.h"
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
namespace analysis {
void zero_copy_run() {
std::string model_dir = FLAGS_infer_model;
std::string prog_file = model_dir + "/model";
std::string params_file = model_dir + "/params";
PD_AnalysisConfig *config = PD_NewAnalysisConfig();
PD_DisableGpu(config);
PD_SetCpuMathLibraryNumThreads(config, 10);
PD_SwitchSpecifyInputNames(config, true);
PD_SwitchIrDebug(config, true);
PD_SetModel(config, prog_file.c_str(), params_file.c_str());
bool use_feed_fetch = PD_UseFeedFetchOpsEnabled(config);
EXPECT_FALSE(use_feed_fetch);
bool specify_input_names = PD_SpecifyInputName(config);
EXPECT_TRUE(specify_input_names);
const int batch_size = 1;
const int channels = 3;
const int height = 318;
const int width = 318;
float *input = new float[batch_size * channels * height * width]();
int shape[4] = {batch_size, channels, height, width};
int shape_size = 4;
int in_size = 1;
int out_size;
PD_ZeroCopyData *inputs = new PD_ZeroCopyData;
PD_ZeroCopyData *outputs = new PD_ZeroCopyData;
inputs->data = static_cast<void *>(input);
inputs->dtype = PD_FLOAT32;
inputs->name = new char[5];
inputs->name[0] = 'd';
inputs->name[1] = 'a';
inputs->name[2] = 't';
inputs->name[3] = 'a';
inputs->name[4] = '\0';
inputs->shape = shape;
inputs->shape_size = shape_size;
PD_PredictorZeroCopyRun(config, inputs, in_size, &outputs, &out_size);
delete[] input;
delete[] inputs;
delete[] outputs;
}
TEST(PD_PredictorZeroCopyRun, zero_copy_run) { zero_copy_run(); }
#ifdef PADDLE_WITH_DNNL
TEST(PD_AnalysisConfig, profile_onednn) {
std::string model_dir = FLAGS_infer_model;
std::string prog_file = model_dir + "/model";
std::string params_file = model_dir + "/params";
PD_AnalysisConfig *config = PD_NewAnalysisConfig();
PD_DisableGpu(config);
PD_SetCpuMathLibraryNumThreads(config, 10);
PD_SwitchSpecifyInputNames(config, true);
PD_SwitchIrDebug(config, true);
PD_EnableONEDNN(config);
bool onednn_enable = PD_OnednnEnabled(config);
EXPECT_TRUE(onednn_enable);
PD_EnableOnednnBfloat16(config);
PD_SetOnednnCacheCapacity(config, 0);
PD_SetModel(config, prog_file.c_str(), params_file.c_str());
PD_DeleteAnalysisConfig(config);
}
#endif
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_capi_xpu_tester.cc
+65
View File
@@ -0,0 +1,65 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <string>
#include <vector>
#include "paddle/common/enforce.h"
#include "paddle/fluid/inference/capi/paddle_c_api.h"
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
namespace analysis {
#ifdef PADDLE_WITH_XPU
TEST(PD_AnalysisConfig, use_xpu) {
std::string model_dir = FLAGS_infer_model + "/mobilenet";
PD_AnalysisConfig *config = PD_NewAnalysisConfig();
PD_SwitchSpecifyInputNames(config, true);
PD_SwitchIrDebug(config, true);
PD_SetModel(config, model_dir.c_str(), nullptr);
PD_SetOptimCacheDir(config, (FLAGS_infer_model + "/OptimCacheDir").c_str());
const char *model_dir_ = PD_ModelDir(config);
LOG(INFO) << model_dir_;
PD_EnableXpu(config, 0xfffc00);
bool use_xpu = PD_UseXpu(config);
PADDLE_ENFORCE_EQ(use_xpu, true, common::errors::PreconditionNotMet("NO"));
int device = PD_XpuDeviceId(config);
PADDLE_ENFORCE_EQ(device, 0, common::errors::PreconditionNotMet("NO"));
PD_SwitchIrOptim(config, true);
bool ir_optim = PD_IrOptim(config);
PADDLE_ENFORCE_EQ(ir_optim, true, common::errors::PreconditionNotMet("NO"));
PD_EnableMemoryOptim(config);
bool memory_optim_enable = PD_MemoryOptimEnabled(config);
PADDLE_ENFORCE_EQ(
memory_optim_enable, true, common::errors::PreconditionNotMet("NO"));
PD_EnableProfile(config);
bool profiler_enable = PD_ProfileEnabled(config);
PADDLE_ENFORCE_EQ(
profiler_enable, true, common::errors::PreconditionNotMet("NO"));
PD_SetInValid(config);
bool is_valid = PD_IsValid(config);
PADDLE_ENFORCE_EQ(is_valid, false, common::errors::PreconditionNotMet("NO"));
PD_DeleteAnalysisConfig(config);
}
#endif
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_dam_tester.cc
+332
View File
@@ -0,0 +1,332 @@
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <vector>
#include "paddle/fluid/inference/analysis/helper.h"
#include "test/cpp/inference/api/tester_helper.h"
const int FLAGS_max_turn_num = 1;
namespace paddle {
namespace inference {
constexpr int32_t kMaxTurnLen = 50;
static std::vector<float> result_data;
struct DataRecord {
std::vector<std::vector<int64_t>> *turns;
std::vector<std::vector<float>> *turns_mask;
std::vector<std::vector<int64_t>> response; // response data : 1
std::vector<std::vector<float>> response_mask; // response mask data : 1
size_t batch_iter{0};
size_t batch_size{1};
size_t num_samples; // total number of samples
DataRecord() { // NOLINT
turns = new std::vector<std::vector<
int64_t>>[FLAGS_max_turn_num]; // turns data : FLAGS_max_turn_num
turns_mask = new std::vector<std::vector<
float>>[FLAGS_max_turn_num]; // turns mask data : FLAGS_max_turn_num
}
explicit DataRecord(const std::string &path, int batch_size = 1)
: DataRecord() {
this->batch_size = batch_size;
Load(path);
}
~DataRecord() { // NOLINT
delete[] turns;
delete[] turns_mask;
}
DataRecord NextBatch() {
DataRecord data;
size_t batch_end = batch_iter + batch_size;
// NOTE skip the final batch, if no enough data is provided.
if (batch_end <= response.size()) {
for (int i = 0; i < FLAGS_max_turn_num; ++i) {
data.turns[i].assign(turns[i].begin() + batch_iter,
turns[i].begin() + batch_end);
}
for (int i = 0; i < FLAGS_max_turn_num; ++i) {
data.turns_mask[i].assign(turns_mask[i].begin() + batch_iter,
turns_mask[i].begin() + batch_end);
}
data.response.assign(response.begin() + batch_iter,
response.begin() + batch_end);
data.response_mask.assign(response_mask.begin() + batch_iter,
response_mask.begin() + batch_end);
PADDLE_ENFORCE_EQ(!data.response.empty(),
true,
common::errors::Fatal(
"Variable `data` response is empty, please check"));
PADDLE_ENFORCE_EQ(
!data.response_mask.empty(),
true,
common::errors::Fatal(
"Variable `data` response mask is empty, please check"));
PADDLE_ENFORCE_EQ(data.response.size(),
data.response_mask.size(),
common::errors::InvalidArgument(
"Required data.response.size() should be equal to "
"data.response_mask.size() . "));
}
batch_iter += batch_size;
return data;
}
void Load(const std::string &path) {
std::ifstream file(path);
std::string line;
size_t num_lines = 0;
result_data.clear();
while (std::getline(file, line)) {
num_lines++;
std::vector<std::string> data;
split(line, ',', &data);
PADDLE_ENFORCE_EQ(data.size(),
(size_t)(2 * FLAGS_max_turn_num + 3),
common::errors::InvalidArgument(
"Required data.size() should be equal to "
"(size_t)(2 * FLAGS_max_turn_num + 3) . "));
// load turn data
std::vector<int64_t> turns_tmp[FLAGS_max_turn_num];
for (int i = 0; i < FLAGS_max_turn_num; ++i) {
split_to_int64(data[i], ' ', &turns_tmp[i]);
turns[i].push_back(std::move(turns_tmp[i]));
}
// load turn_mask data
std::vector<float> turns_mask_tmp[FLAGS_max_turn_num];
for (int i = 0; i < FLAGS_max_turn_num; ++i) {
split_to_float(data[FLAGS_max_turn_num + i], ' ', &turns_mask_tmp[i]);
turns_mask[i].push_back(std::move(turns_mask_tmp[i]));
}
// load response data
std::vector<int64_t> response_tmp;
split_to_int64(data[2 * FLAGS_max_turn_num], ' ', &response_tmp);
response.push_back(std::move(response_tmp));
// load response_mask data
std::vector<float> response_mask_tmp;
split_to_float(data[2 * FLAGS_max_turn_num + 1], ' ', &response_mask_tmp);
response_mask.push_back(std::move(response_mask_tmp));
// load result data
float result_tmp;
result_tmp = std::stof(data[2 * FLAGS_max_turn_num + 2]);
result_data.push_back(result_tmp);
}
num_samples = num_lines;
}
};
void PrepareInputs(std::vector<PaddleTensor> *input_slots,
DataRecord *data,
int batch_size) {
PaddleTensor turns_tensor[FLAGS_max_turn_num]; // NOLINT
PaddleTensor turns_mask_tensor[FLAGS_max_turn_num]; // NOLINT
PaddleTensor response_tensor;
PaddleTensor response_mask_tensor;
std::string turn_pre = "turn_";
std::string turn_mask_pre = "turn_mask_";
auto one_batch = data->NextBatch();
PADDLE_ENFORCE(
!one_batch.response.empty(),
::common::errors::Fatal("The response of one batch is empty."));
int size = one_batch.response[0].size();
PADDLE_ENFORCE_EQ(size,
kMaxTurnLen,
common::errors::InvalidArgument(
"Required size should be equal to kMaxTurnLen . "));
// turn tensor assignment
for (int i = 0; i < FLAGS_max_turn_num; ++i) {
turns_tensor[i].name = turn_pre + std::to_string(i);
turns_tensor[i].shape.assign({batch_size, size, 1});
turns_tensor[i].dtype = PaddleDType::INT64;
TensorAssignData<int64_t>(&turns_tensor[i], one_batch.turns[i]);
}
// turn mask tensor assignment
for (int i = 0; i < FLAGS_max_turn_num; ++i) {
turns_mask_tensor[i].name = turn_mask_pre + std::to_string(i);
turns_mask_tensor[i].shape.assign({batch_size, size, 1});
turns_mask_tensor[i].dtype = PaddleDType::FLOAT32;
TensorAssignData<float>(&turns_mask_tensor[i], one_batch.turns_mask[i]);
}
// response tensor assignment
response_tensor.name = "response";
response_tensor.shape.assign({batch_size, size, 1});
response_tensor.dtype = PaddleDType::INT64;
TensorAssignData<int64_t>(&response_tensor, one_batch.response);
// response mask tensor assignment
response_mask_tensor.name = "response_mask";
response_mask_tensor.shape.assign({batch_size, size, 1});
response_mask_tensor.dtype = PaddleDType::FLOAT32;
TensorAssignData<float>(&response_mask_tensor, one_batch.response_mask);
// Set inputs.
for (auto &item : turns_tensor) {
input_slots->push_back(std::move(item));
}
for (auto &item : turns_mask_tensor) {
input_slots->push_back(std::move(item));
}
input_slots->push_back(std::move(response_tensor));
input_slots->push_back(std::move(response_mask_tensor));
}
/*
* this model is unreasonable, it set a output tensor persistable, so
* ridiculous! so I disable constant_folding_pass
*/
void SetConfig(AnalysisConfig *cfg) {
cfg->SetModel(FLAGS_infer_model + "/__model__", FLAGS_infer_model + "/param");
cfg->SwitchSpecifyInputNames();
cfg->DeletePass("constant_folding_pass");
cfg->SwitchIrOptim(true);
}
void SetOptimConfig(AnalysisConfig *cfg) {
std::string optimModelPath = FLAGS_infer_model + "/saved_optim_model";
cfg->SetModel(optimModelPath + "/model", optimModelPath + "/params");
cfg->SwitchIrOptim(true);
cfg->SwitchSpecifyInputNames();
}
void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
DataRecord data(FLAGS_infer_data, FLAGS_batch_size);
std::vector<PaddleTensor> input_slots;
int test_batch_num =
FLAGS_test_all_data ? data.num_samples / FLAGS_batch_size : 1; // NOLINT
LOG(INFO) << "The number of samples to be test: "
<< test_batch_num * FLAGS_batch_size;
for (int bid = 0; bid < test_batch_num; ++bid) {
input_slots.clear();
PrepareInputs(&input_slots, &data, FLAGS_batch_size);
(*inputs).emplace_back(input_slots);
}
}
// Easy for profiling independently.
void profile(bool use_onednn = false) {
AnalysisConfig cfg;
SetConfig(&cfg);
if (use_onednn) {
cfg.EnableONEDNN();
// Enable all the onednn supported ops except conv3d in dam
std::unordered_set<std::string> op_list = {
"softmax", "elementwise_add", "relu", "fc"};
cfg.SetONEDNNOp(op_list);
} else {
cfg.DisableONEDNN();
}
std::vector<std::vector<PaddleTensor>> outputs;
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all,
&outputs,
FLAGS_num_threads);
if (FLAGS_num_threads == 1 && !FLAGS_test_all_data) {
PADDLE_ENFORCE_GT(outputs.size(),
0,
::common::errors::Fatal(
"The size of outputs should be greater than 0."));
auto output = outputs.back();
PADDLE_ENFORCE_GT(output.size(),
0,
::common::errors::Fatal(
"The size of output should be greater than 0."));
size_t size = GetSize(output[0]);
PADDLE_ENFORCE_GT(size,
0,
::common::errors::Fatal(
"The size of output should be greater than 0."));
float *result = static_cast<float *>(output[0].data.data());
for (size_t i = 0; i < size; i++) {
EXPECT_NEAR(result[i], result_data[i], 1e-3);
}
}
}
TEST(Analyzer_dam, profile) { profile(); }
#ifdef PADDLE_WITH_DNNL
TEST(Analyzer_dam, profile_onednn) { profile(true /* use_onednn */); }
#endif
// Compare result of NativeConfig and AnalysisConfig
void compare(bool use_onednn = false) {
AnalysisConfig cfg;
SetConfig(&cfg);
if (use_onednn) {
cfg.EnableONEDNN();
// Enable all the onednn supported ops except conv3d in dam
std::unordered_set<std::string> op_list = {
"softmax", "elementwise_add", "relu"};
cfg.SetONEDNNOp(op_list);
} else {
cfg.DisableONEDNN();
}
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareNativeAndAnalysis(
reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
}
TEST(Analyzer_dam, compare_with_dynamic_memory_optim) {
// The small dam will core in CI, but works in local.
if (FLAGS_max_turn_num == 9) {
AnalysisConfig cfg, cfg1;
DataRecord data(FLAGS_infer_data, FLAGS_batch_size);
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
// Run the first time to force to update memory cache
SetConfig(&cfg);
cfg.EnableMemoryOptim();
CompareNativeAndAnalysis(
reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all);
}
}
TEST(Analyzer_dam, compare) { compare(); }
#ifdef PADDLE_WITH_DNNL
TEST(Analyzer_dam, compare_onednn) { compare(true /* use_onednn */); }
#endif
// Compare Deterministic result
TEST(Analyzer_dam, compare_determine) {
AnalysisConfig cfg;
SetConfig(&cfg);
cfg.DisableONEDNN();
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all);
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_detect_tester.cc
+137
View File
@@ -0,0 +1,137 @@
/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <gtest/gtest.h>
#include <fstream>
#include <iostream>
#include "test/cpp/inference/api/tester_helper.h"
PD_DEFINE_string(infer_shape, "", "data shape file");
PD_DEFINE_int32(sample, 20, "number of sample");
namespace paddle {
namespace inference {
namespace analysis {
struct Record {
std::vector<float> data;
std::vector<int32_t> shape;
};
Record ProcessALine(const std::string &line, const std::string &shape_line) {
VLOG(3) << "process a line";
std::vector<std::string> columns;
Record record;
std::vector<std::string> data_strs;
split(line, ' ', &data_strs);
for (auto &d : data_strs) {
record.data.push_back(std::stof(d));
}
std::vector<std::string> shape_strs;
split(shape_line, ' ', &shape_strs);
for (auto &s : shape_strs) {
record.shape.push_back(std::stoi(s));
}
return record;
}
void SetConfig(AnalysisConfig *cfg) {
cfg->SetModel(FLAGS_infer_model + "/model", FLAGS_infer_model + "/params");
cfg->DisableGpu();
cfg->SwitchIrDebug();
cfg->SwitchSpecifyInputNames(false);
cfg->SetCpuMathLibraryNumThreads(FLAGS_cpu_num_threads);
}
void SetInput(std::vector<std::vector<PaddleTensor>> *inputs,
const std::string &line,
const std::string &shape_line) {
auto record = ProcessALine(line, shape_line);
PaddleTensor input;
input.shape = record.shape;
input.dtype = PaddleDType::FLOAT32;
size_t input_size = record.data.size() * sizeof(float);
input.data.Resize(input_size);
memcpy(input.data.data(), record.data.data(), input_size);
std::vector<PaddleTensor> input_slots;
input_slots.assign({input});
(*inputs).emplace_back(input_slots);
}
void profile(int cache_capacity = 1) {
AnalysisConfig cfg;
SetConfig(&cfg);
cfg.EnableONEDNN();
cfg.SetOnednnCacheCapacity(cache_capacity);
std::vector<std::vector<PaddleTensor>> outputs;
std::vector<std::vector<PaddleTensor>> input_slots_all;
Timer run_timer;
double elapsed_time = 0;
int num_times = FLAGS_repeat;
int sample = FLAGS_sample;
auto predictor = CreatePaddlePredictor<AnalysisConfig>(cfg);
outputs.resize(sample);
std::vector<std::thread> threads;
std::ifstream file(FLAGS_infer_data);
std::ifstream infer_file(FLAGS_infer_shape);
std::string line;
std::string shape_line;
for (int i = 0; i < sample; i++) {
threads.emplace_back([&, i]() {
std::getline(file, line);
std::getline(infer_file, shape_line);
SetInput(&input_slots_all, line, shape_line);
run_timer.tic();
predictor->Run(input_slots_all[0], &outputs[0], FLAGS_batch_size);
elapsed_time += run_timer.toc();
});
threads[0].join();
threads.clear();
std::vector<std::vector<PaddleTensor>>().swap(input_slots_all);
}
file.close();
infer_file.close();
auto batch_latency = elapsed_time / (sample * num_times);
PrintTime(FLAGS_batch_size,
num_times,
FLAGS_num_threads,
0,
batch_latency,
sample,
VarType::FP32);
}
#ifdef PADDLE_WITH_DNNL
TEST(Analyzer_detect, profile_onednn) {
profile(5 /* cache_capacity */);
profile(10 /* cache_capacity */);
}
#endif
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_dist_model_xpu_tester.cc
+67
View File
@@ -0,0 +1,67 @@
// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "gtest/gtest.h"
#include "paddle/fluid/framework/block_desc.h"
#include "paddle/fluid/framework/op_desc.h"
#include "paddle/fluid/framework/program_desc.h"
#include "paddle/fluid/framework/scope.h"
#include "paddle/fluid/inference/utils/singleton.h"
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
TEST(test_dist_model_xpu, dist_model_xpu) {
std::cout << "Analysis Predictor DistModel XPU test." << std::endl;
AnalysisConfig config;
config.SetModel(FLAGS_infer_model + "/__model__",
FLAGS_infer_model + "/__params__");
config.EnableXpu();
config.SetXpuDeviceId(0);
auto predictor = paddle_infer::CreatePredictor(config);
int batch_size = 1;
int channels = 1;
int height = 48;
int width = 512;
int nums = batch_size * channels * height * width;
std::cout << "Created predictor." << std::endl;
float* input = new float[nums];
for (int i = 0; i < nums; ++i) input[i] = 0;
auto input_names = predictor->GetInputNames();
auto input_t = predictor->GetInputHandle(input_names[0]);
input_t->Reshape({batch_size, channels, height, width});
input_t->CopyFromCpu(input);
std::cout << "Input data." << std::endl;
predictor->Run();
std::cout << "Zero Copy Run." << std::endl;
std::vector<float> out_data;
auto output_names = predictor->GetOutputNames();
auto output_t = predictor->GetOutputHandle(output_names[0]);
std::vector<int> output_shape = output_t->shape();
int out_num = std::accumulate(
output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
out_data.resize(out_num);
output_t->CopyToCpu(out_data.data());
std::cout << "Output data." << std::endl;
delete[] input;
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_image_classification_tester.cc
+99
View File
@@ -0,0 +1,99 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <fstream>
#include <iostream>
#include "test/cpp/inference/api/tester_helper.h"
PD_DEFINE_bool(disable_onednn_fc, false, "Disable usage of ONE-DNN's FC op");
namespace paddle {
namespace inference {
namespace analysis {
void SetConfig(AnalysisConfig *cfg) {
cfg->SetModel(FLAGS_infer_model + "/model", FLAGS_infer_model + "/params");
cfg->DisableGpu();
cfg->SwitchIrOptim();
cfg->SwitchSpecifyInputNames();
cfg->SetCpuMathLibraryNumThreads(FLAGS_cpu_num_threads);
cfg->DeletePass("constant_folding_pass");
}
void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
SetFakeImageInput(inputs, FLAGS_infer_model);
}
// Easy for profiling independently.
void profile(bool use_onednn = false) {
AnalysisConfig cfg;
SetConfig(&cfg);
if (use_onednn) {
cfg.EnableONEDNN();
if (FLAGS_disable_onednn_fc) {
cfg.DisableOnednnFcPasses();
}
}
std::vector<std::vector<PaddleTensor>> outputs;
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all,
&outputs,
FLAGS_num_threads);
}
TEST(Analyzer_resnet50, profile) { profile(); }
#ifdef PADDLE_WITH_DNNL
TEST(Analyzer_resnet50, profile_onednn) { profile(true /* use_onednn */); }
#endif
// Compare result of NativeConfig and AnalysisConfig
void compare(bool use_onednn = false) {
AnalysisConfig cfg;
SetConfig(&cfg);
if (use_onednn) {
cfg.EnableONEDNN();
if (FLAGS_disable_onednn_fc) {
cfg.DisableOnednnFcPasses();
}
}
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareNativeAndAnalysis(
reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
}
TEST(Analyzer_resnet50, compare) { compare(); }
#ifdef PADDLE_WITH_DNNL
TEST(Analyzer_resnet50, compare_onednn) { compare(true /* use_onednn */); }
#endif
// Compare Deterministic result
TEST(Analyzer_resnet50, compare_determine) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all);
}
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_lac_tester.cc
+191
View File
@@ -0,0 +1,191 @@
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
namespace analysis {
struct DataRecord {
std::vector<int64_t> data;
std::vector<size_t> lod;
// for dataset and nextbatch
size_t batch_iter{0};
std::vector<std::vector<size_t>> batched_lods;
std::vector<std::vector<int64_t>> batched_datas;
std::vector<std::vector<int64_t>> datasets;
DataRecord() : data(), lod(), batched_lods(), batched_datas(), datasets() {}
explicit DataRecord(const std::string &path, int batch_size = 1)
: data(), lod(), batched_lods(), batched_datas(), datasets() {
Load(path);
Prepare(batch_size);
batch_iter = 0;
}
void Load(const std::string &path) {
std::ifstream file(path);
std::string line;
int num_lines = 0;
datasets.resize(0);
while (std::getline(file, line)) {
num_lines++;
std::vector<std::string> data;
split(line, ';', &data);
std::vector<int64_t> words_ids;
split_to_int64(data[1], ' ', &words_ids);
datasets.emplace_back(words_ids);
}
}
void Prepare(int bs) {
if (bs == 1) {
batched_datas = datasets;
for (auto one_sentence : datasets) {
batched_lods.push_back({0, one_sentence.size()});
}
} else {
std::vector<int64_t> one_batch;
std::vector<size_t> lod{0};
int bs_id = 0;
for (auto one_sentence : datasets) {
bs_id++;
one_batch.insert(
one_batch.end(), one_sentence.begin(), one_sentence.end());
lod.push_back(lod.back() + one_sentence.size());
if (bs_id == bs) {
bs_id = 0;
batched_datas.push_back(one_batch);
batched_lods.push_back(lod);
one_batch.clear();
one_batch.resize(0);
lod.clear();
lod.resize(0);
lod.push_back(0);
}
}
if (!one_batch.empty()) {
batched_datas.push_back(one_batch);
batched_lods.push_back(lod);
}
}
}
DataRecord NextBatch() {
DataRecord data;
data.data = batched_datas[batch_iter];
data.lod = batched_lods[batch_iter];
batch_iter++;
if (batch_iter >= batched_datas.size()) {
batch_iter = 0;
}
return data;
}
};
void GetOneBatch(std::vector<PaddleTensor> *input_slots,
DataRecord *data,
int batch_size) {
auto one_batch = data->NextBatch();
PaddleTensor input_tensor;
input_tensor.name = "word";
input_tensor.dtype = PaddleDType::INT64;
TensorAssignData<int64_t>(&input_tensor, {one_batch.data}, one_batch.lod);
PADDLE_ENFORCE_EQ(
batch_size,
static_cast<int>(one_batch.lod.size() - 1),
::common::errors::Fatal("The lod size of one batch is invalid."));
input_slots->assign({input_tensor});
}
void SetConfig(AnalysisConfig *cfg) {
cfg->SetModel(FLAGS_infer_model);
cfg->DisableGpu();
cfg->SwitchSpecifyInputNames();
cfg->SwitchIrOptim();
}
void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
DataRecord data(FLAGS_infer_data, FLAGS_batch_size);
std::vector<PaddleTensor> input_slots;
int epoch = FLAGS_test_all_data ? data.batched_datas.size() : 1;
LOG(INFO) << "number of samples: " << epoch;
for (int bid = 0; bid < epoch; ++bid) {
GetOneBatch(&input_slots, &data, FLAGS_batch_size);
(*inputs).emplace_back(input_slots);
}
}
// Easy for profiling independently.
TEST(Analyzer_LAC, profile) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> outputs;
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all,
&outputs,
FLAGS_num_threads);
if (FLAGS_num_threads == 1 && !FLAGS_test_all_data) {
// the first inference result
const std::array<int64_t, 47> lac_ref_data = {
24, 25, 25, 25, 38, 30, 31, 14, 15, 44, 24, 25, 25, 25, 25, 25,
44, 24, 25, 25, 25, 36, 42, 43, 44, 14, 15, 44, 14, 15, 44, 14,
15, 44, 38, 39, 14, 15, 44, 22, 23, 23, 23, 23, 23, 23, 23};
PADDLE_ENFORCE_GT(outputs.size(),
0,
::common::errors::Fatal(
"The size of output should be greater than 0."));
auto output = outputs.back();
PADDLE_ENFORCE_EQ(
output.size(),
1UL,
::common::errors::Fatal("The size of output should be equal to 1."));
size_t size = GetSize(output[0]);
size_t batch1_size = sizeof(lac_ref_data) / sizeof(int64_t);
PADDLE_ENFORCE_GE(size,
batch1_size,
::common::errors::Fatal("The size of batch is invalid."));
int64_t *pdata = static_cast<int64_t *>(output[0].data.data());
for (size_t i = 0; i < batch1_size; ++i) {
EXPECT_EQ(pdata[i], lac_ref_data[i]);
}
}
}
// Compare result of NativeConfig and AnalysisConfig
TEST(Analyzer_LAC, compare) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareNativeAndAnalysis(
reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
}
// Compare Deterministic result
TEST(Analyzer_LAC, compare_determine) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all);
}
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_mmp_tester.cc
+115
View File
@@ -0,0 +1,115 @@
// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <random>
#include "paddle/fluid/framework/transfer_scope_cache.h"
#include "test/cpp/inference/api/tester_helper.h"
// Here add missing commands
PD_DEFINE_string(infer_model2, "", "model path");
PD_DEFINE_string(infer_model3, "", "model path");
namespace paddle {
namespace inference {
// Shape of Input to models
const int N = 1, C = 3, H = 224, W = 224;
void SetConfig(AnalysisConfig* config, const std::string& infer_model) {
config->SetModel(infer_model + "/__model__", infer_model + "/__params__");
config->DisableFCPadding();
config->SwitchSpecifyInputNames(true);
}
std::unique_ptr<PaddlePredictor> InitializePredictor(
const std::string& infer_model,
const std::vector<float>& data,
bool use_onednn) {
AnalysisConfig cfg;
SetConfig(&cfg, infer_model);
if (use_onednn) {
cfg.EnableONEDNN();
}
auto predictor = ::paddle::CreatePaddlePredictor<AnalysisConfig>(cfg);
auto input_name = predictor->GetInputNames()[0];
auto input = predictor->GetInputTensor(input_name);
std::vector<int> shape{N, C, H, W};
input->Reshape(std::move(shape));
input->copy_from_cpu(data.data());
return predictor;
}
// Compare result of NativeConfig and AnalysisConfig
void compare(bool use_onednn = false) {
// Create Input to models
std::vector<float> data(N * C * H * W);
std::default_random_engine re{1234};
std::uniform_real_distribution<float> sampler{0.0, 1.0};
for (auto& v : data) {
v = sampler(re);
}
// Initialize Models predictors
auto predictor_1 = InitializePredictor(FLAGS_infer_model, data, use_onednn);
auto predictor_xx = InitializePredictor(FLAGS_infer_model2, data, use_onednn);
auto predictor_3 = InitializePredictor(FLAGS_infer_model3, data, use_onednn);
// Run single xx model
predictor_xx->ZeroCopyRun();
auto output =
predictor_xx->GetOutputTensor(predictor_xx->GetOutputNames()[0]);
auto output_shape = output->shape();
int numel = std::accumulate(
output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
std::vector<float> xx_output(numel);
output->copy_to_cpu(xx_output.data());
// Initialize xx model's predictor to trigger oneDNN cache clearing
predictor_xx = InitializePredictor(FLAGS_infer_model2, data, use_onednn);
// Run sequence of models
predictor_1->ZeroCopyRun();
predictor_xx->ZeroCopyRun();
predictor_3->ZeroCopyRun();
// Get again output of xx model , but when all three models were executed
std::vector<float> xx2_output(numel);
output = predictor_xx->GetOutputTensor(predictor_xx->GetOutputNames()[0]);
output->copy_to_cpu(xx2_output.data());
// compare results
auto result = std::equal(
xx_output.begin(),
xx_output.end(),
xx2_output.begin(),
[](const float& l, const float& r) { return fabs(l - r) < 1e-4; });
PADDLE_ENFORCE_EQ(
result,
true,
::common::errors::Fatal("Results of model run independently "
"differs from results of the same model "
"run as a sequence of models"));
}
TEST(Analyzer_mmp, compare) { compare(); }
#ifdef PADDLE_WITH_DNNL
TEST(Analyzer_mmp, compare_onednn) { compare(true /* use_onednn */); }
#endif
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_ner_tester.cc
+175
View File
@@ -0,0 +1,175 @@
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
struct DataRecord {
std::vector<std::vector<int64_t>> word, mention;
std::vector<size_t> lod; // two inputs have the same lod info.
size_t batch_iter{0}, batch_size{1}, num_samples; // total number of samples
DataRecord() : word(), mention(), lod(), num_samples(0) {}
explicit DataRecord(const std::string &path, int batch_size = 1)
: word(), mention(), lod(), batch_size(batch_size), num_samples(0) {
Load(path);
}
DataRecord NextBatch() {
DataRecord data;
size_t batch_end = batch_iter + batch_size;
// NOTE skip the final batch, if no enough data is provided.
if (batch_end <= word.size()) {
GetInputPerBatch(word, &data.word, &data.lod, batch_iter, batch_end);
GetInputPerBatch(
mention, &data.mention, &data.lod, batch_iter, batch_end);
}
batch_iter += batch_size;
return data;
}
void Load(const std::string &path) {
std::ifstream file(path);
std::string line;
int num_lines = 0;
while (std::getline(file, line)) {
num_lines++;
std::vector<std::string> data;
split(line, ';', &data);
// load word data
std::vector<int64_t> word_data;
split_to_int64(data[1], ' ', &word_data);
// load mention data
std::vector<int64_t> mention_data;
split_to_int64(data[3], ' ', &mention_data);
word.push_back(std::move(word_data));
mention.push_back(std::move(mention_data));
}
num_samples = num_lines;
}
};
void PrepareInputs(std::vector<PaddleTensor> *input_slots, DataRecord *data) {
PaddleTensor lod_word_tensor, lod_mention_tensor;
lod_word_tensor.name = "word";
lod_mention_tensor.name = "mention";
auto one_batch = data->NextBatch();
// assign data
TensorAssignData<int64_t>(&lod_word_tensor, one_batch.word, one_batch.lod);
TensorAssignData<int64_t>(
&lod_mention_tensor, one_batch.mention, one_batch.lod);
// Set inputs.
input_slots->assign({lod_word_tensor, lod_mention_tensor});
for (auto &tensor : *input_slots) {
tensor.dtype = PaddleDType::INT64;
}
}
void SetConfig(AnalysisConfig *cfg, bool memory_load = false) {
if (memory_load) {
std::string buffer_prog, buffer_param;
ReadBinaryFile(FLAGS_infer_model + "/__model__", &buffer_prog);
ReadBinaryFile(FLAGS_infer_model + "/param", &buffer_param);
cfg->SetModelBuffer(&buffer_prog[0],
buffer_prog.size(),
&buffer_param[0],
buffer_param.size());
} else {
cfg->SetModel(FLAGS_infer_model + "/__model__",
FLAGS_infer_model + "/param");
}
cfg->DisableGpu();
cfg->SwitchSpecifyInputNames();
cfg->SwitchIrOptim();
}
void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
DataRecord data(FLAGS_infer_data, FLAGS_batch_size);
std::vector<PaddleTensor> input_slots;
int epoch =
FLAGS_test_all_data ? data.num_samples / FLAGS_batch_size : 1; // NOLINT
LOG(INFO) << "number of samples: " << epoch * FLAGS_batch_size;
for (int bid = 0; bid < epoch; ++bid) {
PrepareInputs(&input_slots, &data);
(*inputs).emplace_back(input_slots);
}
}
// Easy for profiling independently.
void profile(bool memory_load = false) {
AnalysisConfig cfg;
SetConfig(&cfg, memory_load);
std::vector<std::vector<PaddleTensor>> outputs;
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all,
&outputs,
FLAGS_num_threads);
if (FLAGS_num_threads == 1 && !FLAGS_test_all_data) {
// the first inference result
const std::array<int, 11> chinese_ner_result_data = {
30, 45, 41, 48, 17, 26, 48, 39, 38, 16, 25};
PADDLE_ENFORCE_GT(
outputs.size(),
0,
common::errors::Fatal("The size of output should be greater than 0."));
auto output = outputs.back();
PADDLE_ENFORCE_EQ(
output.size(),
1UL,
common::errors::Fatal("The size of output should be equal to 1."));
size_t size = GetSize(output[0]);
PADDLE_ENFORCE_GT(
size,
0,
common::errors::Fatal("The size of output should be greater than 0."));
int64_t *result = static_cast<int64_t *>(output[0].data.data());
for (size_t i = 0; i < std::min<size_t>(11, size); i++) {
EXPECT_EQ(result[i], chinese_ner_result_data[i]);
}
}
}
TEST(Analyzer_Chinese_ner, profile) { profile(); }
TEST(Analyzer_Chinese_ner, profile_memory_load) {
profile(true /* memory_load */);
}
// Compare result of NativeConfig and AnalysisConfig
TEST(Analyzer_Chinese_ner, compare) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareNativeAndAnalysis(
reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
}
// Compare Deterministic result
TEST(Analyzer_Chinese_ner, compare_determine) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all);
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_openvino_tester.cc
+72
View File
@@ -0,0 +1,72 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <fstream>
#include <iostream>
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
namespace analysis {
void SetConfig(AnalysisConfig *cfg) {
cfg->SetModel(FLAGS_infer_model + "/inference.pdmodel",
FLAGS_infer_model + "/inference.pdiparams");
cfg->DisableGpu();
cfg->SwitchIrOptim();
cfg->EnableOpenVINOEngine(AnalysisConfig::Precision::kFloat32);
if (cfg->openvino_engine_enabled()) {
cfg->SetCpuMathLibraryNumThreads(FLAGS_cpu_num_threads);
}
}
void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
SetFakeImageInput(inputs,
FLAGS_infer_model,
true,
"inference.pdmodel",
"inference.pdiparams");
}
// Easy for profiling independently.
void profile() {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> outputs;
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all,
&outputs,
FLAGS_num_threads);
}
TEST(Analyzer_openvino_resnet50, profile) { profile(); }
#ifdef PADDLE_WITH_OPENVINO
TEST(Analyzer_openvino_resnet50, compare_determine) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all);
}
#endif
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_pyramid_dnn_tester.cc
+205
View File
@@ -0,0 +1,205 @@
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
struct DataRecord {
std::vector<std::vector<int64_t>> query_basic, query_phrase, title_basic,
title_phrase;
std::vector<size_t> lod1, lod2, lod3, lod4;
size_t batch_iter{0}, batch_size{1}, num_samples; // total number of samples
DataRecord() = default;
explicit DataRecord(const std::string &path, int batch_size = 1)
: batch_size(batch_size) {
Load(path);
}
DataRecord NextBatch() {
DataRecord data;
size_t batch_end = batch_iter + batch_size;
// NOTE skip the final batch, if no enough data is provided.
if (batch_end <= query_basic.size()) {
GetInputPerBatch(
query_basic, &data.query_basic, &data.lod1, batch_iter, batch_end);
GetInputPerBatch(
query_phrase, &data.query_phrase, &data.lod2, batch_iter, batch_end);
GetInputPerBatch(
title_basic, &data.title_basic, &data.lod3, batch_iter, batch_end);
GetInputPerBatch(
title_phrase, &data.title_phrase, &data.lod4, batch_iter, batch_end);
}
batch_iter += batch_size;
return data;
}
void Load(const std::string &path) {
std::ifstream file(path);
std::string line;
int num_lines = 0;
while (std::getline(file, line)) {
std::vector<std::string> data;
split(line, ';', &data);
// load query data
std::vector<int64_t> query_basic_data;
split_to_int64(data[1], ' ', &query_basic_data);
std::vector<int64_t> query_phrase_data;
split_to_int64(data[2], ' ', &query_phrase_data);
// load title data
std::vector<int64_t> title_basic_data;
split_to_int64(data[3], ' ', &title_basic_data);
std::vector<int64_t> title_phrase_data;
split_to_int64(data[4], ' ', &title_phrase_data);
// filter the empty data
bool flag =
data[1].size() && data[2].size() && data[3].size() && data[4].size();
if (flag) {
query_basic.push_back(std::move(query_basic_data));
query_phrase.push_back(std::move(query_phrase_data));
title_basic.push_back(std::move(title_basic_data));
title_phrase.push_back(std::move(title_phrase_data));
num_lines++;
}
}
num_samples = num_lines;
}
};
void PrepareInputs(std::vector<PaddleTensor> *input_slots,
DataRecord *data,
int batch_size) {
PaddleTensor query_basic_tensor, query_phrase_tensor, title_basic_tensor,
title_phrase_tensor;
query_basic_tensor.name = "query_basic";
query_phrase_tensor.name = "query_phrase";
title_basic_tensor.name = "pos_title_basic";
title_phrase_tensor.name = "pos_title_phrase";
auto one_batch = data->NextBatch();
// assign data
TensorAssignData<int64_t>(
&query_basic_tensor, one_batch.query_basic, one_batch.lod1);
TensorAssignData<int64_t>(
&query_phrase_tensor, one_batch.query_phrase, one_batch.lod2);
TensorAssignData<int64_t>(
&title_basic_tensor, one_batch.title_basic, one_batch.lod3);
TensorAssignData<int64_t>(
&title_phrase_tensor, one_batch.title_phrase, one_batch.lod4);
// Set inputs.
input_slots->assign({query_basic_tensor,
query_phrase_tensor,
title_basic_tensor,
title_phrase_tensor});
for (auto &tensor : *input_slots) {
tensor.dtype = PaddleDType::INT64;
}
}
void SetConfig(AnalysisConfig *cfg) {
cfg->SetModel(FLAGS_infer_model);
cfg->DisableGpu();
cfg->SwitchSpecifyInputNames();
cfg->SwitchIrOptim();
cfg->SetCpuMathLibraryNumThreads(FLAGS_cpu_num_threads);
}
void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
DataRecord data(FLAGS_infer_data, FLAGS_batch_size);
std::vector<PaddleTensor> input_slots;
int epoch = FLAGS_test_all_data ? data.num_samples / FLAGS_batch_size : 1;
LOG(INFO) << "number of samples: " << epoch * FLAGS_batch_size;
for (int bid = 0; bid < epoch; ++bid) {
PrepareInputs(&input_slots, &data, FLAGS_batch_size);
(*inputs).emplace_back(input_slots);
}
}
// Easy for profiling independently.
TEST(Analyzer_Pyramid_DNN, profile) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> outputs;
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all,
&outputs,
FLAGS_num_threads);
if (FLAGS_num_threads == 1 && !FLAGS_test_all_data && !FLAGS_zero_copy) {
PADDLE_ENFORCE_GT(
outputs.size(),
0,
common::errors::Fatal("The size of output should be greater than 0."));
auto output = outputs.back();
PADDLE_ENFORCE_EQ(
output.size(),
1UL,
common::errors::Fatal("The size of output should be equal to 1."));
size_t size = GetSize(output[0]);
PADDLE_ENFORCE_GT(
size,
0,
common::errors::Fatal("The size of output should be greater than 0."));
float *result = static_cast<float *>(output[0].data.data());
// output is probability, which is in (0, 1).
for (size_t i = 0; i < size; i++) {
EXPECT_GT(result[i], 0);
EXPECT_LT(result[i], 1);
}
}
}
// Compare result of NativeConfig and AnalysisConfig
TEST(Analyzer_Pyramid_DNN, compare) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareNativeAndAnalysis(
reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
}
// Compare result of AnalysisConfig and AnalysisConfig + ZeroCopy
TEST(Analyzer_Pyramid_DNN, compare_zero_copy) {
AnalysisConfig cfg;
SetConfig(&cfg);
AnalysisConfig cfg1;
SetConfig(&cfg1);
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
std::vector<std::string> outputs_name;
outputs_name.emplace_back("cos_sim_2.tmp_0");
CompareAnalysisAndZeroCopy(reinterpret_cast<PaddlePredictor::Config *>(&cfg),
reinterpret_cast<PaddlePredictor::Config *>(&cfg1),
input_slots_all,
outputs_name);
}
// Compare Deterministic result
TEST(Analyzer_Pyramid_DNN, compare_determine) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all);
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_quant_image_classification_tester.cc
+143
View File
@@ -0,0 +1,143 @@
/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <fstream>
#include <iostream>
#include "paddle/fluid/inference/api/paddle_analysis_config.h"
#include "test/cpp/inference/api/tester_helper.h"
PD_DEFINE_bool(enable_onednn, true, "Enable ONEDNN");
namespace paddle {
namespace inference {
namespace analysis {
void SetConfig(AnalysisConfig *cfg, std::string model_path) {
cfg->SetModel(model_path);
cfg->DisableGpu();
cfg->SwitchIrOptim(true);
cfg->SetCpuMathLibraryNumThreads(FLAGS_cpu_num_threads);
cfg->EnableNewIR();
cfg->EnableNewExecutor();
cfg->SetOptimizationLevel(3);
if (FLAGS_enable_onednn) cfg->EnableONEDNN();
}
template <typename T>
class TensorReader {
public:
TensorReader(std::ifstream &file,
size_t beginning_offset,
std::vector<int> shape,
std::string name)
: file_(file),
position_(beginning_offset),
shape_(shape),
name_(name),
numel_(0) {
numel_ = std::accumulate(
shape_.begin(), shape_.end(), size_t{1}, std::multiplies<size_t>());
}
PaddleTensor NextBatch() {
PaddleTensor tensor;
tensor.name = name_;
tensor.shape = shape_;
tensor.dtype = GetPaddleDType<T>();
tensor.data.Resize(numel_ * sizeof(T));
file_.seekg(position_);
file_.read(static_cast<char *>(tensor.data.data()), numel_ * sizeof(T));
position_ = file_.tellg();
if (file_.eof()) LOG(ERROR) << name_ << ": reached end of stream";
if (file_.fail())
throw std::runtime_error(name_ + ": failed reading file.");
return tensor;
}
protected:
std::ifstream &file_;
size_t position_;
std::vector<int> shape_;
std::string name_;
size_t numel_;
};
void SetInput(std::vector<std::vector<PaddleTensor>> *inputs,
bool with_accuracy_layer = FLAGS_with_accuracy_layer,
int32_t batch_size = FLAGS_batch_size) {
std::ifstream file(FLAGS_infer_data, std::ios::binary);
if (!file) {
FAIL() << "Couldn't open file: " << FLAGS_infer_data;
}
int64_t total_images{0};
file.read(reinterpret_cast<char *>(&total_images), sizeof(total_images));
LOG(INFO) << "Total images in file: " << total_images;
std::vector<int> image_batch_shape{batch_size, 3, 224, 224};
std::vector<int> label_batch_shape{batch_size, 1};
auto images_offset_in_file = static_cast<size_t>(file.tellg());
TensorReader<float> image_reader(
file, images_offset_in_file, image_batch_shape, "image");
auto iterations_max = total_images / batch_size;
auto iterations = iterations_max;
if (FLAGS_iterations > 0 && FLAGS_iterations < iterations_max) {
iterations = FLAGS_iterations;
}
auto labels_offset_in_file =
images_offset_in_file + sizeof(float) * total_images * 3 * 224 * 224;
TensorReader<int64_t> label_reader(
file, labels_offset_in_file, label_batch_shape, "label");
for (auto i = 0; i < iterations; i++) {
auto images = image_reader.NextBatch();
std::vector<PaddleTensor> tmp_vec;
tmp_vec.push_back(std::move(images));
if (with_accuracy_layer) {
auto labels = label_reader.NextBatch();
tmp_vec.push_back(std::move(labels));
}
inputs->push_back(std::move(tmp_vec));
}
}
TEST(Analyzer_quant_image_classification, quantization) {
AnalysisConfig fp32_cfg;
SetConfig(&fp32_cfg, FLAGS_fp32_model);
fp32_cfg.EnableONEDNN();
AnalysisConfig int8_cfg;
SetConfig(&int8_cfg, FLAGS_int8_model);
if (FLAGS_enable_int8_qat) int8_cfg.EnableOnednnInt8();
// read data from file and prepare batches with test data
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
// 0 is avg_cost, 1 is top1_accuracy, 2 is top5_accuracy or mAP
CompareAnalysisAndAnalysis(
&fp32_cfg, &int8_cfg, input_slots_all, FLAGS_with_accuracy_layer, 1);
}
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_rnn1_tester.cc
+325
View File
@@ -0,0 +1,325 @@
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "test/cpp/inference/api/tester_helper.h"
#include "paddle/common/enforce.h"
PD_DEFINE_bool(with_precision_check, true, "turn on test");
namespace paddle {
namespace inference {
using namespace framework; // NOLINT
struct DataRecord {
std::vector<std::vector<std::vector<float>>> link_step_data_all;
std::vector<std::vector<float>> week_data_all, minute_data_all;
std::vector<size_t> lod1, lod2, lod3;
std::vector<std::vector<float>> rnn_link_data, rnn_week_datas,
rnn_minute_datas;
size_t num_samples; // total number of samples
size_t batch_iter{0};
size_t batch_size{1};
DataRecord() = default;
explicit DataRecord(const std::string &path, int batch_size = 1)
: batch_size(batch_size) {
Load(path);
}
DataRecord NextBatch() {
DataRecord data;
size_t batch_end = batch_iter + batch_size;
// NOTE skip the final batch, if no enough data is provided.
if (batch_end <= link_step_data_all.size()) {
data.link_step_data_all.assign(link_step_data_all.begin() + batch_iter,
link_step_data_all.begin() + batch_end);
data.week_data_all.assign(week_data_all.begin() + batch_iter,
week_data_all.begin() + batch_end);
data.minute_data_all.assign(minute_data_all.begin() + batch_iter,
minute_data_all.begin() + batch_end);
// Prepare LoDs
data.lod1.push_back(0);
data.lod2.push_back(0);
data.lod3.push_back(0);
PADDLE_ENFORCE_EQ(!data.link_step_data_all.empty(),
true,
common::errors::InvalidArgument(
"link_step_data_all should not be empty."));
PADDLE_ENFORCE_EQ(!data.week_data_all.empty(),
true,
common::errors::InvalidArgument(
"week_data_all should not be empty."));
PADDLE_ENFORCE_EQ(!data.minute_data_all.empty(),
true,
common::errors::InvalidArgument(
"minute_data_all should not be empty."));
PADDLE_ENFORCE_EQ(
data.link_step_data_all.size(),
data.week_data_all.size(),
platform::errors::InvalidArgument(
"The value of data.link_step_data_all.size() is not equal to the "
"value of data.week_data_all.size()."))
PADDLE_ENFORCE_EQ(
data.minute_data_all.size(),
data.link_step_data_all.size(),
platform::errors::InvalidArgument(
"The value of data.minute_data_all.size() is not equal to the "
"value of data.link_step_data_all.size()."))
for (size_t j = 0; j < data.link_step_data_all.size(); j++) {
for (const auto &d : data.link_step_data_all[j]) {
data.rnn_link_data.push_back(d);
}
data.rnn_week_datas.push_back(data.week_data_all[j]);
data.rnn_minute_datas.push_back(data.minute_data_all[j]);
// calculate lod
data.lod1.push_back(data.lod1.back() +
data.link_step_data_all[j].size());
data.lod3.push_back(data.lod3.back() + 1);
for (size_t i = 1; i < data.link_step_data_all[j].size() + 1; i++) {
data.lod2.push_back(data.lod2.back() +
data.link_step_data_all[j].size());
}
}
}
batch_iter += batch_size;
return data;
}
void Load(const std::string &path) {
std::ifstream file(path);
std::string line;
int num_lines = 0;
while (std::getline(file, line)) {
num_lines++;
std::vector<std::string> data;
split(line, ':', &data);
std::vector<std::vector<float>> link_step_data;
std::vector<std::string> link_datas;
split(data[0], '|', &link_datas);
for (auto &step_data : link_datas) {
std::vector<float> tmp;
split_to_float(step_data, ',', &tmp);
link_step_data.push_back(tmp);
}
// load week data
std::vector<float> week_data;
split_to_float(data[2], ',', &week_data);
// load minute data
std::vector<float> minute_data;
split_to_float(data[1], ',', &minute_data);
link_step_data_all.push_back(std::move(link_step_data));
week_data_all.push_back(std::move(week_data));
minute_data_all.push_back(std::move(minute_data));
}
num_samples = num_lines;
}
};
void PrepareInputs(std::vector<PaddleTensor> *input_slots,
DataRecord *data,
int batch_size) {
PaddleTensor lod_attention_tensor, init_zero_tensor, lod_tensor_tensor,
week_tensor, minute_tensor;
lod_attention_tensor.name = "data_lod_attention";
init_zero_tensor.name = "cell_init";
lod_tensor_tensor.name = "data";
week_tensor.name = "week";
minute_tensor.name = "minute";
auto one_batch = data->NextBatch();
std::vector<int> rnn_link_data_shape(
{static_cast<int>(one_batch.rnn_link_data.size()),
static_cast<int>(one_batch.rnn_link_data.front().size())});
lod_attention_tensor.shape.assign({1, 2});
lod_attention_tensor.lod.assign({one_batch.lod1, one_batch.lod2});
init_zero_tensor.shape.assign({batch_size, 15});
init_zero_tensor.lod.assign({one_batch.lod3});
lod_tensor_tensor.shape = rnn_link_data_shape;
lod_tensor_tensor.lod.assign({one_batch.lod1});
week_tensor.shape.assign(
{static_cast<int>(one_batch.rnn_week_datas.size()),
static_cast<int>(one_batch.rnn_week_datas.front().size())});
week_tensor.lod.assign({one_batch.lod3});
minute_tensor.shape.assign(
{static_cast<int>(one_batch.rnn_minute_datas.size()),
static_cast<int>(one_batch.rnn_minute_datas.front().size())});
minute_tensor.lod.assign({one_batch.lod3});
// assign data
TensorAssignData<float>(&lod_attention_tensor,
std::vector<std::vector<float>>({{0, 0}}));
std::vector<float> tmp_zeros(batch_size * 15, 0.);
TensorAssignData<float>(&init_zero_tensor, {tmp_zeros});
TensorAssignData<float>(&lod_tensor_tensor, one_batch.rnn_link_data);
TensorAssignData<float>(&week_tensor, one_batch.rnn_week_datas);
TensorAssignData<float>(&minute_tensor, one_batch.rnn_minute_datas);
// Set inputs.
auto init_zero_tensor1 = init_zero_tensor;
init_zero_tensor1.name = "hidden_init";
input_slots->assign({week_tensor,
init_zero_tensor,
minute_tensor,
init_zero_tensor1,
lod_attention_tensor,
lod_tensor_tensor});
for (auto &tensor : *input_slots) {
tensor.dtype = PaddleDType::FLOAT32;
}
}
void PrepareZeroCopyInputs(ZeroCopyTensor *lod_attention_tensor,
ZeroCopyTensor *cell_init_tensor,
ZeroCopyTensor *data_tensor,
ZeroCopyTensor *hidden_init_tensor,
ZeroCopyTensor *week_tensor,
ZeroCopyTensor *minute_tensor,
DataRecord *data_record,
int batch_size) {
auto one_batch = data_record->NextBatch();
std::vector<int> rnn_link_data_shape(
{static_cast<int>(one_batch.rnn_link_data.size()),
static_cast<int>(one_batch.rnn_link_data.front().size())});
lod_attention_tensor->Reshape({1, 2});
lod_attention_tensor->SetLoD({one_batch.lod1, one_batch.lod2});
cell_init_tensor->Reshape({batch_size, 15});
cell_init_tensor->SetLoD({one_batch.lod3});
hidden_init_tensor->Reshape({batch_size, 15});
hidden_init_tensor->SetLoD({one_batch.lod3});
data_tensor->Reshape(rnn_link_data_shape);
data_tensor->SetLoD({one_batch.lod1});
week_tensor->Reshape(
{static_cast<int>(one_batch.rnn_week_datas.size()),
static_cast<int>(one_batch.rnn_week_datas.front().size())});
week_tensor->SetLoD({one_batch.lod3});
minute_tensor->Reshape(
{static_cast<int>(one_batch.rnn_minute_datas.size()),
static_cast<int>(one_batch.rnn_minute_datas.front().size())});
minute_tensor->SetLoD({one_batch.lod3});
// assign data
std::array<float, 2> arr0 = {0, 0};
std::vector<float> zeros(batch_size * 15, 0);
std::copy_n(arr0.data(),
2,
lod_attention_tensor->mutable_data<float>(PaddlePlace::kCPU));
std::copy_n(
arr0.data(), 2, data_tensor->mutable_data<float>(PaddlePlace::kCPU));
std::copy_n(zeros.begin(),
zeros.size(),
cell_init_tensor->mutable_data<float>(PaddlePlace::kCPU));
std::copy_n(zeros.begin(),
zeros.size(),
hidden_init_tensor->mutable_data<float>(PaddlePlace::kCPU));
ZeroCopyTensorAssignData(data_tensor, one_batch.rnn_link_data);
ZeroCopyTensorAssignData(week_tensor, one_batch.rnn_week_datas);
ZeroCopyTensorAssignData(minute_tensor, one_batch.rnn_minute_datas);
}
void SetConfig(AnalysisConfig *cfg) {
cfg->SetModel(FLAGS_infer_model + "/__model__", FLAGS_infer_model + "/param");
cfg->DisableGpu();
cfg->SwitchSpecifyInputNames();
cfg->SwitchIrOptim();
}
void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
DataRecord data(FLAGS_infer_data, FLAGS_batch_size);
std::vector<PaddleTensor> input_slots;
int epoch =
FLAGS_test_all_data ? data.num_samples / FLAGS_batch_size : 1; // NOLINT
LOG(INFO) << "number of samples: " << epoch * FLAGS_batch_size;
for (int bid = 0; bid < epoch; ++bid) {
PrepareInputs(&input_slots, &data, FLAGS_batch_size);
(*inputs).emplace_back(input_slots);
}
}
// Easy for profiling independently.
TEST(Analyzer_rnn1, profile) {
AnalysisConfig cfg;
SetConfig(&cfg);
cfg.DisableGpu();
cfg.SwitchIrDebug();
std::vector<std::vector<PaddleTensor>> outputs;
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all,
&outputs,
FLAGS_num_threads);
}
// Compare result of NativeConfig and AnalysisConfig
TEST(Analyzer_rnn1, compare) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareNativeAndAnalysis(
reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
}
// Compare Deterministic result
TEST(Analyzer_rnn1, compare_determine) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all);
}
// Test Multi-Thread.
TEST(Analyzer_rnn1, multi_thread) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> outputs;
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all,
&outputs,
2 /* multi_thread */);
}
// Compare result of AnalysisConfig and AnalysisConfig + ZeroCopy
TEST(Analyzer_rnn1, compare_zero_copy) {
AnalysisConfig cfg;
SetConfig(&cfg);
AnalysisConfig cfg1;
SetConfig(&cfg1);
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
std::vector<std::string> outputs_name;
outputs_name.emplace_back("final_output.tmp_1");
CompareAnalysisAndZeroCopy(reinterpret_cast<PaddlePredictor::Config *>(&cfg),
reinterpret_cast<PaddlePredictor::Config *>(&cfg1),
input_slots_all,
outputs_name);
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_rnn2_tester.cc
+194
View File
@@ -0,0 +1,194 @@
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
using namespace framework; // NOLINT
static std::vector<float> result_data;
struct DataRecord {
std::vector<std::vector<std::vector<float>>> link_step_data_all;
std::vector<size_t> lod;
std::vector<std::vector<float>> rnn_link_data;
size_t num_samples; // total number of samples
size_t batch_iter{0};
size_t batch_size{1};
DataRecord() : link_step_data_all(), lod(), rnn_link_data(), num_samples(0) {}
explicit DataRecord(const std::string &path, int batch_size = 1)
: link_step_data_all(),
lod(),
rnn_link_data(),
num_samples(0),
batch_size(batch_size) {
Load(path);
}
DataRecord NextBatch() {
DataRecord data;
size_t batch_end = batch_iter + batch_size;
// NOTE skip the final batch, if no enough data is provided.
if (batch_end <= link_step_data_all.size()) {
data.link_step_data_all.assign(link_step_data_all.begin() + batch_iter,
link_step_data_all.begin() + batch_end);
// Prepare LoDs
data.lod.push_back(0);
PADDLE_ENFORCE_EQ(
!data.link_step_data_all.empty(),
true,
common::errors::InvalidArgument(
"`data.link_step_data_all` is empty, please check"));
for (size_t j = 0; j < data.link_step_data_all.size(); j++) {
for (const auto &d : data.link_step_data_all[j]) {
data.rnn_link_data.push_back(d);
// calculate lod
data.lod.push_back(data.lod.back() + 11);
}
}
}
batch_iter += batch_size;
return data;
}
void Load(const std::string &path) {
std::ifstream file(path);
std::string line;
int num_lines = 0;
result_data.clear();
while (std::getline(file, line)) {
num_lines++;
std::vector<std::string> data;
split(line, ':', &data);
if (num_lines % 2) { // feature
std::vector<std::string> feature_data;
split(data[1], ' ', &feature_data);
std::vector<std::vector<float>> link_step_data;
int feature_count = 1;
std::vector<float> feature;
for (auto &step_data : feature_data) {
std::vector<float> tmp;
split_to_float(step_data, ',', &tmp);
feature.insert(feature.end(), tmp.begin(), tmp.end());
if (feature_count % 11 == 0) { // each sample has 11 features
link_step_data.push_back(feature);
feature.clear();
}
feature_count++;
}
link_step_data_all.push_back(std::move(link_step_data));
} else { // result
std::vector<float> tmp;
split_to_float(data[1], ',', &tmp);
result_data.insert(result_data.end(), tmp.begin(), tmp.end());
}
}
num_samples = num_lines / 2;
}
};
void PrepareInputs(std::vector<PaddleTensor> *input_slots,
DataRecord *data,
int batch_size) {
PaddleTensor feed_tensor;
feed_tensor.name = "feed";
auto one_batch = data->NextBatch();
int token_size = one_batch.rnn_link_data.size();
// each token has 11 features, each feature's dim is 54.
std::vector<int> rnn_link_data_shape({token_size * 11, 54});
feed_tensor.shape = rnn_link_data_shape;
feed_tensor.lod.assign({one_batch.lod});
feed_tensor.dtype = PaddleDType::FLOAT32;
TensorAssignData<float>(&feed_tensor, one_batch.rnn_link_data);
// Set inputs.
input_slots->assign({feed_tensor});
}
void SetConfig(AnalysisConfig *cfg) {
cfg->SetModel(FLAGS_infer_model + "/__model__", FLAGS_infer_model + "/param");
cfg->DisableGpu();
cfg->SwitchSpecifyInputNames();
cfg->SwitchIrOptim();
}
void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
DataRecord data(FLAGS_infer_data, FLAGS_batch_size);
std::vector<PaddleTensor> input_slots;
int epoch =
FLAGS_test_all_data ? data.num_samples / FLAGS_batch_size : 1; // NOLINT
LOG(INFO) << "number of samples: " << epoch * FLAGS_batch_size;
for (int bid = 0; bid < epoch; ++bid) {
PrepareInputs(&input_slots, &data, FLAGS_batch_size);
(*inputs).emplace_back(input_slots);
}
}
// Easy for profiling independently.
TEST(Analyzer_rnn2, profile) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> outputs;
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all,
&outputs,
FLAGS_num_threads);
if (FLAGS_num_threads == 1 && !FLAGS_test_all_data) {
// the first inference result
PADDLE_ENFORCE_GT(
outputs.size(),
0,
common::errors::Fatal("The size of output should be greater than 0."));
auto output = outputs.back();
PADDLE_ENFORCE_GT(
output.size(),
0,
common::errors::Fatal("The size of output should be greater than 0."));
size_t size = GetSize(output[0]);
PADDLE_ENFORCE_GT(
size,
0,
common::errors::Fatal("The size of output should be greater than 0."));
float *result = static_cast<float *>(output[0].data.data());
for (size_t i = 0; i < size; i++) {
EXPECT_NEAR(result[i], result_data[i], 1e-3);
}
}
}
// Compare result of NativeConfig and AnalysisConfig
TEST(Analyzer_rnn2, compare) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareNativeAndAnalysis(
reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
}
// Compare Deterministic result
TEST(Analyzer_rnn2, compare_determine) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all);
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_seq_conv1_tester.cc
+194
View File
@@ -0,0 +1,194 @@
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
struct DataRecord {
std::vector<std::vector<int64_t>> title1, title2, title3, l1;
std::vector<size_t> lod1, lod2, lod3, l1_lod;
size_t batch_iter{0}, batch_size{1}, num_samples; // total number of samples
DataRecord()
: title1(),
title2(),
title3(),
l1(),
lod1(),
lod2(),
lod3(),
l1_lod(),
batch_size(0),
num_samples(0) {}
explicit DataRecord(const std::string &path, int batch_size = 1)
: title1(),
title2(),
title3(),
l1(),
lod1(),
lod2(),
lod3(),
l1_lod(),
batch_size(batch_size),
num_samples(0) {
Load(path);
}
DataRecord NextBatch() {
DataRecord data;
size_t batch_end = batch_iter + batch_size;
// NOTE skip the final batch, if no enough data is provided.
if (batch_end <= title1.size()) {
GetInputPerBatch(title1, &data.title1, &data.lod1, batch_iter, batch_end);
GetInputPerBatch(title2, &data.title2, &data.lod2, batch_iter, batch_end);
GetInputPerBatch(title3, &data.title3, &data.lod3, batch_iter, batch_end);
GetInputPerBatch(l1, &data.l1, &data.l1_lod, batch_iter, batch_end);
}
batch_iter += batch_size;
return data;
}
void Load(const std::string &path) {
std::ifstream file(path);
std::string line;
int num_lines = 0;
while (std::getline(file, line)) {
num_lines++;
std::vector<std::string> data;
split(line, '\t', &data);
PADDLE_ENFORCE_GT(data.size(),
4,
common::errors::Fatal("The size of data is invalid."));
// load title1 data
std::vector<int64_t> title1_data;
split_to_int64(data[0], ' ', &title1_data);
// load title2 data
std::vector<int64_t> title2_data;
split_to_int64(data[1], ' ', &title2_data);
// load title3 data
std::vector<int64_t> title3_data;
split_to_int64(data[2], ' ', &title3_data);
// load l1 data
std::vector<int64_t> l1_data;
split_to_int64(data[3], ' ', &l1_data);
title1.push_back(std::move(title1_data));
title2.push_back(std::move(title2_data));
title3.push_back(std::move(title3_data));
l1.push_back(std::move(l1_data));
}
num_samples = num_lines;
}
};
void PrepareInputs(std::vector<PaddleTensor> *input_slots,
DataRecord *data,
int batch_size) {
PaddleTensor title1_tensor, title2_tensor, title3_tensor, l1_tensor;
title1_tensor.name = "title1";
title2_tensor.name = "title2";
title3_tensor.name = "title3";
l1_tensor.name = "l1";
auto one_batch = data->NextBatch();
// assign data
TensorAssignData<int64_t>(&title1_tensor, one_batch.title1, one_batch.lod1);
TensorAssignData<int64_t>(&title2_tensor, one_batch.title2, one_batch.lod2);
TensorAssignData<int64_t>(&title3_tensor, one_batch.title3, one_batch.lod3);
TensorAssignData<int64_t>(&l1_tensor, one_batch.l1, one_batch.l1_lod);
// Set inputs.
input_slots->assign({title1_tensor, title2_tensor, title3_tensor, l1_tensor});
for (auto &tensor : *input_slots) {
tensor.dtype = PaddleDType::INT64;
}
}
void SetConfig(AnalysisConfig *cfg) {
cfg->SetModel(FLAGS_infer_model);
cfg->DisableGpu();
cfg->SwitchSpecifyInputNames();
cfg->SwitchIrOptim();
}
void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
DataRecord data(FLAGS_infer_data, FLAGS_batch_size);
std::vector<PaddleTensor> input_slots;
int epoch =
FLAGS_test_all_data ? data.num_samples / FLAGS_batch_size : 1; // NOLINT
LOG(INFO) << "number of samples: " << epoch * FLAGS_batch_size;
for (int bid = 0; bid < epoch; ++bid) {
PrepareInputs(&input_slots, &data, FLAGS_batch_size);
(*inputs).emplace_back(input_slots);
}
}
// Easy for profiling independently.
TEST(Analyzer_seq_conv1, profile) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> outputs;
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all,
&outputs,
FLAGS_num_threads);
if (FLAGS_num_threads == 1 && !FLAGS_test_all_data) {
// the first inference result
PADDLE_ENFORCE_GT(
outputs.size(),
0,
common::errors::Fatal("The size of output should be greater than 0."));
auto output = outputs.back();
PADDLE_ENFORCE_EQ(
output.size(),
1UL,
common::errors::Fatal("The size of output should be equal to 0."));
size_t size = GetSize(output[0]);
PADDLE_ENFORCE_GT(
size,
0,
common::errors::Fatal("The size of output should be greater than 0."));
float *result = static_cast<float *>(output[0].data.data());
// output is probability, which is in (0, 1).
for (size_t i = 0; i < size; i++) {
EXPECT_GT(result[i], 0);
EXPECT_LT(result[i], 1);
}
}
}
// Compare result of NativeConfig and AnalysisConfig
TEST(Analyzer_seq_conv1, compare) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareNativeAndAnalysis(
reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
}
// Compare Deterministic result
TEST(Analyzer_seq_conv1, compare_determine) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all);
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_seq_pool1_compare_determine_tester.cc
+41
View File
@@ -0,0 +1,41 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <algorithm>
#include <fstream>
#include <iostream>
#include "test/cpp/inference/api/analyzer_seq_pool1_tester_helper.h"
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
namespace analysis {
namespace seq_pool1_tester {
// Compare Deterministic result
TEST(Analyzer_seq_pool1_compare_determine, compare_determine) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all);
}
} // namespace seq_pool1_tester
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_seq_pool1_compare_tester.cc
+40
View File
@@ -0,0 +1,40 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <algorithm>
#include <fstream>
#include <iostream>
#include "test/cpp/inference/api/analyzer_seq_pool1_tester_helper.h"
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
namespace analysis {
namespace seq_pool1_tester {
TEST(Analyzer_seq_pool1_compare, compare) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareNativeAndAnalysis(
reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
}
} // namespace seq_pool1_tester
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_seq_pool1_fuse_compare_zero_copy_tester.cc
+48
View File
@@ -0,0 +1,48 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <algorithm>
#include <fstream>
#include <iostream>
#include "test/cpp/inference/api/analyzer_seq_pool1_tester_helper.h"
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
namespace analysis {
namespace seq_pool1_tester {
// Compare result of AnalysisConfig and AnalysisConfig + ZeroCopy
TEST(Analyzer_seq_pool1_compare_zero_copy, compare_zero_copy) {
AnalysisConfig cfg;
SetConfig(&cfg);
AnalysisConfig cfg1;
SetConfig(&cfg1);
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
std::vector<std::string> outputs_name;
outputs_name.emplace_back(out_var_name);
CompareAnalysisAndZeroCopy(reinterpret_cast<PaddlePredictor::Config *>(&cfg),
reinterpret_cast<PaddlePredictor::Config *>(&cfg1),
input_slots_all,
outputs_name);
}
} // namespace seq_pool1_tester
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_seq_pool1_profile_tester.cc
+45
View File
@@ -0,0 +1,45 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <algorithm>
#include <fstream>
#include <iostream>
#include "test/cpp/inference/api/analyzer_seq_pool1_tester_helper.h"
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
namespace analysis {
namespace seq_pool1_tester {
void profile(bool use_onednn = false) {
AnalysisConfig cfg;
SetConfig(&cfg, use_onednn);
std::vector<std::vector<PaddleTensor>> outputs;
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all,
&outputs,
FLAGS_num_threads);
}
TEST(Analyzer_seq_pool1_profile, profile) { profile(); }
} // namespace seq_pool1_tester
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_seq_pool1_tester_helper.h
+178
View File
@@ -0,0 +1,178 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#pragma once
#include <algorithm>
#include <fstream>
#include <iostream>
#include <map>
#include <string>
#include <utility>
#include <vector>
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
namespace analysis {
namespace seq_pool1_tester {
// diff: similarity_norm.tmp_0, for speed: fc_4.tmp_1
static const char out_var_name[] = "reduce_sum_0.tmp_0";
// for diff: 154, for speed 111
constexpr int num_slots = 154;
struct OneSlotInBatch {
std::string name;
std::vector<std::vector<float>> data;
std::vector<int> shape;
std::vector<size_t> lod;
};
struct DataRecord {
std::vector<std::vector<OneSlotInBatch>> batched_data;
std::map<std::string, std::vector<std::vector<float>>> datasets;
size_t batch_iter{0}, num_samples; // total number of samples
DataRecord() = default;
explicit DataRecord(const std::string &path, int batch_size = 1) {
Load(path);
Prepare(batch_size);
}
void Load(const std::string &path) {
std::ifstream file(path);
std::string line;
int num_lines = 0;
while (std::getline(file, line)) {
num_lines++;
std::vector<std::string> data;
split(line, '\t', &data);
std::vector<float> slot_data;
split_to_float(data[1], ' ', &slot_data);
std::string name = data[0];
PADDLE_ENFORCE_EQ(
slot_data.size() % 11,
0UL,
::common::errors::Fatal(
"line %d, %s should be divisible", num_lines, name));
datasets[name].emplace_back(std::move(slot_data));
}
num_samples = num_lines / num_slots;
PADDLE_ENFORCE_EQ(
num_samples * num_slots,
static_cast<size_t>(num_lines),
::common::errors::Fatal("num samples should be divisible"));
PADDLE_ENFORCE_GT(num_samples,
0UL,
::common::errors::Fatal(
"The num of samples should be greater than 0."));
}
void Prepare(int bs) {
for (auto it = datasets.begin(); it != datasets.end(); ++it) {
PADDLE_ENFORCE_EQ(
it->second.size(),
num_samples,
::common::errors::Fatal("size of each slot should be equal"));
}
size_t num_batches = num_samples / bs;
EXPECT_GT(num_batches, 0UL);
batched_data.resize(num_batches);
for (auto &one_batch : batched_data) {
one_batch.resize(datasets.size());
size_t i = 0;
for (auto it = datasets.begin(); it != datasets.end(); ++it) {
auto &slot = one_batch[i];
slot.name = it->first;
slot.data.resize(bs);
slot.lod.resize(bs + 1);
slot.lod[0] = 0;
auto &lod = slot.lod;
auto &datas = it->second;
for (int k = 0; k < bs; ++k) {
size_t id = k + batch_iter * bs;
std::copy(datas[id].begin(),
datas[id].end(),
std::back_inserter(slot.data[k]));
size_t len = datas[id].size() / 11;
PADDLE_ENFORCE_EQ(
len * 11,
datas[id].size(),
::common::errors::Fatal(
"%s %d size should be divisible", slot.name, id));
lod[k + 1] = lod[k] + len;
}
slot.shape.assign({static_cast<int>(lod[bs]), 11});
i++;
}
}
}
const std::vector<OneSlotInBatch> &NextBatch() {
if (batch_iter >= batched_data.size() - 1) {
batch_iter = -1;
}
return batched_data[++batch_iter];
}
};
static void TensorAssignSlot(PaddleTensor *tensor, const OneSlotInBatch &slot) {
tensor->name = slot.name + "_embed";
tensor->shape = slot.shape;
tensor->dtype = PaddleDType::FLOAT32;
tensor->lod.clear();
tensor->lod.emplace_back(slot.lod);
TensorAssignData(tensor, slot.data);
}
void PrepareInputs(std::vector<PaddleTensor> *input_slots, DataRecord *data) {
const auto &one_batch = data->NextBatch();
input_slots->resize(one_batch.size());
for (size_t i = 0; i < one_batch.size(); ++i) {
auto &slot = one_batch[i];
TensorAssignSlot(&((*input_slots)[i]), slot);
}
}
void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
DataRecord data(FLAGS_infer_data, FLAGS_batch_size);
std::vector<PaddleTensor> input_slots;
int epoch = FLAGS_test_all_data ? data.batched_data.size() : 1;
LOG(INFO) << "number of samples: "
<< data.batched_data.size() * FLAGS_batch_size;
for (int bid = 0; bid < epoch; ++bid) {
PrepareInputs(&input_slots, &data);
(*inputs).emplace_back(input_slots);
}
}
void SetConfig(AnalysisConfig *cfg, bool use_onednn = false) {
cfg->SetModel(FLAGS_infer_model + "/model", FLAGS_infer_model + "/params");
cfg->DisableGpu();
cfg->SwitchSpecifyInputNames();
cfg->SwitchIrDebug();
cfg->SetCpuMathLibraryNumThreads(FLAGS_cpu_num_threads);
if (use_onednn) {
cfg->EnableONEDNN();
}
// Enable seqpool_concat_fuse_pass, disabled by default since it takes much
// time
cfg->pass_builder()->InsertPass(2, "seqpool_concat_fuse_pass");
}
} // namespace seq_pool1_tester
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_text_classification_tester.cc
+161
View File
@@ -0,0 +1,161 @@
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
struct DataReader {
explicit DataReader(const std::string &path)
: file(new std::ifstream(path)) {}
bool NextBatch(std::vector<PaddleTensor> *input, int batch_size) {
PADDLE_ENFORCE_EQ(
batch_size,
1,
common::errors::Fatal("The size of batch should be equal to 1."));
std::string line;
PaddleTensor tensor;
tensor.dtype = PaddleDType::INT64;
tensor.lod.emplace_back(std::vector<size_t>({0}));
std::vector<int64_t> data;
for (int i = 0; i < batch_size; i++) {
if (!std::getline(*file, line)) return false;
inference::split_to_int64(line, ' ', &data);
}
tensor.lod.front().push_back(data.size());
tensor.data.Resize(data.size() * sizeof(int64_t));
PADDLE_ENFORCE_NE(
tensor.data.data(),
nullptr,
common::errors::Fatal("Variable `tensor.data.data()` is nullptr"));
PADDLE_ENFORCE_NE(
data.data(),
nullptr,
common::errors::Fatal("Variable `data.data()` is nullptr"));
memcpy(tensor.data.data(), data.data(), data.size() * sizeof(int64_t));
tensor.shape.push_back(data.size());
tensor.shape.push_back(1);
input->assign({tensor});
return true;
}
std::unique_ptr<std::ifstream> file = nullptr;
};
void SetConfig(AnalysisConfig *cfg) {
cfg->SetModel(FLAGS_infer_model);
cfg->DisableGpu();
cfg->SwitchSpecifyInputNames();
cfg->SwitchIrOptim();
}
void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
std::vector<PaddleTensor> input_slots;
DataReader reader(FLAGS_infer_data);
int num_batches = 0;
while (reader.NextBatch(&input_slots, FLAGS_batch_size)) {
(*inputs).emplace_back(input_slots);
++num_batches;
if (!FLAGS_test_all_data) return;
}
LOG(INFO) << "total number of samples: " << num_batches * FLAGS_batch_size;
}
// Easy for profiling independently.
TEST(Analyzer_Text_Classification, profile) {
AnalysisConfig cfg;
SetConfig(&cfg);
cfg.SwitchIrDebug();
std::vector<std::vector<PaddleTensor>> outputs;
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all,
&outputs,
FLAGS_num_threads);
if (FLAGS_num_threads == 1) {
// Get output
PADDLE_ENFORCE_GT(
outputs.size(),
0,
common::errors::Fatal("The size of output should be greater than 0."));
LOG(INFO) << "get outputs " << outputs.back().size();
for (auto &output : outputs.back()) {
LOG(INFO) << "output.shape: " << to_string(output.shape);
// no lod ?
PADDLE_ENFORCE_EQ(
output.lod.size(),
0UL,
common::errors::InvalidArgument(
"The 'lod' size of 'output' should be 0, but received size %d.",
output.lod.size()));
LOG(INFO) << "output.dtype: " << output.dtype;
std::stringstream ss;
int num_data = 1;
for (auto i : output.shape) {
num_data *= i;
}
for (int i = 0; i < num_data; i++) {
ss << static_cast<float *>(output.data.data())[i] << " ";
}
LOG(INFO) << "output.data summary: " << ss.str();
// one batch ends
}
}
}
// Compare result of NativeConfig and AnalysisConfig
TEST(Analyzer_Text_Classification, compare) {
AnalysisConfig cfg;
SetConfig(&cfg);
cfg.EnableMemoryOptim();
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareNativeAndAnalysis(
reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
}
// Compare Deterministic result
TEST(Analyzer_Text_Classification, compare_determine) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all);
}
TEST(Analyzer_Text_Classification, compare_against_embedding_fc_lstm_fused) {
AnalysisConfig cfg;
SetConfig(&cfg);
// Enable embedding_fc_lstm_fuse_pass (disabled by default)
cfg.pass_builder()->InsertPass(2, "embedding_fc_lstm_fuse_pass");
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareNativeAndAnalysis(
reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_transformer_tester_helper.h
+203
View File
@@ -0,0 +1,203 @@
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <string>
#include <utility>
#include <vector>
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
namespace analysis {
namespace transformer_tester {
struct DataRecord {
std::vector<std::vector<int64_t>> src_word, src_pos, trg_word, init_idx;
std::vector<std::vector<float>> src_slf_attn_bias, init_score,
trg_src_attn_bias;
std::vector<std::vector<int32_t>> batch_data_shape;
std::vector<std::vector<size_t>> lod;
size_t batch_iter{0}, batch_size{1}, num_samples; // total number of samples
DataRecord() = default;
explicit DataRecord(const std::string &path, int batch_size = 1)
: batch_size(batch_size) {
Load(path);
}
DataRecord NextBatch() {
DataRecord data;
size_t batch_end = batch_iter + batch_size;
// NOTE skip the final batch, if no enough data is provided.
if (batch_end <= src_word.size()) {
data.src_word.assign(src_word.begin() + batch_iter,
src_word.begin() + batch_end);
data.src_pos.assign(src_pos.begin() + batch_iter,
src_pos.begin() + batch_end);
data.src_slf_attn_bias.assign(src_slf_attn_bias.begin() + batch_iter,
src_slf_attn_bias.begin() + batch_end);
data.trg_word.assign(trg_word.begin() + batch_iter,
trg_word.begin() + batch_end);
data.init_score.assign(init_score.begin() + batch_iter,
init_score.begin() + batch_end);
data.init_idx.assign(init_idx.begin() + batch_iter,
init_idx.begin() + batch_end);
data.trg_src_attn_bias.assign(trg_src_attn_bias.begin() + batch_iter,
trg_src_attn_bias.begin() + batch_end);
std::vector<int32_t> batch_shape =
*(batch_data_shape.begin() + batch_iter);
data.batch_data_shape.push_back(batch_shape);
data.lod.resize(2);
for (int i = 0; i < batch_shape[0] + 1; i++) {
data.lod[0].push_back(i);
data.lod[1].push_back(i);
}
}
batch_iter += batch_size;
return data;
}
void Load(const std::string &path) {
std::ifstream file(path);
std::string line;
size_t num_lines = 0;
while (std::getline(file, line)) {
num_lines++;
std::vector<std::string> data;
split(line, ',', &data);
PADDLE_ENFORCE_EQ(data.size(),
static_cast<size_t>(8),
common::errors::InvalidArgument(
"The size of data should be equal to 8. "));
// load src_word
std::vector<int64_t> src_word_data;
split_to_int64(data[0], ' ', &src_word_data);
src_word.push_back(std::move(src_word_data));
// load src_pos
std::vector<int64_t> src_pos_data;
split_to_int64(data[1], ' ', &src_pos_data);
src_pos.push_back(std::move(src_pos_data));
// load src_slf_attn_bias
std::vector<float> src_slf_attn_bias_data;
split_to_float(data[2], ' ', &src_slf_attn_bias_data);
src_slf_attn_bias.push_back(std::move(src_slf_attn_bias_data));
// load trg_word
std::vector<int64_t> trg_word_data;
split_to_int64(data[3], ' ', &trg_word_data);
trg_word.push_back(std::move(trg_word_data));
// load init_score
std::vector<float> init_score_data;
split_to_float(data[4], ' ', &init_score_data);
init_score.push_back(std::move(init_score_data));
// load init_idx
std::vector<int64_t> init_idx_data;
split_to_int64(data[5], ' ', &init_idx_data);
init_idx.push_back(std::move(init_idx_data));
// load trg_src_attn_bias
std::vector<float> trg_src_attn_bias_data;
split_to_float(data[6], ' ', &trg_src_attn_bias_data);
trg_src_attn_bias.push_back(std::move(trg_src_attn_bias_data));
// load shape for variant data shape
std::vector<int> batch_data_shape_data;
split_to_int(data[7], ' ', &batch_data_shape_data);
batch_data_shape.push_back(std::move(batch_data_shape_data));
}
num_samples = num_lines;
}
};
void PrepareInputs(std::vector<PaddleTensor> *input_slots,
DataRecord *data,
int batch_size) {
auto one_batch = data->NextBatch();
batch_size = one_batch.batch_data_shape[0][0];
auto n_head = one_batch.batch_data_shape[0][1];
auto trg_seq_len = one_batch.batch_data_shape[0][2]; // 1 for inference
auto src_seq_len = one_batch.batch_data_shape[0][3];
PaddleTensor src_word, src_pos, src_slf_attn_bias, trg_word, init_score,
init_idx, trg_src_attn_bias;
src_word.name = "src_word";
src_word.shape.assign({batch_size, src_seq_len, 1});
src_word.dtype = PaddleDType::INT64;
TensorAssignData<int64_t>(&src_word, one_batch.src_word);
src_pos.name = "src_pos";
src_pos.shape.assign({batch_size, src_seq_len, 1});
src_pos.dtype = PaddleDType::INT64;
TensorAssignData<int64_t>(&src_pos, one_batch.src_pos);
src_slf_attn_bias.name = "src_slf_attn_bias";
src_slf_attn_bias.shape.assign(
{batch_size, n_head, src_seq_len, src_seq_len});
src_slf_attn_bias.dtype = PaddleDType::FLOAT32;
TensorAssignData<float>(&src_slf_attn_bias, one_batch.src_slf_attn_bias);
trg_word.name = "trg_word";
trg_word.shape.assign({batch_size, 1});
trg_word.dtype = PaddleDType::INT64;
trg_word.lod.assign(one_batch.lod.begin(), one_batch.lod.end());
TensorAssignData<int64_t>(&trg_word, one_batch.trg_word);
init_score.name = "init_score";
init_score.shape.assign({batch_size, 1});
init_score.dtype = PaddleDType::FLOAT32;
init_score.lod.assign(one_batch.lod.begin(), one_batch.lod.end());
TensorAssignData<float>(&init_score, one_batch.init_score);
init_idx.name = "init_idx";
init_idx.shape.assign({batch_size});
init_idx.dtype = PaddleDType::INT64;
TensorAssignData<int64_t>(&init_idx, one_batch.init_idx);
trg_src_attn_bias.name = "trg_src_attn_bias";
trg_src_attn_bias.shape.assign(
{batch_size, n_head, trg_seq_len, src_seq_len});
trg_src_attn_bias.dtype = PaddleDType::FLOAT32;
TensorAssignData<float>(&trg_src_attn_bias, one_batch.trg_src_attn_bias);
input_slots->assign({src_word,
src_pos,
src_slf_attn_bias,
trg_word,
init_score,
init_idx,
trg_src_attn_bias});
}
void SetConfig(AnalysisConfig *cfg) {
cfg->SetModel(FLAGS_infer_model + "/model", FLAGS_infer_model + "/params");
cfg->DisableGpu();
cfg->SwitchSpecifyInputNames();
cfg->SwitchIrOptim();
cfg->SetCpuMathLibraryNumThreads(FLAGS_cpu_num_threads);
}
void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
DataRecord data(FLAGS_infer_data, FLAGS_batch_size);
std::vector<PaddleTensor> input_slots;
int test_batch_num =
FLAGS_test_all_data ? data.num_samples / FLAGS_batch_size : 1;
LOG(INFO) << "The number of samples to be test: "
<< test_batch_num * FLAGS_batch_size;
for (int bid = 0; bid < test_batch_num; ++bid) {
input_slots.clear();
PrepareInputs(&input_slots, &data, FLAGS_batch_size);
(*inputs).emplace_back(input_slots);
}
}
} // namespace transformer_tester
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_vis_tester.cc
+168
View File
@@ -0,0 +1,168 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <gtest/gtest.h>
#include <fstream>
#include <iostream>
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
namespace analysis {
struct Record {
std::vector<float> data;
std::vector<int32_t> shape;
Record() : data(), shape() {}
};
Record ProcessALine(const std::string &line) {
VLOG(3) << "process a line";
std::vector<std::string> columns;
split(line, '\t', &columns);
PADDLE_ENFORCE_EQ(columns.size(),
2UL,
common::errors::InvalidArgument(
"data format error, should be <data>\t<shape>"));
Record record;
std::vector<std::string> data_strs;
split(columns[0], ' ', &data_strs);
for (auto &d : data_strs) {
record.data.push_back(std::stof(d));
}
std::vector<std::string> shape_strs;
split(columns[1], ' ', &shape_strs);
for (auto &s : shape_strs) {
record.shape.push_back(std::stoi(s));
}
VLOG(3) << "data size " << record.data.size();
VLOG(3) << "data shape size " << record.shape.size();
return record;
}
void SetConfig(AnalysisConfig *cfg) {
cfg->SetModel(FLAGS_infer_model + "/__model__",
FLAGS_infer_model + "/__params__");
cfg->DisableGpu();
cfg->SwitchIrDebug();
cfg->SwitchSpecifyInputNames(false);
}
void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
PADDLE_ENFORCE_EQ(FLAGS_test_all_data,
0,
::common::errors::Fatal("Only have single batch of data."));
std::string line;
std::ifstream file(FLAGS_infer_data);
std::getline(file, line);
auto record = ProcessALine(line);
PaddleTensor input;
input.shape = record.shape;
input.dtype = PaddleDType::FLOAT32;
size_t input_size = record.data.size() * sizeof(float);
input.data.Resize(input_size);
memcpy(input.data.data(), record.data.data(), input_size);
std::vector<PaddleTensor> input_slots;
input_slots.assign({input});
(*inputs).emplace_back(input_slots);
}
// Easy for profiling independently.
// ocr, mobilenet and se_resnext50
void profile(bool use_onednn = false) {
AnalysisConfig cfg;
SetConfig(&cfg);
if (use_onednn) {
cfg.EnableONEDNN();
}
// cfg.pass_builder()->TurnOnDebug();
std::vector<std::vector<PaddleTensor>> outputs;
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all,
&outputs,
FLAGS_num_threads);
if (FLAGS_num_threads == 1 && !FLAGS_test_all_data) {
std::string line;
std::ifstream file(FLAGS_refer_result);
std::getline(file, line);
auto refer = ProcessALine(line);
file.close();
PADDLE_ENFORCE_GT(outputs.size(),
0,
::common::errors::Fatal(
"The size of output should be greater than 0."));
auto &output = outputs.back().front();
size_t numel = output.data.length() / PaddleDtypeSize(output.dtype);
PADDLE_ENFORCE_EQ(
numel,
refer.data.size(),
common::errors::InvalidArgument(
"value of numel is wrong, expected %d but received %d",
refer.data.size(),
numel));
for (size_t i = 0; i < numel; ++i) {
EXPECT_NEAR(
static_cast<float *>(output.data.data())[i], refer.data[i], 1e-5);
}
}
}
TEST(Analyzer_vis, profile) { profile(); }
#ifdef PADDLE_WITH_DNNL
TEST(Analyzer_vis, profile_onednn) { profile(true /* use_onednn */); }
#endif
// Compare result of NativeConfig and AnalysisConfig
void compare(bool use_onednn = false) {
AnalysisConfig cfg;
SetConfig(&cfg);
if (use_onednn) {
cfg.EnableONEDNN();
}
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareNativeAndAnalysis(
reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
}
TEST(Analyzer_vis, compare) { compare(); }
#ifdef PADDLE_WITH_DNNL
TEST(Analyzer_vis, compare_onednn) { compare(true /* use_onednn */); }
#endif
// Compare Deterministic result
TEST(Analyzer_vis, compare_determine) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all);
}
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_vit_ocr_tester.cc
+100
View File
@@ -0,0 +1,100 @@
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <fstream>
#include <iostream>
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
namespace analysis {
struct Record {
std::vector<float> data;
std::vector<int32_t> shape;
Record() : data(), shape() {}
};
Record ProcessALine(const std::string &line) {
std::vector<std::string> columns;
split(line, '\t', &columns);
PADDLE_ENFORCE_EQ(columns.size(),
2UL,
common::errors::InvalidArgument(
"Data format is invalid, should be <data>\t<shape>"));
Record record;
std::vector<std::string> data_strs;
split(columns[0], ' ', &data_strs);
for (auto &d : data_strs) {
record.data.push_back(std::stof(d));
}
std::vector<std::string> shape_strs;
split(columns[1], ' ', &shape_strs);
for (auto &s : shape_strs) {
record.shape.push_back(std::stoi(s));
}
return record;
}
void SetInput(std::vector<std::vector<PaddleTensor>> *inputs) {
std::string line;
std::ifstream file(FLAGS_infer_data);
std::getline(file, line);
auto record = ProcessALine(line);
PaddleTensor input;
input.shape = record.shape;
input.dtype = PaddleDType::FLOAT32;
size_t input_size = record.data.size() * sizeof(float);
input.data.Resize(input_size);
memcpy(input.data.data(), record.data.data(), input_size);
std::vector<PaddleTensor> input_slots;
input_slots.assign({input});
(*inputs).emplace_back(input_slots);
}
void SetConfig(AnalysisConfig *cfg, bool use_onednn = false) {
cfg->SetModel(FLAGS_infer_model + "/inference.pdmodel",
FLAGS_infer_model + "/inference.pdiparams");
if (use_onednn) {
cfg->EnableONEDNN();
cfg->SwitchIrOptim();
}
}
// Compare results of NativeConfig and AnalysisConfig
void compare(bool use_onednn = false) {
AnalysisConfig cfg;
SetConfig(&cfg, use_onednn);
std::vector<std::vector<PaddleTensor>> input_slots_all;
SetInput(&input_slots_all);
CompareNativeAndAnalysis(
reinterpret_cast<const PaddlePredictor::Config *>(&cfg), input_slots_all);
}
TEST(Analyzer_vit_ocr, compare) { compare(); }
#ifdef PADDLE_WITH_DNNL
TEST(Analyzer_vit_ocr, compare_onednn) { compare(true /* use_onednn */); }
#endif
} // namespace analysis
} // namespace inference
} // namespace paddle
test/cpp/inference/api/analyzer_zerocopy_tensor_tester.cc
+63
View File
@@ -0,0 +1,63 @@
// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/fluid/framework/block_desc.h"
#include "paddle/fluid/framework/op_desc.h"
#include "paddle/fluid/framework/program_desc.h"
#include "paddle/fluid/framework/scope.h"
#include "paddle/fluid/inference/utils/singleton.h"
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
TEST(test_zerocopy_tensor, zerocopy_tensor) {
AnalysisConfig config;
config.SetModel(FLAGS_infer_model + "/inference.pdmodel",
FLAGS_infer_model + "/inference.pdiparams");
auto predictor = CreatePaddlePredictor(config);
int batch_size = 1;
int channels = 3;
int height = 224;
int width = 224;
int nums = batch_size * channels * height * width;
float* input = new float[nums];
for (int i = 0; i < nums; ++i) input[i] = 0;
auto input_names = predictor->GetInputNames();
PaddlePlace p = PaddlePlace::kCPU;
PaddlePlace* place = &p;
int size;
auto input_t = predictor->GetInputTensor(input_names[0]);
input_t->Reshape({batch_size, channels, height, width});
input_t->copy_from_cpu<float>(input);
input_t->data<float>(place, &size);
input_t->mutable_data<float>(p);
predictor->ZeroCopyRun();
std::vector<float> out_data;
auto output_names = predictor->GetOutputNames();
auto output_t = predictor->GetOutputTensor(output_names[0]);
std::vector<int> output_shape = output_t->shape();
int out_num = std::accumulate(
output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
out_data.resize(out_num);
output_t->copy_to_cpu<float>(out_data.data());
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/api_impl_tester.cc
+343
View File
@@ -0,0 +1,343 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <thread> // NOLINT
#include "paddle/common/flags.h"
#include "paddle/fluid/framework/convert_utils.h"
#include "paddle/fluid/inference/api/api_impl.h"
#include "test/cpp/inference/test_helper.h"
#ifdef __clang__
#define ACC_DIFF 4e-3
#else
#define ACC_DIFF 2e-3
#endif
PD_DEFINE_string(word2vec_dirname,
"",
"Directory of the word2vec inference model.");
PD_DEFINE_string(book_dirname, "", "Directory of the book inference model.");
namespace paddle {
PaddleTensor LodTensorToPaddleTensor(phi::DenseTensor* t) {
PaddleTensor pt;
if (framework::TransToProtoVarType(t->dtype()) ==
framework::proto::VarType::INT64) {
pt.data.Reset(t->data(), t->numel() * sizeof(int64_t));
pt.dtype = PaddleDType::INT64;
} else if (framework::TransToProtoVarType(t->dtype()) ==
framework::proto::VarType::FP32) {
pt.data.Reset(t->data(), t->numel() * sizeof(float));
pt.dtype = PaddleDType::FLOAT32;
} else if (framework::TransToProtoVarType(t->dtype()) ==
framework::proto::VarType::INT32) {
pt.data.Reset(t->data(), t->numel() * sizeof(int32_t));
pt.dtype = PaddleDType::INT32;
} else {
PADDLE_THROW(common::errors::Unimplemented(
"Unsupported tensor date type. Now only supports INT64, FP32, INT32."));
}
pt.shape = common::vectorize<int>(t->dims());
return pt;
}
NativeConfig GetConfig() {
NativeConfig config;
config.model_dir = FLAGS_word2vec_dirname;
LOG(INFO) << "dirname " << config.model_dir;
config.fraction_of_gpu_memory = 0.15;
config.device = 0;
return config;
}
void MainWord2Vec(const ::paddle::PaddlePlace& place) {
NativeConfig config = GetConfig();
auto predictor = CreatePaddlePredictor<NativeConfig>(config);
config.use_gpu = ::paddle::gpu_place_used(place);
config.use_xpu = ::paddle::xpu_place_used(place);
phi::DenseTensor first_word, second_word, third_word, fourth_word;
phi::LegacyLoD lod{{0, 1}};
int64_t dict_size = 2073; // The size of dictionary
SetupDenseTensor(&first_word, lod, static_cast<int64_t>(0), dict_size - 1);
SetupDenseTensor(&second_word, lod, static_cast<int64_t>(0), dict_size - 1);
SetupDenseTensor(&third_word, lod, static_cast<int64_t>(0), dict_size - 1);
SetupDenseTensor(&fourth_word, lod, static_cast<int64_t>(0), dict_size - 1);
std::vector<PaddleTensor> paddle_tensor_feeds;
paddle_tensor_feeds.push_back(LodTensorToPaddleTensor(&first_word));
paddle_tensor_feeds.push_back(LodTensorToPaddleTensor(&second_word));
paddle_tensor_feeds.push_back(LodTensorToPaddleTensor(&third_word));
paddle_tensor_feeds.push_back(LodTensorToPaddleTensor(&fourth_word));
std::vector<PaddleTensor> outputs;
ASSERT_TRUE(predictor->Run(paddle_tensor_feeds, &outputs));
ASSERT_EQ(outputs.size(), 1UL);
size_t len = outputs[0].data.length();
float* data = static_cast<float*>(outputs[0].data.data());
for (size_t j = 0; j < len / sizeof(float); ++j) {
ASSERT_LT(data[j], 1.0);
ASSERT_GT(data[j], -1.0);
}
std::vector<phi::DenseTensor*> cpu_feeds;
cpu_feeds.push_back(&first_word);
cpu_feeds.push_back(&second_word);
cpu_feeds.push_back(&third_word);
cpu_feeds.push_back(&fourth_word);
framework::FetchType output1;
std::vector<::paddle::framework::FetchType*> cpu_fetches1;
cpu_fetches1.push_back(&output1);
TestInference<phi::CPUPlace>(config.model_dir, cpu_feeds, cpu_fetches1);
auto output1_tensor = PADDLE_GET(phi::DenseTensor, output1);
float* lod_data = output1_tensor.data<float>();
for (int i = 0; i < output1_tensor.numel(); ++i) {
EXPECT_LT(lod_data[i] - data[i], ACC_DIFF);
EXPECT_GT(lod_data[i] - data[i], -ACC_DIFF);
}
}
void MainImageClassification(const ::paddle::PaddlePlace& place) {
int batch_size = 2;
bool repeat = false;
NativeConfig config = GetConfig();
config.use_gpu = ::paddle::gpu_place_used(place);
config.use_xpu = ::paddle::xpu_place_used(place);
config.model_dir =
FLAGS_book_dirname + "/image_classification_resnet.inference.model";
const bool is_combined = false;
std::vector<std::vector<int64_t>> feed_target_shapes =
GetFeedTargetShapes(config.model_dir, is_combined);
phi::DenseTensor input;
// Use normilized image pixels as input data,
// which should be in the range [0.0, 1.0].
feed_target_shapes[0][0] = batch_size;
phi::DDim input_dims = common::make_ddim(feed_target_shapes[0]);
SetupTensor<float>(
&input, input_dims, static_cast<float>(0), static_cast<float>(1));
std::vector<phi::DenseTensor*> cpu_feeds;
cpu_feeds.push_back(&input);
framework::FetchType output1;
std::vector<framework::FetchType*> cpu_fetches1;
cpu_fetches1.push_back(&output1);
TestInference<phi::CPUPlace, false, true>(
config.model_dir, cpu_feeds, cpu_fetches1, repeat, is_combined);
auto predictor = CreatePaddlePredictor(config);
std::vector<PaddleTensor> paddle_tensor_feeds;
paddle_tensor_feeds.push_back(LodTensorToPaddleTensor(&input));
std::vector<PaddleTensor> outputs;
ASSERT_TRUE(predictor->Run(paddle_tensor_feeds, &outputs));
ASSERT_EQ(outputs.size(), 1UL);
size_t len = outputs[0].data.length();
float* data = static_cast<float*>(outputs[0].data.data());
float* lod_data = PADDLE_GET(phi::DenseTensor, output1).data<float>();
for (size_t j = 0; j < len / sizeof(float); ++j) {
EXPECT_NEAR(lod_data[j], data[j], ACC_DIFF);
}
}
void MainThreadsWord2Vec(const ::paddle::PaddlePlace& place) {
NativeConfig config = GetConfig();
config.use_gpu = ::paddle::gpu_place_used(place);
config.use_xpu = ::paddle::xpu_place_used(place);
auto main_predictor = CreatePaddlePredictor<NativeConfig>(config);
// prepare inputs data and reference results
constexpr int num_jobs = 3;
std::vector<std::vector<phi::DenseTensor>> jobs(num_jobs);
std::vector<std::vector<PaddleTensor>> paddle_tensor_feeds(num_jobs);
std::vector<framework::FetchType> refs(num_jobs);
for (size_t i = 0; i < jobs.size(); ++i) {
// each job has 4 words
jobs[i].resize(4);
for (size_t j = 0; j < 4; ++j) {
phi::LegacyLoD lod{{0, 1}};
int64_t dict_size = 2073; // The size of dictionary
SetupDenseTensor(
&jobs[i][j], lod, static_cast<int64_t>(0), dict_size - 1);
paddle_tensor_feeds[i].push_back(LodTensorToPaddleTensor(&jobs[i][j]));
}
// get reference result of each job
std::vector<phi::DenseTensor*> ref_feeds;
std::vector<::paddle::framework::FetchType*> ref_fetches(1, &refs[i]);
for (auto& word : jobs[i]) {
ref_feeds.push_back(&word);
}
TestInference<phi::CPUPlace>(config.model_dir, ref_feeds, ref_fetches);
}
// create threads and each thread run 1 job
std::vector<std::thread> threads;
for (int tid = 0; tid < num_jobs; ++tid) {
threads.emplace_back([&, tid]() {
auto predictor = CreatePaddlePredictor(config);
auto& local_inputs = paddle_tensor_feeds[tid];
std::vector<PaddleTensor> local_outputs;
ASSERT_TRUE(predictor->Run(local_inputs, &local_outputs));
// check outputs range
ASSERT_EQ(local_outputs.size(), 1UL);
const size_t len = local_outputs[0].data.length();
float* data = static_cast<float*>(local_outputs[0].data.data());
for (size_t j = 0; j < len / sizeof(float); ++j) {
ASSERT_LT(data[j], 1.0);
ASSERT_GT(data[j], -1.0);
}
// check outputs correctness
auto ref_tensor = PADDLE_GET(phi::DenseTensor, refs[tid]);
float* ref_data = ref_tensor.data<float>();
EXPECT_EQ(ref_tensor.numel(), static_cast<int64_t>(len / sizeof(float)));
for (int i = 0; i < ref_tensor.numel(); ++i) {
EXPECT_NEAR(ref_data[i], data[i], 2e-3);
}
});
}
for (int i = 0; i < num_jobs; ++i) {
threads[i].join();
}
}
void MainThreadsImageClassification(const ::paddle::PaddlePlace& place) {
constexpr int num_jobs = 4; // each job run 1 batch
constexpr int batch_size = 1;
NativeConfig config = GetConfig();
config.use_gpu = ::paddle::gpu_place_used(place);
config.use_xpu = ::paddle::xpu_place_used(place);
config.model_dir =
FLAGS_book_dirname + "/image_classification_resnet.inference.model";
auto main_predictor = CreatePaddlePredictor<NativeConfig>(config);
std::vector<phi::DenseTensor> jobs(num_jobs);
std::vector<std::vector<PaddleTensor>> paddle_tensor_feeds(num_jobs);
std::vector<framework::FetchType> refs(num_jobs);
for (size_t i = 0; i < jobs.size(); ++i) {
// prepare inputs
std::vector<std::vector<int64_t>> feed_target_shapes =
GetFeedTargetShapes(config.model_dir, /*is_combined*/ false);
feed_target_shapes[0][0] = batch_size;
phi::DDim input_dims = common::make_ddim(feed_target_shapes[0]);
SetupTensor<float>(&jobs[i], input_dims, 0.f, 1.f);
paddle_tensor_feeds[i].push_back(LodTensorToPaddleTensor(&jobs[i]));
// get reference result of each job
std::vector<phi::DenseTensor*> ref_feeds(1, &jobs[i]);
std::vector<framework::FetchType*> ref_fetches(1, &refs[i]);
TestInference<phi::CPUPlace>(config.model_dir, ref_feeds, ref_fetches);
}
// create threads and each thread run 1 job
std::vector<std::thread> threads;
for (int tid = 0; tid < num_jobs; ++tid) {
threads.emplace_back([&, tid]() {
auto predictor = CreatePaddlePredictor(config);
auto& local_inputs = paddle_tensor_feeds[tid];
std::vector<PaddleTensor> local_outputs;
ASSERT_TRUE(predictor->Run(local_inputs, &local_outputs));
// check outputs correctness
ASSERT_EQ(local_outputs.size(), 1UL);
const size_t len = local_outputs[0].data.length();
float* data = static_cast<float*>(local_outputs[0].data.data());
auto ref_tensor = PADDLE_GET(phi::DenseTensor, refs[tid]);
float* ref_data = ref_tensor.data<float>();
EXPECT_EQ((size_t)ref_tensor.numel(), len / sizeof(float));
for (int i = 0; i < ref_tensor.numel(); ++i) {
EXPECT_NEAR(ref_data[i], data[i], ACC_DIFF);
}
});
}
for (int i = 0; i < num_jobs; ++i) {
threads[i].join();
}
}
TEST(inference_api_native, word2vec_cpu) {
MainWord2Vec(::paddle::PaddlePlace::kCPU);
}
TEST(inference_api_native, word2vec_cpu_threads) {
MainThreadsWord2Vec(::paddle::PaddlePlace::kCPU);
}
TEST(inference_api_native, image_classification_cpu) {
MainImageClassification(::paddle::PaddlePlace::kCPU);
}
TEST(inference_api_native, image_classification_cpu_threads) {
MainThreadsImageClassification(::paddle::PaddlePlace::kCPU);
}
#ifdef PADDLE_WITH_XPU
TEST(inference_api_native, word2vec_xpu) {
MainWord2Vec(::paddle::PaddlePlace::kXPU);
}
TEST(inference_api_native, image_classification_xpu) {
MainImageClassification(::paddle::PaddlePlace::kXPU);
}
#endif
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
TEST(inference_api_native, word2vec_gpu) {
MainWord2Vec(::paddle::PaddlePlace::kGPU);
}
// Turn off temporarily for the unstable result.
// TEST(inference_api_native, word2vec_gpu_threads) {
// MainThreadsWord2Vec(::paddle::PaddlePlace::kGPU);
// }
TEST(inference_api_native, image_classification_gpu) {
MainImageClassification(::paddle::PaddlePlace::kGPU);
}
// Turn off temporarily for the unstable result.
// TEST(inference_api_native, image_classification_gpu_threads) {
// MainThreadsImageClassification(::paddle::PaddlePlace::kGPU);
// }
#endif
#ifdef PADDLE_WITH_DNNL
TEST(inference_api_native, image_classification_cpu_onednn) {
FLAGS_use_onednn = true;
MainImageClassification(::paddle::PaddlePlace::kCPU);
}
TEST(inference_api_native, word2vec_cpu_onednn) {
FLAGS_use_onednn = true;
MainWord2Vec(::paddle::PaddlePlace::kCPU);
}
#endif
TEST(PassBuilder, Delete) {
AnalysisConfig config;
config.DisableGpu();
config.pass_builder()->DeletePass("attention_lstm_fuse_pass");
const auto& passes = config.pass_builder()->AllPasses();
auto it = std::find(passes.begin(), passes.end(), "attention_lstm_fuse_pass");
ASSERT_EQ(it, passes.end());
}
} // namespace paddle
test/cpp/inference/api/api_tester.cc
+112
View File
@@ -0,0 +1,112 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <exception>
#include <string>
#include "paddle/fluid/inference/api/paddle_analysis_config.h"
#include "paddle/fluid/inference/api/paddle_api.h"
#include "paddle/fluid/platform/enforce.h"
namespace paddle {
/*
* Do not use this, just a demo indicating how to customize a config for a
* specific predictor.
*/
struct DemoConfig : public PaddlePredictor::Config {
float other_config;
DemoConfig() : other_config(0) {}
};
/*
* Do not use this, just a demo indicating how to customize a Predictor.
*/
class DemoPredictor : public PaddlePredictor {
public:
explicit DemoPredictor(const DemoConfig &config) {
LOG(INFO) << "I get other_config " << config.other_config;
}
bool Run(const std::vector<PaddleTensor> &inputs,
std::vector<PaddleTensor> *output_data,
int batch_size = 0) override {
LOG(INFO) << "Run";
return false;
}
std::unique_ptr<PaddlePredictor> Clone(void *stream = nullptr) override {
return nullptr;
}
~DemoPredictor() override = default;
};
template <>
std::unique_ptr<PaddlePredictor> CreatePaddlePredictor<DemoConfig>(
const DemoConfig &config) {
std::unique_ptr<PaddlePredictor> x(new DemoPredictor(config));
return x;
}
TEST(paddle_inference_api, demo) {
DemoConfig config;
config.other_config = 1.7;
auto predictor = CreatePaddlePredictor(config);
std::vector<PaddleTensor> outputs;
predictor->Run({}, &outputs);
predictor->TryShrinkMemory();
}
TEST(paddle_inference_api, get_version) {
LOG(INFO) << "paddle version:\n" << get_version();
auto version = get_version();
ASSERT_FALSE(version.empty());
}
TEST(paddle_inference_api, UpdateDllFlag) {
UpdateDllFlag("paddle_num_threads", "10");
try {
UpdateDllFlag("paddle_num_threads2", "10");
} catch (std::exception &e) {
LOG(INFO) << e.what();
}
}
TEST(paddle_inference_api, AnalysisConfigCopyCtor) {
AnalysisConfig cfg1;
cfg1.EnableUseGpu(10);
#ifdef PADDLE_WITH_TENSORRT
cfg1.EnableTensorRtEngine();
#endif
std::string delete_pass("skip_layernorm_fuse_pass");
cfg1.pass_builder()->DeletePass(delete_pass);
AnalysisConfig cfg2(cfg1);
auto passes = cfg2.pass_builder()->AllPasses();
for (auto const &ps : passes) {
PADDLE_ENFORCE_NE(ps,
delete_pass,
common::errors::InvalidArgument(
"Required ps shouldn't be equal to delete_pass. "));
}
}
#ifdef PADDLE_WITH_CRYPTO
TEST(paddle_inference_api, crypto) { paddle::MakeCipher(""); }
#endif
} // namespace paddle
test/cpp/inference/api/config_printer.h
+86
View File
@@ -0,0 +1,86 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#pragma once
#include <ostream>
#include <string>
#include "paddle/fluid/inference/api/paddle_inference_api.h"
namespace paddle {
namespace inference {
thread_local int num_spaces = 0;
static std::string GenSpaces(int num_spaces) {
std::ostringstream os;
for (int i = 0; i < num_spaces; ++i) {
os << " ";
}
return os.str();
}
std::ostream &operator<<(std::ostream &os,
const PaddlePredictor::Config &config) {
os << GenSpaces(num_spaces) << "PaddlePredictor::Config {\n";
num_spaces++;
os << GenSpaces(num_spaces) << "model_dir: " << config.model_dir << "\n";
num_spaces--;
os << GenSpaces(num_spaces) << "}\n";
return os;
}
std::ostream &operator<<(std::ostream &os, const NativeConfig &config) {
os << GenSpaces(num_spaces) << "NativeConfig {\n";
num_spaces++;
os << *reinterpret_cast<const PaddlePredictor::Config *>(&config);
os << GenSpaces(num_spaces) << "use_gpu: " << config.use_gpu << "\n";
os << GenSpaces(num_spaces) << "device: " << config.device << "\n";
os << GenSpaces(num_spaces)
<< "fraction_of_gpu_memory: " << config.fraction_of_gpu_memory << "\n";
os << GenSpaces(num_spaces)
<< "specify_input_name: " << config.specify_input_name << "\n";
num_spaces--;
os << GenSpaces(num_spaces) << "}\n";
return os;
}
std::ostream &operator<<(std::ostream &os, const AnalysisConfig &config) {
os << GenSpaces(num_spaces) << "AnalysisConfig {\n";
num_spaces++;
os << config.ToNativeConfig();
if (!config.model_from_memory()) {
os << GenSpaces(num_spaces) << "prog_file: " << config.prog_file() << "\n";
os << GenSpaces(num_spaces) << "param_file: " << config.params_file()
<< "\n";
} else {
os << GenSpaces(num_spaces)
<< "prog_file and param_file: load from memory \n";
}
os << GenSpaces(num_spaces) << "enable_ir_optim: " << config.ir_optim()
<< "\n";
os << GenSpaces(num_spaces)
<< "cpu_num_threads: " << config.cpu_math_library_num_threads() << "\n";
os << GenSpaces(num_spaces)
<< "use_tensorrt: " << config.tensorrt_engine_enabled() << "\n";
os << GenSpaces(num_spaces) << "use_onednn: " << config.onednn_enabled()
<< "\n";
num_spaces--;
os << GenSpaces(num_spaces) << "}\n";
return os;
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/full_ILSVRC2012_val_preprocess.py
+275
View File
@@ -0,0 +1,275 @@
# copyright (c) 2019 paddlepaddle authors. all rights reserved.
# licensed under the apache license, version 2.0 (the "license");
# you may not use this file except in compliance with the license.
# you may obtain a copy of the license at
#
# http://www.apache.org/licenses/license-2.0
#
# unless required by applicable law or agreed to in writing, software
# distributed under the license is distributed on an "as is" basis,
# without warranties or conditions of any kind, either express or implied.
# see the license for the specific language governing permissions and
# limitations under the license.
import argparse
import io
import os
import shutil
import sys
import tarfile
import numpy as np
from PIL import Image
from paddle.dataset.common import download
np.random.seed(0)
DATA_DIM = 224
SIZE_FLOAT32 = 4
SIZE_INT64 = 8
FULL_SIZE_BYTES = 30106000008
FULL_IMAGES = 50000
FOLDER_NAME = "ILSVRC2012/"
VALLIST_TAR_NAME = "ILSVRC2012/val_list.txt"
img_mean = np.array([0.485, 0.456, 0.406]).reshape((3, 1, 1))
img_std = np.array([0.229, 0.224, 0.225]).reshape((3, 1, 1))
def resize_short(img, target_size):
percent = float(target_size) / min(img.size[0], img.size[1])
resized_width = int(round(img.size[0] * percent))
resized_height = int(round(img.size[1] * percent))
img = img.resize((resized_width, resized_height), Image.LANCZOS)
return img
def crop_image(img, target_size, center):
width, height = img.size
size = target_size
if center:
w_start = (width - size) // 2
h_start = (height - size) // 2
else:
w_start = np.random.randint(0, width - size + 1)
h_start = np.random.randint(0, height - size + 1)
w_end = w_start + size
h_end = h_start + size
img = img.crop((w_start, h_start, w_end, h_end))
return img
def process_image(img):
img = resize_short(img, target_size=256)
img = crop_image(img, target_size=DATA_DIM, center=True)
if img.mode != 'RGB':
img = img.convert('RGB')
img = np.array(img).astype('float32').transpose((2, 0, 1)) / 255
img -= img_mean
img /= img_std
return img
def download_concat(cache_folder, zip_path):
data_urls = []
data_md5s = []
data_urls.append(
'https://paddle-inference-dist.bj.bcebos.com/int8/ILSVRC2012_img_val.tar.gz.partaa'
)
data_md5s.append('60f6525b0e1d127f345641d75d41f0a8')
data_urls.append(
'https://paddle-inference-dist.bj.bcebos.com/int8/ILSVRC2012_img_val.tar.gz.partab'
)
data_md5s.append('1e9f15f64e015e58d6f9ec3210ed18b5')
file_names = []
print("Downloading full ImageNet Validation dataset ...")
for i in range(0, len(data_urls)):
download(data_urls[i], cache_folder, data_md5s[i])
file_name = os.path.join(cache_folder, data_urls[i].split('/')[-1])
file_names.append(file_name)
print(f"Downloaded part {file_name}\n")
with open(zip_path, "wb") as outfile:
for fname in file_names:
shutil.copyfileobj(open(fname, 'rb'), outfile)
def print_processbar(done_percentage):
done_filled = done_percentage * '='
empty_filled = (100 - done_percentage) * ' '
sys.stdout.write(f"\r[{done_filled}{empty_filled}]{done_percentage}%")
sys.stdout.flush()
def convert_Imagenet_tar2bin(tar_file, output_file):
print('Converting 50000 images to binary file ...\n')
tar = tarfile.open(name=tar_file, mode='r:gz')
print_processbar(0)
dataset = {}
for tarInfo in tar:
if tarInfo.isfile() and tarInfo.name != VALLIST_TAR_NAME:
dataset[tarInfo.name] = tar.extractfile(tarInfo).read()
with open(output_file, "w+b") as ofs:
ofs.seek(0)
num = np.array(int(FULL_IMAGES)).astype('int64')
ofs.write(num.tobytes())
per_percentage = FULL_IMAGES // 100
val_info = tar.getmember(VALLIST_TAR_NAME)
val_list = tar.extractfile(val_info).read().decode("utf-8")
lines = val_list.splitlines()
idx = 0
for imagedata in dataset.values():
img = Image.open(io.BytesIO(imagedata))
img = process_image(img)
np_img = np.array(img)
ofs.write(np_img.astype('float32').tobytes())
if idx % per_percentage == 0:
print_processbar(idx // per_percentage)
idx = idx + 1
val_dict = {}
for line_idx, line in enumerate(lines):
if line_idx == FULL_IMAGES:
break
name, label = line.split()
val_dict[name] = label
for img_name in dataset.keys():
remove_len = len(FOLDER_NAME)
img_name_prim = img_name[remove_len:]
label = val_dict[img_name_prim]
label_int = (int)(label)
np_label = np.array(label_int)
ofs.write(np_label.astype('int64').tobytes())
print_processbar(100)
tar.close()
print("Conversion finished.")
def run_convert():
print('Start to download and convert 50000 images to binary file...')
cache_folder = os.path.expanduser('~/.cache/paddle/dataset/int8/download')
zip_path = os.path.join(cache_folder, 'full_imagenet_val.tar.gz.partaa')
output_file = os.path.join(cache_folder, 'int8_full_val.bin')
retry = 0
try_limit = 3
while not (
os.path.exists(output_file)
and os.path.getsize(output_file) == FULL_SIZE_BYTES
):
if os.path.exists(output_file):
sys.stderr.write(
f"\n\nThe existing binary file[{output_file}] is broken. Start to generate new one...\n\n"
)
os.remove(output_file)
if retry < try_limit:
retry = retry + 1
else:
raise RuntimeError(
f"Can not convert the dataset to binary file with try limit {try_limit}"
)
download_concat(cache_folder, zip_path)
convert_Imagenet_tar2bin(zip_path, output_file)
print(f"\nSuccess! The binary file can be found at {output_file}")
def convert_Imagenet_local2bin(args):
data_dir = args.data_dir
label_list_path = os.path.join(args.data_dir, args.label_list)
bin_file_path = os.path.join(args.data_dir, args.output_file)
assert data_dir, 'Once set --local, user need to provide the --data_dir'
with open(label_list_path) as flist:
lines = [line.strip() for line in flist]
num_images = len(lines)
with open(bin_file_path, "w+b") as of:
of.seek(0)
num = np.array(int(num_images)).astype('int64')
of.write(num.tobytes())
for idx, line in enumerate(lines):
img_path, label = line.split()
img_path = os.path.join(data_dir, img_path)
if not os.path.exists(img_path):
continue
# save image(float32) to file
img = Image.open(img_path)
img = process_image(img)
np_img = np.array(img)
of.seek(
SIZE_INT64 + SIZE_FLOAT32 * DATA_DIM * DATA_DIM * 3 * idx
)
of.write(np_img.astype('float32').tobytes())
# save label(int64_t) to file
label_int = (int)(label)
np_label = np.array(label_int)
of.seek(
SIZE_INT64
+ SIZE_FLOAT32 * DATA_DIM * DATA_DIM * 3 * num_images
+ idx * SIZE_INT64
)
of.write(np_label.astype('int64').tobytes())
# The bin file should contain
# number of images + all images data + all corresponding labels
# so the file target_size should be as follows
target_size = (
SIZE_INT64
+ num_images * 3 * args.data_dim * args.data_dim * SIZE_FLOAT32
+ num_images * SIZE_INT64
)
if os.path.getsize(bin_file_path) == target_size:
print(
f"Success! The user data output binary file can be found at: {bin_file_path}"
)
else:
print("Conversion failed!")
def main_preprocess_Imagenet(args):
parser = argparse.ArgumentParser(
description="Convert the full Imagenet val set or local data to binary file.",
usage=None,
add_help=True,
)
parser.add_argument(
'--local',
action="store_true",
help="If used, user need to set --data_dir and then convert file",
)
parser.add_argument(
"--data_dir", default="", type=str, help="Dataset root directory"
)
parser.add_argument(
"--label_list",
type=str,
default="val_list.txt",
help="List of object labels with same sequence as denoted in the annotation file",
)
parser.add_argument(
"--output_file",
type=str,
default="imagenet_small.bin",
help="File path of the output binary file",
)
parser.add_argument(
"--data_dim",
type=int,
default=DATA_DIM,
help="Image preprocess with data_dim width and height",
)
args = parser.parse_args()
if args.local:
convert_Imagenet_local2bin(args)
else:
run_convert()
if __name__ == '__main__':
main_preprocess_Imagenet(sys.argv)
test/cpp/inference/api/full_pascalvoc_test_preprocess.py
+373
View File
@@ -0,0 +1,373 @@
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import hashlib
import os
import sys
import tarfile
import xml.etree.ElementTree
from io import StringIO
import numpy as np
from PIL import Image
from paddle.dataset.common import download
DATA_URL = (
"http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar"
)
DATA_DIR = os.path.expanduser("~/.cache/paddle/dataset/pascalvoc/")
TAR_FILE = "VOCtest_06-Nov-2007.tar"
TAR_PATH = os.path.join(DATA_DIR, TAR_FILE)
SIZE_FLOAT32 = 4
SIZE_INT64 = 8
RESIZE_H = 300
RESIZE_W = 300
MEAN_VALUE = [127.5, 127.5, 127.5]
AP_VERSION = '11point'
DATA_OUT = 'pascalvoc_full.bin'
DATA_OUT_PATH = os.path.join(DATA_DIR, DATA_OUT)
BIN_TARGETHASH = "f6546cadc42f5ff13178b84ed29b740b"
TAR_TARGETHASH = "b6e924de25625d8de591ea690078ad9f"
TEST_LIST_KEY = "VOCdevkit/VOC2007/ImageSets/Main/test.txt"
BIN_FULLSIZE = 5348678856
def preprocess(img):
img_width, img_height = img.size
img = img.resize((RESIZE_W, RESIZE_H), Image.LANCZOS)
img = np.array(img)
# HWC to CHW
if len(img.shape) == 3:
img = np.swapaxes(img, 1, 2)
img = np.swapaxes(img, 1, 0)
# RBG to BGR
img = img[[2, 1, 0], :, :]
img = img.astype('float32')
img_mean = np.array(MEAN_VALUE)[:, np.newaxis, np.newaxis].astype('float32')
img -= img_mean
img = img * 0.007843
return img
def convert_pascalvoc_local2bin(args):
data_dir = os.path.expanduser(args.data_dir)
label_fpath = os.path.join(data_dir, args.label_file)
assert data_dir, 'Once set --local, user need to provide the --data_dir'
flabel = open(label_fpath)
label_list = [line.strip() for line in flabel]
img_annotation_list_path = os.path.join(data_dir, args.img_annotation_list)
flist = open(img_annotation_list_path)
lines = [line.strip() for line in flist]
output_file_path = os.path.join(data_dir, args.output_file)
f1 = open(output_file_path, "w+b")
f1.seek(0)
image_nums = len(lines)
f1.write(np.array(image_nums).astype('int64').tobytes())
boxes = []
lbls = []
difficulties = []
object_nums = []
for line in lines:
image_path, label_path = line.split()
image_path = os.path.join(data_dir, image_path)
label_path = os.path.join(data_dir, label_path)
im = Image.open(image_path)
if im.mode == 'L':
im = im.convert('RGB')
im_width, im_height = im.size
im = preprocess(im)
np_im = np.array(im)
f1.write(np_im.astype('float32').tobytes())
# layout: label | xmin | ymin | xmax | ymax | difficult
bbox_labels = []
root = xml.etree.ElementTree.parse(label_path).getroot()
objects = root.findall('object')
objects_size = len(objects)
object_nums.append(objects_size)
for object in objects:
bbox_sample = []
# start from 1
bbox_sample.append(
float(label_list.index(object.find('name').text))
)
bbox = object.find('bndbox')
difficult = float(object.find('difficult').text)
bbox_sample.append(float(bbox.find('xmin').text) / im_width)
bbox_sample.append(float(bbox.find('ymin').text) / im_height)
bbox_sample.append(float(bbox.find('xmax').text) / im_width)
bbox_sample.append(float(bbox.find('ymax').text) / im_height)
bbox_sample.append(difficult)
bbox_labels.append(bbox_sample)
bbox_labels = np.array(bbox_labels)
if len(bbox_labels) == 0:
continue
lbls.extend(bbox_labels[:, 0])
boxes.extend(bbox_labels[:, 1:5])
difficulties.extend(bbox_labels[:, -1])
f1.write(np.array(object_nums).astype('uint64').tobytes())
f1.write(np.array(lbls).astype('int64').tobytes())
f1.write(np.array(boxes).astype('float32').tobytes())
f1.write(np.array(difficulties).astype('int64').tobytes())
f1.close()
object_nums_sum = sum(object_nums)
# The data should be contains
# number of images + all images data + an array that represent object numbers of each image
# + labels of all objects in images + bboxes of all objects + difficulties of all objects
# so the target size should be as follows:
target_size = (
SIZE_INT64
+ image_nums * 3 * args.resize_h * args.resize_h * SIZE_FLOAT32
+ image_nums * SIZE_INT64
+ object_nums_sum * (SIZE_INT64 + 4 * SIZE_FLOAT32 + SIZE_INT64)
)
if os.path.getsize(output_file_path) == target_size:
print(
"Success! \nThe local data output binary file can be found at: ",
output_file_path,
)
else:
print("Conversion failed!")
def print_processbar(done_percentage):
done_filled = done_percentage * '='
empty_filled = (100 - done_percentage) * ' '
sys.stdout.write(f"\r[{done_filled}{empty_filled}]{done_percentage}%")
sys.stdout.flush()
def convert_pascalvoc_tar2bin(tar_path, data_out_path):
print("Start converting ...\n")
images = {}
gt_labels = {}
boxes = []
lbls = []
difficulties = []
object_nums = []
# map label to number (index)
label_list = [
"background",
"aeroplane",
"bicycle",
"bird",
"boat",
"bottle",
"bus",
"car",
"cat",
"chair",
"cow",
"diningtable",
"dog",
"horse",
"motorbike",
"person",
"pottedplant",
"sheep",
"sofa",
"train",
"tvmonitor",
]
print_processbar(0)
# read from tar file and write to bin
tar = tarfile.open(tar_path, "r")
f_test = tar.extractfile(TEST_LIST_KEY).read()
lines = f_test.split('\n')
del lines[-1]
image_nums = len(lines)
per_percentage = image_nums / 100
f1 = open(data_out_path, "w+b")
f1.seek(0)
f1.write(np.array(image_nums).astype('int64').tobytes())
for tarInfo in tar:
if tarInfo.isfile():
tmp_filename = tarInfo.name
name_arr = tmp_filename.split('/')
name_prefix = name_arr[-1].split('.')[0]
if name_arr[-2] == 'JPEGImages' and name_prefix in lines:
images[name_prefix] = tar.extractfile(tarInfo).read()
if name_arr[-2] == 'Annotations' and name_prefix in lines:
gt_labels[name_prefix] = tar.extractfile(tarInfo).read()
for line_idx, name_prefix in enumerate(lines):
im = Image.open(StringIO(images[name_prefix]))
if im.mode == 'L':
im = im.convert('RGB')
im_width, im_height = im.size
im = preprocess(im)
np_im = np.array(im)
f1.write(np_im.astype('float32').tobytes())
# layout: label | xmin | ymin | xmax | ymax | difficult
bbox_labels = []
root = xml.etree.ElementTree.fromstring(gt_labels[name_prefix])
objects = root.findall('object')
objects_size = len(objects)
object_nums.append(objects_size)
for object in objects:
bbox_sample = []
bbox_sample.append(
float(label_list.index(object.find('name').text))
)
bbox = object.find('bndbox')
difficult = float(object.find('difficult').text)
bbox_sample.append(float(bbox.find('xmin').text) / im_width)
bbox_sample.append(float(bbox.find('ymin').text) / im_height)
bbox_sample.append(float(bbox.find('xmax').text) / im_width)
bbox_sample.append(float(bbox.find('ymax').text) / im_height)
bbox_sample.append(difficult)
bbox_labels.append(bbox_sample)
bbox_labels = np.array(bbox_labels)
if len(bbox_labels) == 0:
continue
lbls.extend(bbox_labels[:, 0])
boxes.extend(bbox_labels[:, 1:5])
difficulties.extend(bbox_labels[:, -1])
if line_idx % per_percentage:
print_processbar(line_idx / per_percentage)
# The data should be stored in binary in following sequence:
# number of images->all images data->an array that represent object numbers in each image
# ->labels of all objects in images->bboxes of all objects->difficulties of all objects
f1.write(np.array(object_nums).astype('uint64').tobytes())
f1.write(np.array(lbls).astype('int64').tobytes())
f1.write(np.array(boxes).astype('float32').tobytes())
f1.write(np.array(difficulties).astype('int64').tobytes())
f1.close()
print_processbar(100)
print("Conversion finished!\n")
def download_pascalvoc(data_url, data_dir, tar_targethash, tar_path):
print("Downloading pascalvcoc test set...")
download(data_url, data_dir, tar_targethash)
if not os.path.exists(tar_path):
print(f"Failed in downloading pascalvoc test set. URL {data_url}\n")
else:
tmp_hash = hashlib.md5(open(tar_path, 'rb').read()).hexdigest()
if tmp_hash != tar_targethash:
print("Downloaded test set is broken, removing ...\n")
else:
print(f"Downloaded successfully. Path: {tar_path}\n")
def run_convert():
try_limit = 2
retry = 0
while not (
os.path.exists(DATA_OUT_PATH)
and os.path.getsize(DATA_OUT_PATH) == BIN_FULLSIZE
and BIN_TARGETHASH
== hashlib.md5(open(DATA_OUT_PATH, 'rb').read()).hexdigest()
):
if os.path.exists(DATA_OUT_PATH):
sys.stderr.write(
"The existing binary file is broken. It is being removed...\n"
)
os.remove(DATA_OUT_PATH)
if retry < try_limit:
retry = retry + 1
else:
download_pascalvoc(DATA_URL, DATA_DIR, TAR_TARGETHASH, TAR_PATH)
convert_pascalvoc_tar2bin(TAR_PATH, DATA_OUT_PATH)
print(f"Success!\nThe binary file can be found at {DATA_OUT_PATH}\n")
def main_pascalvoc_preprocess(args):
parser = argparse.ArgumentParser(
description="Convert the full pascalvoc val set or local data to binary file.",
usage=None,
add_help=True,
)
parser.add_argument(
'--local',
action="store_true",
help="If used, user need to set --data_dir and then convert file",
)
parser.add_argument(
"--data_dir", default="", type=str, help="Dataset root directory"
)
parser.add_argument(
"--img_annotation_list",
type=str,
default="test_100.txt",
help="A file containing the image file path and corresponding annotation file path",
)
parser.add_argument(
"--label_file",
type=str,
default="label_list",
help="List of object labels with same sequence as denoted in the annotation file",
)
parser.add_argument(
"--output_file",
type=str,
default="pascalvoc_small.bin",
help="File path of the output binary file",
)
parser.add_argument(
"--resize_h",
type=int,
default=RESIZE_H,
help="Image preprocess with resize_h",
)
parser.add_argument(
"--resize_w",
type=int,
default=RESIZE_W,
help="Image prerocess with resize_w",
)
parser.add_argument(
"--mean_value",
type=str,
default=MEAN_VALUE,
help="Image preprocess with mean_value",
)
parser.add_argument(
"--ap_version",
type=str,
default=AP_VERSION,
help="Image preprocess with ap_version",
)
args = parser.parse_args()
if args.local:
convert_pascalvoc_local2bin(args)
else:
run_convert()
if __name__ == "__main__":
main_pascalvoc_preprocess(sys.argv)
test/cpp/inference/api/gpu_ernie_half_test.cc
+290
View File
@@ -0,0 +1,290 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/fluid/inference/api/paddle_inference_api.h"
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
using paddle::PaddleTensor;
template <typename T>
void GetValueFromStream(std::stringstream *ss, T *t) {
(*ss) >> (*t);
}
template <>
void GetValueFromStream<std::string>(std::stringstream *ss, std::string *t) {
*t = ss->str();
}
// Split string to vector
template <typename T>
void Split(const std::string &line, char sep, std::vector<T> *v) {
std::stringstream ss;
T t;
for (auto c : line) {
if (c != sep) {
ss << c;
} else {
GetValueFromStream<T>(&ss, &t);
v->push_back(std::move(t));
ss.str({});
ss.clear();
}
}
if (!ss.str().empty()) {
GetValueFromStream<T>(&ss, &t);
v->push_back(std::move(t));
ss.str({});
ss.clear();
}
}
// Parse tensor from string
template <typename T>
bool ParseTensor(const std::string &field, paddle::PaddleTensor *tensor) {
std::vector<std::string> data;
Split(field, ':', &data);
if (data.size() < 2) return false;
std::string shape_str = data[0];
std::vector<int> shape;
Split(shape_str, ' ', &shape);
std::string mat_str = data[1];
std::vector<T> mat;
Split(mat_str, ' ', &mat);
tensor->shape = shape;
auto size =
std::accumulate(shape.begin(), shape.end(), 1, std::multiplies<int>()) *
sizeof(T);
tensor->data.Resize(size);
std::copy(mat.begin(), mat.end(), static_cast<T *>(tensor->data.data()));
tensor->dtype = GetPaddleDType<T>();
return true;
}
// Parse input tensors from string
bool ParseLine(const std::string &line,
std::vector<paddle::PaddleTensor> *tensors) {
std::vector<std::string> fields;
Split(line, ';', &fields);
tensors->clear();
tensors->reserve(4);
int i = 0;
auto input_name = FLAGS_ernie_large ? "eval_placeholder_" : "placeholder_";
for (; i < 3; i++) {
paddle::PaddleTensor temp;
ParseTensor<int64_t>(fields[i], &temp);
temp.name = input_name + std::to_string(i);
tensors->push_back(temp);
}
// input_mask
paddle::PaddleTensor input_mask;
ParseTensor<float>(fields[i], &input_mask);
input_mask.name = input_name + std::to_string(i);
tensors->push_back(input_mask);
return true;
}
bool LoadInputData(std::vector<std::vector<paddle::PaddleTensor>> *inputs,
int batch_size = 1) {
if (FLAGS_infer_data.empty()) {
LOG(ERROR) << "please set input data path";
return false;
}
std::ifstream fin(FLAGS_infer_data);
std::string line;
int sample = 0;
// The unit-test dataset only have 10 samples, each sample have 5 feeds.
while (std::getline(fin, line)) {
std::vector<paddle::PaddleTensor> feed_data;
ParseLine(line, &feed_data);
inputs->push_back(std::move(feed_data));
sample++;
if (!FLAGS_test_all_data && sample == batch_size) break;
}
LOG(INFO) << "number of samples: " << sample;
return true;
}
// Compare results
TEST(Ernie_gpu_fp16_no_ir, compare_results) {
AnalysisConfig config;
config.SetModel(FLAGS_infer_model);
config.EnableUseGpu(512, 0, paddle_infer::PrecisionType::kHalf);
config.SwitchIrOptim(false);
auto predictor = CreatePaddlePredictor(config);
std::vector<std::vector<PaddleTensor>> input_slots_all;
LoadInputData(&input_slots_all);
std::ifstream fin(FLAGS_refer_result);
std::string line;
std::vector<float> ref;
while (std::getline(fin, line)) {
Split(line, ' ', &ref);
}
std::vector<PaddleTensor> outputs;
for (size_t i = 0; i < input_slots_all.size(); i++) {
outputs.clear();
predictor->Run(input_slots_all[i], &outputs);
auto output = outputs.front();
size_t outputs_size = 1;
for (auto dim : output.shape) {
outputs_size *= dim;
}
float *result = reinterpret_cast<float *>(output.data.data());
for (size_t j = 0; j < outputs_size; ++j) {
EXPECT_NEAR(ref[i * outputs_size + j], result[j], 8e-3);
}
}
}
// Compare results
TEST(Ernie_gpu_fp16_with_ir, compare_results) {
AnalysisConfig config;
config.SetModel(FLAGS_infer_model);
config.EnableUseGpu(512, 0, paddle_infer::PrecisionType::kHalf);
config.SwitchIrOptim(true);
// There is a problem with the model itself, which has nothing to do with
// constant_folding_pass.
config.pass_builder()->DeletePass("constant_folding_pass");
auto predictor = CreatePaddlePredictor(config);
std::vector<std::vector<PaddleTensor>> input_slots_all;
LoadInputData(&input_slots_all);
std::ifstream fin(FLAGS_refer_result);
std::string line;
std::vector<float> ref;
while (std::getline(fin, line)) {
Split(line, ' ', &ref);
}
std::vector<PaddleTensor> outputs;
for (size_t i = 0; i < input_slots_all.size(); i++) {
outputs.clear();
predictor->Run(input_slots_all[i], &outputs);
auto output = outputs.front();
size_t outputs_size = 1;
for (auto dim : output.shape) {
outputs_size *= dim;
}
float *result = reinterpret_cast<float *>(output.data.data());
for (size_t j = 0; j < outputs_size; ++j) {
EXPECT_NEAR(ref[i * outputs_size + j], result[j], 2e-2);
}
}
}
// Compare results
TEST(Ernie_gpu_bf16_no_ir, compare_results) {
AnalysisConfig config;
config.SetModel(FLAGS_infer_model);
config.EnableUseGpu(512, 0, paddle_infer::PrecisionType::kBf16);
config.SwitchIrOptim(false);
auto predictor = CreatePaddlePredictor(config);
std::vector<std::vector<PaddleTensor>> input_slots_all;
LoadInputData(&input_slots_all);
std::ifstream fin(FLAGS_refer_result);
std::string line;
std::vector<float> ref;
while (std::getline(fin, line)) {
Split(line, ' ', &ref);
}
std::vector<PaddleTensor> outputs;
for (size_t i = 0; i < input_slots_all.size(); i++) {
outputs.clear();
predictor->Run(input_slots_all[i], &outputs);
auto output = outputs.front();
size_t outputs_size = 1;
for (auto dim : output.shape) {
outputs_size *= dim;
}
float *result = reinterpret_cast<float *>(output.data.data());
for (size_t j = 0; j < outputs_size; ++j) {
EXPECT_NEAR(ref[i * outputs_size + j], result[j], 1e-2);
}
}
}
// Compare results
TEST(Ernie_gpu_bf16_with_ir, compare_results) {
AnalysisConfig config;
config.SetModel(FLAGS_infer_model);
config.EnableUseGpu(512, 0, paddle_infer::PrecisionType::kBf16);
config.SwitchIrOptim(true);
// There is a problem with the model itself, which has nothing to do with
// constant_folding_pass.
config.pass_builder()->DeletePass("constant_folding_pass");
auto predictor = CreatePaddlePredictor(config);
std::vector<std::vector<PaddleTensor>> input_slots_all;
LoadInputData(&input_slots_all);
std::ifstream fin(FLAGS_refer_result);
std::string line;
std::vector<float> ref;
while (std::getline(fin, line)) {
Split(line, ' ', &ref);
}
std::vector<PaddleTensor> outputs;
for (size_t i = 0; i < input_slots_all.size(); i++) {
outputs.clear();
predictor->Run(input_slots_all[i], &outputs);
auto output = outputs.front();
size_t outputs_size = 1;
for (auto dim : output.shape) {
outputs_size *= dim;
}
float *result = reinterpret_cast<float *>(output.data.data());
for (size_t j = 0; j < outputs_size; ++j) {
EXPECT_NEAR(ref[i * outputs_size + j], result[j], 5e-3);
}
}
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/helper_test.cc
+35
View File
@@ -0,0 +1,35 @@
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/fluid/inference/api/helper.h"
#include "gtest/gtest.h"
namespace paddle {
TEST(inference_api_helper, DataType) {
ASSERT_TRUE(
paddle::inference::IsFloatVar(paddle::framework::proto::VarType::FP64));
ASSERT_TRUE(
paddle::inference::IsFloatVar(paddle::framework::proto::VarType::FP32));
ASSERT_TRUE(
paddle::inference::IsFloatVar(paddle::framework::proto::VarType::FP16));
ASSERT_TRUE(
paddle::inference::IsFloatVar(paddle::framework::proto::VarType::BF16));
ASSERT_FALSE(
paddle::inference::IsFloatVar(paddle::framework::proto::VarType::INT32));
}
} // namespace paddle
test/cpp/inference/api/ipu_ernie_fp16_test.cc
+184
View File
@@ -0,0 +1,184 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
using paddle::PaddleTensor;
template <typename T>
void GetValueFromStream(std::stringstream *ss, T *t) {
(*ss) >> (*t);
}
template <>
void GetValueFromStream<std::string>(std::stringstream *ss, std::string *t) {
*t = ss->str();
}
// Split string to vector
template <typename T>
void Split(const std::string &line, char sep, std::vector<T> *v) {
std::stringstream ss;
T t;
for (auto c : line) {
if (c != sep) {
ss << c;
} else {
GetValueFromStream<T>(&ss, &t);
v->push_back(std::move(t));
ss.str({});
ss.clear();
}
}
if (!ss.str().empty()) {
GetValueFromStream<T>(&ss, &t);
v->push_back(std::move(t));
ss.str({});
ss.clear();
}
}
// Parse tensor from string
template <typename T>
bool ParseTensor(const std::string &field, paddle::PaddleTensor *tensor) {
std::vector<std::string> data;
Split(field, ':', &data);
if (data.size() < 2) return false;
std::string shape_str = data[0];
std::vector<int> shape;
Split(shape_str, ' ', &shape);
std::string mat_str = data[1];
std::vector<T> mat;
Split(mat_str, ' ', &mat);
tensor->shape = shape;
auto size =
std::accumulate(shape.begin(), shape.end(), 1, std::multiplies<int>()) *
sizeof(T);
tensor->data.Resize(size);
std::copy(mat.begin(), mat.end(), static_cast<T *>(tensor->data.data()));
tensor->dtype = GetPaddleDType<T>();
return true;
}
// Parse input tensors from string
bool ParseLine(const std::string &line,
std::vector<paddle::PaddleTensor> *tensors) {
std::vector<std::string> fields;
Split(line, ';', &fields);
tensors->clear();
tensors->reserve(4);
int i = 0;
auto input_name = FLAGS_ernie_large ? "eval_placeholder_" : "placeholder_";
for (; i < 3; i++) {
paddle::PaddleTensor temp;
ParseTensor<int64_t>(fields[i], &temp);
temp.name = input_name + std::to_string(i);
tensors->push_back(temp);
}
// input_mask
paddle::PaddleTensor input_mask;
ParseTensor<float>(fields[i], &input_mask);
// fp32 to fp16
ConvertFP32toFP16(input_mask);
input_mask.name = input_name + std::to_string(i);
tensors->push_back(input_mask);
return true;
}
bool LoadInputData(std::vector<std::vector<paddle::PaddleTensor>> *inputs,
int batch_size = 1) {
if (FLAGS_infer_data.empty()) {
LOG(ERROR) << "please set input data path";
return false;
}
std::ifstream fin(FLAGS_infer_data);
std::string line;
int sample = 0;
// The unit-test dataset only have 10 samples, each sample have 5 feeds.
while (std::getline(fin, line)) {
std::vector<paddle::PaddleTensor> feed_data;
ParseLine(line, &feed_data);
inputs->push_back(std::move(feed_data));
sample++;
if (!FLAGS_test_all_data && sample == batch_size) break;
}
LOG(INFO) << "number of samples: " << sample;
return true;
}
void SetConfig(AnalysisConfig *cfg, int batch_size = 1) {
cfg->SetModel(FLAGS_infer_model);
// ipu_device_num, ipu_micro_batch_size, ipu_enable_pipelining
cfg->EnableIpu(1, batch_size, false);
// ipu_enable_fp16, ipu_replica_num, ipu_available_memory_proportion,
// ipu_enable_half_partial
cfg->SetIpuConfig(true, 1, 1.0, true);
}
// Compare results
TEST(Analyzer_Ernie_ipu, compare_results) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> input_slots_all;
LoadInputData(&input_slots_all);
std::ifstream fin(FLAGS_refer_result);
std::string line;
std::vector<float> ref;
while (std::getline(fin, line)) {
Split(line, ' ', &ref);
}
auto predictor = CreateTestPredictor(
reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
FLAGS_use_analysis);
std::vector<PaddleTensor> outputs;
for (size_t i = 0; i < input_slots_all.size(); i++) {
outputs.clear();
predictor->Run(input_slots_all[i], &outputs);
auto output = outputs.front();
ConvertFP16toFP32(output);
auto outputs_size = 1;
for (auto dim : output.shape) {
outputs_size *= dim;
}
float *fp32_data = reinterpret_cast<float *>(output.data.data());
for (size_t j = 0; j < outputs_size; ++j) {
EXPECT_NEAR(ref[i * outputs_size + j], fp32_data[j], 5e-3);
}
}
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/ipu_ernie_test.cc
+198
View File
@@ -0,0 +1,198 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
using paddle::PaddleTensor;
template <typename T>
void GetValueFromStream(std::stringstream *ss, T *t) {
(*ss) >> (*t);
}
template <>
void GetValueFromStream<std::string>(std::stringstream *ss, std::string *t) {
*t = ss->str();
}
// Split string to vector
template <typename T>
void Split(const std::string &line, char sep, std::vector<T> *v) {
std::stringstream ss;
T t;
for (auto c : line) {
if (c != sep) {
ss << c;
} else {
GetValueFromStream<T>(&ss, &t);
v->push_back(std::move(t));
ss.str({});
ss.clear();
}
}
if (!ss.str().empty()) {
GetValueFromStream<T>(&ss, &t);
v->push_back(std::move(t));
ss.str({});
ss.clear();
}
}
// Parse tensor from string
template <typename T>
bool ParseTensor(const std::string &field, paddle::PaddleTensor *tensor) {
std::vector<std::string> data;
Split(field, ':', &data);
if (data.size() < 2) return false;
std::string shape_str = data[0];
std::vector<int> shape;
Split(shape_str, ' ', &shape);
std::string mat_str = data[1];
std::vector<T> mat;
Split(mat_str, ' ', &mat);
tensor->shape = shape;
auto size =
std::accumulate(shape.begin(), shape.end(), 1, std::multiplies<int>()) *
sizeof(T);
tensor->data.Resize(size);
std::copy(mat.begin(), mat.end(), static_cast<T *>(tensor->data.data()));
tensor->dtype = GetPaddleDType<T>();
return true;
}
// Parse input tensors from string
bool ParseLine(const std::string &line,
std::vector<paddle::PaddleTensor> *tensors) {
std::vector<std::string> fields;
Split(line, ';', &fields);
tensors->clear();
tensors->reserve(4);
int i = 0;
auto input_name = FLAGS_ernie_large ? "eval_placeholder_" : "placeholder_";
for (; i < 3; i++) {
paddle::PaddleTensor temp;
ParseTensor<int64_t>(fields[i], &temp);
temp.name = input_name + std::to_string(i);
tensors->push_back(temp);
}
// input_mask
paddle::PaddleTensor input_mask;
ParseTensor<float>(fields[i], &input_mask);
input_mask.name = input_name + std::to_string(i);
tensors->push_back(input_mask);
return true;
}
bool LoadInputData(std::vector<std::vector<paddle::PaddleTensor>> *inputs,
int batch_size = 1) {
if (FLAGS_infer_data.empty()) {
LOG(ERROR) << "please set input data path";
return false;
}
std::ifstream fin(FLAGS_infer_data);
std::string line;
int sample = 0;
// The unit-test dataset only have 10 samples, each sample have 5 feeds.
while (std::getline(fin, line)) {
std::vector<paddle::PaddleTensor> feed_data;
ParseLine(line, &feed_data);
inputs->push_back(std::move(feed_data));
sample++;
if (!FLAGS_test_all_data && sample == batch_size) break;
}
LOG(INFO) << "number of samples: " << sample;
return true;
}
void SetConfig(AnalysisConfig *cfg, int batch_size = 1) {
cfg->SetModel(FLAGS_infer_model);
// ipu_device_num, ipu_micro_batch_size, ipu_enable_pipelining
cfg->EnableIpu(1, batch_size, false);
}
void profile() {
AnalysisConfig config;
SetConfig(&config);
std::vector<std::vector<PaddleTensor>> outputs;
std::vector<std::vector<PaddleTensor>> inputs;
LoadInputData(&inputs);
TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&config),
inputs,
&outputs,
FLAGS_num_threads);
}
// Compare Deterministic result
TEST(Analyzer_Ernie_ipu, compare_determine) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> input_slots_all;
LoadInputData(&input_slots_all);
CompareDeterministic(reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
input_slots_all);
}
// Compare results
TEST(Analyzer_Ernie_ipu, compare_results) {
AnalysisConfig cfg;
SetConfig(&cfg);
std::vector<std::vector<PaddleTensor>> input_slots_all;
LoadInputData(&input_slots_all);
std::ifstream fin(FLAGS_refer_result);
std::string line;
std::vector<float> ref;
while (std::getline(fin, line)) {
Split(line, ' ', &ref);
}
auto predictor = CreateTestPredictor(
reinterpret_cast<const PaddlePredictor::Config *>(&cfg),
FLAGS_use_analysis);
std::vector<PaddleTensor> outputs;
for (size_t i = 0; i < input_slots_all.size(); i++) {
outputs.clear();
predictor->Run(input_slots_all[i], &outputs);
auto outputs_size = outputs.front().data.length() / (sizeof(float));
for (size_t j = 0; j < outputs_size; ++j) {
EXPECT_NEAR(ref[i * outputs_size + j],
static_cast<float *>(outputs[0].data.data())[j],
FLAGS_accuracy);
}
}
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/ipu_multi_model_profile.cc
+112
View File
@@ -0,0 +1,112 @@
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include "paddle/common/flags.h"
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
void ErnieInputData(const int &total_batch_size,
const bool enable_fp16,
std::vector<PaddleTensor> *inputs) {
const int input_num = total_batch_size * 128 * 1;
std::vector<int64_t> placeholder_012(input_num, 1);
std::vector<float> placeholder_3(input_num, 1);
for (int i = 0; i < 4; i++) {
PaddleTensor in;
in.name = "placeholder_" + std::to_string(i);
in.shape = {total_batch_size, 128, 1};
if (i < 3) {
in.data = PaddleBuf(static_cast<void *>(placeholder_012.data()),
input_num * sizeof(int64_t));
in.dtype = PaddleDType::INT64;
} else {
in.data = PaddleBuf(static_cast<void *>(placeholder_3.data()),
input_num * sizeof(float));
in.dtype = PaddleDType::FLOAT32;
if (enable_fp16) {
ConvertFP32toFP16(in);
}
}
inputs->push_back(std::move(in));
}
}
void Resnet50InputData(const int &total_batch_size,
const bool enable_fp16,
std::vector<paddle::PaddleTensor> *inputs) {
const int input_num = total_batch_size * 3 * 318 * 318;
std::vector<float> input(input_num, 1);
PaddleTensor in;
in.shape = {total_batch_size, 3, 318, 318};
in.data =
PaddleBuf(static_cast<void *>(input.data()), input_num * sizeof(float));
in.dtype = PaddleDType::FLOAT32;
if (enable_fp16) {
ConvertFP32toFP16(in);
}
inputs->push_back(std::move(in));
}
// performance profile
TEST(Analyzer_ipu_fp16, performance_profile) {
AnalysisConfig config;
std::vector<PaddleTensor> inputs;
std::vector<std::vector<PaddleTensor>> outputs;
int total_batch_size = FLAGS_ipu_micro_batch_size * FLAGS_ipu_replica_num;
if (FLAGS_ipu_enable_pipelining) {
// if device_num > 1 and pipelining is enabled, the total batch size =
// micro_batch_size * device_num(batches_per_step) * replica_num
total_batch_size = FLAGS_ipu_micro_batch_size * FLAGS_ipu_batches_per_step *
FLAGS_ipu_replica_num;
}
if (FLAGS_model_name == "Resnet50") {
config.SetModel(FLAGS_infer_model + "/model/model",
FLAGS_infer_model + "/model/params");
Resnet50InputData(total_batch_size, FLAGS_ipu_enable_fp16, &inputs);
} else if (FLAGS_model_name == "Ernie") {
config.SetModel(FLAGS_infer_model + "/model/");
ErnieInputData(total_batch_size, FLAGS_ipu_enable_fp16, &inputs);
} else {
PADDLE_THROW(common::errors::InvalidArgument(
"Only support Resnet50 and Ernie Currently"));
}
// ipu_device_num, ipu_micro_batch_size, ipu_enable_pipelining,
// ipu_batches_per_step
config.EnableIpu(FLAGS_ipu_device_num,
FLAGS_ipu_micro_batch_size,
FLAGS_ipu_enable_pipelining,
FLAGS_ipu_batches_per_step);
// ipu_enable_fp16, ipu_replica_num, ipu_available_memory_proportion,
// ipu_enable_half_partial
config.SetIpuConfig(FLAGS_ipu_enable_fp16,
FLAGS_ipu_replica_num,
FLAGS_ipu_available_memory_proportion,
FLAGS_ipu_enable_half_partial);
TestPrediction(reinterpret_cast<const PaddlePredictor::Config *>(&config),
{inputs},
&outputs,
1);
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/ipu_resnet50_fp16_test.cc
+87
View File
@@ -0,0 +1,87 @@
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <cmath>
#include "paddle/common/flags.h"
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
// Compare results with 1 batch
TEST(Analyzer_Resnet50_ipu, compare_results_1_batch) {
std::string model_dir = FLAGS_infer_model + "/" + "model";
AnalysisConfig config;
// ipu_device_num, ipu_micro_batch_size, ipu_enable_pipelining
config.EnableIpu(1, 1, false);
// ipu_enable_fp16, ipu_replica_num, ipu_available_memory_proportion,
// ipu_enable_half_partial
config.SetIpuConfig(true, 1, 1.0, true);
config.SetModel(model_dir + "/model", model_dir + "/params");
std::vector<PaddleTensor> inputs;
auto predictor = CreatePaddlePredictor(config);
const int batch = 1;
const int channel = 3;
const int height = 318;
const int width = 318;
const int input_num = batch * channel * height * width;
std::vector<float> input(input_num, 1);
PaddleTensor in;
in.shape = {batch, channel, height, width};
in.data =
PaddleBuf(static_cast<void*>(input.data()), input_num * sizeof(float));
in.dtype = PaddleDType::FLOAT32;
ConvertFP32toFP16(in);
inputs.emplace_back(in);
std::vector<PaddleTensor> outputs;
ASSERT_TRUE(predictor->Run(inputs, &outputs));
const std::vector<float> truth_values = {
127.779f, 738.165f, 1013.22f, -438.17f, 366.401f, 927.659f,
736.222f, -633.684f, -329.927f, -430.155f, -633.062f, -146.548f,
-1324.28f, -1349.36f, -242.675f, 117.448f, -801.723f, -391.514f,
-404.818f, 454.16f, 515.48f, -133.031f, 69.293f, 590.096f,
-1434.69f, -1070.89f, 307.074f, 400.525f, -316.12f, -587.125f,
-161.056f, 800.363f, -96.4708f, 748.706f, 868.174f, -447.938f,
112.737f, 1127.2f, 47.4355f, 677.72f, 593.186f, -336.4f,
551.362f, 397.823f, 78.3979f, -715.398f, 405.969f, 404.256f,
246.019f, -8.42969f, 131.365f, -648.051f};
const size_t expected_size = 1;
EXPECT_EQ(outputs.size(), expected_size);
auto output = outputs.front();
ConvertFP16toFP32(output);
auto outputs_size = 1;
for (auto dim : output.shape) {
outputs_size *= dim;
}
float* fp32_data = reinterpret_cast<float*>(output.data.data());
for (size_t j = 0; j < outputs_size; j += 10) {
EXPECT_NEAR(
(fp32_data[j] - truth_values[j / 10]) / truth_values[j / 10], 0., 9e-2);
}
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/ipu_resnet50_test.cc
+178
View File
@@ -0,0 +1,178 @@
/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <cmath>
#include "paddle/common/flags.h"
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
static std::vector<float> truth_values = {
127.779f, 738.165f, 1013.22f, -438.17f, 366.401f, 927.659f, 736.222f,
-633.684f, -329.927f, -430.155f, -633.062f, -146.548f, -1324.28f, -1349.36f,
-242.675f, 117.448f, -801.723f, -391.514f, -404.818f, 454.16f, 515.48f,
-133.031f, 69.293f, 590.096f, -1434.69f, -1070.89f, 307.074f, 400.525f,
-316.12f, -587.125f, -161.056f, 800.363f, -96.4708f, 748.706f, 868.174f,
-447.938f, 112.737f, 1127.2f, 47.4355f, 677.72f, 593.186f, -336.4f,
551.362f, 397.823f, 78.3979f, -715.398f, 405.969f, 404.256f, 246.019f,
-8.42969f, 131.365f, -648.051f};
// Compare results with 1 batch
TEST(Analyzer_Resnet50_ipu, compare_results_1_batch) {
std::string model_dir = FLAGS_infer_model + "/" + "model";
AnalysisConfig config;
// ipu_device_num, ipu_micro_batch_size, ipu_enable_pipelining
config.EnableIpu(1, 1, false);
config.SetModel(model_dir + "/model", model_dir + "/params");
std::vector<PaddleTensor> inputs;
auto predictor = CreatePaddlePredictor(config);
const int batch = 1;
const int channel = 3;
const int height = 318;
const int width = 318;
const int input_num = batch * channel * height * width;
std::vector<float> input(input_num, 1);
PaddleTensor in;
in.shape = {batch, channel, height, width};
in.data =
PaddleBuf(static_cast<void*>(input.data()), input_num * sizeof(float));
in.dtype = PaddleDType::FLOAT32;
inputs.emplace_back(in);
std::vector<PaddleTensor> outputs;
ASSERT_TRUE(predictor->Run(inputs, &outputs));
const size_t expected_size = 1;
EXPECT_EQ(outputs.size(), expected_size);
float* data_o = static_cast<float*>(outputs[0].data.data());
for (size_t j = 0; j < outputs[0].data.length() / sizeof(float); j += 10) {
EXPECT_NEAR(
(data_o[j] - truth_values[j / 10]) / truth_values[j / 10], 0., 12e-5);
}
}
// Compare results with 2 batch
TEST(Analyzer_Resnet50_ipu, compare_results_2_batch) {
std::string model_dir = FLAGS_infer_model + "/" + "model";
AnalysisConfig config;
// ipu_device_num, ipu_micro_batch_size, ipu_enable_pipelining
config.EnableIpu(1, 2, false);
config.SetModel(model_dir + "/model", model_dir + "/params");
std::vector<PaddleTensor> inputs;
auto predictor = CreatePaddlePredictor(config);
const int batch = 2;
const int channel = 3;
const int height = 318;
const int width = 318;
const int input_num = batch * channel * height * width;
std::vector<float> input(input_num, 1);
PaddleTensor in;
in.shape = {batch, channel, height, width};
in.data =
PaddleBuf(static_cast<void*>(input.data()), input_num * sizeof(float));
in.dtype = PaddleDType::FLOAT32;
inputs.emplace_back(in);
std::vector<PaddleTensor> outputs;
ASSERT_TRUE(predictor->Run(inputs, &outputs));
const size_t expected_size = 1;
EXPECT_EQ(outputs.size(), expected_size);
float* data_o = static_cast<float*>(outputs[0].data.data());
auto num_output_per_batch = outputs[0].data.length() / sizeof(float) / 2;
for (size_t j = 0; j < num_output_per_batch; j += 10) {
EXPECT_NEAR(
(data_o[j] - truth_values[j / 10]) / truth_values[j / 10], 0., 12e-5);
EXPECT_NEAR((data_o[j + num_output_per_batch] - truth_values[j / 10]) /
truth_values[j / 10],
0.,
12e-5);
}
}
// multi threading
TEST(Analyzer_Resnet50_ipu, model_runtime_multi_thread) {
std::string model_dir = FLAGS_infer_model + "/" + "model";
AnalysisConfig config;
const int thread_num = 10;
// ipu_device_num, ipu_micro_batch_size, ipu_enable_pipelining
config.EnableIpu(1, 1, false);
config.SetIpuConfig(false, 1, 1.0, false, true);
config.SetModel(model_dir + "/model", model_dir + "/params");
auto main_predictor = CreatePaddlePredictor(config);
std::vector<std::vector<PaddleTensor>> inputs;
std::vector<std::vector<PaddleTensor>> outputs;
std::vector<decltype(main_predictor)> predictors;
std::vector<std::thread> threads;
outputs.resize(thread_num);
inputs.resize(thread_num);
const int batch = 1;
const int channel = 3;
const int height = 318;
const int width = 318;
const int input_num = batch * channel * height * width;
std::vector<float> input(input_num, 1);
PaddleTensor in;
in.shape = {batch, channel, height, width};
in.data =
PaddleBuf(static_cast<void*>(input.data()), input_num * sizeof(float));
in.dtype = PaddleDType::FLOAT32;
for (int i = 0; i < thread_num; ++i) {
inputs[i].emplace_back(in);
predictors.emplace_back(std::move(main_predictor->Clone()));
}
auto run = [](PaddlePredictor* predictor,
std::vector<PaddleTensor>& input,
std::vector<PaddleTensor>& output) {
ASSERT_TRUE(predictor->Run(input, &output));
};
for (int i = 0; i < thread_num; ++i) {
threads.emplace_back(
run, predictors[i].get(), std::ref(inputs[i]), std::ref(outputs[i]));
}
for (int i = 0; i < thread_num; ++i) {
threads[i].join();
}
const size_t expected_size = 1;
for (int i = 0; i < thread_num; ++i) {
EXPECT_EQ(outputs[i].size(), expected_size);
float* data_o = static_cast<float*>(outputs[i][0].data.data());
for (size_t j = 0; j < outputs[i][0].data.length() / sizeof(float);
j += 10) {
EXPECT_NEAR(
(data_o[j] - truth_values[j / 10]) / truth_values[j / 10], 0., 12e-5);
}
}
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/ipu_word2vec_sample.cc
+82
View File
@@ -0,0 +1,82 @@
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
/*
* This file contains a simple demo for how to take a model for inference with
* IPUs.
* Model: wget -q
* http://paddle-inference-dist.bj.bcebos.com/word2vec.inference.model.tar.gz
*/
#include <iostream>
#include <numeric>
#include <string>
#include <vector>
#include "glog/logging.h"
#include "paddle/common/flags.h"
#include "paddle/fluid/inference/api/paddle_inference_api.h"
PD_DEFINE_string(infer_model, "", "Directory of the inference model.");
using paddle_infer::Config;
using paddle_infer::CreatePredictor;
using paddle_infer::Predictor;
void inference(std::string model_path,
bool use_ipu,
std::vector<float> *out_data) {
//# 1. Create Predictor with a config.
Config config;
config.SetModel(FLAGS_infer_model);
if (use_ipu) {
// ipu_device_num, ipu_micro_batch_size
config.EnableIpu(1, 4);
}
auto predictor = CreatePredictor(config);
//# 2. Prepare input/output tensor.
auto input_names = predictor->GetInputNames();
std::vector<int64_t> data{1, 2, 3, 4};
// For simplicity, we set all the slots with the same data.
for (auto input_name : input_names) {
auto input_tensor = predictor->GetInputHandle(input_name);
input_tensor->Reshape({4, 1});
input_tensor->CopyFromCpu(data.data());
}
//# 3. Run
predictor->Run();
//# 4. Get output.
auto output_names = predictor->GetOutputNames();
auto output_tensor = predictor->GetOutputHandle(output_names[0]);
std::vector<int> output_shape = output_tensor->shape();
int out_num = std::accumulate(
output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
out_data->resize(out_num);
output_tensor->CopyToCpu(out_data->data());
}
int main(int argc, char *argv[]) {
::paddle::flags::ParseCommandLineFlags(&argc, &argv);
std::vector<float> ipu_result;
std::vector<float> cpu_result;
inference(FLAGS_infer_model, true, &ipu_result);
inference(FLAGS_infer_model, false, &cpu_result);
for (size_t i = 0; i < ipu_result.size(); i++) {
CHECK_NEAR(ipu_result[i], cpu_result[i], 1e-6);
}
LOG(INFO) << "Finished";
}
test/cpp/inference/api/lite_mul_model_test.cc
+126
View File
@@ -0,0 +1,126 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <cmath>
#include <mutex> // NOLINT
#include <thread> // NOLINT
#include "paddle/common/flags.h"
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
int test_predictor(const AnalysisConfig& config_in,
Barrier* barrier = nullptr) {
static std::mutex mutex;
AnalysisConfig config{config_in};
std::unique_ptr<PaddlePredictor> predictor;
{
std::unique_lock<std::mutex> lock(mutex);
predictor = CreatePaddlePredictor(config);
}
if (barrier) {
barrier->Wait();
}
std::vector<PaddleTensor> inputs;
std::vector<float> input({1});
PaddleTensor in;
in.shape = {1, 1};
in.data = PaddleBuf(static_cast<void*>(input.data()), 1 * sizeof(float));
in.dtype = PaddleDType::FLOAT32;
inputs.emplace_back(in);
std::vector<PaddleTensor> outputs;
predictor->Run(inputs, &outputs);
const std::vector<float> truth_values = {-0.00621776f,
-0.00620937f,
0.00990623f,
-0.0039817f,
-0.00074315f,
0.61229795f,
-0.00491806f,
-0.00068755f,
0.18409646f,
0.30090684f};
const size_t expected_size = 1;
EXPECT_EQ(outputs.size(), expected_size);
float* data_o = static_cast<float*>(outputs[0].data.data());
for (size_t j = 0; j < outputs[0].data.length() / sizeof(float); ++j) {
EXPECT_LT(std::abs(data_o[j] - truth_values[j]), 10e-6);
}
return 0;
}
int test_predictor_zero_copy(const AnalysisConfig& config_in,
Barrier* barrier = nullptr) {
static std::mutex mutex;
AnalysisConfig config{config_in};
std::unique_ptr<PaddlePredictor> predictor;
{
std::unique_lock<std::mutex> lock(mutex);
predictor = CreatePaddlePredictor(config);
}
if (barrier) {
barrier->Wait();
}
std::vector<float> input({1});
auto in_tensor =
predictor->GetInputTensor(predictor->GetInputNames().front());
in_tensor->Reshape({1, 1});
in_tensor->copy_from_cpu(input.data());
predictor->ZeroCopyRun();
auto out_tensor =
predictor->GetOutputTensor(predictor->GetOutputNames().front());
std::vector<float> data_o(10);
out_tensor->copy_to_cpu(data_o.data());
const std::vector<float> truth_values = {-0.00621776f,
-0.00620937f,
0.00990623f,
-0.0039817f,
-0.00074315f,
0.61229795f,
-0.00491806f,
-0.00068755f,
0.18409646f,
0.30090684f};
const size_t expected_size = 1;
EXPECT_EQ(predictor->GetOutputNames().size(), expected_size);
for (size_t j = 0; j < truth_values.size(); ++j) {
EXPECT_LT(std::abs(data_o[j] - truth_values[j]), 10e-6);
}
return 0;
}
#ifdef PADDLE_WITH_XPU
TEST(AnalysisPredictor, native_xpu) {
AnalysisConfig config;
config.EnableXpu();
config.SetModel(FLAGS_infer_model + "/" + "mul_model");
test_predictor(config);
test_predictor_zero_copy(config);
}
#endif
} // namespace inference
} // namespace paddle
test/cpp/inference/api/onnxruntime_predictor_tester.cc
+81
View File
@@ -0,0 +1,81 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <string>
#include <thread> // NOLINT
#include <vector>
#include "paddle/fluid/framework/ir/pass.h"
#include "paddle/fluid/framework/tensor.h"
#include "paddle/fluid/inference/api/helper.h"
#include "paddle/fluid/inference/api/onnxruntime_predictor.h"
#include "paddle/fluid/inference/api/paddle_api.h"
#include "paddle/fluid/inference/api/paddle_inference_api.h"
#include "paddle/fluid/inference/utils/io_utils.h"
#include "paddle/phi/backends/cpu/cpu_info.h"
#include "test/cpp/inference/api/tester_helper.h"
PD_DEFINE_string(dirname, "", "dirname to tests.");
namespace paddle {
TEST(ONNXRuntimePredictor, onnxruntime_on) {
AnalysisConfig config;
config.SetModel(FLAGS_dirname + "/inference.pdmodel",
FLAGS_dirname + "/inference.pdiparams");
config.EnableONNXRuntime();
config.EnableORTOptimization();
config.SetCpuMathLibraryNumThreads(2);
LOG(INFO) << config.Summary();
auto _predictor =
CreatePaddlePredictor<AnalysisConfig,
paddle::PaddleEngineKind::kONNXRuntime>(config);
ASSERT_TRUE(_predictor);
auto* predictor = static_cast<ONNXRuntimePredictor*>(_predictor.get());
ASSERT_TRUE(predictor);
ASSERT_TRUE(!predictor->Clone());
// Dummy Input Data
std::vector<int64_t> input_shape = {-1, 3, 224, 224};
std::vector<float> input_data(1 * 3 * 224 * 224, 1.0);
std::vector<float> out_data;
out_data.resize(1000);
// testing all interfaces
auto input_names = predictor->GetInputNames();
auto output_names = predictor->GetOutputNames();
auto get_input_shape = predictor->GetInputTensorShape();
ASSERT_EQ(input_names.size(), 1UL);
ASSERT_EQ(output_names.size(), 1UL);
ASSERT_EQ(input_names[0], "inputs");
ASSERT_EQ(output_names[0], "save_infer_model/scale_0.tmp_1");
ASSERT_EQ(get_input_shape["inputs"], input_shape);
auto input_tensor = predictor->GetInputTensor(input_names[0]);
input_tensor->Reshape({1, 3, 224, 224});
auto output_tensor = predictor->GetOutputTensor(output_names[0]);
input_tensor->CopyFromCpu(input_data.data());
ASSERT_TRUE(predictor->ZeroCopyRun());
output_tensor->CopyToCpu(out_data.data());
predictor->TryShrinkMemory();
}
} // namespace paddle
test/cpp/inference/api/paddle_infer_api_copy_tensor_tester.cc
+560
View File
@@ -0,0 +1,560 @@
/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <cuda_runtime.h>
#include <gtest/gtest.h>
#include <array>
#include <cstring>
#include <functional>
#include <numeric>
#include "glog/logging.h"
#include "paddle/common/flags.h"
#include "paddle/fluid/inference/api/paddle_infer_contrib.h"
#include "paddle/phi/common/float16.h"
#include "test/cpp/inference/api/trt_test_helper.h"
namespace paddle_infer {
class InferApiTesterUtils {
public:
static std::unique_ptr<Tensor> CreateInferTensorForTest(
const std::string &name, PlaceType place, void *p_scope) {
auto var = static_cast<paddle::framework::Scope *>(p_scope)->Var(name);
var->GetMutable<phi::DenseTensor>();
phi::DeviceContextPool &pool = phi::DeviceContextPool::Instance();
const auto &dev_ctxs = pool.device_contexts();
std::unique_ptr<Tensor> res(new Tensor(p_scope, &dev_ctxs));
res->input_or_output_ = true;
res->SetName(name);
res->SetPlace(place, 0 /*device id*/);
return res;
}
};
TEST(Tensor, copy_to_cpu_async_stream) {
LOG(INFO) << GetVersion();
UpdateDllFlag("conv_workspace_size_limit", "4000");
std::string model_dir = FLAGS_infer_model + "/model";
Config config;
config.EnableNewIR(false);
config.SetModel(model_dir + "/model", model_dir + "/params");
config.EnableUseGpu(100, 0);
auto predictor = CreatePredictor(config);
auto pred_clone = predictor->Clone();
std::vector<int> in_shape = {1, 3, 318, 318};
int in_num = std::accumulate(
in_shape.begin(), in_shape.end(), 1, std::multiplies<int>());
std::vector<float> input(in_num, 1.0);
const auto &input_names = predictor->GetInputNames();
auto input_tensor = predictor->GetInputHandle(input_names[0]);
input_tensor->Reshape(in_shape);
input_tensor->CopyFromCpu(input.data());
predictor->Run();
const auto &output_names = predictor->GetOutputNames();
auto output_tensor = predictor->GetOutputHandle(output_names[0]);
std::vector<int> output_shape = output_tensor->shape();
int out_num = std::accumulate(
output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
float *out_data = static_cast<float *>(
contrib::TensorUtils::CudaMallocPinnedMemory(sizeof(float) * out_num));
memset(out_data, 0, sizeof(float) * out_num);
std::vector<float> correct_out_data = {
127.78,
1.07353,
-229.42,
1127.28,
-177.365,
-292.412,
-271.614,
466.054,
540.436,
-214.223,
};
for (int i = 0; i < 100; i++) {
predictor->Run();
}
cudaStream_t stream;
output_tensor->CopyToCpuAsync(out_data, static_cast<void *>(&stream));
// sync
cudaStreamSynchronize(stream);
for (int i = 0; i < 10; i++) {
EXPECT_NEAR(out_data[i] / correct_out_data[i], 1.0, 1e-3);
}
contrib::TensorUtils::CudaFreePinnedMemory(static_cast<void *>(out_data));
}
TEST(Tensor, copy_to_cpu_async_callback) {
LOG(INFO) << GetVersion();
UpdateDllFlag("conv_workspace_size_limit", "4000");
std::string model_dir = FLAGS_infer_model + "/model";
Config config;
config.SwitchIrOptim(false);
config.EnableNewIR(false);
config.SetModel(model_dir + "/model", model_dir + "/params");
config.EnableUseGpu(100, 0);
auto predictor = CreatePredictor(config);
auto pred_clone = predictor->Clone();
std::vector<int> in_shape = {1, 3, 318, 318};
int in_num = std::accumulate(
in_shape.begin(), in_shape.end(), 1, std::multiplies<int>());
std::vector<float> input(in_num, 1.0);
const auto &input_names = predictor->GetInputNames();
auto input_tensor = predictor->GetInputHandle(input_names[0]);
input_tensor->Reshape(in_shape);
input_tensor->CopyFromCpu(input.data());
predictor->Run();
const auto &output_names = predictor->GetOutputNames();
auto output_tensor = predictor->GetOutputHandle(output_names[0]);
std::vector<int> output_shape = output_tensor->shape();
int out_num = std::accumulate(
output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
float *out_data = static_cast<float *>(
contrib::TensorUtils::CudaMallocPinnedMemory(sizeof(float) * out_num));
memset(out_data, 0, sizeof(float) * out_num);
for (int i = 0; i < 100; i++) {
predictor->Run();
}
cudaDeviceSynchronize();
output_tensor->CopyToCpuAsync(
out_data,
[](void *cb_params) {
float *data = static_cast<float *>(cb_params);
std::vector<float> correct_out_data = {
127.78,
1.07353,
-229.42,
1127.28,
-177.365,
-292.412,
-271.614,
466.054,
540.436,
-214.223,
};
for (int i = 0; i < 10; i++) {
EXPECT_NEAR(data[i] / correct_out_data[i], 1.0, 1e-3);
}
},
static_cast<void *>(out_data));
cudaDeviceSynchronize();
contrib::TensorUtils::CudaFreePinnedMemory(static_cast<void *>(out_data));
}
template <class DTYPE>
static void test_copy_tensor(PlaceType src_place, PlaceType dst_place) {
paddle::framework::Scope scope;
auto tensor_src = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
"tensor_src", src_place, static_cast<void *>(&scope));
auto tensor_dst = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
"tensor_dst", dst_place, static_cast<void *>(&scope));
std::vector<DTYPE> data_src(6, 1);
tensor_src->Reshape({2, 3});
tensor_src->CopyFromCpu(data_src.data());
std::vector<DTYPE> data_dst(4, 2);
tensor_dst->Reshape({2, 2});
tensor_dst->CopyFromCpu(data_dst.data());
paddle_infer::contrib::TensorUtils::CopyTensor(tensor_dst.get(), *tensor_src);
EXPECT_EQ(tensor_dst->shape().size(), (size_t)2);
EXPECT_EQ(tensor_dst->shape()[0], 2);
EXPECT_EQ(tensor_dst->shape()[1], 3);
std::vector<DTYPE> data_check(6, 3);
tensor_dst->CopyToCpu<DTYPE>(static_cast<DTYPE *>(data_check.data()));
for (int i = 0; i < 6; i++) {
EXPECT_NEAR(data_check[i], 1, 1e-5);
}
}
TEST(CopyTensor, float64) {
test_copy_tensor<double>(PlaceType::kCPU, PlaceType::kCPU);
test_copy_tensor<double>(PlaceType::kCPU, PlaceType::kGPU);
test_copy_tensor<double>(PlaceType::kGPU, PlaceType::kCPU);
test_copy_tensor<double>(PlaceType::kGPU, PlaceType::kGPU);
}
TEST(CopyTensor, float32) {
test_copy_tensor<float>(PlaceType::kCPU, PlaceType::kCPU);
test_copy_tensor<float>(PlaceType::kCPU, PlaceType::kGPU);
test_copy_tensor<float>(PlaceType::kGPU, PlaceType::kCPU);
test_copy_tensor<float>(PlaceType::kGPU, PlaceType::kGPU);
}
TEST(CopyTensor, int32) {
test_copy_tensor<int32_t>(PlaceType::kCPU, PlaceType::kCPU);
test_copy_tensor<int32_t>(PlaceType::kCPU, PlaceType::kGPU);
test_copy_tensor<int32_t>(PlaceType::kGPU, PlaceType::kCPU);
test_copy_tensor<int32_t>(PlaceType::kGPU, PlaceType::kGPU);
}
TEST(CopyTensor, int64) {
test_copy_tensor<int64_t>(PlaceType::kCPU, PlaceType::kCPU);
test_copy_tensor<int64_t>(PlaceType::kCPU, PlaceType::kGPU);
test_copy_tensor<int64_t>(PlaceType::kGPU, PlaceType::kCPU);
test_copy_tensor<int64_t>(PlaceType::kGPU, PlaceType::kGPU);
}
TEST(CopyTensor, int8) {
test_copy_tensor<int8_t>(PlaceType::kCPU, PlaceType::kCPU);
test_copy_tensor<int8_t>(PlaceType::kCPU, PlaceType::kGPU);
test_copy_tensor<int8_t>(PlaceType::kGPU, PlaceType::kCPU);
test_copy_tensor<int8_t>(PlaceType::kGPU, PlaceType::kGPU);
}
TEST(CopyTensor, uint8) {
test_copy_tensor<uint8_t>(PlaceType::kCPU, PlaceType::kCPU);
test_copy_tensor<uint8_t>(PlaceType::kCPU, PlaceType::kGPU);
test_copy_tensor<uint8_t>(PlaceType::kGPU, PlaceType::kCPU);
test_copy_tensor<uint8_t>(PlaceType::kGPU, PlaceType::kGPU);
}
TEST(CopyTensor, bool_cpu_to_cpu) {
paddle::framework::Scope scope;
auto tensor_src = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
"tensor_src", PlaceType::kCPU, static_cast<void *>(&scope));
auto tensor_dst = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
"tensor_dst", PlaceType::kCPU, static_cast<void *>(&scope));
std::array<bool, 6> data_src;
data_src.fill(true);
tensor_src->Reshape({2, 3});
tensor_src->CopyFromCpu(data_src.data());
std::array<bool, 4> data_dst;
data_dst.fill(false);
tensor_dst->Reshape({2, 2});
tensor_dst->CopyFromCpu(data_dst.data());
paddle_infer::contrib::TensorUtils::CopyTensor(tensor_dst.get(), *tensor_src);
EXPECT_EQ(tensor_dst->shape().size(), (size_t)2);
EXPECT_EQ(tensor_dst->shape()[0], 2);
EXPECT_EQ(tensor_dst->shape()[1], 3);
std::array<bool, 6> data_check;
data_check.fill(false);
tensor_dst->CopyToCpu<bool>(data_check.data());
for (int i = 0; i < 6; i++) {
EXPECT_TRUE(data_check[i] == true);
}
}
TEST(CopyTensor, bool_gpu_to_gpu) {
paddle::framework::Scope scope;
auto tensor_src = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
"tensor_src", PlaceType::kGPU, static_cast<void *>(&scope));
auto tensor_dst = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
"tensor_dst", PlaceType::kGPU, static_cast<void *>(&scope));
std::array<bool, 6> data_src;
data_src.fill(true);
tensor_src->Reshape({2, 3});
tensor_src->CopyFromCpu(data_src.data());
std::array<bool, 4> data_dst;
data_dst.fill(false);
tensor_dst->Reshape({2, 2});
tensor_dst->CopyFromCpu(data_dst.data());
paddle_infer::contrib::TensorUtils::CopyTensor(tensor_dst.get(), *tensor_src);
EXPECT_EQ(tensor_dst->shape().size(), (size_t)2);
EXPECT_EQ(tensor_dst->shape()[0], 2);
EXPECT_EQ(tensor_dst->shape()[1], 3);
std::array<bool, 6> data_check;
data_check.fill(false);
tensor_dst->CopyToCpu<bool>(data_check.data());
for (int i = 0; i < 6; i++) {
EXPECT_TRUE(data_check[i] == true);
}
}
TEST(CopyTensor, bool_gpu_to_cpu) {
paddle::framework::Scope scope;
auto tensor_src = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
"tensor_src", PlaceType::kGPU, static_cast<void *>(&scope));
auto tensor_dst = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
"tensor_dst", PlaceType::kCPU, static_cast<void *>(&scope));
std::array<bool, 6> data_src;
data_src.fill(true);
tensor_src->Reshape({2, 3});
tensor_src->CopyFromCpu(data_src.data());
std::array<bool, 4> data_dst;
data_dst.fill(false);
tensor_dst->Reshape({2, 2});
tensor_dst->CopyFromCpu(data_dst.data());
paddle_infer::contrib::TensorUtils::CopyTensor(tensor_dst.get(), *tensor_src);
EXPECT_EQ(tensor_dst->shape().size(), (size_t)2);
EXPECT_EQ(tensor_dst->shape()[0], 2);
EXPECT_EQ(tensor_dst->shape()[1], 3);
std::array<bool, 6> data_check;
data_check.fill(false);
tensor_dst->CopyToCpu<bool>(data_check.data());
for (int i = 0; i < 6; i++) {
EXPECT_TRUE(data_check[i] == true);
}
}
TEST(CopyTensor, bool_cpu_to_gpu) {
paddle::framework::Scope scope;
auto tensor_src = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
"tensor_src", PlaceType::kCPU, static_cast<void *>(&scope));
auto tensor_dst = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
"tensor_dst", PlaceType::kGPU, static_cast<void *>(&scope));
std::array<bool, 6> data_src;
data_src.fill(true);
tensor_src->Reshape({2, 3});
tensor_src->CopyFromCpu(data_src.data());
std::array<bool, 4> data_dst;
data_dst.fill(false);
tensor_dst->Reshape({2, 2});
tensor_dst->CopyFromCpu(data_dst.data());
paddle_infer::contrib::TensorUtils::CopyTensor(tensor_dst.get(), *tensor_src);
EXPECT_EQ(tensor_dst->shape().size(), (size_t)2);
EXPECT_EQ(tensor_dst->shape()[0], 2);
EXPECT_EQ(tensor_dst->shape()[1], 3);
std::array<bool, 6> data_check{false};
data_check.fill(false);
tensor_dst->CopyToCpu<bool>(data_check.data());
for (int i = 0; i < 6; i++) {
EXPECT_TRUE(data_check[i] == true);
}
}
TEST(CopyTensor, float16_cpu_to_cpu) {
paddle::framework::Scope scope;
auto tensor_src = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
"tensor_src", PlaceType::kCPU, static_cast<void *>(&scope));
auto tensor_dst = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
"tensor_dst", PlaceType::kCPU, static_cast<void *>(&scope));
using phi::dtype::float16;
std::vector<float16> data_src(6, float16(1.0));
tensor_src->Reshape({2, 3});
tensor_src->CopyFromCpu(data_src.data());
std::vector<float16> data_dst(4, float16(2.0));
tensor_dst->Reshape({2, 2});
tensor_dst->CopyFromCpu(data_dst.data());
paddle_infer::contrib::TensorUtils::CopyTensor(tensor_dst.get(), *tensor_src);
EXPECT_EQ(tensor_dst->shape().size(), (size_t)2);
EXPECT_EQ(tensor_dst->shape()[0], 2);
EXPECT_EQ(tensor_dst->shape()[1], 3);
std::vector<float16> data_check(6, float16(2.0));
tensor_dst->CopyToCpu<float16>(data_check.data());
for (int i = 0; i < 6; i++) {
EXPECT_TRUE(data_check[i] == float16(1.0));
}
}
TEST(CopyTensor, float16_gpu_to_gpu) {
paddle::framework::Scope scope;
auto tensor_src = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
"tensor_src", PlaceType::kGPU, static_cast<void *>(&scope));
auto tensor_dst = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
"tensor_dst", PlaceType::kGPU, static_cast<void *>(&scope));
using phi::dtype::float16;
std::vector<float16> data_src(6, float16(1.0));
tensor_src->Reshape({2, 3});
tensor_src->CopyFromCpu(data_src.data());
std::vector<float16> data_dst(4, float16(2.0));
tensor_dst->Reshape({2, 2});
tensor_dst->CopyFromCpu(data_dst.data());
paddle_infer::contrib::TensorUtils::CopyTensor(tensor_dst.get(), *tensor_src);
EXPECT_EQ(tensor_dst->shape().size(), (size_t)2);
EXPECT_EQ(tensor_dst->shape()[0], 2);
EXPECT_EQ(tensor_dst->shape()[1], 3);
std::vector<float16> data_check(6, float16(2.0));
tensor_dst->CopyToCpu<float16>(data_check.data());
for (int i = 0; i < 6; i++) {
EXPECT_TRUE(data_check[i] == float16(1.0));
}
}
TEST(CopyTensor, float16_cpu_to_gpu) {
paddle::framework::Scope scope;
auto tensor_src = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
"tensor_src", PlaceType::kCPU, static_cast<void *>(&scope));
auto tensor_dst = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
"tensor_dst", PlaceType::kGPU, static_cast<void *>(&scope));
using phi::dtype::float16;
std::vector<float16> data_src(6, float16(1.0));
tensor_src->Reshape({2, 3});
tensor_src->CopyFromCpu(data_src.data());
std::vector<float16> data_dst(4, float16(2.0));
tensor_dst->Reshape({2, 2});
tensor_dst->CopyFromCpu(data_dst.data());
paddle_infer::contrib::TensorUtils::CopyTensor(tensor_dst.get(), *tensor_src);
EXPECT_EQ(tensor_dst->shape().size(), (size_t)2);
EXPECT_EQ(tensor_dst->shape()[0], 2);
EXPECT_EQ(tensor_dst->shape()[1], 3);
std::vector<float16> data_check(6, float16(2.0));
tensor_dst->CopyToCpu<float16>(data_check.data());
for (int i = 0; i < 6; i++) {
EXPECT_TRUE(data_check[i] == float16(1.0));
}
}
TEST(CopyTensor, float16_gpu_to_cpu) {
paddle::framework::Scope scope;
auto tensor_src = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
"tensor_src", PlaceType::kGPU, static_cast<void *>(&scope));
auto tensor_dst = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
"tensor_dst", PlaceType::kCPU, static_cast<void *>(&scope));
using phi::dtype::float16;
std::vector<float16> data_src(6, float16(1.0));
tensor_src->Reshape({2, 3});
tensor_src->CopyFromCpu(data_src.data());
std::vector<float16> data_dst(4, float16(2.0));
tensor_dst->Reshape({2, 2});
tensor_dst->CopyFromCpu(data_dst.data());
paddle_infer::contrib::TensorUtils::CopyTensor(tensor_dst.get(), *tensor_src);
EXPECT_EQ(tensor_dst->shape().size(), (size_t)2);
EXPECT_EQ(tensor_dst->shape()[0], 2);
EXPECT_EQ(tensor_dst->shape()[1], 3);
std::vector<float16> data_check(6, float16(2.0));
tensor_dst->CopyToCpu<float16>(data_check.data());
for (int i = 0; i < 6; i++) {
EXPECT_TRUE(data_check[i] == float16(1.0));
}
}
TEST(CopyTensor, async_stream) {
paddle::framework::Scope scope;
auto tensor_src = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
"tensor_src", PlaceType::kGPU, static_cast<void *>(&scope));
auto tensor_dst = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
"tensor_dst", PlaceType::kGPU, static_cast<void *>(&scope));
std::vector<float> data_src(6, 1.0);
tensor_src->Reshape({2, 3});
tensor_src->CopyFromCpu(data_src.data());
std::vector<float> data_dst(4, 2.0);
tensor_dst->Reshape({2, 2});
tensor_dst->CopyFromCpu(data_dst.data());
cudaStream_t stream;
paddle_infer::contrib::TensorUtils::CopyTensorAsync(
tensor_dst.get(), *tensor_src, static_cast<void *>(&stream));
EXPECT_EQ(tensor_dst->shape().size(), (size_t)2);
EXPECT_EQ(tensor_dst->shape()[0], 2);
EXPECT_EQ(tensor_dst->shape()[1], 3);
cudaStreamSynchronize(stream);
std::vector<float> data_check(6, 1.0);
tensor_dst->CopyToCpu<float>(data_check.data());
for (int i = 0; i < 6; i++) {
EXPECT_NEAR(data_check[i], static_cast<float>(1.0), 1e-5);
}
}
TEST(CopyTensor, async_callback) {
paddle::framework::Scope scope;
auto tensor_src = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
"tensor_src", PlaceType::kCPU, static_cast<void *>(&scope));
auto tensor_dst = paddle_infer::InferApiTesterUtils::CreateInferTensorForTest(
"tensor_dst", PlaceType::kGPU, static_cast<void *>(&scope));
std::vector<float> data_src(6, 1.0);
tensor_src->Reshape({2, 3});
tensor_src->CopyFromCpu(data_src.data());
std::vector<float> data_dst(4, 2.0);
tensor_dst->Reshape({2, 2});
tensor_dst->CopyFromCpu(data_dst.data());
paddle_infer::contrib::TensorUtils::CopyTensorAsync(
tensor_dst.get(),
*tensor_src,
[](void *cb_params) {
Tensor *tensor = static_cast<Tensor *>(cb_params);
EXPECT_EQ(tensor->shape().size(), (size_t)2);
EXPECT_EQ(tensor->shape()[0], 2);
EXPECT_EQ(tensor->shape()[1], 3);
},
static_cast<void *>(&(*tensor_dst)));
cudaDeviceSynchronize();
}
} // namespace paddle_infer
test/cpp/inference/api/paddle_infer_api_errors_tester.cc
+90
View File
@@ -0,0 +1,90 @@
// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "glog/logging.h"
#include "gtest/gtest.h"
#include "paddle/common/flags.h"
#include "paddle/fluid/inference/api/paddle_infer_contrib.h"
#include "paddle/fluid/platform/enforce.h"
namespace paddle_infer {
namespace contrib {
TEST(Status, ctor) { CHECK(Status::OK().ok()); }
struct FakeException {
void pd_exception(int a) const {
PADDLE_ENFORCE_NE(a,
a,
common::errors::InvalidArgument(
"This is a preset error message used to verify "
"whether the exception meets expectations: %d, %d.",
a,
a));
}
[[noreturn]] void base_exception() const { throw std::exception(); }
void no_exception() const noexcept {}
};
TEST(Status, pd_exception) {
FakeException e;
Status status = get_status([&]() { e.pd_exception(1); });
PADDLE_ENFORCE_EQ(
status.ok(),
false,
common::errors::PreconditionNotMet("Status should not be OK."));
PADDLE_ENFORCE_EQ(
status == status,
true,
common::errors::PreconditionNotMet("Status should be equal to itself."));
PADDLE_ENFORCE_EQ(status != status,
false,
common::errors::PreconditionNotMet(
"Status should not be different from itself."));
PADDLE_ENFORCE_EQ(
status.code(),
common::ErrorCode::INVALID_ARGUMENT + 1,
common::errors::InvalidArgument(
"Required status.code() should be equal to INVALID_ARGUMENT + 1. "));
LOG(INFO) << status.error_message();
}
TEST(Status, basic_exception) {
FakeException e;
Status status;
status = get_status([&]() { e.base_exception(); });
PADDLE_ENFORCE_EQ(
status.ok(),
false,
common::errors::PreconditionNotMet("Status should not be OK."));
LOG(INFO) << status.error_message();
}
TEST(Status, no_exception) {
FakeException e;
Status status;
status = get_status([&]() { e.no_exception(); });
PADDLE_ENFORCE_EQ(status.ok(),
true,
common::errors::PreconditionNotMet("Status should be OK."));
}
TEST(Status, copy) {
Status status;
Status status_1(status);
status_1 = status;
}
} // namespace contrib
} // namespace paddle_infer
test/cpp/inference/api/paddle_infer_api_test.cc
+97
View File
@@ -0,0 +1,97 @@
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <functional>
#include <numeric>
#include "paddle/common/flags.h"
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle_infer {
TEST(Predictor, use_gpu) {
LOG(INFO) << GetVersion();
UpdateDllFlag("conv_workspace_size_limit", "4000");
std::string model_dir = FLAGS_infer_model + "/model";
Config config;
config.EnableNewIR(false);
config.SetModel(model_dir + "/model", model_dir + "/params");
config.EnableUseGpu(100, 0);
auto predictor = CreatePredictor(config);
auto pred_clone = predictor->Clone();
std::vector<int> in_shape = {1, 3, 318, 318};
int in_num = std::accumulate(
in_shape.begin(), in_shape.end(), 1, std::multiplies<int>());
std::vector<float> input(in_num, 0);
auto input_names = predictor->GetInputNames();
auto input_t = predictor->GetInputHandle(input_names[0]);
input_t->Reshape(in_shape);
input_t->CopyFromCpu(input.data());
predictor->Run();
auto output_names = predictor->GetOutputNames();
auto output_t = predictor->GetOutputHandle(output_names[0]);
std::vector<int> output_shape = output_t->shape();
int out_num = std::accumulate(
output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
std::vector<float> out_data;
out_data.resize(out_num);
output_t->CopyToCpu(out_data.data());
predictor->ClearIntermediateTensor();
}
TEST(PredictorPool, basic) {
LOG(INFO) << GetVersion();
UpdateDllFlag("conv_workspace_size_limit", "4000");
std::string model_dir = FLAGS_infer_model + "/model";
Config config;
config.EnableNewIR(false);
config.SetModel(model_dir + "/model", model_dir + "/params");
config.EnableUseGpu(100, 0);
services::PredictorPool pred_pool(config, 4);
auto pred = pred_pool.Retrieve(2);
std::vector<int> in_shape = {1, 3, 318, 318};
int in_num = std::accumulate(
in_shape.begin(), in_shape.end(), 1, std::multiplies<int>());
std::vector<float> input(in_num, 0);
auto in_names = pred->GetInputNames();
auto input_t = pred->GetInputHandle(in_names[0]);
input_t->name();
input_t->Reshape(in_shape);
input_t->CopyFromCpu(input.data());
pred->Run();
auto out_names = pred->GetOutputNames();
auto output_t = pred->GetOutputHandle(out_names[0]);
auto out_type = output_t->type();
LOG(INFO) << GetNumBytesOfDataType(out_type);
if (out_type == DataType::FLOAT32) {
PlaceType place;
int size;
output_t->data<float>(&place, &size);
}
}
} // namespace paddle_infer
test/cpp/inference/api/test_detection_dataset_preprocess.py
+30
View File
@@ -0,0 +1,30 @@
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import unittest
class Test_Preprocess(unittest.TestCase):
def test_local_convert(self):
os.system("python full_pascalvoc_test_preprocess.py --choice=local")
def test_online_convert(self):
os.system(
"python full_pascalvoc_test_preprocess.py --choice=VOC_test_2007"
)
if __name__ == '__main__':
unittest.main()
test/cpp/inference/api/tester_helper.h
+1209
View File
File diff suppressed because it is too large Load Diff
test/cpp/inference/api/trt_cascade_rcnn_test.cc
+62
View File
@@ -0,0 +1,62 @@
/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include "paddle/common/flags.h"
#include "test/cpp/inference/api/trt_test_helper.h"
namespace paddle {
namespace inference {
TEST(TensorRT, cascade_rcnn) {
std::string model_dir = FLAGS_infer_model + "/cascade_rcnn";
AnalysisConfig config;
int batch_size = 1;
config.EnableNewIR(false);
config.EnableUseGpu(100, 0);
config.SetModel(model_dir + "/model", model_dir + "/params");
config.EnableTensorRtEngine(
1 << 30, batch_size, 40, AnalysisConfig::Precision::kFloat32, false);
auto predictor = CreatePaddlePredictor(config);
int channels = 3;
int height = 640;
int width = 640;
int input_num = batch_size * channels * height * width;
float *input = new float[input_num];
memset(input, 1.0, input_num * sizeof(float));
float *im_shape = new float[3];
im_shape[0] = 3.0;
im_shape[1] = 640.0;
im_shape[2] = 640.0;
auto input_names = predictor->GetInputNames();
auto input_t = predictor->GetInputTensor(input_names[0]);
input_t->Reshape({batch_size, channels, height, width});
input_t->copy_from_cpu(input);
auto input_t1 = predictor->GetInputTensor(input_names[1]);
input_t1->Reshape({batch_size, 3});
input_t1->copy_from_cpu(im_shape);
ASSERT_TRUE(predictor->ZeroCopyRun());
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/trt_disable_tensorrt_half_ops_test.cc
+43
View File
@@ -0,0 +1,43 @@
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include "test/cpp/inference/api/trt_test_helper.h"
namespace paddle {
namespace inference {
TEST(TensorRT, disable_tensorrt_half_ops) {
std::string model_dir = FLAGS_infer_model + "/resnet50";
AnalysisConfig config;
config.SetModel(model_dir);
config.EnableUseGpu(100, 0);
config.EnableTensorRtEngine(
1 << 30, 1, 5, AnalysisConfig::Precision::kHalf, false, false);
paddle_infer::experimental::InternalUtils::DisableTensorRtHalfOps(&config,
{"conv2d"});
std::vector<std::vector<PaddleTensor>> inputs_all;
auto predictor = CreatePaddlePredictor(config);
SetFakeImageInput(&inputs_all, model_dir, false, "__model__", "");
std::vector<PaddleTensor> outputs;
for (auto &input : inputs_all) {
ASSERT_TRUE(predictor->Run(input, &outputs));
predictor->ClearIntermediateTensor();
}
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/trt_dynamic_shape_ernie_fp16_serialize_deserialize_test.cc
+38
View File
@@ -0,0 +1,38 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <dirent.h>
#ifndef _WIN32
#include <unistd.h>
#else // headers below are substitute of unistd.h in windows
#include <io.h>
#include <process.h>
#endif
#include <glog/logging.h>
#include <gtest/gtest.h>
#include "paddle/common/flags.h"
#include "test/cpp/inference/api/trt_dynamic_shape_ernie_serialize_deserialize_test.h"
namespace paddle {
namespace inference {
TEST(AnalysisPredictor, fp16) {
std::vector<float> result = {0.59923654, 0.21923761, 0.18152587};
trt_ernie(true, result);
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/trt_dynamic_shape_ernie_serialize_deserialize_test.cc
+40
View File
@@ -0,0 +1,40 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <dirent.h>
#ifndef _WIN32
#include <unistd.h>
#else // headers below are substitute of unistd.h in windows
#include <io.h>
#include <process.h>
#endif
#include <glog/logging.h>
#include <gtest/gtest.h>
#include "paddle/common/flags.h"
#include "test/cpp/inference/api/trt_dynamic_shape_ernie_serialize_deserialize_test.h"
namespace paddle {
namespace inference {
#if defined _WIN32
#else
TEST(AnalysisPredictor, no_fp16) {
std::vector<float> result = {0.597841, 0.219972, 0.182187};
trt_ernie(false, result);
}
#endif
} // namespace inference
} // namespace paddle
test/cpp/inference/api/trt_dynamic_shape_ernie_serialize_deserialize_test.h
+153
View File
@@ -0,0 +1,153 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#pragma once
#include <glog/logging.h>
#include <gtest/gtest.h>
#ifndef _WIN32
#include <unistd.h>
#else // headers below are substitute of unistd.h in windows
#include <io.h>
#include <process.h>
#endif
#include <functional>
#include <map>
#include <string>
#include <vector>
#include "paddle/common/flags.h"
#include "test/cpp/inference/api/trt_test_helper.h"
namespace paddle {
namespace inference {
static void run(const AnalysisConfig& config, std::vector<float>* out_data) {
auto predictor = CreatePaddlePredictor(config);
auto input_names = predictor->GetInputNames();
int run_batch = 1;
const int run_seq_len = 128;
std::vector<int32_t> tmp_input;
std::vector<float> tmp_four_input;
tmp_input.reserve(run_batch * run_seq_len);
tmp_four_input.reserve(run_batch * run_seq_len);
int32_t i0[run_seq_len] = {
1, 3558, 4, 75, 491, 89, 340, 313, 93, 4, 255, 10, 75, 321,
4095, 1902, 4, 134, 49, 75, 311, 14, 44, 178, 543, 15, 12043, 2,
75, 201, 340, 9, 14, 44, 486, 218, 1140, 279, 12043, 2};
int32_t i1[run_seq_len] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
int32_t i2[run_seq_len] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
30, 31, 32, 33, 34, 35, 36, 37, 38, 39};
float i3[run_seq_len] = {1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0};
// first input
auto input_t = predictor->GetInputTensor(input_names[0]);
input_t->Reshape({run_batch, run_seq_len, 1});
input_t->copy_from_cpu(i0);
// second input
auto input_t2 = predictor->GetInputTensor(input_names[1]);
input_t2->Reshape({run_batch, run_seq_len, 1});
input_t2->copy_from_cpu(i1);
// third input.
auto input_t3 = predictor->GetInputTensor(input_names[2]);
input_t3->Reshape({run_batch, run_seq_len, 1});
input_t3->copy_from_cpu(i2);
auto input_t4 = predictor->GetInputTensor(input_names[3]);
input_t4->Reshape({run_batch, run_seq_len, 1});
input_t4->copy_from_cpu(i3);
ASSERT_TRUE(predictor->ZeroCopyRun());
auto output_names = predictor->GetOutputNames();
auto output_t = predictor->GetOutputTensor(output_names[0]);
std::vector<int> output_shape = output_t->shape();
int out_num = std::accumulate(
output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
out_data->resize(out_num);
output_t->copy_to_cpu(out_data->data());
}
static void trt_ernie(bool with_fp16, std::vector<float> result) {
AnalysisConfig config;
std::string model_dir = FLAGS_infer_model;
// Delete serialization cache to perform serialization first rather than
// deserialization.
std::string opt_cache_dir = FLAGS_infer_model + "/opt_cache";
delete_cache_files(opt_cache_dir);
config.SetOptimCacheDir(opt_cache_dir);
SetConfig(&config, model_dir, true /* use_gpu */);
int batch = 1;
int min_seq_len = 1;
int max_seq_len = 128;
int opt_seq_len = 128;
std::vector<int> min_shape = {batch, min_seq_len, 1};
std::vector<int> max_shape = {batch, max_seq_len, 1};
std::vector<int> opt_shape = {batch, opt_seq_len, 1};
// Set the input's min, max, opt shape
std::map<std::string, std::vector<int>> min_input_shape = {
{"read_file_0.tmp_0", min_shape},
{"read_file_0.tmp_1", min_shape},
{"read_file_0.tmp_2", min_shape},
{"read_file_0.tmp_4", min_shape}};
std::map<std::string, std::vector<int>> max_input_shape = {
{"read_file_0.tmp_0", max_shape},
{"read_file_0.tmp_1", max_shape},
{"read_file_0.tmp_2", max_shape},
{"read_file_0.tmp_4", max_shape}};
std::map<std::string, std::vector<int>> opt_input_shape = {
{"read_file_0.tmp_0", opt_shape},
{"read_file_0.tmp_1", opt_shape},
{"read_file_0.tmp_2", opt_shape},
{"read_file_0.tmp_4", opt_shape}};
auto precision = AnalysisConfig::Precision::kFloat32;
if (with_fp16) {
precision = AnalysisConfig::Precision::kHalf;
}
config.EnableTensorRtEngine(1 << 30, 1, 5, precision, true, false);
config.SetTRTDynamicShapeInfo(
min_input_shape, max_input_shape, opt_input_shape);
paddle_infer::experimental::InternalUtils::SetTransformerMaskid(
&config, "read_file_0.tmp_4");
AnalysisConfig* config_deser = new AnalysisConfig(config);
std::vector<float> out_data;
run(config, &out_data); // serialize
run(*config_deser, &out_data); // deserialize
for (size_t i = 0; i < out_data.size(); i++) {
EXPECT_NEAR(result[i], out_data[i], 1e-2);
}
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/trt_dynamic_shape_ernie_test.cc
+443
View File
@@ -0,0 +1,443 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include "paddle/common/flags.h"
#include "paddle/common/enforce.h"
#include "paddle/fluid/inference/tensorrt/helper.h"
#include "test/cpp/inference/api/trt_test_helper.h"
namespace paddle {
namespace inference {
void run(const AnalysisConfig& config, std::vector<float>* out_data, int bs) {
#if !defined(_WIN32)
setenv("NVIDIA_TF32_OVERRIDE", "0", 1);
#endif
auto predictor = CreatePaddlePredictor(config);
auto input_names = predictor->GetInputNames();
int run_batch = bs;
const int run_seq_len = 128;
size_t len = run_batch * run_seq_len;
std::array<int32_t, 128> i0_bs1 = {
1, 3558, 4, 75, 491, 89, 340, 313, 93, 4, 255, 10, 75, 321,
4095, 1902, 4, 134, 49, 75, 311, 14, 44, 178, 543, 15, 12043, 2,
75, 201, 340, 9, 14, 44, 486, 218, 1140, 279, 12043, 2};
std::array<int32_t, 128> i1_bs1 = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
std::array<int32_t, 128> i2_bs1 = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
30, 31, 32, 33, 34, 35, 36, 37, 38, 39};
std::array<float, 128> i3_bs1 = {
1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0};
std::vector<int32_t> i0_data(len), i1_data(len), i2_data(len);
std::vector<float> i3_data(len);
for (size_t i = 0; i < len; i++) {
i0_data[i] = i0_bs1[i % run_seq_len];
i1_data[i] = i1_bs1[i % run_seq_len];
i2_data[i] = i2_bs1[i % run_seq_len];
i3_data[i] = i3_bs1[i % run_seq_len];
}
// first input
auto input_t = predictor->GetInputTensor(input_names[0]);
input_t->Reshape({run_batch, run_seq_len, 1});
input_t->copy_from_cpu(i0_data.data());
// second input
auto input_t2 = predictor->GetInputTensor(input_names[1]);
input_t2->Reshape({run_batch, run_seq_len, 1});
input_t2->copy_from_cpu(i1_data.data());
// third input.
auto input_t3 = predictor->GetInputTensor(input_names[2]);
input_t3->Reshape({run_batch, run_seq_len, 1});
input_t3->copy_from_cpu(i2_data.data());
auto input_t4 = predictor->GetInputTensor(input_names[3]);
input_t4->Reshape({run_batch, run_seq_len, 1});
input_t4->copy_from_cpu(i3_data.data());
ASSERT_TRUE(predictor->ZeroCopyRun());
auto output_names = predictor->GetOutputNames();
auto output_t = predictor->GetOutputTensor(output_names[0]);
std::vector<int> output_shape = output_t->shape();
int out_num = std::accumulate(
output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
out_data->resize(out_num);
output_t->copy_to_cpu(out_data->data());
}
void trt_ernie(bool with_fp16,
std::vector<float> result,
float near_tolerance,
int batch_size = 1) {
AnalysisConfig config;
std::string model_dir = FLAGS_infer_model;
SetConfig(&config, model_dir, true);
int batch = 32;
int min_seq_len = 1;
int max_seq_len = 128;
int opt_seq_len = 128;
std::vector<int> min_shape = {1, min_seq_len, 1};
std::vector<int> max_shape = {batch, max_seq_len, 1};
std::vector<int> opt_shape = {batch, opt_seq_len, 1};
// Set the input's min, max, opt shape
std::map<std::string, std::vector<int>> min_input_shape = {
{"read_file_0.tmp_0", min_shape},
{"read_file_0.tmp_1", min_shape},
{"read_file_0.tmp_2", min_shape},
{"read_file_0.tmp_4", min_shape}};
std::map<std::string, std::vector<int>> max_input_shape = {
{"read_file_0.tmp_0", max_shape},
{"read_file_0.tmp_1", max_shape},
{"read_file_0.tmp_2", max_shape},
{"read_file_0.tmp_4", max_shape}};
std::map<std::string, std::vector<int>> opt_input_shape = {
{"read_file_0.tmp_0", opt_shape},
{"read_file_0.tmp_1", opt_shape},
{"read_file_0.tmp_2", opt_shape},
{"read_file_0.tmp_4", opt_shape}};
auto precision = AnalysisConfig::Precision::kFloat32;
if (with_fp16) {
precision = AnalysisConfig::Precision::kHalf;
}
config.EnableTensorRtEngine(1 << 30, 1, 5, precision, false, false);
config.SetTRTDynamicShapeInfo(
min_input_shape, max_input_shape, opt_input_shape);
paddle_infer::experimental::InternalUtils::SetTransformerMaskid(
&config, "read_file_0.tmp_4");
std::vector<float> out_data;
run(config, &out_data, batch_size);
for (size_t i = 0; i < out_data.size(); i++) {
EXPECT_NEAR(result[i], out_data[i], near_tolerance);
}
}
TEST(AnalysisPredictor, no_fp16) {
std::vector<float> result = {0.597841, 0.219972, 0.182187};
trt_ernie(false, result, 1e-4);
}
TEST(AnalysisPredictor, fp16) {
#ifdef PADDLE_WITH_CUDA
std::vector<float> result = {0.598, 0.219, 0.182};
trt_ernie(true, result, 4e-3);
#endif
}
TEST(AnalysisPredictor, no_fp16_bs2) {
std::vector<float> result = {
0.597841, 0.219972, 0.182187, 0.597841, 0.219972, 0.182187};
trt_ernie(false, result, 1e-4, 2);
}
TEST(AnalysisPredictor, fp16_bs2) {
#ifdef PADDLE_WITH_CUDA
std::vector<float> result = {0.598, 0.219, 0.182, 0.598, 0.219, 0.182};
trt_ernie(true, result, 4e-3, 2);
#endif
}
// ernie_varlen
std::shared_ptr<paddle_infer::Predictor> InitPredictor() {
paddle_infer::Config config;
config.SetModel(FLAGS_infer_model);
config.EnableUseGpu(100, 0);
// Open the memory optim.
config.EnableMemoryOptim();
int max_batch = 32;
int max_single_seq_len = 128;
int opt_single_seq_len = 64;
int min_batch_seq_len = 1;
int max_batch_seq_len = 512;
int opt_batch_seq_len = 256;
std::string input_name0 = "read_file_0.tmp_0";
std::string input_name1 = "read_file_0.tmp_1";
std::string input_name2 = "read_file_0.tmp_2";
std::string input_name3 = "read_file_0.tmp_4";
std::vector<int> min_shape = {min_batch_seq_len};
std::vector<int> max_shape = {max_batch_seq_len};
std::vector<int> opt_shape = {opt_batch_seq_len};
// Set the input's min, max, opt shape
std::map<std::string, std::vector<int>> min_input_shape = {
{input_name0, min_shape},
{input_name1, min_shape},
{input_name2, {1}},
{input_name3, {1, 1, 1}}};
std::map<std::string, std::vector<int>> max_input_shape = {
{input_name0, max_shape},
{input_name1, max_shape},
{input_name2, {max_batch + 1}},
{input_name3, {1, max_single_seq_len, 1}}};
std::map<std::string, std::vector<int>> opt_input_shape = {
{input_name0, opt_shape},
{input_name1, opt_shape},
{input_name2, {max_batch + 1}},
{input_name3, {1, opt_single_seq_len, 1}}};
// only kHalf supported
config.EnableTensorRtEngine(
1 << 30, 1, 5, paddle_infer::Config::Precision::kHalf, false, false);
// erinie varlen must be used with dynamic shape
config.SetTRTDynamicShapeInfo(
min_input_shape, max_input_shape, opt_input_shape);
// erinie varlen must be used with oss
config.EnableVarseqlen();
paddle_infer::experimental::InternalUtils::SetTransformerPosid(&config,
input_name2);
paddle_infer::experimental::InternalUtils::SetTransformerMaskid(&config,
input_name3);
return paddle_infer::CreatePredictor(config);
}
void run(paddle_infer::Predictor* predictor, std::vector<float>* out_data) {
#if !defined(_WIN32)
setenv("NVIDIA_TF32_OVERRIDE", "0", 1);
#endif
const int run_batch = 2;
const int run_seq_len = 71;
const int max_seq_len = 128;
std::vector<int32_t> i1 = {
// sentence 1
1,
3558,
4,
75,
491,
89,
340,
313,
93,
4,
255,
10,
75,
321,
4095,
1902,
4,
134,
49,
75,
311,
14,
44,
178,
543,
15,
12043,
2,
75,
201,
340,
9,
14,
44,
486,
218,
1140,
279,
12043,
2,
// sentence 2
101,
2054,
2234,
2046,
2486,
2044,
1996,
2047,
4552,
2001,
9536,
1029,
102,
2004,
1997,
2008,
2154,
1010,
1996,
2047,
4552,
9536,
2075,
1996,
2117,
3072,
2234,
2046,
2486,
1012,
102,
};
std::vector<int32_t> i2 = {// sentence 1
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
// sentence 2
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1};
// shape info of this batch
std::vector<int32_t> i3 = {0, 40, 71};
// max_seq_len represents the max sentence length of all the sentences, only
// length of
// input i4 is useful, data means nothing.
std::vector<float> i4(max_seq_len, 0);
auto input_names = predictor->GetInputNames();
// first input
auto input_t1 = predictor->GetInputHandle(input_names[0]);
input_t1->Reshape({run_seq_len});
input_t1->CopyFromCpu(i1.data());
// second input
auto input_t2 = predictor->GetInputHandle(input_names[1]);
input_t2->Reshape({run_seq_len});
input_t2->CopyFromCpu(i2.data());
// third input
auto input_t3 = predictor->GetInputHandle(input_names[2]);
input_t3->Reshape({run_batch + 1});
input_t3->CopyFromCpu(i3.data());
// fourth input
auto input_t4 = predictor->GetInputHandle(input_names[3]);
input_t4->Reshape({1, max_seq_len, 1});
input_t4->CopyFromCpu(i4.data());
PADDLE_ENFORCE(
predictor->Run(),
common::errors::PreconditionNotMet("Predictor is not runnable"));
auto output_names = predictor->GetOutputNames();
auto output_t = predictor->GetOutputHandle(output_names[0]);
std::vector<int> output_shape = output_t->shape();
int out_num = std::accumulate(
output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
out_data->resize(out_num);
output_t->CopyToCpu(out_data->data());
return;
}
TEST(AnalysisPredictor, ernie_varlen) {
#if IS_TRT_VERSION_GE(7234)
if (platform::GetGPUComputeCapability(platform::GetCurrentDeviceId()) >= 75) {
auto predictor = InitPredictor();
std::vector<float> out_data;
run(predictor.get(), &out_data);
std::vector<float> ref_data{
0.59814, 0.219882, 0.181978, 0.359796, 0.577414, 0.0627908};
float near_tolerance = 4e-3;
for (size_t i = 0; i < out_data.size(); i++) {
EXPECT_NEAR(ref_data[i], out_data[i], near_tolerance);
}
}
#endif
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/trt_dynamic_shape_test.cc
+307
View File
@@ -0,0 +1,307 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include "paddle/common/flags.h"
#include "test/cpp/inference/api/trt_test_helper.h"
namespace paddle {
namespace inference {
void TestDynamic(bool with_dynamic = true,
bool delete_cache = true,
bool delete_conv_bn = false) {
std::string model_dir =
FLAGS_infer_model + "/conv_bn_swish_split_gelu/conv_bn_swish_split_gelu";
std::string opt_cache_dir = model_dir + "/my_cache";
if (delete_cache) {
delete_cache_files(opt_cache_dir);
}
AnalysisConfig config;
config.EnableNewIR(false);
config.EnableUseGpu(100, 0);
std::string buffer_prog, buffer_param;
ReadBinaryFile(model_dir + "/model", &buffer_prog);
ReadBinaryFile(model_dir + "/params", &buffer_param);
config.SetModelBuffer(&buffer_prog[0],
buffer_prog.size(),
&buffer_param[0],
buffer_param.size());
config.SetOptimCacheDir(opt_cache_dir);
// Set the input's min, max, opt shape
config.EnableTensorRtEngine(
1 << 30, 1, 1, AnalysisConfig::Precision::kFloat32, true, true);
if (delete_conv_bn) {
config.pass_builder()->DeletePass("conv_bn_fuse_pass");
}
if (with_dynamic) {
std::map<std::string, std::vector<int>> min_input_shape = {
{"image", {1, 1, 3, 3}}};
std::map<std::string, std::vector<int>> max_input_shape = {
{"image", {1, 1, 10, 10}}};
std::map<std::string, std::vector<int>> opt_input_shape = {
{"image", {1, 1, 3, 3}}};
config.SetTRTDynamicShapeInfo(
min_input_shape, max_input_shape, opt_input_shape);
}
auto predictor = CreatePaddlePredictor(config);
auto input_names = predictor->GetInputNames();
int channels = 1;
int height = 3;
int width = 3;
int input_num = channels * height * width * 1;
float *input = new float[input_num];
memset(input, 0, input_num * sizeof(float));
auto input_t = predictor->GetInputTensor(input_names[0]);
input_t->Reshape({1, channels, height, width});
input_t->copy_from_cpu(input);
ASSERT_TRUE(predictor->ZeroCopyRun());
std::vector<float> out_data;
auto output_names = predictor->GetOutputNames();
auto output_t = predictor->GetOutputTensor(output_names[0]);
std::vector<int> output_shape = output_t->shape();
int out_num = std::accumulate(
output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
out_data.resize(out_num);
output_t->copy_to_cpu(out_data.data());
}
void TestDynamic2() {
std::string model_dir =
FLAGS_infer_model + "/complex_model_dynamic/complex_model_dynamic2";
AnalysisConfig config;
config.EnableUseGpu(100, 0);
config.SetModel(model_dir + "/model", model_dir + "/params");
// Set the input's min, max, opt shape
int batch_size = 1;
std::map<std::string, std::vector<int>> min_input_shape = {
{"image", {1, 3, 3, 3}}, {"in1", {1, 2, 1, 1}}, {"in2", {1, 2, 1, 1}}};
std::map<std::string, std::vector<int>> max_input_shape = {
{"image", {1, 3, 10, 10}}, {"in1", {1, 2, 1, 1}}, {"in2", {1, 2, 1, 1}}};
std::map<std::string, std::vector<int>> opt_input_shape = {
{"image", {1, 3, 5, 5}}, {"in1", {1, 2, 1, 1}}, {"in2", {1, 2, 1, 1}}};
config.EnableTensorRtEngine(
1 << 30, batch_size, 0, AnalysisConfig::Precision::kFloat32, false, true);
config.SetTRTDynamicShapeInfo(
min_input_shape, max_input_shape, opt_input_shape);
auto predictor = CreatePaddlePredictor(config);
int channels = 3;
int height = 5;
int width = 5;
int input_num = channels * height * width * 1;
float *input = new float[input_num];
memset(input, 0, input_num * sizeof(float));
auto input_names = predictor->GetInputNames();
auto input_t = predictor->GetInputTensor(input_names[0]);
input_t->Reshape({batch_size, channels, height, width});
input_t->copy_from_cpu(input);
auto input_t1 = predictor->GetInputTensor(input_names[1]);
input_t1->Reshape({batch_size, 2, 1, 1});
std::vector<float> first;
for (int i = 0; i < batch_size * 2; i++) first.push_back(1.0);
input_t1->copy_from_cpu(first.data());
auto input_t2 = predictor->GetInputTensor(input_names[2]);
input_t2->Reshape({batch_size, 2, 1, 1});
input_t2->copy_from_cpu(first.data());
ASSERT_TRUE(predictor->ZeroCopyRun());
std::vector<float> out_data;
auto output_names = predictor->GetOutputNames();
auto output_t = predictor->GetOutputTensor(output_names[0]);
std::vector<int> output_shape = output_t->shape();
int out_num = std::accumulate(
output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
out_data.resize(out_num);
output_t->copy_to_cpu(out_data.data());
std::vector<float> result = {0.617728, 1.63504, 2.15771, 0.535556};
for (size_t i = 0; i < out_data.size(); i++) {
EXPECT_NEAR(result[i], out_data[i], 1e-5);
}
}
void TestTunedDynamic() {
std::string model_dir =
FLAGS_infer_model + "/complex_model_dynamic/complex_model_dynamic2";
AnalysisConfig config_tuned;
const std::string shape_range = "shape_range.pbtxt";
config_tuned.EnableUseGpu(100, 0);
config_tuned.SetModel(model_dir + "/model", model_dir + "/params");
config_tuned.CollectShapeRangeInfo(shape_range);
int batch_size = 1;
auto predictor_tuned = CreatePaddlePredictor(config_tuned);
auto check_func = [batch_size](PaddlePredictor *predictor) {
int channels = 3;
int height = 5;
int width = 5;
int input_num = channels * height * width * 1;
float *input = new float[input_num];
memset(input, 0, input_num * sizeof(float));
auto input_names = predictor->GetInputNames();
auto input_t = predictor->GetInputTensor(input_names[0]);
input_t->Reshape({batch_size, channels, height, width});
input_t->copy_from_cpu(input);
auto input_t1 = predictor->GetInputTensor(input_names[1]);
input_t1->Reshape({batch_size, 2, 1, 1});
std::vector<float> first;
for (int i = 0; i < batch_size * 2; i++) first.push_back(1.0);
input_t1->copy_from_cpu(first.data());
auto input_t2 = predictor->GetInputTensor(input_names[2]);
input_t2->Reshape({batch_size, 2, 1, 1});
input_t2->copy_from_cpu(first.data());
ASSERT_TRUE(predictor->ZeroCopyRun());
std::vector<float> out_data;
auto output_names = predictor->GetOutputNames();
auto output_t = predictor->GetOutputTensor(output_names[0]);
std::vector<int> output_shape = output_t->shape();
int out_num = std::accumulate(
output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
out_data.resize(out_num);
output_t->copy_to_cpu(out_data.data());
};
check_func(predictor_tuned.get());
predictor_tuned.reset(nullptr);
// check tuned_dynamic_shape
AnalysisConfig config;
config.EnableUseGpu(100, 0);
std::string cache_dir = "tuned_cache";
config.SetOptimCacheDir(cache_dir);
delete_cache_files(cache_dir);
config.SetModel(model_dir + "/model", model_dir + "/params");
config.EnableTunedTensorRtDynamicShape(shape_range, true);
config.EnableTensorRtEngine(
1 << 30, batch_size, 0, AnalysisConfig::Precision::kFloat32, true, false);
auto test_predictor = CreatePaddlePredictor(config);
check_func(test_predictor.get());
}
void TestDynamicClone(bool with_dynamic = true,
bool delete_cache = true,
bool delete_conv_bn = false) {
std::string model_dir =
FLAGS_infer_model + "/conv_bn_swish_split_gelu/conv_bn_swish_split_gelu";
std::string opt_cache_dir = model_dir + "/my_cache";
if (delete_cache) {
delete_cache_files(opt_cache_dir);
}
AnalysisConfig config;
config.EnableUseGpu(100, 0);
std::string buffer_prog, buffer_param;
ReadBinaryFile(model_dir + "/model", &buffer_prog);
ReadBinaryFile(model_dir + "/params", &buffer_param);
config.SetModelBuffer(&buffer_prog[0],
buffer_prog.size(),
&buffer_param[0],
buffer_param.size());
config.SetOptimCacheDir(opt_cache_dir);
// Set the input's min, max, opt shape
config.EnableTensorRtEngine(
1 << 30, 1, 1, AnalysisConfig::Precision::kFloat32, false, false);
if (delete_conv_bn) {
config.pass_builder()->DeletePass("conv_bn_fuse_pass");
}
if (with_dynamic) {
std::map<std::string, std::vector<int>> min_input_shape = {
{"image", {1, 1, 3, 3}}};
std::map<std::string, std::vector<int>> max_input_shape = {
{"image", {1, 1, 10, 10}}};
std::map<std::string, std::vector<int>> opt_input_shape = {
{"image", {1, 1, 3, 3}}};
config.SetTRTDynamicShapeInfo(
min_input_shape, max_input_shape, opt_input_shape);
}
auto predictor = CreatePaddlePredictor(config);
auto input_names = predictor->GetInputNames();
int channels = 1;
int height = 3;
int width = 3;
int input_num = channels * height * width * 1;
float *input = new float[input_num];
memset(input, 0, input_num * sizeof(float));
auto input_t = predictor->GetInputTensor(input_names[0]);
input_t->Reshape({1, channels, height, width});
input_t->copy_from_cpu(input);
ASSERT_TRUE(predictor->ZeroCopyRun());
std::vector<float> out_data;
auto output_names = predictor->GetOutputNames();
auto output_t = predictor->GetOutputTensor(output_names[0]);
std::vector<int> output_shape = output_t->shape();
int out_num = std::accumulate(
output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
out_data.resize(out_num);
output_t->copy_to_cpu(out_data.data());
auto predictor2 = predictor->Clone();
auto input_t2 = predictor2->GetInputTensor(input_names[0]);
input_t2->Reshape({1, channels, height, width});
input_t2->copy_from_cpu(input);
ASSERT_TRUE(predictor2->ZeroCopyRun());
std::vector<float> out_data2;
auto output_t2 = predictor2->GetOutputTensor(output_names[0]);
std::vector<int> output_shape2 = output_t2->shape();
int out_num2 = std::accumulate(
output_shape2.begin(), output_shape2.end(), 1, std::multiplies<int>());
out_data2.resize(out_num2);
output_t2->copy_to_cpu(out_data2.data());
ASSERT_TRUE(out_data2.size() == out_data.size());
for (size_t i = 0; i < out_data.size(); i++) {
EXPECT_NEAR(out_data2[i], out_data[i], 1e-5);
}
}
TEST(AnalysisPredictor, trt_dynamic) { TestDynamic(true); }
TEST(AnalysisPredictor, trt_memory_serialize) {
// serialize
TestDynamic(true, true, true);
// deserialize
TestDynamic(true, false, true);
}
TEST(AnalysisPredictor, trt_dynamic2) { TestDynamic2(); }
TEST(AnalysisPredictor, trt_tuned_dynamic) { TestTunedDynamic(); }
TEST(AnalysisPredictor, trt_dynamic_clone) { TestDynamicClone(); }
} // namespace inference
} // namespace paddle
test/cpp/inference/api/trt_instance_norm_converter_test.cc
+49
View File
@@ -0,0 +1,49 @@
/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include "paddle/common/flags.h"
#include "test/cpp/inference/api/trt_test_helper.h"
namespace paddle {
namespace inference {
TEST(TensorRT, instance_norm) {
std::string model_dir = FLAGS_infer_model + "/instance_norm";
AnalysisConfig config;
int batch_size = 4;
config.EnableUseGpu(100, 0);
config.SetModel(model_dir);
config.EnableTensorRtEngine(
1 << 20, batch_size, 0, AnalysisConfig::Precision::kFloat32, false);
auto predictor = CreatePaddlePredictor(config);
int length = 4;
int input_num = batch_size * length;
float *input = new float[input_num];
memset(input, 1.0, input_num * sizeof(float));
auto input_names = predictor->GetInputNames();
auto input_t = predictor->GetInputTensor(input_names[0]);
input_t->Reshape({batch_size, length});
input_t->copy_from_cpu(input);
ASSERT_TRUE(predictor->ZeroCopyRun());
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/trt_mark_trt_engine_outputs_test.cc
+43
View File
@@ -0,0 +1,43 @@
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include "test/cpp/inference/api/trt_test_helper.h"
namespace paddle {
namespace inference {
TEST(TensorRT, mark_trt_engine_outputs) {
std::string model_dir = FLAGS_infer_model + "/resnet50";
AnalysisConfig config;
config.SetModel(model_dir);
config.EnableUseGpu(100, 0);
config.EnableTensorRtEngine(
1 << 30, 1, 5, AnalysisConfig::Precision::kFloat32, false, false);
// The name of the tensor that needs to be marked
std::vector<std::string> markOutput = {"pool2d_0.tmp_0"};
config.MarkTrtEngineOutputs(markOutput);
std::vector<std::vector<PaddleTensor>> inputs_all;
auto predictor = CreatePaddlePredictor(config);
SetFakeImageInput(&inputs_all, model_dir, false, "__model__", "");
std::vector<PaddleTensor> outputs;
for (auto &input : inputs_all) {
ASSERT_TRUE(predictor->Run(input, &outputs));
predictor->ClearIntermediateTensor();
}
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/trt_mobilenet_test.cc
+72
View File
@@ -0,0 +1,72 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <functional>
#include <numeric>
#include "paddle/common/flags.h"
#include "test/cpp/inference/api/trt_test_helper.h"
namespace paddle_infer {
TEST(PredictorPool, use_gpu) {
std::string model_dir = FLAGS_infer_model + "/" + "mobilenet";
Config config;
config.EnableUseGpu(100, 0);
config.SetModel(model_dir);
config.EnableTensorRtEngine();
config.Exp_DisableTensorRtOPs({"fc"});
config.EnableTensorRtDLA(0);
services::PredictorPool pred_pool(config, 1);
auto predictor = pred_pool.Retrieve(0);
auto input_names = predictor->GetInputNames();
auto input_t = predictor->GetInputHandle(input_names[0]);
std::vector<int> in_shape = {1, 3, 224, 224};
int in_num = std::accumulate(
in_shape.begin(), in_shape.end(), 1, std::multiplies<int>());
std::vector<float> input(in_num, 0);
input_t->Reshape(in_shape);
input_t->CopyFromCpu(input.data());
predictor->Run();
}
TEST(PredictorPool, use_trt_cuda_graph) {
std::string model_dir = FLAGS_infer_model + "/" + "mobilenet";
Config config;
config.EnableUseGpu(100, 0);
config.SetModel(model_dir);
config.EnableTensorRtEngine(
1 << 20, 1, 3, PrecisionType::kFloat32, false, false, true);
config.Exp_DisableTensorRtOPs({"fc"});
config.EnableTensorRtDLA(0);
services::PredictorPool pred_pool(config, 1);
auto predictor = pred_pool.Retrieve(0);
auto input_names = predictor->GetInputNames();
auto input_t = predictor->GetInputHandle(input_names[0]);
std::vector<int> in_shape = {1, 3, 224, 224};
int in_num = std::accumulate(
in_shape.begin(), in_shape.end(), 1, std::multiplies<int>());
std::vector<float> input(in_num, 0);
input_t->Reshape(in_shape);
input_t->CopyFromCpu(input.data());
predictor->Run();
}
} // namespace paddle_infer
test/cpp/inference/api/trt_quant_int8_test.cc
+68
View File
@@ -0,0 +1,68 @@
/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <numeric>
#include "paddle/common/flags.h"
#include "test/cpp/inference/api/trt_test_helper.h"
namespace paddle {
namespace inference {
TEST(quant_int8, resnet50) {
std::string model_dir = FLAGS_infer_model;
AnalysisConfig config;
config.EnableUseGpu(1000, 0);
config.SetModel(model_dir);
config.EnableTensorRtEngine(
1 << 30, 1, 1, AnalysisConfig::Precision::kInt8, false, false);
std::map<std::string, std::vector<int>> min_input_shape = {
{"image", {1, 1, 3, 3}}};
std::map<std::string, std::vector<int>> max_input_shape = {
{"image", {1, 1, 10, 10}}};
std::map<std::string, std::vector<int>> opt_input_shape = {
{"image", {1, 1, 3, 3}}};
config.SetTRTDynamicShapeInfo(
min_input_shape, max_input_shape, opt_input_shape);
auto predictor = CreatePaddlePredictor(config);
auto input_names = predictor->GetInputNames();
int channels = 1;
int height = 3;
int width = 3;
int input_num = channels * height * width * 1;
float *input = new float[input_num];
memset(input, 0, input_num * sizeof(float));
auto input_t = predictor->GetInputTensor(input_names[0]);
input_t->Reshape({1, channels, height, width});
input_t->copy_from_cpu(input);
ASSERT_TRUE(predictor->ZeroCopyRun());
std::vector<float> out_data;
auto output_names = predictor->GetOutputNames();
auto output_t = predictor->GetOutputTensor(output_names[0]);
std::vector<int> output_shape = output_t->shape();
int out_num = std::accumulate(
output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
out_data.resize(out_num);
output_t->copy_to_cpu(out_data.data());
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/trt_quant_int8_yolov3_r50_test.cc
+64
View File
@@ -0,0 +1,64 @@
/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <numeric>
#include "paddle/common/flags.h"
#include "test/cpp/inference/api/trt_test_helper.h"
namespace paddle {
namespace inference {
TEST(quant_int8, yolov3_resnet50) {
AnalysisConfig config;
config.EnableNewIR(false);
config.EnableUseGpu(100, 0);
config.SetModel(FLAGS_infer_model + "/model", FLAGS_infer_model + "/params");
config.EnableTensorRtEngine(
1 << 30, 1, 3, AnalysisConfig::Precision::kInt8, false, false);
auto predictor = CreatePaddlePredictor(config);
auto input_names = predictor->GetInputNames();
int channels = 3;
int height = 608;
int width = 608;
int input_num = channels * height * width * 1;
float *input = new float[input_num];
int32_t *im_shape = new int32_t[2];
im_shape[0] = 608;
im_shape[1] = 608;
memset(input, 1.0, input_num * sizeof(float));
auto input_t = predictor->GetInputTensor(input_names[0]);
input_t->Reshape({1, channels, height, width});
input_t->copy_from_cpu(input);
auto input_t1 = predictor->GetInputTensor(input_names[1]);
input_t1->Reshape({1, 2});
input_t1->copy_from_cpu(im_shape);
ASSERT_TRUE(predictor->ZeroCopyRun());
std::vector<float> out_data;
auto output_names = predictor->GetOutputNames();
auto output_t = predictor->GetOutputTensor(output_names[0]);
std::vector<int> output_shape = output_t->shape();
int out_num = std::accumulate(
output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
out_data.resize(out_num);
output_t->copy_to_cpu(out_data.data());
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/trt_rebind_stream_test.cc
+183
View File
@@ -0,0 +1,183 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <thread>
#include "paddle/common/flags.h"
#include "paddle/fluid/inference/api/paddle_inference_api.h"
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
// TODO(inference): This case failed in windows with a SEH error, we need to fix
// it.
TEST(ReBindStream_single, use_gpu) {
std::string model_dir = FLAGS_infer_model + "/mobilenet";
AnalysisConfig config;
config.EnableUseGpu(100, 0);
config.SetModel(model_dir);
config.EnableTensorRtEngine();
cudaStream_t stream1, stream2, stream3;
cudaStreamCreate(&stream1);
cudaStreamCreate(&stream2);
cudaStreamCreate(&stream3);
config.SetExecStream(stream1);
auto predictor = paddle_infer::CreatePredictor(config);
auto x_t = predictor->GetInputHandle("x");
x_t->Reshape({1, 3, 224, 224});
std::array<float, 3 * 224 * 224> x_data = {0};
x_t->CopyFromCpu(x_data.data());
ASSERT_TRUE(predictor->Run());
cudaDeviceSynchronize();
ASSERT_TRUE(paddle_infer::experimental::InternalUtils::RunWithExternalStream(
predictor.get(), stream2));
cudaDeviceSynchronize();
ASSERT_TRUE(paddle_infer::experimental::InternalUtils::RunWithExternalStream(
predictor.get(), stream3));
cudaDeviceSynchronize();
}
TEST(ReBindStream_multi, use_gpu) {
std::string model_dir = FLAGS_infer_model + "/mobilenet";
AnalysisConfig config1;
config1.EnableUseGpu(100, 0);
config1.SetModel(model_dir);
config1.EnableTensorRtEngine();
AnalysisConfig config2;
config2.EnableUseGpu(100, 0);
config2.EnableTensorRtEngine();
config2.SetModel(model_dir);
cudaStream_t stream1, stream2, stream3;
cudaStreamCreate(&stream1);
cudaStreamCreate(&stream2);
cudaStreamCreate(&stream3);
config1.SetExecStream(stream1);
config2.SetExecStream(stream1);
auto predictor1 = paddle_infer::CreatePredictor(config1);
auto predictor2 = paddle_infer::CreatePredictor(config2);
std::vector<float> x1(3 * 224 * 224, 1.0);
auto x_t1 = predictor1->GetInputHandle("x");
x_t1->Reshape({1, 3, 224, 224});
x_t1->CopyFromCpu(x1.data());
std::vector<float> x2(3 * 224 * 224, 2.0);
auto x_t2 = predictor2->GetInputHandle("x");
x_t2->Reshape({1, 3, 224, 224});
x_t2->CopyFromCpu(x2.data());
ASSERT_TRUE(predictor1->Run());
cudaStreamSynchronize(stream1);
ASSERT_TRUE(predictor2->Run());
cudaStreamSynchronize(stream1);
ASSERT_TRUE(paddle_infer::experimental::InternalUtils::RunWithExternalStream(
predictor1.get(), stream2));
cudaDeviceSynchronize();
ASSERT_TRUE(paddle_infer::experimental::InternalUtils::RunWithExternalStream(
predictor2.get(), stream2));
cudaDeviceSynchronize();
ASSERT_TRUE(paddle_infer::experimental::InternalUtils::RunWithExternalStream(
predictor1.get(), stream3));
cudaStreamSynchronize(stream3);
ASSERT_TRUE(paddle_infer::experimental::InternalUtils::RunWithExternalStream(
predictor2.get(), stream3));
cudaStreamSynchronize(stream3);
}
TEST(SwitchStream_multi, use_gpu) {
std::string model_dir = FLAGS_infer_model + "/mobilenet";
AnalysisConfig config1;
config1.EnableUseGpu(100, 0);
config1.SetModel(model_dir);
AnalysisConfig config2;
config2.EnableUseGpu(100, 0);
config2.SetModel(model_dir);
AnalysisConfig config3;
config3.EnableUseGpu(100, 0);
config3.SetModel(model_dir);
// config1.EnableTensorRtEngine();
// config2.EnableTensorRtEngine();
// config3.EnableTensorRtEngine();
cudaStream_t stream1, stream2, stream3;
cudaStreamCreate(&stream1);
cudaStreamCreate(&stream2);
cudaStreamCreate(&stream3);
config1.SetExecStream(stream1);
config2.SetExecStream(stream1);
config3.SetExecStream(stream1);
auto predictor1 = paddle_infer::CreatePredictor(config1);
auto predictor2 = paddle_infer::CreatePredictor(config2);
auto predictor3 = paddle_infer::CreatePredictor(config3);
std::vector<float> x1(3 * 224 * 224, 1.0);
auto x_t1 = predictor1->GetInputHandle("x");
x_t1->Reshape({1, 3, 224, 224});
x_t1->CopyFromCpu(x1.data());
std::vector<float> x2(3 * 224 * 224, 2.0);
auto x_t2 = predictor2->GetInputHandle("x");
x_t2->Reshape({1, 3, 224, 224});
x_t2->CopyFromCpu(x2.data());
std::vector<float> x3(3 * 224 * 224, 2.5);
auto x_t3 = predictor3->GetInputHandle("x");
x_t3->Reshape({1, 3, 224, 224});
x_t3->CopyFromCpu(x3.data());
// TODO(wilber): fix.
// NOTE: Must run once on master thread, but why?
// if remove the code, the unit test fail.
ASSERT_TRUE(predictor1->Run());
cudaStreamSynchronize(stream1);
ASSERT_TRUE(predictor2->Run());
cudaStreamSynchronize(stream1);
ASSERT_TRUE(predictor3->Run());
cudaStreamSynchronize(stream1);
auto Run = [&](paddle_infer::Predictor* p,
std::vector<cudaStream_t> streams) {
for (auto s : streams) {
paddle_infer::experimental::InternalUtils::RunWithExternalStream(p, s);
}
};
std::thread p1(Run,
predictor1.get(),
std::vector<cudaStream_t>{
stream1, stream2, stream3, stream3, stream2, stream2});
std::thread p2(Run,
predictor2.get(),
std::vector<cudaStream_t>{
stream1, stream3, stream1, stream2, stream1, stream3});
std::thread p3(Run,
predictor3.get(),
std::vector<cudaStream_t>{
stream1, stream1, stream2, stream3, stream3, stream2});
p1.join();
p2.join();
p3.join();
cudaDeviceSynchronize();
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/trt_split_converter_test.cc
+46
View File
@@ -0,0 +1,46 @@
/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include "paddle/common/flags.h"
#include "test/cpp/inference/api/trt_test_helper.h"
namespace paddle {
namespace inference {
TEST(TensorRT, split_converter) {
std::string model_dir = FLAGS_infer_model + "/split_converter";
std::string opt_cache_dir = model_dir + "/_opt_cache";
delete_cache_files(opt_cache_dir);
AnalysisConfig config;
int batch_size = 4;
int channels = 4;
int height = 4;
int width = 4;
config.EnableUseGpu(100, 0);
config.SetModel(model_dir);
config.EnableTensorRtEngine(
1 << 20, batch_size, 1, AnalysisConfig::Precision::kInt8, false, true);
std::map<std::string, std::vector<int>> input_shape;
input_shape["x"] = {batch_size, channels, height, width};
config.SetTRTDynamicShapeInfo(input_shape, input_shape, input_shape, false);
auto predictor = CreatePaddlePredictor(config);
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/trt_test_helper.h
+178
View File
@@ -0,0 +1,178 @@
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#pragma once
#include <dirent.h>
#include <string>
#include <vector>
#include "glog/logging.h"
#include "gtest/gtest.h"
#include "paddle/common/flags.h"
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle {
namespace inference {
PD_DEFINE_bool(use_tensorrt, true, "Test the performance of TensorRT engine.");
PD_DEFINE_string(prog_filename, "", "Name of model file.");
PD_DEFINE_string(param_filename, "", "Name of parameters file.");
template <typename ConfigType>
void SetConfig(ConfigType* config,
std::string model_dir,
bool use_gpu,
bool use_tensorrt = false,
int batch_size = -1) {
if (!FLAGS_prog_filename.empty() && !FLAGS_param_filename.empty()) {
config->prog_file = model_dir + "/" + FLAGS_prog_filename;
config->param_file = model_dir + "/" + FLAGS_param_filename;
} else {
config->model_dir = model_dir;
}
if (use_gpu) {
config->use_gpu = true;
config->device = 0;
config->fraction_of_gpu_memory = 0.15;
}
}
template <>
void SetConfig<AnalysisConfig>(AnalysisConfig* config,
std::string model_dir,
bool use_gpu,
bool use_tensorrt,
int batch_size) {
if (!FLAGS_prog_filename.empty() && !FLAGS_param_filename.empty()) {
config->SetModel(model_dir + "/" + FLAGS_prog_filename,
model_dir + "/" + FLAGS_param_filename);
} else {
config->SetModel(model_dir);
}
if (use_gpu) {
config->EnableUseGpu(100, 0);
if (use_tensorrt) {
config->EnableTensorRtEngine(
1 << 10, batch_size, 3, AnalysisConfig::Precision::kFloat32, false);
config->pass_builder()->DeletePass("conv_bn_fuse_pass");
config->pass_builder()->DeletePass("fc_fuse_pass");
config->pass_builder()->TurnOnDebug();
} else {
config->EnableCUDNN();
config->SwitchIrOptim();
}
}
}
void profile(std::string model_dir, bool use_analysis, bool use_tensorrt) {
std::vector<std::vector<PaddleTensor>> inputs_all;
if (!FLAGS_prog_filename.empty() && !FLAGS_param_filename.empty()) {
SetFakeImageInput(&inputs_all,
model_dir,
true,
FLAGS_prog_filename,
FLAGS_param_filename);
} else {
SetFakeImageInput(&inputs_all, model_dir, false, "__model__", "");
}
std::vector<std::vector<PaddleTensor>> outputs;
if (use_analysis || use_tensorrt) {
AnalysisConfig config;
config.EnableUseGpu(100, 0);
config.pass_builder()->TurnOnDebug();
SetConfig<AnalysisConfig>(
&config, model_dir, true, use_tensorrt, FLAGS_batch_size);
TestPrediction(reinterpret_cast<PaddlePredictor::Config*>(&config),
inputs_all,
&outputs,
FLAGS_num_threads,
true);
} else {
NativeConfig config;
SetConfig<NativeConfig>(&config, model_dir, true, false);
TestPrediction(reinterpret_cast<PaddlePredictor::Config*>(&config),
inputs_all,
&outputs,
FLAGS_num_threads,
false);
}
}
void compare(std::string model_dir, bool use_tensorrt) {
std::vector<std::vector<PaddleTensor>> inputs_all;
if (!FLAGS_prog_filename.empty() && !FLAGS_param_filename.empty()) {
SetFakeImageInput(&inputs_all,
model_dir,
true,
FLAGS_prog_filename,
FLAGS_param_filename);
} else {
SetFakeImageInput(&inputs_all, model_dir, false, "__model__", "");
}
AnalysisConfig analysis_config;
SetConfig<AnalysisConfig>(
&analysis_config, model_dir, true, use_tensorrt, FLAGS_batch_size);
CompareNativeAndAnalysis(
reinterpret_cast<const PaddlePredictor::Config*>(&analysis_config),
inputs_all);
}
void compare_continuous_input(std::string model_dir, bool use_tensorrt) {
AnalysisConfig analysis_config;
SetConfig<AnalysisConfig>(
&analysis_config, model_dir, true, use_tensorrt, FLAGS_batch_size);
auto config =
reinterpret_cast<const PaddlePredictor::Config*>(&analysis_config);
auto native_pred = CreateTestPredictor(config, false);
auto analysis_pred = CreateTestPredictor(config, true);
for (int i = 0; i < 20; i++) {
std::vector<std::vector<PaddleTensor>> inputs_all;
if (!FLAGS_prog_filename.empty() && !FLAGS_param_filename.empty()) {
SetFakeImageInput(&inputs_all,
model_dir,
true,
FLAGS_prog_filename,
FLAGS_param_filename,
nullptr,
i);
} else {
SetFakeImageInput(
&inputs_all, model_dir, false, "__model__", "", nullptr, i);
}
CompareNativeAndAnalysis(
native_pred.get(), analysis_pred.get(), inputs_all);
}
}
void delete_cache_files(std::string path) {
DIR* dir = opendir(path.c_str());
if (dir == NULL) return;
struct dirent* ptr;
while ((ptr = readdir(dir)) != NULL) {
if (std::strcmp(ptr->d_name, ".") == 0 ||
std::strcmp(ptr->d_name, "..") == 0) {
continue;
} else if (ptr->d_type == 8) {
std::string file_rm = path + "/" + ptr->d_name;
remove(file_rm.c_str());
}
}
closedir(dir);
remove(path.c_str());
}
} // namespace inference
} // namespace paddle
test/cpp/inference/api/xpu_config_resnet50_test.cc
+89
View File
@@ -0,0 +1,89 @@
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <cmath>
#include "paddle/common/flags.h"
#include "test/cpp/inference/api/tester_helper.h"
namespace paddle_infer {
static const std::vector<float> TRUTH_VALUES = {
127.779f, 738.165f, 1013.22f, -438.17f, 366.401f, 927.659f, 736.222f,
-633.684f, -329.927f, -430.155f, -633.062f, -146.548f, -1324.28f, -1349.36f,
-242.675f, 117.448f, -801.723f, -391.514f, -404.818f, 454.16f, 515.48f,
-133.031f, 69.293f, 590.096f, -1434.69f, -1070.89f, 307.074f, 400.525f,
-316.12f, -587.125f, -161.056f, 800.363f, -96.4708f, 748.706f, 868.174f,
-447.938f, 112.737f, 1127.2f, 47.4355f, 677.72f, 593.186f, -336.4f,
551.362f, 397.823f, 78.3979f, -715.398f, 405.969f, 404.256f, 246.019f,
-8.42969f, 131.365f, -648.051f};
void PrepareInput(std::shared_ptr<Predictor> predictor) {
const int batch = 1;
const int channel = 3;
const int height = 318;
const int width = 318;
const int input_num = batch * channel * height * width;
std::vector<float> input(input_num, 1);
auto input_names = predictor->GetInputNames();
auto input_t = predictor->GetInputHandle(input_names[0]);
input_t->Reshape({batch, channel, height, width});
input_t->CopyFromCpu(input.data());
}
void CompareOutput(std::shared_ptr<Predictor> predictor) {
auto output_names = predictor->GetOutputNames();
auto output_t = predictor->GetOutputHandle(output_names[0]);
std::vector<int> output_shape = output_t->shape();
size_t out_num = std::accumulate(
output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
std::vector<float> out_data;
out_data.resize(out_num);
output_t->CopyToCpu(out_data.data());
float* data_o = out_data.data();
for (size_t j = 0; j < out_num; j += 10) {
EXPECT_NEAR(
(data_o[j] - TRUTH_VALUES[j / 10]) / TRUTH_VALUES[j / 10], 0., 10e-3);
}
}
TEST(xpu_config, inference) {
size_t l3_size = 10 * 1024 * 1024;
XpuConfig xpu_config;
xpu_config.l3_size = l3_size;
std::string model_dir = FLAGS_infer_model + "/" + "model";
Config config;
config.SetModel(model_dir + "/model", model_dir + "/params");
config.EnableXpu();
config.SetXpuConfig(xpu_config);
XpuConfig xpu_config_test = config.xpu_config();
PADDLE_ENFORCE_EQ(xpu_config_test.l3_size,
l3_size,
common::errors::InvalidArgument(
"xpu_config_test.l3_size %d is different from our "
"expected value l3_size %d.",
xpu_config_test.l3_size,
l3_size));
auto predictor = CreatePredictor(config);
PrepareInput(predictor);
predictor->Run();
CompareOutput(predictor);
}
} // namespace paddle_infer
test/cpp/inference/api/xpu_runtime_config_resnet50_test.cc
+269
View File
@@ -0,0 +1,269 @@
/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <cmath>
#include "paddle/common/enforce.h"
#include "paddle/common/flags.h"
#include "test/cpp/inference/api/tester_helper.h"
#include "xpu/runtime.h"
#include "xpu/xdnn.h"
namespace paddle_infer {
static const std::vector<float> TRUTH_VALUES = {
127.779f, 738.165f, 1013.22f, -438.17f, 366.401f, 927.659f, 736.222f,
-633.684f, -329.927f, -430.155f, -633.062f, -146.548f, -1324.28f, -1349.36f,
-242.675f, 117.448f, -801.723f, -391.514f, -404.818f, 454.16f, 515.48f,
-133.031f, 69.293f, 590.096f, -1434.69f, -1070.89f, 307.074f, 400.525f,
-316.12f, -587.125f, -161.056f, 800.363f, -96.4708f, 748.706f, 868.174f,
-447.938f, 112.737f, 1127.2f, 47.4355f, 677.72f, 593.186f, -336.4f,
551.362f, 397.823f, 78.3979f, -715.398f, 405.969f, 404.256f, 246.019f,
-8.42969f, 131.365f, -648.051f};
void PrepareInput(std::shared_ptr<Predictor> predictor) {
const int batch = 1;
const int channel = 3;
const int height = 318;
const int width = 318;
const int input_num = batch * channel * height * width;
std::vector<float> input(input_num, 1);
auto input_names = predictor->GetInputNames();
auto input_t = predictor->GetInputHandle(input_names[0]);
input_t->Reshape({batch, channel, height, width});
input_t->CopyFromCpu(input.data());
}
void CompareOutput(std::shared_ptr<Predictor> predictor) {
auto output_names = predictor->GetOutputNames();
auto output_t = predictor->GetOutputHandle(output_names[0]);
std::vector<int> output_shape = output_t->shape();
size_t out_num = std::accumulate(
output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
std::vector<float> out_data;
out_data.resize(out_num);
output_t->CopyToCpu(out_data.data());
float* data_o = out_data.data();
for (size_t j = 0; j < out_num; j += 10) {
EXPECT_NEAR(
(data_o[j] - TRUTH_VALUES[j / 10]) / TRUTH_VALUES[j / 10], 0., 10e-3);
}
}
Config InferXpuConfig() {
std::string model_dir = FLAGS_infer_model + "/" + "model";
Config config;
config.SetModel(model_dir + "/model", model_dir + "/params");
config.EnableXpu();
return config;
}
TEST(resnet50_xpu, basic) {
Config config = InferXpuConfig();
auto predictor = CreatePredictor(config);
PrepareInput(predictor);
predictor->Run();
CompareOutput(predictor);
}
#define RUN_WITH_RUNTIME_CONFIG(idx_, config_) \
Config config##idx_ = InferXpuConfig(); \
auto predictor##idx_ = CreatePredictor(config##idx_); \
PrepareInput(predictor##idx_); \
experimental::InternalUtils::RunWithRuntimeConfig(predictor##idx_.get(), \
&config_); \
CompareOutput(predictor##idx_); \
PADDLE_ENFORCE_EQ( \
predictor##idx_->GetExecStream(), \
config_.stream, \
common::errors::InvalidArgument( \
"predictor##idx_->GetExecStream() is not equal with " \
"config_.stream while predictor##idx_->GetExecStream() " \
"is %d and config_.stream is %d", \
predictor##idx_->GetExecStream(), \
config_.stream));
TEST(runtime_stream, null_stream) {
experimental::XpuRuntimeConfig xpu_runtime_config;
xpu_runtime_config.context = nullptr;
xpu_runtime_config.stream = nullptr;
xpu_runtime_config.l3_size = 0;
xpu_runtime_config.l3_ptr = nullptr;
xpu_runtime_config.l3_autotune_size = 0;
RUN_WITH_RUNTIME_CONFIG(0, xpu_runtime_config);
}
TEST(runtime_stream, new_stream) {
void* stream = nullptr;
xpu_stream_create(&stream);
CHECK_NOTNULL(stream);
{
experimental::XpuRuntimeConfig xpu_runtime_config;
xpu_runtime_config.context = nullptr;
xpu_runtime_config.stream = stream;
xpu_runtime_config.l3_size = 0;
xpu_runtime_config.l3_ptr = nullptr;
xpu_runtime_config.l3_autotune_size = 0;
RUN_WITH_RUNTIME_CONFIG(0, xpu_runtime_config);
}
xpu_stream_destroy(stream);
}
TEST(runtime_stream, 2_null_stream) {
experimental::XpuRuntimeConfig xpu_runtime_config;
xpu_runtime_config.context = nullptr;
xpu_runtime_config.stream = nullptr;
xpu_runtime_config.l3_size = 0;
xpu_runtime_config.l3_ptr = nullptr;
xpu_runtime_config.l3_autotune_size = 0;
RUN_WITH_RUNTIME_CONFIG(0, xpu_runtime_config);
RUN_WITH_RUNTIME_CONFIG(1, xpu_runtime_config);
}
TEST(runtime_stream, null_and_new_stream) {
experimental::XpuRuntimeConfig xpu_runtime_config0;
xpu_runtime_config0.context = nullptr;
xpu_runtime_config0.stream = nullptr;
xpu_runtime_config0.l3_size = 0;
xpu_runtime_config0.l3_ptr = nullptr;
xpu_runtime_config0.l3_autotune_size = 0;
void* stream = nullptr;
xpu_stream_create(&stream);
CHECK_NOTNULL(stream);
{
experimental::XpuRuntimeConfig xpu_runtime_config1;
xpu_runtime_config1.context = nullptr;
xpu_runtime_config1.stream = stream;
xpu_runtime_config1.l3_size = 0;
xpu_runtime_config1.l3_ptr = nullptr;
xpu_runtime_config1.l3_autotune_size = 0;
RUN_WITH_RUNTIME_CONFIG(0, xpu_runtime_config0);
RUN_WITH_RUNTIME_CONFIG(1, xpu_runtime_config1);
}
xpu_stream_destroy(stream);
}
TEST(runtime_stream, 2_new_same_stream) {
void* stream = nullptr;
xpu_stream_create(&stream);
CHECK_NOTNULL(stream);
experimental::XpuRuntimeConfig xpu_runtime_config;
xpu_runtime_config.context = nullptr;
xpu_runtime_config.stream = stream;
xpu_runtime_config.l3_size = 0;
xpu_runtime_config.l3_ptr = nullptr;
xpu_runtime_config.l3_autotune_size = 0;
{
RUN_WITH_RUNTIME_CONFIG(0, xpu_runtime_config);
RUN_WITH_RUNTIME_CONFIG(1, xpu_runtime_config);
}
xpu_stream_destroy(stream);
}
TEST(runtime_stream, 2_new_different_stream) {
void* stream0 = nullptr;
xpu_stream_create(&stream0);
CHECK_NOTNULL(stream0);
experimental::XpuRuntimeConfig xpu_runtime_config0;
xpu_runtime_config0.context = nullptr;
xpu_runtime_config0.stream = stream0;
xpu_runtime_config0.l3_size = 0;
xpu_runtime_config0.l3_ptr = nullptr;
xpu_runtime_config0.l3_autotune_size = 0;
void* stream1 = nullptr;
xpu_stream_create(&stream1);
CHECK_NOTNULL(stream1);
experimental::XpuRuntimeConfig xpu_runtime_config1;
xpu_runtime_config1.context = nullptr;
xpu_runtime_config1.stream = stream1;
xpu_runtime_config1.l3_size = 0;
xpu_runtime_config1.l3_ptr = nullptr;
xpu_runtime_config1.l3_autotune_size = 0;
{
RUN_WITH_RUNTIME_CONFIG(0, xpu_runtime_config0);
RUN_WITH_RUNTIME_CONFIG(1, xpu_runtime_config1);
}
xpu_stream_destroy(stream0);
xpu_stream_destroy(stream1);
}
void RunPredictorWithRuntimeConfig(
std::shared_ptr<Predictor> predictor,
experimental::XpuRuntimeConfig runtime_config) {
PrepareInput(predictor);
experimental::InternalUtils::RunWithRuntimeConfig(predictor.get(),
&runtime_config);
CompareOutput(predictor);
PADDLE_ENFORCE_EQ(predictor->GetExecStream(),
runtime_config.stream,
common::errors::InvalidArgument(
"predictor->GetExecStream() is not equal with "
"runtime_config.stream"));
}
TEST(runtime_stream, 2_thread) {
void* stream0 = nullptr;
xpu_stream_create(&stream0);
CHECK_NOTNULL(stream0);
experimental::XpuRuntimeConfig xpu_runtime_config0;
xpu_runtime_config0.context = nullptr;
xpu_runtime_config0.stream = stream0;
xpu_runtime_config0.l3_size = 0;
xpu_runtime_config0.l3_ptr = nullptr;
xpu_runtime_config0.l3_autotune_size = 0;
void* stream1 = nullptr;
xpu_stream_create(&stream1);
CHECK_NOTNULL(stream1);
experimental::XpuRuntimeConfig xpu_runtime_config1;
xpu_runtime_config1.context = nullptr;
xpu_runtime_config1.stream = stream1;
xpu_runtime_config1.l3_size = 0;
xpu_runtime_config1.l3_ptr = nullptr;
xpu_runtime_config1.l3_autotune_size = 0;
{
RUN_WITH_RUNTIME_CONFIG(0, xpu_runtime_config0);
RUN_WITH_RUNTIME_CONFIG(1, xpu_runtime_config1);
std::thread t0(
RunPredictorWithRuntimeConfig, predictor0, xpu_runtime_config0);
std::thread t1(
RunPredictorWithRuntimeConfig, predictor1, xpu_runtime_config1);
t0.join();
t1.join();
}
xpu_stream_destroy(stream0);
xpu_stream_destroy(stream1);
}
TEST(runtime_context, new_context) {
auto* context = baidu::xpu::api::create_context();
CHECK_NOTNULL(context);
{
experimental::XpuRuntimeConfig xpu_runtime_config;
xpu_runtime_config.context = context;
xpu_runtime_config.stream = nullptr;
xpu_runtime_config.l3_size = 0;
xpu_runtime_config.l3_ptr = nullptr;
xpu_runtime_config.l3_autotune_size = 0;
RUN_WITH_RUNTIME_CONFIG(0, xpu_runtime_config);
}
baidu::xpu::api::destroy_context(context);
}
} // namespace paddle_infer