wehub/paddlepaddle--paddle
1
0
Fork 0
Code Issues Pull Requests Actions 2 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:40:42 +08:00
commit e25996e7db
15472 changed files with 3536181 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/quantization/CMakeLists.txt
+500
View File
@@ -0,0 +1,500 @@
include(../cpp/inference/test.cmake)
file(
GLOB TEST_OPS
RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
"test_*.py")
string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}")
function(_inference_analysis_python_api_int8_test target model_dir data_path
filename use_onednn)
py_test(
${target}
SRCS ${filename}
ENVS
CPU_NUM_THREADS=${CPU_NUM_THREADS_ON_CI}
FLAGS_use_onednn=${use_onednn}
ARGS
--infer_model
${model_dir}/model
--infer_data
${data_path}
--int8_model_save_path
int8_models/${target}
--warmup_batch_size
${WARMUP_BATCH_SIZE}
--batch_size
50)
endfunction()
function(inference_analysis_python_api_int8_test target model_dir data_path
filename)
_inference_analysis_python_api_int8_test(${target} ${model_dir} ${data_path}
${filename} False)
endfunction()
function(inference_analysis_python_api_int8_test_custom_warmup_batch_size
target model_dir data_dir filename warmup_batch_size)
set(WARMUP_BATCH_SIZE ${warmup_batch_size})
inference_analysis_python_api_int8_test(${target} ${model_dir} ${data_dir}
${filename})
endfunction()
function(inference_analysis_python_api_int8_test_onednn target model_dir
data_path filename)
_inference_analysis_python_api_int8_test(${target} ${model_dir} ${data_path}
${filename} True)
endfunction()
function(download_data install_dir url data_file check_sum)
if(NOT EXISTS ${install_dir}/${data_file})
inference_download_and_uncompress(${install_dir} ${url} ${data_file}
${check_sum})
endif()
endfunction()
function(download_quant_data install_dir data_file check_sum)
if(NOT EXISTS ${install_dir}/${data_file})
inference_download_and_uncompress(${install_dir} ${INFERENCE_URL}/int8
${data_file} ${check_sum})
endif()
endfunction()
function(download_quant_model install_dir data_file check_sum)
if(NOT EXISTS ${install_dir}/${data_file})
inference_download_and_uncompress(
${install_dir} ${INFERENCE_URL}/int8/QAT_models ${data_file} ${check_sum})
endif()
endfunction()
function(download_quant_fp32_model install_dir data_file check_sum)
if(NOT EXISTS ${install_dir}/${data_file})
inference_download_and_uncompress(
${install_dir} ${INFERENCE_URL}/int8/QAT_models/fp32 ${data_file}
${check_sum})
endif()
endfunction()
function(download_lstm_model install_dir data_file check_sum)
if(NOT EXISTS ${install_dir}/${data_file})
inference_download_and_uncompress(${install_dir} ${INFERENCE_URL}/lstm
${data_file} ${check_sum})
endif()
endfunction()
function(inference_quant_int8_image_classification_test target quant_model_dir
dataset_path)
py_test(
${target}
SRCS "${CMAKE_CURRENT_SOURCE_DIR}/quant_int8_image_classification_comparison.py"
ENVS
FLAGS_OMP_NUM_THREADS=${CPU_NUM_THREADS_ON_CI}
OMP_NUM_THREADS=${CPU_NUM_THREADS_ON_CI}
FLAGS_use_onednn=true
ARGS
--quant_model
${quant_model_dir}
--infer_data
${dataset_path}
--batch_size
25
--batch_num
2
--acc_diff_threshold
0.1)
endfunction()
# set batch_size 10 for UT only (avoid OOM).
# For whole dataset, use batch_size 25
function(inference_quant2_int8_image_classification_test target quant_model_dir
fp32_model_dir dataset_path)
py_test(
${target}
SRCS "${CMAKE_CURRENT_SOURCE_DIR}/quant2_int8_image_classification_comparison.py"
ENVS
FLAGS_OMP_NUM_THREADS=${CPU_NUM_THREADS_ON_CI}
OMP_NUM_THREADS=${CPU_NUM_THREADS_ON_CI}
FLAGS_use_onednn=true
ARGS
--quant_model
${quant_model_dir}
--fp32_model
${fp32_model_dir}
--infer_data
${dataset_path}
--batch_size
50
--batch_num
2
--acc_diff_threshold
0.1)
endfunction()
# set batch_size 10 for UT only (avoid OOM).
# For whole dataset, use batch_size 20
function(
inference_quant2_int8_nlp_test
target
quant_model_dir
fp32_model_dir
dataset_path
labels_path
ops_to_quantize)
py_test(
${target}
SRCS "${CMAKE_CURRENT_SOURCE_DIR}/quant2_int8_nlp_comparison.py"
ENVS
FLAGS_OMP_NUM_THREADS=${CPU_NUM_THREADS_ON_CI}
OMP_NUM_THREADS=${CPU_NUM_THREADS_ON_CI}
FLAGS_use_onednn=true
ARGS
--quant_model
${quant_model_dir}
--fp32_model
${fp32_model_dir}
--infer_data
${dataset_path}
--labels
${labels_path}
--batch_size
10
--batch_num
2
--acc_diff_threshold
0.1
--ops_to_quantize
${ops_to_quantize})
endfunction()
function(inference_quant2_int8_lstm_model_test target fp32_model quant_model
dataset_path)
py_test(
${target}
SRCS "${CMAKE_CURRENT_SOURCE_DIR}/quant2_int8_lstm_model.py"
ARGS
--fp32_model
${fp32_model}
--quant_model
${quant_model}
--infer_data
${dataset_path}
--num_threads
1
--onednn_cache_capacity
100
--warmup_iter
100
--acc_diff_threshold
0.11)
endfunction()
function(download_quant_data install_dir data_file check_sum)
if(NOT EXISTS ${install_dir}/${data_file})
inference_download_and_uncompress(${install_dir} ${INFERENCE_URL}/int8
${data_file} ${check_sum})
endif()
endfunction()
function(download_quant_model install_dir data_file check_sum)
if(NOT EXISTS ${install_dir}/${data_file})
inference_download_and_uncompress(
${install_dir} ${INFERENCE_URL}/int8/QAT_models ${data_file} ${check_sum})
endif()
endfunction()
function(convert_model2dot_test target model_path save_graph_dir
save_graph_name)
py_test(
${target}
SRCS ${CMAKE_CURRENT_SOURCE_DIR}/convert_model2dot.py
ARGS
--model_path
${model_path}
--save_graph_dir
${save_graph_dir}
--save_graph_name
${save_graph_name})
endfunction()
if(WIN32)
list(REMOVE_ITEM TEST_OPS test_light_nas)
list(REMOVE_ITEM TEST_OPS test_post_training_quantization_mnist)
list(REMOVE_ITEM TEST_OPS test_ptq)
list(REMOVE_ITEM TEST_OPS test_post_training_quantization_mobilenetv1)
list(REMOVE_ITEM TEST_OPS test_post_training_quantization_resnet50)
list(REMOVE_ITEM TEST_OPS test_post_training_quantization_program_resnet50)
list(REMOVE_ITEM TEST_OPS test_post_training_quantization_lstm_model)
list(REMOVE_ITEM TEST_OPS test_imperative_ptq)
list(REMOVE_ITEM TEST_OPS test_weight_quantization_mobilenetv1)
list(REMOVE_ITEM TEST_OPS test_imperative_qat_amp)
list(REMOVE_ITEM TEST_OPS test_imperative_qat_lsq)
list(REMOVE_ITEM TEST_OPS test_imperative_qat_matmul)
list(REMOVE_ITEM TEST_OPS test_weight_only_linear)
list(REMOVE_ITEM TEST_OPS test_weight_quantize)
list(REMOVE_ITEM TEST_OPS test_llm_int8_linear)
list(REMOVE_ITEM TEST_OPS test_quant_aware)
list(REMOVE_ITEM TEST_OPS test_quant_post_quant_aware)
list(REMOVE_ITEM TEST_OPS test_quant_aware_user_defined)
list(REMOVE_ITEM TEST_OPS test_quant_aware_config)
list(REMOVE_ITEM TEST_OPS test_quant_amp)
list(REMOVE_ITEM TEST_OPS test_apply_per_channel_scale)
endif()
if(NOT WITH_GPU)
list(REMOVE_ITEM TEST_OPS test_weight_only_linear)
list(REMOVE_ITEM TEST_OPS test_weight_quantize)
list(REMOVE_ITEM TEST_OPS test_llm_int8_linear)
list(REMOVE_ITEM TEST_OPS test_apply_per_channel_scale)
endif()
#
if(WITH_GPU)
if(${CUDA_ARCH_NAME} STREQUAL "Hopper")
list(REMOVE_ITEM TEST_OPS test_weight_only_linear)
endif()
endif()
if(LINUX AND WITH_ONEDNN)
#### Image classification dataset: ImageNet (small)
# The dataset should already be downloaded for INT8v2 unit tests
set(IMAGENET_DATA_PATH "${INFERENCE_DEMO_INSTALL_DIR}/imagenet/data.bin")
#### INT8 image classification python api test
# Models should be already downloaded for INT8v2 unit tests
set(INT8_INSTALL_DIR "${INFERENCE_DEMO_INSTALL_DIR}/int8v2")
#### QUANT & INT8 comparison python api tests
set(QUANT_INSTALL_DIR "${INFERENCE_DEMO_INSTALL_DIR}/quant")
### Quant1 for image classification
# Quant ResNet50
set(QUANT_RESNET50_MODEL_DIR "${QUANT_INSTALL_DIR}/ResNet50_quant")
set(QUANT_RESNET50_MODEL_ARCHIVE "ResNet50_qat_model.tar.gz")
download_quant_model(
${QUANT_RESNET50_MODEL_DIR} ${QUANT_RESNET50_MODEL_ARCHIVE}
ff89b934ab961c3a4a844193ece2e8a7)
inference_quant_int8_image_classification_test(
test_quant_int8_resnet50_onednn ${QUANT_RESNET50_MODEL_DIR}/model
${IMAGENET_DATA_PATH})
# Quant ResNet101
set(QUANT_RESNET101_MODEL_DIR "${QUANT_INSTALL_DIR}/ResNet101_quant")
set(QUANT_RESNET101_MODEL_ARCHIVE "ResNet101_qat_model.tar.gz")
download_quant_model(
${QUANT_RESNET101_MODEL_DIR} ${QUANT_RESNET101_MODEL_ARCHIVE}
95c6d01e3aeba31c13efb2ba8057d558)
# inference_quant_int8_image_classification_test( \
# test_quant_int8_resnet101_onednn \
# ${QUANT_RESNET101_MODEL_DIR}/model \
# ${IMAGENET_DATA_PATH})
# Quant GoogleNet
set(QUANT_GOOGLENET_MODEL_DIR "${QUANT_INSTALL_DIR}/GoogleNet_quant")
set(QUANT_GOOGLENET_MODEL_ARCHIVE "GoogleNet_qat_model.tar.gz")
download_quant_model(
${QUANT_GOOGLENET_MODEL_DIR} ${QUANT_GOOGLENET_MODEL_ARCHIVE}
1d4a7383baa63e7d1c423e8db2b791d5)
#inference_quant_int8_image_classification_test(
# test_quant_int8_googlenet_onednn ${QUANT_GOOGLENET_MODEL_DIR}/model
# ${IMAGENET_DATA_PATH})
# Quant MobileNetV1
set(QUANT_MOBILENETV1_MODEL_DIR "${QUANT_INSTALL_DIR}/MobileNetV1_quant")
set(QUANT_MOBILENETV1_MODEL_ARCHIVE "MobileNetV1_qat_model.tar.gz")
download_quant_model(
${QUANT_MOBILENETV1_MODEL_DIR} ${QUANT_MOBILENETV1_MODEL_ARCHIVE}
3b774d94a9fcbb604d09bdb731fc1162)
# Quant MobileNetV2
set(QUANT_MOBILENETV2_MODEL_DIR "${QUANT_INSTALL_DIR}/MobileNetV2_quant")
set(QUANT_MOBILENETV2_MODEL_ARCHIVE "MobileNetV2_qat_model.tar.gz")
download_quant_model(
${QUANT_MOBILENETV2_MODEL_DIR} ${QUANT_MOBILENETV2_MODEL_ARCHIVE}
758a99d9225d8b73e1a8765883f96cdd)
inference_quant_int8_image_classification_test(
test_quant_int8_mobilenetv2_onednn ${QUANT_MOBILENETV2_MODEL_DIR}/model
${IMAGENET_DATA_PATH})
# Quant VGG16
set(QUANT_VGG16_MODEL_DIR "${QUANT_INSTALL_DIR}/VGG16_quant")
set(QUANT_VGG16_MODEL_ARCHIVE "VGG16_qat_model.tar.gz")
download_quant_model(${QUANT_VGG16_MODEL_DIR} ${QUANT_VGG16_MODEL_ARCHIVE}
c37e63ca82a102f47be266f8068b0b55)
# inference_quant_int8_image_classification_test( \
# test_quant_int8_vgg16_onednn \
# ${QUANT_VGG16_MODEL_DIR}/model \
# ${IMAGENET_DATA_PATH})
# Quant VGG19
set(QUANT_VGG19_MODEL_DIR "${QUANT_INSTALL_DIR}/VGG19_quant")
set(QUANT_VGG19_MODEL_ARCHIVE "VGG19_qat_model.tar.gz")
download_quant_model(${QUANT_VGG19_MODEL_DIR} ${QUANT_VGG19_MODEL_ARCHIVE}
62bcd4b6c3ca2af67e8251d1c96ea18f)
# inference_quant_int8_image_classification_test( \
# test_quant_int8_vgg19_onednn ${QUANT_VGG19_MODEL_DIR}/model \
# ${IMAGENET_DATA_PATH})
### Quant2 for image classification
# Quant2 ResNet50 with input/output scales in
# `fake_quantize_moving_average_abs_max` operators,
# with weight scales in `fake_dequantize_max_abs` operators
set(QUANT2_RESNET50_MODEL_DIR "${QUANT_INSTALL_DIR}/ResNet50_quant2")
set(QUANT2_RESNET50_MODEL_ARCHIVE "ResNet50_qat_perf.tar.gz")
download_quant_model(
${QUANT2_RESNET50_MODEL_DIR} ${QUANT2_RESNET50_MODEL_ARCHIVE}
e87309457e8c462a579340607f064d66)
set(FP32_RESNET50_MODEL_DIR "${INT8_INSTALL_DIR}/resnet50")
# Quant2 ResNet50 with input/output scales in `fake_quantize_range_abs_max`
# operators and the `out_threshold` attributes,
# with weight scales in `fake_dequantize_max_abs` operators
set(QUANT2_RESNET50_RANGE_MODEL_DIR
"${QUANT_INSTALL_DIR}/ResNet50_quant2_range")
set(QUANT2_RESNET50_RANGE_MODEL_ARCHIVE "ResNet50_qat_range.tar.gz")
download_quant_model(
${QUANT2_RESNET50_RANGE_MODEL_DIR} ${QUANT2_RESNET50_RANGE_MODEL_ARCHIVE}
2fdc8a139f041c0d270abec826b2d304)
# Quant2 ResNet50 with input/output scales in `fake_quantize_range_abs_max`
# operators and the `out_threshold` attributes,
# with weight scales in `fake_channel_wise_dequantize_max_abs` operators
set(QUANT2_RESNET50_CHANNELWISE_MODEL_DIR
"${QUANT_INSTALL_DIR}/ResNet50_quant2_channelwise")
set(QUANT2_RESNET50_CHANNELWISE_MODEL_ARCHIVE
"ResNet50_qat_channelwise.tar.gz")
download_quant_model(
${QUANT2_RESNET50_CHANNELWISE_MODEL_DIR}
${QUANT2_RESNET50_CHANNELWISE_MODEL_ARCHIVE}
887a1b1b0e9a4efd10f263a43764db26)
# Quant2 MobileNetV1
set(QUANT2_MOBILENETV1_MODEL_DIR "${QUANT_INSTALL_DIR}/MobileNetV1_quant2")
set(QUANT2_MOBILENETV1_MODEL_ARCHIVE "MobileNet_qat_perf.tar.gz")
download_quant_model(
${QUANT2_MOBILENETV1_MODEL_DIR} ${QUANT2_MOBILENETV1_MODEL_ARCHIVE}
7f626e453db2d56fed6c2538621ffacf)
set(FP32_MOBILENETV1_MODEL_DIR "${INT8_INSTALL_DIR}/mobilenetv1")
### Quant2 for NLP
set(NLP_DATA_ARCHIVE "Ernie_dataset.tar.gz")
set(NLP_DATA_DIR "${INFERENCE_DEMO_INSTALL_DIR}/Ernie_dataset")
set(NLP_DATA_PATH "${NLP_DATA_DIR}/Ernie_dataset/1.8w.bs1")
set(NLP_LABELS_PATH "${NLP_DATA_DIR}/Ernie_dataset/label.xnli.dev")
download_quant_data(${NLP_DATA_DIR} ${NLP_DATA_ARCHIVE}
e650ce0cbc1fadbed5cc2c01d4e734dc)
# Quant2 Ernie
set(QUANT2_ERNIE_MODEL_ARCHIVE "ernie_qat.tar.gz")
set(QUANT2_ERNIE_MODEL_DIR "${QUANT_INSTALL_DIR}/Ernie_quant2")
download_quant_model(${QUANT2_ERNIE_MODEL_DIR} ${QUANT2_ERNIE_MODEL_ARCHIVE}
f7cdf4720755ecf66efbc8044e9922d9)
set(FP32_ERNIE_MODEL_ARCHIVE "ernie_fp32_model.tar.gz")
set(FP32_ERNIE_MODEL_DIR "${QUANT_INSTALL_DIR}/Ernie_float")
download_quant_fp32_model(${FP32_ERNIE_MODEL_DIR} ${FP32_ERNIE_MODEL_ARCHIVE}
114f38804a3ef8c45e7259e68bbd838b)
set(QUANT2_ERNIE_OPS_TO_QUANTIZE "fused_matmul,matmul,matmul_v2,slice")
# Quant2 GRU
set(QUANT2_GRU_MODEL_DIR "${QUANT_INSTALL_DIR}/GRU_quant2")
set(QUANT2_GRU_OPS_TO_QUANTIZE "multi_gru")
# Quant2 LSTM
set(QUANT2_LSTM_MODEL_ARCHIVE "lstm_quant.tar.gz")
set(QUANT2_LSTM_MODEL_DIR "${QUANT_INSTALL_DIR}/lstm_quant_test")
download_quant_model(${QUANT2_LSTM_MODEL_DIR} ${QUANT2_LSTM_MODEL_ARCHIVE}
40a693803b12ee9e251258f32559abcb)
# Convert Quant2 model to dot and pdf files
set(QUANT2_INT8_ERNIE_DOT_SAVE_PATH
"${QUANT_INSTALL_DIR}/Ernie_quant2_int8_dot_file")
### PTQ INT8
# PTQ int8 lstm model
set(QUANT2_INT8_LSTM_SAVE_PATH "${QUANT_INSTALL_DIR}/lstm_quant2_int8")
set(LSTM_DATA_FILE "quant_lstm_input_data.tar.gz")
set(LSTM_URL "${INFERENCE_URL}/int8/unittest_model_data")
download_data(${QUANT2_INT8_LSTM_SAVE_PATH} ${LSTM_URL} ${LSTM_DATA_FILE}
add84c754e9b792fea1fbd728d134ab7)
set(QUANT2_FP32_LSTM_MODEL_ARCHIVE "lstm_fp32_model.tar.gz")
download_lstm_model(
${QUANT2_INT8_LSTM_SAVE_PATH} ${QUANT2_FP32_LSTM_MODEL_ARCHIVE}
eecd9f44d69a84acc1cf2235c4b8b743)
inference_quant2_int8_lstm_model_test(
test_quant2_int8_lstm_onednn ${QUANT2_INT8_LSTM_SAVE_PATH}/lstm_fp32_model
${QUANT2_LSTM_MODEL_DIR}/lstm_quant
${QUANT2_INT8_LSTM_SAVE_PATH}/quant_lstm_input_data)
endif()
# Since the tests for Quant & INT8 comparison support only testing on Linux
# with One-DNN, we remove it here to not test it on other systems.
list(REMOVE_ITEM TEST_OPS test_onednn_int8_quantization_strategy
quant_int8_image_classification_comparison quant_int8_nlp_comparison)
#TODO(wanghaoshuang): Fix this unittest failed on GCC8.
list(REMOVE_ITEM TEST_OPS test_auto_pruning)
list(REMOVE_ITEM TEST_OPS test_filter_pruning)
# fix
if(WIN32)
set(SINGLE_CARD_TEST_OPS
test_imperative_qat_channelwise test_imperative_qat
test_imperative_qat_fuse test_imperative_qat_lsq
test_imperative_qat_matmul test_imperative_out_scale)
list(REMOVE_ITEM TEST_OPS ${SINGLE_CARD_TEST_OPS})
foreach(src ${SINGLE_CARD_TEST_OPS})
py_test(${src} SRCS ${src}.py ENVS CUDA_VISIBLE_DEVICES=0)
endforeach()
endif()
foreach(src ${TEST_OPS})
py_test(${src} SRCS ${src}.py)
endforeach()
# setting timeout value for old unittests
if(NOT WIN32)
set_tests_properties(test_post_training_quantization_lstm_model
PROPERTIES TIMEOUT 120)
set_tests_properties(test_post_training_quantization_program_resnet50
PROPERTIES TIMEOUT 240)
set_tests_properties(test_post_training_quantization_mobilenetv1
PROPERTIES TIMEOUT 900 LABELS "RUN_TYPE=NIGHTLY")
set_tests_properties(test_post_training_quantization_resnet50
PROPERTIES TIMEOUT 600 LABELS "RUN_TYPE=NIGHTLY")
set_tests_properties(test_post_training_quantization_mnist PROPERTIES TIMEOUT
150)
set_tests_properties(test_imperative_ptq PROPERTIES TIMEOUT 120)
set_tests_properties(test_ptq PROPERTIES TIMEOUT 200)
set_tests_properties(test_quant_aware_config PROPERTIES TIMEOUT 200)
endif()
set_tests_properties(test_imperative_qat_user_defined PROPERTIES TIMEOUT 200)
set_tests_properties(test_imperative_qat_lsq PROPERTIES TIMEOUT 300)
set_tests_properties(test_imperative_qat_matmul PROPERTIES TIMEOUT 300)
set_tests_properties(test_imperative_qat PROPERTIES TIMEOUT 200)
set_tests_properties(test_imperative_qat_fuse PROPERTIES TIMEOUT 200)
set_tests_properties(test_imperative_qat_channelwise PROPERTIES TIMEOUT 200)
set_tests_properties(test_imperative_out_scale PROPERTIES TIMEOUT 200)
set_tests_properties(test_imperative_skip_op PROPERTIES TIMEOUT 300)
if(LINUX AND WITH_ONEDNN)
set_tests_properties(test_quant_int8_mobilenetv2_onednn PROPERTIES TIMEOUT
120)
set_tests_properties(test_quant_int8_resnet50_onednn PROPERTIES TIMEOUT 120)
#set_tests_properties(test_quant_int8_googlenet_onednn PROPERTIES TIMEOUT 120)
set_tests_properties(test_quant2_int8_lstm_onednn PROPERTIES TIMEOUT 120)
endif()
if(APPLE)
set_tests_properties(test_post_training_quantization_mnist PROPERTIES TIMEOUT
300)
set_tests_properties(test_imperative_ptq PROPERTIES TIMEOUT 300)
set_tests_properties(test_imperative_skip_op PROPERTIES TIMEOUT 300)
set_tests_properties(test_ptq PROPERTIES TIMEOUT 300)
endif()
test/quantization/README.md
+288
View File
@@ -0,0 +1,288 @@
# SLIM Quantization-aware training (QAT) for INT8 MKL-DNN
This document describes how to use [Paddle Slim](https://paddlepaddle.github.io/PaddleSlim/index.html) to convert a quantization-aware trained model (Quant model) into INT8 MKL-DNN quantized model and run it.
In **Release 1.5**, we have released the first approach to the MKL-DNN-based quantization of Quant models, called Quant1. It enabled the `conv2d` and `mul` INT8 MKL-DNN kernels for Quant trained models (GoogleNet, MobileNetV1, MobileNetV2, ResNet50, ResNet101, VGG16, and VGG19) with 0.05% accuracy diff.
In **Release 1.6**, a new approach was introduced, called Quant2, which adds support for more performance optimizations and more INT8 MKL-DNN kernels. INT8 MKL-DNN models obtained using Quant2 have much better inference performance than using Quant1, with only a little bit bigger accuracy diff.
In **Release 1.7**, a support for [Ernie (NLP) Quant trained model](https://github.com/PaddlePaddle/benchmark/tree/master/Inference/c%2B%2B/ernie/mkldnn) was added to the Quant2.
In **Release 2.0**, further optimizations were added to the Quant2: INT8 `matmul` kernel, inplace execution of activation and `elementwise_add` operators, and broader support for quantization aware strategy from PaddleSlim.
In this document we focus on the Quant2 approach only.
## 0. Prerequisites
* PaddlePaddle in version 2.0 or higher is required. For instructions on how to install it see the [installation document](https://www.paddlepaddle.org.cn/install/quick).
* MKL-DNN and MKL are required. The highest performance gain can be observed using CPU servers supporting AVX512 instructions.
* INT8 accuracy is best on CPU servers supporting AVX512 VNNI extension (e.g. CLX class Intel processors). A linux server supports AVX512 VNNI instructions if the output of the command `lscpu` contains the `avx512_vnni` entry in the `Flags` section. AVX512 VNNI support on Windows can be checked using the [`coreinfo`]( https://docs.microsoft.com/en-us/sysinternals/downloads/coreinfo) tool.
## 1. Introduction
There are two approaches to quantization supported in PaddlePaddle: [post-training quantization](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/fluid/inference/tests/api/int8_mkldnn_quantization.md) (PTQ) and quantization-aware training (QAT). Using both PTQ and QAT a user can convert models created by PaddleSlim into INT8 models and run INT8 inference on CPU. PTQ is more automatic and requires less model preparation. However, QAT usually gives better accuracy with similar performance. In this document we focus on a transformation from intermediate models obtained during the QAT process (Quant models) into MKL-DNN INT8 models. We call this procedure Quant2.
## 2. How to turn an FP32 model into a Quant model?
A procedure on how to transform an FP32 model into a Quant model supported by the Quant2 approach is described in [this document](https://github.com/PaddlePaddle/PaddleSlim/blob/develop/demo/mkldnn_quant/README.md).
## 3. How to turn a Quant model into an INT8 MKL-DNN model?
A Quant model can be transformed into an INT8 quantized model if it contains enough information about quantization scales for every quantized operator in the graph. The process of quantization is done by the `Quant2Int8OnednnPass` pass which comprises several steps:
### Gathering scales
The information about the quantization scales is collected from two sources:
1. the `out_threshold` attribute of quantizable operators - it contains a single value quantization scale for the operator's output,
2. fake quantize/dequantize operators - they imitate quantization from FP32 into INT8, or dequantization in reverse direction, but keep the quantized tensor values as floats.
There are three types of fake quantize/dequantize operators:
* `fake_quantize_moving_average_abs_max` and `fake_quantize_range_abs_max` - used before quantized operator (e.g. `conv2d`), gather single value scale information for the op's input,
* `fake_dequantize_max_abs` and `fake_channel_wise_dequantize_max_abs` - used after quantized operators, contain scales used for the operators' weights dequantization; the first one collects a single value scale for the weights tensor, whereas the second one collects a vector of scales for each output channel of the weights,
* `fake_quantize_dequantize_moving_average_abs_max` - used after a quantized operator to get the scale value for the op's output; imitates immediate quantization and dequantization.
Scale values gathered from the fake quantize/dequantize operators have precedence over the scales collected from the `out_threshold` attributes.
Notes:
1. As the next steps describe, quantization will be applied later to an optimized FP32 model. It means that quantization scales for inputs and outputs of each quantized operator have to be gathered for tensors which are inputs and outputs of already optimized or fused operators. For example, if a model contains the following sequence of tensors and operators in the graph
```... → input1 → conv2d → output1 → batch_norm → output2 → relu → output3 → ...```
and we want to quantize the `conv2d` op, then after applying FP32 optimizations the sequence will become
```... → input1 → conv2d → output3 → ...```
and the quantization scales have to be collected for the `input1` and `output3` tensors in the Quant model.
2. Quantization of the following operators is supported: `conv2d`, `depthwise_conv2d`, `mul`, `fc`, `matmul`, `pool2d`, `reshape2`, `transpose2`, `concat`.
3. The longest sequence of consecutive quantizable operators in the model, the biggest performance boost can be achieved through quantization:
```... → conv2d → conv2d → pool2d → conv2d → conv2d → ...```
Quantizing single operator separated from other quantizable operators can give no performance benefits or even slow down the inference:
```... → swish → fc → softmax → ...`
### Removing fake operators
All the `fake_quantize_*` and `fake_dequantize_*` operators are being removed from the graph.
### Dequantizing weights
Weights of `conv2d`, `depthwise_conv2d` and `mul` operators are assumed to be fake-quantized (with integer values in the `int8` range, but kept as `float`s) in Quant models. Here, the information about the scale from `fake_dequantize_max_abs` and `fake_channel_wise_dequantize_max_abs` operators is used to fake-dequantize the weights back to the full float range of values. At this moment the model becomes an unoptimized clean FP32 inference model.
### Optimizing FP32 graph
A series of standard optimization passes are being applied to the FP32 graph. This gives us an optimized FP32 inference model and we can proceed with INT8 quantization.
### Computing weight scales
After optimization fuses, the weight tensors of `conv2d` or `fc` operators are likely to have different values and require new quantization scales. The weights are static, i.e. they do not change during the inference process, and the scales can be calculated simply as a maximum of absolute values from the tensor. To improve the inference accuracy we calculate the scales for each output channel separately, getting an array of quantization scales for a weight tensor.
### Taking activations into account
The basic datatype used during INT8 inference is signed INT8, with possible values from -128 to 127. However, if `conv2d` or `fc` operator has `relu` or `relu6` activation integrated in it, the output of the operator is known to have non-negative values. In that case we use unsigned INT8 datatype for output tensors, with a wider range for positive values (0 to 255), improving the inference accuracy further.
### Propagation of scales
Some of the operators (e.g. `reshape2`, `transpose2`, `pool2d` with max pooling) transform the data without changing the quantization scale. For this reason we propagate the quantization scale values through these operators without any modifications. We propagate the quantization scales also through the `scale` operator, updating the quantization scale accordingly. This approach lets us minimize the number of fake quantize/dequantize operators in the graph, because the information about the scales required for the quantization process to succeed spreads between quantized operators.
### Applying quantization passes
Having gathered all the data needed for quantization we apply the `cpu_quantize_pass` which quantizes the graph, and the `cpu_quantize_squash_pass` which optimizes the INT8 graph.
## 4. Code example
The code snipped shows how the `Quant2Int8OnednnPass` can be applied to a model graph:
```python
import paddle
import paddle.static as static
from paddle.static.quantization import Quant2Int8OnednnPass
from paddle.base.framework import IrGraph
from paddle.framework import core
# Create the IrGraph by Program
graph = IrGraph(core.Graph(static.Program().desc), for_test=False)
place = paddle.CPUPlace()
# Convert the IrGraph to MKL-DNN supported INT8 IrGraph using the
# Quant2Int8OnednnPass. It requires a list of operators to be quantized
onednn_pass = Quant2Int8OnednnPass({'conv2d', 'pool2d'}, static.global_scope(), place, core, False)
# Apply Quant2Int8OnednnPass to IrGraph
onednn_pass.apply(graph)
```
## 5. Accuracy and Performance benchmark
This section contain Quant2 MKL-DNN accuracy and performance benchmark results measured on the following server:
* Intel(R) Xeon(R) Gold 6271 (with AVX512 VNNI support),
Performance benchmarks were run with the following environment settings:
* The benchmark threads were assigned to cores by setting
```bash
export KMP_AFFINITY=granularity=fine,compact,1,0
export KMP_BLOCKTIME=1
```
* Turbo Boost was set to OFF using the command
```bash
echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo
```
### Image classification models benchmark results
#### Accuracy
>**Intel(R) Xeon(R) Gold 6271**
| Model | FP32 Top1 Accuracy | INT8 Quant Top1 Accuracy | Top1 Diff | FP32 Top5 Accuracy | INT8 Quant Top5 Accuracy | Top5 Diff |
| :----------: | :----------------: | :--------------------: | :-------: | :----------------: | :--------------------: | :-------: |
| MobileNet-V1 | 70.78% | 70.71% | -0.07% | 89.69% | 89.41% | -0.28% |
| MobileNet-V2 | 71.90% | 72.11% | +0.21% | 90.56% | 90.62% | +0.06% |
| ResNet101 | 77.50% | 77.64% | +0.14% | 93.58% | 93.58% | 0.00% |
| ResNet50 | 76.63% | 76.47% | -0.16% | 93.10% | 92.98% | -0.12% |
| VGG16 | 72.08% | 71.73% | -0.35% | 90.63% | 89.71% | -0.92% |
| VGG19 | 72.57% | 72.12% | -0.45% | 90.84% | 90.15% | -0.69% |
#### Performance
Image classification models performance was measured using a single thread. The setting is included in the benchmark reproduction commands below.
>**Intel(R) Xeon(R) Gold 6271**
| Model | FP32 (images/s) | INT8 Quant (images/s) | Ratio (INT8/FP32) |
| :----------: | :-------------: | :-----------------: | :---------------: |
| MobileNet-V1 | 74.05 | 196.98 | 2.66 |
| MobileNet-V2 | 88.60 | 187.67 | 2.12 |
| ResNet101 | 7.20 | 26.43 | 3.67 |
| ResNet50 | 13.23 | 47.44 | 3.59 |
| VGG16 | 3.47 | 10.20 | 2.94 |
| VGG19 | 2.83 | 8.67 | 3.06 |
Notes:
* Performance FP32 (images/s) values come from [INT8 MKL-DNN post-training quantization](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/fluid/inference/tests/api/int8_mkldnn_quantization.md) document.
### NLP models benchmark results
#### Accuracy
>**Intel(R) Xeon(R) Gold 6271**
| Model | FP32 Accuracy | Quant INT8 Accuracy | Accuracy Diff |
|:------------:|:----------------------:|:----------------------:|:---------:|
| Ernie | 80.20% | 79.44% | -0.76% |
#### Performance
>**Intel(R) Xeon(R) Gold 6271**
| Model | Threads | FP32 Latency (ms) | Quant INT8 Latency (ms) | Ratio (FP32/INT8) |
|:------------:|:----------------------:|:-------------------:|:---------:|:---------:|
| Ernie | 1 thread | 237.21 | 79.26 | 2.99x |
| Ernie | 20 threads | 22.08 | 12.57 | 1.76x |
## 6. How to reproduce the results
The steps below show, taking ResNet50 as an example, how to reproduce the above accuracy and performance results for Image Classification models.
To reproduce NLP models results (Ernie), please follow [How to reproduce Ernie Quant results on MKL-DNN](https://github.com/PaddlePaddle/benchmark/tree/master/Inference/c%2B%2B/ernie/mkldnn/README.md).
### Prepare dataset
Download the dataset for image classification models benchmarking by executing:
```bash
cd /PATH/TO/PADDLE
python paddle/fluid/inference/tests/api/full_ILSVRC2012_val_preprocess.py
```
The converted data binary file is saved by default in `$HOME/.cache/paddle/dataset/int8/download/int8_full_val.bin`
### Prepare models
Run the following commands to download and extract Quant model:
```bash
mkdir -p /PATH/TO/DOWNLOAD/MODEL/
cd /PATH/TO/DOWNLOAD/MODEL/
export QUANT_MODEL_NAME=ResNet50
export QUANT_MODEL_ARCHIVE=${QUANT_MODEL_NAME}_qat_model.tar.gz
wget http://paddle-inference-dist.bj.bcebos.com/int8/QAT_models/${QUANT_MODEL_ARCHIVE}
mkdir ${QUANT_MODEL_NAME} && tar -xvf ${QUANT_MODEL_ARCHIVE} -C ${QUANT_MODEL_NAME}
```
To download other Quant models, set the `QUANT_MODEL_NAME` variable in the above commands to one of the values: `ResNet101`, `MobileNetV1`, `MobileNetV2`, `VGG16`, `VGG19`.
Moreover, there are other variations of these Quant models that use different methods to obtain scales during training, run these commands to download and extract Quant model:
```bash
mkdir -p /PATH/TO/DOWNLOAD/MODEL/
cd /PATH/TO/DOWNLOAD/MODEL/
export QUANT_MODEL_NAME=ResNet50_qat_perf
export QUANT_MODEL_ARCHIVE=${QUANT_MODEL_NAME}.tar.gz
wget http://paddle-inference-dist.bj.bcebos.com/int8/QAT_models/${QUANT_MODEL_ARCHIVE}
mkdir ${QUANT_MODEL_NAME} && tar -xvf ${QUANT_MODEL_ARCHIVE} -C ${QUANT_MODEL_NAME}
```
To download other Quant models, set the `QUANT_MODEL_NAME` variable to on of the values: `ResNet50_qat_perf`, `ResNet50_qat_range`, `ResNet50_qat_channelwise`, `MobileNet_qat_perf`, where:
- `ResNet50_qat_perf`, `MobileNet_qat_perf` with input/output scales in `fake_quantize_moving_average_abs_max` operators, with weight scales in `fake_dequantize_max_abs` operators
- `ResNet50_qat_range`, with input/output scales in `fake_quantize_range_abs_max` operators and the `out_threshold` attributes, with weight scales in `fake_dequantize_max_abs` operators
- `ResNet50_qat_channelwise`, with input/output scales in `fake_quantize_range_abs_max` operators and the `out_threshold` attributes, with weight scales in `fake_channel_wise_dequantize_max_abs` operators
Download clean FP32 model for accuracy comparison against the INT8 model:
```bash
cd /PATH/TO/DOWNLOAD/MODEL/
export FP32_MODEL_NAME=resnet50
export FP32_MODEL_ARCHIVE=${FP32_MODEL_NAME}_int8_model.tar.gz
wget http://paddle-inference-dist.bj.bcebos.com/int8/${FP32_MODEL_ARCHIVE}
mkdir ${FP32_MODEL_NAME} && tar -xzvf ${FP32_MODEL_ARCHIVE} -C ${FP32_MODEL_NAME}
```
To download other FP32 models, set the `FP32_MODEL_NAME` variable to on of the values: `Res101`, `mobilenetv1`, `mobilenet_v2`, `VGG16`, and `VGG19`.
### Run benchmark
#### Accuracy benchmark commands
You can use the `quant2_int8_image_classification_comparison.py` script to reproduce the accuracy result of the INT8 Quant models. The following options are required:
* `--quant_model` - a path to a Quant model that will be transformed into INT8 model.
* `--fp32_model` - a path to an FP32 model whose accuracy will be measured and compared to the accuracy of the INT8 model.
* `--infer_data` - a path to the validation dataset.
The following options are also accepted:
* `--ops_to_quantize` - a comma-separated list of operator types to quantize. If the option is not used, an attempt to quantize all quantizable operators will be made, and in that case only quantizable operators which have quantization scales provided in the Quant model will be quantized. When deciding which operators to put on the list, the following have to be considered:
* Only operators which support quantization will be taken into account.
* All the quantizable operators from the list, which are present in the model, must have quantization scales provided in the model. Otherwise, quantization of the operator will be skipped with a message saying which variable is missing a quantization scale.
* Sometimes it may be suboptimal to quantize all quantizable operators in the model (cf. *Notes* in the **Gathering scales** section above). To find the optimal configuration for this option, user can run benchmark a few times with different lists of quantized operators present in the model and compare the results. For Image Classification models mentioned above the list usually comprises of `conv2d` and `pool2d` operators.
* `--op_ids_to_skip` - a comma-separated list of operator ids to skip in quantization. To get an id of a particular operator run the script with the `--debug` option first (see below for the description of the option), and having opened the generated file `int8_<some_number>_cpu_quantize_placement_pass.dot` find the id number written in parentheses next to the name of the operator.
* `--debug` - add this option to generate a series of `*.dot` files containing the model graphs after each step of the transformation. For a description of the DOT format see [DOT]( https://graphviz.gitlab.io/_pages/doc/info/lang.html). The files will be saved in the current location. To open the `*.dot` files use any of the Graphviz tools available on your system (e.g. `xdot` tool on Linux or `dot` tool on Windows, for documentation see [Graphviz](http://www.graphviz.org/documentation/)).
```bash
cd /PATH/TO/PADDLE
OMP_NUM_THREADS=28 FLAGS_use_onednn=true python python/paddle/static/quantization/slim/tests/quant2_int8_image_classification_comparison.py --quant_model=/PATH/TO/DOWNLOADED/QUANT/MODEL --fp32_model=/PATH/TO/DOWNLOADED/FP32/MODEL --infer_data=$HOME/.cache/paddle/dataset/int8/download/int8_full_val.bin --batch_size=50 --batch_num=1000 --acc_diff_threshold=0.01 --ops_to_quantize="conv2d,pool2d"
```
> Notes: Due to a large amount of images in the `int8_full_val.bin` dataset (50 000), the accuracy benchmark may last long. To accelerate accuracy measuring, it is recommended to set `OMP_NUM_THREADS` to the maximum number of physical cores available on the server.
#### Performance benchmark commands
To reproduce the performance results, the environment variable `OMP_NUM_THREADS=1` and `--batch_size=1` option should be set.
1. Transform the Quant model into INT8 model by applying the `Quant2Int8OnednnPass` pass and save the result. You can use the script `save_quant_model.py` for this purpose. It also accepts the option `--ops_to_quantize` with a list of operators to quantize.
```bash
cd /PATH/TO/PADDLE/build
python ../python/paddle/static/quantization/slim/tests/save_quant_model.py --quant_model_path=/PATH/TO/DOWNLOADED/QUANT/MODEL --int8_model_save_path=/PATH/TO/SAVE/QUANT/INT8/MODEL --ops_to_quantize="conv2d,pool2d"
```
2. Run the C-API test for performance benchmark.
```bash
cd /PATH/TO/PADDLE/build
OMP_NUM_THREADS=1 paddle/fluid/inference/tests/api/test_analyzer_quant_image_classification ARGS --enable_fp32=false --with_accuracy_layer=false --int8_model=/PATH/TO/SAVED/QUANT/INT8/MODEL --infer_data=$HOME/.cache/paddle/dataset/int8/download/int8_full_val.bin --batch_size=1 --paddle_num_threads=1
```
test/quantization/__init__.py
+13
View File
@@ -0,0 +1,13 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
test/quantization/convert_model2dot.py
+92
View File
@@ -0,0 +1,92 @@
# copyright (c) 2022 paddlepaddle authors. all rights reserved.
#
# licensed under the apache license, version 2.0 (the "license");
# you may not use this file except in compliance with the license.
# you may obtain a copy of the license at
#
# http://www.apache.org/licenses/license-2.0
#
# unless required by applicable law or agreed to in writing, software
# distributed under the license is distributed on an "as is" basis,
# without warranties or conditions of any kind, either express or implied.
# see the license for the specific language governing permissions and
# limitations under the license.
import argparse
import os
import sys
import unittest
import paddle
from paddle.base.framework import IrGraph
from paddle.framework import core
paddle.enable_static()
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument(
'--model_path', type=str, default='', help='A path to a model.'
)
parser.add_argument(
'--save_graph_dir',
type=str,
default='',
help='A path to save the graph.',
)
parser.add_argument(
'--save_graph_name',
type=str,
default='',
help='A name to save the graph. Default - name from model path will be used',
)
test_args, args = parser.parse_known_args(namespace=unittest)
return test_args, sys.argv[:1] + args
def generate_dot_for_model(model_path, save_graph_dir, save_graph_name):
place = paddle.CPUPlace()
exe = paddle.static.Executor(place)
inference_scope = paddle.static.global_scope()
with paddle.static.scope_guard(inference_scope):
if os.path.exists(os.path.join(model_path, '__model__')):
[
inference_program,
feed_target_names,
fetch_targets,
] = paddle.static.io.load_inference_model(
model_path, exe, model_filename='__model__'
)
else:
[
inference_program,
feed_target_names,
fetch_targets,
] = paddle.static.load_inference_model(
model_path,
exe,
model_filename='model',
params_filename='params',
)
graph = IrGraph(core.Graph(inference_program.desc), for_test=True)
if not os.path.exists(save_graph_dir):
os.makedirs(save_graph_dir)
model_name = os.path.basename(os.path.normpath(save_graph_dir))
if save_graph_name == '':
save_graph_name = model_name
graph.draw(save_graph_dir, save_graph_name, graph.all_op_nodes())
print(
f"Success! Generated dot and pdf files for {model_name} model, that can be found at {save_graph_dir} named {save_graph_name}.\n"
)
if __name__ == '__main__':
global test_args
test_args, remaining_args = parse_args()
generate_dot_for_model(
test_args.model_path,
test_args.save_graph_dir,
test_args.save_graph_name,
)
test/quantization/imperative_test_utils.py
+316
View File
@@ -0,0 +1,316 @@
# copyright (c) 2021 paddlepaddle authors. all rights reserved.
#
# licensed under the apache license, version 2.0 (the "license");
# you may not use this file except in compliance with the license.
# you may obtain a copy of the license at
#
# http://www.apache.org/licenses/license-2.0
#
# unless required by applicable law or agreed to in writing, software
# distributed under the license is distributed on an "as is" basis,
# without warranties or conditions of any kind, either express or implied.
# see the license for the specific language governing permissions and
# limitations under the license.
import logging
import numpy as np
import paddle
from paddle.framework import ParamAttr
from paddle.nn import (
BatchNorm1D,
BatchNorm2D,
Conv2D,
LeakyReLU,
Linear,
MaxPool2D,
PReLU,
ReLU,
ReLU6,
Sequential,
Sigmoid,
Softmax,
)
from paddle.static.log_helper import get_logger
_logger = get_logger(
__name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s'
)
def fix_model_dict(model):
fixed_state = {}
for name, param in model.named_parameters():
p_shape = param.numpy().shape
p_value = param.numpy()
if name.endswith("bias"):
value = np.zeros_like(p_value).astype('float32')
else:
value = (
np.random.normal(loc=0.0, scale=0.01, size=np.prod(p_shape))
.reshape(p_shape)
.astype('float32')
)
fixed_state[name] = value
model.set_dict(fixed_state)
return model
def pre_hook(layer, input):
input_return = input[0] * 2
return input_return
def post_hook(layer, input, output):
return output * 2
def train_lenet(lenet, reader, optimizer):
loss_list = []
lenet.train()
for batch_id, data in enumerate(reader()):
x_data = np.array([x[0].reshape(1, 28, 28) for x in data]).astype(
'float32'
)
y_data = np.array([x[1] for x in data]).astype('int64').reshape(-1, 1)
img = paddle.to_tensor(x_data)
label = paddle.to_tensor(y_data)
out = lenet(img)
loss = paddle.nn.functional.cross_entropy(
out, label, reduction='none', use_softmax=False
)
avg_loss = paddle.mean(loss)
avg_loss.backward()
optimizer.minimize(avg_loss)
lenet.clear_gradients()
if batch_id % 100 == 0:
loss_list.append(float(avg_loss))
_logger.info('{}: {}'.format('loss', float(avg_loss)))
return loss_list
class ImperativeLenet(paddle.nn.Layer):
def __init__(self, num_classes=10):
super().__init__()
conv2d_w1_attr = ParamAttr(name="conv2d_w_1")
conv2d_w2_attr = ParamAttr(name="conv2d_w_2")
fc_w1_attr = ParamAttr(name="fc_w_1")
fc_w2_attr = ParamAttr(name="fc_w_2")
fc_w3_attr = ParamAttr(name="fc_w_3")
conv2d_b2_attr = ParamAttr(name="conv2d_b_2")
fc_b1_attr = ParamAttr(name="fc_b_1")
fc_b2_attr = ParamAttr(name="fc_b_2")
fc_b3_attr = ParamAttr(name="fc_b_3")
self.features = Sequential(
Conv2D(
in_channels=1,
out_channels=6,
kernel_size=3,
stride=1,
padding=1,
weight_attr=conv2d_w1_attr,
bias_attr=False,
),
BatchNorm2D(6),
ReLU(),
MaxPool2D(kernel_size=2, stride=2),
Conv2D(
in_channels=6,
out_channels=16,
kernel_size=5,
stride=1,
padding=0,
weight_attr=conv2d_w2_attr,
bias_attr=conv2d_b2_attr,
),
BatchNorm2D(16),
PReLU(),
MaxPool2D(kernel_size=2, stride=2),
)
self.fc = Sequential(
Linear(
in_features=400,
out_features=120,
weight_attr=fc_w1_attr,
bias_attr=fc_b1_attr,
),
LeakyReLU(),
Linear(
in_features=120,
out_features=84,
weight_attr=fc_w2_attr,
bias_attr=fc_b2_attr,
),
Sigmoid(),
Linear(
in_features=84,
out_features=num_classes,
weight_attr=fc_w3_attr,
bias_attr=fc_b3_attr,
),
Softmax(),
)
self.add = paddle.nn.quant.add()
self.quant_stub = paddle.nn.quant.QuantStub()
def forward(self, inputs):
x = self.quant_stub(inputs)
x = self.features(x)
x = paddle.flatten(x, 1)
x = self.add(x, paddle.to_tensor([0.0])) # For CI
x = self.fc(x)
return x
class ImperativeLenetWithSkipQuant(paddle.nn.Layer):
def __init__(self, num_classes=10):
super().__init__()
conv2d_w1_attr = ParamAttr(name="conv2d_w_1")
conv2d_w2_attr = ParamAttr(name="conv2d_w_2")
fc_w1_attr = ParamAttr(name="fc_w_1")
fc_w2_attr = ParamAttr(name="fc_w_2")
fc_w3_attr = ParamAttr(name="fc_w_3")
conv2d_b1_attr = ParamAttr(name="conv2d_b_1")
conv2d_b2_attr = ParamAttr(name="conv2d_b_2")
fc_b1_attr = ParamAttr(name="fc_b_1")
fc_b2_attr = ParamAttr(name="fc_b_2")
fc_b3_attr = ParamAttr(name="fc_b_3")
self.conv2d_0 = Conv2D(
in_channels=1,
out_channels=6,
kernel_size=3,
stride=1,
padding=1,
weight_attr=conv2d_w1_attr,
bias_attr=conv2d_b1_attr,
)
self.conv2d_0.skip_quant = True
self.batch_norm_0 = BatchNorm2D(6)
self.relu_0 = ReLU()
self.pool2d_0 = MaxPool2D(kernel_size=2, stride=2)
self.conv2d_1 = Conv2D(
in_channels=6,
out_channels=16,
kernel_size=5,
stride=1,
padding=0,
weight_attr=conv2d_w2_attr,
bias_attr=conv2d_b2_attr,
)
self.conv2d_1.skip_quant = False
self.batch_norm_1 = BatchNorm2D(16)
self.relu6_0 = ReLU6()
self.pool2d_1 = MaxPool2D(kernel_size=2, stride=2)
self.linear_0 = Linear(
in_features=400,
out_features=120,
weight_attr=fc_w1_attr,
bias_attr=fc_b1_attr,
)
self.linear_0.skip_quant = True
self.leaky_relu_0 = LeakyReLU()
self.linear_1 = Linear(
in_features=120,
out_features=84,
weight_attr=fc_w2_attr,
bias_attr=fc_b2_attr,
)
self.linear_1.skip_quant = False
self.sigmoid_0 = Sigmoid()
self.linear_2 = Linear(
in_features=84,
out_features=num_classes,
weight_attr=fc_w3_attr,
bias_attr=fc_b3_attr,
)
self.linear_2.skip_quant = False
self.softmax_0 = Softmax()
def forward(self, inputs):
x = self.conv2d_0(inputs)
x = self.batch_norm_0(x)
x = self.relu_0(x)
x = self.pool2d_0(x)
x = self.conv2d_1(x)
x = self.batch_norm_1(x)
x = self.relu6_0(x)
x = self.pool2d_1(x)
x = paddle.flatten(x, 1)
x = self.linear_0(x)
x = self.leaky_relu_0(x)
x = self.linear_1(x)
x = self.sigmoid_0(x)
x = self.linear_2(x)
x = self.softmax_0(x)
return x
class ImperativeLinearBn(paddle.nn.Layer):
def __init__(self):
super().__init__()
fc_w_attr = paddle.ParamAttr(
name="fc_weight",
initializer=paddle.nn.initializer.Constant(value=0.5),
)
fc_b_attr = paddle.ParamAttr(
name="fc_bias",
initializer=paddle.nn.initializer.Constant(value=1.0),
)
bn_w_attr = paddle.ParamAttr(
name="bn_weight",
initializer=paddle.nn.initializer.Constant(value=0.5),
)
self.linear = Linear(
in_features=10,
out_features=10,
weight_attr=fc_w_attr,
bias_attr=fc_b_attr,
)
self.bn = BatchNorm1D(10, weight_attr=bn_w_attr)
def forward(self, inputs):
x = self.linear(inputs)
x = self.bn(x)
return x
class ImperativeLinearBn_hook(paddle.nn.Layer):
def __init__(self):
super().__init__()
fc_w_attr = paddle.ParamAttr(
name="linear_weight",
initializer=paddle.nn.initializer.Constant(value=0.5),
)
self.linear = Linear(
in_features=10, out_features=10, weight_attr=fc_w_attr
)
self.bn = BatchNorm1D(10)
forward_pre = self.linear.register_forward_pre_hook(pre_hook)
forward_post = self.bn.register_forward_post_hook(post_hook)
def forward(self, inputs):
x = self.linear(inputs)
x = self.bn(x)
return x
test/quantization/quant2_int8_image_classification_comparison.py
+492
View File
@@ -0,0 +1,492 @@
# copyright (c) 2019 paddlepaddle authors. all rights reserved.
#
# licensed under the apache license, version 2.0 (the "license");
# you may not use this file except in compliance with the license.
# you may obtain a copy of the license at
#
# http://www.apache.org/licenses/license-2.0
#
# unless required by applicable law or agreed to in writing, software
# distributed under the license is distributed on an "as is" basis,
# without warranties or conditions of any kind, either express or implied.
# see the license for the specific language governing permissions and
# limitations under the license.
import argparse
import logging
import os
import struct
import sys
import time
import unittest
import numpy as np
import paddle
from paddle.base.framework import IrGraph
from paddle.framework import core
from paddle.static.quantization import Quant2Int8OnednnPass
paddle.enable_static()
logging.basicConfig(format='%(asctime)s-%(levelname)s: %(message)s')
_logger = logging.getLogger(__name__)
_logger.setLevel(logging.INFO)
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('--batch_size', type=int, default=1, help='Batch size.')
parser.add_argument(
'--skip_batch_num',
type=int,
default=0,
help='Number of the first minibatches to skip in performance statistics.',
)
parser.add_argument(
'--quant_model', type=str, default='', help='A path to a Quant model.'
)
parser.add_argument(
'--fp32_model', type=str, default='', help='A path to an FP32 model.'
)
parser.add_argument('--infer_data', type=str, default='', help='Data file.')
parser.add_argument(
'--batch_num',
type=int,
default=0,
help='Number of batches to process. 0 or less means whole dataset. Default: 0.',
)
parser.add_argument(
'--acc_diff_threshold',
type=float,
default=0.01,
help='Accepted accuracy difference threshold.',
)
parser.add_argument(
'--ops_to_quantize',
type=str,
default='',
help='A comma separated list of operators to quantize. Only quantizable operators are taken into account. If the option is not used, an attempt to quantize all quantizable operators will be made.',
)
parser.add_argument(
'--op_ids_to_skip',
type=str,
default='',
help='A comma separated list of operator ids to skip in quantization.',
)
parser.add_argument(
'--targets',
type=str,
default='quant,int8,fp32',
help='A comma separated list of inference types to run ("int8", "fp32", "quant"). Default: "quant,int8,fp32"',
)
parser.add_argument(
'--debug',
action='store_true',
help='If used, the graph of Quant model is drawn.',
)
test_args, args = parser.parse_known_args(namespace=unittest)
return test_args, sys.argv[:1] + args
class Quant2Int8ImageClassificationComparisonTest(unittest.TestCase):
"""
Test for accuracy comparison of FP32 and Quant2 INT8 Image Classification inference.
"""
def _reader_creator(self, data_file='data.bin'):
def reader():
with open(data_file, 'rb') as fp:
num = fp.read(8)
num = struct.unpack('q', num)[0]
imgs_offset = 8
img_ch = 3
img_w = 224
img_h = 224
img_pixel_size = 4
img_size = img_ch * img_h * img_w * img_pixel_size
label_size = 8
labels_offset = imgs_offset + num * img_size
step = 0
while step < num:
fp.seek(imgs_offset + img_size * step)
img = fp.read(img_size)
img = struct.unpack_from(f'{img_ch * img_w * img_h}f', img)
img = np.array(img)
img.shape = (img_ch, img_w, img_h)
fp.seek(labels_offset + label_size * step)
label = fp.read(label_size)
label = struct.unpack('q', label)[0]
yield img, int(label)
step += 1
return reader
def _get_batch_accuracy(self, batch_output=None, labels=None):
total = 0
correct = 0
correct_5 = 0
for n, result in enumerate(batch_output):
index = result.argsort()
top_1_index = index[-1]
top_5_index = index[-5:]
total += 1
if top_1_index == labels[n]:
correct += 1
if labels[n] in top_5_index:
correct_5 += 1
acc1 = float(correct) / float(total)
acc5 = float(correct_5) / float(total)
return acc1, acc5
def _prepare_for_fp32_onednn(self, graph):
ops = graph.all_op_nodes()
for op_node in ops:
name = op_node.name()
if name in ['depthwise_conv2d']:
input_var_node = graph._find_node_by_name(
op_node.inputs, op_node.input("Input")[0]
)
weight_var_node = graph._find_node_by_name(
op_node.inputs, op_node.input("Filter")[0]
)
output_var_node = graph._find_node_by_name(
graph.all_var_nodes(), op_node.output("Output")[0]
)
attrs = {
name: op_node.op().attr(name)
for name in op_node.op().attr_names()
}
conv_op_node = graph.create_op_node(
op_type='conv2d',
attrs=attrs,
inputs={'Input': input_var_node, 'Filter': weight_var_node},
outputs={'Output': output_var_node},
)
graph.link_to(input_var_node, conv_op_node)
graph.link_to(weight_var_node, conv_op_node)
graph.link_to(conv_op_node, output_var_node)
graph.safe_remove_nodes(op_node)
return graph
def _predict(
self,
test_reader=None,
model_path=None,
batch_size=1,
batch_num=1,
skip_batch_num=0,
target='quant',
):
assert target in ['quant', 'int8', 'fp32']
place = paddle.CPUPlace()
exe = paddle.static.Executor(place)
inference_scope = paddle.static.global_scope()
with paddle.static.scope_guard(inference_scope):
if os.path.exists(os.path.join(model_path, '__model__')):
[
inference_program,
feed_target_names,
fetch_targets,
] = paddle.static.io.load_inference_model(
model_path, exe, model_filename=None, params_filename=None
)
else:
[
inference_program,
feed_target_names,
fetch_targets,
] = paddle.static.load_inference_model(
model_path,
exe,
model_filename='model',
params_filename='params',
)
graph = IrGraph(core.Graph(inference_program.desc), for_test=True)
if self._debug:
graph.draw('.', 'quant_orig', graph.all_op_nodes())
quant_transform_pass = Quant2Int8OnednnPass(
self._quantized_ops,
_op_ids_to_skip=self._op_ids_to_skip,
_scope=inference_scope,
_place=place,
_core=core,
_debug=self._debug,
)
if target == 'quant':
graph = self._prepare_for_fp32_onednn(graph)
elif target == 'int8':
graph = quant_transform_pass.apply(graph)
else: # target == fp32
graph = quant_transform_pass.prepare_and_optimize_fp32(graph)
inference_program = graph.to_program()
dshape = [3, 224, 224]
outputs = []
infer_accs1 = []
infer_accs5 = []
batch_acc1 = 0.0
batch_acc5 = 0.0
fpses = []
batch_times = []
batch_time = 0.0
total_samples = 0
iters = 0
infer_start_time = time.time()
for data in test_reader():
if batch_num > 0 and iters >= batch_num:
break
if iters == skip_batch_num:
total_samples = 0
infer_start_time = time.time()
images = [x[0].reshape(dshape) for x in data]
images = np.array(images).astype('float32')
labels = np.array([x[1] for x in data]).astype('int64')
if target == 'fp32':
# FP32 models have accuracy measuring layers
labels = labels.reshape([-1, 1])
start = time.time()
out = exe.run(
inference_program,
feed={
feed_target_names[0]: images,
feed_target_names[1]: labels,
},
fetch_list=fetch_targets,
)
batch_time = (time.time() - start) * 1000 # in milliseconds
batch_acc1, batch_acc5 = out[1], out[2]
outputs.append(batch_acc1)
else:
# Quant INT8 models do not have accuracy measuring layers
start = time.time()
out = exe.run(
inference_program,
feed={feed_target_names[0]: images},
fetch_list=fetch_targets,
)
batch_time = (time.time() - start) * 1000 # in milliseconds
outputs.append(out[0])
# Calculate accuracy result
batch_acc1, batch_acc5 = self._get_batch_accuracy(
out[0], labels
)
infer_accs1.append(batch_acc1)
infer_accs5.append(batch_acc5)
samples = len(data)
total_samples += samples
batch_times.append(batch_time)
fps = samples / batch_time * 1000
fpses.append(fps)
iters += 1
appx = ' (warm-up)' if iters <= skip_batch_num else ''
_logger.info(
f'batch {iters}{appx}, acc1: {batch_acc1:.4f}, acc5: {batch_acc5:.4f}, '
f'latency: {batch_time / batch_size:.4f} ms, fps: {fps:.2f}'
)
# Postprocess benchmark data
batch_latencies = batch_times[skip_batch_num:]
batch_latency_avg = np.average(batch_latencies)
latency_avg = batch_latency_avg / batch_size
fpses = fpses[skip_batch_num:]
fps_avg = np.average(fpses)
infer_total_time = time.time() - infer_start_time
acc1_avg = np.mean(infer_accs1)
acc5_avg = np.mean(infer_accs5)
_logger.info(f'Total inference run time: {infer_total_time:.2f} s')
return outputs, acc1_avg, acc5_avg, fps_avg, latency_avg
def _print_performance(self, title, fps, lat):
_logger.info(f'{title}: avg fps: {fps:.2f}, avg latency: {lat:.4f} ms')
def _print_accuracy(self, title, acc1, acc5):
_logger.info(
f'{title}: avg top1 accuracy: {acc1:.4f}, avg top5 accuracy: {acc5:.4f}'
)
def _summarize_performance(self, int8_fps, int8_lat, fp32_fps, fp32_lat):
_logger.info('--- Performance summary ---')
self._print_performance('INT8', int8_fps, int8_lat)
if fp32_lat >= 0:
self._print_performance('FP32', fp32_fps, fp32_lat)
def _summarize_accuracy(
self, quant_acc1, quant_acc5, int8_acc1, int8_acc5, fp32_acc1, fp32_acc5
):
_logger.info('--- Accuracy summary ---')
self._print_accuracy('Quant', quant_acc1, quant_acc5)
self._print_accuracy('INT8', int8_acc1, int8_acc5)
if fp32_acc1 >= 0:
self._print_accuracy('FP32', fp32_acc1, fp32_acc5)
def _compare_accuracy(self, threshold, quant_acc1, int8_acc1):
_logger.info(
f'Accepted top1 accuracy drop threshold: {threshold}. (condition: (Quant_top1_acc - IN8_top1_acc) <= threshold && Quant_top1_acc > 0.5 && INT8_top1_acc > 0.5)'
)
# We assume valid accuracy to be at least 0.5
assert quant_acc1 > 0.5
assert int8_acc1 > 0.5
assert quant_acc1 - int8_acc1 <= threshold
def _strings_from_csv(self, string):
return {s.strip() for s in string.split(',')}
def _ints_from_csv(self, string):
return set(map(int, string.split(',')))
def test_graph_transformation(self):
if not core.is_compiled_with_onednn():
return
quant_model_path = test_case_args.quant_model
assert quant_model_path, (
'The Quant model path cannot be empty. Please, use the --quant_model option.'
)
data_path = test_case_args.infer_data
assert data_path, (
'The dataset path cannot be empty. Please, use the --infer_data option.'
)
fp32_model_path = test_case_args.fp32_model
batch_size = test_case_args.batch_size
batch_num = test_case_args.batch_num
skip_batch_num = test_case_args.skip_batch_num
acc_diff_threshold = test_case_args.acc_diff_threshold
self._debug = test_case_args.debug
self._quantized_ops = set()
if test_case_args.ops_to_quantize:
self._quantized_ops = self._strings_from_csv(
test_case_args.ops_to_quantize
)
self._op_ids_to_skip = {-1}
if test_case_args.op_ids_to_skip:
self._op_ids_to_skip = self._ints_from_csv(
test_case_args.op_ids_to_skip
)
self._targets = self._strings_from_csv(test_case_args.targets)
assert self._targets.intersection({'quant', 'int8', 'fp32'}), (
'The --targets option, if used, must contain at least one of the targets: "quant", "int8", "fp32".'
)
_logger.info('Quant & INT8 prediction run.')
_logger.info(f'Quant model: {quant_model_path}')
if fp32_model_path:
_logger.info(f'FP32 model: {fp32_model_path}')
_logger.info(f'Dataset: {data_path}')
_logger.info(f'Batch size: {batch_size}')
_logger.info(f'Batch number: {batch_num}')
_logger.info(f'Accuracy drop threshold: {acc_diff_threshold}.')
_logger.info(
'Quantized ops: {}.'.format(
','.join(self._quantized_ops)
if self._quantized_ops
else 'all quantizable'
)
)
_logger.info(
'Op ids to skip quantization: {}.'.format(
','.join(map(str, self._op_ids_to_skip))
if test_case_args.op_ids_to_skip
else 'none'
)
)
_logger.info('Targets: {}.'.format(','.join(self._targets)))
if 'quant' in self._targets:
_logger.info('--- Quant prediction start ---')
val_reader = paddle.batch(
self._reader_creator(data_path), batch_size=batch_size
)
(
quant_output,
quant_acc1,
quant_acc5,
quant_fps,
quant_lat,
) = self._predict(
val_reader,
quant_model_path,
batch_size,
batch_num,
skip_batch_num,
target='quant',
)
self._print_performance('Quant', quant_fps, quant_lat)
self._print_accuracy('Quant', quant_acc1, quant_acc5)
if 'int8' in self._targets:
_logger.info('--- INT8 prediction start ---')
val_reader = paddle.batch(
self._reader_creator(data_path), batch_size=batch_size
)
(
int8_output,
int8_acc1,
int8_acc5,
int8_fps,
int8_lat,
) = self._predict(
val_reader,
quant_model_path,
batch_size,
batch_num,
skip_batch_num,
target='int8',
)
self._print_performance('INT8', int8_fps, int8_lat)
self._print_accuracy('INT8', int8_acc1, int8_acc5)
fp32_acc1 = fp32_acc5 = fp32_fps = fp32_lat = -1
if 'fp32' in self._targets and fp32_model_path:
_logger.info('--- FP32 prediction start ---')
val_reader = paddle.batch(
self._reader_creator(data_path), batch_size=batch_size
)
(
fp32_output,
fp32_acc1,
fp32_acc5,
fp32_fps,
fp32_lat,
) = self._predict(
val_reader,
fp32_model_path,
batch_size,
batch_num,
skip_batch_num,
target='fp32',
)
self._print_performance('FP32', fp32_fps, fp32_lat)
self._print_accuracy('FP32', fp32_acc1, fp32_acc5)
if {'int8', 'fp32'}.issubset(self._targets):
self._summarize_performance(int8_fps, int8_lat, fp32_fps, fp32_lat)
if {'int8', 'quant'}.issubset(self._targets):
self._summarize_accuracy(
quant_acc1,
quant_acc5,
int8_acc1,
int8_acc5,
fp32_acc1,
fp32_acc5,
)
self._compare_accuracy(acc_diff_threshold, quant_acc1, int8_acc1)
if __name__ == '__main__':
global test_case_args
test_case_args, remaining_args = parse_args()
unittest.main(argv=remaining_args)
test/quantization/quant2_int8_lstm_model.py
+276
View File
@@ -0,0 +1,276 @@
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import struct
import sys
import time
import unittest
import numpy as np
import paddle
from paddle.framework import core
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument(
'--fp32_model', type=str, default='', help='A path to a FP32 model.'
)
parser.add_argument(
'--quant_model', type=str, default='', help='A path to a quant model.'
)
parser.add_argument('--infer_data', type=str, default='', help='Data file.')
parser.add_argument(
'--warmup_iter',
type=int,
default=1,
help='Number of the first iterations to skip in performance statistics.',
)
parser.add_argument(
'--acc_diff_threshold',
type=float,
default=0.01,
help='Accepted accuracy difference threshold.',
)
parser.add_argument(
'--num_threads', type=int, default=1, help='Number of threads.'
)
parser.add_argument(
'--onednn_cache_capacity',
type=int,
default=0,
help='Onednn cache capacity. The default value in Python API is 15, which can slow down int8 models. Default 0 means unlimited cache.',
)
test_args, args = parser.parse_known_args(namespace=unittest)
return test_args, sys.argv[:1] + args
class TestLstmModelPTQ(unittest.TestCase):
def get_warmup_tensor(self, data_path, place):
data = []
with open(data_path, 'rb') as in_f:
while True:
plen = in_f.read(4)
if plen is None or len(plen) != 4:
break
alllen = struct.unpack('i', plen)[0]
label_len = alllen & 0xFFFF
seq_len = (alllen >> 16) & 0xFFFF
label = in_f.read(4 * label_len)
label = np.frombuffer(label, dtype=np.int32).reshape(
[len(label) // 4]
)
feat = in_f.read(4 * seq_len * 8)
feat = np.frombuffer(feat, dtype=np.float32).reshape(
[len(feat) // 4 // 8, 8]
)
lod_feat = [feat.shape[0]]
minputs = paddle.base.create_lod_tensor(feat, [lod_feat], place)
infer_data = core.PaddleTensor()
infer_data.lod = minputs.lod()
infer_data.data = core.PaddleBuf(np.array(minputs))
infer_data.shape = minputs.shape()
infer_data.dtype = core.PaddleDType.FLOAT32
infer_label = core.PaddleTensor()
infer_label.data = core.PaddleBuf(np.array(label))
infer_label.shape = label.shape
infer_label.dtype = core.PaddleDType.INT32
data.append([infer_data, infer_label])
warmup_data = data[:1]
inputs = data[1:]
return warmup_data, inputs
def set_config(
self,
model_path,
num_threads,
onednn_cache_capacity,
warmup_data=None,
use_analysis=False,
mode="fp32",
):
config = core.AnalysisConfig(model_path)
config.set_cpu_math_library_num_threads(num_threads)
if use_analysis:
config.disable_gpu()
config.switch_use_feed_fetch_ops(True)
config.switch_ir_optim(True)
config.enable_onednn()
config.disable_onednn_fc_passes() # fc passes caused dnnl error
config.pass_builder().insert_pass(5, "fc_lstm_fuse_pass")
config.set_onednn_cache_capacity(onednn_cache_capacity)
if mode == "ptq":
config.enable_quantizer()
config.quantizer_config().set_quant_data(warmup_data)
config.quantizer_config().set_quant_batch_size(1)
elif mode == "qat":
config.enable_onednn_int8()
return config
def run_program(
self,
model_path,
data_path,
num_threads,
onednn_cache_capacity,
warmup_iter,
use_analysis=False,
mode="fp32",
):
place = paddle.CPUPlace()
warmup_data, inputs = self.get_warmup_tensor(data_path, place)
warmup_data = [item[0] for item in warmup_data]
config = self.set_config(
model_path,
num_threads,
onednn_cache_capacity,
warmup_data,
use_analysis,
mode,
)
predictor = core.create_paddle_predictor(config)
data = [item[0] for item in inputs]
label = np.array([item[1] for item in inputs])
all_hz_num = 0
ok_hz_num = 0
all_ctc_num = 0
ok_ctc_num = 0
dataset_size = len(data)
start = time.time()
for i in range(dataset_size):
if i == warmup_iter:
start = time.time()
hz_out, ctc_out = predictor.run([data[i]])
np_hz_out = np.array(hz_out.data.float_data()).reshape(-1)
np_ctc_out = np.array(ctc_out.data.int64_data()).reshape(-1)
out_hz_label = np.argmax(np_hz_out)
this_label = label[i]
this_label_data = np.array(this_label.data.int32_data()).reshape(-1)
if this_label.shape[0] == 1:
all_hz_num += 1
best = this_label_data[0]
if out_hz_label == best:
ok_hz_num += 1
if this_label_data[0] <= 6350:
all_ctc_num += 1
if (
np_ctc_out.shape[0] == 1
and np_ctc_out.all() == this_label_data.all()
):
ok_ctc_num += 1
else:
all_ctc_num += 1
if (
np_ctc_out.shape[0] == this_label.shape[0]
and np_ctc_out.all() == this_label_data.all()
):
ok_ctc_num += 1
if all_ctc_num > 1000 or all_hz_num > 1000:
break
end = time.time()
fps = (dataset_size - warmup_iter) / (end - start)
hx_acc = ok_hz_num / all_hz_num
ctc_acc = ok_ctc_num / all_ctc_num
return hx_acc, ctc_acc, fps
def test_lstm_model(self):
if not core.is_compiled_with_onednn():
return
fp32_model = test_case_args.fp32_model
assert fp32_model, (
'The FP32 model path cannot be empty. Please, use the --fp32_model option.'
)
quant_model = test_case_args.quant_model
assert quant_model, (
'The quant model path cannot be empty. Please, use the --quant_model option.'
)
infer_data = test_case_args.infer_data
assert infer_data, (
'The dataset path cannot be empty. Please, use the --infer_data option.'
)
num_threads = test_case_args.num_threads
onednn_cache_capacity = test_case_args.onednn_cache_capacity
warmup_iter = test_case_args.warmup_iter
acc_diff_threshold = test_case_args.acc_diff_threshold
(fp32_hx_acc, fp32_ctc_acc, fp32_fps) = self.run_program(
fp32_model,
infer_data,
num_threads,
onednn_cache_capacity,
warmup_iter,
False,
mode="fp32",
)
(int8_hx_acc, int8_ctc_acc, int8_fps) = self.run_program(
fp32_model,
infer_data,
num_threads,
onednn_cache_capacity,
warmup_iter,
True,
mode="ptq",
)
(quant_hx_acc, quant_ctc_acc, quant_fps) = self.run_program(
quant_model,
infer_data,
num_threads,
onednn_cache_capacity,
warmup_iter,
True,
mode="qat",
)
print(
f"FP32: fps {fp32_fps}, hx_acc {fp32_hx_acc}, ctc_acc {fp32_ctc_acc}"
)
print(
f"PTQ_INT8: fps {int8_fps}, hx_acc {int8_hx_acc}, ctc_acc {int8_ctc_acc}"
)
print(
f"QAT: fps {quant_fps}, hx_acc {quant_hx_acc}, ctc_acc {quant_ctc_acc}"
)
sys.stdout.flush()
self.assertLess(fp32_hx_acc - int8_hx_acc, acc_diff_threshold)
self.assertLess(fp32_ctc_acc - int8_ctc_acc, acc_diff_threshold)
self.assertLess(fp32_hx_acc - quant_hx_acc, acc_diff_threshold)
self.assertLess(fp32_ctc_acc - quant_ctc_acc, acc_diff_threshold)
if __name__ == "__main__":
global test_case_args
test_case_args, remaining_args = parse_args()
unittest.main(argv=remaining_args)
test/quantization/quant2_int8_nlp_comparison.py
+411
View File
@@ -0,0 +1,411 @@
# copyright (c) 2020 paddlepaddle authors. all rights reserved.
#
# licensed under the apache license, version 2.0 (the "license");
# you may not use this file except in compliance with the license.
# you may obtain a copy of the license at
#
# http://www.apache.org/licenses/license-2.0
#
# unless required by applicable law or agreed to in writing, software
# distributed under the license is distributed on an "as is" basis,
# without warranties or conditions of any kind, either express or implied.
# see the license for the specific language governing permissions and
# limitations under the license.
import argparse
import logging
import sys
import time
import unittest
import numpy as np
import paddle
from paddle import base
from paddle.inference import Config, create_predictor
paddle.enable_static()
logging.basicConfig(format='%(asctime)s-%(levelname)s: %(message)s')
_logger = logging.getLogger(__name__)
_logger.setLevel(logging.INFO)
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('--batch_size', type=int, default=1, help='Batch size.')
parser.add_argument(
'--skip_batch_num',
type=int,
default=0,
help='Number of the first minibatches to skip in performance statistics.',
)
parser.add_argument(
'--quant_model', type=str, default='', help='A path to a Quant model.'
)
parser.add_argument(
'--fp32_model',
type=str,
default='',
help='A path to an FP32 model. If empty, the Quant model will be used for FP32 inference.',
)
parser.add_argument('--infer_data', type=str, default='', help='Data file.')
parser.add_argument(
'--labels', type=str, default='', help='File with labels.'
)
parser.add_argument(
'--batch_num',
type=int,
default=0,
help='Number of batches to process. 0 or less means whole dataset. Default: 0.',
)
parser.add_argument(
'--acc_diff_threshold',
type=float,
default=0.01,
help='Accepted accuracy difference threshold.',
)
parser.add_argument(
'--ops_to_quantize',
type=str,
default='',
help='A comma separated list of operators to quantize. Only quantizable operators are taken into account. If the option is not used, an attempt to quantize all quantizable operators will be made.',
)
parser.add_argument(
'--op_ids_to_skip',
type=str,
default='',
help='A comma separated list of operator ids to skip in quantization.',
)
parser.add_argument(
'--targets',
type=str,
default='quant,int8,fp32',
help='A comma separated list of inference types to run ("int8", "fp32", "quant"). Default: "quant,int8,fp32"',
)
parser.add_argument(
'--debug',
action='store_true',
help='If used, the graph of Quant model is drawn.',
)
test_args, args = parser.parse_known_args(namespace=unittest)
return test_args, sys.argv[:1] + args
class QuantInt8NLPComparisonTest(unittest.TestCase):
"""
Test for accuracy comparison of Quant FP32 and INT8 NLP inference.
"""
def _reader_creator(self, data_file=None, labels_file=None):
assert data_file, "The dataset file is missing."
assert labels_file, "The labels file is missing."
def reader():
with (
open(data_file, 'r') as df,
open(labels_file, 'r') as lf,
):
data_lines = df.readlines()
labels_lines = lf.readlines()
assert len(data_lines) == len(labels_lines), (
"The number of labels does not match the length of the dataset."
)
for i in range(len(data_lines)):
data_fields = data_lines[i].split(';')
assert len(data_fields) >= 2, (
"The number of data fields in the dataset is less than 2"
)
buffers = []
shape = []
for j in range(2):
data = data_fields[j].split(':')
assert len(data) >= 2, (
"Size of data in the dataset is less than 2"
)
# Shape is stored under index 0, while data under 1
shape = data[0].split()
shape.pop(0)
shape_np = np.array(shape).astype("int64")
buffer_i = data[1].split()
buffer_np = np.array(buffer_i).astype("int64")
buffer_np.shape = tuple(shape_np)
buffers.append(buffer_np)
yield buffers[0], buffers[1], int(labels_lines[i])
return reader
def _get_batch_correct(self, batch_output=None, labels=None):
total = len(batch_output)
assert total > 0, "The batch output is empty."
correct = 0
for n, output in enumerate(batch_output):
max_idx = np.where(output == output.max())
if max_idx[0] == labels[n]:
correct += 1
return correct
def set_config(
self,
model_path,
target='quant',
):
config = Config(model_path)
config.disable_gpu()
config.switch_specify_input_names(True)
config.switch_ir_optim(True)
config.switch_use_feed_fetch_ops(True)
config.enable_onednn()
if target == 'int8':
config.enable_onednn_int8(self._quantized_ops)
config.delete_pass(
"constant_folding_pass"
) # same reason as in analyzer_ernie_int8_tester.cc
return config
def _predict(
self,
test_reader=None,
model_path=None,
batch_size=1,
batch_num=1,
skip_batch_num=0,
target='fp32',
):
assert target in ['quant', 'int8', 'fp32']
print(f"target: {target}, model path: {model_path}")
config = self.set_config(
model_path,
target,
)
predictor = create_predictor(config)
input_names = predictor.get_input_names()
output_names = predictor.get_output_names()
total_correct = 0
total_samples = 0
batch_times = []
ppses = [] # predictions per second
iters = 0
infer_start_time = time.time()
for data in test_reader():
if batch_num > 0 and iters >= batch_num:
break
if iters == skip_batch_num:
total_samples = 0
infer_start_time = time.time()
# check data
inputs = []
inputs.append(np.array([x[0] for x in data]))
inputs.append(np.array([x[1] for x in data]))
labels = np.array([x[2] for x in data])
for i, name in enumerate(input_names):
input_tensor = predictor.get_input_handle(name)
input_tensor.reshape(inputs[i].shape)
input_tensor.copy_from_cpu(inputs[i].copy())
start = time.time()
predictor.run()
batch_time = (time.time() - start) * 1000 # in milliseconds
out = []
out = predictor.get_output_handle(output_names[0]).copy_to_cpu()
batch_times.append(batch_time)
batch_correct = self._get_batch_correct(out, labels)
batch_len = len(labels)
total_samples += batch_len
total_correct += batch_correct
batch_acc = float(batch_correct) / float(batch_len)
pps = batch_len / batch_time * 1000
ppses.append(pps)
latency = batch_time / batch_len
iters += 1
appx = ' (warm-up)' if iters <= skip_batch_num else ''
_logger.info(
f'batch {iters}{appx}, acc: {batch_acc:.4f}, latency: {latency:.4f} ms, predictions per sec: {pps:.2f}'
)
# Postprocess benchmark data
infer_total_time = time.time() - infer_start_time
batch_latencies = batch_times[skip_batch_num:]
batch_latency_avg = np.average(batch_latencies)
latency_avg = batch_latency_avg / batch_size
ppses = ppses[skip_batch_num:]
pps_avg = np.average(ppses)
acc_avg = float(np.sum(total_correct)) / float(total_samples)
_logger.info(f'Total inference run time: {infer_total_time:.2f} s')
return acc_avg, pps_avg, latency_avg
def _print_performance(self, title, pps, lat):
_logger.info(
f'{title}: avg predictions per sec: {pps:.2f}, avg latency: {lat:.4f} ms'
)
def _print_accuracy(self, title, acc):
_logger.info(f'{title}: avg accuracy: {acc:.6f}')
def _summarize_performance(
self, quant_pps, quant_lat, int8_pps, int8_lat, fp32_pps, fp32_lat
):
_logger.info('--- Performance summary ---')
self._print_performance('QUANT', quant_pps, quant_lat)
self._print_performance('INT8', int8_pps, int8_lat)
if fp32_lat >= 0:
self._print_performance('FP32', fp32_pps, fp32_lat)
def _summarize_accuracy(self, quant_acc, int8_acc, fp32_acc):
_logger.info('--- Accuracy summary ---')
self._print_accuracy('Quant', quant_acc)
self._print_accuracy('INT8', int8_acc)
if fp32_acc >= 0:
self._print_accuracy('FP32', fp32_acc)
def _compare_accuracy(self, threshold, quant_acc, int8_acc):
_logger.info(
f'Accepted accuracy drop threshold: {threshold}. (condition: (Quant_acc - INT8_acc) <= threshold)'
)
# Random outputs give accuracy about 0.33, we assume valid accuracy to be at least 0.5
assert quant_acc > 0.5
assert int8_acc > 0.5
assert quant_acc - int8_acc <= threshold
def _strings_from_csv(self, string):
return {s.strip() for s in string.split(',')}
def _ints_from_csv(self, string):
return set(map(int, string.split(',')))
def test_graph_transformation(self):
if not base.core.is_compiled_with_onednn():
return
quant_model_path = test_case_args.quant_model
assert quant_model_path, (
'The Quant model path cannot be empty. Please, use the --quant_model option.'
)
data_path = test_case_args.infer_data
assert data_path, (
'The dataset path cannot be empty. Please, use the --infer_data option.'
)
fp32_model_path = test_case_args.fp32_model
labels_path = test_case_args.labels
batch_size = test_case_args.batch_size
batch_num = test_case_args.batch_num
skip_batch_num = test_case_args.skip_batch_num
acc_diff_threshold = test_case_args.acc_diff_threshold
self._debug = test_case_args.debug
self._quantized_ops = set()
if test_case_args.ops_to_quantize:
self._quantized_ops = self._strings_from_csv(
test_case_args.ops_to_quantize
)
self._op_ids_to_skip = {-1}
if test_case_args.op_ids_to_skip:
self._op_ids_to_skip = self._ints_from_csv(
test_case_args.op_ids_to_skip
)
self._targets = self._strings_from_csv(test_case_args.targets)
assert self._targets.intersection({'quant', 'int8', 'fp32'}), (
'The --targets option, if used, must contain at least one of the targets: "quant", "int8", "fp32".'
)
_logger.info('Quant & INT8 prediction run.')
_logger.info(f'Quant model: {quant_model_path}')
if fp32_model_path:
_logger.info(f'FP32 model: {fp32_model_path}')
_logger.info(f'Dataset: {data_path}')
_logger.info(f'Labels: {labels_path}')
_logger.info(f'Batch size: {batch_size}')
_logger.info(f'Batch number: {batch_num}')
_logger.info(f'Accuracy drop threshold: {acc_diff_threshold}.')
_logger.info(
'Quantized ops: {}.'.format(
','.join(self._quantized_ops)
if self._quantized_ops
else 'all quantizable'
)
)
_logger.info(
'Op ids to skip quantization: {}.'.format(
','.join(map(str, self._op_ids_to_skip))
if test_case_args.op_ids_to_skip
else 'none'
)
)
_logger.info('Targets: {}.'.format(','.join(self._targets)))
if 'quant' in self._targets:
_logger.info('--- Quant prediction start ---')
val_reader = paddle.batch(
self._reader_creator(data_path, labels_path),
batch_size=batch_size,
)
quant_acc, quant_pps, quant_lat = self._predict(
val_reader,
quant_model_path,
batch_size,
batch_num,
skip_batch_num,
target='quant',
)
self._print_performance('Quant', quant_pps, quant_lat)
self._print_accuracy('Quant', quant_acc)
if 'int8' in self._targets:
_logger.info('--- INT8 prediction start ---')
val_reader = paddle.batch(
self._reader_creator(data_path, labels_path),
batch_size=batch_size,
)
int8_acc, int8_pps, int8_lat = self._predict(
val_reader,
quant_model_path,
batch_size,
batch_num,
skip_batch_num,
target='int8',
)
self._print_performance('INT8', int8_pps, int8_lat)
self._print_accuracy('INT8', int8_acc)
fp32_acc = fp32_pps = fp32_lat = -1
if 'fp32' in self._targets and fp32_model_path:
_logger.info('--- FP32 prediction start ---')
val_reader = paddle.batch(
self._reader_creator(data_path, labels_path),
batch_size=batch_size,
)
fp32_acc, fp32_pps, fp32_lat = self._predict(
val_reader,
fp32_model_path,
batch_size,
batch_num,
skip_batch_num,
target='fp32',
)
self._print_performance('FP32', fp32_pps, fp32_lat)
self._print_accuracy('FP32', fp32_acc)
if {'int8', 'quant', 'fp32'}.issubset(self._targets):
self._summarize_performance(
quant_pps, quant_lat, int8_pps, int8_lat, fp32_pps, fp32_lat
)
if {'int8', 'quant'}.issubset(self._targets):
self._summarize_accuracy(quant_acc, int8_acc, fp32_acc)
self._compare_accuracy(acc_diff_threshold, quant_acc, int8_acc)
if __name__ == '__main__':
global test_case_args
test_case_args, remaining_args = parse_args()
unittest.main(argv=remaining_args)
test/quantization/quant_int8_image_classification_comparison.py
+344
View File
@@ -0,0 +1,344 @@
# copyright (c) 2019 paddlepaddle authors. all rights reserved.
#
# licensed under the apache license, version 2.0 (the "license");
# you may not use this file except in compliance with the license.
# you may obtain a copy of the license at
#
# http://www.apache.org/licenses/license-2.0
#
# unless required by applicable law or agreed to in writing, software
# distributed under the license is distributed on an "as is" basis,
# without warranties or conditions of any kind, either express or implied.
# see the license for the specific language governing permissions and
# limitations under the license.
import argparse
import logging
import os
import struct
import sys
import time
import unittest
import numpy as np
import paddle
from paddle.base.framework import IrGraph
from paddle.framework import core
from paddle.static.quantization import QuantInt8OnednnPass
paddle.enable_static()
logging.basicConfig(format='%(asctime)s-%(levelname)s: %(message)s')
_logger = logging.getLogger(__name__)
_logger.setLevel(logging.INFO)
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('--batch_size', type=int, default=1, help='Batch size.')
parser.add_argument(
'--skip_batch_num',
type=int,
default=0,
help='Number of the first minibatches to skip in performance statistics.',
)
parser.add_argument(
'--debug',
action='store_true',
help='If used, the graph of Quant model is drawn.',
)
parser.add_argument(
'--quant_model', type=str, default='', help='A path to a Quant model.'
)
parser.add_argument('--infer_data', type=str, default='', help='Data file.')
parser.add_argument(
'--batch_num',
type=int,
default=0,
help='Number of batches to process. 0 or less means whole dataset. Default: 0.',
)
parser.add_argument(
'--acc_diff_threshold',
type=float,
default=0.01,
help='Accepted accuracy difference threshold.',
)
test_args, args = parser.parse_known_args(namespace=unittest)
return test_args, sys.argv[:1] + args
class QuantInt8ImageClassificationComparisonTest(unittest.TestCase):
"""
Test for accuracy comparison of Quant FP32 and INT8 Image Classification inference.
"""
def _reader_creator(self, data_file='data.bin'):
def reader():
with open(data_file, 'rb') as fp:
num = fp.read(8)
num = struct.unpack('q', num)[0]
imgs_offset = 8
img_ch = 3
img_w = 224
img_h = 224
img_pixel_size = 4
img_size = img_ch * img_h * img_w * img_pixel_size
label_size = 8
labels_offset = imgs_offset + num * img_size
step = 0
while step < num:
fp.seek(imgs_offset + img_size * step)
img = fp.read(img_size)
img = struct.unpack_from(f'{img_ch * img_w * img_h}f', img)
img = np.array(img)
img.shape = (img_ch, img_w, img_h)
fp.seek(labels_offset + label_size * step)
label = fp.read(label_size)
label = struct.unpack('q', label)[0]
yield img, int(label)
step += 1
return reader
def _get_batch_accuracy(self, batch_output=None, labels=None):
total = 0
correct = 0
correct_5 = 0
for n, result in enumerate(batch_output):
index = result.argsort()
top_1_index = index[-1]
top_5_index = index[-5:]
total += 1
if top_1_index == labels[n]:
correct += 1
if labels[n] in top_5_index:
correct_5 += 1
acc1 = float(correct) / float(total)
acc5 = float(correct_5) / float(total)
return acc1, acc5
def _prepare_for_fp32_onednn(self, graph):
ops = graph.all_op_nodes()
for op_node in ops:
name = op_node.name()
if name in ['depthwise_conv2d']:
input_var_node = graph._find_node_by_name(
op_node.inputs, op_node.input("Input")[0]
)
weight_var_node = graph._find_node_by_name(
op_node.inputs, op_node.input("Filter")[0]
)
output_var_node = graph._find_node_by_name(
graph.all_var_nodes(), op_node.output("Output")[0]
)
attrs = {
name: op_node.op().attr(name)
for name in op_node.op().attr_names()
}
conv_op_node = graph.create_op_node(
op_type='conv2d',
attrs=attrs,
inputs={'Input': input_var_node, 'Filter': weight_var_node},
outputs={'Output': output_var_node},
)
graph.link_to(input_var_node, conv_op_node)
graph.link_to(weight_var_node, conv_op_node)
graph.link_to(conv_op_node, output_var_node)
graph.safe_remove_nodes(op_node)
return graph
def _predict(
self,
test_reader=None,
model_path=None,
batch_size=1,
batch_num=1,
skip_batch_num=0,
transform_to_int8=False,
):
place = paddle.CPUPlace()
exe = paddle.static.Executor(place)
inference_scope = paddle.static.global_scope()
with paddle.static.scope_guard(inference_scope):
if os.path.exists(os.path.join(model_path, '__model__')):
[
inference_program,
feed_target_names,
fetch_targets,
] = paddle.static.io.load_inference_model(
model_path, exe, model_filename=None, params_filename=None
)
else:
[
inference_program,
feed_target_names,
fetch_targets,
] = paddle.static.load_inference_model(
model_path,
exe,
model_filename='model',
params_filename='params',
)
graph = IrGraph(core.Graph(inference_program.desc), for_test=True)
if self._debug:
graph.draw('.', 'quant_orig', graph.all_op_nodes())
if transform_to_int8:
onednn_int8_pass = QuantInt8OnednnPass(
_scope=inference_scope, _place=place
)
graph = onednn_int8_pass.apply(graph)
else:
graph = self._prepare_for_fp32_onednn(graph)
inference_program = graph.to_program()
dshape = [3, 224, 224]
outputs = []
infer_accs1 = []
infer_accs5 = []
fpses = []
batch_times = []
total_samples = 0
iters = 0
infer_start_time = time.time()
for data in test_reader():
if batch_num > 0 and iters >= batch_num:
break
if iters == skip_batch_num:
total_samples = 0
infer_start_time = time.time()
images = [x[0].reshape(dshape) for x in data]
images = np.array(images).astype('float32')
labels = np.array([x[1] for x in data]).astype('int64')
start = time.time()
out = exe.run(
inference_program,
feed={feed_target_names[0]: images},
fetch_list=fetch_targets,
)
batch_time = (time.time() - start) * 1000 # in milliseconds
outputs.append(out[0])
batch_acc1, batch_acc5 = self._get_batch_accuracy(
out[0], labels
)
infer_accs1.append(batch_acc1)
infer_accs5.append(batch_acc5)
samples = len(data)
total_samples += samples
batch_times.append(batch_time)
fps = samples / batch_time * 1000
fpses.append(fps)
iters += 1
appx = ' (warm-up)' if iters <= skip_batch_num else ''
_logger.info(
f'batch {iters}{appx}, acc1: {batch_acc1:.4f}, acc5: {batch_acc5:.4f}, '
f'latency: {batch_time / batch_size:.4f} ms, fps: {fps:.2f}'
)
# Postprocess benchmark data
batch_latencies = batch_times[skip_batch_num:]
batch_latency_avg = np.average(batch_latencies)
latency_avg = batch_latency_avg / batch_size
fpses = fpses[skip_batch_num:]
fps_avg = np.average(fpses)
infer_total_time = time.time() - infer_start_time
acc1_avg = np.mean(infer_accs1)
acc5_avg = np.mean(infer_accs5)
_logger.info(f'Total inference run time: {infer_total_time:.2f} s')
return outputs, acc1_avg, acc5_avg, fps_avg, latency_avg
def _summarize_performance(self, fp32_fps, fp32_lat, int8_fps, int8_lat):
_logger.info('--- Performance summary ---')
_logger.info(
f'FP32: avg fps: {fp32_fps:.2f}, avg latency: {fp32_lat:.4f} ms'
)
_logger.info(
f'INT8: avg fps: {int8_fps:.2f}, avg latency: {int8_lat:.4f} ms'
)
def _compare_accuracy(
self, fp32_acc1, fp32_acc5, int8_acc1, int8_acc5, threshold
):
_logger.info('--- Accuracy summary ---')
_logger.info(
f'Accepted top1 accuracy drop threshold: {threshold}. (condition: (FP32_top1_acc - IN8_top1_acc) <= threshold)'
)
_logger.info(
f'FP32: avg top1 accuracy: {fp32_acc1:.4f}, avg top5 accuracy: {fp32_acc5:.4f}'
)
_logger.info(
f'INT8: avg top1 accuracy: {int8_acc1:.4f}, avg top5 accuracy: {int8_acc5:.4f}'
)
assert fp32_acc1 > 0.0
assert int8_acc1 > 0.0
assert fp32_acc1 - int8_acc1 <= threshold
def test_graph_transformation(self):
if not core.is_compiled_with_onednn():
return
quant_model_path = test_case_args.quant_model
assert quant_model_path, (
'The Quant model path cannot be empty. Please, use the --quant_model option.'
)
data_path = test_case_args.infer_data
assert data_path, (
'The dataset path cannot be empty. Please, use the --infer_data option.'
)
batch_size = test_case_args.batch_size
batch_num = test_case_args.batch_num
skip_batch_num = test_case_args.skip_batch_num
acc_diff_threshold = test_case_args.acc_diff_threshold
self._debug = test_case_args.debug
_logger.info('Quant FP32 & INT8 prediction run.')
_logger.info(f'Quant model: {quant_model_path}')
_logger.info(f'Dataset: {data_path}')
_logger.info(f'Batch size: {batch_size}')
_logger.info(f'Batch number: {batch_num}')
_logger.info(f'Accuracy drop threshold: {acc_diff_threshold}.')
_logger.info('--- Quant FP32 prediction start ---')
val_reader = paddle.batch(
self._reader_creator(data_path), batch_size=batch_size
)
fp32_output, fp32_acc1, fp32_acc5, fp32_fps, fp32_lat = self._predict(
val_reader,
quant_model_path,
batch_size,
batch_num,
skip_batch_num,
transform_to_int8=False,
)
_logger.info('--- Quant INT8 prediction start ---')
val_reader = paddle.batch(
self._reader_creator(data_path), batch_size=batch_size
)
int8_output, int8_acc1, int8_acc5, int8_fps, int8_lat = self._predict(
val_reader,
quant_model_path,
batch_size,
batch_num,
skip_batch_num,
transform_to_int8=True,
)
self._summarize_performance(fp32_fps, fp32_lat, int8_fps, int8_lat)
self._compare_accuracy(
fp32_acc1, fp32_acc5, int8_acc1, int8_acc5, acc_diff_threshold
)
if __name__ == '__main__':
global test_case_args
test_case_args, remaining_args = parse_args()
unittest.main(argv=remaining_args)
test/quantization/test_apply_per_channel_scale.py
+127
View File
@@ -0,0 +1,127 @@
# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import struct
import unittest
import numpy as np
import paddle
import paddle.nn.quant as Q
from paddle.base import core
def convert_uint16_to_float(in_list):
in_list = np.asarray(in_list)
out = np.vectorize(
lambda x: struct.unpack(
'<f', struct.pack('<I', np.uint32(x) << np.uint32(16))
)[0],
otypes=[np.float32],
)(in_list.flat)
return np.reshape(out, in_list.shape)
class ApplyPerChannelScaleTest(unittest.TestCase):
def config(self):
self.rows = 32
self.cols = 128
self.rtol = 1e-5
self.atol = 1e-8
self.dtype = 'float16'
self.static = False
def setUp(self):
self.config()
paddle.set_default_dtype(self.dtype)
self.x = paddle.to_tensor(
np.random.random(size=(self.rows, self.cols)), self.dtype
)
self.scales = paddle.to_tensor(
np.random.uniform(0, 1, size=(self.cols)), self.dtype
)
self.out_expected = paddle.multiply(self.x, self.scales)
def get_out_static(self):
paddle.enable_static()
main = paddle.static.Program()
start = paddle.static.Program()
with paddle.static.program_guard(main, start):
x = paddle.static.data("x", self.x.shape, dtype=self.dtype)
scales = paddle.static.data(
"scales", self.scales.shape, dtype=self.dtype
)
x.stop_gradient = True
scales.stop_gradient = True
out = Q.apply_per_channel_scale(x, scales)
feed_dict = {
'x': self.x.numpy(),
'scales': self.scales.numpy(),
}
exe = paddle.static.Executor(paddle.CUDAPlace(0))
exe.run(start)
(out,) = exe.run(main, feed=feed_dict, fetch_list=[out])
paddle.disable_static()
return out
def test_apply_per_channel_scale(self):
if self.static:
self.out_real = self.get_out_static()
else:
paddle.disable_static()
self.out_real = Q.apply_per_channel_scale(
x=self.x,
scales=self.scales,
)
out_expected = self.out_expected
if self.dtype == 'bfloat16' and isinstance(
self.out_real, paddle.Tensor
):
self.out_real = convert_uint16_to_float(self.out_real)
out_expected = convert_uint16_to_float(self.out_expected)
np.testing.assert_allclose(
out_expected, self.out_real, rtol=self.rtol, atol=self.atol
)
@unittest.skipIf(
not core.is_compiled_with_cuda()
or paddle.device.cuda.get_device_capability()[0] < 8
or not core.is_bfloat16_supported(core.CUDAPlace(0)),
"quantized_matmul requires CUDA >= 11.2 and CUDA_ARCH >= 8 or core is not support bfloat16",
)
class ApplyPerChannelScaleTestCase1(ApplyPerChannelScaleTest):
def config(self):
super().config()
self.dtype = 'bfloat16'
class ApplyPerChannelScaleTestCase2(ApplyPerChannelScaleTest):
def config(self):
super().config()
self.rows = 1024
self.cols = 128
class ApplyPerChannelScaleStaticTest(ApplyPerChannelScaleTest):
def config(self):
super().config()
self.static = True
if __name__ == '__main__':
unittest.main()
test/quantization/test_customized_quanter.py
+72
View File
@@ -0,0 +1,72 @@
# copyright (c) 2022 paddlepaddle authors. all rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import unittest
from typing import TYPE_CHECKING
from paddle.nn import Linear
from paddle.quantization.base_quanter import BaseQuanter
from paddle.quantization.factory import quanter
if TYPE_CHECKING:
from collections.abc import Iterable
import numpy as np
import paddle
linear_quant_axis = 1
@quanter("CustomizedQuanter")
class CustomizedQuanterLayer(BaseQuanter):
def __init__(self, layer, bit_length=8, kwargs1=None):
super().__init__()
self._layer = layer
self._bit_length = bit_length
self._kwargs1 = kwargs1
def scales(self) -> paddle.Tensor | np.ndarray:
return None
def bit_length(self):
return self._bit_length
def quant_axis(self) -> int | Iterable:
return linear_quant_axis if isinstance(self._layer, Linear) else None
def zero_points(self) -> paddle.Tensor | np.ndarray:
return None
def forward(self, input):
return input
class TestCustomizedQuanter(unittest.TestCase):
def test_details(self):
layer = Linear(5, 5)
bit_length = 4
quanter = CustomizedQuanter( # noqa: F821
bit_length=bit_length, kwargs1="test"
)
quanter = quanter._instance(layer)
self.assertEqual(quanter.bit_length(), bit_length)
self.assertEqual(quanter.quant_axis(), linear_quant_axis)
self.assertEqual(quanter._kwargs1, 'test')
if __name__ == '__main__':
unittest.main()
test/quantization/test_imperative_out_scale.py
+198
View File
@@ -0,0 +1,198 @@
# copyright (c) 2018 paddlepaddle authors. all rights reserved.
#
# licensed under the apache license, version 2.0 (the "license");
# you may not use this file except in compliance with the license.
# you may obtain a copy of the license at
#
# http://www.apache.org/licenses/license-2.0
#
# unless required by applicable law or agreed to in writing, software
# distributed under the license is distributed on an "as is" basis,
# without warranties or conditions of any kind, either express or implied.
# see the license for the specific language governing permissions and
# limitations under the license.
import os
import sys
import tempfile
import unittest
import numpy as np
sys.path.append("../../quantization")
from imperative_test_utils import fix_model_dict, train_lenet
import paddle
from paddle import base
from paddle.framework import core, set_flags
from paddle.nn import (
BatchNorm2D,
Conv2D,
Linear,
MaxPool2D,
Sequential,
Softmax,
)
from paddle.nn.layer import LeakyReLU, PReLU, ReLU, Sigmoid
from paddle.quantization import ImperativeQuantAware
paddle.enable_static()
os.environ["CPU_NUM"] = "1"
if core.is_compiled_with_cuda():
set_flags({"FLAGS_cudnn_deterministic": True})
def get_valid_warning_num(warning, w):
num = 0
for i in range(len(w)):
if warning in str(w[i].message):
num += 1
return num
class ImperativeLenet(paddle.nn.Layer):
def __init__(self, num_classes=10):
super().__init__()
conv2d_w1_attr = paddle.ParamAttr(name="conv2d_w_1")
conv2d_w2_attr = paddle.ParamAttr(name="conv2d_w_2")
fc_w1_attr = paddle.ParamAttr(name="fc_w_1")
fc_w2_attr = paddle.ParamAttr(name="fc_w_2")
fc_w3_attr = paddle.ParamAttr(name="fc_w_3")
conv2d_b2_attr = paddle.ParamAttr(name="conv2d_b_2")
fc_b1_attr = paddle.ParamAttr(name="fc_b_1")
fc_b2_attr = paddle.ParamAttr(name="fc_b_2")
fc_b3_attr = paddle.ParamAttr(name="fc_b_3")
self.features = Sequential(
Conv2D(
in_channels=1,
out_channels=6,
kernel_size=3,
stride=1,
padding=1,
weight_attr=conv2d_w1_attr,
bias_attr=False,
),
BatchNorm2D(6),
ReLU(),
MaxPool2D(kernel_size=2, stride=2),
Conv2D(
in_channels=6,
out_channels=16,
kernel_size=5,
stride=1,
padding=0,
weight_attr=conv2d_w2_attr,
bias_attr=conv2d_b2_attr,
),
BatchNorm2D(16),
PReLU(),
MaxPool2D(kernel_size=2, stride=2),
)
self.fc = Sequential(
Linear(
in_features=400,
out_features=120,
weight_attr=fc_w1_attr,
bias_attr=fc_b1_attr,
),
LeakyReLU(),
Linear(
in_features=120,
out_features=84,
weight_attr=fc_w2_attr,
bias_attr=fc_b2_attr,
),
Sigmoid(),
Linear(
in_features=84,
out_features=num_classes,
weight_attr=fc_w3_attr,
bias_attr=fc_b3_attr,
),
Softmax(),
)
def forward(self, inputs):
x = self.features(inputs)
x = paddle.flatten(x, 1)
x = self.fc(x)
return x
class TestImperativeOutScale(unittest.TestCase):
def setUp(self):
self.root_path = tempfile.TemporaryDirectory()
self.param_save_path = os.path.join(
self.root_path.name, "lenet.pdparams"
)
self.save_path = os.path.join(
self.root_path.name, "lenet_dynamic_outscale_infer_model"
)
def tearDown(self):
self.root_path.cleanup()
def test_out_scale_acc(self):
seed = 1
lr = 0.001
weight_quantize_type = 'abs_max'
activation_quantize_type = 'moving_average_abs_max'
imperative_out_scale = ImperativeQuantAware(
weight_quantize_type=weight_quantize_type,
activation_quantize_type=activation_quantize_type,
)
with base.dygraph.guard():
np.random.seed(seed)
paddle.seed(seed)
lenet = ImperativeLenet()
lenet = fix_model_dict(lenet)
imperative_out_scale.quantize(lenet)
reader = paddle.batch(
paddle.dataset.mnist.test(), batch_size=32, drop_last=True
)
adam = paddle.optimizer.Adam(
learning_rate=lr, parameters=lenet.parameters()
)
loss_list = train_lenet(lenet, reader, adam)
lenet.eval()
save_dict = lenet.state_dict()
paddle.save(save_dict, self.param_save_path)
for i in range(len(loss_list) - 1):
self.assertTrue(
loss_list[i] > loss_list[i + 1],
msg='Failed to do the imperative qat.',
)
with base.dygraph.guard():
lenet = ImperativeLenet()
load_dict = paddle.load(self.param_save_path)
imperative_out_scale.quantize(lenet)
lenet.set_dict(load_dict)
reader = paddle.batch(
paddle.dataset.mnist.test(), batch_size=32, drop_last=True
)
adam = paddle.optimizer.Adam(
learning_rate=lr, parameters=lenet.parameters()
)
loss_list = train_lenet(lenet, reader, adam)
lenet.eval()
for i in range(len(loss_list) - 1):
self.assertTrue(
loss_list[i] > loss_list[i + 1],
msg='Failed to do the imperative qat.',
)
if __name__ == '__main__':
unittest.main()
test/quantization/test_imperative_ptq.py
+395
View File
@@ -0,0 +1,395 @@
# copyright (c) 2022 paddlepaddle authors. all rights reserved.
#
# licensed under the apache license, version 2.0 (the "license");
# you may not use this file except in compliance with the license.
# you may obtain a copy of the license at
#
# http://www.apache.org/licenses/license-2.0
#
# unless required by applicable law or agreed to in writing, software
# distributed under the license is distributed on an "as is" basis,
# without warranties or conditions of any kind, either express or implied.
# see the license for the specific language governing permissions and
# limitations under the license.
import logging
import os
import tempfile
import time
import unittest
import numpy as np
from imperative_test_utils import (
ImperativeLenet,
ImperativeLinearBn,
ImperativeLinearBn_hook,
)
import paddle
from paddle import nn
from paddle.dataset.common import download
from paddle.quantization import (
AbsmaxQuantizer,
HistQuantizer,
ImperativePTQ,
KLQuantizer,
PerChannelAbsmaxQuantizer,
PTQConfig,
)
from paddle.static.log_helper import get_logger
_logger = get_logger(
__name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s'
)
class TestFuseLinearBn(unittest.TestCase):
"""
Fuse the linear and bn layers, and then quantize the model.
"""
def test_fuse(self):
model = ImperativeLinearBn()
model_h = ImperativeLinearBn_hook()
inputs = paddle.randn((3, 10), dtype="float32")
config = PTQConfig(AbsmaxQuantizer(), AbsmaxQuantizer())
ptq = ImperativePTQ(config)
f_l = [['linear', 'bn']]
quant_model = ptq.quantize(model, fuse=True, fuse_list=f_l)
quant_h = ptq.quantize(model_h, fuse=True, fuse_list=f_l)
for name, layer in quant_model.named_sublayers():
if name in f_l:
assert not (isinstance(layer, (nn.BatchNorm1D, nn.BatchNorm2D)))
out = model(inputs)
out_h = model_h(inputs)
out_quant = quant_model(inputs)
out_quant_h = quant_h(inputs)
cos_sim_func = nn.CosineSimilarity(axis=0)
print(
'fuse linear+bn', cos_sim_func(out.flatten(), out_quant.flatten())
)
print(cos_sim_func(out_h.flatten(), out_quant_h.flatten()))
class TestImperativePTQ(unittest.TestCase):
""" """
@classmethod
def setUpClass(cls):
cls.download_path = 'dygraph_int8/download'
cls.cache_folder = os.path.expanduser(
'~/.cache/paddle/dataset/' + cls.download_path
)
cls.lenet_url = "https://paddle-inference-dist.cdn.bcebos.com/int8/unittest_model_data/lenet_pretrained.tar.gz"
cls.lenet_md5 = "953b802fb73b52fae42896e3c24f0afb"
seed = 1
np.random.seed(seed)
paddle.seed(seed)
def cache_unzipping(self, target_folder, zip_path):
if not os.path.exists(target_folder):
cmd = (
f'mkdir {target_folder} && tar xf {zip_path} -C {target_folder}'
)
os.system(cmd)
def download_model(self, data_url, data_md5, folder_name):
download(data_url, self.download_path, data_md5)
file_name = data_url.split('/')[-1]
zip_path = os.path.join(self.cache_folder, file_name)
print(f'Data is downloaded at {zip_path}')
data_cache_folder = os.path.join(self.cache_folder, folder_name)
self.cache_unzipping(data_cache_folder, zip_path)
return data_cache_folder
def set_vars(self):
config = PTQConfig(AbsmaxQuantizer(), AbsmaxQuantizer())
self.ptq = ImperativePTQ(config)
self.batch_num = 10
self.batch_size = 10
self.eval_acc_top1 = 0.95
# the input, output and weight thresholds of quantized op
self.gt_thresholds = {
'conv2d_0': [[1.0], [0.37673383951187134], [0.10933732241392136]],
'batch_norm2d_0': [[0.37673383951187134], [0.44249194860458374]],
're_lu_0': [[0.44249194860458374], [0.25804123282432556]],
'max_pool2d_0': [[0.25804123282432556], [0.25804123282432556]],
'linear_0': [
[1.7058950662612915],
[14.405526161193848],
[0.4373355209827423],
],
'add_0': [[1.7058950662612915, 0.0], [1.7058950662612915]],
}
def model_test(self, model, batch_num=-1, batch_size=8):
model.eval()
test_reader = paddle.batch(
paddle.dataset.mnist.test(), batch_size=batch_size
)
eval_acc_top1_list = []
for batch_id, data in enumerate(test_reader()):
x_data = np.array([x[0].reshape(1, 28, 28) for x in data]).astype(
'float32'
)
y_data = (
np.array([x[1] for x in data]).astype('int64').reshape(-1, 1)
)
img = paddle.to_tensor(x_data)
label = paddle.to_tensor(y_data)
out = model(img)
acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1)
acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5)
eval_acc_top1_list.append(float(acc_top1.numpy()))
if batch_id % 50 == 0:
_logger.info(
f"Test | At step {batch_id}: acc1 = {acc_top1.numpy()}, acc5 = {acc_top5.numpy()}"
)
if batch_num > 0 and batch_id + 1 >= batch_num:
break
eval_acc_top1 = sum(eval_acc_top1_list) / len(eval_acc_top1_list)
return eval_acc_top1
def program_test(self, program_path, batch_num=-1, batch_size=8):
exe = paddle.static.Executor(paddle.CPUPlace())
[
inference_program,
feed_target_names,
fetch_targets,
] = paddle.static.load_inference_model(program_path, exe)
test_reader = paddle.batch(
paddle.dataset.mnist.test(), batch_size=batch_size
)
top1_correct_num = 0.0
total_num = 0.0
for batch_id, data in enumerate(test_reader()):
img = np.array([x[0].reshape(1, 28, 28) for x in data]).astype(
'float32'
)
label = np.array([x[1] for x in data]).astype('int64')
feed = {feed_target_names[0]: img}
results = exe.run(
inference_program, feed=feed, fetch_list=fetch_targets
)
pred = np.argmax(results[0], axis=1)
top1_correct_num += np.sum(np.equal(pred, label))
total_num += len(img)
if total_num % 50 == 49:
_logger.info(
f"Test | Test num {total_num}: acc1 = {top1_correct_num / total_num}"
)
if batch_num > 0 and batch_id + 1 >= batch_num:
break
return top1_correct_num / total_num
def func_ptq(self):
start_time = time.time()
self.set_vars()
# Load model
params_path = self.download_model(
self.lenet_url, self.lenet_md5, "lenet"
)
params_path += "/lenet_pretrained/lenet.pdparams"
model = ImperativeLenet()
model_state_dict = paddle.load(params_path)
model.set_state_dict(model_state_dict)
# Quantize, calibrate and save
quant_model = self.ptq.quantize(model)
before_acc_top1 = self.model_test(
quant_model, self.batch_num, self.batch_size
)
input_spec = [
paddle.static.InputSpec(shape=[None, 1, 28, 28], dtype='float32')
]
with tempfile.TemporaryDirectory(prefix="imperative_ptq_") as tmpdir:
save_path = os.path.join(tmpdir, "model")
self.ptq.save_quantized_model(
model=quant_model, path=save_path, input_spec=input_spec
)
print(f'Quantized model saved in {{{save_path}}}')
after_acc_top1 = self.model_test(
quant_model, self.batch_num, self.batch_size
)
paddle.enable_static()
infer_acc_top1 = self.program_test(
save_path, self.batch_num, self.batch_size
)
paddle.disable_static()
# Check
print(f'Before converted acc_top1: {before_acc_top1}')
print(f'After converted acc_top1: {after_acc_top1}')
print(f'Infer acc_top1: {infer_acc_top1}')
self.assertTrue(
after_acc_top1 >= self.eval_acc_top1,
msg=f"The test acc {{{after_acc_top1:f}}} is less than {{{self.eval_acc_top1:f}}}.",
)
self.assertTrue(
infer_acc_top1 >= after_acc_top1,
msg='The acc is lower after converting model.',
)
end_time = time.time()
print("total time: %ss \n" % (end_time - start_time))
def test_ptq(self):
self.func_ptq()
class TestImperativePTQfuse(TestImperativePTQ):
def func_ptq(self):
start_time = time.time()
self.set_vars()
# Load model
params_path = self.download_model(
self.lenet_url, self.lenet_md5, "lenet"
)
params_path += "/lenet_pretrained/lenet.pdparams"
model = ImperativeLenet()
model_state_dict = paddle.load(params_path)
model.set_state_dict(model_state_dict)
# Quantize, calibrate and save
f_l = [['features.0', 'features.1'], ['features.4', 'features.5']]
quant_model = self.ptq.quantize(model, fuse=True, fuse_list=f_l)
for name, layer in quant_model.named_sublayers():
if name in f_l:
assert not (isinstance(layer, (nn.BatchNorm1D, nn.BatchNorm2D)))
before_acc_top1 = self.model_test(
quant_model, self.batch_num, self.batch_size
)
input_spec = [
paddle.static.InputSpec(shape=[None, 1, 28, 28], dtype='float32')
]
with tempfile.TemporaryDirectory(prefix="imperative_ptq_") as tmpdir:
save_path = os.path.join(tmpdir, "model")
self.ptq.save_quantized_model(
model=quant_model, path=save_path, input_spec=input_spec
)
print(f'Quantized model saved in {{{save_path}}}')
after_acc_top1 = self.model_test(
quant_model, self.batch_num, self.batch_size
)
paddle.enable_static()
infer_acc_top1 = self.program_test(
save_path, self.batch_num, self.batch_size
)
paddle.disable_static()
# Check
print(f'Before converted acc_top1: {before_acc_top1}')
print(f'After converted acc_top1: {after_acc_top1}')
print(f'Infer acc_top1: {infer_acc_top1}')
# Check whether the quant_model is correct after converting.
# The acc of quantized model should be higher than 0.95.
self.assertTrue(
after_acc_top1 >= self.eval_acc_top1,
msg=f"The test acc {{{after_acc_top1:f}}} is less than {{{self.eval_acc_top1:f}}}.",
)
# Check the saved infer_model.The acc of infer model
# should not be lower than the one of dygraph model.
self.assertTrue(
infer_acc_top1 >= after_acc_top1,
msg='The acc is lower after converting model.',
)
end_time = time.time()
print("total time: %ss \n" % (end_time - start_time))
def test_ptq(self):
self.func_ptq()
class TestImperativePTQHist(TestImperativePTQ):
def set_vars(self):
config = PTQConfig(HistQuantizer(), AbsmaxQuantizer())
self.ptq = ImperativePTQ(config)
self.batch_num = 10
self.batch_size = 10
self.eval_acc_top1 = 0.98
self.gt_thresholds = {
'conv2d_0': [
[0.99853515625],
[0.35732391771364225],
[0.10933732241392136],
],
'batch_norm2d_0': [[0.35732391771364225], [0.4291427868761275]],
're_lu_0': [[0.4291427868761275], [0.2359918110742001]],
'max_pool2d_0': [[0.2359918110742001], [0.25665526917146053]],
'linear_0': [
[1.7037603475152991],
[14.395224522473026],
[0.4373355209827423],
],
'add_0': [[1.7037603475152991, 0.0], [1.7037603475152991]],
}
class TestImperativePTQKL(TestImperativePTQ):
def set_vars(self):
config = PTQConfig(KLQuantizer(), PerChannelAbsmaxQuantizer())
self.ptq = ImperativePTQ(config)
self.batch_num = 10
self.batch_size = 10
self.eval_acc_top1 = 0.98
conv2d_1_wt_thresholds = [
0.18116560578346252,
0.17079241573810577,
0.1702047884464264,
0.179476797580719,
0.1454375684261322,
0.22981858253479004,
]
self.gt_thresholds = {
'conv2d_0': [[0.99267578125], [0.37695913558696836]],
'conv2d_1': [
[0.19189296757394914],
[0.24514256547263358],
[conv2d_1_wt_thresholds],
],
'batch_norm2d_0': [[0.37695913558696836], [0.27462541429440535]],
're_lu_0': [[0.27462541429440535], [0.19189296757394914]],
'max_pool2d_0': [[0.19189296757394914], [0.19189296757394914]],
'linear_0': [[1.2839322163611087], [8.957185942414352]],
'add_0': [[1.2839322163611087, 0.0], [1.2839322163611087]],
}
if __name__ == '__main__':
unittest.main()
test/quantization/test_imperative_qat.py
+209
View File
@@ -0,0 +1,209 @@
# copyright (c) 2018 paddlepaddle authors. all rights reserved.
#
# licensed under the apache license, version 2.0 (the "license");
# you may not use this file except in compliance with the license.
# you may obtain a copy of the license at
#
# http://www.apache.org/licenses/license-2.0
#
# unless required by applicable law or agreed to in writing, software
# distributed under the license is distributed on an "as is" basis,
# without warranties or conditions of any kind, either express or implied.
# see the license for the specific language governing permissions and
# limitations under the license.
import logging
import os
import sys
import unittest
import numpy as np
sys.path.append("../../quantization")
from imperative_test_utils import ImperativeLenet, fix_model_dict
import paddle
from paddle import base
from paddle.framework import core, set_flags
from paddle.nn import Conv2D, Conv2DTranspose
from paddle.nn.quant.quant_layers import (
QuantizedConv2D,
QuantizedConv2DTranspose,
)
from paddle.optimizer import Adam
from paddle.quantization import ImperativeQuantAware
from paddle.static.log_helper import get_logger
INFER_MODEL_SUFFIX = ".pdmodel"
INFER_PARAMS_SUFFIX = ".pdiparams"
paddle.enable_static()
os.environ["CPU_NUM"] = "1"
if core.is_compiled_with_cuda():
set_flags({"FLAGS_cudnn_deterministic": True})
_logger = get_logger(
__name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s'
)
class TestImperativeQat(unittest.TestCase):
"""
QAT = quantization-aware training
"""
def set_vars(self):
self.weight_quantize_type = 'abs_max'
self.activation_quantize_type = 'moving_average_abs_max'
self.onnx_format = False
self.check_export_model_accuracy = True
# The original model and quantized model may have different prediction.
# There are 32 test data and we allow at most one is different.
# Hence, the diff_threshold is 1 / 32 = 0.03125
self.diff_threshold = 0.03125
self.fuse_conv_bn = False
def test_qat(self):
self.set_vars()
imperative_qat = ImperativeQuantAware(
weight_quantize_type=self.weight_quantize_type,
activation_quantize_type=self.activation_quantize_type,
fuse_conv_bn=self.fuse_conv_bn,
onnx_format=self.onnx_format,
)
with base.dygraph.guard():
# For CI coverage
conv1 = Conv2D(
in_channels=3,
out_channels=2,
kernel_size=3,
stride=1,
padding=1,
padding_mode='replicate',
)
quant_conv1 = QuantizedConv2D(conv1)
data = np.random.uniform(-1, 1, [10, 3, 32, 32]).astype('float32')
quant_conv1(paddle.to_tensor(data))
conv_transpose = Conv2DTranspose(4, 6, (3, 3))
quant_conv_transpose = QuantizedConv2DTranspose(conv_transpose)
x_var = paddle.uniform(
(2, 4, 8, 8), dtype='float32', min=-1.0, max=1.0
)
quant_conv_transpose(x_var)
seed = 1
np.random.seed(seed)
paddle.seed(seed)
lenet = ImperativeLenet()
lenet = fix_model_dict(lenet)
imperative_qat.quantize(lenet)
adam = Adam(learning_rate=0.001, parameters=lenet.parameters())
train_reader = paddle.batch(
paddle.dataset.mnist.train(), batch_size=32, drop_last=True
)
test_reader = paddle.batch(
paddle.dataset.mnist.test(), batch_size=32
)
epoch_num = 1
for epoch in range(epoch_num):
lenet.train()
for batch_id, data in enumerate(train_reader()):
x_data = np.array(
[x[0].reshape(1, 28, 28) for x in data]
).astype('float32')
y_data = (
np.array([x[1] for x in data])
.astype('int64')
.reshape(-1, 1)
)
img = paddle.to_tensor(x_data)
label = paddle.to_tensor(y_data)
out = lenet(img)
acc = paddle.metric.accuracy(out, label)
loss = paddle.nn.functional.cross_entropy(
out, label, reduction='none', use_softmax=False
)
avg_loss = paddle.mean(loss)
avg_loss.backward()
adam.minimize(avg_loss)
lenet.clear_gradients()
if batch_id % 100 == 0:
_logger.info(
f"Train | At epoch {epoch} step {batch_id}: loss = {avg_loss.numpy()}, acc= {acc.numpy()}"
)
if batch_id == 500: # For shortening CI time
break
lenet.eval()
eval_acc_top1_list = []
for batch_id, data in enumerate(test_reader()):
x_data = np.array(
[x[0].reshape(1, 28, 28) for x in data]
).astype('float32')
y_data = (
np.array([x[1] for x in data])
.astype('int64')
.reshape(-1, 1)
)
img = paddle.to_tensor(x_data)
label = paddle.to_tensor(y_data)
out = lenet(img)
acc_top1 = paddle.metric.accuracy(
input=out, label=label, k=1
)
acc_top5 = paddle.metric.accuracy(
input=out, label=label, k=5
)
if batch_id % 100 == 0:
eval_acc_top1_list.append(float(acc_top1.numpy()))
_logger.info(
f"Test | At epoch {epoch} step {batch_id}: acc1 = {acc_top1.numpy()}, acc5 = {acc_top5.numpy()}"
)
# check eval acc
eval_acc_top1 = sum(eval_acc_top1_list) / len(
eval_acc_top1_list
)
print('eval_acc_top1', eval_acc_top1)
self.assertTrue(
eval_acc_top1 > 0.9,
msg=f"The test acc {{{eval_acc_top1:f}}} is less than 0.9.",
)
# test the correctness of `paddle.jit.save`
data = next(test_reader())
test_data = np.array(
[x[0].reshape(1, 28, 28) for x in data]
).astype('float32')
y_data = (
np.array([x[1] for x in data]).astype('int64').reshape(-1, 1)
)
test_img = paddle.to_tensor(test_data)
label = paddle.to_tensor(y_data)
lenet.eval()
fp32_out = lenet(test_img)
fp32_acc = paddle.metric.accuracy(fp32_out, label).numpy()
class TestImperativeQatONNXFormat(unittest.TestCase):
def set_vars(self):
self.weight_quantize_type = 'abs_max'
self.activation_quantize_type = 'moving_average_abs_max'
self.onnx_format = True
self.diff_threshold = 0.03125
self.fuse_conv_bn = False
if __name__ == '__main__':
unittest.main()
test/quantization/test_imperative_qat_amp.py
+239
View File
@@ -0,0 +1,239 @@
# copyright (c) 2022 paddlepaddle authors. all rights reserved.
#
# licensed under the apache license, version 2.0 (the "license");
# you may not use this file except in compliance with the license.
# you may obtain a copy of the license at
#
# http://www.apache.org/licenses/license-2.0
#
# unless required by applicable law or agreed to in writing, software
# distributed under the license is distributed on an "as is" basis,
# without warranties or conditions of any kind, either express or implied.
# see the license for the specific language governing permissions and
# limitations under the license.
import logging
import os
import tempfile
import time
import unittest
import numpy as np
from imperative_test_utils import ImperativeLenet
import paddle
from paddle import base
from paddle.dataset.common import download
from paddle.framework import set_flags
from paddle.quantization import ImperativeQuantAware
from paddle.static.log_helper import get_logger
os.environ["CPU_NUM"] = "1"
if paddle.is_compiled_with_cuda():
set_flags({"FLAGS_cudnn_deterministic": True})
_logger = get_logger(
__name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s'
)
class TestImperativeQatAmp(unittest.TestCase):
"""
Test the combination of qat and amp.
"""
@classmethod
def setUpClass(cls):
cls.root_path = tempfile.TemporaryDirectory(
prefix="imperative_qat_amp_"
)
cls.save_path = os.path.join(cls.root_path.name, "model")
cls.download_path = 'dygraph_int8/download'
cls.cache_folder = os.path.expanduser(
'~/.cache/paddle/dataset/' + cls.download_path
)
cls.lenet_url = "https://paddle-inference-dist.cdn.bcebos.com/int8/unittest_model_data/lenet_pretrained.tar.gz"
cls.lenet_md5 = "953b802fb73b52fae42896e3c24f0afb"
seed = 1
np.random.seed(seed)
paddle.seed(seed)
@classmethod
def tearDownClass(cls):
cls.root_path.cleanup()
def cache_unzipping(self, target_folder, zip_path):
if not os.path.exists(target_folder):
cmd = (
f'mkdir {target_folder} && tar xf {zip_path} -C {target_folder}'
)
os.system(cmd)
def download_model(self, data_url, data_md5, folder_name):
download(data_url, self.download_path, data_md5)
file_name = data_url.split('/')[-1]
zip_path = os.path.join(self.cache_folder, file_name)
print(f'Data is downloaded at {zip_path}')
data_cache_folder = os.path.join(self.cache_folder, folder_name)
self.cache_unzipping(data_cache_folder, zip_path)
return data_cache_folder
def set_vars(self):
self.qat = ImperativeQuantAware()
self.train_batch_num = 30
self.train_batch_size = 32
self.test_batch_num = 100
self.test_batch_size = 32
self.eval_acc_top1 = 0.99
def model_train(self, model, batch_num=-1, batch_size=32, use_amp=False):
model.train()
train_reader = paddle.batch(
paddle.dataset.mnist.train(), batch_size=batch_size
)
adam = paddle.optimizer.Adam(
learning_rate=0.001, parameters=model.parameters()
)
scaler = paddle.amp.GradScaler(init_loss_scaling=500)
for batch_id, data in enumerate(train_reader()):
x_data = np.array([x[0].reshape(1, 28, 28) for x in data]).astype(
'float32'
)
y_data = (
np.array([x[1] for x in data]).astype('int64').reshape(-1, 1)
)
img = paddle.to_tensor(x_data)
label = paddle.to_tensor(y_data)
if use_amp:
with paddle.amp.auto_cast():
out = model(img)
acc = paddle.metric.accuracy(out, label)
loss = paddle.nn.functional.cross_entropy(
out, label, reduction='none', use_softmax=False
)
avg_loss = paddle.mean(loss)
scaled_loss = scaler.scale(avg_loss)
scaled_loss.backward()
scaler.minimize(adam, scaled_loss)
adam.clear_gradients()
else:
out = model(img)
acc = paddle.metric.accuracy(out, label)
loss = paddle.nn.functional.cross_entropy(
out, label, reduction='none', use_softmax=False
)
avg_loss = paddle.mean(loss)
avg_loss.backward()
adam.minimize(avg_loss)
model.clear_gradients()
if batch_id % 100 == 0:
_logger.info(
f"Train | step {batch_id}: loss = {avg_loss.numpy()}, acc= {acc.numpy()}"
)
if batch_num > 0 and batch_id + 1 >= batch_num:
break
def model_test(self, model, batch_num=-1, batch_size=32, use_amp=False):
model.eval()
test_reader = paddle.batch(
paddle.dataset.mnist.test(), batch_size=batch_size
)
acc_top1_list = []
for batch_id, data in enumerate(test_reader()):
x_data = np.array([x[0].reshape(1, 28, 28) for x in data]).astype(
'float32'
)
y_data = (
np.array([x[1] for x in data]).astype('int64').reshape(-1, 1)
)
img = paddle.to_tensor(x_data)
label = paddle.to_tensor(y_data)
with paddle.amp.auto_cast(use_amp):
out = model(img)
acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1)
acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5)
acc_top1_list.append(float(acc_top1.numpy()))
if batch_id % 100 == 0:
_logger.info(
f"Test | At step {batch_id}: acc1 = {acc_top1.numpy()}, acc5 = {acc_top5.numpy()}"
)
if batch_num > 0 and batch_id + 1 >= batch_num:
break
acc_top1 = sum(acc_top1_list) / len(acc_top1_list)
return acc_top1
def test_ptq(self):
start_time = time.time()
self.set_vars()
params_path = self.download_model(
self.lenet_url, self.lenet_md5, "lenet"
)
params_path += "/lenet_pretrained/lenet.pdparams"
with base.dygraph.guard():
model = ImperativeLenet()
model_state_dict = paddle.load(params_path)
model.set_state_dict(model_state_dict)
_logger.info("Test fp32 model")
fp32_acc_top1 = self.model_test(
model, self.test_batch_num, self.test_batch_size
)
self.qat.quantize(model)
use_amp = True
self.model_train(
model, self.train_batch_num, self.train_batch_size, use_amp
)
_logger.info("Test int8 model")
int8_acc_top1 = self.model_test(
model, self.test_batch_num, self.test_batch_size, use_amp
)
_logger.info(
f'fp32_acc_top1: {fp32_acc_top1:f}, int8_acc_top1: {int8_acc_top1:f}'
)
self.assertTrue(
int8_acc_top1 > fp32_acc_top1 - 0.01,
msg=f'fp32_acc_top1: {fp32_acc_top1:f}, int8_acc_top1: {int8_acc_top1:f}',
)
input_spec = [
paddle.static.InputSpec(shape=[None, 1, 28, 28], dtype='float32')
]
with paddle.pir_utils.OldIrGuard():
paddle.jit.save(
layer=model, path=self.save_path, input_spec=input_spec
)
print(f'Quantized model saved in {{{self.save_path}}}')
end_time = time.time()
print("total time: %ss" % (end_time - start_time))
if __name__ == '__main__':
unittest.main()
test/quantization/test_imperative_qat_channelwise.py
+51
View File
@@ -0,0 +1,51 @@
# copyright (c) 2022 paddlepaddle authors. all rights reserved.
#
# licensed under the apache license, version 2.0 (the "license");
# you may not use this file except in compliance with the license.
# you may obtain a copy of the license at
#
# http://www.apache.org/licenses/license-2.0
#
# unless required by applicable law or agreed to in writing, software
# distributed under the license is distributed on an "as is" basis,
# without warranties or conditions of any kind, either express or implied.
# see the license for the specific language governing permissions and
# limitations under the license.
import os
import unittest
from test_imperative_qat import TestImperativeQat
import paddle
from paddle.framework import core, set_flags
paddle.enable_static()
os.environ["CPU_NUM"] = "1"
if core.is_compiled_with_cuda():
set_flags({"FLAGS_cudnn_deterministic": True})
class TestImperativeQatChannelWise(TestImperativeQat):
def set_vars(self):
self.weight_quantize_type = 'channel_wise_abs_max'
self.activation_quantize_type = 'moving_average_abs_max'
self.diff_threshold = 0.03125
self.onnx_format = False
self.fuse_conv_bn = False
print('weight_quantize_type', self.weight_quantize_type)
class TestImperativeQatChannelWiseONNXFormat(TestImperativeQat):
def set_vars(self):
self.weight_quantize_type = 'channel_wise_abs_max'
self.activation_quantize_type = 'moving_average_abs_max'
self.onnx_format = True
self.diff_threshold = 0.03125
self.fuse_conv_bn = False
print('weight_quantize_type', self.weight_quantize_type)
if __name__ == '__main__':
unittest.main()
test/quantization/test_imperative_qat_fuse.py
+40
View File
@@ -0,0 +1,40 @@
# copyright (c) 2018 paddlepaddle authors. all rights reserved.
#
# licensed under the apache license, version 2.0 (the "license");
# you may not use this file except in compliance with the license.
# you may obtain a copy of the license at
#
# http://www.apache.org/licenses/license-2.0
#
# unless required by applicable law or agreed to in writing, software
# distributed under the license is distributed on an "as is" basis,
# without warranties or conditions of any kind, either express or implied.
# see the license for the specific language governing permissions and
# limitations under the license.
import os
import unittest
from test_imperative_qat import TestImperativeQat
import paddle
from paddle.framework import core, set_flags
paddle.enable_static()
os.environ["CPU_NUM"] = "1"
if core.is_compiled_with_cuda():
set_flags({"FLAGS_cudnn_deterministic": True})
class TestImperativeQatfuseBN(TestImperativeQat):
def set_vars(self):
self.weight_quantize_type = 'abs_max'
self.activation_quantize_type = 'moving_average_abs_max'
self.diff_threshold = 0.03125
self.onnx_format = False
self.fuse_conv_bn = True
if __name__ == '__main__':
unittest.main()
test/quantization/test_imperative_qat_lsq.py
+224
View File
@@ -0,0 +1,224 @@
# copyright (c) 2022 paddlepaddle authors. all rights reserved.
#
# licensed under the apache license, version 2.0 (the "license");
# you may not use this file except in compliance with the license.
# you may obtain a copy of the license at
#
# http://www.apache.org/licenses/license-2.0
#
# unless required by applicable law or agreed to in writing, software
# distributed under the license is distributed on an "as is" basis,
# without warranties or conditions of any kind, either express or implied.
# see the license for the specific language governing permissions and
# limitations under the license.
import logging
import os
import unittest
import numpy as np
from imperative_test_utils import fix_model_dict
import paddle
from paddle.framework import core, set_flags
from paddle.nn import (
BatchNorm2D,
Conv2D,
LeakyReLU,
Linear,
MaxPool2D,
PReLU,
ReLU,
Sequential,
Sigmoid,
Softmax,
)
from paddle.quantization import ImperativeQuantAware
from paddle.static.log_helper import get_logger
paddle.enable_static()
os.environ["CPU_NUM"] = "1"
if core.is_compiled_with_cuda():
set_flags({"FLAGS_cudnn_deterministic": True})
_logger = get_logger(
__name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s'
)
class ImperativeLenet(paddle.nn.Layer):
def __init__(self, num_classes=10):
super().__init__()
conv2d_w1_attr = paddle.ParamAttr(name="conv2d_w_1")
conv2d_w2_attr = paddle.ParamAttr(name="conv2d_w_2")
fc_w1_attr = paddle.ParamAttr(name="fc_w_1")
fc_w2_attr = paddle.ParamAttr(name="fc_w_2")
fc_w3_attr = paddle.ParamAttr(name="fc_w_3")
conv2d_b2_attr = paddle.ParamAttr(name="conv2d_b_2")
fc_b1_attr = paddle.ParamAttr(name="fc_b_1")
fc_b2_attr = paddle.ParamAttr(name="fc_b_2")
fc_b3_attr = paddle.ParamAttr(name="fc_b_3")
self.features = Sequential(
Conv2D(
in_channels=1,
out_channels=6,
kernel_size=3,
stride=1,
padding=1,
weight_attr=conv2d_w1_attr,
bias_attr=False,
),
BatchNorm2D(6),
ReLU(),
MaxPool2D(kernel_size=2, stride=2),
Conv2D(
in_channels=6,
out_channels=16,
kernel_size=5,
stride=1,
padding=0,
weight_attr=conv2d_w2_attr,
bias_attr=conv2d_b2_attr,
),
BatchNorm2D(16),
PReLU(),
MaxPool2D(kernel_size=2, stride=2),
)
self.fc = Sequential(
Linear(
in_features=400,
out_features=120,
weight_attr=fc_w1_attr,
bias_attr=fc_b1_attr,
),
LeakyReLU(),
Linear(
in_features=120,
out_features=84,
weight_attr=fc_w2_attr,
bias_attr=fc_b2_attr,
),
Sigmoid(),
Linear(
in_features=84,
out_features=num_classes,
weight_attr=fc_w3_attr,
bias_attr=fc_b3_attr,
),
Softmax(),
)
def forward(self, inputs):
x = self.features(inputs)
x = paddle.flatten(x, 1)
x = self.fc(x)
return x
class TestImperativeQatLSQ(unittest.TestCase):
def set_vars(self):
self.weight_quantize_type = 'channel_wise_lsq_weight'
self.activation_quantize_type = 'lsq_act'
self.onnx_format = False
self.fuse_conv_bn = False
def func_qat(self):
self.set_vars()
imperative_qat = ImperativeQuantAware(
weight_quantize_type=self.weight_quantize_type,
activation_quantize_type=self.activation_quantize_type,
fuse_conv_bn=self.fuse_conv_bn,
)
seed = 100
np.random.seed(seed)
paddle.seed(seed)
paddle.disable_static()
lenet = ImperativeLenet()
lenet = fix_model_dict(lenet)
imperative_qat.quantize(lenet)
optimizer = paddle.optimizer.Momentum(
learning_rate=0.1, parameters=lenet.parameters(), momentum=0.9
)
train_reader = paddle.batch(
paddle.dataset.mnist.train(), batch_size=64, drop_last=True
)
test_reader = paddle.batch(paddle.dataset.mnist.test(), batch_size=32)
epoch_num = 2
for epoch in range(epoch_num):
lenet.train()
for batch_id, data in enumerate(train_reader()):
x_data = np.array(
[x[0].reshape(1, 28, 28) for x in data]
).astype('float32')
y_data = (
np.array([x[1] for x in data])
.astype('int64')
.reshape(-1, 1)
)
img = paddle.to_tensor(x_data)
label = paddle.to_tensor(y_data)
out = lenet(img)
acc = paddle.metric.accuracy(out, label)
loss = paddle.nn.functional.cross_entropy(
out, label, reduction='none', use_softmax=False
)
avg_loss = paddle.mean(loss)
avg_loss.backward()
optimizer.minimize(avg_loss)
lenet.clear_gradients()
if batch_id % 100 == 0:
_logger.info(
f"Train | At epoch {epoch} step {batch_id}: loss = {avg_loss.numpy()}, acc= {acc.numpy()}"
)
lenet.eval()
eval_acc_top1_list = []
with paddle.no_grad():
for batch_id, data in enumerate(test_reader()):
x_data = np.array(
[x[0].reshape(1, 28, 28) for x in data]
).astype('float32')
y_data = (
np.array([x[1] for x in data])
.astype('int64')
.reshape(-1, 1)
)
img = paddle.to_tensor(x_data)
label = paddle.to_tensor(y_data)
out = lenet(img)
acc_top1 = paddle.metric.accuracy(
input=out, label=label, k=1
)
acc_top5 = paddle.metric.accuracy(
input=out, label=label, k=5
)
if batch_id % 100 == 0:
eval_acc_top1_list.append(float(acc_top1.numpy()))
_logger.info(
f"Test | At epoch {epoch} step {batch_id}: acc1 = {acc_top1.numpy()}, acc5 = {acc_top5.numpy()}"
)
# check eval acc
eval_acc_top1 = sum(eval_acc_top1_list) / len(eval_acc_top1_list)
print('eval_acc_top1', eval_acc_top1)
self.assertTrue(
eval_acc_top1 > 0.9,
msg=f"The test acc {{{eval_acc_top1:f}}} is less than 0.9.",
)
def test_qat(self):
self.func_qat()
if __name__ == '__main__':
unittest.main()
test/quantization/test_imperative_qat_matmul.py
+224
View File
@@ -0,0 +1,224 @@
# copyright (c) 2022 paddlepaddle authors. all rights reserved.
#
# licensed under the apache license, version 2.0 (the "license");
# you may not use this file except in compliance with the license.
# you may obtain a copy of the license at
#
# http://www.apache.org/licenses/license-2.0
#
# unless required by applicable law or agreed to in writing, software
# distributed under the license is distributed on an "as is" basis,
# without warranties or conditions of any kind, either express or implied.
# see the license for the specific language governing permissions and
# limitations under the license.
import logging
import os
import unittest
import numpy as np
from imperative_test_utils import fix_model_dict
import paddle
from paddle.framework import core, set_flags
from paddle.nn import (
BatchNorm2D,
Conv2D,
LeakyReLU,
Linear,
MaxPool2D,
PReLU,
ReLU,
Sequential,
Sigmoid,
Softmax,
)
from paddle.nn.quant.quant_layers import QuantizedMatmul
from paddle.optimizer import Momentum
from paddle.quantization import ImperativeQuantAware
from paddle.static.log_helper import get_logger
paddle.enable_static()
os.environ["CPU_NUM"] = "1"
if core.is_compiled_with_cuda():
set_flags({"FLAGS_cudnn_deterministic": True})
_logger = get_logger(
__name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s'
)
class ImperativeLenet(paddle.nn.Layer):
def __init__(self, num_classes=10):
super().__init__()
conv2d_w1_attr = paddle.ParamAttr(name="conv2d_w_1")
conv2d_w2_attr = paddle.ParamAttr(name="conv2d_w_2")
fc_w1_attr = paddle.ParamAttr(name="fc_w_1")
fc_w2_attr = paddle.ParamAttr(name="fc_w_2")
fc_w3_attr = paddle.ParamAttr(name="fc_w_3")
conv2d_b2_attr = paddle.ParamAttr(name="conv2d_b_2")
fc_b1_attr = paddle.ParamAttr(name="fc_b_1")
fc_b2_attr = paddle.ParamAttr(name="fc_b_2")
fc_b3_attr = paddle.ParamAttr(name="fc_b_3")
self.features = Sequential(
Conv2D(
in_channels=1,
out_channels=6,
kernel_size=3,
stride=1,
padding=1,
weight_attr=conv2d_w1_attr,
bias_attr=False,
),
BatchNorm2D(6),
ReLU(),
MaxPool2D(kernel_size=2, stride=2),
Conv2D(
in_channels=6,
out_channels=16,
kernel_size=5,
stride=1,
padding=0,
weight_attr=conv2d_w2_attr,
bias_attr=conv2d_b2_attr,
),
BatchNorm2D(16),
PReLU(),
MaxPool2D(kernel_size=2, stride=2),
)
self.matmul = QuantizedMatmul()
self.fc = Sequential(
Linear(
in_features=400,
out_features=120,
weight_attr=fc_w1_attr,
bias_attr=fc_b1_attr,
),
LeakyReLU(),
Linear(
in_features=120,
out_features=84,
weight_attr=fc_w2_attr,
bias_attr=fc_b2_attr,
),
Sigmoid(),
Linear(
in_features=84,
out_features=num_classes,
weight_attr=fc_w3_attr,
bias_attr=fc_b3_attr,
),
Softmax(),
)
def forward(self, inputs):
inputs = self.features(inputs)
inputs = self.matmul(inputs, inputs, transpose_y=True)
inputs = paddle.flatten(inputs, 1)
x = self.fc(inputs)
return x
class TestImperativeQatMatmul(unittest.TestCase):
def set_vars(self):
self.weight_quantize_type = 'abs_max'
self.activation_quantize_type = 'moving_average_abs_max'
self.onnx_format = True
self.fuse_conv_bn = False
def func_qat(self):
self.set_vars()
imperative_qat = ImperativeQuantAware(
weight_quantize_type=self.weight_quantize_type,
activation_quantize_type=self.activation_quantize_type,
fuse_conv_bn=self.fuse_conv_bn,
)
seed = 100
np.random.seed(seed)
paddle.seed(seed)
paddle.disable_static()
lenet = ImperativeLenet()
lenet = fix_model_dict(lenet)
imperative_qat.quantize(lenet)
optimizer = Momentum(
learning_rate=0.1, parameters=lenet.parameters(), momentum=0.9
)
train_reader = paddle.batch(
paddle.dataset.mnist.train(), batch_size=64, drop_last=True
)
test_reader = paddle.batch(paddle.dataset.mnist.test(), batch_size=32)
epoch_num = 1
for epoch in range(epoch_num):
lenet.train()
for batch_id, data in enumerate(train_reader()):
x_data = np.array(
[x[0].reshape(1, 28, 28) for x in data]
).astype('float32')
y_data = (
np.array([x[1] for x in data])
.astype('int64')
.reshape(-1, 1)
)
img = paddle.to_tensor(x_data)
label = paddle.to_tensor(y_data)
out = lenet(img)
acc = paddle.metric.accuracy(out, label)
loss = paddle.nn.functional.cross_entropy(
out, label, reduction='none', use_softmax=False
)
avg_loss = paddle.mean(loss)
avg_loss.backward()
optimizer.step()
optimizer.clear_grad()
if batch_id % 100 == 0:
_logger.info(
f"Train | At epoch {epoch} step {batch_id}: loss = {avg_loss.numpy()}, acc= {acc.numpy()}"
)
lenet.eval()
eval_acc_top1_list = []
with paddle.no_grad():
for batch_id, data in enumerate(test_reader()):
x_data = np.array(
[x[0].reshape(1, 28, 28) for x in data]
).astype('float32')
y_data = (
np.array([x[1] for x in data])
.astype('int64')
.reshape(-1, 1)
)
img = paddle.to_tensor(x_data)
label = paddle.to_tensor(y_data)
out = lenet(img)
acc_top1 = paddle.metric.accuracy(
input=out, label=label, k=1
)
acc_top5 = paddle.metric.accuracy(
input=out, label=label, k=5
)
if batch_id % 100 == 0:
eval_acc_top1_list.append(float(acc_top1.numpy()))
_logger.info(
f"Test | At epoch {epoch} step {batch_id}: acc1 = {acc_top1.numpy()}, acc5 = {acc_top5.numpy()}"
)
# check eval acc
eval_acc_top1 = sum(eval_acc_top1_list) / len(eval_acc_top1_list)
print('eval_acc_top1', eval_acc_top1)
def test_qat(self):
self.func_qat()
if __name__ == '__main__':
unittest.main()
test/quantization/test_imperative_qat_user_defined.py
+281
View File
@@ -0,0 +1,281 @@
# copyright (c) 2020 paddlepaddle authors. all rights reserved.
#
# licensed under the apache license, version 2.0 (the "license");
# you may not use this file except in compliance with the license.
# you may obtain a copy of the license at
#
# http://www.apache.org/licenses/license-2.0
#
# unless required by applicable law or agreed to in writing, software
# distributed under the license is distributed on an "as is" basis,
# without warranties or conditions of any kind, either express or implied.
# see the license for the specific language governing permissions and
# limitations under the license.
import logging
import os
import unittest
import numpy as np
import paddle
from paddle import nn
from paddle.nn import Sequential
from paddle.optimizer import Adam
from paddle.quantization import ImperativeQuantAware
from paddle.static.log_helper import get_logger
os.environ["CPU_NUM"] = "1"
_logger = get_logger(
__name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s'
)
class PACT(nn.Layer):
def __init__(self, init_value=20):
super().__init__()
alpha_attr = paddle.ParamAttr(
name=self.full_name() + ".pact",
initializer=paddle.nn.initializer.Constant(value=init_value),
)
self.alpha = self.create_parameter(
shape=[1], attr=alpha_attr, dtype='float32'
)
def forward(self, x):
out_left = paddle.nn.functional.relu(x - self.alpha)
out_right = paddle.nn.functional.relu(-self.alpha - x)
x = x - out_left + out_right
return x
class CustomQAT(nn.Layer):
def __init__(self):
super().__init__()
attr = paddle.ParamAttr(
initializer=paddle.nn.initializer.Constant(value=1.0)
)
self.u_param = self.create_parameter(
shape=[1], attr=attr, dtype='float32'
)
self.l_param = self.create_parameter(
shape=[1], attr=attr, dtype='float32'
)
self.alpha_param = self.create_parameter(
shape=[1], attr=attr, dtype='float32'
)
self.upper = self.create_parameter(
shape=[1], attr=attr, dtype='float32'
)
self.upper.stop_gradient = True
self.lower = self.create_parameter(
shape=[1], attr=attr, dtype='float32'
)
self.lower.stop_gradient = True
def forward(self, x):
def clip(x, upper, lower):
x = x + paddle.nn.functional.relu(lower - x)
x = x - paddle.nn.functional.relu(x - upper)
return x
def phi_function(x, mi, alpha, delta):
s = 1 / (1 - alpha)
k = paddle.log(2 / alpha - 1) * (1 / delta)
x = (paddle.tanh((x - mi) * k)) * s
return x
def dequantize(x, lower_bound, delta, interval):
x = ((x + 1) / 2 + interval) * delta + lower_bound
return x
bit = 8
bit_range = 2**bit - 1
paddle.assign(self.upper * 0.9 + self.u_param * 0.1, self.upper)
paddle.assign(self.lower * 0.9 + self.l_param * 0.1, self.lower)
x = clip(x, self.upper, self.lower)
delta = (self.upper - self.lower) / bit_range
interval = (x - self.lower) / delta
mi = (interval + 0.5) * delta + self.l_param
x = phi_function(x, mi, self.alpha_param, delta)
x = dequantize(x, self.l_param, delta, interval)
return x
class ModelForConv2dT(nn.Layer):
def __init__(self, num_classes=10):
super().__init__()
self.features = nn.Conv2DTranspose(4, 6, (3, 3))
self.fc = nn.Linear(in_features=600, out_features=num_classes)
def forward(self, inputs):
x = self.features(inputs)
x = paddle.flatten(x, 1)
x = self.fc(x)
return x
class ImperativeLenet(paddle.nn.Layer):
def __init__(self, num_classes=10, classifier_activation='softmax'):
super().__init__()
self.features = Sequential(
nn.Conv2D(
in_channels=1,
out_channels=6,
kernel_size=3,
stride=1,
padding=1,
),
nn.MaxPool2D(kernel_size=2, stride=2),
nn.Conv2D(
in_channels=6,
out_channels=16,
kernel_size=5,
stride=1,
padding=0,
),
nn.MaxPool2D(kernel_size=2, stride=2),
)
self.fc = Sequential(
nn.Linear(in_features=400, out_features=120),
nn.Linear(in_features=120, out_features=84),
nn.Linear(in_features=84, out_features=num_classes),
)
def forward(self, inputs):
x = self.features(inputs)
x = paddle.flatten(x, 1)
x = self.fc(x)
return x
class TestUserDefinedActPreprocess(unittest.TestCase):
def setUp(self):
_logger.info("test act_preprocess")
self.imperative_qat = ImperativeQuantAware(act_preprocess_layer=PACT)
def func_quant_aware_training(self):
imperative_qat = self.imperative_qat
seed = 1
np.random.seed(seed)
paddle.seed(seed)
lenet = ImperativeLenet()
fixed_state = {}
param_init_map = {}
for name, param in lenet.named_parameters():
p_shape = np.array(param).shape
p_value = np.array(param)
if name.endswith("bias"):
value = np.zeros_like(p_value).astype('float32')
else:
value = (
np.random.normal(loc=0.0, scale=0.01, size=np.prod(p_shape))
.reshape(p_shape)
.astype('float32')
)
fixed_state[name] = value
param_init_map[param.name] = value
lenet.set_dict(fixed_state)
imperative_qat.quantize(lenet)
adam = Adam(learning_rate=0.001, parameters=lenet.parameters())
dynamic_loss_rec = []
# for CI coverage
conv_transpose = ModelForConv2dT()
imperative_qat.quantize(conv_transpose)
x_var = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1.0, max=1.0)
conv_transpose(x_var)
def train(model):
adam = Adam(learning_rate=0.001, parameters=model.parameters())
epoch_num = 1
for epoch in range(epoch_num):
model.train()
for batch_id, data in enumerate(train_reader()):
x_data = np.array(
[x[0].reshape(1, 28, 28) for x in data]
).astype('float32')
y_data = (
np.array([x[1] for x in data])
.astype('int64')
.reshape(-1, 1)
)
img = paddle.to_tensor(x_data)
label = paddle.to_tensor(y_data)
out = model(img)
acc = paddle.metric.accuracy(out, label, k=1)
loss = nn.functional.loss.cross_entropy(out, label)
avg_loss = paddle.mean(loss)
avg_loss.backward()
adam.step()
adam.clear_grad()
if batch_id % 50 == 0:
_logger.info(
f"Train | At epoch {epoch} step {batch_id}: loss = {avg_loss.numpy()}, acc= {acc.numpy()}"
)
break
def test(model):
model.eval()
avg_acc = [[], []]
for batch_id, data in enumerate(test_reader()):
x_data = np.array(
[x[0].reshape(1, 28, 28) for x in data]
).astype('float32')
y_data = (
np.array([x[1] for x in data])
.astype('int64')
.reshape(-1, 1)
)
img = paddle.to_tensor(x_data)
label = paddle.to_tensor(y_data)
out = model(img)
acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1)
acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5)
avg_acc[0].append(acc_top1.numpy())
avg_acc[1].append(acc_top5.numpy())
if batch_id % 100 == 0:
_logger.info(
f"Test | step {batch_id}: acc1 = {acc_top1.numpy()}, acc5 = {acc_top5.numpy()}"
)
train_reader = paddle.batch(
paddle.dataset.mnist.train(), batch_size=512, drop_last=True
)
test_reader = paddle.batch(paddle.dataset.mnist.test(), batch_size=512)
train(lenet)
test(lenet)
def test_quant_aware_training(self):
self.func_quant_aware_training()
class TestUserDefinedWeightPreprocess(TestUserDefinedActPreprocess):
def setUp(self):
_logger.info("test weight_preprocess")
self.imperative_qat = ImperativeQuantAware(weight_preprocess_layer=PACT)
class TestUserDefinedActQuantize(TestUserDefinedActPreprocess):
def setUp(self):
_logger.info("test act_quantize")
self.imperative_qat = ImperativeQuantAware(act_quantize_layer=CustomQAT)
class TestUserDefinedWeightQuantize(TestUserDefinedActPreprocess):
def setUp(self):
_logger.info("test weight_quantize")
self.imperative_qat = ImperativeQuantAware(
weight_quantize_layer=CustomQAT
)
if __name__ == '__main__':
unittest.main()
test/quantization/test_imperative_skip_op.py
+77
View File
@@ -0,0 +1,77 @@
# copyright (c) 2022 paddlepaddle authors. all rights reserved.
#
# licensed under the apache license, version 2.0 (the "license");
# you may not use this file except in compliance with the license.
# you may obtain a copy of the license at
#
# http://www.apache.org/licenses/license-2.0
#
# unless required by applicable law or agreed to in writing, software
# distributed under the license is distributed on an "as is" basis,
# without warranties or conditions of any kind, either express or implied.
# see the license for the specific language governing permissions and
# limitations under the license.
import os
import sys
import unittest
import numpy as np
sys.path.append("../../quantization")
from imperative_test_utils import (
ImperativeLenetWithSkipQuant,
fix_model_dict,
train_lenet,
)
import paddle
from paddle.framework import core, set_flags
from paddle.optimizer import Adam
from paddle.quantization import ImperativeQuantAware
INFER_MODEL_SUFFIX = ".pdmodel"
INFER_PARAMS_SUFFIX = ".pdiparams"
os.environ["CPU_NUM"] = "1"
if core.is_compiled_with_cuda():
set_flags({"FLAGS_cudnn_deterministic": True})
class TestImperativeOutSclae(unittest.TestCase):
def test_out_scale_acc(self):
paddle.disable_static()
seed = 1000
lr = 0.1
qat = ImperativeQuantAware()
np.random.seed(seed)
reader = paddle.batch(
paddle.dataset.mnist.test(), batch_size=512, drop_last=True
)
lenet = ImperativeLenetWithSkipQuant()
lenet = fix_model_dict(lenet)
qat.quantize(lenet)
adam = Adam(learning_rate=lr, parameters=lenet.parameters())
dynamic_loss_rec = []
lenet.train()
loss_list = train_lenet(lenet, reader, adam)
lenet.eval()
path = "./save_dynamic_quant_infer_model/lenet"
save_dir = "./save_dynamic_quant_infer_model"
paddle.enable_static()
if core.is_compiled_with_cuda():
place = core.CUDAPlace(0)
else:
place = core.CPUPlace()
exe = paddle.static.Executor(place)
if __name__ == '__main__':
unittest.main()
test/quantization/test_llm_int8_linear.py
+304
View File
@@ -0,0 +1,304 @@
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import numpy as np
from test_weight_only_linear import convert_uint16_to_float
import paddle
import paddle.nn.quant as Q
from paddle import base
from paddle.base import core
from paddle.framework import set_default_dtype
@unittest.skipIf(
not core.is_compiled_with_cuda()
or paddle.device.cuda.get_device_capability()[0] < 8,
"quantized_matmul requires CUDA_ARCH >= 8",
)
class LLMInt8LinearTestCase(unittest.TestCase):
def config(self):
self.dtype = 'float16'
self.rtol = 1e-5
self.atol = 1e-1
self.bias = True
self.batch = 1
self.token = 32
self.in_features = 64
self.out_features = 128
self.threshold = 6.0
self.static = False
def setUp(self):
np.random.seed(123)
paddle.seed(42)
self.config()
x = np.random.random((self.batch, self.token, self.in_features))
self.x = paddle.to_tensor(x, dtype=self.dtype)
if self.bias:
bias_attr = base.ParamAttr(
trainable=False,
regularizer=None,
initializer=paddle.nn.initializer.Constant(value=1.0),
)
else:
bias_attr = None
set_default_dtype(self.dtype)
self.linear = paddle.nn.Linear(
self.in_features, self.out_features, bias_attr=bias_attr
)
self.weight = self.linear.weight
self.weight_scale = None
self.weight, self.weight_scale = Q.weight_quantize(
self.weight, algo="llm.int8"
)
def dynamic_quant(self, x):
row_ranges = paddle.max(x, axis=[-1]).astype('float32')
row_ranges = row_ranges.unsqueeze(-1)
quant_x = paddle.round(
paddle.clip(
x.astype('float32') * 127.0 * (1 / row_ranges),
min=-127.0,
max=127.0,
)
).astype('int8')
return quant_x, row_ranges
def get_linear_out(self):
outlier_cols = (
paddle.nonzero(paddle.max(self.x, axis=[0, 1]) > self.threshold)
.reshape([-1])
.numpy()
.tolist()
)
x_int8 = self.x
if len(outlier_cols) > 0:
x_fp = self.x[:, :, outlier_cols]
w_fp = self.linear.weight[outlier_cols]
res_fp = paddle.matmul(x_fp, w_fp)
x_int8[:, :, outlier_cols] = 0
x_int8, row_ranges = self.dynamic_quant(x_int8)
res_int8 = paddle.matmul(x_int8, self.weight.transpose((1, 0)))
dequant_scale = row_ranges * self.weight_scale / 127.0
res_dequant = (res_int8.astype('float32') * dequant_scale).astype(
self.dtype
)
if len(outlier_cols) > 0:
out = res_dequant + res_fp
else:
out = res_dequant
if self.bias:
out += self.bias
return out.numpy()
def get_llm_int8_linear_out(self):
out = Q.llm_int8_linear(
self.x,
self.weight,
bias=self.linear.bias,
weight_scale=self.weight_scale,
threshold=self.threshold,
)
return out.numpy()
def llm_int8_linear_out_static(self, out_expect):
paddle.enable_static()
main = paddle.static.Program()
start = paddle.static.Program()
with paddle.static.program_guard(main, start):
x = paddle.static.data("x", self.x.shape, dtype=self.dtype)
weight = paddle.static.data(
"weight", self.weight.shape, dtype='int8'
)
bias = paddle.static.data(
"bias", self.linear.bias.shape, dtype=self.dtype
)
x_np = self.x.numpy()
weight_np = self.weight.numpy()
bias_np = self.linear.bias.numpy()
if self.weight_scale is not None:
weight_scale = paddle.static.data(
"weight_scale",
self.weight_scale.shape,
dtype='float32',
)
weight_scale_np = self.weight_scale.numpy()
else:
weight_scale = None
weight_scale_np = None
out = Q.llm_int8_linear(
x,
weight,
bias,
weight_scale,
self.threshold,
)
feed_dict = {
'x': x_np,
'weight': weight_np,
'bias': bias_np,
"weight_scale": weight_scale_np,
}
exe = base.Executor(paddle.CUDAPlace(0))
exe.run(start)
(out_real,) = exe.run(main, feed=feed_dict, fetch_list=[out])
paddle.disable_static()
if self.dtype == "bfloat16":
out_real = convert_uint16_to_float(out_real)
out_expect = convert_uint16_to_float(out_expect)
np.testing.assert_allclose(
out_real, out_expect, rtol=self.rtol, atol=self.atol
)
def test_llm_int8_linear(self):
out_expect = self.get_linear_out()
if self.static:
self.llm_int8_linear_out_static(out_expect)
return
else:
out_real = self.get_llm_int8_linear_out()
if self.dtype == "bfloat16":
out_real = convert_uint16_to_float(out_real)
out_expect = convert_uint16_to_float(out_expect)
np.testing.assert_allclose(
out_real, out_expect, rtol=self.rtol, atol=self.atol
)
@unittest.skipIf(
not core.is_compiled_with_cuda()
or paddle.device.cuda.get_device_capability()[0] < 8,
"quantized_matmul requires CUDA_ARCH >= 8",
)
class LLMInt8LinearTestCase1(LLMInt8LinearTestCase):
def config(self):
super().config()
self.dtype = 'float16'
self.weight_dtype = "int8"
@unittest.skipIf(
not core.is_compiled_with_cuda()
or paddle.device.cuda.get_device_capability()[0] < 8,
"quantized_matmul requires CUDA_ARCH >= 8",
)
class LLMInt8LinearTestCase2(LLMInt8LinearTestCase):
def config(self):
super().config()
self.dtype = 'float16'
self.bias = False
self.weight_dtype = "int8"
@unittest.skipIf(
not core.is_compiled_with_cuda()
or paddle.device.cuda.get_device_capability()[0] < 8
or not core.is_bfloat16_supported(core.CUDAPlace(0)),
"quantized_matmul requires CUDA_ARCH >= 8 or core is not support bfloat16",
)
class LLMInt8LinearTestCase4(LLMInt8LinearTestCase):
def config(self):
super().config()
self.dtype = 'float16'
self.weight_dtype = "int4"
@unittest.skipIf(
not core.is_compiled_with_cuda()
or paddle.device.cuda.get_device_capability()[0] < 8,
"quantized_matmul requires CUDA_ARCH >= 8",
)
class LLMInt8LinearTestCase5(LLMInt8LinearTestCase):
def config(self):
super().config()
self.dtype = 'float16'
self.bias = False
self.weight_dtype = "int4"
@unittest.skipIf(
not core.is_compiled_with_cuda()
or paddle.device.cuda.get_device_capability()[0] < 8,
"quantized_matmul requires CUDA_ARCH >= 8",
)
class LLMInt8LinearTestCase7(LLMInt8LinearTestCase):
def config(self):
super().config()
self.dtype = 'float16'
self.weight_dtype = "int8"
self.batch = 1
self.token = 1
@unittest.skipIf(
not core.is_compiled_with_cuda()
or paddle.device.cuda.get_device_capability()[0] < 8,
"quantized_matmul requires CUDA_ARCH >= 8",
)
class LLMInt8LinearTestCase8(LLMInt8LinearTestCase):
def config(self):
super().config()
self.dtype = 'float16'
self.weight_dtype = "int8"
self.bias = False
self.batch = 1
self.token = 1
@unittest.skipIf(
not core.is_compiled_with_cuda()
or paddle.device.cuda.get_device_capability()[0] < 8,
"quantized_matmul requires CUDA_ARCH >= 8",
)
class LLMInt8LinearTestCase10(LLMInt8LinearTestCase):
def config(self):
super().config()
self.dtype = 'bfloat16'
self.weight_dtype = "int8"
self.bias = False
self.batch = 1
self.token = 1
@unittest.skipIf(
not core.is_compiled_with_cuda()
or not core.is_compiled_with_cuda()
or paddle.device.cuda.get_device_capability()[0] < 8,
"quantized_matmul requires CUDA_ARCH >= 8",
)
class LLMInt8LinearTestCaseStatic(LLMInt8LinearTestCase):
def config(self):
super().config()
self.static = True
if __name__ == '__main__':
unittest.main()
test/quantization/test_observer_convert.py
+90
View File
@@ -0,0 +1,90 @@
# copyright (c) 2023 paddlepaddle authors. all rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import tempfile
import unittest
import paddle
from paddle.nn import Linear, Sequential
from paddle.quantization import PTQ, QuantConfig
from paddle.quantization.observers import (
AbsmaxObserver,
GroupWiseWeightObserver,
)
class LinearDygraph(paddle.nn.Layer):
def __init__(self):
super().__init__()
self.fc = Sequential(
Linear(128, 128), Linear(128, 128), Linear(128, 128)
)
def forward(self, inputs):
out = self.fc(inputs)
return out
class TestPTQGroupWise(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.path = os.path.join(self.temp_dir.name, 'ptq')
def tearDown(self):
self.temp_dir.cleanup()
def _get_model_for_ptq_groupwise(self):
observer = GroupWiseWeightObserver(quant_bits=4, group_size=128)
model = LinearDygraph()
model.eval()
q_config = QuantConfig(activation=None, weight=observer)
ptq = PTQ(q_config)
quant_model = ptq.quantize(model)
inputs = paddle.rand([128, 128], dtype="float32")
out = model(inputs)
return quant_model, ptq
def _get_model_for_ptq_absmax(self):
observer = AbsmaxObserver(quant_bits=8)
model = LinearDygraph()
model.eval()
q_config = QuantConfig(activation=observer, weight=observer)
ptq = PTQ(q_config)
quant_model = ptq.quantize(model)
inputs = paddle.rand([128, 128], dtype="float32")
out = model(inputs)
return quant_model, ptq
def test_quantize(self):
ptq_model, ptq = self._get_model_for_ptq_groupwise()
inputs = paddle.rand([128, 128], dtype="float32")
out = ptq_model(inputs)
self.assertIsNotNone(out)
converted_model = ptq.convert(ptq_model)
out = converted_model(inputs)
self.assertIsNotNone(out)
def test_quantize_absmax(self):
ptq_model, ptq = self._get_model_for_ptq_absmax()
inputs = paddle.rand([128, 128], dtype="float32")
out = ptq_model(inputs)
self.assertIsNotNone(out)
converted_model = ptq.convert(ptq_model)
out = converted_model(inputs)
self.assertIsNotNone(out)
if __name__ == '__main__':
unittest.main()
test/quantization/test_post_training_quantization_lstm_model.py
+384
View File
@@ -0,0 +1,384 @@
# copyright (c) 2018 paddlepaddle authors. all rights reserved.
#
# licensed under the apache license, version 2.0 (the "license");
# you may not use this file except in compliance with the license.
# you may obtain a copy of the license at
#
# http://www.apache.org/licenses/license-2.0
#
# unless required by applicable law or agreed to in writing, software
# distributed under the license is distributed on an "as is" basis,
# without warranties or conditions of any kind, either express or implied.
# see the license for the specific language governing permissions and
# limitations under the license.
import os
import random
import struct
import sys
import tempfile
import time
import unittest
import numpy as np
import paddle
from paddle import base
from paddle.dataset.common import download
from paddle.static.quantization import PostTrainingQuantization
paddle.enable_static()
random.seed(0)
np.random.seed(0)
class TestPostTrainingQuantization(unittest.TestCase):
def setUp(self):
self.download_path = 'int8/download'
self.cache_folder = os.path.expanduser(
'~/.cache/paddle/dataset/' + self.download_path
)
self.root_path = tempfile.TemporaryDirectory()
self.int8_model_path = os.path.join(
self.root_path.name, "post_training_quantization"
)
try:
os.system("mkdir -p " + self.int8_model_path)
except Exception as e:
print(f"Failed to create {self.int8_model_path} due to {e}")
sys.exit(-1)
def tearDown(self):
self.root_path.cleanup()
def cache_unzipping(self, target_folder, zip_path):
if not os.path.exists(target_folder):
cmd = (
f'mkdir {target_folder} && tar xf {zip_path} -C {target_folder}'
)
os.system(cmd)
def download_model(self, data_url, data_md5, folder_name):
download(data_url, self.download_path, data_md5)
file_name = data_url.split('/')[-1]
zip_path = os.path.join(self.cache_folder, file_name)
print(f'Data is downloaded at {zip_path}')
data_cache_folder = os.path.join(self.cache_folder, folder_name)
self.cache_unzipping(data_cache_folder, zip_path)
return data_cache_folder
def get_batch_reader(self, data_path, place):
def reader():
with open(data_path, 'rb') as in_file:
while True:
plen = in_file.read(4)
if plen is None or len(plen) != 4:
break
all_len = struct.unpack('i', plen)[0]
label_len = all_len & 0xFFFF
seq_len = (all_len >> 16) & 0xFFFF
label = in_file.read(4 * label_len)
label = np.frombuffer(label, dtype=np.int32).reshape(
[len(label) // 4]
)
if label.shape[0] != 1 or label[0] > 6350:
continue
feat = in_file.read(4 * seq_len * 8)
feat = np.frombuffer(feat, dtype=np.float32).reshape(
[len(feat) // 4 // 8, 8]
)
lod_feat = [feat.shape[0]]
minputs = base.create_lod_tensor(feat, [lod_feat], place)
yield [minputs]
return reader
def get_simple_reader(self, data_path, place):
def reader():
with open(data_path, 'rb') as in_file:
while True:
plen = in_file.read(4)
if plen is None or len(plen) != 4:
break
all_len = struct.unpack('i', plen)[0]
label_len = all_len & 0xFFFF
seq_len = (all_len >> 16) & 0xFFFF
label = in_file.read(4 * label_len)
label = np.frombuffer(label, dtype=np.int32).reshape(
[len(label) // 4]
)
if label.shape[0] != 1 or label[0] > 6350:
continue
feat = in_file.read(4 * seq_len * 8)
feat = np.frombuffer(feat, dtype=np.float32).reshape(
[len(feat) // 4 // 8, 8]
)
lod_feat = [feat.shape[0]]
minputs = base.create_lod_tensor(feat, [lod_feat], place)
yield minputs, label
return reader
def run_program(
self,
model_path,
model_filename,
params_filename,
data_path,
infer_iterations,
):
print("test model path:" + model_path)
place = paddle.CPUPlace()
exe = paddle.static.Executor(place)
[
infer_program,
feed_dict,
fetch_targets,
] = paddle.static.load_inference_model(
model_path,
exe,
model_filename=model_filename,
params_filename=params_filename,
)
val_reader = self.get_simple_reader(data_path, place)
all_num = 0
right_num = 0
periods = []
for batch_id, (data, label) in enumerate(val_reader()):
t1 = time.time()
cls_out, ctc_out = exe.run(
infer_program,
feed={feed_dict[0]: data},
fetch_list=fetch_targets,
return_numpy=False,
)
t2 = time.time()
periods.append(t2 - t1)
cls_out = np.array(cls_out).reshape(-1)
out_cls_label = np.argmax(cls_out)
all_num += 1
if out_cls_label == label[0]:
right_num += 1
if (batch_id + 1) == infer_iterations:
break
latency = np.average(periods)
acc = right_num / all_num
return (latency, acc)
def generate_quantized_model(
self,
model_path,
model_filename,
params_filename,
data_path,
algo="KL",
round_type="round",
quantizable_op_type=["conv2d"],
is_full_quantize=False,
is_use_cache_file=False,
is_optimize_model=False,
batch_size=10,
batch_nums=10,
onnx_format=False,
):
place = paddle.CPUPlace()
exe = paddle.static.Executor(place)
scope = paddle.static.global_scope()
batch_generator = self.get_batch_reader(data_path, place)
ptq = PostTrainingQuantization(
executor=exe,
model_dir=model_path,
model_filename=model_filename,
params_filename=params_filename,
batch_generator=batch_generator,
batch_nums=batch_nums,
algo=algo,
quantizable_op_type=quantizable_op_type,
round_type=round_type,
is_full_quantize=is_full_quantize,
optimize_model=is_optimize_model,
onnx_format=onnx_format,
is_use_cache_file=is_use_cache_file,
)
ptq.quantize()
if onnx_format:
ptq._clip_extra = False
ptq.save_quantized_model(self.int8_model_path)
def run_test(
self,
model_name,
model_filename,
params_filename,
model_url,
model_md5,
data_name,
data_url,
data_md5,
algo,
round_type,
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
infer_iterations,
quant_iterations,
onnx_format=False,
):
fp32_model_path = self.download_model(model_url, model_md5, model_name)
fp32_model_path = os.path.join(fp32_model_path, model_name)
data_path = self.download_model(data_url, data_md5, data_name)
data_path = os.path.join(data_path, data_name)
print(
f"Start FP32 inference for {model_name} on {infer_iterations} samples ..."
)
(fp32_latency, fp32_acc) = self.run_program(
fp32_model_path,
model_filename,
params_filename,
data_path,
infer_iterations,
)
print(
f"Start post training quantization for {model_name} on {quant_iterations} samples ..."
)
self.generate_quantized_model(
fp32_model_path,
model_filename,
params_filename,
data_path,
algo,
round_type,
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
10,
quant_iterations,
onnx_format,
)
print(
f"Start INT8 inference for {model_name} on {infer_iterations} samples ..."
)
(int8_latency, int8_acc) = self.run_program(
self.int8_model_path,
'model.pdmodel',
'model.pdiparams',
data_path,
infer_iterations,
)
print(f"---Post training quantization of {algo} method---")
print(
f"FP32 {model_name}: batch_size {1}, latency {fp32_latency} s, acc {fp32_acc}."
)
print(
f"INT8 {model_name}: batch_size {1}, latency {int8_latency} s, acc1 {int8_acc}.\n"
)
sys.stdout.flush()
delta_value = fp32_acc - int8_acc
self.assertLess(delta_value, diff_threshold)
class TestPostTrainingAvgForLSTM(TestPostTrainingQuantization):
def test_post_training_avg(self):
model_name = "nlp_lstm_fp32_model"
model_url = "https://paddle-inference-dist.cdn.bcebos.com/int8/unittest_model_data/nlp_lstm_fp32_model_combined.tar.gz"
model_md5 = "5b47cd7ba2afcf24120d9727ed3f05a7"
data_name = "quant_lstm_input_data"
data_url = "https://paddle-inference-dist.cdn.bcebos.com/int8/unittest_model_data/quant_lstm_input_data.tar.gz"
data_md5 = "add84c754e9b792fea1fbd728d134ab7"
algo = "avg"
round_type = "round"
quantizable_op_type = ["mul", "lstm"]
is_full_quantize = False
is_use_cache_file = False
is_optimize_model = False
diff_threshold = 0.02
infer_iterations = 100
quant_iterations = 10
self.run_test(
model_name,
'model.pdmodel',
'model.pdiparams',
model_url,
model_md5,
data_name,
data_url,
data_md5,
algo,
round_type,
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
infer_iterations,
quant_iterations,
)
class TestPostTrainingAvgForLSTMONNXFormat(TestPostTrainingQuantization):
def not_test_post_training_avg_onnx_format(self):
model_name = "nlp_lstm_fp32_model"
model_url = "https://paddle-inference-dist.cdn.bcebos.com/int8/unittest_model_data/nlp_lstm_fp32_model_combined.tar.gz"
model_md5 = "5b47cd7ba2afcf24120d9727ed3f05a7"
data_name = "quant_lstm_input_data"
data_url = "https://paddle-inference-dist.cdn.bcebos.com/int8/unittest_model_data/quant_lstm_input_data.tar.gz"
data_md5 = "add84c754e9b792fea1fbd728d134ab7"
algo = "avg"
round_type = "round"
quantizable_op_type = ["mul", "lstm"]
is_full_quantize = False
is_use_cache_file = False
is_optimize_model = False
diff_threshold = 0.02
infer_iterations = 100
quant_iterations = 10
onnx_format = True
self.run_test(
model_name,
'model.pdmodel',
'model.pdiparams',
model_url,
model_md5,
data_name,
data_url,
data_md5,
algo,
round_type,
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
infer_iterations,
quant_iterations,
onnx_format=onnx_format,
)
if __name__ == '__main__':
unittest.main()
test/quantization/test_post_training_quantization_mnist.py
+728
View File
@@ -0,0 +1,728 @@
# copyright (c) 2022 paddlepaddle authors. all rights reserved.
#
# licensed under the apache license, version 2.0 (the "license");
# you may not use this file except in compliance with the license.
# you may obtain a copy of the license at
#
# http://www.apache.org/licenses/license-2.0
#
# unless required by applicable law or agreed to in writing, software
# distributed under the license is distributed on an "as is" basis,
# without warranties or conditions of any kind, either express or implied.
# see the license for the specific language governing permissions and
# limitations under the license.
import os
import random
import sys
import tempfile
import time
import unittest
import numpy as np
import paddle
from paddle.dataset.common import md5file
from paddle.static.quantization import PostTrainingQuantization
paddle.enable_static()
random.seed(0)
np.random.seed(0)
class TransedMnistDataSet(paddle.io.Dataset):
def __init__(self, mnist_data):
self.mnist_data = mnist_data
def __getitem__(self, idx):
img = (
np.array(self.mnist_data[idx][0])
.astype('float32')
.reshape(1, 28, 28)
)
batch = img / 127.5 - 1.0
return {"img": batch}
def __len__(self):
return len(self.mnist_data)
class TestPostTrainingQuantization(unittest.TestCase):
def setUp(self):
self.root_path = tempfile.TemporaryDirectory()
self.int8_model_path = os.path.join(
self.root_path.name, "post_training_quantization"
)
self.download_path = f'download_model_{time.time()}'
self.cache_folder = os.path.join(
self.root_path.name, self.download_path
)
try:
os.system("mkdir -p " + self.int8_model_path)
os.system("mkdir -p " + self.cache_folder)
except Exception as e:
print(f"Failed to create {self.int8_model_path} due to {e}")
sys.exit(-1)
def tearDown(self):
self.root_path.cleanup()
def cache_unzipping(self, target_folder, zip_path):
if not os.path.exists(target_folder):
cmd = (
f'mkdir {target_folder} && tar xf {zip_path} -C {target_folder}'
)
os.system(cmd)
def download(self, url, dirname, md5sum, save_name=None):
import shutil
import httpx
filename = os.path.join(
dirname, url.split('/')[-1] if save_name is None else save_name
)
if os.path.exists(filename) and md5file(filename) == md5sum:
return filename
retry = 0
retry_limit = 3
while not (os.path.exists(filename) and md5file(filename) == md5sum):
if os.path.exists(filename):
sys.stderr.write(f"file {md5file(filename)} md5 {md5sum}\n")
if retry < retry_limit:
retry += 1
else:
raise RuntimeError(
f"Cannot download {url} within retry limit {retry_limit}"
)
sys.stderr.write(
f"Cache file {filename} not found, downloading {url} \n"
)
sys.stderr.write("Begin to download\n")
try:
with httpx.stream("GET", url) as r:
total_length = r.headers.get('content-length')
if total_length is None:
with open(filename, 'wb') as f:
shutil.copyfileobj(r.raw, f)
else:
with open(filename, 'wb') as f:
chunk_size = 4096
total_length = int(total_length)
total_iter = total_length / chunk_size + 1
log_interval = (
total_iter // 20 if total_iter > 20 else 1
)
log_index = 0
bar = paddle.hapi.progressbar.ProgressBar(
total_iter, name='item'
)
for data in r.iter_bytes(chunk_size=chunk_size):
f.write(data)
log_index += 1
bar.update(log_index, {})
if log_index % log_interval == 0:
bar.update(log_index)
except Exception as e:
# re-try
continue
sys.stderr.write("\nDownload finished\n")
sys.stdout.flush()
return filename
def download_model(self, data_url, data_md5, folder_name):
self.download(data_url, self.cache_folder, data_md5)
os.system(f'wget -q {data_url}')
file_name = data_url.split('/')[-1]
zip_path = os.path.join(self.cache_folder, file_name)
print(
f'Data is downloaded at {zip_path}. File exists: {os.path.exists(zip_path)}'
)
data_cache_folder = os.path.join(self.cache_folder, folder_name)
self.cache_unzipping(data_cache_folder, zip_path)
return data_cache_folder
def run_program(
self,
model_path,
model_filename,
params_filename,
batch_size,
infer_iterations,
):
print(
f"test model path: {model_path}. File exists: {os.path.exists(model_path)}"
)
place = paddle.CPUPlace()
exe = paddle.static.Executor(place)
[
infer_program,
feed_dict,
fetch_targets,
] = paddle.static.load_inference_model(
model_path,
exe,
model_filename=model_filename,
params_filename=params_filename,
)
val_reader = paddle.batch(paddle.dataset.mnist.test(), batch_size)
img_shape = [1, 28, 28]
test_info = []
cnt = 0
periods = []
for batch_id, data in enumerate(val_reader()):
image = np.array([x[0].reshape(img_shape) for x in data]).astype(
"float32"
)
input_label = np.array([x[1] for x in data]).astype("int64")
t1 = time.time()
out = exe.run(
infer_program,
feed={feed_dict[0]: image},
fetch_list=fetch_targets,
)
t2 = time.time()
period = t2 - t1
periods.append(period)
out_label = np.argmax(np.array(out[0]), axis=1)
top1_num = sum(input_label == out_label)
test_info.append(top1_num)
cnt += len(data)
if (batch_id + 1) == infer_iterations:
break
throughput = cnt / np.sum(periods)
latency = np.average(periods)
acc1 = np.sum(test_info) / cnt
return (throughput, latency, acc1)
def generate_quantized_model(
self,
model_path,
model_filename,
params_filename,
algo="KL",
round_type="round",
quantizable_op_type=["conv2d"],
is_full_quantize=False,
is_use_cache_file=False,
is_optimize_model=False,
batch_size=10,
batch_nums=10,
onnx_format=False,
skip_tensor_list=None,
bias_correction=False,
):
place = paddle.CPUPlace()
exe = paddle.static.Executor(place)
train_dataset = paddle.vision.datasets.MNIST(
mode='train', transform=None
)
train_dataset = TransedMnistDataSet(train_dataset)
BatchSampler = paddle.io.BatchSampler(
train_dataset, batch_size=batch_size
)
val_data_generator = paddle.io.DataLoader(
train_dataset,
batch_sampler=BatchSampler,
places=paddle.static.cpu_places(),
)
ptq = PostTrainingQuantization(
executor=exe,
model_dir=model_path,
model_filename=model_filename,
params_filename=params_filename,
sample_generator=None,
data_loader=val_data_generator,
batch_size=batch_size,
batch_nums=batch_nums,
algo=algo,
quantizable_op_type=quantizable_op_type,
round_type=round_type,
is_full_quantize=is_full_quantize,
optimize_model=is_optimize_model,
bias_correction=bias_correction,
onnx_format=onnx_format,
skip_tensor_list=skip_tensor_list,
is_use_cache_file=is_use_cache_file,
)
ptq.quantize()
ptq.save_quantized_model(self.int8_model_path)
def run_test(
self,
model_name,
model_filename,
params_filename,
data_url,
data_md5,
algo,
round_type,
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
batch_size=10,
infer_iterations=10,
quant_iterations=5,
bias_correction=False,
onnx_format=False,
skip_tensor_list=None,
):
origin_model_path = self.download_model(data_url, data_md5, model_name)
origin_model_path = os.path.join(origin_model_path, model_name)
print(
f"Start FP32 inference for {model_name} on {infer_iterations * batch_size} images ..."
)
(fp32_throughput, fp32_latency, fp32_acc1) = self.run_program(
origin_model_path,
model_filename,
params_filename,
batch_size,
infer_iterations,
)
print(
f"Start INT8 post training quantization for {model_name} on {quant_iterations * batch_size} images ..."
)
self.generate_quantized_model(
origin_model_path,
model_filename,
params_filename,
algo,
round_type,
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
batch_size,
quant_iterations,
onnx_format,
skip_tensor_list,
bias_correction,
)
print(
f"Start INT8 inference for {model_name} on {infer_iterations * batch_size} images ..."
)
(int8_throughput, int8_latency, int8_acc1) = self.run_program(
self.int8_model_path,
'model.pdmodel',
'model.pdiparams',
batch_size,
infer_iterations,
)
print(f"---Post training quantization of {algo} method---")
print(
f"FP32 {model_name}: batch_size {batch_size}, throughput {fp32_throughput} img/s, latency {fp32_latency} s, acc1 {fp32_acc1}."
)
print(
f"INT8 {model_name}: batch_size {batch_size}, throughput {int8_throughput} img/s, latency {int8_latency} s, acc1 {int8_acc1}.\n"
)
sys.stdout.flush()
delta_value = fp32_acc1 - int8_acc1
self.assertLess(delta_value, diff_threshold)
class TestPostTrainingKLForMnist(TestPostTrainingQuantization):
def test_post_training_kl(self):
model_name = "mnist_model"
data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model_combined.tar.gz"
data_md5 = "a49251d3f555695473941e5a725c6014"
algo = "KL"
round_type = "round"
quantizable_op_type = ["conv2d", "depthwise_conv2d", "mul"]
is_full_quantize = False
is_use_cache_file = False
is_optimize_model = True
diff_threshold = 0.01
batch_size = 10
infer_iterations = 50
quant_iterations = 5
self.run_test(
model_name,
'model.pdmodel',
'model.pdiparams',
data_url,
data_md5,
algo,
round_type,
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
batch_size,
infer_iterations,
quant_iterations,
)
class TestPostTraininghistForMnist(TestPostTrainingQuantization):
def test_post_training_hist(self):
model_name = "mnist_model"
data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model_combined.tar.gz"
data_md5 = "a49251d3f555695473941e5a725c6014"
algo = "hist"
round_type = "round"
quantizable_op_type = ["conv2d", "depthwise_conv2d", "mul"]
is_full_quantize = False
is_use_cache_file = False
is_optimize_model = True
diff_threshold = 0.01
batch_size = 10
infer_iterations = 50
quant_iterations = 5
self.run_test(
model_name,
'model.pdmodel',
'model.pdiparams',
data_url,
data_md5,
algo,
round_type,
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
batch_size,
infer_iterations,
quant_iterations,
)
class TestPostTrainingmseForMnist(TestPostTrainingQuantization):
def test_post_training_mse(self):
model_name = "mnist_model"
data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model_combined.tar.gz"
data_md5 = "a49251d3f555695473941e5a725c6014"
algo = "mse"
round_type = "round"
quantizable_op_type = ["conv2d", "depthwise_conv2d", "mul"]
is_full_quantize = False
is_use_cache_file = False
is_optimize_model = True
diff_threshold = 0.01
batch_size = 10
infer_iterations = 50
quant_iterations = 5
self.run_test(
model_name,
'model.pdmodel',
'model.pdiparams',
data_url,
data_md5,
algo,
round_type,
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
batch_size,
infer_iterations,
quant_iterations,
)
class TestPostTrainingemdForMnist(TestPostTrainingQuantization):
def test_post_training_mse(self):
model_name = "mnist_model"
data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model_combined.tar.gz"
data_md5 = "a49251d3f555695473941e5a725c6014"
algo = "emd"
round_type = "round"
quantizable_op_type = ["conv2d", "depthwise_conv2d", "mul"]
is_full_quantize = False
is_use_cache_file = False
is_optimize_model = True
diff_threshold = 0.01
batch_size = 10
infer_iterations = 50
quant_iterations = 5
self.run_test(
model_name,
'model.pdmodel',
'model.pdiparams',
data_url,
data_md5,
algo,
round_type,
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
batch_size,
infer_iterations,
quant_iterations,
)
class TestPostTrainingavgForMnist(TestPostTrainingQuantization):
def test_post_training_avg(self):
model_name = "mnist_model"
data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model_combined.tar.gz"
data_md5 = "a49251d3f555695473941e5a725c6014"
algo = "avg"
round_type = "round"
quantizable_op_type = ["conv2d", "depthwise_conv2d", "mul"]
is_full_quantize = False
is_use_cache_file = False
is_optimize_model = True
diff_threshold = 0.01
batch_size = 10
infer_iterations = 50
quant_iterations = 5
self.run_test(
model_name,
'model.pdmodel',
'model.pdiparams',
data_url,
data_md5,
algo,
round_type,
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
batch_size,
infer_iterations,
quant_iterations,
)
class TestPostTrainingAbsMaxForMnist(TestPostTrainingQuantization):
def test_post_training_abs_max(self):
model_name = "mnist_model"
data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model_combined.tar.gz"
data_md5 = "a49251d3f555695473941e5a725c6014"
algo = "abs_max"
round_type = "round"
quantizable_op_type = ["conv2d", "mul"]
is_full_quantize = True
is_use_cache_file = False
is_optimize_model = True
diff_threshold = 0.01
batch_size = 10
infer_iterations = 50
quant_iterations = 10
self.run_test(
model_name,
'model.pdmodel',
'model.pdiparams',
data_url,
data_md5,
algo,
round_type,
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
batch_size,
infer_iterations,
quant_iterations,
)
class TestPostTrainingmseAdaroundForMnist(TestPostTrainingQuantization):
def test_post_training_mse(self):
model_name = "mnist_model"
data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model_combined.tar.gz"
data_md5 = "a49251d3f555695473941e5a725c6014"
algo = "mse"
round_type = "adaround"
quantizable_op_type = ["conv2d", "depthwise_conv2d", "mul"]
is_full_quantize = False
is_use_cache_file = False
is_optimize_model = True
diff_threshold = 0.01
batch_size = 10
infer_iterations = 50
quant_iterations = 5
bias_correction = True
self.run_test(
model_name,
'model.pdmodel',
'model.pdiparams',
data_url,
data_md5,
algo,
round_type,
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
batch_size,
infer_iterations,
quant_iterations,
bias_correction=bias_correction,
)
class TestPostTrainingKLAdaroundForMnist(TestPostTrainingQuantization):
def test_post_training_kl(self):
model_name = "mnist_model"
data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model_combined.tar.gz"
data_md5 = "a49251d3f555695473941e5a725c6014"
algo = "KL"
round_type = "adaround"
quantizable_op_type = ["conv2d", "depthwise_conv2d", "mul"]
is_full_quantize = False
is_use_cache_file = False
is_optimize_model = True
diff_threshold = 0.01
batch_size = 10
infer_iterations = 50
quant_iterations = 5
self.run_test(
model_name,
'model.pdmodel',
'model.pdiparams',
data_url,
data_md5,
algo,
round_type,
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
batch_size,
infer_iterations,
quant_iterations,
)
class TestPostTrainingmseForMnistONNXFormat(TestPostTrainingQuantization):
def test_post_training_mse_onnx_format(self):
model_name = "mnist_model"
data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model_combined.tar.gz"
data_md5 = "a49251d3f555695473941e5a725c6014"
algo = "mse"
round_type = "round"
quantizable_op_type = ["conv2d", "depthwise_conv2d", "mul"]
is_full_quantize = False
is_use_cache_file = False
is_optimize_model = True
onnx_format = True
diff_threshold = 0.01
batch_size = 10
infer_iterations = 50
quant_iterations = 5
self.run_test(
model_name,
'model.pdmodel',
'model.pdiparams',
data_url,
data_md5,
algo,
round_type,
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
batch_size,
infer_iterations,
quant_iterations,
onnx_format=onnx_format,
)
class TestPostTrainingmseForMnistONNXFormatFullQuant(
TestPostTrainingQuantization
):
def test_post_training_mse_onnx_format_full_quant(self):
model_name = "mnist_model"
data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model_combined.tar.gz"
data_md5 = "a49251d3f555695473941e5a725c6014"
algo = "mse"
round_type = "round"
quantizable_op_type = ["conv2d", "depthwise_conv2d", "mul"]
is_full_quantize = True
is_use_cache_file = False
is_optimize_model = False
onnx_format = True
diff_threshold = 0.01
batch_size = 10
infer_iterations = 50
quant_iterations = 5
self.run_test(
model_name,
'model.pdmodel',
'model.pdiparams',
data_url,
data_md5,
algo,
round_type,
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
batch_size,
infer_iterations,
quant_iterations,
onnx_format=onnx_format,
)
class TestPostTrainingavgForMnistSkipOP(TestPostTrainingQuantization):
def test_post_training_avg_skip_op(self):
model_name = "mnist_model"
data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model_combined.tar.gz"
data_md5 = "a49251d3f555695473941e5a725c6014"
algo = "avg"
round_type = "round"
quantizable_op_type = ["conv2d", "depthwise_conv2d", "mul"]
is_full_quantize = False
is_use_cache_file = False
is_optimize_model = True
diff_threshold = 0.01
batch_size = 10
infer_iterations = 50
quant_iterations = 5
skip_tensor_list = ["fc_0.w_0"]
self.run_test(
model_name,
'model.pdmodel',
'model.pdiparams',
data_url,
data_md5,
algo,
round_type,
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
batch_size,
infer_iterations,
quant_iterations,
skip_tensor_list=skip_tensor_list,
)
if __name__ == '__main__':
unittest.main()
test/quantization/test_post_training_quantization_mobilenetv1.py
+911
View File
@@ -0,0 +1,911 @@
# copyright (c) 2022 paddlepaddle authors. all rights reserved.
#
# licensed under the apache license, version 2.0 (the "license");
# you may not use this file except in compliance with the license.
# you may obtain a copy of the license at
#
# http://www.apache.org/licenses/license-2.0
#
# unless required by applicable law or agreed to in writing, software
# distributed under the license is distributed on an "as is" basis,
# without warranties or conditions of any kind, either express or implied.
# see the license for the specific language governing permissions and
# limitations under the license.
import functools
import logging
import os
import random
import sys
import tempfile
import time
import unittest
import numpy as np
from PIL import Image
import paddle
from paddle.dataset.common import download
from paddle.io import Dataset
from paddle.static.log_helper import get_logger
from paddle.static.quantization import PostTrainingQuantization
paddle.enable_static()
random.seed(0)
np.random.seed(0)
DATA_DIM = 224
THREAD = 1
BUF_SIZE = 102400
DATA_DIR = 'data/ILSVRC2012'
img_mean = np.array([0.485, 0.456, 0.406]).reshape((3, 1, 1))
img_std = np.array([0.229, 0.224, 0.225]).reshape((3, 1, 1))
_logger = get_logger(
__name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s'
)
def resize_short(img, target_size):
percent = float(target_size) / min(img.size[0], img.size[1])
resized_width = int(round(img.size[0] * percent))
resized_height = int(round(img.size[1] * percent))
img = img.resize((resized_width, resized_height), Image.LANCZOS)
return img
def crop_image(img, target_size, center):
width, height = img.size
size = target_size
if center is True:
w_start = (width - size) / 2
h_start = (height - size) / 2
else:
w_start = np.random.randint(0, width - size + 1)
h_start = np.random.randint(0, height - size + 1)
w_end = w_start + size
h_end = h_start + size
img = img.crop((w_start, h_start, w_end, h_end))
return img
def process_image(sample, mode, color_jitter, rotate):
img_path = sample[0]
img = Image.open(img_path)
img = resize_short(img, target_size=256)
img = crop_image(img, target_size=DATA_DIM, center=True)
if img.mode != 'RGB':
img = img.convert('RGB')
img = np.array(img).astype('float32').transpose((2, 0, 1)) / 255
img -= img_mean
img /= img_std
return img, sample[1]
def _reader_creator(
file_list,
mode,
shuffle=False,
color_jitter=False,
rotate=False,
data_dir=DATA_DIR,
):
def reader():
with open(file_list) as flist:
full_lines = [line.strip() for line in flist]
if shuffle:
np.random.shuffle(full_lines)
lines = full_lines
for line in lines:
img_path, label = line.split()
img_path = os.path.join(data_dir, img_path)
if not os.path.exists(img_path):
continue
yield img_path, int(label)
mapper = functools.partial(
process_image, mode=mode, color_jitter=color_jitter, rotate=rotate
)
return paddle.reader.xmap_readers(mapper, reader, THREAD, BUF_SIZE)
def val(data_dir=DATA_DIR):
file_list = os.path.join(data_dir, 'val_list.txt')
return _reader_creator(file_list, 'val', shuffle=False, data_dir=data_dir)
class ImageNetDataset(Dataset):
def __init__(self, data_dir=DATA_DIR, shuffle=False, need_label=False):
super().__init__()
self.need_label = need_label
self.data_dir = data_dir
val_file_list = os.path.join(data_dir, 'val_list.txt')
with open(val_file_list) as flist:
lines = [line.strip() for line in flist]
if shuffle:
np.random.shuffle(lines)
self.data = [line.split() for line in lines]
def __getitem__(self, index):
sample = self.data[index]
data_path = os.path.join(self.data_dir, sample[0])
data, label = process_image(
[data_path, sample[1]], mode='val', color_jitter=False, rotate=False
)
if self.need_label:
return data, np.array([label]).astype('int64')
else:
return data
def __len__(self):
return len(self.data)
class TestPostTrainingQuantization(unittest.TestCase):
def setUp(self):
self.int8_download = 'int8/download'
self.cache_folder = os.path.expanduser(
'~/.cache/paddle/dataset/' + self.int8_download
)
self.data_cache_folder = ''
data_urls = []
data_md5s = []
if os.environ.get('DATASET') == 'full':
data_urls.append(
'https://paddle-inference-dist.bj.bcebos.com/int8/ILSVRC2012_img_val.tar.gz.partaa'
)
data_md5s.append('60f6525b0e1d127f345641d75d41f0a8')
data_urls.append(
'https://paddle-inference-dist.bj.bcebos.com/int8/ILSVRC2012_img_val.tar.gz.partab'
)
data_md5s.append('1e9f15f64e015e58d6f9ec3210ed18b5')
self.data_cache_folder = self.download_data(
data_urls, data_md5s, "full_data", False
)
else:
data_urls.append(
'http://paddle-inference-dist.bj.bcebos.com/int8/calibration_test_data.tar.gz'
)
data_md5s.append('1b6c1c434172cca1bf9ba1e4d7a3157d')
self.data_cache_folder = self.download_data(
data_urls, data_md5s, "small_data", False
)
# reader/decorator.py requires the relative path to the data folder
if not os.path.exists("./data/ILSVRC2012"):
cmd = 'rm -rf {0} && ln -s {1} {0}'.format(
"data", self.data_cache_folder
)
os.system(cmd)
self.batch_size = 1 if os.environ.get('DATASET') == 'full' else 50
self.infer_iterations = (
50000 if os.environ.get('DATASET') == 'full' else 2
)
self.root_path = tempfile.TemporaryDirectory()
self.int8_model = os.path.join(
self.root_path.name, "post_training_quantization"
)
def tearDown(self):
self.root_path.cleanup()
def cache_unzipping(self, target_folder, zip_path):
if not os.path.exists(target_folder):
cmd = (
f'mkdir {target_folder} && tar xf {zip_path} -C {target_folder}'
)
os.system(cmd)
def download_data(self, data_urls, data_md5s, folder_name, is_model=True):
data_cache_folder = os.path.join(self.cache_folder, folder_name)
zip_path = ''
if os.environ.get('DATASET') == 'full':
file_names = []
for i in range(0, len(data_urls)):
download(data_urls[i], self.int8_download, data_md5s[i])
file_names.append(data_urls[i].split('/')[-1])
zip_path = os.path.join(
self.cache_folder, 'full_imagenet_val.tar.gz'
)
if not os.path.exists(zip_path):
cat_command = 'cat'
for file_name in file_names:
cat_command += ' ' + os.path.join(
self.cache_folder, file_name
)
cat_command += ' > ' + zip_path
os.system(cat_command)
if os.environ.get('DATASET') != 'full' or is_model:
download(data_urls[0], self.int8_download, data_md5s[0])
file_name = data_urls[0].split('/')[-1]
zip_path = os.path.join(self.cache_folder, file_name)
_logger.info(f'Data is downloaded at {zip_path}')
self.cache_unzipping(data_cache_folder, zip_path)
return data_cache_folder
def download_model(self):
pass
def run_program(
self,
model_path,
model_filename,
params_filename,
batch_size,
infer_iterations,
):
image_shape = [3, 224, 224]
place = paddle.CPUPlace()
exe = paddle.static.Executor(place)
[
infer_program,
feed_dict,
fetch_targets,
] = paddle.static.load_inference_model(
model_path,
exe,
model_filename=model_filename,
params_filename=params_filename,
)
val_reader = paddle.batch(val(), batch_size)
iterations = infer_iterations
test_info = []
cnt = 0
periods = []
for batch_id, data in enumerate(val_reader()):
image = np.array([x[0].reshape(image_shape) for x in data]).astype(
"float32"
)
label = np.array([x[1] for x in data]).astype("int64")
label = label.reshape([-1, 1])
t1 = time.time()
pred = exe.run(
infer_program,
feed={feed_dict[0]: image},
fetch_list=fetch_targets,
)
t2 = time.time()
period = t2 - t1
periods.append(period)
pred = np.array(pred[0])
sort_array = pred.argsort(axis=1)
top_1_pred = sort_array[:, -1:][:, ::-1]
top_1 = np.mean(label == top_1_pred)
test_info.append(np.mean(top_1) * len(data))
cnt += len(data)
if (batch_id + 1) % 100 == 0:
_logger.info(f"{batch_id + 1} images,")
sys.stdout.flush()
if (batch_id + 1) == iterations:
break
throughput = cnt / np.sum(periods)
latency = np.average(periods)
acc1 = np.sum(test_info) / cnt
return (throughput, latency, acc1, feed_dict)
def generate_quantized_model(
self,
model_path,
model_filename,
params_filename,
quantizable_op_type,
batch_size,
algo="KL",
round_type="round",
is_full_quantize=False,
is_use_cache_file=False,
is_optimize_model=False,
batch_nums=1,
onnx_format=False,
deploy_backend=None,
feed_name="inputs",
):
try:
os.system("mkdir " + self.int8_model)
except Exception as e:
_logger.info(f"Failed to create {self.int8_model} due to {e}")
sys.exit(-1)
place = paddle.CPUPlace()
exe = paddle.static.Executor(place)
image = paddle.static.data(
name=feed_name[0], shape=[None, 3, 224, 224], dtype='float32'
)
feed_list = [image]
if len(feed_name) == 2:
label = paddle.static.data(
name='label', shape=[None, 1], dtype='int64'
)
feed_list.append(label)
val_dataset = ImageNetDataset(need_label=len(feed_list) == 2)
data_loader = paddle.io.DataLoader(
val_dataset,
places=place,
feed_list=feed_list,
drop_last=False,
return_list=False,
batch_size=2,
shuffle=False,
)
ptq = PostTrainingQuantization(
executor=exe,
data_loader=data_loader,
model_dir=model_path,
model_filename=model_filename,
params_filename=params_filename,
batch_size=batch_size,
batch_nums=batch_nums,
algo=algo,
quantizable_op_type=quantizable_op_type,
round_type=round_type,
is_full_quantize=is_full_quantize,
optimize_model=is_optimize_model,
onnx_format=onnx_format,
is_use_cache_file=is_use_cache_file,
deploy_backend=deploy_backend,
)
ptq.quantize()
ptq.save_quantized_model(
self.int8_model,
model_filename=model_filename,
params_filename=params_filename,
)
def run_test(
self,
model,
model_filename,
params_filename,
algo,
round_type,
data_urls,
data_md5s,
data_name,
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
onnx_format=False,
batch_nums=1,
deploy_backend=None,
):
infer_iterations = self.infer_iterations
batch_size = self.batch_size
model_cache_folder = self.download_data(data_urls, data_md5s, model)
model_path = os.path.join(model_cache_folder, data_name)
_logger.info(
f"Start FP32 inference for {model} on {infer_iterations * batch_size} images ..."
)
(
fp32_throughput,
fp32_latency,
fp32_acc1,
feed_name,
) = self.run_program(
model_path,
model_filename,
params_filename,
batch_size,
infer_iterations,
)
self.generate_quantized_model(
model_path,
model_filename,
params_filename,
quantizable_op_type,
batch_size,
algo,
round_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
batch_nums,
onnx_format,
deploy_backend,
feed_name,
)
_logger.info(
f"Start INT8 inference for {model} on {infer_iterations * batch_size} images ..."
)
(int8_throughput, int8_latency, int8_acc1, _) = self.run_program(
self.int8_model,
model_filename,
params_filename,
batch_size,
infer_iterations,
)
_logger.info(f"---Post training quantization of {algo} method---")
_logger.info(
f"FP32 {model}: batch_size {batch_size}, throughput {fp32_throughput} images/second, latency {fp32_latency} second, accuracy {fp32_acc1}."
)
_logger.info(
f"INT8 {model}: batch_size {batch_size}, throughput {int8_throughput} images/second, latency {int8_latency} second, accuracy {int8_acc1}.\n"
)
sys.stdout.flush()
delta_value = fp32_acc1 - int8_acc1
self.assertLess(delta_value, diff_threshold)
class TestPostTrainingKLForMobilenetv1(TestPostTrainingQuantization):
def test_post_training_kl_mobilenetv1(self):
model = "MobileNet-V1"
algo = "KL"
round_type = "round"
data_urls = [
'https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/inference/MobileNetV1_infer.tar'
]
data_md5s = ['5ee2b1775b11dc233079236cdc216c2e']
quantizable_op_type = [
"conv2d",
"depthwise_conv2d",
"mul",
"pool2d",
]
is_full_quantize = False
is_use_cache_file = False
is_optimize_model = True
diff_threshold = 0.025
batch_nums = 2
self.run_test(
model,
'inference.pdmodel',
'inference.pdiparams',
algo,
round_type,
data_urls,
data_md5s,
"MobileNetV1_infer",
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
)
class TestPostTrainingavgForMobilenetv1(TestPostTrainingQuantization):
def test_post_training_avg_mobilenetv1(self):
model = "MobileNet-V1"
algo = "avg"
round_type = "round"
data_urls = [
'https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/inference/MobileNetV1_infer.tar'
]
data_md5s = ['5ee2b1775b11dc233079236cdc216c2e']
quantizable_op_type = [
"conv2d",
"depthwise_conv2d",
"mul",
]
is_full_quantize = False
is_use_cache_file = False
is_optimize_model = True
diff_threshold = 0.025
self.run_test(
model,
'inference.pdmodel',
'inference.pdiparams',
algo,
round_type,
data_urls,
data_md5s,
"MobileNetV1_infer",
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
batch_nums=2,
)
class TestPostTrainingavgForPwganCsmsc(TestPostTrainingQuantization):
def setUp(self):
self.int8_download = 'int8/download'
self.cache_folder = os.path.expanduser(
'~/.cache/paddle/dataset/' + self.int8_download
)
self.data_cache_folder = ''
data_urls = []
data_md5s = []
if os.environ.get('DATASET') == 'full':
data_urls.append(
'https://paddlespeech.bj.bcebos.com/tmp/csmsc_voc1.npy'
)
data_md5s.append('47950146167ca8d885a78d71e74f1a2b')
self.data_cache_folder = self.download_data(
data_urls, data_md5s, "full_data", False
)
else:
data_urls.append(
'https://paddlespeech.bj.bcebos.com/tmp/csmsc_voc1.npy'
)
data_md5s.append('47950146167ca8d885a78d71e74f1a2b')
self.data_cache_folder = self.download_data(
data_urls, data_md5s, "small_data", False
)
# reader/decorator.py requires the relative path to the data folder
if not os.path.exists("./data/BZNSYP"):
cmd = 'rm -rf {0} && ln -s {1} {0}'.format(
"data", self.data_cache_folder
)
os.system(cmd)
self.batch_size = 1 if os.environ.get('DATASET') == 'full' else 50
self.infer_iterations = (
50000 if os.environ.get('DATASET') == 'full' else 2
)
self.root_path = tempfile.TemporaryDirectory()
self.int8_model = os.path.join(
self.root_path.name, "post_training_quantization"
)
def cache_unzipping(self, target_folder, zip_path):
if not os.path.exists(target_folder):
if zip_path.endswith('.tar.gz'):
cmd = f'mkdir {target_folder} && tar xf {zip_path} -C {target_folder}'
elif zip_path.endswith('.zip'):
cmd = f'mkdir {target_folder} && unzip -o {zip_path} -d {target_folder}'
else:
cmd = f'mkdir {target_folder}'
os.system(cmd)
def generate_quantized_model(
self,
model_path,
model_filename,
params_filename,
quantizable_op_type,
batch_size,
algo="avg",
round_type="round",
is_full_quantize=False,
is_use_cache_file=False,
is_optimize_model=False,
batch_nums=1,
onnx_format=False,
deploy_backend=None,
feed_name="inputs",
):
try:
os.system("mkdir " + self.int8_model)
except Exception as e:
_logger.info(f"Failed to create {self.int8_model} due to {e}")
sys.exit(-1)
place = paddle.CPUPlace()
exe = paddle.static.Executor(place)
val_dataset = "~/.cache/paddle/dataset/int8/download/csmsc_voc1.npy"
data_loader = paddle.io.DataLoader(
val_dataset,
places=place,
drop_last=False,
batch_size=2,
)
ptq = PostTrainingQuantization(
executor=exe,
data_loader=data_loader,
model_dir=model_path,
model_filename=model_filename,
params_filename=params_filename,
batch_size=batch_size,
batch_nums=batch_nums,
algo=algo,
quantizable_op_type=quantizable_op_type,
round_type=round_type,
is_full_quantize=is_full_quantize,
optimize_model=is_optimize_model,
onnx_format=onnx_format,
is_use_cache_file=is_use_cache_file,
deploy_backend=deploy_backend,
)
ptq.quantize()
ptq.save_quantized_model(
self.int8_model,
model_filename=model_filename,
params_filename=params_filename,
)
class TestPostTraininghistForNoneShape(TestPostTrainingavgForPwganCsmsc):
def test_post_training_avg_pwgancsmsc(self):
model = "pwg_baker_static_0.4"
algo = "avg"
round_type = "round"
data_urls = [
'https://paddlespeech.bj.bcebos.com/Parakeet/released_models/pwgan/pwg_baker_static_0.4.zip'
]
data_md5s = ['e3504aed9c5a290be12d1347836d2742']
quantizable_op_type = [
"conv2d",
"depthwise_conv2d",
"mul",
]
is_full_quantize = False
is_use_cache_file = False
is_optimize_model = True
diff_threshold = 0.05
self.run_test(
model,
'pwgan_csmsc.pdmodel',
'pwgan_csmsc.pdiparams',
algo,
round_type,
data_urls,
data_md5s,
"pwg_baker_static_0.4",
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
batch_nums=2,
)
class TestPostTraininghistForMobilenetv1(TestPostTrainingQuantization):
def test_post_training_hist_mobilenetv1(self):
model = "MobileNet-V1"
algo = "hist"
round_type = "round"
data_urls = [
'https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/inference/MobileNetV1_infer.tar'
]
data_md5s = ['5ee2b1775b11dc233079236cdc216c2e']
quantizable_op_type = [
"conv2d",
"depthwise_conv2d",
"mul",
]
is_full_quantize = False
is_use_cache_file = False
is_optimize_model = True
diff_threshold = 0.03
batch_nums = 1
self.run_test(
model,
'inference.pdmodel',
'inference.pdiparams',
algo,
round_type,
data_urls,
data_md5s,
"MobileNetV1_infer",
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
batch_nums=batch_nums,
)
class TestPostTrainingAbsMaxForMobilenetv1(TestPostTrainingQuantization):
def test_post_training_abs_max_mobilenetv1(self):
model = "MobileNet-V1"
algo = "abs_max"
round_type = "round"
data_urls = [
'https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/inference/MobileNetV1_infer.tar'
]
data_md5s = ['5ee2b1775b11dc233079236cdc216c2e']
quantizable_op_type = [
"conv2d",
"mul",
]
is_full_quantize = False
is_use_cache_file = False
is_optimize_model = False
# The accuracy diff of post-training quantization (abs_max) maybe bigger
diff_threshold = 0.05
self.run_test(
model,
'inference.pdmodel',
'inference.pdiparams',
algo,
round_type,
data_urls,
data_md5s,
"MobileNetV1_infer",
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
)
class TestPostTrainingAvgONNXFormatForMobilenetv1(TestPostTrainingQuantization):
def test_post_training_onnx_format_mobilenetv1(self):
model = "MobileNet-V1"
algo = "emd"
round_type = "round"
data_urls = [
'https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/inference/MobileNetV1_infer.tar'
]
data_md5s = ['5ee2b1775b11dc233079236cdc216c2e']
quantizable_op_type = [
"conv2d",
"depthwise_conv2d",
"mul",
]
is_full_quantize = False
is_use_cache_file = False
is_optimize_model = True
onnx_format = True
diff_threshold = 0.05
batch_nums = 1
self.run_test(
model,
'inference.pdmodel',
'inference.pdiparams',
algo,
round_type,
data_urls,
data_md5s,
"MobileNetV1_infer",
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
onnx_format=onnx_format,
batch_nums=batch_nums,
)
class TestPostTrainingAvgONNXFormatForMobilenetv1TensorRT(
TestPostTrainingQuantization
):
def test_post_training_onnx_format_mobilenetv1_tensorrt(self):
model = "MobileNet-V1"
algo = "KL"
round_type = "round"
data_urls = [
'https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/inference/MobileNetV1_infer.tar'
]
data_md5s = ['5ee2b1775b11dc233079236cdc216c2e']
quantizable_op_type = [
"conv2d",
"depthwise_conv2d",
"mul",
]
is_full_quantize = False
is_use_cache_file = False
is_optimize_model = False
onnx_format = True
diff_threshold = 0.05
batch_nums = 12
deploy_backend = "tensorrt"
self.run_test(
model,
'inference.pdmodel',
'inference.pdiparams',
algo,
round_type,
data_urls,
data_md5s,
"MobileNetV1_infer",
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
onnx_format=onnx_format,
batch_nums=batch_nums,
deploy_backend=deploy_backend,
)
class TestPostTrainingKLONNXFormatForMobilenetv1ONEDNN(
TestPostTrainingQuantization
):
def test_post_training_onnx_format_mobilenetv1_onednn(self):
model = "MobileNet-V1"
algo = "ptf"
round_type = "round"
data_urls = [
'https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/inference/MobileNetV1_infer.tar'
]
data_md5s = ['5ee2b1775b11dc233079236cdc216c2e']
quantizable_op_type = [
"conv2d",
"depthwise_conv2d",
"mul",
]
is_full_quantize = False
is_use_cache_file = False
is_optimize_model = False
onnx_format = True
diff_threshold = 0.05
batch_nums = 12
deploy_backend = "onednn"
self.run_test(
model,
'inference.pdmodel',
'inference.pdiparams',
algo,
round_type,
data_urls,
data_md5s,
"MobileNetV1_infer",
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
onnx_format=onnx_format,
batch_nums=batch_nums,
deploy_backend=deploy_backend,
)
class TestPostTrainingAvgONNXFormatForMobilenetv1ARMCPU(
TestPostTrainingQuantization
):
def test_post_training_onnx_format_mobilenetv1_armcpu(self):
model = "MobileNet-V1"
algo = "avg"
round_type = "round"
data_urls = [
'https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/inference/MobileNetV1_infer.tar'
]
data_md5s = ['5ee2b1775b11dc233079236cdc216c2e']
quantizable_op_type = [
"conv2d",
"depthwise_conv2d",
"mul",
]
is_full_quantize = False
is_use_cache_file = False
is_optimize_model = True
onnx_format = True
diff_threshold = 0.05
batch_nums = 1
deploy_backend = "arm"
self.run_test(
model,
'inference.pdmodel',
'inference.pdiparams',
algo,
round_type,
data_urls,
data_md5s,
"MobileNetV1_infer",
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
onnx_format=onnx_format,
batch_nums=batch_nums,
deploy_backend=deploy_backend,
)
if __name__ == '__main__':
unittest.main()
test/quantization/test_post_training_quantization_program_resnet50.py
+352
View File
@@ -0,0 +1,352 @@
# copyright (c) 2018 paddlepaddle authors. all rights reserved.
#
# licensed under the apache license, version 2.0 (the "license");
# you may not use this file except in compliance with the license.
# you may obtain a copy of the license at
#
# http://www.apache.org/licenses/license-2.0
#
# unless required by applicable law or agreed to in writing, software
# distributed under the license is distributed on an "as is" basis,
# without warranties or conditions of any kind, either express or implied.
# see the license for the specific language governing permissions and
# limitations under the license.
import functools
import os
import random
import sys
import time
import unittest
import numpy as np
from PIL import Image
from test_post_training_quantization_mobilenetv1 import (
TestPostTrainingQuantization,
)
import paddle
from paddle.static.quantization import PostTrainingQuantizationProgram
paddle.enable_static()
random.seed(0)
np.random.seed(0)
THREAD = 1
DATA_DIM = 224
BUF_SIZE = 102400
DATA_DIR = 'data/ILSVRC2012'
img_mean = np.array([0.485, 0.456, 0.406]).reshape((3, 1, 1))
img_std = np.array([0.229, 0.224, 0.225]).reshape((3, 1, 1))
def resize_short(img, target_size):
percent = float(target_size) / min(img.size[0], img.size[1])
resized_width = int(round(img.size[0] * percent))
resized_height = int(round(img.size[1] * percent))
img = img.resize((resized_width, resized_height), Image.LANCZOS)
return img
def crop_image(img, target_size, center):
width, height = img.size
size = target_size
if center is True:
w_start = (width - size) / 2
h_start = (height - size) / 2
else:
w_start = np.random.randint(0, width - size + 1)
h_start = np.random.randint(0, height - size + 1)
w_end = w_start + size
h_end = h_start + size
img = img.crop((w_start, h_start, w_end, h_end))
return img
def process_image(sample, mode, color_jitter, rotate):
img_path = sample[0]
img = Image.open(img_path)
img = resize_short(img, target_size=256)
img = crop_image(img, target_size=DATA_DIM, center=True)
if img.mode != 'RGB':
img = img.convert('RGB')
img = np.array(img).astype('float32').transpose((2, 0, 1)) / 255
img -= img_mean
img /= img_std
return img, sample[1]
def _reader_creator(
file_list,
mode,
shuffle=False,
color_jitter=False,
rotate=False,
data_dir=DATA_DIR,
):
def reader():
with open(file_list) as flist:
full_lines = [line.strip() for line in flist]
if shuffle:
np.random.shuffle(full_lines)
lines = full_lines
for line in lines:
img_path, label = line.split()
img_path = os.path.join(data_dir, img_path)
if not os.path.exists(img_path):
continue
yield img_path, int(label)
mapper = functools.partial(
process_image, mode=mode, color_jitter=color_jitter, rotate=rotate
)
return paddle.reader.xmap_readers(mapper, reader, THREAD, BUF_SIZE)
def val(data_dir=DATA_DIR):
file_list = os.path.join(data_dir, 'val_list.txt')
return _reader_creator(file_list, 'val', shuffle=False, data_dir=data_dir)
class TestPostTrainingQuantizationProgram(TestPostTrainingQuantization):
def run_program(
self,
model_path,
model_filename,
params_filename,
batch_size,
infer_iterations,
):
image_shape = [3, 224, 224]
place = paddle.CPUPlace()
exe = paddle.static.Executor(place)
[
infer_program,
feed_dict,
fetch_targets,
] = paddle.static.load_inference_model(
model_path,
exe,
model_filename=model_filename,
params_filename=params_filename,
)
val_reader = paddle.batch(val(), batch_size)
iterations = infer_iterations
test_info = []
cnt = 0
periods = []
for batch_id, data in enumerate(val_reader()):
image = np.array([x[0].reshape(image_shape) for x in data]).astype(
"float32"
)
label = np.array([x[1] for x in data]).astype("int64")
label = label.reshape([-1, 1])
t1 = time.time()
_, acc1, _ = exe.run(
infer_program,
feed={feed_dict[0]: image, feed_dict[1]: label},
fetch_list=fetch_targets,
)
t2 = time.time()
period = t2 - t1
periods.append(period)
test_info.append(np.mean(acc1) * len(data))
cnt += len(data)
if (batch_id + 1) % 100 == 0:
print(f"{batch_id + 1} images,")
sys.stdout.flush()
if (batch_id + 1) == iterations:
break
throughput = cnt / np.sum(periods)
latency = np.average(periods)
acc1 = np.sum(test_info) / cnt
[
infer_program,
feed_dict,
fetch_targets,
] = paddle.static.load_inference_model(
model_path,
exe,
model_filename=model_filename,
params_filename=params_filename,
)
return (
throughput,
latency,
acc1,
infer_program,
feed_dict,
fetch_targets,
)
def generate_quantized_model(
self,
program,
quantizable_op_type,
feed_list,
fetch_list,
algo="KL",
round_type="round",
is_full_quantize=False,
is_use_cache_file=False,
is_optimize_model=False,
onnx_format=False,
):
try:
os.system("mkdir " + self.int8_model)
except Exception as e:
print(f"Failed to create {self.int8_model} due to {e}")
sys.exit(-1)
place = paddle.CPUPlace()
exe = paddle.static.Executor(place)
val_reader = val()
same_scale_tensor_list = [
['batch_norm_3.tmp_2#/#1', 'batch_norm_4.tmp_2#*#1'],
['batch_norm_27.tmp_2', 'batch_norm_26.tmp_2'],
[
'test_scale_name_not_in_scale_dict1',
'test_scale_name_not_in_scale_dict2',
],
[
'test_scale_name_not_in_scale_dict1#/#1',
'test_scale_name_not_in_scale_dict2#/#1',
],
]
ptq = PostTrainingQuantizationProgram(
executor=exe,
program=program,
sample_generator=val_reader,
batch_nums=10,
algo=algo,
quantizable_op_type=quantizable_op_type,
round_type=round_type,
is_full_quantize=is_full_quantize,
optimize_model=is_optimize_model,
onnx_format=onnx_format,
is_use_cache_file=is_use_cache_file,
feed_list=feed_list,
fetch_list=fetch_list,
same_scale_tensor_list=same_scale_tensor_list,
)
ptq.quantize()
ptq.save_quantized_model(self.int8_model)
def run_test(
self,
model,
model_filename,
params_filename,
algo,
round_type,
data_urls,
data_md5s,
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
onnx_format=False,
):
infer_iterations = self.infer_iterations
batch_size = self.batch_size
model_cache_folder = self.download_data(data_urls, data_md5s, model)
print(
f"Start FP32 inference for {model} on {infer_iterations * batch_size} images ..."
)
(
fp32_throughput,
fp32_latency,
fp32_acc1,
infer_program,
feed_dict,
fetch_targets,
) = self.run_program(
os.path.join(model_cache_folder, "model"),
model_filename,
params_filename,
batch_size,
infer_iterations,
)
self.generate_quantized_model(
infer_program,
quantizable_op_type,
feed_dict,
fetch_targets,
algo,
round_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
onnx_format,
)
print(
f"Start INT8 inference for {model} on {infer_iterations * batch_size} images ..."
)
(int8_throughput, int8_latency, int8_acc1, _, _, _) = self.run_program(
self.int8_model,
model_filename,
params_filename,
batch_size,
infer_iterations,
)
print(f"---Post training quantization of {algo} method---")
print(
f"FP32 {model}: batch_size {batch_size}, throughput {fp32_throughput} images/second, latency {fp32_latency} second, accuracy {fp32_acc1}."
)
print(
f"INT8 {model}: batch_size {batch_size}, throughput {int8_throughput} images/second, latency {int8_latency} second, accuracy {int8_acc1}.\n"
)
sys.stdout.flush()
delta_value = fp32_acc1 - int8_acc1
self.assertLess(delta_value, diff_threshold)
class TestPostTrainingProgramAbsMaxForResnet50(
TestPostTrainingQuantizationProgram
):
def test_post_training_abs_max_resnet50(self):
model = "ResNet-50"
algo = "abs_max"
round_type = "round"
data_urls = [
'http://paddle-inference-dist.bj.bcebos.com/int8/resnet50_int8_model_combined.tar.gz'
]
data_md5s = ['db212fd4e9edc83381aef4533107e60c']
quantizable_op_type = ["conv2d", "mul"]
is_full_quantize = False
is_use_cache_file = False
is_optimize_model = False
diff_threshold = 0.025
self.run_test(
model,
'model.pdmodel',
'model.pdiparams',
algo,
round_type,
data_urls,
data_md5s,
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
)
if __name__ == '__main__':
unittest.main()
test/quantization/test_post_training_quantization_resnet50.py
+153
View File
@@ -0,0 +1,153 @@
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import time
import unittest
import numpy as np
from test_post_training_quantization_mobilenetv1 import (
TestPostTrainingQuantization,
val,
)
import paddle
paddle.enable_static()
class TestPostTrainingForResnet50(TestPostTrainingQuantization):
def test_post_training_resnet50(self):
model = "ResNet-50"
algo = "min_max"
round_type = "round"
data_urls = [
'http://paddle-inference-dist.bj.bcebos.com/int8/resnet50_int8_model_combined.tar.gz'
]
data_md5s = ['db212fd4e9edc83381aef4533107e60c']
quantizable_op_type = ["conv2d", "mul"]
is_full_quantize = False
is_use_cache_file = False
is_optimize_model = False
diff_threshold = 0.025
self.run_test(
model,
'model.pdmodel',
'model.pdiparams',
algo,
round_type,
data_urls,
data_md5s,
"model",
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
)
def run_program(
self,
model_path,
model_filename,
params_filename,
batch_size,
infer_iterations,
):
image_shape = [3, 224, 224]
place = paddle.CPUPlace()
exe = paddle.static.Executor(place)
[
infer_program,
feed_dict,
fetch_targets,
] = paddle.static.load_inference_model(
model_path,
exe,
model_filename=model_filename,
params_filename=params_filename,
)
val_reader = paddle.batch(val(), batch_size)
iterations = infer_iterations
test_info = []
cnt = 0
periods = []
for batch_id, data in enumerate(val_reader()):
image = np.array([x[0].reshape(image_shape) for x in data]).astype(
"float32"
)
label = np.array([x[1] for x in data]).astype("int64")
label = label.reshape([-1, 1])
t1 = time.time()
_, acc1, _ = exe.run(
infer_program,
feed={feed_dict[0]: image, feed_dict[1]: label},
fetch_list=fetch_targets,
)
t2 = time.time()
period = t2 - t1
periods.append(period)
test_info.append(np.mean(acc1) * len(data))
cnt += len(data)
if (batch_id + 1) % 100 == 0:
print(f"{batch_id + 1} images,")
sys.stdout.flush()
if (batch_id + 1) == iterations:
break
throughput = cnt / np.sum(periods)
latency = np.average(periods)
acc1 = np.sum(test_info) / cnt
return (throughput, latency, acc1, feed_dict)
class TestPostTrainingForResnet50ONNXFormat(TestPostTrainingForResnet50):
def test_post_training_resnet50(self):
model = "ResNet-50"
algo = "min_max"
round_type = "round"
data_urls = [
'http://paddle-inference-dist.bj.bcebos.com/int8/resnet50_int8_model_combined.tar.gz'
]
data_md5s = ['db212fd4e9edc83381aef4533107e60c']
quantizable_op_type = ["conv2d", "mul"]
is_full_quantize = False
is_use_cache_file = False
is_optimize_model = False
diff_threshold = 0.025
onnx_format = True
self.run_test(
model,
'model.pdmodel',
'model.pdiparams',
algo,
round_type,
data_urls,
data_md5s,
"model",
quantizable_op_type,
is_full_quantize,
is_use_cache_file,
is_optimize_model,
diff_threshold,
onnx_format=onnx_format,
)
if __name__ == '__main__':
unittest.main()
test/quantization/test_ptq.py
+297
View File
@@ -0,0 +1,297 @@
# copyright (c) 2023 paddlepaddle authors. all rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import tempfile
import unittest
import numpy as np
import paddle
import paddle.nn.functional as F
from paddle.nn import Conv2D, Linear, ReLU, Sequential
from paddle.nn.quant.format import LinearDequanter, LinearQuanter
from paddle.quantization import PTQ, QuantConfig
from paddle.quantization.observers import AbsmaxObserver
from paddle.quantization.observers.abs_max import AbsmaxObserverLayer
class LeNetDygraph(paddle.nn.Layer):
def __init__(self, num_classes=10):
super().__init__()
self.num_classes = num_classes
self.features = Sequential(
Conv2D(1, 6, 3, stride=1, padding=1),
ReLU(),
paddle.nn.MaxPool2D(2, 2),
Conv2D(6, 16, 5, stride=1, padding=0),
ReLU(),
paddle.nn.MaxPool2D(2, 2),
)
if num_classes > 0:
self.fc = Sequential(
Linear(576, 120), Linear(120, 84), Linear(84, 10)
)
def forward(self, inputs):
x = self.features(inputs)
if self.num_classes > 0:
x = paddle.flatten(x, 1)
x = self.fc(x)
out = F.relu(x)
return out
class TestPTQ(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.path = os.path.join(self.temp_dir.name, 'ptq')
def tearDown(self):
self.temp_dir.cleanup()
def _get_model_for_ptq(self):
observer = AbsmaxObserver(quant_bits=8)
model = LeNetDygraph()
model.eval()
q_config = QuantConfig(activation=observer, weight=observer)
ptq = PTQ(q_config)
quant_model = ptq.quantize(model)
return quant_model, ptq
def _count_layers(self, model, layer_type):
count = 0
for _layer in model.sublayers(True):
if isinstance(_layer, layer_type):
count += 1
return count
def test_quantize(self):
ptq_model, _ = self._get_model_for_ptq()
image = paddle.rand([1, 1, 32, 32], dtype="float32")
out = ptq_model(image)
self.assertIsNotNone(out)
observer_count = self._count_layers(ptq_model, AbsmaxObserverLayer)
self.assertEqual(observer_count, 14)
def test_convert(self):
quant_model, ptq = self._get_model_for_ptq()
image = paddle.rand([1, 1, 32, 32], dtype="float32")
out = quant_model(image)
converted_model = ptq.convert(quant_model)
out = converted_model(image)
self.assertIsNotNone(out)
observer_count = self._count_layers(
converted_model, AbsmaxObserverLayer
)
quanter_count = self._count_layers(converted_model, LinearQuanter)
dequanter_count = self._count_layers(converted_model, LinearDequanter)
self.assertEqual(observer_count, 0)
self.assertEqual(dequanter_count, 14)
self.assertEqual(quanter_count, 9)
save_path = os.path.join(self.temp_dir.name, 'int8_infer')
paddle.jit.save(converted_model, save_path, [image])
paddle.enable_static()
exe = paddle.static.Executor(paddle.CPUPlace())
main_program = paddle.static.Program()
startup_program = paddle.static.Program()
with paddle.static.program_guard(main_program, startup_program):
[
inference_program,
feed_target_names,
fetch_targets,
] = paddle.static.load_inference_model(save_path, exe)
tensor_img = np.array(
np.random.random((1, 1, 32, 32)), dtype=np.float32
)
results = exe.run(
inference_program,
feed={feed_target_names[0]: tensor_img},
fetch_list=fetch_targets,
)
self.assertIsNotNone(results)
paddle.disable_static()
def test_convert_2times(self):
quant_model, ptq = self._get_model_for_ptq()
image = paddle.rand([1, 1, 32, 32], dtype="float32")
out = quant_model(image)
converted_model = ptq.convert(quant_model)
converted_model = ptq.convert(converted_model)
out = converted_model(image)
self.assertIsNotNone(out)
observer_count = self._count_layers(
converted_model, AbsmaxObserverLayer
)
quanter_count = self._count_layers(converted_model, LinearQuanter)
dequanter_count = self._count_layers(converted_model, LinearDequanter)
self.assertEqual(observer_count, 0)
self.assertEqual(dequanter_count, 14)
self.assertEqual(quanter_count, 9)
save_path = os.path.join(self.temp_dir.name, 'int8_infer')
paddle.jit.save(converted_model, save_path, [image])
paddle.enable_static()
exe = paddle.static.Executor(paddle.CPUPlace())
main_program = paddle.static.Program()
startup_program = paddle.static.Program()
with paddle.static.program_guard(main_program, startup_program):
[
inference_program,
feed_target_names,
fetch_targets,
] = paddle.static.load_inference_model(save_path, exe)
tensor_img = np.array(
np.random.random((1, 1, 32, 32)), dtype=np.float32
)
results = exe.run(
inference_program,
feed={feed_target_names[0]: tensor_img},
fetch_list=fetch_targets,
)
self.assertIsNotNone(results)
paddle.disable_static()
class TestPTQFP8(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.path = os.path.join(self.temp_dir.name, 'ptq')
def tearDown(self):
self.temp_dir.cleanup()
def _get_model_for_ptq(self):
weight_observer = AbsmaxObserver(quant_bits=(4, 3))
act_observer = AbsmaxObserver(quant_bits=(5, 2))
model = LeNetDygraph()
model.eval()
q_config = QuantConfig(activation=act_observer, weight=weight_observer)
ptq = PTQ(q_config)
quant_model = ptq.quantize(model)
return quant_model, ptq
def _count_layers(self, model, layer_type):
count = 0
for _layer in model.sublayers(True):
if isinstance(_layer, layer_type):
count += 1
return count
def test_quantize(self):
ptq_model, _ = self._get_model_for_ptq()
image = paddle.rand([1, 1, 32, 32], dtype="float32")
out = ptq_model(image)
self.assertIsNotNone(out)
observer_count = self._count_layers(ptq_model, AbsmaxObserverLayer)
self.assertEqual(observer_count, 14)
def test_convert(self):
quant_model, ptq = self._get_model_for_ptq()
image = paddle.rand([1, 1, 32, 32], dtype="float32")
out = quant_model(image)
converted_model = ptq.convert(quant_model)
out = converted_model(image)
self.assertIsNotNone(out)
observer_count = self._count_layers(
converted_model, AbsmaxObserverLayer
)
quanter_count = self._count_layers(converted_model, LinearQuanter)
dequanter_count = self._count_layers(converted_model, LinearDequanter)
self.assertEqual(observer_count, 0)
self.assertEqual(dequanter_count, 14)
self.assertEqual(quanter_count, 9)
save_path = os.path.join(self.temp_dir.name, 'int8_infer')
paddle.jit.save(converted_model, save_path, [image])
paddle.enable_static()
exe = paddle.static.Executor(paddle.CPUPlace())
main_program = paddle.static.Program()
startup_program = paddle.static.Program()
with paddle.static.program_guard(main_program, startup_program):
[
inference_program,
feed_target_names,
fetch_targets,
] = paddle.static.load_inference_model(save_path, exe)
tensor_img = np.array(
np.random.random((1, 1, 32, 32)), dtype=np.float32
)
results = exe.run(
inference_program,
feed={feed_target_names[0]: tensor_img},
fetch_list=fetch_targets,
)
self.assertIsNotNone(results)
paddle.disable_static()
def test_convert_2times(self):
quant_model, ptq = self._get_model_for_ptq()
image = paddle.rand([1, 1, 32, 32], dtype="float32")
out = quant_model(image)
converted_model = ptq.convert(quant_model)
converted_model = ptq.convert(converted_model)
out = converted_model(image)
self.assertIsNotNone(out)
observer_count = self._count_layers(
converted_model, AbsmaxObserverLayer
)
quanter_count = self._count_layers(converted_model, LinearQuanter)
dequanter_count = self._count_layers(converted_model, LinearDequanter)
self.assertEqual(observer_count, 0)
self.assertEqual(dequanter_count, 14)
self.assertEqual(quanter_count, 9)
save_path = os.path.join(self.temp_dir.name, 'int8_infer')
paddle.jit.save(converted_model, save_path, [image])
paddle.enable_static()
exe = paddle.static.Executor(paddle.CPUPlace())
main_program = paddle.static.Program()
startup_program = paddle.static.Program()
with paddle.static.program_guard(main_program, startup_program):
[
inference_program,
feed_target_names,
fetch_targets,
] = paddle.static.load_inference_model(save_path, exe)
tensor_img = np.array(
np.random.random((1, 1, 32, 32)), dtype=np.float32
)
results = exe.run(
inference_program,
feed={feed_target_names[0]: tensor_img},
fetch_list=fetch_targets,
)
self.assertIsNotNone(results)
paddle.disable_static()
if __name__ == '__main__':
unittest.main()
test/quantization/test_qat.py
+100
View File
@@ -0,0 +1,100 @@
# copyright (c) 2022 paddlepaddle authors. all rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import numpy as np
import paddle
import paddle.nn.functional as F
from paddle.io import Dataset
from paddle.nn import Conv2D, Linear, ReLU, Sequential
from paddle.quantization import QAT, QuantConfig
from paddle.quantization.quanters import FakeQuanterWithAbsMaxObserver
from paddle.quantization.quanters.abs_max import (
FakeQuanterWithAbsMaxObserverLayer,
)
class RandomDataset(Dataset):
def __init__(self, num_samples):
self.num_samples = num_samples
def __getitem__(self, idx):
data = np.random.random([3, 32, 32]).astype('float32')
return data
def __len__(self):
return self.num_samples
class Model(paddle.nn.Layer):
def __init__(self, num_classes=10):
super().__init__()
self.num_classes = num_classes
self.features = Sequential(
Conv2D(3, 6, 3, stride=1, padding=1),
ReLU(),
paddle.nn.MaxPool2D(2, stride=2),
Conv2D(6, 16, 5, stride=1, padding=0),
ReLU(),
paddle.nn.MaxPool2D(2, stride=2),
)
if num_classes > 0:
self.fc = Sequential(
Linear(576, 120), Linear(120, 84), Linear(84, 10)
)
def forward(self, inputs):
x = self.features(inputs)
if self.num_classes > 0:
x = paddle.flatten(x, 1)
x = self.fc(x)
out = F.relu(x)
return out
class TestQAT(unittest.TestCase):
def test_qat(self):
nums_batch = 100
batch_size = 32
dataset = RandomDataset(nums_batch * batch_size)
loader = paddle.io.DataLoader(
dataset,
batch_size=batch_size,
shuffle=False,
drop_last=True,
num_workers=0,
)
model = Model()
quanter = FakeQuanterWithAbsMaxObserver(moving_rate=0.9)
q_config = QuantConfig(activation=quanter, weight=quanter)
qat = QAT(q_config)
print(model)
quant_model = qat.quantize(model)
print(quant_model)
quanter_count = 0
for _layer in quant_model.sublayers(True):
if isinstance(_layer, FakeQuanterWithAbsMaxObserverLayer):
quanter_count += 1
self.assertEqual(quanter_count, 14)
for _, data in enumerate(loader):
out = quant_model(data)
out.backward()
if __name__ == '__main__':
unittest.main()
test/quantization/test_quant.py
+151
View File
@@ -0,0 +1,151 @@
# copyright (c) 2022 paddlepaddle authors. all rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import paddle
import paddle.nn.functional as F
from paddle.nn import Conv2D, Linear, ReLU, Sequential
from paddle.quantization import QuantConfig
from paddle.quantization.base_quanter import BaseQuanter
from paddle.quantization.quanters import FakeQuanterWithAbsMaxObserver
class LeNetDygraph(paddle.nn.Layer):
def __init__(self, num_classes=10):
super().__init__()
self.num_classes = num_classes
self.features = Sequential(
Conv2D(3, 6, 3, stride=1, padding=1),
ReLU(),
paddle.nn.MaxPool2D(2, 2),
Conv2D(6, 16, 5, stride=1, padding=0),
ReLU(),
paddle.nn.MaxPool2D(2, 2),
)
if num_classes > 0:
self.fc = Sequential(
Linear(576, 120), Linear(120, 84), Linear(84, 10)
)
def forward(self, inputs):
x = self.features(inputs)
if self.num_classes > 0:
x = paddle.flatten(x, 1)
x = self.fc(x)
out = F.relu(x)
out = F.relu(out)
return out
class TestQuantConfig(unittest.TestCase):
def setUp(self):
self.model = LeNetDygraph()
self.quanter = FakeQuanterWithAbsMaxObserver(moving_rate=0.9)
def test_global_config(self):
self.q_config = QuantConfig(
activation=self.quanter, weight=self.quanter
)
self.q_config._specify(self.model)
self.assertIsNotNone(self.q_config.global_config.activation)
self.assertIsNotNone(self.q_config.global_config.weight)
for layer in self.model.sublayers():
config = self.q_config._get_config_by_layer(layer)
self.assertTrue(config.activation == self.quanter)
self.assertTrue(config.weight == self.quanter)
def assert_just_linear_weight_configure(self, model, config):
for layer in model.sublayers():
layer_config = config._get_config_by_layer(layer)
if type(layer) == Linear:
self.assertIsNone(layer_config.activation)
self.assertEqual(layer_config.weight, self.quanter)
self.assertTrue(config._is_quantifiable(layer))
elif type(layer) == Conv2D:
self.assertIsNone(layer_config)
self.assertFalse(config._is_quantifiable(layer))
def test_add_layer_config(self):
self.q_config = QuantConfig(activation=None, weight=None)
self.q_config.add_layer_config(
[self.model.fc], activation=None, weight=self.quanter
)
self.q_config._specify(self.model)
self.assert_just_linear_weight_configure(self.model, self.q_config)
def test_add_name_config(self):
self.q_config = QuantConfig(activation=None, weight=None)
self.q_config.add_name_config(
[self.model.fc.full_name()], activation=None, weight=self.quanter
)
self.q_config._specify(self.model)
self.assert_just_linear_weight_configure(self.model, self.q_config)
def test_add_type_config(self):
self.q_config = QuantConfig(activation=None, weight=None)
self.q_config.add_type_config(
[Linear], activation=None, weight=self.quanter
)
self.q_config._specify(self.model)
self.assert_just_linear_weight_configure(self.model, self.q_config)
def test_add_qat_layer_mapping(self):
self.q_config = QuantConfig(activation=None, weight=None)
self.q_config.add_qat_layer_mapping(Sequential, Conv2D)
self.assertTrue(Sequential in self.q_config.qat_layer_mappings)
self.assertTrue(
Sequential not in self.q_config.default_qat_layer_mapping
)
def test_add_customized_leaf(self):
self.q_config = QuantConfig(activation=None, weight=None)
self.q_config.add_customized_leaf(Sequential)
self.assertTrue(Sequential in self.q_config.customized_leaves)
self.assertTrue(self.q_config._is_customized_leaf(self.model.fc))
self.assertTrue(self.q_config._is_leaf(self.model.fc))
self.assertFalse(self.q_config._is_default_leaf(self.model.fc))
self.assertFalse(self.q_config._is_real_leaf(self.model.fc))
def test_need_observe(self):
self.q_config = QuantConfig(activation=None, weight=None)
self.q_config.add_layer_config(
[self.model.fc], activation=self.quanter, weight=self.quanter
)
self.q_config.add_customized_leaf(Sequential)
self.q_config._specify(self.model)
self.assertTrue(self.q_config._has_observer_config(self.model.fc))
self.assertTrue(self.q_config._need_observe(self.model.fc))
def test__get_observer(self):
self.q_config = QuantConfig(activation=None, weight=None)
self.q_config.add_layer_config(
[self.model.fc], activation=self.quanter, weight=self.quanter
)
self.q_config._specify(self.model)
observer = self.q_config._get_observer(self.model.fc)
self.assertIsInstance(observer, BaseQuanter)
def test_details(self):
self.q_config = QuantConfig(
activation=self.quanter, weight=self.quanter
)
self.q_config._specify(self.model)
self.assertIsNotNone(self.q_config.details())
self.assertIsNotNone(self.q_config.__str__())
if __name__ == '__main__':
unittest.main()
test/quantization/test_quant_aware_config.py
+214
View File
@@ -0,0 +1,214 @@
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import os
import unittest
import numpy as np
from test_quant_aware import MobileNet
import paddle
from paddle.static.quantization.quanter import convert, quant_aware
logging.basicConfig(level="INFO", format="%(message)s")
class TestQuantAwareBase(unittest.TestCase):
def setUp(self):
paddle.enable_static()
def get_save_int8(self):
return False
def generate_config(self):
config = {
'weight_quantize_type': 'channel_wise_abs_max',
'activation_quantize_type': 'moving_average_abs_max',
'quantize_op_types': ['depthwise_conv2d', 'mul', 'conv2d'],
'onnx_format': False,
}
return config
def test_accuracy(self):
main_prog = paddle.static.Program()
with paddle.static.program_guard(main_prog):
image = paddle.static.data(
name='image', shape=[None, 1, 28, 28], dtype='float32'
)
label = paddle.static.data(
name='label', shape=[None, 1], dtype='int64'
)
model = MobileNet()
out = model.net(input=image, class_dim=10)
cost = paddle.nn.functional.loss.cross_entropy(
input=out, label=label
)
avg_cost = paddle.mean(x=cost)
acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1)
acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5)
optimizer = paddle.optimizer.Momentum(
momentum=0.9,
learning_rate=0.01,
weight_decay=paddle.regularizer.L2Decay(4e-5),
)
optimizer.minimize(avg_cost)
val_prog = main_prog.clone(for_test=True)
place = (
paddle.CUDAPlace(0)
if paddle.is_compiled_with_cuda()
else paddle.CPUPlace()
)
exe = paddle.static.Executor(place)
exe.run(paddle.static.default_startup_program())
def transform(x):
return np.reshape(x, [1, 28, 28])
train_dataset = paddle.vision.datasets.MNIST(
mode='train', backend='cv2', transform=transform
)
test_dataset = paddle.vision.datasets.MNIST(
mode='test', backend='cv2', transform=transform
)
batch_size = 64 if os.environ.get('DATASET') == 'full' else 8
train_loader = paddle.io.DataLoader(
train_dataset,
places=place,
feed_list=[image, label],
drop_last=True,
return_list=False,
batch_size=batch_size,
)
valid_loader = paddle.io.DataLoader(
test_dataset,
places=place,
feed_list=[image, label],
batch_size=batch_size,
return_list=False,
)
def train(program):
iter = 0
stop_iter = None if os.environ.get('DATASET') == 'full' else 10
for data in train_loader():
cost, top1, top5 = exe.run(
program,
feed=data,
fetch_list=[avg_cost, acc_top1, acc_top5],
)
iter += 1
if iter % 100 == 0:
logging.info(
f'train iter={iter}, avg loss {cost}, acc_top1 {top1}, acc_top5 {top5}'
)
if stop_iter is not None and iter == stop_iter:
break
def test(program):
iter = 0
stop_iter = None if os.environ.get('DATASET') == 'full' else 10
result = [[], [], []]
for data in valid_loader():
cost, top1, top5 = exe.run(
program,
feed=data,
fetch_list=[avg_cost, acc_top1, acc_top5],
)
iter += 1
if iter % 100 == 0:
logging.info(
f'eval iter={iter}, avg loss {cost}, acc_top1 {top1}, acc_top5 {top5}'
)
result[0].append(cost)
result[1].append(top1)
result[2].append(top5)
if stop_iter is not None and iter == stop_iter:
break
logging.info(
f' avg loss {np.mean(result[0])}, acc_top1 {np.mean(result[1])}, acc_top5 {np.mean(result[2])}'
)
return np.mean(result[1]), np.mean(result[2])
train(main_prog)
top1_1, top5_1 = test(main_prog)
config = self.generate_config()
quant_train_prog = quant_aware(main_prog, place, config, for_test=False)
quant_eval_prog = quant_aware(val_prog, place, config, for_test=True)
train(quant_train_prog)
save_int8 = self.get_save_int8()
if save_int8:
convert_eval_prog, _ = convert(
quant_eval_prog, place, config, save_int8=save_int8
)
else:
convert_eval_prog = convert(
quant_eval_prog, place, config, save_int8=save_int8
)
top1_2, top5_2 = test(convert_eval_prog)
# values before quantization and after quantization should be close
logging.info(f"before quantization: top1: {top1_1}, top5: {top5_1}")
logging.info(f"after quantization: top1: {top1_2}, top5: {top5_2}")
class TestQuantAwareNone(TestQuantAwareBase):
def generate_config(self):
config = None
return config
class TestQuantAwareTRT(TestQuantAwareBase):
def generate_config(self):
config = {
'weight_quantize_type': 'channel_wise_abs_max',
'activation_quantize_type': 'moving_average_abs_max',
'quantize_op_types': ['depthwise_conv2d', 'mul', 'conv2d'],
'onnx_format': False,
'for_tensorrt': True,
}
return config
class TestQuantAwareFullQuantize(TestQuantAwareBase):
def generate_config(self):
config = {
'weight_quantize_type': 'channel_wise_abs_max',
'activation_quantize_type': 'moving_average_abs_max',
'quantize_op_types': ['depthwise_conv2d', 'mul', 'conv2d'],
'onnx_format': False,
'is_full_quantize': True,
}
return config
class TestQuantAwareSaveInt8(TestQuantAwareBase):
def generate_config(self):
config = {
'weight_quantize_type': 'channel_wise_abs_max',
'activation_quantize_type': 'moving_average_abs_max',
'quantize_op_types': ['depthwise_conv2d', 'mul', 'conv2d'],
'onnx_format': False,
}
return config
def get_save_int8(self):
return True
if __name__ == '__main__':
unittest.main()
test/quantization/test_stub.py
+64
View File
@@ -0,0 +1,64 @@
# copyright (c) 2023 paddlepaddle authors. all rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import paddle
from paddle.nn import Conv2D
from paddle.nn.quant import Stub
from paddle.quantization import QAT, QuantConfig
from paddle.quantization.quanters import FakeQuanterWithAbsMaxObserver
from paddle.quantization.quanters.abs_max import (
FakeQuanterWithAbsMaxObserverLayer,
)
quanter = FakeQuanterWithAbsMaxObserver(moving_rate=0.9)
class Model(paddle.nn.Layer):
def __init__(self, num_classes=10):
super().__init__()
self.quant_in = Stub()
self.conv = Conv2D(3, 6, 3, stride=1, padding=1)
self.quant = Stub(quanter)
self.quant_out = Stub()
def forward(self, inputs):
out = self.conv(inputs)
out = self.quant(out)
out = paddle.nn.functional.relu(out)
return self.quant_out(out)
class TestStub(unittest.TestCase):
def test_stub(self):
model = Model()
q_config = QuantConfig(activation=quanter, weight=quanter)
qat = QAT(q_config)
q_config.add_layer_config(model.quant_in, activation=None, weight=None)
quant_model = qat.quantize(model)
image = paddle.rand([1, 3, 32, 32], dtype="float32")
out = model(image)
out = quant_model(image)
out.backward()
quanter_count = 0
for _layer in quant_model.sublayers(True):
if isinstance(_layer, FakeQuanterWithAbsMaxObserverLayer):
quanter_count += 1
self.assertEqual(quanter_count, 5)
if __name__ == '__main__':
unittest.main()
test/quantization/test_trace_quanter.py
+99
View File
@@ -0,0 +1,99 @@
# copyright (c) 2023 paddlepaddle authors. all rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The quantizer layers should be traced by paddle.jit.save function."""
import os
import tempfile
import unittest
import paddle
from paddle.quantization import QAT, QuantConfig
from paddle.quantization.quanters import FakeQuanterWithAbsMaxObserver
from paddle.quantization.quanters.abs_max import (
FakeQuanterWithAbsMaxObserverLayer,
)
from paddle.vision.models import resnet18
class TestPTQ(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory(dir="./")
self.path = os.path.join(self.temp_dir.name, 'ptq')
def tearDown(self):
self.temp_dir.cleanup()
def _get_model_for_qat(self):
observer = FakeQuanterWithAbsMaxObserver()
model = resnet18()
model.train()
q_config = QuantConfig(activation=None, weight=None)
q_config.add_type_config(
paddle.nn.Conv2D, activation=observer, weight=observer
)
qat = QAT(q_config)
quant_model = qat.quantize(model)
return quant_model, qat
def _count_layers(self, model, layer_type):
count = 0
for _layer in model.sublayers(True):
if isinstance(_layer, layer_type):
count += 1
return count
def test_trace(self):
quant_model, ptq = self._get_model_for_qat()
image = paddle.rand([1, 3, 32, 32], dtype="float32")
quantizer_count_in_dygraph = self._count_layers(
quant_model, FakeQuanterWithAbsMaxObserverLayer
)
save_path = os.path.join(self.path, 'int8_infer')
paddle.jit.save(quant_model, save_path, [image])
print(f"quant_model is saved into {save_path}")
paddle.enable_static()
exe = paddle.static.Executor(paddle.CPUPlace())
main_program = paddle.static.Program()
startup_program = paddle.static.Program()
with paddle.static.program_guard(main_program, startup_program):
[
inference_program,
feed_target_names,
fetch_targets,
] = paddle.static.load_inference_model(save_path, exe)
quantizer_count_in_static_model = 0
if paddle.base.framework.in_pir_mode():
for _op in inference_program.global_block().ops:
if (
"fake_quantize_dequantize_moving_average_abs_max"
in _op.name()
):
quantizer_count_in_static_model += 1
else:
for _op in inference_program.global_block().ops:
if (
_op.type
== "fake_quantize_dequantize_moving_average_abs_max"
):
quantizer_count_in_static_model += 1
self.assertEqual(
quantizer_count_in_dygraph, quantizer_count_in_static_model
)
paddle.disable_static()
if __name__ == '__main__':
unittest.main()
test/quantization/test_weight_only_linear.py
+1099
View File
File diff suppressed because it is too large Load Diff
test/quantization/test_weight_quantize.py
+167
View File
@@ -0,0 +1,167 @@
# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import numpy as np
import paddle
from paddle.nn.quant import weight_quantize
paddle.seed(3)
np.random.seed(3)
# fmt: off
# 预先计算得到的权重矩阵，作为ref用于测试
ref_out = [[-103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103],
[-86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86],
[-69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69],
[-52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52],
[-35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35],
[-18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18],
[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17],
[34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34],
[51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51],
[68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68],
[85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85],
[102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102],
[119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119],
[-103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103],
[-86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86],
[-69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69],
[-52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52],
[-35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35],
[-18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18],
[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17],
[34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34],
[51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51],
[68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68],
[85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85],
[102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102],
[119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119],
[-103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103],
[-86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86],
[-69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69],
[-52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52],
[-35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35],
[-18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18],
[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17],
[34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34],
[51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51],
[68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68],
[85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85],
[102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102],
[119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119],
[-103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103],
[-86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86],
[-69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69],
[-52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52],
[-35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35, -35],
[-18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18, -18],
[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17],
[34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34],
[51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51],
[68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68],
[85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85],
[102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102],
[119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119],
[-103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103, -103],
[-86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86, -86],
[-69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69, -69],
[-52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52, -52]]
# fmt: off
np.set_printoptions(threshold=100000000)
paddle.set_printoptions(threshold=100000000)
def arrange_cols(rows, cols):
weight = []
for i in range(rows):
for j in range(cols):
weight.append((j % 15) - 7)
# pack them
res = np.array(weight).reshape([rows, cols])
res0 = res[0::2,:] & 0x0F
res1 = (res[1::2,:] & 0x0F) << 4
res = res0 | res1
return res
class WeightQuantizeW4a8TestCase(unittest.TestCase):
def setUp(self):
self.rows = 64
self.cols = 64
self.weight = arrange_cols(self.rows, self.cols)
self.weight = paddle.to_tensor(self.weight).astype('int8')
def run_test(self):
out = weight_quantize(self.weight, algo="w4a8")[0]
out = out.reshape([-1])
baseline = np.array(ref_out).reshape([-1])
np.allclose(baseline, out.astype("int32").numpy(), atol=1e-2)
class WeightQuantizeW4a8ZeroSizeTensorTestCase(unittest.TestCase):
def setUp(self):
self.x_shape = [1, 0, 10]
self.dtype = "float32"
def _test_dygraph(self):
paddle.disable_static()
x = paddle.rand(shape= self.x_shape, dtype=paddle.float16)
x.stop_gradient=False
out, scale = weight_quantize(x, algo='weight_only_int8')
loss = out.sum()
loss.backward()
np.assert_allclose(x.grad.shape,x.shape)
def run_test(self):
self.setUp()
self._test_dygraph()
class WeightQuantizeW4afp8TestCase(unittest.TestCase):
def setUp(self):
self.rows = 128
self.cols = 128
weight = np.random.randint(-7, 7, size=[self.rows, self.cols], dtype='int8') # shape: [K, N]
self.weight_trans = weight.transpose() + 7
weight1 = weight[0::2, :] & 0x0F
weight2 = (weight[1::2, :] & 0x0F) << 4
weight_packed = weight1 | weight2
self.weight_packed = paddle.to_tensor(weight_packed)
def test(self):
out = weight_quantize(self.weight_packed, algo="w4afp8")[0] # shape: [N, K/2]
out_np = np.array(out.reshape([-1, 32]))
out_np_1 = (out_np >> 4) & 0x0F
out_np_2 = out_np & 0x0F
result = np.zeros((out_np_1.shape[0], out_np_1.shape[1]*2), dtype=out_np.dtype)
result[:, 1::2] = out_np_1
result[:, 0::2] = out_np_2
# ref out
tmp = self.weight_trans.reshape([-1, 64])
tmp1 = tmp[:, 0:32] & 0x0F
tmp2 = (tmp[:, 32:64] & 0x0F) << 4
ref_out = tmp1 | tmp2
ref_out = ref_out.reshape([-1, self.rows])
np.allclose(ref_out.astype("int32"), out.astype("int32").numpy(), atol=1e-2)
if __name__ == '__main__':
unittest.main()