wehub/tensorflow--tensorflow
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
cffconvert / validate (push) Has been skipped
Details
License Check / license-check (push) Failing after 2s
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:14:16 +08:00
commit 8a852e4b4e
36502 changed files with 9277225 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tensorflow/lite/delegates/nnapi/BUILD
+352
View File
@@ -0,0 +1,352 @@
load("@rules_cc//cc:cc_library.bzl", "cc_library")
load("@rules_cc//cc:cc_test.bzl", "cc_test")
load("//tensorflow:tensorflow.default.bzl", "get_compatible_with_portable")
load("//tensorflow/lite:build_def.bzl", "tflite_cc_library_with_c_headers_test")
load("//tensorflow/lite:special_rules.bzl", "tflite_portable_test_suite")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
default_visibility = [
"//visibility:public",
],
licenses = ["notice"],
)
# Client of this library is responsible to provide an implementation of
# for headers in //third_party/tensorflow/lite/nnapi:nnapi_implementation_headers
cc_library(
name = "nnapi_delegate_no_nnapi_implementation",
srcs = select({
"//tensorflow:ios": [
"nnapi_delegate_disabled.cc",
],
"//tensorflow:windows": [
"nnapi_delegate_disabled.cc",
],
"//conditions:default": [
"nnapi_delegate.cc",
"nnapi_delegate_c_api.cc",
"quant_lstm_sup.cc",
"quant_lstm_sup.h",
],
}),
hdrs = [
"nnapi_delegate.h",
"nnapi_delegate_c_api.h",
"nnapi_delegate_kernel.h",
"nnapi_delegate_plugin.h",
],
compatible_with = get_compatible_with_portable(),
deps = [
"//tensorflow/lite:allocation",
"//tensorflow/lite:array",
"//tensorflow/lite:kernel_api",
"//tensorflow/lite:minimal_logging",
"//tensorflow/lite:util",
"//tensorflow/lite/c:c_api_types",
"//tensorflow/lite/c:common",
"//tensorflow/lite/core/c:c_api_types",
"//tensorflow/lite/core/c:common",
"//tensorflow/lite/delegates:serialization",
"//tensorflow/lite/delegates:utils",
"//tensorflow/lite/kernels:kernel_util",
"//tensorflow/lite/kernels/internal/utils:sparsity_format_converter",
"//tensorflow/lite/nnapi:nnapi_implementation_headers",
"//tensorflow/lite/nnapi:nnapi_lib",
"//tensorflow/lite/nnapi:nnapi_util",
"//tensorflow/lite/nnapi/sl:nnapi_support_library_headers",
"//tensorflow/lite/schema:schema_fbs",
"@FP16",
"@eigen_archive//:eigen3",
],
)
cc_library(
name = "nnapi_delegate",
hdrs = [
"nnapi_delegate.h",
"nnapi_delegate_c_api.h",
"nnapi_delegate_kernel.h",
"nnapi_delegate_plugin.h",
],
compatible_with = get_compatible_with_portable(),
deps = [
":nnapi_delegate_no_nnapi_implementation",
"//tensorflow/lite:allocation",
"//tensorflow/lite/core/c:common",
"//tensorflow/lite/delegates:serialization",
"//tensorflow/lite/nnapi:nnapi_implementation",
"//tensorflow/lite/nnapi:nnapi_lib",
],
)
tflite_cc_library_with_c_headers_test(
name = "nnapi_delegate_c_api",
hdrs = ["nnapi_delegate_c_api.h"],
compatible_with = get_compatible_with_portable(),
visibility = ["//visibility:private"],
deps = [
":nnapi_delegate",
"//tensorflow/lite/core/c:common",
],
)
exports_files(["nnapi_delegate_c_api.h"])
cc_library(
name = "nnapi_delegate_verbose_validation",
srcs = select({
"//tensorflow:ios": [
"nnapi_delegate_disabled.cc",
],
"//tensorflow:windows": [
"nnapi_delegate_disabled.cc",
],
"//conditions:default": [
"nnapi_delegate.cc",
"quant_lstm_sup.cc",
"quant_lstm_sup.h",
],
}),
hdrs = [
"nnapi_delegate.h",
"nnapi_delegate_kernel.h",
"nnapi_delegate_plugin.h",
],
copts = ["-DNNAPI_VERBOSE_VALIDATION"],
deps = [
"//tensorflow/lite:allocation",
"//tensorflow/lite:array",
"//tensorflow/lite:kernel_api",
"//tensorflow/lite:minimal_logging",
"//tensorflow/lite:util",
"//tensorflow/lite/c:common",
"//tensorflow/lite/core/c:c_api_types",
"//tensorflow/lite/core/c:common",
"//tensorflow/lite/delegates:serialization",
"//tensorflow/lite/delegates:utils",
"//tensorflow/lite/kernels:kernel_util",
"//tensorflow/lite/kernels/internal/utils:sparsity_format_converter",
"//tensorflow/lite/nnapi:nnapi_implementation",
"//tensorflow/lite/nnapi:nnapi_lib",
"//tensorflow/lite/nnapi:nnapi_util",
"//tensorflow/lite/nnapi/sl:nnapi_support_library_headers",
"//tensorflow/lite/schema:schema_fbs",
"@FP16",
"@eigen_archive//:eigen3",
],
)
cc_library(
name = "acceleration_test_util",
testonly = 1,
srcs = [
"acceleration_test_list.cc",
"acceleration_test_util.cc",
],
hdrs = ["acceleration_test_util.h"],
deps = [
":nnapi_delegate",
"//tensorflow/lite/kernels:acceleration_test_util_internal",
"@com_google_absl//absl/types:optional",
"@com_google_googletest//:gtest",
],
)
cc_library(
name = "nnapi_delegate_mock_test",
testonly = 1,
hdrs = [
"nnapi_delegate_mock_test.h",
],
copts = select({
"//tensorflow:ios": [
"-DNNAPI_DELEGATE_DISABLED",
],
"//tensorflow:windows": [
"-DNNAPI_DELEGATE_DISABLED",
],
"//conditions:default": [],
}),
deps = [
":nnapi_delegate",
"//tensorflow/lite/nnapi:nnapi_handler",
"//tensorflow/lite/nnapi:nnapi_implementation",
"//tensorflow/lite/nnapi:nnapi_lib",
"@com_google_absl//absl/memory",
"@com_google_googletest//:gtest",
],
)
cc_test(
name = "nnapi_delegate_test",
size = "small",
srcs = [
"nnapi_delegate_test.cc",
],
tags = [
"no_windows",
"tflite_not_portable_ios",
],
deps = [
":nnapi_delegate",
"//tensorflow/lite:framework",
"//tensorflow/lite:string",
"//tensorflow/lite:string_util",
"//tensorflow/lite/core:framework",
"//tensorflow/lite/core/c:common",
"//tensorflow/lite/kernels:builtin_ops",
"//tensorflow/lite/kernels:deprecated_backends",
"//tensorflow/lite/kernels:test_util",
"//tensorflow/lite/nnapi:nnapi_implementation",
"//tensorflow/lite/nnapi:nnapi_lib",
"//tensorflow/lite/schema:schema_fbs",
"@com_google_absl//absl/log:check",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "nnapi_delegate_c_api_test",
size = "small",
srcs = [
"nnapi_delegate_c_api_test.cc",
],
tags = [
"no_windows",
# TODO(b/401046768): Re-enable once the test is fixed.
"notap",
"tflite_not_portable_ios",
],
visibility = ["//visibility:private"],
deps = [
":nnapi_delegate",
"//tensorflow/lite/core:framework",
"//tensorflow/lite/core/c:common",
"//tensorflow/lite/kernels:builtin_ops",
"//tensorflow/lite/kernels:deprecated_backends",
"//tensorflow/lite/kernels:test_util",
"//tensorflow/lite/schema:schema_fbs",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "nnapi_delegate_errno_test",
size = "small",
srcs = [
"nnapi_delegate_errno_test.cc",
],
tags = [
"no_mac",
"no_windows",
"tflite_not_portable_ios",
],
deps = [
":nnapi_delegate",
":nnapi_delegate_mock_test",
"//tensorflow/lite:framework",
"//tensorflow/lite/core:framework",
"//tensorflow/lite/core/c:common",
"//tensorflow/lite/kernels:test_util",
"//tensorflow/lite/nnapi:nnapi_implementation",
"//tensorflow/lite/schema:schema_fbs",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "nnapi_delegate_device_selection_test",
size = "small",
srcs = [
"nnapi_delegate_device_selection_test.cc",
],
tags = [
"no_mac",
"no_windows",
"tflite_not_portable_ios",
],
deps = [
":nnapi_delegate",
":nnapi_delegate_mock_test",
"//tensorflow/lite:framework",
"//tensorflow/lite:kernel_api",
"//tensorflow/lite/core:framework",
"//tensorflow/lite/core/c:common",
"//tensorflow/lite/kernels:test_util",
"//tensorflow/lite/nnapi:nnapi_implementation",
"//tensorflow/lite/nnapi:nnapi_lib",
"//tensorflow/lite/schema:schema_fbs",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "nnapi_delegate_nnapi_failure_handling_test",
size = "small",
srcs = [
"nnapi_delegate_nnapi_failure_handling_test.cc",
],
tags = [
"no_mac",
"no_windows",
"tflite_not_portable_ios",
],
deps = [
":nnapi_delegate",
":nnapi_delegate_mock_test",
"//tensorflow/lite:framework",
"//tensorflow/lite/core:framework",
"//tensorflow/lite/core/c:common",
"//tensorflow/lite/kernels:test_util",
"//tensorflow/lite/nnapi:nnapi_implementation",
"//tensorflow/lite/nnapi:nnapi_lib",
"//tensorflow/lite/schema:schema_fbs",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "nnapi_delegate_signed_quantization_test",
size = "small",
srcs = [
"nnapi_delegate_signed_quantization_test.cc",
],
tags = [
"no_mac",
"no_windows",
"tflite_not_portable_ios",
],
deps = [
":nnapi_delegate",
":nnapi_delegate_mock_test",
"//tensorflow/lite:framework",
"//tensorflow/lite/core:framework",
"//tensorflow/lite/core/c:common",
"//tensorflow/lite/kernels:builtin_ops",
"//tensorflow/lite/kernels:test_util",
"//tensorflow/lite/nnapi:nnapi_implementation",
"//tensorflow/lite/nnapi:nnapi_lib",
"//tensorflow/lite/schema:schema_fbs",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "quant_lstm_sup_test",
size = "small",
srcs = [
"quant_lstm_sup.cc",
"quant_lstm_sup.h",
"quant_lstm_sup_test.cc",
],
deps = [
"//tensorflow/lite:framework",
"//tensorflow/lite/c:common",
"//tensorflow/lite/core/c:common",
"//tensorflow/lite/kernels:kernel_util",
"@com_google_googletest//:gtest_main",
],
)
tflite_portable_test_suite()
tensorflow/lite/delegates/nnapi/acceleration_test_list.cc
+504
View File
@@ -0,0 +1,504 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/lite/delegates/nnapi/acceleration_test_util.h"
namespace tflite {
const char* const NnapiAccelerationTestParams::acceleration_test_config_ =
R"(
## Every Test can be allowlisted or denylisted using a regexp on its test_id
## Test_id
#
# The test_id is test_suite_name / test_name, this differs from the
# name used by the build because of the / separator instead of .
# Parameterized tests names are composed by the base test name / test / ordinal
# the ordinal is the position in the list of parameters generated by the
# cardinal product of all the different parameter sets
# Denylist/Allowlist
# To denylist an element simply add - before the test_id regex
## Rules evaluation
#
# Rules are checked in order, the first matching completes the browsing
# This can be useful to put more specific rules first and generic default
# ones below
## Test Arguments
#
# The test can be parameterized with the minimum Android SDK version
# to apply the acceleration validation for.
# If omitted will use 27
#test-id,min-android-sdk-version
# activations_test
FloatActivationsOpTest/Softmax[13]D,29
QuantizedActivationsOpTest/Softmax[13]D.+nt8,29
FloatActivationsOpTest/Softmax\dD
QuantizedActivationsOpTest/Softmax\dD.+nt8
FloatActivationsOpTest/LogSoftmax,29
FloatActivationsOpTest/PRelu,29
LogisticOpTest/LogisticOpTest/Sigmoid(.+nt8)?/\d+
LogisticOpTest/LogisticOpTest/Sigmoid/\d+
TanhOpTest/TanhOpTest/Tanh(.+nt8)?/\d+,29
FloatActivationsOpTest/Elu,30
FloatActivationsOpTest/Relu
FloatActivationsOpTest/Relu1
FloatActivationsOpTest/Relu6
FloatActivationsOpTest/HardSwish
FloatActivationsOpTest/LeakyRelu,29
QuantizedActivationsOpTest/HardSwish
QuantizedActivationsOpTest/HardSwishBias
-QuantizedActivationsOpTest/Relu.?Int16
QuantizedActivationsOpTest/Relu.*
-QuantizedActivationsOpTest/LeakyReluInt16,30
QuantizedActivationsOpTest/LeakyRelu.*,30
QuantizedActivationsOpTest/Relu.+nt8
QuantizedActivationsOpTest/PRelu,29
QuantizedActivationsOpTest/PReluSameShapes,29
QuantizedActivationsOpTest/PReluInt8.+,30
PReluOpTest/.*,29
# add_test
FloatAddOpModel/.+
QuantizedAddOpModel/QuantizedTestsNoActivation.+nt8
QuantizedAddOpModel/QuantizedVariousInputShapes.+
QuantizedAddOpModel/QuantizedWithScalarBroadcast.+nt8
QuantizedAddOpModel/QuantizedWithMixedBroadcas.+nt8
# arg_min_max_test
# Only tests with ConstantAxis && OutputType == TensorType_INT32
# (element 4 and 6 in the test parameter list)
# Supported only from NNAPI 1.2
ArgMinMaxOpTest/ArgMinMaxOpTest/Get.+ArgFloat/[46],29
ArgMinMaxOpTest/ArgMinMaxOpTest/Get.+Arg.+nt8/[46],29
ArgMinMaxOpTest/ArgMinMaxOpTest/Get.+ArgInt/[46],29
ArgMinMaxOpTest/ArgMinMaxOpTest/Get.+ArgMulDimensions/[46],29
ArgMinMaxOpTest/ArgMinMaxOpTest/Get.+ArgNegativeAxis/[46],29
ArgMinMaxOpTest/ArgMinMaxOpTest/Get.+ArgOutput64/[46],29
# basic_rnn_test
RnnOpTest/BlackBoxTest
# batch_matmul_test
# broadcasting is not supported
-BatchMatMulOpTest/BatchMatMulOpTest/.+Broadcast.+
BatchMatMulOpTest/BatchMatMulOpTest/.+,1000006
QuantizedBatchMatMulOpTest/QuantizedBatchMatMulOpTest/SimpleTestQuantizedInt8/.+,1000006
# batch_to_space_nd_test
BatchToSpaceNDOpTest/SimpleConstTest.*
BatchToSpaceNDOpTest/BatchOneConstTest.*
# bidirectional_sequence_lstm_test
# Only test with non quantized weights
LSTMOpTest/LSTMOpTest/BlackBoxTestNoCifgNoPeepholeNoProjectionNoClipping/0,29
# Only test with non quantized weights
LSTMOpTest/LSTMOpTest/BlackBoxTestMergedOutput/0,29
LSTMOpTest/BlackBoxTestNoCifgNoPeepholeNoProjectionNoClippingReverse,29
LSTMOpTest/BlackBoxTestWithCifgWithPeepholeNoProjectionNoClipping,29
LSTMOpTest/BlackBoxTestWithCifgWithPeepholeNoProjectionNoClippingReversed,29
LSTMOpTest/BlackBoxTestWithPeepholeWithProjectionNoClipping,29
LSTMOpTest/BlackBoxTestWithPeepholeWithProjectionNoClippingBatchMajor,29
# Only test with non quantized weights
LSTMOpTest/LSTMOpTest/BlackBoxTestWithAuxInputZeroAuxWeight/0,29
QuantizationOrNot/LSTMOpTest/BlackBoxTestNoCifgNoPeepholeNoProjectionNoClipping/0,29
QuantizationOrNot/LSTMOpTest/BlackBoxTestMergedOutput/0,29
QuantizationOrNot/LSTMOpTest/BlackBoxTestWithAuxInputZeroAuxWeight/0,29
LSTMOpTest/LSTMOpTest/BlackBoxTestWithAuxInput/0,29
# cast_test
CastOpModel/CastFloatToIn32
CastOpModel/CastInt32ToFloat,29
CastOpModel/CastFloatToUInt8,29
CastOpModel/CastUInt8ToFloat,29
CastOpModel/CastInt32ToUInt8,29
CastOpModel/CastUInt8ToInt32,29
# comparisons_test
ComparisonsTest/.+,29
# concatenation_test
ConcatenationOpTest/ThreeDimensionalOneInput
ConcatenationOpTest/OneTrivialInput
ConcatenationOpTest/TwoDimensionalOneInput
ConcatenationOpTest/TwoInputsTwoAxesNegativeAxes
ConcatenationOpTest/TwoInputsTwoAxesNegativeAxesNonQuantized
ConcatenationOpTest/FourInputs
ConcatenationOpTest/FourInputsQuantizedUint8
ConcatenationOpTest/FourInputsQuantizedInt8
ConcatenationOpTest/ThreeDimensionalNonQuantizedOneInput
ConcatenationOpTest/OneTrivialNonQuantizedInput
ConcatenationOpTest/TwoDimensionalNonQuantizedOneInput
ConcatenationOpTest/FourInputsQuantizedMixedRange,29
ConcatenationOpTest/FourInputsQuantizedMixedRangeClampingLogic,29
# conv_test
-ConvolutionOpTest/ConvolutionOpTest.SimplePerTensorTest/.+
ConvolutionOpTest/ConvolutionOpTest.SimpleTestFloatWithDilation/.+,29
ConvolutionOpTest/ConvolutionOpTest.SimpleTestLargeIrregularQuantized/.+,29
ConvolutionOpTest/ConvolutionOpTest.SimpleTestQuantizedOutputMultiplierGreaterThan1/.+,29
ConvolutionOpTest/ConvolutionOpTest.SimpleTestQuantizedWithDilation/.+,29
ConvolutionOpTest/ConvolutionOpTest.SimplePerChannelTest/.+,29
ConvolutionOpTest/ConvolutionOpTest.SimpleTestQuantizedGrouped/.+,29
ConvolutionOpTest/ConvolutionOpTest/.+Hybrid.+,29/
ConvolutionOpTest/ConvolutionOpTest/.+/\d+
# depthwise_conv_test
.+ConvolutionOpTest/.+/\d+,29
# dequantize_test
DequantizeOpTest/Uint8
DequantizeOpTest/Int8,30
# depth_to_space_test
DepthToSpaceOpModel/Float32
DepthToSpaceOpModel/UInt8
DepthToSpaceOpModel/int8
# div_test
-FloatDivOpTest/WithBroadcast5D
FloatDivOpTest/.+
# elementwise_test
ElementWise/Abs,29
ElementWise/Sin,29
ElementWise/Log,29
ElementWise/Sqrt,29
ElementWise/Rsqrt,29
ElementWise/LogicalNot,29
# embedding_lookup_test
EmbeddingLookupOpTest/SimpleTest
# exp_test
ExpOpTest/FloatTest,29
# expand_dims_test
# Only constant tensors models
ExpandDimsOpTest/.+/1,29
# fill_test
FillOpTest/FillOpTest/FillInt32/0,30
FillOpTest/FillOpTest/FillFloat/0,30
FillOpTest/FillOpTest/FillFloatInt32Dims/0,30
# floor_test
FloorOpTest/.+
# fully_connected_test
FloatFullyConnectedOpTest/FloatFullyConnectedOpTest/SimpleTest/\d+
FloatFullyConnectedOpTest/FloatFullyConnectedOpTest/SimpleTest2/\d+
QuantizedFullyConnectedOpTest/QuantizedFullyConnectedOpTest/SimpleTestQuantized.+nt8/\d+,29
QuantizedFullyConnectedOpTest/QuantizedFullyConnectedOpTest/SimpleTestSingleBatchQuantizedInt8/\d+,29
QuantizedFullyConnectedOpTest/SimpleTestQuantizedOutputMultiplierGreaterThan1Uint8/\d+,29
QuantizedFullyConnectedOpTest/SimpleTestQuantizedOutputMultiplierGreaterThan1Int8/\d+,29
HybridFullyConnectedOpTest/SimpleTestQuantizedUint8,29
HybridFullyConnectedOpTest/SimpleTestQuantizedInt8,29
HybridAsymmetricInputFullyConnectedOpTest.SimpleTestQuantizedUint8,29
FloatFullyConnectedOpTest/FloatFullyConnectedOpTest/SimpleTest4DInput/\d+
QuantizedFullyConnectedOpTest/QuantizedFullyConnectedOpTest/SimpleTest4dInputQuantizedUint8/\d+
QuantizedFullyConnectedOpTest/QuantizedFullyConnectedOpTest/SimpleTest4dInputQuantizedOutputMultiplierGreaterThan1Uint8/\d+,29
FloatFullyConnectedOpTest/FloatFullyConnectedOpTest/BlackBoxTest/\d+
FloatFullyConnectedOpTest/SimpleTestNoBias
# gather_test
GatherOpTest/Shuffle,29
GatherOpTest/Test1DInput1DIndex,29
GatherOpTest/Test2DIndexWith2DResult,29
FloatGatherOpTest/Duplicate,29
FloatGatherOpTest/Slice,29
FloatGatherOpTest/Axis1,29
FloatGatherOpTest/Axis1Slice,29
FloatGatherOpTest/LastAxis,29
TypesGatherOpTest/Float32Int32,29
TypesGatherOpTest/Int32Int32,29
TypesGatherOpTest/Uint8Int32,29
TypesGatherOpTest/Int8Int32,29
-TypesGatherOpTest/.*Int16.*
# hashtable_lookup_test
# All test excepted the string one should be accelerated
-HashtableLookupOpTest/TestString
HashtableLookupOpTest/.+
# l2norm_test
L2NormOpTest/.+,29
# local_response_norm_test
LocalResponseNormOpTest/.+
# logical_test
LogicalTest/.+,29
# lsh_projection_test
-LSHProjectionOpTest2/Sparse3DInputs
LSHProjectionOpTest2/Sparse1DInputs,29
LSHProjectionOpTest2/.+
# Before the lstm because of clashing with matchers
# unidirectional_sequence_lstm_test
NoCifgNoPeepholeNoProjectionNoClippingUnidirectionalLstmTest/LstmBlackBoxTest,29
CifgPeepholeNoProjectionNoClippingUnidirectionalLstmTest/NonLayerNormLstmBlackBoxTest,29
# Only the two tests above, disabling all possible matches from the lstm tests
# coming after
-.+UnidirectionalLstmTest/.+
# lstm_test
-LstmOpTest/InvalidTypes
# Float
Parameterized/LstmOpTest.+/0,29
Parameterized/LstmOpTest.+/1,29
Parameterized/LstmOpTest.+/2,29
Parameterized/LstmOpTest.+/3,29
# HybridUint8
Parameterized/LstmOpTest.+/4,29
Parameterized/LstmOpTest.+/5,29
Parameterized/LstmOpTest.+/6,29
Parameterized/LstmOpTest.+/7,29
# HybridInt8
-Parameterized/LstmOpTest.+/8
-Parameterized/LstmOpTest.+/9
-Parameterized/LstmOpTest.+/10
-Parameterized/LstmOpTest.+/11
# maximum_minimum_test
MaxMinOpTest/.+nt8Test,29
MaximumOpTest/.+,29
# mirror_pad_test
MirrorPadTest/.+,1000007
# mul_test
FloatMulOpTest/.+
# neg_test
-NegOpModel/.+Int64
NegOpModel/.+,29
# pack_test
# int32 and uint8 are supported since NNAPI FL6
PackOpTest/Int32.+,1000006
PackOpTestInt/1/.+,1000006
# PACK along last axis is supported since NNAPI FL6
PackOpTest/FloatThreeInputsDifferentAxis,1000006
PackOpTest/FloatThreeInputsNegativeAxis,1000006
PackOpTestInt/0/ThreeInputsDifferentAxis,1000006
PackOpTestInt/0/ThreeInputsNegativeAxis,1000006
# f32 and int8 are supported since NNAPI 1.3 by decomposition
PackOpTest/Float.+,30
PackOpTestInt/0/.+,30
# pad_test
-PadOpTest/TooManyDimensions
-PadOpTest/UnequalDimensions
-PadOpTest/InvalidPadValue
# Zero height or width is not supported
-PadOpTest/Zero.+ConstImageStyleTest
# Dynamic tensors are not supported
-.*Pad.*OpTest/.+Dynamic.*Test
-QuantizedPad.*OpTest/.+ZeroNotInQuantizationRange
# 16-bit tests are not supported
-QuantizedPadOpTest/.*Int16.*
QuantizedPadOpTest/.+,29
QuantizedPadV2OpTest/.+,29
PadOpTest/.+,29
# pooling_test
FloatPoolingOpTest/L2PoolActivationRelu.*,29
FloatPoolingOpTest/.+
# Image is too big
-QuantizedPoolingOpTest/AveragePoolImageSize17
# Int16 unsupported
-QuantizedPoolingOpTest/SymmetricAveragePool16
QuantizedPoolingOpTest/.+
QuantizedUInt8PoolingOpTest/.+
# pow_test
-PowOpModel/Simple
-PowOpModel/NegativeAndZeroValue
-PowOpModel/BroadcastTest
-PowOpModel/IntSingleIntegerExponentTest
PowOpModel/.+,29
# quant_basic_lstm_test
QuantizedLstmTest/BasicQuantizedLstmTest/29
# quantized_lstm op test
# Temporary disabled due to b/188515203
//IntegerLstmOpTest/NoCifg_NoPeephole_Projection_LayerNorm,30
# quantize_test
QuantizeOpTest/UINT8,29
QuantizeOpTest/UInt8UInt8.+,29
QuantizeOpTest/Int8Int8.+,30
QuantizeOpTest/INT8,30
# rank
# reduce_test
-Dynamic.+(Mean|Sum|Prod|Max|Min)OpTest/.+
-ConstUint8SumOpTest/.+
ConstUint8(Max|Min)OpTest/.+,29
ConstUint8(Mean)OpTest/.+,29
-ConstInt8(Max|Min)OpTest/.+,29
ConstInt8MeanOpTest/.+,29
-ConstMeanOpTest.*/.+Int16
ConstMeanOpTest.*/.+,29
-MeanOpTestQuantized.*/.+Int16
MeanOpTestQuantized.*/.+,29
ConstFloat(Sum|Prod|Max|Min)OpTest/NotKeepDims,29
ConstFloat(Sum|Prod|Max|Min)OpTest/KeepDims,29
ConstFloat(Mean|Any)OpTest/NotKeepDims
ConstFloat(Mean|Any)OpTest/KeepDims
ConstFloat(Sum|Prod|Max|Min)OpTest/ScalarAxis,29
# reshape_test
# Acceleration would be only for the test with shape being a constant tensor or
# as hardcoded options.
ReshapeOpTest/[01]/InvalidShape
ReshapeOpTest/[01]/RegularShapes
ReshapeOpTest/[01]/WithStretchDimension
# int32 is supported since NNAPI FL6
ReshapeOpTest/3/InvalidShape,1000006
ReshapeOpTest/3/RegularShapes,1000006
ReshapeOpTest/3/WithStretchDimension,1000006
# resize_bilinear_test
// align_corners & half_pixel_centers are not implemented in NNAPI before API 30
ResizeBilinearOpTest/ResizeBilinearOpTest.+HalfPixelCenters.*/0,30
// Only models with constant size tensor are accelerated
ResizeBilinearOpTest/ResizeBilinearOpTest/.+/0,29
# resize_nearest_neighbor_test
// align_corners & half_pixel_centers are not implemented in NNAPI before API 30
ResizeNearestNeighborOpTest/ResizeNearestNeighborOpTest.+AlignCorners.*/0,30
ResizeNearestNeighborOpTest/ResizeNearestNeighborOpTest.+HalfPixelCenters.*/0,30
// 16-bit tests are not supported
-ResizeNearestNeighborOpTest.+Int16/.+
// Only models with constant size tensor are accelerated
ResizeNearestNeighborOpTest/ResizeNearestNeighborOpTest/.+/0,29
# reverse_test
-ReverseOpTest/Int64.+
-ReverseOpTest/Int16.+
ReverseOpTest/.+,1000007
# select_test
-SelectOpTest/SelectBool
-SelectOpTest.SelectInt16
-SelectOpTest/RankZero.+
-SelectOpTest/RankOne.+
SelectOpTest/.+,29
# slice_test
-SliceOpTest/SliceOpTest/IndexInt64/.+
-SliceOpTest/SliceOpTest/SliceString/.+
-SliceOpTest/SliceOpTest/SliceInt64/.+
-SliceOpTest/SliceOpTest/SliceBool/.+
-SliceOpTest/SliceOpTest/SliceInt16/.+
-SliceOpTest/SliceOpTest/SliceInt64StaticOutput/.*
# Only constant tensors
SliceOpTest/SliceOpTest/.+/0,29
# softmax_test
SoftmaxOpTest/CompareWithTFminiBetaEq1
SoftmaxOpTest/CompareWithTFminiBetaNotEq1
# space_to_depth_test
SpaceToDepthOpModel/Float32
SpaceToDepthOpModel/Uint8
SpaceToDepthOpModel/int8
# split_test
-SplitOpTest/SplitOpTest/.+Int8/.+
# Only accelerated when axis is a constant tensor
SplitOpTest/SplitOpTest/.+/0,29
# split_v_test
# NNAPI does not support int16
-SplitVOpTypedTest/3/.+
# NNAPI does not support zero-sized slice
-SplitVOpTypedTest/.+OneDimensional2
# Only accelerated when both split_sizes and axis are constant
SplitVOpTypedTest/.+/ConstSplits.+,30
# squared_difference_test
FloatSquaredDifferenceOpTest/.+,28
(Integer|Quantized)SquaredDifferenceOpTest/.+,30
# squeeze_test
FloatSqueezeOpTest/.+,29
# sub_test
-FloatSubOpModel/WithBroadcast5D
FloatSubOpModel/.+
-QuantizedSubOpModel/.+Int16
-QuantizedSubOpModel/.+Int8
QuantizedSubOpModel/.+
# svdf_test
SVDFOpTest/BlackBoxTestRank1
SVDFOpTest/BlackBoxTestRank2
# tile_test
-TileTest/TileTest/Int64.+/.+
-TileTest/TileTest/Boolean.+/.+
-TileTest/TileTest/String.+/.+
# Const tensor only
TileTest/TileTest/.+/0,29
# topk_v2_test
-TopKV2OpTest/TopKV2OpTest/.+Int64/.+
# Const tensor only
TopKV2OpTest/TopKV2OpTest/.+/0,29
# transpose_test
# death test
-TransposeTest/Test6DInputTensor
-TransposeTest/5DDividedIntoTwo2Ds.*
-TransposeTest/Complex5DTest.*
-TransposeTest/.+DynamicTensor
-TransposeTest/TestRefOps4DInt16
TransposeTest/.+
# transpose_conv_test
-TransposeConvOpTest/TransposeConvOpTest.SimpleTestQuantizedPerChannel16x8/.+
TransposeConvOpTest/TransposeConvOpTest..*Bias.*/0,29
# Const tensor only
TransposeConvOpTest/TransposeConvOpTest/.+/0,29
# unidirectional_sequence_rnn_test
UnidirectionalRNNOpTest/BlackBoxTest,29
UnidirectionalRNNOpTest.TimeMajorBlackBoxTest,29
# unpack_test
# Unpacking along the last axis is not supported
-UnpackOpTest/.+/ThreeOutputsAxisOne
-UnpackOpTest/.+/ThreeOutputsNegativeAxisOne
-UnpackOpTest/.+/ThreeDimensionsOutputs
# Unpacking 5D tensor is not supported
-UnpackOpTest/.+/FiveDimensionsOutputs
# Unpacking a vector to scalar is not supported
-UnpackOpTest/.+/VectorToScalar
# float, int8, uint8 only
UnpackOpTest/(0|2|3)/.+,30
)";
} // namespace tflite
tensorflow/lite/delegates/nnapi/acceleration_test_util.cc
+41
View File
@@ -0,0 +1,41 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/lite/delegates/nnapi/acceleration_test_util.h"
#include <optional>
#include <string>
#include "tensorflow/lite/kernels/acceleration_test_util_internal.h"
namespace tflite {
std::optional<NnapiAccelerationTestParams> GetNnapiAccelerationTestParam(
std::string test_id) {
return GetAccelerationTestParam<NnapiAccelerationTestParams>(test_id);
}
// static
NnapiAccelerationTestParams NnapiAccelerationTestParams::ParseConfigurationLine(
const std::string& conf_line) {
if (conf_line.empty()) {
return NnapiAccelerationTestParams();
}
int min_sdk_version = std::stoi(conf_line);
return NnapiAccelerationTestParams{min_sdk_version};
}
} // namespace tflite
tensorflow/lite/delegates/nnapi/acceleration_test_util.h
+61
View File
@@ -0,0 +1,61 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_DELEGATES_NNAPI_ACCELERATION_TEST_UTIL_H_
#define TENSORFLOW_LITE_DELEGATES_NNAPI_ACCELERATION_TEST_UTIL_H_
// WARNING: this header file is DEPRECATED.
// See https://developer.android.com/ndk/guides/neuralnetworks/migration-guide.
#include <optional>
#include <string>
#include <gtest/gtest.h>
#include "absl/types/optional.h"
#include "tensorflow/lite/delegates/nnapi/nnapi_delegate_kernel.h"
namespace tflite {
// NNAPI specific configuration for the validation allowlist.
class NnapiAccelerationTestParams {
public:
static const char* AccelerationTestConfig() {
return acceleration_test_config_;
}
static NnapiAccelerationTestParams ParseConfigurationLine(
const std::string& conf_line);
explicit NnapiAccelerationTestParams(int min_android_sdk_version)
: min_android_sdk_version_{min_android_sdk_version} {};
NnapiAccelerationTestParams()
: min_android_sdk_version_{delegate::nnapi::kMinSdkVersionForNNAPI} {};
// Minimum SDK version to apply the acceleration validation to.
int MinAndroidSdkVersion() { return min_android_sdk_version_; }
private:
// Content in nnapi_acceleration_test_list.cc.
static const char* const acceleration_test_config_;
int min_android_sdk_version_;
};
// Returns the NNAPI acceleration test configuration for the given test id.
std::optional<NnapiAccelerationTestParams> GetNnapiAccelerationTestParam(
std::string test_id);
} // namespace tflite
#endif // TENSORFLOW_LITE_DELEGATES_NNAPI_ACCELERATION_TEST_UTIL_H_
tensorflow/lite/delegates/nnapi/java/src/main/java/org/tensorflow/lite/nnapi/BUILD
+28
View File
@@ -0,0 +1,28 @@
# WARNING: this package is DEPRECATED.
# See https://developer.android.com/ndk/guides/neuralnetworks/migration-guide.
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
licenses = ["notice"],
)
filegroup(
name = "nnapi_delegate_src",
srcs = [
"NnApiDelegate.java",
"NnApiDelegateImpl.java",
],
visibility = ["//visibility:public"],
)
filegroup(
name = "nnapi_delegate_api_src",
srcs = ["NnApiDelegate.java"],
visibility = ["//visibility:public"],
)
filegroup(
name = "nnapi_delegate_impl_src",
srcs = ["NnApiDelegateImpl.java"],
visibility = ["//visibility:public"],
)
tensorflow/lite/delegates/nnapi/java/src/main/java/org/tensorflow/lite/nnapi/NnApiDelegate.java
+299
View File
@@ -0,0 +1,299 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.nnapi;
import org.tensorflow.lite.Delegate;
import org.tensorflow.lite.InterpreterFactoryApi;
import org.tensorflow.lite.TensorFlowLite;
/**
* {@link Delegate} for NNAPI inference.
*
* @deprecated See https://developer.android.com/ndk/guides/neuralnetworks/migration-guide.
*/
@Deprecated
public class NnApiDelegate implements Delegate, AutoCloseable {
/**
* Interface used for implementing NnApiDelegate. This is for use only by packages in
* org.tensorflow.lite.
*
* @hide
*/
public static interface PrivateInterface extends Delegate, AutoCloseable {
/** See NnApiDelegate#getNnApiErrno. */
public int getNnapiErrno();
@Override
public void close();
}
/** Delegate options. */
public static final class Options {
public Options() {}
/**
* undefined, specifies default behavior. so far, the default setting of NNAPI is
* EXECUTION_PREFERENCE_FAST_SINGLE_ANSWER
*/
public static final int EXECUTION_PREFERENCE_UNDEFINED = -1;
/**
* Prefer executing in a way that minimizes battery drain. This is desirable for compilations
* that will be executed often.
*/
public static final int EXECUTION_PREFERENCE_LOW_POWER = 0;
/**
* Prefer returning a single answer as fast as possible, even if this causes more power
* consumption.
*/
public static final int EXECUTION_PREFERENCE_FAST_SINGLE_ANSWER = 1;
/**
* Prefer maximizing the throughput of successive frames, for example when processing successive
* frames coming from the camera.
*/
public static final int EXECUTION_PREFERENCE_SUSTAINED_SPEED = 2;
/**
* Sets the inference preference for precision/compilation/runtime tradeoffs.
*
* @param preference One of EXECUTION_PREFERENCE_LOW_POWER,
* EXECUTION_PREFERENCE_FAST_SINGLE_ANSWER, or EXECUTION_PREFERENCE_SUSTAINED_SPEED.
*/
public Options setExecutionPreference(int preference) {
this.executionPreference = preference;
return this;
}
/**
* Specifies the name of the target accelerator to be used by NNAPI. If this parameter is
* specified the {@link #setUseNnapiCpu(boolean)} method won't have any effect.
*
* <p>Only effective on Android 10 (API level 29) and above.
*/
public Options setAcceleratorName(String name) {
this.acceleratorName = name;
return this;
}
/**
* Configure the location to be used to store model compilation cache entries. If either {@code
* cacheDir} or {@code modelToken} parameters are unset NNAPI caching will be disabled.
*
* <p>Only effective on Android 10 (API level 29) and above.
*/
public Options setCacheDir(String cacheDir) {
this.cacheDir = cacheDir;
return this;
}
/**
* Sets the token to be used to identify this model in the model compilation cache. If either
* {@code cacheDir} or {@code modelToken} parameters are unset NNAPI caching will be disabled.
*
* <p>Only effective on Android 10 (API level 29) and above.
*/
public Options setModelToken(String modelToken) {
this.modelToken = modelToken;
return this;
}
/**
* Sets the maximum number of graph partitions that the delegate will try to delegate. If more
* partitions could be delegated than the limit, the ones with the larger number of nodes will
* be chosen. If unset it will use the NNAPI default limit.
*/
public Options setMaxNumberOfDelegatedPartitions(int limit) {
this.maxDelegatedPartitions = limit;
return this;
}
/**
* Enable or disable the NNAPI CPU Device "nnapi-reference". If unset it will use the NNAPI
* default settings.
*
* <p>Only effective on Android 10 (API level 29) and above.
*/
public Options setUseNnapiCpu(boolean enable) {
this.useNnapiCpu = enable;
return this;
}
/**
* Enable or disable to allow fp32 computation to be run in fp16 in NNAPI. See
* https://source.android.com/devices/neural-networks#android-9
*
* <p>Only effective on Android 9 (API level 28) and above.
*/
public Options setAllowFp16(boolean enable) {
this.allowFp16 = enable;
return this;
}
public Options setNnApiSupportLibraryHandle(long handle) {
this.nnApiSupportLibraryHandle = handle;
return this;
}
/**
* Returns the inference preference for precision/compilation/runtime tradeoffs.
*
* @return One of EXECUTION_PREFERENCE_LOW_POWER, EXECUTION_PREFERENCE_FAST_SINGLE_ANSWER, or
* EXECUTION_PREFERENCE_SUSTAINED_SPEED.
*/
public int getExecutionPreference() {
return this.executionPreference;
}
/** Returns the name of the target accelerator to be used by NNAPI. */
public String getAcceleratorName() {
return this.acceleratorName;
}
/** Returns the location to be used to store model compilation cache entries. */
public String getCacheDir() {
return this.cacheDir;
}
/** Returns the token to be used to identify this model in the model compilation cache. */
public String getModelToken() {
return this.modelToken;
}
/**
* Returns the maximum number of graph partitions that the delegate will try to delegate. Or -1
* if no maximum has neen set.
*/
public int getMaxNumberOfDelegatedPartitions() {
return this.maxDelegatedPartitions == null ? -1 : this.maxDelegatedPartitions;
}
/**
* Returns whether the NNAPI CPU Device "nnapi-reference" is enabled.
*
* @return true if the NNAPI CPU Device "nnapi-reference" is explicitly enabled.
* @return false if the NNAPI CPU Device "nnapi-reference" is explicitly disabled.
* @return null if use of the NNAPI CPU Device "nnapi-reference" will be determined by the NNAPI
* default settings.
*/
public Boolean getUseNnapiCpu() {
return this.useNnapiCpu;
}
/** Returns true if allowing fp32 computation to be run in fp16 in NNAPI is enabled. */
public boolean getAllowFp16() {
return this.allowFp16 != null && this.allowFp16;
}
/** Returns the native handle to the NNAPI Support Library. */
public long getNnApiSupportLibraryHandle() {
return this.nnApiSupportLibraryHandle;
}
private int executionPreference = EXECUTION_PREFERENCE_UNDEFINED;
private String acceleratorName = null;
private String cacheDir = null;
private String modelToken = null;
private Integer maxDelegatedPartitions = null;
private Boolean useNnapiCpu = null;
private Boolean allowFp16 = null;
private long nnApiSupportLibraryHandle = 0;
}
private Options options;
private PrivateInterface impl;
private boolean initialized;
public NnApiDelegate(Options options) {
// Ensure the native TensorFlow Lite libraries are available.
TensorFlowLite.init();
this.options = options;
// this.impl will be initialized later.
}
public NnApiDelegate() {
this(new Options());
}
/**
* Called when an interpreter is constructed with this delegate. This method is only for use by TF
* Lite itself, and should not be used by application developers.
*
* @hide
*/
public void initWithInterpreterFactoryApi(InterpreterFactoryApi interpreterFactoryApi) {
impl = interpreterFactoryApi.createNnApiDelegateImpl(options);
initialized = true;
}
@Override
public long getNativeHandle() {
checkNotClosed();
return impl.getNativeHandle();
}
/**
* Frees TFLite resources in C runtime.
*
* <p>User is expected to call this method explicitly.
*/
@Override
public void close() {
if (impl != null) {
impl.close();
impl = null;
}
}
/**
* Returns the latest error code returned by an NNAPI call or zero if NO calls to NNAPI failed.
* The error code is reset when the delegate is associated with an <a
* href=https://www.tensorflow.org/lite/api_docs/java/org/tensorflow/lite/Interpreter>interpreter</a>.
*
* <p>For details on NNAPI error codes see <a
* href="https://developer.android.com/ndk/reference/group/neural-networks#resultcode">the NNAPI
* documentation</a>.
*
* @throws IllegalStateException if the method is called after {@link #close() close}.
*/
public int getNnapiErrno() {
if (!initialized) {
return 0 /*ANEURALNETWORKS_NO_ERROR*/;
}
checkNotClosed();
return impl.getNnapiErrno();
}
/**
* Returns true if any NNAPI call failed since this delegate was associated with an <a
* href=https://www.tensorflow.org/lite/api_docs/java/org/tensorflow/lite/Interpreter>interpreter</a>.
*
* @throws IllegalStateException if the method is called after {@link #close() close}.
*/
public boolean hasErrors() {
return getNnapiErrno() != 0 /*ANEURALNETWORKS_NO_ERROR*/;
}
private void checkNotClosed() {
if (impl == null) {
throw new IllegalStateException(
initialized
? "Should not access delegate after delegate has been closed."
: "Should not access delegate before interpreter has been constructed.");
}
}
}
tensorflow/lite/delegates/nnapi/java/src/main/java/org/tensorflow/lite/nnapi/NnApiDelegateImpl.java
+106
View File
@@ -0,0 +1,106 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.nnapi;
import org.tensorflow.lite.Delegate;
import org.tensorflow.lite.TensorFlowLite;
/**
* Implementation of {@link Delegate} for NNAPI inference. Only for use by packages in
* org.tensorflow.lite.
*
* @hide
*/
public class NnApiDelegateImpl implements NnApiDelegate.PrivateInterface, Delegate, AutoCloseable {
private static final long INVALID_DELEGATE_HANDLE = 0;
private long delegateHandle;
public NnApiDelegateImpl(NnApiDelegate.Options options) {
// Ensure the native TensorFlow Lite libraries are available.
TensorFlowLite.init();
delegateHandle =
createDelegate(
options.getExecutionPreference(),
options.getAcceleratorName(),
options.getCacheDir(),
options.getModelToken(),
options.getMaxNumberOfDelegatedPartitions(),
/*overrideDisallowCpu=*/ options.getUseNnapiCpu() != null,
/*disallowCpuValue=*/ options.getUseNnapiCpu() != null
? !options.getUseNnapiCpu().booleanValue()
: true,
options.getAllowFp16(),
options.getNnApiSupportLibraryHandle());
}
@Override
public long getNativeHandle() {
return delegateHandle;
}
/**
* Frees TFLite resources in C runtime.
*
* <p>User is expected to call this method explicitly.
*/
@Override
public void close() {
if (delegateHandle != INVALID_DELEGATE_HANDLE) {
deleteDelegate(delegateHandle);
delegateHandle = INVALID_DELEGATE_HANDLE;
}
}
/**
* Returns the latest error code returned by an NNAPI call or zero if NO calls to NNAPI failed.
* The error code is reset when the delegate is associated with an <a
* href=https://www.tensorflow.org/lite/api_docs/java/org/tensorflow/lite/Interpreter>interpreter</a>.
*
* <p>For details on NNAPI error codes see <a
* href="https://developer.android.com/ndk/reference/group/neural-networks#resultcode">the NNAPI
* documentation</a>.
*
* @throws IllegalStateException if the method is called after {@link #close() close}.
*/
@Override
public int getNnapiErrno() {
checkNotClosed();
return getNnapiErrno(delegateHandle);
}
private void checkNotClosed() {
if (delegateHandle == INVALID_DELEGATE_HANDLE) {
throw new IllegalStateException("Should not access delegate after it has been closed.");
}
}
private static native long createDelegate(
int preference,
String deviceName,
String cacheDir,
String modelToken,
int maxDelegatedPartitions,
boolean overrideDisallowCpu,
boolean disallowCpuValue,
boolean allowFp16,
long nnApiSupportLibraryHandle);
private static native void deleteDelegate(long delegateHandle);
private static native int getNnapiErrno(long delegateHandle);
}
tensorflow/lite/delegates/nnapi/java/src/main/native/BUILD
+34
View File
@@ -0,0 +1,34 @@
# Description:
# Java Native Interface (JNI) library intended for implementing the
# TensorFlow Lite NNAPI Delegate Java API using the TensorFlow Lite CC library.
load("//tensorflow/lite:build_def.bzl", "tflite_copts")
load("//tensorflow/lite:special_rules.bzl", "nnapi_native_srcs_visibility_allowlist")
load("//tensorflow/lite/core/shims:cc_library_with_tflite.bzl", "cc_library_with_tflite")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
default_visibility = ["//visibility:public"],
licenses = ["notice"],
)
cc_library_with_tflite(
name = "native",
srcs = ["nnapi_delegate_impl_jni.cc"],
copts = tflite_copts(),
tags = [
"manual",
"notap",
],
deps = [
"//tensorflow/lite/delegates/nnapi:nnapi_delegate",
"//tensorflow/lite/java/jni",
],
alwayslink = 1,
)
filegroup(
name = "native_srcs",
srcs = ["nnapi_delegate_impl_jni.cc"],
visibility = nnapi_native_srcs_visibility_allowlist(),
)
tensorflow/lite/delegates/nnapi/java/src/main/native/nnapi_delegate_impl_jni.cc
+211
View File
@@ -0,0 +1,211 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <jni.h>
#include <memory>
#include <type_traits>
#if TFLITE_DISABLE_SELECT_JAVA_APIS
#include "flatbuffers/flatbuffers.h" // from @flatbuffers
#include "tensorflow/lite/acceleration/configuration/c/nnapi_plugin.h"
#include "tensorflow/lite/acceleration/configuration/configuration_generated.h"
#include "tensorflow/lite/c/common.h"
#else
#include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
#endif
#if TFLITE_DISABLE_SELECT_JAVA_APIS
using flatbuffers::FlatBufferBuilder;
using tflite::NNAPISettings;
using tflite::NNAPISettingsBuilder;
using tflite::TFLiteSettings;
using tflite::TFLiteSettingsBuilder;
#else
using tflite::StatefulNnApiDelegate;
#endif
extern "C" {
JNIEXPORT jlong JNICALL
Java_org_tensorflow_lite_nnapi_NnApiDelegateImpl_createDelegate(
JNIEnv* env, jclass clazz, jint preference, jstring accelerator_name,
jstring cache_dir, jstring model_token, jint max_delegated_partitions,
jboolean override_disallow_cpu, jboolean disallow_cpu_value,
jboolean allow_fp16, jlong nnapi_support_library_handle) {
#if TFLITE_DISABLE_SELECT_JAVA_APIS
// Use NNAPI Delegate via Delegate Plugin API.
// This approach would work for the !TFLITE_DISABLE_SELECT_JAVA_APIS case too,
// but may have a slightly higher overhead due to the need to construct
// a FlatBuffer for the configuration parameters.
// Construct a FlatBuffer that contains the following:
// TFLiteSettings {
// NnapiSettings {
// accelerator_name : <accelerator_name>,
// cache_directory : <cache_dir>,
// model_token : <model_token>,
// allow_nnapi_cpu_on_android_10_plus: !<disallow_cpu_value>,
// allow_fp16_precision_for_fp32: <allow_fp16>,
// support_library_handle: <nnapi_support_library_handle>,
// }
// max_delegate_partitions: <max_delegated_partitions>
// }
// where the values in angle brackets are the parameters to this function,
// except that we set the 'allow_nnapi_cpu_on_android_10_plus' field only if
// <override_disallow_cpu> is true, and that we only set the other fields
// if they have non-default values.
FlatBufferBuilder flatbuffer_builder;
flatbuffers::Offset<flatbuffers::String> accelerator_name_fb_string = 0;
if (accelerator_name) {
const char* accelerator_name_c_string =
env->GetStringUTFChars(accelerator_name, nullptr);
accelerator_name_fb_string =
flatbuffer_builder.CreateString(accelerator_name_c_string);
env->ReleaseStringUTFChars(accelerator_name, accelerator_name_c_string);
}
flatbuffers::Offset<flatbuffers::String> cache_directory_fb_string = 0;
if (cache_dir) {
const char* cache_directory_c_string =
env->GetStringUTFChars(cache_dir, nullptr);
cache_directory_fb_string =
flatbuffer_builder.CreateString(cache_directory_c_string);
env->ReleaseStringUTFChars(cache_dir, cache_directory_c_string);
}
flatbuffers::Offset<flatbuffers::String> model_token_fb_string = 0;
if (model_token) {
const char* model_token_c_string =
env->GetStringUTFChars(model_token, nullptr);
model_token_fb_string =
flatbuffer_builder.CreateString(model_token_c_string);
env->ReleaseStringUTFChars(model_token, model_token_c_string);
}
NNAPISettingsBuilder nnapi_settings_builder(flatbuffer_builder);
nnapi_settings_builder.add_execution_preference(
static_cast<tflite::NNAPIExecutionPreference>(preference));
if (accelerator_name) {
nnapi_settings_builder.add_accelerator_name(accelerator_name_fb_string);
}
if (cache_dir) {
nnapi_settings_builder.add_cache_directory(cache_directory_fb_string);
}
if (model_token) {
nnapi_settings_builder.add_model_token(model_token_fb_string);
}
if (override_disallow_cpu) {
nnapi_settings_builder.add_allow_nnapi_cpu_on_android_10_plus(
!disallow_cpu_value);
}
if (allow_fp16) {
nnapi_settings_builder.add_allow_fp16_precision_for_fp32(allow_fp16);
}
if (nnapi_support_library_handle) {
nnapi_settings_builder.add_support_library_handle(
nnapi_support_library_handle);
}
flatbuffers::Offset<NNAPISettings> nnapi_settings =
nnapi_settings_builder.Finish();
TFLiteSettingsBuilder tflite_settings_builder(flatbuffer_builder);
tflite_settings_builder.add_nnapi_settings(nnapi_settings);
if (max_delegated_partitions >= 0) {
tflite_settings_builder.add_max_delegated_partitions(
max_delegated_partitions);
}
flatbuffers::Offset<TFLiteSettings> tflite_settings =
tflite_settings_builder.Finish();
flatbuffer_builder.Finish(tflite_settings);
const TFLiteSettings* settings = flatbuffers::GetRoot<TFLiteSettings>(
flatbuffer_builder.GetBufferPointer());
// Construct the delegate using the Delegate Plugin C API,
// and passing in the flatbuffer settings that we constructed above.
TfLiteOpaqueDelegate* nnapi_delegate =
TfLiteNnapiDelegatePluginCApi()->create(settings);
return reinterpret_cast<jlong>(nnapi_delegate);
#else
// Use NNAPI Delegate directly.
// Construct an Options object for the parameter settings.
StatefulNnApiDelegate::Options options = StatefulNnApiDelegate::Options();
options.execution_preference =
(StatefulNnApiDelegate::Options::ExecutionPreference)preference;
if (accelerator_name) {
options.accelerator_name =
env->GetStringUTFChars(accelerator_name, nullptr);
}
if (cache_dir) {
options.cache_dir = env->GetStringUTFChars(cache_dir, nullptr);
}
if (model_token) {
options.model_token = env->GetStringUTFChars(model_token, nullptr);
}
if (max_delegated_partitions >= 0) {
options.max_number_delegated_partitions = max_delegated_partitions;
}
if (override_disallow_cpu) {
options.disallow_nnapi_cpu = disallow_cpu_value;
}
if (allow_fp16) {
options.allow_fp16 = allow_fp16;
}
// Construct the delegate, using the options object constructed earlier.
auto delegate =
nnapi_support_library_handle
? new StatefulNnApiDelegate(reinterpret_cast<NnApiSLDriverImplFL5*>(
nnapi_support_library_handle),
options)
: new StatefulNnApiDelegate(options);
// Deallocate temporary strings.
if (options.accelerator_name) {
env->ReleaseStringUTFChars(accelerator_name, options.accelerator_name);
}
if (options.cache_dir) {
env->ReleaseStringUTFChars(cache_dir, options.cache_dir);
}
if (options.model_token) {
env->ReleaseStringUTFChars(model_token, options.model_token);
}
return reinterpret_cast<jlong>(delegate);
#endif
}
JNIEXPORT jint JNICALL
Java_org_tensorflow_lite_nnapi_NnApiDelegateImpl_getNnapiErrno(JNIEnv* env,
jclass clazz,
jlong delegate) {
#if TFLITE_DISABLE_SELECT_JAVA_APIS
TfLiteOpaqueDelegate* nnapi_delegate =
reinterpret_cast<TfLiteOpaqueDelegate*>(delegate);
return TfLiteNnapiDelegatePluginCApi()->get_delegate_errno(nnapi_delegate);
#else
StatefulNnApiDelegate* nnapi_delegate =
reinterpret_cast<StatefulNnApiDelegate*>(delegate);
return nnapi_delegate->GetNnApiErrno();
#endif
}
JNIEXPORT void JNICALL
Java_org_tensorflow_lite_nnapi_NnApiDelegateImpl_deleteDelegate(
JNIEnv* env, jclass clazz, jlong delegate) {
#if TFLITE_DISABLE_SELECT_JAVA_APIS
TfLiteOpaqueDelegate* nnapi_delegate =
reinterpret_cast<TfLiteOpaqueDelegate*>(delegate);
TfLiteNnapiDelegatePluginCApi()->destroy(nnapi_delegate);
#else
delete reinterpret_cast<StatefulNnApiDelegate*>(delegate);
#endif
}
} // extern "C"
tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
+7101
View File
File diff suppressed because it is too large Load Diff
tensorflow/lite/delegates/nnapi/nnapi_delegate.h
+463
View File
@@ -0,0 +1,463 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_DELEGATES_NNAPI_NNAPI_DELEGATE_H_
#define TENSORFLOW_LITE_DELEGATES_NNAPI_NNAPI_DELEGATE_H_
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
#include "tensorflow/lite/core/c/common.h"
#include "tensorflow/lite/delegates/serialization.h"
#include "tensorflow/lite/nnapi/NeuralNetworksTypes.h"
#include "tensorflow/lite/nnapi/nnapi_implementation.h"
// WARNING: this header file is DEPRECATED.
// See https://developer.android.com/ndk/guides/neuralnetworks/migration-guide.
struct NnApiSLDriverImplFL5;
struct NnapiDelegateVendorPlugin;
typedef struct ANeuralNetworksMemory ANeuralNetworksMemory;
namespace tflite {
namespace delegate {
namespace nnapi {
class NNAPIDelegateKernel;
} // namespace nnapi
} // namespace delegate
using tflite::delegate::nnapi::NNAPIDelegateKernel;
// TFliteDelegate to interface with NNAPI.
class StatefulNnApiDelegate : public TfLiteDelegate {
public:
// Encapsulates all options that are specific to NNAPI delegate.
struct Options {
// Preferred Power/perf trade-off. For more details please see
// ANeuralNetworksCompilation_setPreference documentation in :
// https://developer.android.com/ndk/reference/group/neural-networks.html
enum ExecutionPreference {
kUndefined = -1,
kLowPower = 0,
kFastSingleAnswer = 1,
kSustainedSpeed = 2,
};
// Preferred Power/perf trade-off.
ExecutionPreference execution_preference = kUndefined;
// Selected NNAPI accelerator with nul-terminated name.
// Default to nullptr, which implies the NNAPI default behavior: NNAPI
// runtime is allowed to use all available accelerators. If the selected
// accelerator cannot be found, NNAPI will not be used.
// It is the caller's responsibility to ensure the string is valid for the
// duration of the Options object lifetime.
const char* accelerator_name = nullptr;
// The nul-terminated cache dir for NNAPI model.
// Default to nullptr, which implies the NNAPI will not try caching the
// compilation.
const char* cache_dir = nullptr;
// The unique nul-terminated token string for NNAPI model.
// Default to nullptr, which implies the NNAPI will not try caching the
// compilation. It is the caller's responsibility to ensure there is no
// clash of the tokens.
// NOTE: when using compilation caching, it is not recommended to use the
// same delegate instance for multiple models.
const char* model_token = nullptr;
// Whether to disallow NNAPI CPU usage. Only effective on Android 10 and
// above. The NNAPI CPU typically performs less well than built-in TfLite
// kernels, but allowing CPU allows partial acceleration of models. If this
// is set to true, NNAPI is only used if the whole model is accelerated.
bool disallow_nnapi_cpu = true;
// Specifies the max number of partitions to delegate. A value <= 0 means
// no limit.
// If the delegation of the full set of supported nodes would generate a
// number of partition greater than this parameter, only
// <max_number_delegated_partitions> of them will be actually accelerated.
// The selection is currently done sorting partitions in decreasing order
// of number of nodes and selecting them until the limit is reached.
int max_number_delegated_partitions = 3;
// allow fp32 computation to be run in fp16.
bool allow_fp16 = false;
// Specifies the relative priority for executions of the model.
// Available values are {ANEURALNETWORKS_PRIORITY_LOW,
// ANEURALNETWORKS_PRIORITY_MEDIUM, ANEURALNETWORKS_PRIORITY_HIGH,
// ANEURALNETWORKS_PRIORITY_DEFAULT}.
int execution_priority = ANEURALNETWORKS_PRIORITY_DEFAULT;
// Specifies the maximum expected duration in nanosecond for compiling the
// model. If the device is not able to complete the compilation within the
// specified duration, the compilation may be aborted. If set to 0, the
// timeout duration is considered infinite.
uint64_t max_compilation_timeout_duration_ns = 0;
// Specifies the maximum expected duration in nanosecond for executing the
// model. If the device is not able to complete the execution within the
// specified duration, the execution may be aborted. If set to 0, the
// timeout duration is considered infinite.
uint64_t max_execution_timeout_duration_ns = 0;
// Specifies the maximum expected duration in nanosecond for WHILE loops in
// the execution. If a WHILE loop condition model does not output false
// within the specified duration, the execution will be aborted. If set to
// 0, the default timeout for loops will be used.
uint64_t max_execution_loop_timeout_duration_ns = 0;
// Whether to allow dynamic dimension sizes without re-compilation.
// A tensor of with dynamic dimension must have a valid dim_signature
// defined.
// Only supported in NNAPI 1.1 and newer versions.
// WARNING: Setting this flag to true may result in model being rejected by
// accelerator. This should only be enabled if the target device supports
// dynamic dimensions of the model.
bool allow_dynamic_dimensions = false;
// Force using NNAPI Burst mode if supported.
// Burst mode allows accelerators to efficiently manage resources, which
// would significantly reduce overhead especially if the same delegate
// instance is to be used for multiple inferences.
// If NNAPI devices are specified and are of NNAPI feature level 5 or
// higher, NNAPI delegate will automatically enable burst mode for better
// performance.
// Default: Disabled for devices with NNAPI feature level 4 or lower.
bool use_burst_computation = false;
// Specifies the max number of NNAPI reusable executions to cache. An
// execution can be reused if the input and output tensors are using the
// same buffer handles, and all dynamic dimensions are unchanged. Setting
// this field to 0 means do not reuse execution.
uint32_t max_execution_cache_size = 4;
// Provides hints about the max size of tensors with dynamic shapes. The key
// of the map is the tensor index, and the value is the max size of the
// tensor in bytes. If a vendor plugin is supplied, this field is required
// for all output tensors with dynamic shapes because the output size cannot
// be inferred. Otherwise, this field is optional and any provided
// information may be used to guide the memory allocation. This field has no
// effect on tensors with static shapes.
std::map<int, size_t> tensor_max_size_hints;
// The optional null-terminated vendor specific compilation hints string.
// It is the vendor_plugin's responsibility to parse the hint string and
// decide whether the hints should be respected or not. If no vendor_plugin
// provided, the hints will be ignored.
const char* vendor_compilation_hints = nullptr;
// The optional null-terminated vendor specific execution hints string.
// It is the vendor_plugin's responsibility to parse the hint string and
// decide whether the hints should be respected or not. If no vendor_plugin
// provided, the hints will be ignored.
const char* vendor_execution_hints = nullptr;
// It is the users responsibility to make sure that
// vendor_plugin outlives the delegate instance.
// If a vendor plugin is supplied, and the model has dynamic dimensions, the
// delegate is not able to propagate tensor shapes. In such a case, the user
// must provide max tensor size in the "tensor_max_size_hints" field for all
// output tensors with dynamic shapes.
NnapiDelegateVendorPlugin* vendor_plugin = nullptr;
// Controls disabling of the debugging diagnostics callbacks that only print
// debug logs, which are otherwise enabled by default.
// Use this in case different callbacks are being registered elsewhere, such
// as for example to send logs through some logger.
bool disable_debugging_diagnostics_callbacks = false;
};
// Uses default options.
StatefulNnApiDelegate();
// The ownership of the NnApi instance is left to the caller of the
// StatefulNnApiDelegate constructor; the caller must ensure that the lifetime
// of the NnApi instance exceeds the lifetime of the StatefulNnApiDelegate.
explicit StatefulNnApiDelegate(const NnApi* nnapi);
// The constructor that accepts options from user.
// This makes a copy of any data that it needs from Options, so
// the caller can safely deallocate any storage pointed to by
// the 'const char *' members of Options immediately after calling this.
explicit StatefulNnApiDelegate(Options options);
// Constructor that accepts both an NnApi instance and options.
// The ownership of the NnApi instance is left to the caller of the
// StatefulNnApiDelegate constructor; the caller must ensure that the lifetime
// of the NnApi instance exceeds the lifetime of the StatefulNnApiDelegate.
// This constructor makes a copy of any data that it needs from Options, so
// the caller can safely deallocate any storage pointed to by
// the 'const char *' members of Options immediately after calling this.
StatefulNnApiDelegate(const NnApi* nnapi, Options options);
// Constructor that accepts an NnApiSLDriverImplFL5 instance and options.
// The ownership of the NnApiSLDriverImplFL5 instance is left to the caller of
// the StatefulNnApiDelegate constructor; the caller must ensure that the
// lifetime of the NnApiSLDriverImplFL5 instance encompasses all calls to
// methods on the StatefulNnApiDelegate instance, other than the destructor.
// This constructor makes a copy of any data that it needs from Options, so
// the caller can safely deallocate any storage pointed to by
// the 'const char *' members of Options immediately after calling this.
//
// The NN API Support Library Driver must support at least NNAPI Feature Level
// 5 (introduced in SDK level 31), but this might point to a compatible struct
// that also supports a higher NNAPI Feature Level. These cases can be
// distinguished by examining the base.implFeatureLevel field, which should be
// set to the supported feature level (which must be >=
// ANEURALNETWORKS_FEATURE_LEVEL_5).
//
// Please note that since NNAPI Support Library doesn't implement some of the
// functions (see CreateNnApiFromSupportLibrary implementation and NNAPI SL
// documentation for details), the underlying NnApi structure will have
// nullptr stored in some of the function pointers. Calling such functions
// will result in a crash.
//
// WARNING: This is an experimental interface that is subject to change.
StatefulNnApiDelegate(
const NnApiSLDriverImplFL5* nnapi_support_library_driver,
Options options);
~StatefulNnApiDelegate() = default;
// Returns the delegate options.
// The lifetime of the storage pointed to by the 'const char *' members of the
// returned Options object is the same as the lifetime of the supplied
// TfLiteDelegate instance.
static const Options GetOptions(TfLiteDelegate* delegate);
// Callback function which copies data from ANeuralNetworksMemory to host
// tensor CPU buffer. It is the users responsibility to implement these
// callbacks for the specific types of shared memory they intend to use.
// WARNING: This is an experimental interface that is subject to change.
typedef TfLiteStatus (*CopyToHostTensorFnPtr)(TfLiteTensor* tensor,
ANeuralNetworksMemory* memory,
size_t memory_offset,
size_t byte_size,
void* callback_context);
// Encapsulates all fields related to memory registration for internal
// bookkeeping only.
struct MemoryRegistration {
ANeuralNetworksMemory* memory;
CopyToHostTensorFnPtr callback;
void* callback_context;
// The registration timestamp. It is unique for each registered memory in
// the lifetime of a StatefulNnApiDelegate.
uint64_t timestamp;
};
// Register the ANeuralNetworksMemory handle with the delegate. A
// TfLiteBufferHandle will be returned to be used with
// Interpreter::SetBufferHandle. The callback_context will be passed to the
// callback function when invoked.
// Note: the returned TfLiteBufferHandle can only be used with a single
// Interpreter instance. However, the caller can register the same memory
// multiple times to get different handles to use with difference Interpreter
// instances
// WARNING: This is an experimental interface that is subject to change.
TfLiteBufferHandle RegisterNnapiMemory(ANeuralNetworksMemory* memory,
CopyToHostTensorFnPtr callback,
void* callback_context);
// Returns the vector of known ANeuralNetworksMemory handles.
// Note: this function is not intended to be called by developers.
// WARNING: This is an experimental interface that is subject to change.
static const std::vector<MemoryRegistration>& GetTensorMemoryMap(
TfLiteDelegate* delegate);
// Returns ptr to delegates::Serialization, if caching is enabled by user via
// cache_dir & model_token.
static delegates::Serialization* GetCache(TfLiteDelegate* delegate);
// Returns the int value of the ResultCode returned by the latest
// failed call to NNAPI, if any. Zero only in case of NO failed calls since
// the construction of this instance of StatefulNnApiDelegate.
// The error code is reset when the delegate is re-initialized
// (i.e. when calling interpreter.ModifyGraphWithDelegate(delegate)).
int GetNnApiErrno() const;
private:
// Encapsulates all delegate data.
struct Data {
// Pointer to NNAPI implementation to be used by this delegate as
// set when building the StatefulNnApiDelegate instance.
// Will generally be the NnApiInstance() singleton but can be overridden
// for testing or for users needing to wrap or stub parts of NNAPI.
// The ownership of the nnapi instance is left to the caller of
// the StatefulNnApiDelegate constructor.
const NnApi* nnapi;
// Preferred Power/perf trade-off.
Options::ExecutionPreference execution_preference;
// Selected NNAPI accelerator name.
std::string accelerator_name;
// The cache dir for NNAPI model.
std::string cache_dir;
// The unique token string for NNAPI model.
std::string model_token;
// Whether to disallow NNAPI CPU.
bool disallow_nnapi_cpu;
// Tensor to ANeuralNetworksMemory mapping.
std::vector<MemoryRegistration> tensor_memory_map;
// The next timestamp for buffer handle registration.
uint64_t next_buffer_handle_timestamp = 1;
// Contains a non zero value if any NNAPI method call
// operation returned a non zero result code.
int nnapi_errno = ANEURALNETWORKS_NO_ERROR;
// Cache of kernels already built in StatefulNnApiDelegate::DoPrepare
// when trying to understand if all nodes are supported by the target
// accelerators.
// The key is the index of the first node in the partition.
// Couldn't use unique_ptr because of problems building on gcc
std::unordered_map<int, NNAPIDelegateKernel*> delegate_state_cache;
// Maximum number of NNAPI partition to delegate. Zero or negative means
// no limit. Copied from StatefulNnApiDelegate::Options
int max_number_delegated_partitions;
// allow fp32 computation to be run in fp16.
bool allow_fp16;
// Specifies the relative priority for executions of the model.
int execution_priority = ANEURALNETWORKS_PRIORITY_DEFAULT;
// Specifies the maximum expected duration in nanosecond for compiling the
// model.
uint64_t max_compilation_timeout_duration_ns = 0;
// Specifies the maximum expected duration in nanosecond for executing the
// model.
uint64_t max_execution_timeout_duration_ns = 0;
// Specifies the maximum expected duration in nanosecond for WHILE loops in
// the execution
uint64_t max_execution_loop_timeout_duration_ns = 0;
// Whether to allow dynamic dimension sizes without re-compilation.
bool allow_dynamic_dimensions = false;
// Whether to use NNAPI Burst mode.
bool use_burst_computation = false;
// Specifies the max number of NNAPI reusable executions to cache.
uint32_t max_execution_cache_size = 4;
// Provides hints about the max size of tensors with dynamic shapes.
std::map<int, size_t> tensor_max_size_hints;
// The null-terminated vendor specific compilation hints string
const char* vendor_compilation_hints = nullptr;
// The null-terminated vendor specific execution hints string.
const char* vendor_execution_hints = nullptr;
// It is the users responsibility to make sure that
// vendor_plugin outlives the delegate instance.
NnapiDelegateVendorPlugin* vendor_plugin = nullptr;
// Smart pointer for automatically cleaning up NnApi structure in case the
// delegate was constructed from an NNAPI support library
std::unique_ptr<const NnApi> owned_nnapi = nullptr;
// TFLite Serialization in case caching has been enabled by the user through
// Options.
std::unique_ptr<delegates::Serialization> cache;
// Controls disabling of the default diagnostics callbacks that only print
// debug logs, which are otherwise enabled by default.
// Use this in case different callbacks are being registered elsewhere, such
// as for example to send logs through some logger.
bool disable_debugging_diagnostics_callbacks = false;
explicit Data(const NnApi* nnapi);
explicit Data(std::unique_ptr<const NnApi> nnapi);
~Data();
// Caches an initialised NNAPIDelegateKernel.
void CacheDelegateKernel(const TfLiteDelegateParams* delegate_params,
NNAPIDelegateKernel* delegate_state);
// Returns a cached NNAPIDelegateKernel if available and removes it
// from the cache transferring the ownership to the caller.
NNAPIDelegateKernel* MaybeGetCachedDelegateKernel(
const TfLiteDelegateParams* delegate_params);
};
// Implements TfLiteDelegate::Prepare. Please refer to TFLiteDelegate
// documentation for more info.
static TfLiteStatus DoPrepare(TfLiteContext* context,
TfLiteDelegate* delegate);
// Copy the data from delegate buffer handle into raw memory of the given
// 'tensor'. The delegate is allowed to allocate the raw
// bytes as long as it follows the rules for kTfLiteDynamic tensors.
static TfLiteStatus DoCopyFromBufferHandle(TfLiteContext* context,
TfLiteDelegate* delegate,
TfLiteBufferHandle buffer_handle,
TfLiteTensor* tensor);
// Copy the data from raw memory of the given 'tensor' to delegate buffer
// handle. Currently this function is not supported, and calling the function
// will result in an error.
static TfLiteStatus DoCopyToBufferHandle(TfLiteContext* context,
TfLiteDelegate* delegate,
TfLiteBufferHandle buffer_handle,
TfLiteTensor* tensor);
// Free the Delegate Buffer Handle. Note: This only frees the handle, but
// this doesn't release the underlying resource (e.g. textures). The
// resources are either owned by application layer or the delegate.
static void DoFreeBufferHandle(TfLiteContext* context,
TfLiteDelegate* delegate,
TfLiteBufferHandle* handle);
// Returns the nodes that can be delegated via NNAPI to the accelerator
// specified in the delegate options and information about the way the
// graph will be partitioned if the supported nodes will be delegated.
// Partition information is composed by the number of partitions and
// the delegate parameters associated to each partition.
// The method also caches in delegate->data the NNApiDelegateKernel instances
// that have been created during the device evaluation.
// All arguments are expected to be non-null.
static TfLiteStatus GetNodesSupportedByAccelerator(
TfLiteContext* context, TfLiteDelegate* delegate, const NnApi* nnapi,
const std::vector<int>& supported_nodes,
std::vector<int>* device_supported_nodes, int* num_partitions,
TfLiteDelegateParams** params_array, int* nnapi_errno);
// Alters the given array of nodes_to_delegate to limit the number of NNAPI
// owned partition to be less or equal than num_partitions. If num_partitions
// is less or equal to zero the input is left unaltered.
// The nodes_to_delegate array is expected to contain at element 0 the number
// of nodes to delegate and in remaining elements the set of nodes
// that would be delegated to NNAPI if this function wouldn't be
// called. It will be altered storing in the first element the count of
// nodes to actually delegate and in the remainder of the array the indexes.
// The params_array params might be altered during the functions execution.
static TfLiteStatus LimitDelegatedPartitions(
int max_partitions,
std::vector<TfLiteDelegateParams> partition_params_array,
std::vector<int>* nodes_to_delegate);
void StatefulNnApiDelegateConstructorImpl(const Options& options);
// Delegate data presented through TfLiteDelegate::data_.
Data delegate_data_;
};
// DEPRECATED: Please use StatefulNnApiDelegate class instead.
//
// Returns a singleton delegate that can be used to use the NN API.
// e.g.
// TfLiteDelegate* delegate = NnApiDelegate();
// interpreter->ModifyGraphWithDelegate(delegate);
// NnApiDelegate() returns a singleton, so you should not free this
// pointer or worry about its lifetime.
TfLiteDelegate* NnApiDelegate();
} // namespace tflite
#endif // TENSORFLOW_LITE_DELEGATES_NNAPI_NNAPI_DELEGATE_H_
tensorflow/lite/delegates/nnapi/nnapi_delegate_c_api.cc
+69
View File
@@ -0,0 +1,69 @@
/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/lite/delegates/nnapi/nnapi_delegate_c_api.h"
#include "tensorflow/lite/c/c_api_types.h"
#include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
#include "tensorflow/lite/nnapi/sl/public/NeuralNetworksSupportLibraryImpl.h"
TfLiteDelegate* TfLiteNnapiDelegateCreate(
const TfLiteNnapiDelegateOptions* options) {
tflite::StatefulNnApiDelegate::StatefulNnApiDelegate::Options
internal_options;
internal_options.execution_preference =
static_cast<tflite::StatefulNnApiDelegate::StatefulNnApiDelegate::
Options::ExecutionPreference>(
options->execution_preference);
internal_options.accelerator_name = options->accelerator_name;
internal_options.cache_dir = options->cache_dir;
internal_options.model_token = options->model_token;
internal_options.disallow_nnapi_cpu = options->disallow_nnapi_cpu;
internal_options.max_number_delegated_partitions =
options->max_number_delegated_partitions;
internal_options.allow_fp16 = options->allow_fp16;
tflite::StatefulNnApiDelegate* delegate = nullptr;
if (options->nnapi_support_library_handle) {
delegate = new tflite::StatefulNnApiDelegate(
static_cast<NnApiSLDriverImplFL5*>(
options->nnapi_support_library_handle),
internal_options);
} else {
delegate = new tflite::StatefulNnApiDelegate(internal_options);
}
return delegate;
}
TfLiteNnapiDelegateOptions TfLiteNnapiDelegateOptionsDefault() {
TfLiteNnapiDelegateOptions result = {};
tflite::StatefulNnApiDelegate::Options options;
result.execution_preference =
static_cast<TfLiteNnapiDelegateOptions::ExecutionPreference>(
options.execution_preference);
result.accelerator_name = options.accelerator_name;
result.cache_dir = options.cache_dir;
result.model_token = options.model_token;
result.disallow_nnapi_cpu = options.disallow_nnapi_cpu;
result.max_number_delegated_partitions =
options.max_number_delegated_partitions;
result.allow_fp16 = options.allow_fp16;
result.nnapi_support_library_handle = nullptr;
return result;
}
void TfLiteNnapiDelegateDelete(TfLiteDelegate* delegate) {
delete static_cast<tflite::StatefulNnApiDelegate*>(delegate);
}
tensorflow/lite/delegates/nnapi/nnapi_delegate_c_api.h
+105
View File
@@ -0,0 +1,105 @@
/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_DELEGATES_NNAPI_NNAPI_DELEGATE_C_API_H_
#define TENSORFLOW_LITE_DELEGATES_NNAPI_NNAPI_DELEGATE_C_API_H_
#include "tensorflow/lite/core/c/common.h"
// WARNING: this header file is DEPRECATED.
// See https://developer.android.com/ndk/guides/neuralnetworks/migration-guide.
#ifdef __cplusplus
extern "C" {
#endif // __cplusplus
// Use TfLiteNnapiDelegateOptionsDefault() for Default options.
// WARNING: This is an experimental API and subject to change.
typedef struct TfLiteNnapiDelegateOptions {
// Preferred Power/perf trade-off. For more details please see
// ANeuralNetworksCompilation_setPreference documentation in :
// https://developer.android.com/ndk/reference/group/neural-networks.html
enum ExecutionPreference {
kUndefined = -1,
kLowPower = 0,
kFastSingleAnswer = 1,
kSustainedSpeed = 2,
} execution_preference; // Preferred Power/perf trade-off.
// Default to kUndefined.
// Selected NNAPI accelerator with nul-terminated name.
// Default to nullptr, which implies the NNAPI default behavior: NNAPI
// runtime is allowed to use all available accelerators. If the selected
// accelerator cannot be found, NNAPI will not be used.
// It is the caller's responsibility to ensure the string is valid for the
// duration of the Options object lifetime.
const char* accelerator_name;
// The nul-terminated cache dir for NNAPI model.
// Default to nullptr, which implies the NNAPI will not try caching the
// compilation.
const char* cache_dir;
// The unique nul-terminated token string for NNAPI model.
// Default to nullptr, which implies the NNAPI will not try caching the
// compilation. It is the caller's responsibility to ensure there is no
// clash of the tokens.
// NOTE: when using compilation caching, it is not recommended to use the
// same delegate instance for multiple models.
const char* model_token;
// Whether to disallow NNAPI CPU usage. Default to 1 (true). Only effective on
// Android 10 and above. The NNAPI CPU typically performs less well than
// built-in TfLite kernels, but allowing CPU allows partial acceleration of
// models. If this is set to true, NNAPI is only used if the whole model is
// accelerated.
int disallow_nnapi_cpu;
// Whether to allow fp32 computation to be run in fp16. Default to 0 (false).
int allow_fp16;
// Specifies the max number of partitions to delegate. A value <= 0 means
// no limit. Default to 3.
// If the delegation of the full set of supported nodes would generate a
// number of partition greater than this parameter, only
// <max_number_delegated_partitions> of them will be actually accelerated.
// The selection is currently done sorting partitions in decreasing order
// of number of nodes and selecting them until the limit is reached.
int max_number_delegated_partitions;
// The pointer to NNAPI support lib implementation. Default to nullptr.
// If specified, NNAPI delegate will use the support lib instead of NNAPI in
// Android OS.
void* nnapi_support_library_handle;
} TfLiteNnapiDelegateOptions;
// Returns a delegate that uses NNAPI for ops execution.
// Must outlive the interpreter.
// WARNING: This is an experimental API and subject to change.
TFL_CAPI_EXPORT TfLiteDelegate* TfLiteNnapiDelegateCreate(
const TfLiteNnapiDelegateOptions* options);
// Returns TfLiteNnapiDelegateOptions populated with default values.
// WARNING: This is an experimental API and subject to change.
TFL_CAPI_EXPORT TfLiteNnapiDelegateOptions TfLiteNnapiDelegateOptionsDefault();
// Does any needed cleanup and deletes 'delegate'.
// WARNING: This is an experimental API and subject to change.
TFL_CAPI_EXPORT void TfLiteNnapiDelegateDelete(TfLiteDelegate* delegate);
#ifdef __cplusplus
}
#endif // __cplusplus
#endif // TENSORFLOW_LITE_DELEGATES_NNAPI_NNAPI_DELEGATE_C_API_H_
tensorflow/lite/delegates/nnapi/nnapi_delegate_c_api_test.cc
+163
View File
@@ -0,0 +1,163 @@
/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/lite/delegates/nnapi/nnapi_delegate_c_api.h"
#include <sys/mman.h>
#include <initializer_list>
#include <utility>
#include <vector>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "tensorflow/lite/core/c/common.h"
#include "tensorflow/lite/core/interpreter.h"
#include "tensorflow/lite/kernels/test_util.h"
#include "tensorflow/lite/schema/schema_generated.h"
namespace tflite {
namespace {
using ::testing::ElementsAreArray;
class SingleOpModelWithNnapiDelegateCApi : public SingleOpModel {
public:
SingleOpModelWithNnapiDelegateCApi() {
options_ = TfLiteNnapiDelegateOptionsDefault();
options_.disallow_nnapi_cpu = false;
}
explicit SingleOpModelWithNnapiDelegateCApi(
const TfLiteNnapiDelegateOptions& options) {
options_ = options;
options_.disallow_nnapi_cpu = false;
}
~SingleOpModelWithNnapiDelegateCApi() = default;
protected:
void BuildInterpreterWithNNAPI(std::vector<std::vector<int>> input_shapes) {
tflite::Interpreter::TfLiteDelegatePtr nnapi_delegate{
TfLiteNnapiDelegateCreate(&options_), TfLiteNnapiDelegateDelete};
SetDelegate(std::move(nnapi_delegate));
BuildInterpreter(input_shapes, /*num_threads=*/-1, options_.allow_fp16,
/*apply_delegate=*/true, /*allocate_and_delegate=*/true);
}
private:
TfLiteNnapiDelegateOptions options_;
};
class FloatAddOpModel : public SingleOpModelWithNnapiDelegateCApi {
public:
FloatAddOpModel(const TensorData& input1, const TensorData& input2,
const TensorData& output,
ActivationFunctionType activation_type) {
Init(input1, input2, output, activation_type);
}
FloatAddOpModel(const TfLiteNnapiDelegateOptions& options,
const TensorData& input1, const TensorData& input2,
const TensorData& output,
ActivationFunctionType activation_type)
: SingleOpModelWithNnapiDelegateCApi(options) {
Init(input1, input2, output, activation_type);
}
int input1() { return input1_; }
int input2() { return input2_; }
std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
protected:
int input1_;
int input2_;
int output_;
private:
// Performs initialization logic shared across all constructors.
void Init(const TensorData& input1, const TensorData& input2,
const TensorData& output, ActivationFunctionType activation_type) {
input1_ = AddInput(input1);
input2_ = AddInput(input2);
output_ = AddOutput(output);
SetBuiltinOp(BuiltinOperator_ADD, BuiltinOptions_AddOptions,
CreateAddOptions(builder_, activation_type).Union());
BuildInterpreterWithNNAPI({GetShape(input1_), GetShape(input2_)});
}
};
// Basic test for the NNAPI delegate C APIs.
TEST(NNAPIDelegate, C_API) {
TfLiteNnapiDelegateOptions options = TfLiteNnapiDelegateOptionsDefault();
options.execution_preference =
TfLiteNnapiDelegateOptions::ExecutionPreference::kLowPower;
FloatAddOpModel m(options, {TensorType_FLOAT32, {1, 2, 2, 1}},
{TensorType_FLOAT32, {1, 2, 2, 1}},
{TensorType_FLOAT32, {}}, ActivationFunctionType_NONE);
m.PopulateTensor<float>(m.input1(), {-2.0, 0.2, 0.7, 0.8});
m.PopulateTensor<float>(m.input2(), {0.1, 0.2, 0.3, 0.5});
ASSERT_EQ(m.Invoke(), kTfLiteOk);
EXPECT_THAT(m.GetOutput(), ElementsAreArray({-1.9, 0.4, 1.0, 1.3}));
}
// Basic test for the NNAPI delegate C API with accelerator_name specified.
TEST(NNAPIDelegate, C_API_WithAcceleratorName) {
TfLiteNnapiDelegateOptions options = TfLiteNnapiDelegateOptionsDefault();
options.execution_preference =
TfLiteNnapiDelegateOptions::ExecutionPreference::kLowPower;
options.accelerator_name = "nnapi-reference";
FloatAddOpModel m(options, {TensorType_FLOAT32, {1, 2, 2, 1}},
{TensorType_FLOAT32, {1, 2, 2, 1}},
{TensorType_FLOAT32, {}}, ActivationFunctionType_NONE);
m.PopulateTensor<float>(m.input1(), {-2.0, 0.2, 0.7, 0.8});
m.PopulateTensor<float>(m.input2(), {0.1, 0.2, 0.3, 0.5});
ASSERT_EQ(m.Invoke(), kTfLiteOk);
EXPECT_THAT(m.GetOutput(), ElementsAreArray({-1.9, 0.4, 1.0, 1.3}));
}
// Basic test for the NNAPI delegate C API with compilation caching enabled.
TEST(NNAPIDelegate, C_API_WithCompilationCaching) {
TfLiteNnapiDelegateOptions options = TfLiteNnapiDelegateOptionsDefault();
options.execution_preference =
TfLiteNnapiDelegateOptions::ExecutionPreference::kLowPower;
options.cache_dir = "/data/local/tmp";
options.model_token = "NNAPIDelegate.C_API_WithCompilationCaching";
// 1st run
{
FloatAddOpModel m(options, {TensorType_FLOAT32, {1, 2, 2, 1}},
{TensorType_FLOAT32, {1, 2, 2, 1}},
{TensorType_FLOAT32, {}}, ActivationFunctionType_NONE);
m.PopulateTensor<float>(m.input1(), {-2.0, 0.2, 0.7, 0.8});
m.PopulateTensor<float>(m.input2(), {0.1, 0.2, 0.3, 0.5});
ASSERT_EQ(m.Invoke(), kTfLiteOk);
EXPECT_THAT(m.GetOutput(), ElementsAreArray({-1.9, 0.4, 1.0, 1.3}));
}
// 2nd run
{
FloatAddOpModel m(options, {TensorType_FLOAT32, {1, 2, 2, 1}},
{TensorType_FLOAT32, {1, 2, 2, 1}},
{TensorType_FLOAT32, {}}, ActivationFunctionType_NONE);
m.PopulateTensor<float>(m.input1(), {-1.0, 0.1, 0.7, 0.8});
m.PopulateTensor<float>(m.input2(), {0.2, 0.2, 0.4, 0.2});
ASSERT_EQ(m.Invoke(), kTfLiteOk);
EXPECT_THAT(m.GetOutput(), ElementsAreArray({-0.8, 0.3, 1.1, 1.0}));
}
}
} // namespace
} // namespace tflite
tensorflow/lite/delegates/nnapi/nnapi_delegate_device_selection_test.cc
+1092
View File
File diff suppressed because it is too large Load Diff
tensorflow/lite/delegates/nnapi/nnapi_delegate_disabled.cc
+75
View File
@@ -0,0 +1,75 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
namespace tflite {
// Return a non-functional NN API Delegate struct.
TfLiteDelegate* NnApiDelegate() {
static StatefulNnApiDelegate* delegate = new StatefulNnApiDelegate();
return delegate;
}
StatefulNnApiDelegate::StatefulNnApiDelegate(const NnApi* /* nnapi */)
: StatefulNnApiDelegate() {}
StatefulNnApiDelegate::StatefulNnApiDelegate(Options /* options */)
: StatefulNnApiDelegate() {}
StatefulNnApiDelegate::StatefulNnApiDelegate(const NnApi* /* nnapi */,
Options /* options */)
: StatefulNnApiDelegate() {}
StatefulNnApiDelegate::StatefulNnApiDelegate(
const NnApiSLDriverImplFL5* /* nnapi_support_library_driver */,
Options /* options */)
: StatefulNnApiDelegate() {}
StatefulNnApiDelegate::StatefulNnApiDelegate()
: TfLiteDelegate(TfLiteDelegateCreate()),
delegate_data_(/*nnapi=*/nullptr) {
Prepare = DoPrepare;
}
TfLiteStatus StatefulNnApiDelegate::DoPrepare(TfLiteContext* /* context */,
TfLiteDelegate* /* delegate */) {
return kTfLiteOk;
}
TfLiteBufferHandle StatefulNnApiDelegate::RegisterNnapiMemory(
ANeuralNetworksMemory* memory, CopyToHostTensorFnPtr callback,
void* callback_context) {
return kTfLiteNullBufferHandle;
}
int StatefulNnApiDelegate::GetNnApiErrno() const { return 0; }
using ::tflite::delegate::nnapi::NNAPIDelegateKernel;
StatefulNnApiDelegate::Data::Data(const NnApi* nnapi) : nnapi(nnapi) {}
StatefulNnApiDelegate::Data::~Data() {}
void StatefulNnApiDelegate::Data::CacheDelegateKernel(
const TfLiteDelegateParams* delegate_params,
NNAPIDelegateKernel* delegate_state) {}
NNAPIDelegateKernel* StatefulNnApiDelegate::Data::MaybeGetCachedDelegateKernel(
const TfLiteDelegateParams* delegate_params) {
return nullptr;
}
} // namespace tflite
tensorflow/lite/delegates/nnapi/nnapi_delegate_errno_test.cc
+191
View File
@@ -0,0 +1,191 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <sys/mman.h>
#include <memory>
#include <vector>
#include <gtest/gtest.h>
#include "tensorflow/lite/core/c/common.h"
#include "tensorflow/lite/core/subgraph.h"
#include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
#include "tensorflow/lite/delegates/nnapi/nnapi_delegate_mock_test.h"
#include "tensorflow/lite/interpreter.h"
#include "tensorflow/lite/kernels/test_util.h"
#include "tensorflow/lite/nnapi/nnapi_implementation.h"
#include "tensorflow/lite/schema/schema_generated.h"
namespace tflite {
namespace {
void StatefulNnApiDelegateDelete(TfLiteDelegate* delegate) {
delete static_cast<StatefulNnApiDelegate*>(delegate);
}
class SingleOpModelWithNNAPI : public SingleOpModel {
public:
explicit SingleOpModelWithNNAPI(const NnApi* nnapi) {
options_.disallow_nnapi_cpu = false;
this->SetDelegate({new StatefulNnApiDelegate(nnapi, options_),
StatefulNnApiDelegateDelete});
}
StatefulNnApiDelegate* GetDelegate() {
return static_cast<StatefulNnApiDelegate*>(delegate_.get());
}
void SetBufferHandle(int index, TfLiteBufferHandle handle) {
interpreter_->SetBufferHandle(index, handle, delegate_.get());
}
private:
StatefulNnApiDelegate::Options options_;
};
class FloatAddOpModel : public SingleOpModelWithNNAPI {
public:
FloatAddOpModel(const NnApi* nnapi, const TensorData& input1,
const TensorData& input2, const TensorData& output,
ActivationFunctionType activation_type,
bool allow_fp32_relax_to_fp16 = false)
: SingleOpModelWithNNAPI(nnapi) {
Init(input1, input2, output, activation_type, allow_fp32_relax_to_fp16);
}
int input1() { return input1_; }
int input2() { return input2_; }
std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
protected:
int input1_;
int input2_;
int output_;
private:
// Performs initialization logic shared across all constructors.
void Init(const TensorData& input1, const TensorData& input2,
const TensorData& output, ActivationFunctionType activation_type,
bool allow_fp32_relax_to_fp16 = false) {
input1_ = AddInput(input1);
input2_ = AddInput(input2);
output_ = AddOutput(output);
SetBuiltinOp(BuiltinOperator_ADD, BuiltinOptions_AddOptions,
CreateAddOptions(builder_, activation_type).Union());
BuildInterpreter({GetShape(input1_), GetShape(input2_)}, /*num_threads=*/-1,
allow_fp32_relax_to_fp16, /*apply_delegate=*/true);
}
};
struct NnApiErrnoTest : ::tflite::delegate::nnapi::NnApiDelegateMockTest {};
TEST_F(NnApiErrnoTest, IsZeroWhenNoErrorOccurs) {
FloatAddOpModel m(nnapi_mock_->GetNnApi(), {TensorType_FLOAT32, {1, 2, 2, 1}},
{TensorType_FLOAT32, {1, 2, 2, 1}},
{TensorType_FLOAT32, {}}, ActivationFunctionType_NONE);
m.PopulateTensor<float>(m.input1(), {-2.0, 0.2, 0.7, 0.8});
m.PopulateTensor<float>(m.input2(), {0.1, 0.2, 0.3, 0.5});
ASSERT_EQ(m.Invoke(), kTfLiteOk);
EXPECT_EQ(m.GetDelegate()->GetNnApiErrno(), 0);
}
TEST_F(NnApiErrnoTest, HasTheStatusOfTheNnApiCallFailedCallingInit) {
nnapi_mock_->ExecutionCreateReturns<8>();
FloatAddOpModel m(nnapi_mock_->GetNnApi(), {TensorType_FLOAT32, {1, 2, 2, 1}},
{TensorType_FLOAT32, {1, 2, 2, 1}},
{TensorType_FLOAT32, {}}, ActivationFunctionType_NONE);
m.PopulateTensor<float>(m.input1(), {-2.0, 0.2, 0.7, 0.8});
m.PopulateTensor<float>(m.input2(), {0.1, 0.2, 0.3, 0.5});
EXPECT_EQ(m.Invoke(), kTfLiteError);
EXPECT_EQ(m.GetDelegate()->GetNnApiErrno(), 8);
}
TEST_F(NnApiErrnoTest, HasTheStatusOfTheNnApiCallFailedCallingInvoke) {
nnapi_mock_->ModelFinishReturns<-4>();
FloatAddOpModel m(nnapi_mock_->GetNnApi(), {TensorType_FLOAT32, {1, 2, 2, 1}},
{TensorType_FLOAT32, {1, 2, 2, 1}},
{TensorType_FLOAT32, {}}, ActivationFunctionType_NONE);
m.PopulateTensor<float>(m.input1(), {-2.0, 0.2, 0.7, 0.8});
m.PopulateTensor<float>(m.input2(), {0.1, 0.2, 0.3, 0.5});
// Failure is detected and the delegate is disabled.
// Execution runs without it and succeeds
EXPECT_EQ(m.Invoke(), kTfLiteOk);
// The delegate should store the value of the failure
EXPECT_EQ(m.GetDelegate()->GetNnApiErrno(), -4);
}
TEST_F(NnApiErrnoTest, ErrnoIsResetWhenRestoringDelegateForModel) {
nnapi_mock_->ModelFinishReturns<-4>();
FloatAddOpModel m(nnapi_mock_->GetNnApi(), {TensorType_FLOAT32, {1, 2, 2, 1}},
{TensorType_FLOAT32, {1, 2, 2, 1}},
{TensorType_FLOAT32, {}}, ActivationFunctionType_NONE);
m.PopulateTensor<float>(m.input1(), {-2.0, 0.2, 0.7, 0.8});
m.PopulateTensor<float>(m.input2(), {0.1, 0.2, 0.3, 0.5});
// Failure is detected and the delegate is disabled.
// Execution runs without it and succeeds
EXPECT_EQ(m.Invoke(), kTfLiteOk);
// The delegate should store the value of the failure
EXPECT_EQ(m.GetDelegate()->GetNnApiErrno(), -4);
nnapi_mock_->ModelFinishReturns<0>();
// Need to restore the delegate since it was disabled because of the
// previous failure.
m.ApplyDelegate();
EXPECT_EQ(m.Invoke(), kTfLiteOk);
// The error is still the last one recorded
EXPECT_EQ(m.GetDelegate()->GetNnApiErrno(), 0);
}
TEST_F(NnApiErrnoTest, ErrnoIsUpdatedInCaseOfAnotherFailure) {
nnapi_mock_->ModelFinishReturns<-4>();
FloatAddOpModel m(nnapi_mock_->GetNnApi(), {TensorType_FLOAT32, {1, 2, 2, 1}},
{TensorType_FLOAT32, {1, 2, 2, 1}},
{TensorType_FLOAT32, {}}, ActivationFunctionType_NONE);
m.PopulateTensor<float>(m.input1(), {-2.0, 0.2, 0.7, 0.8});
m.PopulateTensor<float>(m.input2(), {0.1, 0.2, 0.3, 0.5});
// Failure is detected and the delegate is disabled.
// Execution runs without it and succeeds
EXPECT_EQ(m.Invoke(), kTfLiteOk);
// The delegate should store the value of the failure
EXPECT_EQ(m.GetDelegate()->GetNnApiErrno(), -4);
nnapi_mock_->ModelFinishReturns<-5>();
// Need to restore the delegate since it was disabled because of the
// previous failure.
m.ApplyDelegate();
EXPECT_EQ(m.Invoke(), kTfLiteOk);
// The error is still the last one recorded
EXPECT_EQ(m.GetDelegate()->GetNnApiErrno(), -5);
}
} // namespace
} // namespace tflite
tensorflow/lite/delegates/nnapi/nnapi_delegate_kernel.h
+458
View File
@@ -0,0 +1,458 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_DELEGATES_NNAPI_NNAPI_DELEGATE_KERNEL_H_
#define TENSORFLOW_LITE_DELEGATES_NNAPI_NNAPI_DELEGATE_KERNEL_H_
// WARNING: this header file is DEPRECATED.
// See https://developer.android.com/ndk/guides/neuralnetworks/migration-guide.
#include <cstddef>
#include <cstdint>
#include <list>
#include <map>
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
#include "tensorflow/lite/allocation.h"
#include "tensorflow/lite/core/c/common.h"
#include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
#include "tensorflow/lite/delegates/nnapi/nnapi_delegate_plugin.h"
#include "tensorflow/lite/nnapi/nnapi_implementation.h"
namespace tflite {
namespace delegate {
namespace nnapi {
constexpr int32_t kMinSdkVersionForNNAPI = 27;
constexpr int32_t kMinSdkVersionForNNAPI11 = 28;
constexpr int32_t kMinSdkVersionForNNAPI12 = 29;
constexpr int32_t kMinSdkVersionForNNAPI13 = 30;
constexpr int32_t kNNAPIRuntimeFeatureLevel5 = 31;
constexpr int32_t kNNAPIRuntimeFeatureLevel6 = 1000006;
constexpr int32_t kNNAPIRuntimeFeatureLevel7 = 1000007;
constexpr int32_t kNNAPIRuntimeFeatureLevel8 = 1000008;
class NNAPIOpBuilder;
// The kernel that represents the node sub set of TF Lite being run on NN API.
struct NNAPIOpMappingArgs {
TfLiteContext* context;
NNAPIOpBuilder* builder;
TfLiteNode* node;
int node_index;
std::vector<int>* model_state_outputs;
std::vector<int>* model_state_tfl_inputs;
std::vector<std::tuple<int, int>>* feedback_loops;
int* nnapi_errno;
};
// RAII NN API Model Destructor for use with std::unique_ptr
class NNFreeModel {
public:
explicit NNFreeModel(const NnApi* nnapi) : nnapi_(nnapi) {}
void operator()(ANeuralNetworksModel* model) {
nnapi_->ANeuralNetworksModel_free(model);
}
private:
// NnApi instance to use. Not owned by this object.
const NnApi* nnapi_;
};
// RAII NN API Compilation Destructor for use with std::unique_ptr
class NNFreeCompilation {
public:
explicit NNFreeCompilation(const NnApi* nnapi) : nnapi_(nnapi) {}
void operator()(ANeuralNetworksCompilation* model) {
nnapi_->ANeuralNetworksCompilation_free(model);
}
private:
// NnApi instance to use. Not owned by this object.
const NnApi* nnapi_;
};
// RAII NN API Execution Destructor for use with std::unique_ptr
class NNFreeExecution {
public:
explicit NNFreeExecution(const NnApi* nnapi) : nnapi_(nnapi) {}
void operator()(ANeuralNetworksExecution* execution) {
nnapi_->ANeuralNetworksExecution_free(execution);
}
private:
// NnApi instance to use. Not owned by this object.
const NnApi* nnapi_;
};
// RAII NN API Burst Destructor for use with std::unique_ptr
class NNFreeBurst {
public:
explicit NNFreeBurst(const NnApi* nnapi) : nnapi_(nnapi) {}
void operator()(ANeuralNetworksBurst* model) {
nnapi_->ANeuralNetworksBurst_free(model);
}
private:
// NnApi instance to use. Not owned by this object.
const NnApi* nnapi_;
};
using UniqueExecution =
std::unique_ptr<ANeuralNetworksExecution, NNFreeExecution>;
// RAII NN API MappingUtil Destructor for use with std::unique_ptr
class NNFreeMappingUtil {
public:
void operator()(NnapiMappingUtilCInterface* mapping_util);
};
// Manage NNAPI shared memory handle
class NNMemory {
public:
static std::unique_ptr<NNMemory> Create(const NnApi* nnapi, const char* name,
size_t size);
~NNMemory();
NNMemory(const NNMemory&) = delete;
void operator=(const NNMemory&) = delete;
ANeuralNetworksMemory* get_handle() { return nn_memory_handle_; }
uint8_t* get_data_ptr() { return data_ptr_; }
size_t get_byte_size() { return byte_size_; }
private:
// Private constructor. Use Create() to create an instance.
NNMemory(const NnApi* nnapi, int fd, size_t byte_size, uint8_t* data_ptr,
ANeuralNetworksMemory* nn_memory_handle
#ifndef __ANDROID__
,
const std::string& shm_region_name
#endif
)
: nnapi_(nnapi),
fd_(fd),
byte_size_(byte_size),
data_ptr_(data_ptr),
nn_memory_handle_(nn_memory_handle)
#ifndef __ANDROID__
,
shm_region_name_(shm_region_name)
#endif
{};
// NnApi instance to use. Not owned by this object.
const NnApi* nnapi_;
int fd_ = 0;
size_t byte_size_ = 0;
uint8_t* data_ptr_ = nullptr;
ANeuralNetworksMemory* nn_memory_handle_ = nullptr;
#ifndef __ANDROID__
std::string shm_region_name_;
#endif
};
// LINT.IfChange
enum class NNAPIValidationFailureType : int {
// The operator is not supported by either NNAPI or the NNAPI Delegate.
kUnsupportedOperator = 0,
// The given operation or operands are not supported on the specified
// Android SDK version. The min supported version is specified in the
// validation failure message.
kUnsupportedAndroidVersion = 1,
// The version of the operator (value of TfLiteRegistration::version)
// for the given op is not supported. The max supported version
// is specified in the validation failure message.
// For more details on each operator version see
// the GetBuiltinOperatorVersion function in
// tensorflow/compiler/mlir/lite/tools/versioning/op_version.cc.
kUnsupportedOperatorVersion = 2,
// The given input operand type is not supported for the current combination
// of operator type and sdk version.
kUnsupportedInputType = 3,
// When using NN API version 1.0 or 1.1, the condition
// input_scale * filter_scale < output_scale
// must be true for quantized versions of the following ops:
// * CONV_2D
// * DEPTHWISE_CONV_2D
// * FULLY_CONNECTED (where filter actually stands for weights)
// The condition is relaxed and no longer required since version 1.2.
kNotRestrictedScaleCompliant = 4,
// The given output operand type is not supported for the current combination
// of operator type and sdk version.
kUnsupportedOutputType = 5,
// The size of the operand tensor is too large.
kUnsupportedOperandSize = 6,
// The value of one of the operands or of a combination of operands is
// not supported. Details are provided in the failure message.
kUnsupportedOperandValue = 7,
// The combination of float inputs and quantized weights or filters
// is not supported
kUnsupportedHybridOperator = 8,
// The quantization type (for example per-channel quantization) is not
// supported.
kUnsupportedQuantizationType = 9,
// The accelerated version of operation requires a specific operand to be
// specified.
kMissingRequiredOperand = 10,
// The rank of the operand is not supported. Details in the failure message.
kUnsupportedOperandRank = 11,
// The input tensor cannot be dynamically-sized.
kInputTensorShouldHaveConstantShape = 12,
// The operator has a different number of inputs of the one or ones that
// are supported by NNAPI.
kUnsupportedOperatorVariant = 13,
// The accelerated version of the operator cannot specify an activation
// function.
kNoActivationExpected = 14,
// Quantization scale and/or zero point are not in the supported value(s)
// for the accelerated operation.
kUnsupportedQuantizationParameters = 15,
};
// LINT.ThenChange(nnapi_linter/linter.proto)
struct NNAPIValidationFailure {
NNAPIValidationFailureType type;
std::string message;
NNAPIValidationFailure(NNAPIValidationFailureType type, const char* message)
: type(type), message(message) {}
};
// LRU cache of reusable NNAPI executions.
class NNAPIExecutionCache {
public:
// The cache signature. Uniquely identifies an execution request.
struct Signature {
std::vector<uint64_t> tensor_handle_timestamps;
std::vector<int> dynamic_dimensions;
bool operator==(const Signature& other) const;
struct Hasher {
std::size_t operator()(const Signature& signature) const;
};
};
explicit NNAPIExecutionCache(uint32_t max_cache_size)
: max_cache_size_(max_cache_size) {}
// Gets the cached execution by signature.
// On cache hit, the target execution is set to be the most recently used one.
// On cache miss, nullptr is returned.
ANeuralNetworksExecution* Get(const Signature& signature);
// Puts the execution in cache and set it to be the most recently used one.
// If the cache is full, the least recently used entry will be released.
void Put(const Signature& signature, UniqueExecution execution);
// Clears all cache entries.
void Clear();
// Resets the max cache size.
void SetMaxCacheSize(uint32_t max_cache_size);
private:
// Releases the least recently used cache.
void ReleaseLRU();
// The maximum number of reusable executions to cache.
uint32_t max_cache_size_;
// Cache signatures in the order of most recent use. The most recently used
// signature is at the front of the list.
std::list<Signature> order_;
// A hash map to lookup a managed execution by its signature.
std::unordered_map<Signature,
std::pair<std::list<Signature>::iterator, UniqueExecution>,
Signature::Hasher>
lookup_;
};
// The kernel that represents the node sub set of TF Lite being run on NN API.
class NNAPIDelegateKernel {
public:
explicit NNAPIDelegateKernel(
const NnApi* nnapi, NnapiDelegateVendorPlugin* vendor_plugin = nullptr)
: initialised_(false),
nnapi_(nnapi),
nn_model_(nullptr, NNFreeModel(nnapi_)),
nn_compilation_(nullptr, NNFreeCompilation(nnapi_)),
nn_burst_(nullptr, NNFreeBurst(nnapi_)),
nn_execution_cache_(/*max_cache_size=*/4),
mapping_util_(NnapiMappingUtilCInterfaceCreate(), NNFreeMappingUtil()),
vendor_plugin_(vendor_plugin) {}
NNAPIDelegateKernel() : NNAPIDelegateKernel(NnApiImplementation()) {}
~NNAPIDelegateKernel() {
for (auto content : allocation_memory_mapping_) {
nnapi_->ANeuralNetworksMemory_free(content.second);
}
}
static NnapiMappingUtilCInterface* NnapiMappingUtilCInterfaceCreate();
// Translate a node into its operands
// It assumes that the call to Validate for has been successful for
// the operation.
// In case of success it returns kTfLiteOk and stores in n_op_type the
// NNAPI Operation code.
// Returns kTfLiteError in case of failures during mapping.
static TfLiteStatus Map(TfLiteContext* context, int builtin_code, int version,
int android_sdk_version,
const NNAPIOpMappingArgs& mapping_args,
ANeuralNetworksOperationType* nn_op_type,
NnapiDelegateVendorPlugin* vendor_plugin = nullptr);
// Returns true if the node can be accelerated with NNAPI.
static bool Validate(
const TfLiteContext* context, const TfLiteRegistration* registration,
int android_sdk_version, const TfLiteNode* node,
bool is_accelerator_specified,
NnapiDelegateVendorPlugin* vendor_plugin = nullptr,
// Collects lists of failures collected during
// the validation of the possibility of accelerating
// the given node
std::vector<NNAPIValidationFailure>* map_failures = nullptr);
// Initialize the kernel (a NN model) and builds the NN Model.
// Any NNAPI Related error causing this method to fail will have the
// associated error number stored in nnapi_errno
TfLiteStatus Init(TfLiteContext* context, const TfLiteDelegateParams* params,
int* nnapi_errno);
// Creates the NNAPI Compilation for the NN model. It assumes that Init has
// been called and completed successfully.
// Any NNAPI Related error causing this method to fail will have the
// associated error number stored in nnapi_errno
TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node,
int* nnapi_errno);
// Invoke the NN Model. Expects Init and Prepare to have been completed
// successfully.
// Any NNAPI Related error causing this method to fail will have the
// associated error number stored in nnapi_errno
TfLiteStatus Invoke(TfLiteContext* context, TfLiteNode* node,
int* nnapi_errno);
// Returns the list of operations supported by the current NNAPI model as
// built in Prepare. Every operation is identified by the index as provided
// in the delegate parameters given to the delegate during the Init call.
// It expects the Init method has been called and completed successfully and
// returns kTfLiteError if not. Returns an error if any of the NNAPI
// operations fails or if the
// ANeuralNetworksModel_getSupportedOperationsForDevices function is not
// available in the NnApi object.
TfLiteStatus GetOperationsSupportedByTargetNnApiDevices(
TfLiteContext* context, std::vector<int>* supported_nodes,
int* nnapi_errno);
private:
// True if initialization has been completed successfully
bool initialised_;
// Access to NNApi.
const NnApi* nnapi_;
// ANN device handle.
std::vector<ANeuralNetworksDevice*> nnapi_devices_;
// Name of the nnapi device, empty if nnapi_devices_ is empty;
std::string device_name_;
// ANN API state.
std::unique_ptr<ANeuralNetworksModel, NNFreeModel> nn_model_;
std::unique_ptr<ANeuralNetworksCompilation, NNFreeCompilation>
nn_compilation_;
std::unique_ptr<ANeuralNetworksBurst, NNFreeBurst> nn_burst_;
NNAPIExecutionCache nn_execution_cache_;
// The mappings of tenor id to BufferHandle. Needed to track BufferHandle
// change and alter nn_reusable_execution_ if necessary.
std::vector<int> tensor_handle_map_;
// Node indices that this delegate is responsible for. Indices here
// indexes into the nodes array in the TfLiteContext.
std::vector<int> nodes_;
// Track indices we use
std::unique_ptr<NnapiMappingUtilCInterface, NNFreeMappingUtil> mapping_util_;
std::map<const MMAPAllocation*, ANeuralNetworksMemory*>
allocation_memory_mapping_;
// Track memory map
const std::vector<StatefulNnApiDelegate::MemoryRegistration>*
tensor_memory_map_;
std::vector<int> model_state_outputs_;
std::vector<int> model_state_tfl_inputs_;
// This is the equivalent of the pair model_state_outputs_,
// model_state_tfl_inputs_ for all tensors where we have to keep the output
// data available for TFLite model users
std::vector<std::tuple<int, int>> feedback_loops_;
// The mappings of tenor id to max size in bytes. If the hint is not provided
// for a tensor, it is set to 0.
std::vector<size_t> tensor_max_size_hints_;
std::unique_ptr<NNMemory> nn_input_memory_;
std::unique_ptr<NNMemory> nn_output_memory_;
std::vector<uint8_t> nn_compilation_cache_token_;
// Map of DENSIFY output tensor id to node id.
std::vector<int> densify_output_to_node_mapping_;
// Map of DEQUANTIZE output tensor id to node id.
// Only contains DEQUANTIZE nodes with non-const input.
std::vector<int> non_const_dequantize_output_to_node_mapping_;
NnapiDelegateVendorPlugin* vendor_plugin_ = nullptr;
// Fully initialized in NNAPIDelegateKernel::AddOpsAndTensors
int target_feature_level_ = 27; // kMinSdkVersionForNNAPI10
void AddDequantizeOperatorsWhereNeeded(
const TfLiteContext* context, int builtin_code, const TfLiteNode* node,
int tflite_node_index, NNAPIOpBuilder* builder, int* nnapi_errno);
TfLiteStatus DensifyAndDequantizeConstTensor(TfLiteContext* context,
int densify_node_id,
bool should_dequantize,
NNAPIOpBuilder& builder);
TfLiteStatus AddOpsAndTensors(TfLiteContext* context, int* nnapi_errno,
bool allow_dynamic_dimensions);
TfLiteStatus BuildGraph(TfLiteContext* context,
const StatefulNnApiDelegate::Options& options,
const TfLiteIntArray* input_tensors,
const TfLiteIntArray* output_tensors,
int* nnapi_errno);
// Log the compilation info provided by the support library at the end of
// a compilation (failed or successful).
// To avoid output spamming, logging is done only once, on the first call to
// this method, subsequent runs will only retrieve the information but not
// log it.
//
// This method is registered as a callback with the SL which calls it.
static void LogCompilationInfoOnce(
const NnApi* nnapi, const ANeuralNetworksDiagnosticCompilationInfo* info);
// Log the execution info provided by the support library at the end of
// an execution (failed or successful).
// To avoid output spamming, logging is done only once, on the first call to
// this method, subsequent runs will only retrieve the information but not
// log it.
//
// This method is registered as a callback with the SL which calls it.
static void LogExecutionInfoOnce(
const NnApi* nnapi, const ANeuralNetworksDiagnosticExecutionInfo* info);
};
} // namespace nnapi
} // namespace delegate
} // namespace tflite
#endif // TENSORFLOW_LITE_DELEGATES_NNAPI_NNAPI_DELEGATE_KERNEL_H_
tensorflow/lite/delegates/nnapi/nnapi_delegate_mock_test.h
+105
View File
@@ -0,0 +1,105 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_DELEGATES_NNAPI_NNAPI_DELEGATE_MOCK_TEST_H_
#define TENSORFLOW_LITE_DELEGATES_NNAPI_NNAPI_DELEGATE_MOCK_TEST_H_
// WARNING: this header file is DEPRECATED.
// See https://developer.android.com/ndk/guides/neuralnetworks/migration-guide.
// Cannot mock the delegate when using the disabled version
// (see the condition in the BUILD file).
#ifndef NNAPI_DELEGATE_DISABLED
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <memory>
#include <gtest/gtest.h>
#include "absl/memory/memory.h"
#include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
#include "tensorflow/lite/nnapi/NeuralNetworksTypes.h"
#include "tensorflow/lite/nnapi/nnapi_handler.h"
#include "tensorflow/lite/nnapi/nnapi_implementation.h"
namespace tflite {
namespace delegate {
namespace nnapi {
class NnApiMock : public ::tflite::nnapi::NnApiHandler {
public:
explicit NnApiMock(NnApi* nnapi, int android_sdk_version = 29)
: ::tflite::nnapi::NnApiHandler(nnapi) {
nnapi_->nnapi_exists = true;
nnapi_->android_sdk_version = android_sdk_version;
nnapi_->nnapi_runtime_feature_level = android_sdk_version;
nnapi_->ANeuralNetworksCompilation_free =
[](ANeuralNetworksCompilation* compilation) {};
nnapi_->ANeuralNetworksMemory_free = [](ANeuralNetworksMemory* memory) {};
nnapi_->ANeuralNetworksModel_free = [](ANeuralNetworksModel* model) {};
nnapi_->ANeuralNetworksExecution_free =
[](ANeuralNetworksExecution* execution) {};
nnapi_->ASharedMemory_create = [](const char* name, size_t size) -> int {
return open("/dev/zero", O_RDWR);
};
nnapi_->ANeuralNetworksEvent_free = [](ANeuralNetworksEvent* event) {};
ModelCreateReturns<ANEURALNETWORKS_NO_ERROR>();
AddOperandReturns<ANEURALNETWORKS_NO_ERROR>();
SetOperandValueReturns<ANEURALNETWORKS_NO_ERROR>();
AddOperationReturns<ANEURALNETWORKS_NO_ERROR>();
IdentifyInputAndOutputsReturns<ANEURALNETWORKS_NO_ERROR>();
RelaxComputationFloatReturns<ANEURALNETWORKS_NO_ERROR>();
ModelFinishReturns<ANEURALNETWORKS_NO_ERROR>();
MemoryCreateFromFdReturns<ANEURALNETWORKS_NO_ERROR>();
CompilationCreateReturns<ANEURALNETWORKS_NO_ERROR>();
CompilationCreateForDevicesReturns<ANEURALNETWORKS_NO_ERROR>();
CompilationFinishReturns<ANEURALNETWORKS_NO_ERROR>();
ExecutionCreateReturns<ANEURALNETWORKS_NO_ERROR>();
ExecutionSetInputFromMemoryReturns<ANEURALNETWORKS_NO_ERROR>();
ExecutionSetOutputFromMemoryReturns<ANEURALNETWORKS_NO_ERROR>();
ExecutionComputeReturns<ANEURALNETWORKS_NO_ERROR>();
ExecutionStartComputeReturns<ANEURALNETWORKS_NO_ERROR>();
EventWaitReturns<ANEURALNETWORKS_NO_ERROR>();
SetPriorityReturns<ANEURALNETWORKS_NO_ERROR>();
SetOperandSymmPerChannelQuantParamsReturns<ANEURALNETWORKS_NO_ERROR>();
SetNnapiSupportedDevice("test-device", android_sdk_version);
}
~NnApiMock() { Reset(); }
};
class NnApiDelegateMockTest : public ::testing::Test {
protected:
void SetUp() override {
nnapi_ = *NnApiImplementation();
nnapi_mock_ = std::make_unique<NnApiMock>(&nnapi_);
}
std::unique_ptr<NnApiMock> nnapi_mock_;
private:
NnApi nnapi_;
};
} // namespace nnapi
} // namespace delegate
} // namespace tflite
#endif // #ifndef NNAPI_DELEGATE_DISABLED
#endif // TENSORFLOW_LITE_DELEGATES_NNAPI_NNAPI_DELEGATE_MOCK_TEST_H_
tensorflow/lite/delegates/nnapi/nnapi_delegate_nnapi_failure_handling_test.cc
+132
View File
@@ -0,0 +1,132 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <sys/mman.h>
#include <cstdint>
#include <memory>
#include <string>
#include <vector>
#include <gtest/gtest.h>
#include "tensorflow/lite/core/c/common.h"
#include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
#include "tensorflow/lite/delegates/nnapi/nnapi_delegate_mock_test.h"
#include "tensorflow/lite/interpreter.h"
#include "tensorflow/lite/kernels/test_util.h"
#include "tensorflow/lite/nnapi/NeuralNetworksTypes.h"
#include "tensorflow/lite/nnapi/nnapi_implementation.h"
#include "tensorflow/lite/schema/schema_generated.h"
namespace tflite {
namespace {
void StatefulNnApiDelegateDelete(TfLiteDelegate* delegate) {
delete static_cast<StatefulNnApiDelegate*>(delegate);
}
struct NnApiFailureHandlingTest
: ::tflite::delegate::nnapi::NnApiDelegateMockTest {};
// This is a model with two ops:
//
// input1 ---->
// ADD --
// input2 --> |
// -->
// SUB --> output
// input3 ---------------->
//
class AddSubOpsAcceleratedModel : public MultiOpModel {
public:
AddSubOpsAcceleratedModel(const TensorData& input1, const TensorData& input2,
const TensorData& input3, const TensorData& output,
ActivationFunctionType activation_type,
const NnApi* nnapi,
const std::string& accelerator_name,
bool allow_fp32_relax_to_fp16 = false)
: MultiOpModel() {
StatefulNnApiDelegate::Options options;
options.accelerator_name = accelerator_name.c_str();
SetDelegate({new StatefulNnApiDelegate(nnapi, options),
StatefulNnApiDelegateDelete});
Init(input1, input2, input3, output, activation_type,
allow_fp32_relax_to_fp16);
}
int input1() { return input1_; }
int input2() { return input2_; }
int input3() { return input3_; }
std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
protected:
int input1_;
int input2_;
int input3_;
int output_;
private:
// Performs initialization logic shared across all constructors.
void Init(const TensorData& input1, const TensorData& input2,
const TensorData& input3, const TensorData& output,
ActivationFunctionType activation_type,
bool allow_fp32_relax_to_fp16 = false) {
input1_ = AddInput(input1);
input2_ = AddInput(input2);
input3_ = AddInput(input3);
const int add_output = AddInnerTensor<float>(output);
output_ = AddOutput(output);
AddBuiltinOp(BuiltinOperator_ADD, BuiltinOptions_AddOptions,
CreateAddOptions(builder_, activation_type).Union(),
{input1_, input2_}, {add_output});
AddBuiltinOp(BuiltinOperator_SUB, BuiltinOptions_SubOptions,
CreateSubOptions(builder_, activation_type).Union(),
{add_output, input3_}, {output_});
BuildInterpreter({GetShape(input1_), GetShape(input2_), GetShape(input3_)},
/*num_threads=*/-1, allow_fp32_relax_to_fp16,
/*apply_delegate=*/false);
ApplyDelegate();
}
};
TEST_F(NnApiFailureHandlingTest, DelegateShouldFailImmediatelyIfUnableToAddOp) {
static int add_op_invocation_count = 0;
nnapi_mock_->SetNnapiSupportedDevice("test-device");
nnapi_mock_->StubAddOperationWith(
[](ANeuralNetworksModel* model, ANeuralNetworksOperationType type,
uint32_t inputCount, const uint32_t* inputs, uint32_t outputCount,
const uint32_t* outputs) -> int {
++add_op_invocation_count;
return ANEURALNETWORKS_BAD_DATA;
});
AddSubOpsAcceleratedModel m(
{TensorType_FLOAT32, {1, 2, 2, 1}}, {TensorType_FLOAT32, {1, 2, 2, 1}},
{TensorType_FLOAT32, {1, 2, 2, 1}}, {TensorType_FLOAT32, {}},
ActivationFunctionType_NONE, nnapi_mock_->GetNnApi(),
/*accelerator_name=*/"test-device");
std::vector<float> input1{-2.0, 0.2, 0.7, 0.9};
std::vector<float> input2{0.1, 0.2, 0.3, 0.5};
m.PopulateTensor<float>(m.input1(), input1);
m.PopulateTensor<float>(m.input2(), input2);
m.PopulateTensor<float>(m.input3(), input2);
ASSERT_EQ(m.Invoke(), kTfLiteOk);
EXPECT_EQ(add_op_invocation_count, 1);
}
} // namespace
} // namespace tflite
tensorflow/lite/delegates/nnapi/nnapi_delegate_plugin.h
+107
View File
@@ -0,0 +1,107 @@
/* Copyright 2021 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_DELEGATES_NNAPI_NNAPI_DELEGATE_PLUGIN_H_
#define TENSORFLOW_LITE_DELEGATES_NNAPI_NNAPI_DELEGATE_PLUGIN_H_
#include "tensorflow/lite/core/c/common.h"
#include "tensorflow/lite/nnapi/NeuralNetworksTypes.h"
// WARNING: this header file is DEPRECATED.
// See https://developer.android.com/ndk/guides/neuralnetworks/migration-guide.
#ifdef __cplusplus
extern "C" {
#endif // __cplusplus
// The mapping utils intended for vendor plugin to track:
// - TFLite tensor indices to NN API tensor indices mapping.
// - TFLite node indices to NN API operation indices mapping.
// WARNING: This is an experimental interface that is subject to change.
typedef struct NnapiMappingUtilCInterface {
// Given a TFLite index, return the ANN index. If it doesn't exist
// return -1.
int (*TfLiteIndexToNnIndex)(NnapiMappingUtilCInterface* mapping, int index);
// When adding a non-tensor TFLite node parameter to NNAPI as an
// ANeuralNetworksOperand, notify NNAPI delegate to increment the operand
// count.
int (*AddNewNonTensorOperand)(NnapiMappingUtilCInterface* mapping);
// When adding a TFLite tensor to NNAPI as an ANeuralNetworksOperand, notify
// NNAPI delegate to add a new mapping from `tflite_index` and return the NN
// API tensor index.
int (*AddNewNnTensorIndex)(NnapiMappingUtilCInterface* mapping,
int tflite_index);
// When adding a TFLite tensor to NNAPI as multiple ANeuralNetworksOperand
// objects, for example when splitting one input into several ones, notify
// NNAPI delegate to increment the operand count.
int (*AddDelegateGeneratedInputAnnTensorOperand)(
NnapiMappingUtilCInterface* mapping);
// Given a TFLite index returns a TFLite type to which a tensor must be
// converted during copying the data to the memory allocated for NN API.
// kTfLiteNoType means no conversion is needed.
TfLiteType (*TfLiteIndexToNnTypeConversion)(
NnapiMappingUtilCInterface* mapping, int index);
// Add a new mapping from TFLite tensor index to a type conversion.
void (*AddTypeConversion)(NnapiMappingUtilCInterface* mapping,
int tflite_index, TfLiteType tflite_type);
// Add a new mapping from TFLite node index to NNAPI op index.
void (*AddNnapiToTfliteOpMapping)(NnapiMappingUtilCInterface* mapping,
int tflite_node_index);
// opaque handle for the mapping context. Only intended for the NNAPI Delegate
// to use.
void* context;
} NnapiMappingUtilCInterface;
// The interface for NNAPI Vendor Plugin.
// The interface exposes necessary functionalities for NNAPI delegate to
// interact with the vendor plugin.
// WARNING: This is an experimental interface that is subject to change.
typedef struct NnapiDelegateVendorPlugin {
// Validate whether the given TFLite node is supported by the plugin.
bool (*ValidateNode)(const TfLiteContext* context,
const TfLiteRegistration* registration,
const TfLiteNode* node);
// Translate a TFLite node into corresponding NNAPI operands and operation.
// It assumes that the call to Validate for has been successful for
// the operation. In case of success it returns kTfLiteOk and stores the
// corresponding NNAPI operand indices and operation code through the mapping
// utility interface. Returns kTfLiteError in case of failures during mapping.
TfLiteStatus (*MapNode)(TfLiteContext* context, const TfLiteNode* node,
int node_index, NnapiMappingUtilCInterface* mapping,
ANeuralNetworksModel* model);
// Parse the provided compilation_hints string and configure it for the given
// ANeuralNetworksCompilation handle.
TfLiteStatus (*ConfigureCompilationHints)(
const char* compilation_hints, ANeuralNetworksCompilation* compilation);
// Parse the provided execution_hints string and configure it for the given
// ANeuralNetworksExecution handle.
TfLiteStatus (*ConfigureExecutionHints)(const char* execution_hints,
ANeuralNetworksExecution* execution);
} NnapiDelegateVendorPlugin;
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
#endif // TENSORFLOW_LITE_DELEGATES_NNAPI_NNAPI_DELEGATE_PLUGIN_H_
tensorflow/lite/delegates/nnapi/nnapi_delegate_signed_quantization_test.cc
+922
View File
@@ -0,0 +1,922 @@
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <cstddef>
#include <cstdint>
#include <initializer_list>
#include <map>
#include <memory>
#include <vector>
#include <gtest/gtest.h>
#include "tensorflow/lite/core/c/common.h"
#include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
#include "tensorflow/lite/delegates/nnapi/nnapi_delegate_mock_test.h"
#include "tensorflow/lite/interpreter.h"
#include "tensorflow/lite/kernels/fully_connected.h"
#include "tensorflow/lite/kernels/test_util.h"
#include "tensorflow/lite/nnapi/NeuralNetworksTypes.h"
#include "tensorflow/lite/nnapi/nnapi_implementation.h"
#include "tensorflow/lite/schema/schema_generated.h"
namespace tflite {
namespace ops {
namespace builtin {
TfLiteRegistration* Register_CONVOLUTION_REF();
TfLiteRegistration* Register_DEQUANTIZE();
} // namespace builtin
} // namespace ops
namespace {
void StatefulNnApiDelegateDelete(TfLiteDelegate* delegate) {
delete static_cast<StatefulNnApiDelegate*>(delegate);
}
class SingleOpModelWithNNAPI : public SingleOpModel {
public:
SingleOpModelWithNNAPI() = default;
void Init(const NnApi* nnapi) {
options_.disallow_nnapi_cpu = false;
SetDelegate({new StatefulNnApiDelegate(nnapi, options_),
StatefulNnApiDelegateDelete});
}
void SetBufferHandle(int index, TfLiteBufferHandle handle) {
interpreter_->SetBufferHandle(index, handle, delegate_.get());
}
TfLiteStatus GetCompilationStatus() { return compilation_status_; }
protected:
StatefulNnApiDelegate::Options options_;
TfLiteStatus compilation_status_;
};
class HybridFullyConnectedOpModel : public SingleOpModelWithNNAPI {
public:
HybridFullyConnectedOpModel(const NnApi* nnapi, int units, int batches,
const TensorData& input,
const TensorData& weights,
const TensorData& output = {TensorType_FLOAT32},
bool asymmetric_inputs = false)
: batches_(batches), units_(units) {
SingleOpModelWithNNAPI::Init(nnapi);
int total_input_size = 1;
for (size_t i = 0; i < input.shape.size(); ++i) {
total_input_size *= input.shape[i];
}
input_size_ = total_input_size / batches_;
input_ = AddInput(input);
weights_ = AddInput(weights);
TensorData bias{TensorType_FLOAT32, {units_}};
bias_ = AddInput(bias);
output_ = AddOutput(output);
auto options = CreateFullyConnectedOptions(
builder_, ActivationFunctionType_RELU,
tflite::FullyConnectedOptionsWeightsFormat_DEFAULT,
false, asymmetric_inputs)
.Union();
SetBuiltinOp(BuiltinOperator_FULLY_CONNECTED,
BuiltinOptions_FullyConnectedOptions, options);
resolver_ = std::make_unique<SingleOpResolver>(
BuiltinOperator_FULLY_CONNECTED,
ops::builtin::Register_FULLY_CONNECTED_GENERIC_OPT());
BuildInterpreter({GetShape(input_), GetShape(weights_), GetShape(bias_)},
/*num_threads=*/-1,
/*allow_fp32_relax_to_fp16=*/false,
/*apply_delegate=*/false);
compilation_status_ = ApplyDelegate();
}
void SetBias(const std::vector<float>& f) { PopulateTensor(bias_, f); }
void SetWeights(const std::vector<float>& data) {
SymmetricQuantizeAndPopulate(weights_, data);
}
void SetSignedWeights(std::initializer_list<float> f) {
SignedSymmetricQuantizeAndPopulate(weights_, f);
}
void SetInput(const std::vector<float>& f) { PopulateTensor(input_, f); }
std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
int input_size() { return input_size_; }
int num_units() { return units_; }
int num_batches() { return batches_; }
protected:
int input_;
int weights_;
int bias_;
int output_;
int batches_;
int units_;
int input_size_;
};
struct NnApiSignedQuantizationTest
: ::tflite::delegate::nnapi::NnApiDelegateMockTest {
static void SetUpTestSuite() { tensors_count = new std::map<int, int>(); }
void SetUp() override {
::tflite::delegate::nnapi::NnApiDelegateMockTest::SetUp();
nnapi_mock_->StubAddOperandWith(
[](ANeuralNetworksModel* model,
const ANeuralNetworksOperandType* type) -> int {
const auto nn_tensor_type = type->type;
if (tensors_count->find(nn_tensor_type) == tensors_count->end()) {
tensors_count->insert({nn_tensor_type, 0});
}
tensors_count->at(nn_tensor_type)++;
return ANEURALNETWORKS_NO_ERROR;
});
}
void TearDown() override { tensors_count->clear(); }
static void TearDownTestSuite() {
delete tensors_count;
tensors_count = nullptr;
}
static std::map<int, int>* tensors_count;
};
std::map<int, int>* NnApiSignedQuantizationTest::tensors_count = nullptr;
TEST_F(NnApiSignedQuantizationTest,
HybridFullyConnectedMapsToSignedSymmOnSdk29) {
nnapi_mock_->SetAndroidSdkVersion(29);
HybridFullyConnectedOpModel m(
nnapi_mock_->GetNnApi(), /*units=*/3, /*batches=*/2,
/*input=*/{TensorType_FLOAT32, {2, 10}},
/*weights=*/{TensorType_INT8, {3, 10}, 0, 0, 10.0 / 127.0, 0});
m.SetSignedWeights({
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, // u = 0
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, // u = 1
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, // u = 2
});
m.SetBias({1, 2, 3});
m.SetInput({
1, 2, 3, 4, 5, 6, 7, 8, -9, -10, // b = 0
1, 2, 3, 4, 5, 6, 7, -8, 9, -10, // b = 1
});
ASSERT_EQ(m.Invoke(), kTfLiteOk);
EXPECT_EQ(m.GetCompilationStatus(), kTfLiteOk);
ASSERT_EQ(tensors_count->size(), 3);
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_FLOAT32),
tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_INT32), tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_QUANT8_SYMM),
tensors_count->end());
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_FLOAT32),
4); // fc_input, fc_weights, fc_bias, fc_output
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_INT32), 1); // activation
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_QUANT8_SYMM),
1); // dequantize_weights_input
}
TEST_F(NnApiSignedQuantizationTest,
HybridFullyConnectedMapsToSignedSymmOnSdk30) {
nnapi_mock_->SetAndroidSdkVersion(30);
HybridFullyConnectedOpModel m(
nnapi_mock_->GetNnApi(), /*units=*/3, /*batches=*/2,
/*input=*/{TensorType_FLOAT32, {2, 10}},
/*weights=*/{TensorType_INT8, {3, 10}, 0, 0, 10.0 / 127.0, 0});
m.SetSignedWeights({
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, // u = 0
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, // u = 1
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, // u = 2
});
m.SetBias({1, 2, 3});
m.SetInput({
1, 2, 3, 4, 5, 6, 7, 8, -9, -10, // b = 0
1, 2, 3, 4, 5, 6, 7, -8, 9, -10, // b = 1
});
ASSERT_EQ(m.Invoke(), kTfLiteOk);
EXPECT_EQ(m.GetCompilationStatus(), kTfLiteOk);
ASSERT_EQ(tensors_count->size(), 3);
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_FLOAT32),
tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_INT32), tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_QUANT8_SYMM),
tensors_count->end());
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_FLOAT32),
4); // fc_input, fc_weights, fc_bias, fc_output
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_INT32), 1); // activation
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_QUANT8_SYMM),
1); // dequantize_weights_input
}
template <typename FilterType>
class BaseConvolutionOpModel : public SingleOpModelWithNNAPI {
public:
BaseConvolutionOpModel(
const NnApi* nnapi, TfLiteRegistration* registration,
const TensorData& input, const TensorData& filter,
const TensorData& output, int stride_width = 2, int stride_height = 2,
enum Padding padding = Padding_VALID,
enum ActivationFunctionType activation = ActivationFunctionType_NONE,
int dilation_width_factor = 1, int dilation_height_factor = 1,
std::initializer_list<FilterType> filter_data = {}) {
SingleOpModelWithNNAPI::Init(nnapi);
input_ = AddInput(input);
if (filter_data.size()) {
filter_ = AddConstInput(filter, filter_data);
} else {
filter_ = AddInput(filter);
}
int bias_size = GetShape(filter_)[0];
if (input.type == TensorType_FLOAT32) {
bias_ = AddInput({TensorType_FLOAT32, {bias_size}});
} else {
// This is a quantized version. The scale of 'bias' depends on the scales
// of input and filter. Supposedly this is correctly set during quantized
// training.
if (filter.per_channel_quantization) {
// per channel quantization.
std::vector<float> bias_scale(
filter.per_channel_quantization_scales.size());
std::vector<int64_t> bias_zero_points(
filter.per_channel_quantization_scales.size());
for (size_t i = 0; i < filter.per_channel_quantization_scales.size();
++i) {
bias_scale[i] =
input.scale * filter.per_channel_quantization_scales[i];
bias_zero_points[i] = 0;
}
tflite::TensorType bias_type = TensorType_INT32;
if (input.type == TensorType_INT16) {
// In case of 16-bit, the bias type is set to be int 64.
bias_type = TensorType_INT64;
}
TensorData bias{bias_type,
{bias_size},
/*min=*/0,
/*max=*/0,
/*scale=*/0,
/*zero_point=*/0,
true,
/*per_channel_quantization_scales=*/bias_scale,
/*per_channel_quantization_offsets=*/bias_zero_points,
/*channel_index==*/0};
bias_ = AddInput(bias);
} else {
// per tensor quantization.
auto bias_scale = GetScale(input_) * GetScale(filter_);
TensorData bias{TensorType_INT32, {bias_size}, 0, 0, bias_scale};
bias_ = AddInput(bias);
}
}
output_ = AddOutput(output);
SetBuiltinOp(BuiltinOperator_CONV_2D, BuiltinOptions_Conv2DOptions,
CreateConv2DOptions(
builder_, padding, stride_width, stride_height, activation,
dilation_width_factor, dilation_height_factor)
.Union());
resolver_ = std::make_unique<SingleOpResolver>(BuiltinOperator_CONV_2D,
registration);
BuildInterpreter({GetShape(input_), GetShape(filter_), GetShape(bias_)},
/*num_threads=*/-1,
/* allow_fp32_relax_to_fp16 */ false,
/*apply_delegate=*/false);
compilation_status_ = ApplyDelegate();
}
protected:
int input_;
int filter_;
int bias_;
int output_;
};
class QuantizedConvolutionOpModel : public BaseConvolutionOpModel<uint8_t> {
public:
using BaseConvolutionOpModel::BaseConvolutionOpModel;
void SetInput(std::initializer_list<float> data) {
QuantizeAndPopulate<uint8_t>(input_, data);
}
void SetFilter(std::initializer_list<float> data) {
QuantizeAndPopulate<uint8_t>(filter_, data);
}
void SetBias(std::initializer_list<float> data) {
QuantizeAndPopulate<int32_t>(bias_, data);
}
std::vector<uint8_t> GetOutput() { return ExtractVector<uint8_t>(output_); }
std::vector<float> GetDequantizedOutput() {
return Dequantize<uint8_t>(ExtractVector<uint8_t>(output_),
GetScale(output_), GetZeroPoint(output_));
}
};
TEST_F(NnApiSignedQuantizationTest,
Conv2DUnsignedPerTensorMapsToUnsignedOnSdk29) {
QuantizedConvolutionOpModel m(nnapi_mock_->GetNnApi(),
ops::builtin::Register_CONVOLUTION_REF(),
{TensorType_UINT8, {2, 2, 4, 1}, -63.5, 64},
{TensorType_UINT8, {3, 2, 2, 1}, -63.5, 64},
{TensorType_UINT8, {}, -127, 128});
m.SetInput({
// First batch
1, 1, 1, 1, // row = 1
2, 2, 2, 2, // row = 2
// Second batch
1, 2, 3, 4, // row = 1
1, 2, 3, 4, // row = 2
});
m.SetFilter({
1, 2, 3, 4, // first 2x2 filter
-1, 1, -1, 1, // second 2x2 filter
-1, -1, 1, 1, // third 2x2 filter
});
m.SetBias({1, 2, 3});
ASSERT_EQ(m.Invoke(), kTfLiteOk);
EXPECT_EQ(m.GetCompilationStatus(), kTfLiteOk);
ASSERT_EQ(tensors_count->size(), 3);
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM),
tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_INT32),
tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_INT32), tensors_count->end());
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM),
3); // input, filter, output
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_INT32), 1); // bias
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_INT32),
4); // padding, stride_width, stride_height, activation
}
TEST_F(NnApiSignedQuantizationTest,
Conv2dUnsignedPerTensorMapsToUnsignedOnSdk30) {
nnapi_mock_->SetAndroidSdkVersion(30);
QuantizedConvolutionOpModel m(nnapi_mock_->GetNnApi(),
ops::builtin::Register_CONVOLUTION_REF(),
{TensorType_UINT8, {2, 2, 4, 1}, -63.5, 64},
{TensorType_UINT8, {3, 2, 2, 1}, -63.5, 64},
{TensorType_UINT8, {}, -127, 128});
m.SetInput({
// First batch
1, 1, 1, 1, // row = 1
2, 2, 2, 2, // row = 2
// Second batch
1, 2, 3, 4, // row = 1
1, 2, 3, 4, // row = 2
});
m.SetFilter({
1, 2, 3, 4, // first 2x2 filter
-1, 1, -1, 1, // second 2x2 filter
-1, -1, 1, 1, // third 2x2 filter
});
m.SetBias({1, 2, 3});
ASSERT_EQ(m.Invoke(), kTfLiteOk);
EXPECT_EQ(m.GetCompilationStatus(), kTfLiteOk);
ASSERT_EQ(tensors_count->size(), 3);
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM),
tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_INT32),
tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_INT32), tensors_count->end());
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM),
3); // input, filter, output
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_INT32), 1); // bias
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_INT32),
4); // padding, stride_width, stride_height, activation
}
class PerChannelQuantizedConvolutionOpModel
: public BaseConvolutionOpModel<int8_t> {
public:
using BaseConvolutionOpModel::BaseConvolutionOpModel;
void SetInput(std::initializer_list<float> data) {
QuantizeAndPopulate<int8_t>(input_, data);
}
void SetFilter(std::initializer_list<float> data) {
PerChannelSymmetricQuantizeAndPopulate(filter_, data);
}
void SetBias(std::initializer_list<float> data) {
PerChannelQuantizeBias(bias_, data);
}
std::vector<int8_t> GetOutput() { return ExtractVector<int8_t>(output_); }
std::vector<float> GetDequantizedOutput() {
return Dequantize<int8_t>(ExtractVector<int8_t>(output_), GetScale(output_),
GetZeroPoint(output_));
}
};
TEST_F(NnApiSignedQuantizationTest,
Conv2dSignedPerTensorMapsToUnsignedOnSdk29) {
nnapi_mock_->SetAndroidSdkVersion(29);
PerChannelQuantizedConvolutionOpModel m(
nnapi_mock_->GetNnApi(), ops::builtin::Register_CONVOLUTION_REF(),
{TensorType_INT8, {1, 2, 3, 2}, -63.5, 64, 0.5, -1},
{TensorType_INT8,
// [2 * 2 * 2 * 2] as [output_channel, y, x, input_channel]
{2, 2, 2, 2},
0,
0,
0,
0,
/*per_channel_quantization=*/true,
/*per_channel_quantization_scales=*/{1},
/*per_channel_quantization_offsets=*/{0},
/*channel_index=*/0},
{TensorType_INT8, {}, -63.5, 64, 0.5, -1},
/*stride_width=*/1, /*stride_height=*/1);
m.SetInput({
// [1 * 2 * 3 * 2] as [batch, y, x, input_channel]
3, 2, // batch = 0, y = 0, x = 0
1, -1, // batch = 0, y = 0, x = 1
-2, -3, // batch = 0, y = 0, x = 2
4, 3, // batch = 0, y = 1, x = 0
2, -2, // batch = 0, y = 1, x = 1
-3, -4, // batch = 0, y = 1, x = 2
});
m.SetFilter(
// [2 * 2 * 2 * 2] as [output_channel, y, x, input_channel]
{
1, 2, // out channel = 0, y = 0, x = 0
3, 4, // out channel = 0, y = 0, x = 1
3, 4, // out channel = 0, y = 1, x = 0
5, 6, // out channel = 0, y = 1, x = 1
7, 8, // out channel = 1, y = 0, x = 0
5, 6, // out channel = 1, y = 0, x = 1
3, 4, // out channel = 1, y = 1, x = 0
1, 2, // out channel = 1, y = 1, x = 1
});
m.SetBias({3, -2});
// Invoke and verify output.
// output has dimension [1 * 1 * 2 * 2] as [batch, y, x, output_channel]
ASSERT_EQ(m.Invoke(), kTfLiteOk);
EXPECT_EQ(m.GetCompilationStatus(), kTfLiteOk);
ASSERT_EQ(tensors_count->size(), 3);
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM),
tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_INT32),
tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_INT32), tensors_count->end());
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM),
3); // input, filter, output
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_INT32), 1); // bias
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_INT32),
4); // padding, stride_width, stride_height, activation
}
TEST_F(NnApiSignedQuantizationTest,
Conv2dSignedPerTensorMapsToUnsignedOnSdk30) {
nnapi_mock_->SetAndroidSdkVersion(30);
PerChannelQuantizedConvolutionOpModel m(
nnapi_mock_->GetNnApi(), ops::builtin::Register_CONVOLUTION_REF(),
{TensorType_INT8, {1, 2, 3, 2}, -63.5, 64, 0.5, -1},
{TensorType_INT8,
// [2 * 2 * 2 * 2] as [output_channel, y, x, input_channel]
{2, 2, 2, 2},
0,
0,
0,
0,
/*per_channel_quantization=*/true,
/*per_channel_quantization_scales=*/{1},
/*per_channel_quantization_offsets=*/{0},
/*channel_index=*/0},
{TensorType_INT8, {}, -63.5, 64, 0.5, -1},
/*stride_width=*/1, /*stride_height=*/1);
m.SetInput({
// [1 * 2 * 3 * 2] as [batch, y, x, input_channel]
3, 2, // batch = 0, y = 0, x = 0
1, -1, // batch = 0, y = 0, x = 1
-2, -3, // batch = 0, y = 0, x = 2
4, 3, // batch = 0, y = 1, x = 0
2, -2, // batch = 0, y = 1, x = 1
-3, -4, // batch = 0, y = 1, x = 2
});
m.SetFilter(
// [2 * 2 * 2 * 2] as [output_channel, y, x, input_channel]
{
1, 2, // out channel = 0, y = 0, x = 0
3, 4, // out channel = 0, y = 0, x = 1
3, 4, // out channel = 0, y = 1, x = 0
5, 6, // out channel = 0, y = 1, x = 1
7, 8, // out channel = 1, y = 0, x = 0
5, 6, // out channel = 1, y = 0, x = 1
3, 4, // out channel = 1, y = 1, x = 0
1, 2, // out channel = 1, y = 1, x = 1
});
m.SetBias({3, -2});
// Invoke and verify output.
// output has dimension [1 * 1 * 2 * 2] as [batch, y, x, output_channel]
ASSERT_EQ(m.Invoke(), kTfLiteOk);
EXPECT_EQ(m.GetCompilationStatus(), kTfLiteOk);
ASSERT_EQ(tensors_count->size(), 3);
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM_SIGNED),
tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_INT32),
tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_INT32), tensors_count->end());
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM_SIGNED),
3); // input, filter, output
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_INT32), 1); // bias
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_INT32),
4); // padding, stride_width, stride_height, activation
}
TEST_F(NnApiSignedQuantizationTest,
Conv2dSignedPerChannelMapsToUnsignedOnSdk29) {
PerChannelQuantizedConvolutionOpModel m(
nnapi_mock_->GetNnApi(), ops::builtin::Register_CONVOLUTION_REF(),
{TensorType_INT8, {1, 2, 3, 2}, -63.5, 64, 0.5, -1},
{TensorType_INT8,
// [2 * 2 * 2 * 2] as [output_channel, y, x, input_channel]
{2, 2, 2, 2},
0,
0,
0,
0,
/*per_channel_quantization=*/true,
/*per_channel_quantization_scales=*/{1, 2},
/*per_channel_quantization_offsets=*/{0, 0},
/*channel_index=*/0},
{TensorType_INT8, {}, -63.5, 64, 0.5, -1},
/*stride_width=*/1, /*stride_height=*/1);
m.SetInput({
// [1 * 2 * 3 * 2] as [batch, y, x, input_channel]
3, 2, // batch = 0, y = 0, x = 0
1, -1, // batch = 0, y = 0, x = 1
-2, -3, // batch = 0, y = 0, x = 2
4, 3, // batch = 0, y = 1, x = 0
2, -2, // batch = 0, y = 1, x = 1
-3, -4, // batch = 0, y = 1, x = 2
});
m.SetFilter(
// [2 * 2 * 2 * 2] as [output_channel, y, x, input_channel]
{
1, 2, // out channel = 0, y = 0, x = 0
3, 4, // out channel = 0, y = 0, x = 1
3, 4, // out channel = 0, y = 1, x = 0
5, 6, // out channel = 0, y = 1, x = 1
7, 8, // out channel = 1, y = 0, x = 0
5, 6, // out channel = 1, y = 0, x = 1
3, 4, // out channel = 1, y = 1, x = 0
1, 2, // out channel = 1, y = 1, x = 1
});
m.SetBias({3, -2});
// Invoke and verify output.
// output has dimension [1 * 1 * 2 * 2] as [batch, y, x, output_channel]
ASSERT_EQ(m.Invoke(), kTfLiteOk);
EXPECT_EQ(m.GetCompilationStatus(), kTfLiteOk);
ASSERT_EQ(tensors_count->size(), 4);
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM),
tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_QUANT8_SYMM_PER_CHANNEL),
tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_INT32),
tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_INT32), tensors_count->end());
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM),
2); // input, output
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_QUANT8_SYMM_PER_CHANNEL),
1); // filter
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_INT32), 1); // bias
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_INT32),
4); // padding, stride_width, stride_height, activation
}
TEST_F(NnApiSignedQuantizationTest, Conv2dSignedPerChannelMapsToSignedOnSdk30) {
nnapi_mock_->SetAndroidSdkVersion(30);
PerChannelQuantizedConvolutionOpModel m(
nnapi_mock_->GetNnApi(), ops::builtin::Register_CONVOLUTION_REF(),
{TensorType_INT8, {1, 2, 3, 2}, -63.5, 64, 0.5, -1},
{TensorType_INT8,
// [2 * 2 * 2 * 2] as [output_channel, y, x, input_channel]
{2, 2, 2, 2},
0,
0,
0,
0,
/*per_channel_quantization=*/true,
/*per_channel_quantization_scales=*/{1, 2},
/*per_channel_quantization_offsets=*/{0, 0},
/*channel_index=*/0},
{TensorType_INT8, {}, -63.5, 64, 0.5, -1},
/*stride_width=*/1, /*stride_height=*/1);
m.SetInput({
// [1 * 2 * 3 * 2] as [batch, y, x, input_channel]
3, 2, // batch = 0, y = 0, x = 0
1, -1, // batch = 0, y = 0, x = 1
-2, -3, // batch = 0, y = 0, x = 2
4, 3, // batch = 0, y = 1, x = 0
2, -2, // batch = 0, y = 1, x = 1
-3, -4, // batch = 0, y = 1, x = 2
});
m.SetFilter(
// [2 * 2 * 2 * 2] as [output_channel, y, x, input_channel]
{
1, 2, // out channel = 0, y = 0, x = 0
3, 4, // out channel = 0, y = 0, x = 1
3, 4, // out channel = 0, y = 1, x = 0
5, 6, // out channel = 0, y = 1, x = 1
7, 8, // out channel = 1, y = 0, x = 0
5, 6, // out channel = 1, y = 0, x = 1
3, 4, // out channel = 1, y = 1, x = 0
1, 2, // out channel = 1, y = 1, x = 1
});
m.SetBias({3, -2});
// Invoke and verify output.
// output has dimension [1 * 1 * 2 * 2] as [batch, y, x, output_channel]
ASSERT_EQ(m.Invoke(), kTfLiteOk);
EXPECT_EQ(m.GetCompilationStatus(), kTfLiteOk);
ASSERT_EQ(tensors_count->size(), 4);
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM_SIGNED),
tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_QUANT8_SYMM_PER_CHANNEL),
tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_INT32),
tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_INT32), tensors_count->end());
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM_SIGNED),
2); // input, output
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_QUANT8_SYMM_PER_CHANNEL),
1); // filter
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_INT32), 1); // bias
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_INT32),
4); // padding, stride_width, stride_height, activation
}
class QuantizeOpModel : public SingleOpModelWithNNAPI {
public:
QuantizeOpModel(const NnApi* nnapi, const TensorData& input,
const TensorData& output) {
SingleOpModelWithNNAPI::Init(nnapi);
input_ = AddInput(input);
output_ = AddOutput(output);
SetBuiltinOp(BuiltinOperator_QUANTIZE, BuiltinOptions_QuantizeOptions,
CreateQuantizeOptions(builder_).Union());
BuildInterpreter({GetShape(input_)}, /*num_threads=*/-1,
/* allow_fp32_relax_to_fp16 */ false,
/*apply_delegate=*/false);
compilation_status_ = ApplyDelegate();
}
void SetInput(std::initializer_list<float> data) {
PopulateTensor(input_, data);
}
template <typename T>
void SetInputAndQuantize(std::initializer_list<float> data) {
QuantizeAndPopulate<T>(input_, data);
}
template <typename T>
std::vector<T> GetOutput() {
return ExtractVector<T>(output_);
}
private:
int input_;
int output_;
};
TEST_F(NnApiSignedQuantizationTest, QuantizeUint8MapsToUint8OnSdk29) {
// [-63.5, 64] -> scale=0.5 zero_point=127 for UINT8
QuantizeOpModel m(nnapi_mock_->GetNnApi(), {TensorType_FLOAT32, {2, 5}},
{TensorType_UINT8, {2, 5}, 0, 0, 0.5, 127});
m.SetInput({-63.5, -63, -62.5, -62, -61.5, 62, 62.5, 63, 63.5, 64});
ASSERT_EQ(m.Invoke(), kTfLiteOk);
EXPECT_EQ(m.GetCompilationStatus(), kTfLiteOk);
ASSERT_EQ(tensors_count->size(), 2);
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_FLOAT32),
tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM),
tensors_count->end());
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_FLOAT32),
1); // input
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM),
1); // output
}
TEST_F(NnApiSignedQuantizationTest, QuantizeUint8MapsToUint8OnSdk30) {
nnapi_mock_->SetAndroidSdkVersion(30);
// [-63.5, 64] -> scale=0.5 zero_point=127 for UINT8
QuantizeOpModel m(nnapi_mock_->GetNnApi(), {TensorType_FLOAT32, {2, 5}},
{TensorType_UINT8, {2, 5}, 0, 0, 0.5, 127});
m.SetInput({-63.5, -63, -62.5, -62, -61.5, 62, 62.5, 63, 63.5, 64});
ASSERT_EQ(m.Invoke(), kTfLiteOk);
EXPECT_EQ(m.GetCompilationStatus(), kTfLiteOk);
ASSERT_EQ(tensors_count->size(), 2);
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_FLOAT32),
tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM),
tensors_count->end());
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_FLOAT32),
1); // input
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM),
1); // output
}
// Quantize with Int8 output is only supported since SDK level 30.
TEST_F(NnApiSignedQuantizationTest, QuantizeInt8MapsToInt8OnSdk30) {
nnapi_mock_->SetAndroidSdkVersion(30);
// [-63.5, 64] -> scale=0.5 zero_point=1 for INT8
QuantizeOpModel m(nnapi_mock_->GetNnApi(), {TensorType_FLOAT32, {2, 5}},
{TensorType_INT8, {2, 5}, 0, 0, 0.5, -1});
m.SetInput({-63.5, -63, -62.5, -62, -61.5, 62, 62.5, 63, 63.5, 64});
ASSERT_EQ(m.Invoke(), kTfLiteOk);
EXPECT_EQ(m.GetCompilationStatus(), kTfLiteOk);
ASSERT_EQ(tensors_count->size(), 2);
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_FLOAT32),
tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM_SIGNED),
tensors_count->end());
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_FLOAT32),
1); // input
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM_SIGNED),
1); // output
}
class DequantizeOpModel : public SingleOpModelWithNNAPI {
public:
DequantizeOpModel(const NnApi* nnapi, TensorType type,
std::initializer_list<int> shape, float scale,
int32_t zero_point, int version) {
SingleOpModelWithNNAPI::Init(nnapi);
const TensorData input_tensor_data = {type, shape, 0, 0, scale, zero_point};
input_ = AddInput(input_tensor_data);
output_ = AddOutput({TensorType_FLOAT32, shape});
SetBuiltinOp(BuiltinOperator_DEQUANTIZE, BuiltinOptions_DequantizeOptions,
CreateDequantizeOptions(builder_).Union());
resolver_ = std::make_unique<SingleOpResolver>(
BuiltinOperator_DEQUANTIZE, ops::builtin::Register_DEQUANTIZE(),
version);
BuildInterpreter({GetShape(input_)}, /*num_threads=*/-1,
/* allow_fp32_relax_to_fp16 */ false,
/*apply_delegate=*/false);
compilation_status_ = ApplyDelegate();
}
template <typename T>
void SetInput(std::initializer_list<T> data) {
PopulateTensor(input_, data);
}
std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
private:
int input_;
int output_;
};
TEST_F(NnApiSignedQuantizationTest, DequantizeUint8MapsToUint8OnSdk29) {
// [-63.5, 64] -> scale=0.5 zero_point=127 for UINT8
DequantizeOpModel m(nnapi_mock_->GetNnApi(), TensorType_UINT8, {2, 5}, 0.5,
127, 1);
m.SetInput<uint8_t>({0, 1, 2, 3, 4, 251, 252, 253, 254, 255});
ASSERT_EQ(m.Invoke(), kTfLiteOk);
EXPECT_EQ(m.GetCompilationStatus(), kTfLiteOk);
ASSERT_EQ(tensors_count->size(), 2);
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM),
tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_FLOAT32),
tensors_count->end());
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM),
1); // input
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_FLOAT32),
1); // output
}
TEST_F(NnApiSignedQuantizationTest, DequantizeUint8MapsToUint8OnSdk30) {
nnapi_mock_->SetAndroidSdkVersion(30);
// [-63.5, 64] -> scale=0.5 zero_point=127 for UINT8
DequantizeOpModel m(nnapi_mock_->GetNnApi(), TensorType_UINT8, {2, 5}, 0.5,
127, 1);
m.SetInput<uint8_t>({0, 1, 2, 3, 4, 251, 252, 253, 254, 255});
ASSERT_EQ(m.Invoke(), kTfLiteOk);
EXPECT_EQ(m.GetCompilationStatus(), kTfLiteOk);
ASSERT_EQ(tensors_count->size(), 2);
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM),
tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_FLOAT32),
tensors_count->end());
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM),
1); // input
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_FLOAT32),
1); // output
}
// Dequantize with Int8 input is only supported for symmetric quantization on
// SDK level 29
TEST_F(NnApiSignedQuantizationTest,
DequantizeTestInt8SymmMapsToInt8SymmOnSdk29) {
// [-63.5, 64] -> scale=0.5, zero_point=0 for INT8
DequantizeOpModel m(nnapi_mock_->GetNnApi(), TensorType_INT8, {2, 5}, 0.5, 0,
2);
m.SetInput<int8_t>({-128, -127, -126, -125, -124, 123, 124, 125, 126, 127});
ASSERT_EQ(m.Invoke(), kTfLiteOk);
EXPECT_EQ(m.GetCompilationStatus(), kTfLiteOk);
ASSERT_EQ(tensors_count->size(), 2);
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_QUANT8_SYMM),
tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_FLOAT32),
tensors_count->end());
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_QUANT8_SYMM),
1); // input
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_FLOAT32),
1); // output
}
// Dequantize with Int8 input is only supported since SDK level 30.
TEST_F(NnApiSignedQuantizationTest, DequantizeTestInt8MapsToInt8OnSdk30) {
nnapi_mock_->SetAndroidSdkVersion(30);
// [-63.5, 64] -> scale=0.5, zero_point=1 for INT8
DequantizeOpModel m(nnapi_mock_->GetNnApi(), TensorType_INT8, {2, 5}, 0.5, -1,
2);
m.SetInput<int8_t>({-128, -127, -126, -125, -124, 123, 124, 125, 126, 127});
ASSERT_EQ(m.Invoke(), kTfLiteOk);
EXPECT_EQ(m.GetCompilationStatus(), kTfLiteOk);
ASSERT_EQ(tensors_count->size(), 2);
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM_SIGNED),
tensors_count->end());
ASSERT_NE(tensors_count->find(ANEURALNETWORKS_TENSOR_FLOAT32),
tensors_count->end());
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_QUANT8_ASYMM_SIGNED),
1); // input
EXPECT_EQ(tensors_count->at(ANEURALNETWORKS_TENSOR_FLOAT32),
1); // output
}
} // namespace
} // namespace tflite
tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
+5759
View File
File diff suppressed because it is too large Load Diff
tensorflow/lite/delegates/nnapi/quant_lstm_sup.cc
+155
View File
@@ -0,0 +1,155 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/lite/delegates/nnapi/quant_lstm_sup.h"
#include <algorithm>
#include <cstdint>
#include <vector>
#include "tensorflow/lite/c/common.h"
#include "tensorflow/lite/kernels/kernel_util.h"
namespace tflite {
namespace delegate {
namespace nnapi {
// The function extracts a submatrix of the weights at a given row
// and column offsets from a 2D matrix
void ExtractQuantLstmWeightsSubmatrix(const TfLiteIntArray* submatrix_dims,
const int32_t offset_row,
const int32_t offset_column,
const TfLiteIntArray* weight_dims,
const uint8_t* weights,
std::vector<uint8_t>* submatrix) {
auto const& submatrix_rows = submatrix_dims->data[0];
auto const& submatrix_cols = submatrix_dims->data[1];
auto const& weight_cols = weight_dims->data[1];
submatrix->resize(NumElements(submatrix_dims));
for (uint32_t i = 0, end = submatrix_rows * submatrix_cols; i < end; ++i) {
const uint32_t row = i / submatrix_cols;
const uint32_t column = i % submatrix_cols;
(*submatrix)[i] =
weights[(row + offset_row) * weight_cols + column + offset_column];
}
}
inline int OutputDepth(const TfLiteIntArray* weight_dims) {
return weight_dims->data[0] / 4;
}
inline int InputDepth(const TfLiteIntArray* weight_dims) {
return weight_dims->data[1] - OutputDepth(weight_dims);
}
void SetWeightSubmatrixDims(const TfLiteIntArray* weight_dims,
TfLiteIntArray* recurrent_submatrix_dims,
TfLiteIntArray* input_submatrix_dims) {
const auto input_depth = InputDepth(weight_dims);
const auto output_depth = OutputDepth(weight_dims);
recurrent_submatrix_dims->data[0] = output_depth;
recurrent_submatrix_dims->data[1] = output_depth;
input_submatrix_dims->data[0] = output_depth;
input_submatrix_dims->data[1] = input_depth;
}
// Doing exactly the opposite work of QuantizedLSTMCell::concatenateWeights
// in NNAPI, decomposing the concat_weights tensor data into its 8 components
// according to the following diagram
//
// +-----------------------------------+
// | recurrentToInput | inputToInput |
// |-------------------+---------------|
// | recurrentToCell | inputToCell |
// |-------------------+---------------|
// | recurrentToForget | inputToForget |
// |-------------------+---------------|
// | recurrentToOutput | inputToOutput |
// +-----------------------------------+
void DecomposeQuantLstmWeightsTensor(const uint8_t* concat_weights,
const TfLiteIntArray* weight_dims,
std::vector<uint8_t>* recurrent_to_input,
std::vector<uint8_t>* input_to_input,
std::vector<uint8_t>* recurrent_to_cell,
std::vector<uint8_t>* input_to_cell,
std::vector<uint8_t>* recurrent_to_forget,
std::vector<uint8_t>* input_to_forget,
std::vector<uint8_t>* recurrent_to_output,
std::vector<uint8_t>* input_to_output) {
const auto output_depth = OutputDepth(weight_dims);
TfLiteIntArray* recurrent_submatrix_dims = TfLiteIntArrayCreate(2);
TfLiteIntArray* input_submatrix_dims = TfLiteIntArrayCreate(2);
SetWeightSubmatrixDims(weight_dims, recurrent_submatrix_dims,
input_submatrix_dims);
ExtractQuantLstmWeightsSubmatrix(recurrent_submatrix_dims, 0 * output_depth,
0, weight_dims, concat_weights,
recurrent_to_input);
ExtractQuantLstmWeightsSubmatrix(input_submatrix_dims, 0 * output_depth,
output_depth, weight_dims, concat_weights,
input_to_input);
ExtractQuantLstmWeightsSubmatrix(recurrent_submatrix_dims, 1 * output_depth,
0, weight_dims, concat_weights,
recurrent_to_cell);
ExtractQuantLstmWeightsSubmatrix(input_submatrix_dims, 1 * output_depth,
output_depth, weight_dims, concat_weights,
input_to_cell);
ExtractQuantLstmWeightsSubmatrix(recurrent_submatrix_dims, 2 * output_depth,
0, weight_dims, concat_weights,
recurrent_to_forget);
ExtractQuantLstmWeightsSubmatrix(input_submatrix_dims, 2 * output_depth,
output_depth, weight_dims, concat_weights,
input_to_forget);
ExtractQuantLstmWeightsSubmatrix(recurrent_submatrix_dims, 3 * output_depth,
0, weight_dims, concat_weights,
recurrent_to_output);
ExtractQuantLstmWeightsSubmatrix(input_submatrix_dims, 3 * output_depth,
output_depth, weight_dims, concat_weights,
input_to_output);
TfLiteIntArrayFree(recurrent_submatrix_dims);
TfLiteIntArrayFree(input_submatrix_dims);
}
void DecomposeBiasTensor(const int32_t* biases, int bias_size,
std::vector<int32_t>* input_bias,
std::vector<int32_t>* cell_bias,
std::vector<int32_t>* forget_bias,
std::vector<int32_t>* output_bias) {
input_bias->resize(bias_size);
std::copy(biases, biases + bias_size, input_bias->begin());
cell_bias->resize(bias_size);
std::copy(biases + bias_size, biases + 2 * bias_size, cell_bias->begin());
forget_bias->resize(bias_size);
std::copy(biases + 2 * bias_size, biases + 3 * bias_size,
forget_bias->begin());
output_bias->resize(bias_size);
std::copy(biases + 3 * bias_size, biases + 4 * bias_size,
output_bias->begin());
}
} // namespace nnapi
} // namespace delegate
} // namespace tflite
tensorflow/lite/delegates/nnapi/quant_lstm_sup.h
+58
View File
@@ -0,0 +1,58 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_DELEGATES_NNAPI_QUANT_LSTM_SUP_H_
#define TENSORFLOW_LITE_DELEGATES_NNAPI_QUANT_LSTM_SUP_H_
#include <vector>
#include "tensorflow/lite/core/c/common.h"
namespace tflite {
namespace delegate {
namespace nnapi {
void ExtractQuantLstmWeightsSubmatrix(const TfLiteIntArray* submatrix_dims,
const int32_t offset_row,
const int32_t offset_column,
const TfLiteIntArray* weight_dims,
const uint8_t* weights,
std::vector<uint8_t>* submatrix);
void DecomposeQuantLstmWeightsTensor(const uint8_t* concat_weights,
const TfLiteIntArray* weight_dims,
std::vector<uint8_t>* recurrent_to_input,
std::vector<uint8_t>* input_to_input,
std::vector<uint8_t>* recurrent_to_cell,
std::vector<uint8_t>* input_to_cell,
std::vector<uint8_t>* recurrent_to_forget,
std::vector<uint8_t>* input_to_forget,
std::vector<uint8_t>* recurrent_to_output,
std::vector<uint8_t>* input_to_output);
void SetWeightSubmatrixDims(const TfLiteIntArray* weight_dims,
TfLiteIntArray* recurrent_submatrix_dims,
TfLiteIntArray* input_submatrix_dims);
void DecomposeBiasTensor(const int32_t* biases, int bias_size,
std::vector<int32_t>* input_bias,
std::vector<int32_t>* cell_bias,
std::vector<int32_t>* forget_bias,
std::vector<int32_t>* output_bias);
} // namespace nnapi
} // namespace delegate
} // namespace tflite
#endif // TENSORFLOW_LITE_DELEGATES_NNAPI_QUANT_LSTM_SUP_H_
tensorflow/lite/delegates/nnapi/quant_lstm_sup_test.cc
+336
View File
@@ -0,0 +1,336 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/lite/delegates/nnapi/quant_lstm_sup.h"
#include <cstdint>
#include <initializer_list>
#include <vector>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "tensorflow/lite/core/c/common.h"
namespace {
using ::testing::ElementsAreArray;
using ::testing::Test;
class DimsAllocatingTest : public Test {
protected:
DimsAllocatingTest() : allocated_dims_() {}
~DimsAllocatingTest() override {
for (TfLiteIntArray* dim : allocated_dims_) {
TfLiteIntArrayFree(dim);
}
}
TfLiteIntArray* CreateDimArray(int size,
std::initializer_list<int> dimensions) {
TfLiteIntArray* dims = TfLiteIntArrayCreate(size);
allocated_dims_.push_back(dims);
int i = 0;
for (const int dimension : dimensions) {
dims->data[i++] = dimension;
}
return dims;
}
private:
std::vector<TfLiteIntArray*> allocated_dims_;
};
using tflite::delegate::nnapi::ExtractQuantLstmWeightsSubmatrix;
class ExtractQuantLstmWeightsSubmatrixTest : public DimsAllocatingTest {};
TEST_F(ExtractQuantLstmWeightsSubmatrixTest, TopLeftSubmatrixIsExtracted) {
std::vector<uint8_t> weights = {1, 2, 3, 4, 5, //
11, 12, 13, 14, 15, //
101, 102, 103, 104, 105, //
111, 112, 113, 114, 115, //
201, 202, 203, 204, 205, //
211, 212, 213, 214, 215, //
221, 222, 223, 224, 225, //
231, 232, 233, 234, 235};
const TfLiteIntArray* weight_dims = CreateDimArray(2, {8, 5});
std::vector<uint8_t> submatrix;
const TfLiteIntArray* submatrix_dims = CreateDimArray(2, {2, 3});
ExtractQuantLstmWeightsSubmatrix(submatrix_dims, 0 /* offset_row */,
0 /* offset_column */, weight_dims,
weights.data(), &submatrix);
EXPECT_THAT(submatrix, ElementsAreArray({1, 2, 3, 11, 12, 13}));
}
TEST_F(ExtractQuantLstmWeightsSubmatrixTest, TopRightSubmatrixIsExtracted) {
std::vector<uint8_t> weights = {1, 2, 3, 4, 5, //
11, 12, 13, 14, 15, //
101, 102, 103, 104, 105, //
111, 112, 113, 114, 115, //
201, 202, 203, 204, 205, //
211, 212, 213, 214, 215, //
221, 222, 223, 224, 225, //
231, 232, 233, 234, 235};
const TfLiteIntArray* weight_dims = CreateDimArray(2, {8, 5});
std::vector<uint8_t> submatrix;
const TfLiteIntArray* submatrix_dims = CreateDimArray(2, {2, 2});
ExtractQuantLstmWeightsSubmatrix(submatrix_dims, 0 /* offset_row */,
3 /* offset_column */, weight_dims,
weights.data(), &submatrix);
EXPECT_THAT(submatrix, ElementsAreArray({4, 5, 14, 15}));
}
TEST_F(ExtractQuantLstmWeightsSubmatrixTest, RightCentralSubmatrixIsExtracted) {
std::vector<uint8_t> weights = {1, 2, 3, 4, 5, //
11, 12, 13, 14, 15, //
101, 102, 103, 104, 105, //
111, 112, 113, 114, 115, //
201, 202, 203, 204, 205, //
211, 212, 213, 214, 215, //
221, 222, 223, 224, 225, //
231, 232, 233, 234, 235};
const TfLiteIntArray* weight_dims = CreateDimArray(2, {8, 5});
std::vector<uint8_t> submatrix;
const TfLiteIntArray* submatrix_dims = CreateDimArray(2, {2, 2});
ExtractQuantLstmWeightsSubmatrix(
submatrix_dims, 1 * submatrix_dims->data[0] /* offset_row */,
3 /* offset_column */, weight_dims, weights.data(), &submatrix);
EXPECT_THAT(submatrix, ElementsAreArray({104, 105, 114, 115}));
}
using tflite::delegate::nnapi::DecomposeQuantLstmWeightsTensor;
class QuantLstmWeightDecompTest : public DimsAllocatingTest {
protected:
QuantLstmWeightDecompTest()
: weights_({1, 2, 3, 4, 5, //
11, 12, 13, 14, 15, //
101, 102, 103, 104, 105, //
111, 112, 113, 114, 115, //
201, 202, 203, 204, 205, //
211, 212, 213, 214, 215, //
221, 222, 223, 224, 225, //
231, 232, 233, 234, 235}),
// Creating the arrays empty, the size is set by the decomposition
// function
recurrent_to_input_(),
input_to_input_(),
recurrent_to_cell_(),
input_to_cell_(),
recurrent_to_forget_(),
input_to_forget_(),
recurrent_to_output_(),
input_to_output_() {
weight_dims_ = CreateDimArray(2, {8, 5});
}
const std::vector<uint8_t> weights_;
const TfLiteIntArray* weight_dims_;
std::vector<uint8_t> recurrent_to_input_;
std::vector<uint8_t> input_to_input_;
std::vector<uint8_t> recurrent_to_cell_;
std::vector<uint8_t> input_to_cell_;
std::vector<uint8_t> recurrent_to_forget_;
std::vector<uint8_t> input_to_forget_;
std::vector<uint8_t> recurrent_to_output_;
std::vector<uint8_t> input_to_output_;
};
TEST_F(QuantLstmWeightDecompTest, ExtractRecurrentToInput) {
DecomposeQuantLstmWeightsTensor(
weights_.data(), weight_dims_, &recurrent_to_input_, &input_to_input_,
&recurrent_to_cell_, &input_to_cell_, &recurrent_to_forget_,
&input_to_forget_, &recurrent_to_output_, &input_to_output_);
EXPECT_THAT(recurrent_to_input_, ElementsAreArray({1, 2, //
11, 12}));
}
TEST_F(QuantLstmWeightDecompTest, ExtractInputToInput) {
DecomposeQuantLstmWeightsTensor(
weights_.data(), weight_dims_, &recurrent_to_input_, &input_to_input_,
&recurrent_to_cell_, &input_to_cell_, &recurrent_to_forget_,
&input_to_forget_, &recurrent_to_output_, &input_to_output_);
EXPECT_THAT(input_to_input_, ElementsAreArray({3, 4, 5, //
13, 14, 15}));
}
TEST_F(QuantLstmWeightDecompTest, ExtractRecurrentToCell) {
DecomposeQuantLstmWeightsTensor(
weights_.data(), weight_dims_, &recurrent_to_input_, &input_to_input_,
&recurrent_to_cell_, &input_to_cell_, &recurrent_to_forget_,
&input_to_forget_, &recurrent_to_output_, &input_to_output_);
EXPECT_THAT(recurrent_to_cell_, ElementsAreArray({101, 102, //
111, 112}));
}
TEST_F(QuantLstmWeightDecompTest, ExtractInputToCell) {
DecomposeQuantLstmWeightsTensor(
weights_.data(), weight_dims_, &recurrent_to_input_, &input_to_input_,
&recurrent_to_cell_, &input_to_cell_, &recurrent_to_forget_,
&input_to_forget_, &recurrent_to_output_, &input_to_output_);
EXPECT_THAT(input_to_cell_, ElementsAreArray({103, 104, 105, //
113, 114, 115}));
}
TEST_F(QuantLstmWeightDecompTest, ExtractRecurrentToForget) {
DecomposeQuantLstmWeightsTensor(
weights_.data(), weight_dims_, &recurrent_to_input_, &input_to_input_,
&recurrent_to_cell_, &input_to_cell_, &recurrent_to_forget_,
&input_to_forget_, &recurrent_to_output_, &input_to_output_);
EXPECT_THAT(recurrent_to_forget_, ElementsAreArray({201, 202, //
211, 212}));
}
TEST_F(QuantLstmWeightDecompTest, ExtractInputToForget) {
DecomposeQuantLstmWeightsTensor(
weights_.data(), weight_dims_, &recurrent_to_input_, &input_to_input_,
&recurrent_to_cell_, &input_to_cell_, &recurrent_to_forget_,
&input_to_forget_, &recurrent_to_output_, &input_to_output_);
EXPECT_THAT(input_to_forget_, ElementsAreArray({203, 204, 205, //
213, 214, 215}));
}
TEST_F(QuantLstmWeightDecompTest, ExtractRecurrentToOutput) {
DecomposeQuantLstmWeightsTensor(
weights_.data(), weight_dims_, &recurrent_to_input_, &input_to_input_,
&recurrent_to_cell_, &input_to_cell_, &recurrent_to_forget_,
&input_to_forget_, &recurrent_to_output_, &input_to_output_);
EXPECT_THAT(recurrent_to_output_, ElementsAreArray({221, 222, //
231, 232}));
}
TEST_F(QuantLstmWeightDecompTest, ExtractInputToOutput) {
DecomposeQuantLstmWeightsTensor(
weights_.data(), weight_dims_, &recurrent_to_input_, &input_to_input_,
&recurrent_to_cell_, &input_to_cell_, &recurrent_to_forget_,
&input_to_forget_, &recurrent_to_output_, &input_to_output_);
EXPECT_THAT(input_to_output_, ElementsAreArray({223, 224, 225, //
233, 234, 235}));
}
using tflite::delegate::nnapi::DecomposeBiasTensor;
TEST(DecomposeBiasTensor, ExtractInputBias) {
// clang-format off
std::vector<int32_t> biases
// inputGateBias
{-7876, 13488, -726, 32839,
// cellGateBias
39481, 48624, 48976, -21419,
// forgetGateBias
9206, -46884, -11693, -38724,
// outputGateBias
-58999, -17050, -41852, -40538};
// clang-format on
std::vector<int32_t> input_bias;
std::vector<int32_t> cell_bias;
std::vector<int32_t> forget_bias;
std::vector<int32_t> output_bias;
DecomposeBiasTensor(biases.data(), 4, &input_bias, &cell_bias, &forget_bias,
&output_bias);
EXPECT_THAT(input_bias, ElementsAreArray({-7876, 13488, -726, 32839}));
}
TEST(DecomposeBiasTensor, ExtractCellBias) {
// clang-format off
std::vector<int32_t> biases
// inputGateBias
{-7876, 13488, -726, 32839,
// cellGateBias
39481, 48624, 48976, -21419,
// forgetGateBias
9206, -46884, -11693, -38724,
// outputGateBias
-58999, -17050, -41852, -40538};
// clang-format on
std::vector<int32_t> input_bias;
std::vector<int32_t> cell_bias;
std::vector<int32_t> forget_bias;
std::vector<int32_t> output_bias;
DecomposeBiasTensor(biases.data(), 4, &input_bias, &cell_bias, &forget_bias,
&output_bias);
EXPECT_THAT(cell_bias, ElementsAreArray({39481, 48624, 48976, -21419}));
}
TEST(DecomposeBiasTensor, ExtractForgetBias) {
// clang-format off
std::vector<int32_t> biases
// inputGateBias
{-7876, 13488, -726, 32839,
// cellGateBias
39481, 48624, 48976, -21419,
// forgetGateBias
9206, -46884, -11693, -38724,
// outputGateBias
-58999, -17050, -41852, -40538};
// clang-format on
std::vector<int32_t> input_bias;
std::vector<int32_t> cell_bias;
std::vector<int32_t> forget_bias;
std::vector<int32_t> output_bias;
DecomposeBiasTensor(biases.data(), 4, &input_bias, &cell_bias, &forget_bias,
&output_bias);
EXPECT_THAT(forget_bias, ElementsAreArray({9206, -46884, -11693, -38724}));
}
TEST(DecomposeBiasTensor, ExtractOutputBias) {
// clang-format off
std::vector<int32_t> biases
// inputGateBias
{-7876, 13488, -726, 32839,
// cellGateBias
39481, 48624, 48976, -21419,
// forgetGateBias
9206, -46884, -11693, -38724,
// outputGateBias
-58999, -17050, -41852, -40538};
// clang-format on
std::vector<int32_t> input_bias;
std::vector<int32_t> cell_bias;
std::vector<int32_t> forget_bias;
std::vector<int32_t> output_bias;
DecomposeBiasTensor(biases.data(), 4, &input_bias, &cell_bias, &forget_bias,
&output_bias);
EXPECT_THAT(output_bias, ElementsAreArray({-58999, -17050, -41852, -40538}));
}
} // namespace