wehub/tensorflow--tensorflow
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
cffconvert / validate (push) Has been skipped
Details
License Check / license-check (push) Failing after 2s
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:14:16 +08:00
commit 8a852e4b4e
36502 changed files with 9277225 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tensorflow/lite/tools/BUILD
+471
View File
@@ -0,0 +1,471 @@
load("@rules_cc//cc:cc_binary.bzl", "cc_binary")
load("@rules_cc//cc:cc_library.bzl", "cc_library")
load("@rules_cc//cc:cc_test.bzl", "cc_test")
load("@rules_python//python:proto.bzl", "py_proto_library")
load("@xla//third_party/rules_python/python:py_binary.bzl", "py_binary")
load("@xla//third_party/rules_python/python:py_library.bzl", "py_library")
load("//tensorflow:tensorflow.bzl", "py_test", "tf_cc_binary", "tf_cc_test")
load("//tensorflow:tensorflow.default.bzl", "get_compatible_with_portable")
load(
"//tensorflow/core/platform:build_config.bzl",
"tf_proto_library",
)
load("//tensorflow/lite:build_def.bzl", "tflite_copts", "tflite_copts_warnings")
load("//tensorflow/lite:special_rules.bzl", "tflite_portable_test_suite")
load("//tensorflow/lite/core/shims:cc_library_with_tflite.bzl", "cc_library_with_tflite")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
default_visibility = [
"//visibility:public",
],
licenses = ["notice"],
)
exports_files([
"logging.h",
])
py_binary(
name = "visualize",
srcs = ["visualize.py"],
strict_deps = True,
deps = [
"//tensorflow/lite/python:schema_py",
"//third_party/py/numpy",
],
)
py_library(
name = "visualize_lib",
srcs = ["visualize.py"],
strict_deps = True,
deps = [
"//tensorflow/lite/python:schema_py",
"//third_party/py/numpy",
],
)
py_test(
name = "visualize_test",
srcs = ["visualize_test.py"],
strict_deps = True,
deps = [
":test_utils",
":visualize_lib",
#internal proto upb dep
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/platform:client_testlib",
],
)
py_binary(
name = "convert_image_to_csv",
srcs = ["convert_image_to_csv.py"],
strict_deps = True,
deps = [
"//tensorflow/python/client:session",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:ops",
"//tensorflow/python/ops:image_ops",
"//tensorflow/python/ops:io_ops",
"@absl_py//absl:app",
"@absl_py//absl/flags",
],
)
py_library(
name = "convert_image_to_csv_lib",
srcs = ["convert_image_to_csv.py"],
strict_deps = True,
deps = [
"//tensorflow/python/client:session",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:ops",
"//tensorflow/python/ops:image_ops",
"//tensorflow/python/ops:io_ops",
"@absl_py//absl:app",
"@absl_py//absl/flags",
],
)
py_test(
name = "convert_image_to_csv_test",
srcs = ["convert_image_to_csv_test.py"],
data = ["//tensorflow/core:image_testdata"],
strict_deps = True,
deps = [
":convert_image_to_csv_lib",
#internal proto upb dep
"//third_party/py/numpy",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/platform:resource_loader",
],
)
py_binary(
name = "strip_strings",
srcs = ["strip_strings.py"],
strict_deps = True,
deps = [
":flatbuffer_utils",
"@absl_py//absl:app",
"@absl_py//absl/flags",
],
)
py_binary(
name = "reverse_xxd_dump_from_cc",
srcs = ["reverse_xxd_dump_from_cc.py"],
strict_deps = True,
deps = [
":flatbuffer_utils",
"@absl_py//absl:app",
"@absl_py//absl/flags",
],
)
py_binary(
name = "randomize_weights",
srcs = ["randomize_weights.py"],
strict_deps = True,
deps = [
":flatbuffer_utils",
"@absl_py//absl:app",
"@absl_py//absl/flags",
],
)
py_library(
name = "flatbuffer_utils",
srcs = ["flatbuffer_utils.py"],
strict_deps = True,
deps = [
"//tensorflow/lite/python:schema_py",
"//tensorflow/lite/python:schema_util",
"//tensorflow/python/platform:gfile",
"@pypi//flatbuffers",
],
)
py_test(
name = "flatbuffer_utils_test",
srcs = ["flatbuffer_utils_test.py"],
strict_deps = True,
deps = [
":flatbuffer_utils",
":test_utils",
#internal proto upb dep
"//tensorflow/lite/python:schema_py",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/platform:client_testlib",
],
)
py_library(
name = "test_utils",
srcs = ["test_utils.py"],
strict_deps = True,
deps = [
"//tensorflow/lite/python:schema_py",
"@pypi//flatbuffers",
],
)
cc_binary(
name = "generate_op_registrations",
srcs = ["gen_op_registration_main.cc"],
deps = [
":command_line_flags",
":gen_op_registration",
"//tensorflow/lite:util",
"@com_google_absl//absl/strings",
],
)
cc_library(
name = "gen_op_registration",
srcs = ["gen_op_registration.cc"],
hdrs = ["gen_op_registration.h"],
deps = [
"//tensorflow/lite:framework",
"//tensorflow/lite:string",
"//tensorflow/lite/core:framework",
"//tensorflow/lite/schema:schema_utils",
"@com_googlesource_code_re2//:re2",
],
)
cc_test(
name = "gen_op_registration_test",
srcs = ["gen_op_registration_test.cc"],
data = [
"//tensorflow/lite:testdata/0_subgraphs.bin",
"//tensorflow/lite:testdata/2_subgraphs.bin",
"//tensorflow/lite:testdata/empty_model.bin",
"//tensorflow/lite:testdata/test_model.bin",
"//tensorflow/lite:testdata/test_model_broken.bin",
"//tensorflow/lite:testdata/test_model_versioned_ops.bin",
],
tags = [
"tflite_not_portable_android",
"tflite_not_portable_ios",
],
deps = [
":gen_op_registration",
"@com_google_googletest//:gtest_main",
],
)
cc_library_with_tflite(
name = "verifier",
hdrs = ["verifier.h"],
compatible_with = get_compatible_with_portable(),
copts = tflite_copts_warnings(),
tflite_deps = [
"//tensorflow/lite:framework_stable",
],
deps = [
":verifier_internal",
"//tensorflow/lite:schema_fbs_version",
"//tensorflow/lite:string_util",
"//tensorflow/lite:util",
"//tensorflow/lite/core/api:error_reporter",
"//tensorflow/lite/core/api:op_resolver",
"//tensorflow/lite/core/c:common",
"//tensorflow/lite/core/tools:verifier",
"//tensorflow/lite/schema:schema_fbs",
"//tensorflow/lite/schema:schema_utils",
"@com_google_absl//absl/container:flat_hash_set",
],
)
cc_library_with_tflite(
name = "verifier_internal",
hdrs = ["verifier_internal.h"],
compatible_with = get_compatible_with_portable(),
copts = tflite_copts_warnings(),
deps = ["//tensorflow/lite/core/tools:verifier_internal"],
)
cc_library(
name = "logging",
hdrs = ["logging.h"],
compatible_with = get_compatible_with_portable(),
copts = tflite_copts_warnings(),
)
cc_library(
name = "tool_params",
srcs = ["tool_params.cc"],
hdrs = ["tool_params.h"],
compatible_with = get_compatible_with_portable(),
copts = tflite_copts(),
deps = [":logging"],
)
cc_test(
name = "tool_params_test",
srcs = ["tool_params_test.cc"],
copts = tflite_copts(),
visibility = ["//visibility:private"],
deps = [
":tool_params",
"@com_google_googletest//:gtest_main",
],
)
cc_library(
name = "command_line_flags",
srcs = ["command_line_flags.cc"],
hdrs = ["command_line_flags.h"],
compatible_with = get_compatible_with_portable(),
copts = tflite_copts(),
deps = [
":logging",
"@com_google_absl//absl/strings",
],
)
cc_test(
name = "command_line_flags_test",
srcs = ["command_line_flags_test.cc"],
copts = tflite_copts(),
visibility = ["//visibility:private"],
deps = [
":command_line_flags",
":tool_params",
"@com_google_googletest//:gtest_main",
],
)
cc_library(
name = "list_flex_ops",
srcs = ["list_flex_ops.cc"],
hdrs = ["list_flex_ops.h"],
deps = [
"//tensorflow/core:framework",
"//tensorflow/core:lib",
"//tensorflow/core:protos_all_cc",
"//tensorflow/core:tensorflow",
"//tensorflow/lite:framework",
"//tensorflow/lite:util",
"//tensorflow/lite/core:framework",
"//tensorflow/lite/schema:schema_utils",
"@flatbuffers",
"@jsoncpp_git//:jsoncpp",
],
)
# This tool list flex ops and kernels inside a TFLite file.
# It is used to generate header file for selective registration.
tf_cc_binary(
name = "list_flex_ops_main",
srcs = ["list_flex_ops_main.cc"],
visibility = ["//visibility:public"],
deps = [
":command_line_flags",
":list_flex_ops",
"@com_google_absl//absl/strings",
],
)
cc_library(
name = "list_flex_ops_main_lib",
srcs = ["list_flex_ops_main.cc"],
visibility = ["//visibility:public"],
deps = [
":command_line_flags",
":list_flex_ops",
"@com_google_absl//absl/strings",
],
)
cc_library(
name = "list_flex_ops_no_kernel",
srcs = ["list_flex_ops_no_kernel.cc"],
hdrs = ["list_flex_ops.h"],
deps = [
"//tensorflow/lite:framework",
"//tensorflow/lite/core:framework",
"//tensorflow/lite/schema:schema_utils",
"@jsoncpp_git//:jsoncpp",
],
)
tf_cc_binary(
name = "list_flex_ops_no_kernel_main",
srcs = ["list_flex_ops_main.cc"],
visibility = ["//visibility:public"],
deps = [
":command_line_flags",
":list_flex_ops_no_kernel",
"@com_google_absl//absl/strings",
],
)
tf_cc_test(
name = "list_flex_ops_test",
srcs = ["list_flex_ops_test.cc"],
data = [
"//tensorflow/lite:testdata/0_subgraphs.bin",
"//tensorflow/lite:testdata/multi_add_flex.bin",
"//tensorflow/lite:testdata/softplus_flex.bin",
"//tensorflow/lite:testdata/test_model.bin",
"//tensorflow/lite:testdata/test_model_broken.bin",
],
tags = [
"tflite_not_portable_android",
"tflite_not_portable_ios",
],
deps = [
":list_flex_ops",
"//tensorflow/core:protos_all_cc",
"//tensorflow/core/platform:protobuf",
"//tensorflow/core/platform:resource_loader",
"//tensorflow/lite/kernels:test_util",
"@com_google_googletest//:gtest_main",
"@flatbuffers",
],
)
py_binary(
name = "zip_files",
srcs = ["zip_files.py"],
strict_deps = True,
visibility = ["//visibility:public"],
deps = [
"@absl_py//absl:app",
"@absl_py//absl/flags",
],
)
cc_library(
name = "utils",
srcs = ["utils.cc"],
hdrs = ["utils.h"],
deps = [
":logging",
"//tensorflow/lite/c:c_api_types",
"//tensorflow/lite/c:common",
"//tensorflow/lite/core/c:common",
"//tensorflow/lite/kernels:kernel_util",
"//tensorflow/lite/types:half",
"@com_google_absl//absl/types:span",
"@eigen_archive//:eigen3",
],
)
cc_test(
name = "utils_test",
srcs = ["utils_test.cc"],
copts = tflite_copts(),
deps = [
":utils",
"//tensorflow/lite/c:common",
"//tensorflow/lite/types:half",
"@com_google_absl//absl/types:span",
"@com_google_googletest//:gtest_main",
],
)
cc_library(
name = "model_loader",
srcs = ["model_loader.cc"],
hdrs = ["model_loader.h"],
deps = [
"//tensorflow/lite:minimal_logging",
"//tensorflow/lite/core:model_builder",
"@com_google_absl//absl/strings",
],
)
cc_test(
name = "model_loader_test",
srcs = ["model_loader_test.cc"],
data = ["@tflite_mobilenet_float//:mobilenet_v1_1.0_224.tflite"],
deps = [
":model_loader",
"//tensorflow/lite:model_builder",
"//tensorflow/lite/schema:schema_fbs",
"@com_google_absl//absl/strings:str_format",
"@com_google_googletest//:gtest_main",
"@flatbuffers//:runtime_cc",
],
)
tflite_portable_test_suite()
# copybara:uncomment_begin(google-only)
# tf_proto_library(
# name = "op_kernel_set_proto",
# srcs = ["op_kernel_set.proto"],
# )
#
# py_proto_library(
# name = "op_kernel_set_py_pb2",
# deps = [":op_kernel_set_proto"],
# )
# copybara:uncomment_end
tensorflow/lite/tools/benchmark/BUILD
+321
View File
@@ -0,0 +1,321 @@
load("@rules_cc//cc:cc_binary.bzl", "cc_binary")
load("@rules_cc//cc:cc_library.bzl", "cc_library")
load("@rules_cc//cc:cc_test.bzl", "cc_test")
load("//tensorflow:tensorflow.bzl", "tf_cc_binary")
load("//tensorflow/lite:build_def.bzl", "tflite_copts", "tflite_copts_warnings", "tflite_linkopts")
load("//tensorflow/lite:special_rules.bzl", "tflite_portable_test_suite")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
default_visibility = [
"//visibility:public",
],
licenses = ["notice"],
)
common_copts = tflite_copts() + tflite_copts_warnings()
# We create a library for benchmark_main.cc to faciliate the creation of a
# customized benchmark model binary that only needs linking with extra
# dependency, e.g., enabling creating of benchmark binaries with a custom
# delegate provider.
cc_library(
name = "benchmark_model_main",
srcs = [
"benchmark_main.cc",
],
copts = common_copts,
deps = [
":benchmark_tflite_model_lib",
"//tensorflow/lite/tools:logging",
],
)
cc_binary(
name = "benchmark_model",
copts = common_copts,
linkopts = tflite_linkopts() + select({
"//tensorflow:android": [
"-pie", # Android 5.0 and later supports only PIE
"-lm", # some builtin ops, e.g., tanh, need -lm
"-Wl,--rpath=/data/local/tmp/", # Hexagon delegate libraries should be in /data/local/tmp
],
"//conditions:default": [],
}),
tags = ["builder_default_android_arm64"],
deps = [
":benchmark_model_main",
],
)
cc_binary(
name = "benchmark_model_performance_options",
srcs = [
"benchmark_tflite_performance_options_main.cc",
],
copts = common_copts,
linkopts = tflite_linkopts() + select({
"//tensorflow:android": [
"-pie", # Android 5.0 and later supports only PIE
"-lm", # some builtin ops, e.g., tanh, need -lm
"-Wl,--rpath=/data/local/tmp/", # Hexagon delegate libraries should be in /data/local/tmp
],
"//conditions:default": [],
}),
tags = ["builder_default_android_arm64"],
deps = [
":benchmark_performance_options",
":benchmark_tflite_model_lib",
"//tensorflow/lite/tools:logging",
],
)
# As with most target binaries that use flex, this should be built with the
# `--config=monolithic` build flag, e.g.,
# bazel build --config=monolithic --config=android_arm64 \
# -c opt --cxxopt='--std=c++17' :benchmark_model_plus_flex
tf_cc_binary(
name = "benchmark_model_plus_flex",
srcs = [
"benchmark_plus_flex_main.cc",
],
copts = common_copts,
linkopts = tflite_linkopts() + select({
"//tensorflow:android": [
"-pie", # Android 5.0 and later supports only PIE
"-lm", # some builtin ops, e.g., tanh, need -lm
],
"//conditions:default": [],
}),
deps = [
":benchmark_tflite_model_lib",
"//tensorflow/lite/delegates/flex:delegate",
"//tensorflow/lite/testing:init_tensorflow",
"//tensorflow/lite/tools:logging",
],
)
cc_test(
name = "benchmark_test",
srcs = ["benchmark_test.cc"],
args = [
"--fp32_graph=$(location //tensorflow/lite:testdata/multi_add.bin)",
"--int8_graph=$(location //tensorflow/lite:testdata/add_quantized_int8.bin)",
"--string_graph_with_signature=$(location //tensorflow/lite:testdata/string_input_model_with_signature.bin)",
"--string_graph_without_signature=$(location //tensorflow/lite:testdata/string_input_model.bin)",
"--multi_signature_graph=$(location //tensorflow/lite:testdata/multi_signatures.bin)",
],
data = [
"//tensorflow/lite:testdata/add_quantized_int8.bin",
"//tensorflow/lite:testdata/multi_add.bin",
"//tensorflow/lite:testdata/multi_signatures.bin",
"//tensorflow/lite:testdata/string_input_model.bin",
"//tensorflow/lite:testdata/string_input_model_with_signature.bin",
],
tags = [
"tflite_not_portable_android",
"tflite_not_portable_ios",
],
deps = [
":benchmark_performance_options",
":benchmark_tflite_model_lib",
"//tensorflow/lite:framework",
"//tensorflow/lite:string_util",
"//tensorflow/lite/core/c:c_api_types",
"//tensorflow/lite/core/c:common",
"//tensorflow/lite/testing:util",
"//tensorflow/lite/tools:command_line_flags",
"//tensorflow/lite/tools:logging",
"//tensorflow/lite/tools/benchmark/proto:benchmark_result_cc",
"//tensorflow/lite/tools/delegates:delegate_provider_hdr",
"@com_google_absl//absl/algorithm",
"@com_google_absl//absl/log",
"@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/strings:string_view",
"@com_google_googletest//:gtest",
],
)
cc_library(
name = "profiling_listener",
srcs = ["profiling_listener.cc"],
hdrs = ["profiling_listener.h"],
copts = common_copts,
deps = [
":benchmark_model_lib",
":benchmark_params",
"//tensorflow/lite:framework_stable",
"//tensorflow/lite/profiling:profile_summarizer",
"//tensorflow/lite/profiling:profile_summary_formatter",
"//tensorflow/lite/profiling:profiler",
"//tensorflow/lite/tools:logging",
],
)
cc_library(
name = "benchmark_tflite_model_lib",
srcs = ["benchmark_tflite_model.cc"],
hdrs = ["benchmark_tflite_model.h"],
copts = common_copts + select({
"//tensorflow:ios": [
"-xobjective-c++",
],
"//conditions:default": [],
}),
deps = [
":benchmark_model_lib",
":benchmark_params",
":benchmark_utils",
":profiling_listener",
"//tensorflow/core/example:example_protos_cc_impl",
"//tensorflow/lite:framework",
"//tensorflow/lite:simple_memory_arena_debug_dump",
"//tensorflow/lite:string_util",
"//tensorflow/lite/core:cc_api_stable",
"//tensorflow/lite/core:framework",
"//tensorflow/lite/core/c:c_api_types",
"//tensorflow/lite/core/c:common",
"//tensorflow/lite/core/kernels:builtin_ops",
"//tensorflow/lite/kernels:cpu_backend_context",
"//tensorflow/lite/profiling:model_runtime_info",
"//tensorflow/lite/profiling:profile_summary_formatter",
"//tensorflow/lite/profiling:profiler",
"//tensorflow/lite/tools:logging",
"//tensorflow/lite/tools:model_loader",
"//tensorflow/lite/tools:utils",
"//tensorflow/lite/tools/benchmark/proto:benchmark_result_cc",
"//tensorflow/lite/tools/delegates:delegate_provider_hdr",
"//tensorflow/lite/tools/delegates:tflite_execution_providers",
"@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:string_view",
"@com_google_absl//absl/types:span",
"@ruy//ruy/profiler",
],
)
cc_test(
name = "benchmark_tflite_model_lib_test",
srcs = [
"benchmark_tflite_model_test.cc",
],
data = ["@tflite_mobilenet_float//:mobilenet_v1_1.0_224.tflite"],
tags = [
"no_oss", # TODO: b/361565588 - Re-enable.
],
deps = [
":benchmark_model_lib",
":benchmark_params",
":benchmark_tflite_model_lib",
"//tensorflow/lite/core/c:c_api_types",
"//tensorflow/lite/tools:tool_params",
"@com_google_absl//absl/strings",
"@com_google_googletest//:gtest_main",
],
)
cc_library(
name = "benchmark_multirun_stats_recorder",
hdrs = ["benchmark_multirun_stats_recorder.h"],
copts = common_copts,
deps = [":benchmark_model_lib"],
)
cc_library(
name = "benchmark_performance_options",
srcs = [
"benchmark_performance_options.cc",
],
hdrs = ["benchmark_performance_options.h"],
copts = common_copts,
deps = [
":benchmark_model_lib",
":benchmark_multirun_stats_recorder",
":benchmark_params",
":benchmark_utils",
"//tensorflow/core/util:stats_calculator_portable",
"//tensorflow/lite/core/c:c_api_types",
"//tensorflow/lite/core/c:common",
"//tensorflow/lite/delegates/hexagon:hexagon_delegate",
"//tensorflow/lite/nnapi:nnapi_util",
"//tensorflow/lite/profiling:time",
"//tensorflow/lite/tools:command_line_flags",
"//tensorflow/lite/tools:logging",
"@com_google_absl//absl/memory",
] + select({
"//tensorflow:android": [
"//tensorflow/lite/delegates/gpu:delegate",
],
"//conditions:default": [],
}),
)
cc_library(
name = "benchmark_params",
hdrs = ["benchmark_params.h"],
copts = common_copts,
deps = ["//tensorflow/lite/tools:tool_params"],
)
cc_library(
name = "benchmark_model_lib",
srcs = [
"benchmark_model.cc",
],
hdrs = ["benchmark_model.h"],
copts = common_copts,
deps = [
":benchmark_params",
":benchmark_utils",
"//tensorflow/core/util:stats_calculator_portable",
"//tensorflow/lite:framework",
"//tensorflow/lite/core/c:common",
"//tensorflow/lite/profiling:memory_info",
"//tensorflow/lite/profiling:memory_usage_monitor",
"//tensorflow/lite/profiling:time",
"//tensorflow/lite/tools:command_line_flags",
"//tensorflow/lite/tools:logging",
],
)
cc_library(
name = "register_custom_op",
srcs = [
"register_custom_op.cc",
],
hdrs = [
"register_custom_op.h",
],
copts = common_copts,
deps = [
"//tensorflow/lite:op_resolver",
"@com_google_absl//absl/base:core_headers",
],
alwayslink = 1,
)
exports_files(["register_custom_op.h"])
cc_library(
name = "benchmark_utils",
srcs = [
"benchmark_utils.cc",
],
hdrs = ["benchmark_utils.h"],
copts = common_copts,
deps = ["//tensorflow/lite/profiling:time"],
)
cc_test(
name = "benchmark_utils_test",
srcs = [
"benchmark_utils_test.cc",
],
deps = [
":benchmark_utils",
"//tensorflow/lite/profiling:time",
"@com_google_googletest//:gtest_main",
],
)
tflite_portable_test_suite()
tensorflow/lite/tools/benchmark/CMakeLists.txt
+119
View File
@@ -0,0 +1,119 @@
#
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# The benchmark tool for Tensorflow Lite.
populate_source_vars("${TFLITE_SOURCE_DIR}/tools/benchmark"
TFLITE_BENCHMARK_SRCS
FILTER "(_test|_plus_flex_main|_performance_options.*)\\.cc$"
)
list(APPEND TFLITE_BENCHMARK_SRCS
${XLA_SOURCE_DIR}/xla/tsl/util/stats_calculator.cc
${TFLITE_SOURCE_DIR}/kernels/internal/utils/sparsity_format_converter.cc
${TFLITE_SOURCE_DIR}/profiling/memory_info.cc
${TFLITE_SOURCE_DIR}/profiling/memory_usage_monitor.cc
${TFLITE_SOURCE_DIR}/profiling/model_runtime_info.cc
${TFLITE_SOURCE_DIR}/profiling/profile_buffer.cc
${TFLITE_SOURCE_DIR}/profiling/profile_summarizer.cc
${TFLITE_SOURCE_DIR}/profiling/profile_summary_formatter.cc
${TFLITE_SOURCE_DIR}/profiling/root_profiler.cc
${TFLITE_SOURCE_DIR}/profiling/telemetry/profiler.cc
${TFLITE_SOURCE_DIR}/profiling/telemetry/telemetry.cc
${TFLITE_SOURCE_DIR}/profiling/time.cc
${TFLITE_SOURCE_DIR}/tools/command_line_flags.cc
${TFLITE_SOURCE_DIR}/tools/delegates/default_execution_provider.cc
${TFLITE_SOURCE_DIR}/tools/delegates/delegate_provider.cc
${TFLITE_SOURCE_DIR}/tools/evaluation/utils.cc
${TFLITE_SOURCE_DIR}/tools/model_loader.cc
${TFLITE_SOURCE_DIR}/tools/tool_params.cc
${TFLITE_SOURCE_DIR}/tools/utils.cc
)
list(APPEND TFLITE_BENCHMARK_LIBS
tensorflow-lite
)
list(APPEND TFLITE_BENCHMARK_LIBS
profiling_info_proto
feature_proto
example_proto
model_runtime_info_proto
benchmark_result_proto
protobuf::libprotobuf
absl::log
)
# TODO(b/171007016): Enable performance options on Windows.
if(NOT "${CMAKE_SYSTEM_NAME}" STREQUAL "Windows")
list(APPEND TFLITE_BENCHMARK_SRCS
${TFLITE_SOURCE_DIR}/tools/benchmark/benchmark_performance_options.cc
)
endif()
if(TFLITE_ENABLE_XNNPACK)
list(APPEND TFLITE_BENCHMARK_SRCS
${TFLITE_SOURCE_DIR}/tools/delegates/xnnpack_delegate_provider.cc
${TFLITE_SOURCE_DIR}/core/acceleration/configuration/c/xnnpack_plugin.cc)
else()
set(TFLITE_BENCHMARK_CC_OPTIONS "-DTFLITE_WITHOUT_XNNPACK")
endif() # TFLITE_ENABLE_XNNPACK
if(TFLITE_ENABLE_EXTERNAL_DELEGATE)
list(APPEND TFLITE_BENCHMARK_SRCS
${TFLITE_SOURCE_DIR}/tools/delegates/external_delegate_provider.cc
)
endif() # TFLITE_ENABLE_EXTERNAL_DELEGATE
if(CMAKE_SYSTEM_NAME MATCHES "Android")
if(_TFLITE_ENABLE_NNAPI)
list(APPEND TFLITE_BENCHMARK_SRCS
${TFLITE_SOURCE_DIR}/tools/delegates/nnapi_delegate_provider.cc
)
endif() # _TFLITE_ENABLE_NNAPI
list(APPEND TFLITE_BENCHMARK_LIBS
${ANDROID_LOG_LIB}
absl::strings
)
endif() # Android
if(TFLITE_ENABLE_GPU)
list(APPEND TFLITE_BENCHMARK_SRCS
${TFLITE_SOURCE_DIR}/tools/delegates/gpu_delegate_provider.cc
)
endif() # TFLITE_ENABLE_GPU
add_executable(benchmark_model
${TFLITE_BENCHMARK_SRCS}
)
if(TFLITE_ENABLE_BENCHMARK_MODEL)
set_target_properties(benchmark_model PROPERTIES EXCLUDE_FROM_ALL FALSE)
if(TFLITE_ENABLE_INSTALL)
install(TARGETS benchmark_model)
endif() # TFLITE_ENABLE_INSTALL
else()
set_target_properties(benchmark_model PROPERTIES EXCLUDE_FROM_ALL TRUE)
endif() # TFLITE_ENABLE_BENCHMARK_MODEL
target_compile_options(benchmark_model
PRIVATE
${TFLITE_BENCHMARK_CC_OPTIONS}
)
target_include_directories(benchmark_model
PUBLIC
${CMAKE_BINARY_DIR}
)
target_link_libraries(benchmark_model
${TFLITE_BENCHMARK_LIBS}
)
tensorflow/lite/tools/benchmark/README.md
+554
View File
@@ -0,0 +1,554 @@
# TFLite Model Benchmark Tool with C++ Binary
## Description
A simple C++ binary to benchmark a TFLite model and its individual operators,
both on desktop machines and on Android. The binary takes a TFLite model,
generates random inputs and then repeatedly runs the model for specified number
of runs. Aggregate latency statistics are reported after running the benchmark.
The instructions below are for running the binary on Desktop and Android,
for iOS please use the
[iOS benchmark app](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark/ios).
An experimental Android APK wrapper for the benchmark model utility offers more
faithful execution behavior on Android (via a foreground Activity). It is
located
[here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark/android).
## Parameters
The binary takes the following required parameters:
* `graph`: `string` \
The path to the TFLite model file.
and the following optional parameters:
* `signature_to_run_for`: `string` (default="") \
If the model contains multiple signatures, use this flag to specify the
signature to benchmark.
- If multiple signatures are present and this flag is not specified, the
benchmark will throw an error.
- If only one signature is present and this flag is not specified, the
default signature will be used.
* `num_threads`: `int` (default=-1) \
The number of threads to use for running TFLite interpreter. By default,
this is set to the platform default value -1.
* `warmup_runs`: `int` (default=1) \
The number of warmup runs to do before starting the benchmark.
* `num_runs`: `int` (default=50) \
The number of runs. Increase this to reduce variance.
* `max_secs` : float (default=150.0) \
The maximum number of seconds the benchmark will run before being
terminated.
* `run_delay`: `float` (default=-1.0) \
The delay in seconds between subsequent benchmark runs. Non-positive values
mean use no delay.
* `run_frequency`: `float` (default=-1.0) \
The frequency of running a benchmark run as the number of prorated runs per
second. If the targeted rate per second cannot be reached, the benchmark
would start the next run immediately, trying its best to catch up. If set,
this will override the `run_delay` parameter. A non-positive value means
there is no delay between subsequent runs.
* `enable_op_profiling`: `bool` (default=false) \
Whether to enable per-operator profiling measurement.
* `max_profiling_buffer_entries`: `int` (default=1024) \
The initial max number of profiling events that will be stored during each
inference run. It is only meaningful when `enable_op_profiling` is set to
`true`. Note, the actual value of this parameter will be adjusted if the
model has more nodes than the specified value of this parameter. Also, when
`allow_dynamic_profiling_buffer_increase` is set to `true`, the number of
profiling buffer entries will be increased dynamically.
* `allow_dynamic_profiling_buffer_increase`: `bool` (default=false) \
Whether allowing dynamic increase on the number of profiling buffer entries.
It is only meaningful when `enable_op_profiling` is set to `true`. Note,
allowing dynamic buffer size increase may cause more profiling overhead,
thus it is preferred to set `max_profiling_buffer_entries` to a large-enough
value.
* `op_profiling_output_mode`: `str` (default="stdout") \
The output mode for the profiling information generated. Requires
`enable_op_profiling` to be `true`. Takes one of the following 3 values:
- `stdout` : Print profiling information to STDOUT.
- `csv` : Print the profiling information in a CSV format.
- `proto` : Print the profiling information in a proto format as specified
in `tensorflow/lite/profiling/proto/profiling_info.proto`.
* `op_profiling_output_file`: `str` (default="") \
File path to export profile data to. The results are printed to
`stdout` if option is not set. Requires `enable_op_profiling` to be `true`
and the path to include the name of the output file; otherwise results are
printed to `stdout`.
* `export_model_runtime_info`: `bool` (default="false") \
Exports the model runtime information in a proto format as specified
in `tensorflow/lite/profiling/proto/model_runtime_info.proto`.
* `model_runtime_info_output_file`: `str` (default="") \
File path to export model runtime data to. The results are printed to
`stdout` if option is not set. Requires `export_model_runtime_info` to be
`true` and the path to include the name of the output file; otherwise
results are printed to `stdout`.
* `profiling_output_csv_file`: `str` (default="") \
WARNING: Deprecated, prefer using `op_profiling_output_mode` and
`op_profiling_output_file` instead.
File path to export profile data to as CSV. The results are printed to
`stdout` if option is not set. Requires `enable_op_profiling` to be `true`
and the path to include the name of the output CSV; otherwise results are
printed to `stdout`.
* `output_filepath`: `str` (default="") \
File path to save output tensor data to. If specified, the output tensor
values are saved as binary data in the file.
* `output_proto_filepath`: `str` (default="") \
File path to save output tensor data as tensorflow example proto. If
specified, the output tensor values are saved in tensorflow example and then
serialized to the file.
* `print_preinvoke_state`: `bool` (default=false) \
Whether to print out the TfLite interpreter internals just before calling
tflite::Interpreter::Invoke. The internals will include allocated memory
size of each tensor etc. Enabling this could help understand TfLite graph
and memory usage.
* `print_postinvoke_state`: `bool` (default=false) \
Whether to print out the TfLite interpreter internals just before benchmark
completes (i.e. after all repeated Invoke calls complete). The internals
will include allocated memory size of each tensor etc. Enabling this could
help understand TfLite graph and memory usage, particularly when there are
dynamic-shaped tensors in the graph.
* `report_peak_memory_footprint`: `bool` (default=false) \
Whether to report the peak memory footprint by periodically checking the
memory footprint. Internally, a separate thread will be spawned for this
periodic check. Therefore, the performance benchmark result could be
affected.
* `memory_footprint_check_interval_ms`: `int` (default=50) \
The interval in millisecond between two consecutive memory footprint checks.
This is only used when --report_peak_memory_footprint is set to true.
* `dry_run`: `bool` (default=false) \
Whether to run the tool just with simply loading the model, allocating
tensors etc. but without actually invoking any op kernels.
* `verbose`: `bool` (default=false) \
Whether to log parameters whose values are not set. By default, only log
those parameters that are set by parsing their values from the commandline
flags.
* `release_dynamic_tensors`: `bool` (default=false) \
Whether to configure the Interpreter to immediately release the memory of
dynamic tensors in the graph once they are not used.
* `optimize_memory_for_large_tensors`: `int` (default=0) \
Whether to optimize memory usage for large tensors with sacrificing latency.
When the feature is enabled, `release_dynamic_tensors` is also enabled.
* `enable_builtin_cast_constant_cache`: `bool` (default=false) \
Configure the builtin TFLite CAST operation to cache its output if its input
is a constant tensor.
WARNING: This is an experimental option that may be removed at any time.
This list of parameters is not exhaustive. See
[here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/tools/benchmark/benchmark_model.cc)
and
[here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc)
for all parameters that the binary takes.
### Model input parameters
By default, the tool will use randomized data for model inputs. The following
parameters allow users to specify customized input values to the model when
running the benchmark tool:
* `input_layer`: `string` \
A comma-separated list of input layer names, e.g. 'input1,input2'. Note all
inputs of the model graph need to be specified. However, the input name
does not need to match that encoded in the model. Additionally, the order
of input layer names specified here is assumed to be same with that is seen
by the Tensorflow Lite interpreter. This is a bit inconvenient but the
[visualization tool](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/tools/visualize.py)
should help to find this order.
* `input_layer_shape`: `string` \
A colon-separated list of input layer shapes, where each shape is a
comma-separated list, e.g. '1,30:1,10'. Similar to `input_layer`, this
parameter also requires shapes of all inputs be specified, and the order of
inputs be same with that is seen by the interpreter.
* `input_layer_value_range`: `string` \
A map-like string representing value range for *integer* input layers. Each
item is separated by ':', and the item value consists of input layer name
and integer-only range values (both low and high are inclusive) separated by
',', e.g. 'input1,1,2:input2,0,254'. Note that the input layer name must
exist in the list of names specified by `input_layer`.
* `input_layer_value_files`: `string` \
A map-like string representing files that contain input values. Each
item is separated by ',', and the item value consists of input layer name
and the file path separated by ':',
e.g. 'input1:file_path1,input2:file_path2'. In case the input layer name
contains ':' e.g. "input:0", escape it with "::" literal,
e.g. `input::0:file_path1`. If a input name appears in both
`input_layer_value_range` and `input_layer_value_files`, the corresponding
input value range specified by`input_layer_value_range` will be ignored.
The file format is binary, and the content should be either a byte array or
null-separated strings. Note that the input layer name must also exist in
the list of names specified by `input_layer`.
### TFLite delegate parameters
The tool supports all runtime/delegate parameters introduced by
[the delegate registrar](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/delegates).
The following simply lists the names of these parameters and additional notes
where applicable. For details about each parameter, please refer to
[this page](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/tools/delegates/README.md#tflite-delegate-registrar).
#### Common parameters
* `max_delegated_partitions`: `int` (default=0)
* `min_nodes_per_partition`:`int` (default=0)
* `delegate_serialize_dir`: `str` (default="")
* `delegate_serialize_token`: `str` (default="")
#### GPU delegate
* `use_gpu`: `bool` (default=false)
* `gpu_precision_loss_allowed`: `bool` (default=true)
* `gpu_experimental_enable_quant`: `bool` (default=true)
* `gpu_inference_for_sustained_speed`: `bool` (default=false)
* `gpu_backend`: `string` (default="")
* `gpu_wait_type`: `str` (default="")
#### NNAPI delegate
* `use_nnapi`: `bool` (default=false) \
Note some Android P devices will fail to use NNAPI for models in
`/data/local/tmp/` and this benchmark tool will not correctly use NNAPI.
* `nnapi_execution_preference`: `str` (default="") \
Should be one of: `fast_single_answer`, `sustained_speed`, `low_power`,
`undefined`.
* `nnapi_execution_priority`: `str` (default="") \
Note this requires Android 11+.
* `nnapi_accelerator_name`: `str` (default="") \
Note this requires Android 10+.
* `disable_nnapi_cpu`: `bool` (default=true)
* `nnapi_allow_fp16`: `bool` (default=false)
* `nnapi_allow_dynamic_dimensions`:`bool` (default=false)
* `nnapi_use_burst_mode`:`bool` (default=false)
#### Hexagon delegate
* `use_hexagon`: `bool` (default=false)
* `hexagon_profiling`: `bool` (default=false) \
Note enabling this option will not produce profiling results outputs unless
`enable_op_profiling` is also turned on. When both parameters are set to true,
the profile of ops on hexagon DSP will be added to the profile table. Note that,
the reported data on hexagon is in cycles, not in ms like on cpu.
* `hexagon_lib_path`: `string` (default="/data/local/tmp/") \
The library path for the underlying Hexagon libraries.
This is where libhexagon_nn_skel*.so files should be.
For libhexagon_interface.so it needs to be on a path that can be loaded from
example: put it in LD_LIBRARY_PATH.
#### XNNPACK delegate
* `use_xnnpack`: `bool` (default=false) \
Note if this option is explicitly set to `false`, the TfLite runtime will
use its original CPU kernels for model execution. In other words, after
enabling the feature that the XNNPACK delegate is applied by default in
TfLite runtime, explicitly setting this flag to `false` will cause the
benchmark tool to disable the feature at runtime, and to use the original
non-delegated CPU execution path for model benchmarking.
* `xnnpack_force_fp16`: `bool` (default=false) \
Enforce float16 inference.
#### CoreML delegate
* `use_coreml`: `bool` (default=false)
* `coreml_version`: `int` (default=0)
#### External delegate
* `external_delegate_path`: `string` (default="")
* `external_delegate_options`: `string` (default="")
#### Stable delegate [Experimental]
* `stable_delegate_loader_settings`: `string` (default="") A path to the
JSON-encoded delegate [`TFLiteSettings`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/acceleration/configuration/configuration.proto#L488) file, which is defined in `configuration.proto`.
As some delegates are only available on certain platforms, when running the
benchmark tool on a particular platform, specifying `--help` will print out all
supported parameters.
### Use multiple delegates
When multiple delegates are specified to be used in the commandline flags, the
order of delegates applied to the TfLite runtime will be same as their enabling
commandline flag is specified. For example, "--use_xnnpack=true --use_gpu=true"
means applying the XNNPACK delegate first, and then the GPU delegate secondly.
In comparison, "--use_gpu=true --use_xnnpack=true" means applying the GPU
delegate first, and then the XNNPACK delegate secondly.
## To build/install/run
Note: The benchmarking tool must be compiled with a TFLite runtime that
supports the ops found in the model to be tested.<br/>
If Tensorflow Ops ("flex ops")
or other custom ops are used in the model, please see the section [below](#build-the-benchmark-tool-with-tensorflow-ops-support).
### On Android:
(0) Refer to https://www.tensorflow.org/lite/guide/build_android to edit the
`WORKSPACE` to configure the android NDK/SDK.
(1) Build for your specific platform, e.g.:
```
bazel build -c opt \
--config=android_arm64 \
tensorflow/lite/tools/benchmark:benchmark_model
```
(2) Connect your phone. Push the binary to your phone with adb push
(make the directory if required):
```
adb push bazel-bin/tensorflow/lite/tools/benchmark/benchmark_model /data/local/tmp
```
(3) Make the binary executable.
```
adb shell chmod +x /data/local/tmp/benchmark_model
```
(4) Push the compute graph that you need to test. For example:
```
adb push mobilenet_quant_v1_224.tflite /data/local/tmp
```
(5) Optionally, install Hexagon libraries on device.
That step is only needed when using the Hexagon delegate.
```
bazel build --config=android_arm64 \
tensorflow/lite/delegates/hexagon/hexagon_nn:libhexagon_interface.so
adb push bazel-bin/tensorflow/lite/delegates/hexagon/hexagon_nn/libhexagon_interface.so /data/local/tmp
adb push libhexagon_nn_skel*.so /data/local/tmp
```
(6) Run the benchmark. For example:
```
adb shell /data/local/tmp/benchmark_model \
--graph=/data/local/tmp/mobilenet_quant_v1_224.tflite \
--num_threads=4
```
### On desktop:
(1) build the binary
```
bazel build -c opt tensorflow/lite/tools/benchmark:benchmark_model
```
(2) Run on your compute graph, similar to the Android case but without the need
of adb shell. For example:
```
bazel-bin/tensorflow/lite/tools/benchmark/benchmark_model \
--graph=mobilenet_quant_v1_224.tflite \
--num_threads=4
```
The MobileNet graph used as an example here may be downloaded from [here](https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip).
## Reducing variance between runs on Android.
Most modern Android phones use [ARM big.LITTLE](https://en.wikipedia.org/wiki/ARM_big.LITTLE)
architecture where some cores are more power hungry but faster than other cores.
When running benchmarks on these phones there can be significant variance
between different runs of the benchmark. One way to reduce variance between runs
is to set the [CPU affinity](https://en.wikipedia.org/wiki/Processor_affinity)
before running the benchmark. On Android this can be done using the `taskset`
command.
E.g. for running the benchmark on big cores on Pixel 2 with a single thread one
can use the following command:
```
adb shell taskset f0 /data/local/tmp/benchmark_model \
--graph=/data/local/tmp/mobilenet_quant_v1_224.tflite \
--num_threads=1
```
where `f0` is the affinity mask for big cores on Pixel 2.
Note: The affinity mask varies with the device.
## Profiling model operators
The benchmark model binary also allows you to profile operators and give
execution times of each operator. To do this, pass the flag
`--enable_op_profiling=true` to `benchmark_model` during invocation, e.g.,
```
adb shell taskset f0 /data/local/tmp/benchmark_model \
--graph=/data/local/tmp/mobilenet_quant_v1_224.tflite \
--enable_op_profiling=true
```
When enabled, the `benchmark_model` binary will produce detailed statistics for
each operation similar to those shown below:
```
============================== Run Order ==============================
[node type] [start] [first] [avg ms] [%] [cdf%] [mem KB] [times called] [Name]
CONV_2D 0.000 4.269 4.269 0.107% 0.107% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_0/Relu6]
DEPTHWISE_CONV_2D 4.270 2.150 2.150 0.054% 0.161% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_1_depthwise/Relu6]
CONV_2D 6.421 6.107 6.107 0.153% 0.314% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_1_pointwise/Relu6]
DEPTHWISE_CONV_2D 12.528 1.366 1.366 0.034% 0.348% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_2_depthwise/Relu6]
CONV_2D 13.895 4.195 4.195 0.105% 0.454% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_2_pointwise/Relu6]
DEPTHWISE_CONV_2D 18.091 1.260 1.260 0.032% 0.485% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_3_depthwise/Relu6]
CONV_2D 19.352 6.652 6.652 0.167% 0.652% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_3_pointwise/Relu6]
DEPTHWISE_CONV_2D 26.005 0.698 0.698 0.018% 0.670% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_4_depthwise/Relu6]
CONV_2D 26.703 3.344 3.344 0.084% 0.754% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_4_pointwise/Relu6]
DEPTHWISE_CONV_2D 30.047 0.646 0.646 0.016% 0.770% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_5_depthwise/Relu6]
CONV_2D 30.694 5.800 5.800 0.145% 0.915% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_5_pointwise/Relu6]
DEPTHWISE_CONV_2D 36.495 0.331 0.331 0.008% 0.924% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_6_depthwise/Relu6]
CONV_2D 36.826 2.838 2.838 0.071% 0.995% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_6_pointwise/Relu6]
DEPTHWISE_CONV_2D 39.665 0.439 0.439 0.011% 1.006% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_7_depthwise/Relu6]
CONV_2D 40.105 5.293 5.293 0.133% 1.139% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_7_pointwise/Relu6]
DEPTHWISE_CONV_2D 45.399 0.352 0.352 0.009% 1.147% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_8_depthwise/Relu6]
CONV_2D 45.752 5.322 5.322 0.133% 1.281% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_8_pointwise/Relu6]
DEPTHWISE_CONV_2D 51.075 0.357 0.357 0.009% 1.290% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_9_depthwise/Relu6]
CONV_2D 51.432 5.693 5.693 0.143% 1.433% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_9_pointwise/Relu6]
DEPTHWISE_CONV_2D 57.126 0.366 0.366 0.009% 1.442% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_10_depthwise/Relu6]
CONV_2D 57.493 5.472 5.472 0.137% 1.579% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_10_pointwise/Relu6]
DEPTHWISE_CONV_2D 62.966 0.364 0.364 0.009% 1.588% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_11_depthwise/Relu6]
CONV_2D 63.330 5.404 5.404 0.136% 1.724% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_11_pointwise/Relu6]
DEPTHWISE_CONV_2D 68.735 0.155 0.155 0.004% 1.728% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_12_depthwise/Relu6]
CONV_2D 68.891 2.970 2.970 0.074% 1.802% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_12_pointwise/Relu6]
DEPTHWISE_CONV_2D 71.862 0.206 0.206 0.005% 1.807% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_13_depthwise/Relu6]
CONV_2D 72.069 5.888 5.888 0.148% 1.955% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_13_pointwise/Relu6]
AVERAGE_POOL_2D 77.958 0.036 0.036 0.001% 1.956% 0.000 0 [MobilenetV1/Logits/AvgPool_1a/AvgPool]
CONV_2D 77.994 1.445 1.445 0.036% 1.992% 0.000 0 [MobilenetV1/Logits/Conv2d_1c_1x1/BiasAdd]
RESHAPE 79.440 0.002 0.002 0.000% 1.992% 0.000 0 [MobilenetV1/Predictions/Reshape]
SOFTMAX 79.443 0.029 0.029 0.001% 1.993% 0.000 0 [MobilenetV1/Predictions/Softmax]
============================== Top by Computation Time ==============================
[node type] [start] [first] [avg ms] [%] [cdf%] [mem KB] [times called] [Name]
CONV_2D 19.352 6.652 6.652 0.167% 0.167% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_3_pointwise/Relu6]
CONV_2D 6.421 6.107 6.107 0.153% 0.320% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_1_pointwise/Relu6]
CONV_2D 72.069 5.888 5.888 0.148% 0.468% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_13_pointwise/Relu6]
CONV_2D 30.694 5.800 5.800 0.145% 0.613% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_5_pointwise/Relu6]
CONV_2D 51.432 5.693 5.693 0.143% 0.756% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_9_pointwise/Relu6]
CONV_2D 57.493 5.472 5.472 0.137% 0.893% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_10_pointwise/Relu6]
CONV_2D 63.330 5.404 5.404 0.136% 1.029% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_11_pointwise/Relu6]
CONV_2D 45.752 5.322 5.322 0.133% 1.162% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_8_pointwise/Relu6]
CONV_2D 40.105 5.293 5.293 0.133% 1.295% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_7_pointwise/Relu6]
CONV_2D 0.000 4.269 4.269 0.107% 1.402% 0.000 0 [MobilenetV1/MobilenetV1/Conv2d_0/Relu6]
Number of nodes executed: 31
============================== Summary by node type ==============================
[Node type] [count] [avg ms] [avg %] [cdf %] [mem KB] [times called]
CONV_2D 15 1.406 89.270% 89.270% 0.000 0
DEPTHWISE_CONV_2D 13 0.169 10.730% 100.000% 0.000 0
SOFTMAX 1 0.000 0.000% 100.000% 0.000 0
RESHAPE 1 0.000 0.000% 100.000% 0.000 0
AVERAGE_POOL_2D 1 0.000 0.000% 100.000% 0.000 0
Timings (microseconds): count=50 first=79449 curr=81350 min=77385 max=88213 avg=79732 std=1929
Memory (bytes): count=0
31 nodes observed
Average inference timings in us: Warmup: 83235, Init: 38467, Inference: 79760.9
```
## Benchmark multiple performance options in a single run
A convenient and simple C++ binary is also provided to benchmark multiple
performance options in a single run. This binary is built based on the
aforementioned benchmark tool that could only benchmark a single performance
option at a time. They share the same build/install/run process, but the BUILD
target name of this binary is `benchmark_model_performance_options` and it takes
some additional parameters as detailed below.
### Additional Parameters
* `perf_options_list`: `string` (default='all') \
A comma-separated list of TFLite performance options to benchmark.
* `option_benchmark_run_delay`: `float` (default=-1.0) \
The delay between two consecutive runs of benchmarking performance options
in seconds.
* `random_shuffle_benchmark_runs`: `bool` (default=true) \
Whether to perform all benchmark runs, each of which has different
performance options, in a random order.
## Build the benchmark tool with Tensorflow ops support
If you see an error that says: `ERROR: Select TensorFlow op(s), included in the
given model, is(are) not supported by this interpreter.` you will need to
build with [Tensorflow operators support](https://www.tensorflow.org/lite/guide/ops_select).
Having Tensorflow ops in the TFLite file works when the benchmark tool is built
with Tensorflow ops support. It doesn't require any additional option to use it.
### How to build
To build the tool, you need to use the `benchmark_model_plus_flex` target with
the `--config=monolithic` flag.
**Desktop**
```
bazel build -c opt \
--config=monolithic \
tensorflow/lite/tools/benchmark:benchmark_model_plus_flex
```
**Android**
```
bazel build -c opt \
--config=monolithic --config=android_arm64 \
tensorflow/lite/tools/benchmark:benchmark_model_plus_flex
```
### How to benchmark tflite model with Tensorflow ops
Follow the further instructions [above](#to-buildinstallrun) replacing
`benchmark_model` with the `benchmark_model_plus_flex` file created here.
For example, on desktop it's very easy:
```
bazel-bin/tensorflow/lite/tools/benchmark/benchmark_model_plus_flex \
--graph=model_converted_with_TF_ops.tflite \
```
## Build the benchmark tool with Custom ops support
If you see an error that says `ERROR: Op type not registered 'XXXXXXXX'
in binary running on localhost.` for custom ops running in your TFLite model,
you will need to manually build the tool to include your libraries providing
the custom ops.
### How to build
While possible, this is not necessarily supported.
However, you should be able to create a new `cc_binary` rule that depends on
`tensorflow/lite/tools/benchmark:benchmark_model_main` along with your custom op
rules.
```
cc_binary(
name = "benchmark_model_plus_custom_ops",
deps = [
":my_custom_ops_provider",
"//tensorflow/lite/tools/benchmark:benchmark_model_main",
],
)
```
### How to benchmark tflite model with Custom ops
Use the `benchmark_model_plus_custom_ops` (or whatever) file created by your
custom rule instead of the `benchmark_model` file in the instructions,
[above](#to-buildinstallrun).
tensorflow/lite/tools/benchmark/android/AndroidManifest.xml
+45
View File
@@ -0,0 +1,45 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="org.tensorflow.lite.benchmark">
<!-- Necessary for loading custom models from disk. -->
<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE"/>
<uses-sdk
android:minSdkVersion="23"
android:targetSdkVersion="31" />
<application
android:debuggable="true">
<!-- This Activity runs the TensorFlow Lite benchmark at creation, using
a provided set of arguments, then immediately terminates. -->
<activity
android:name=".BenchmarkModelActivity"
android:screenOrientation="portrait"
android:label="TFLite Benchmark"
android:theme="@android:style/Theme.NoDisplay"
android:exported="true"
android:noHistory="true" />
<uses-library android:name="libOpenCL.so"
android:required="false"/>
<uses-library android:name="libOpenCL-pixel.so"
android:required="false"/>
</application>
</manifest>
tensorflow/lite/tools/benchmark/android/BUILD
+74
View File
@@ -0,0 +1,74 @@
# Description:
# BenchmarkModel Android harness for TensorFlow Lite benchmarks.
load("@build_bazel_rules_android//android:rules.bzl", "android_binary")
load("@rules_cc//cc:cc_library.bzl", "cc_library")
load("//tensorflow/lite:build_def.bzl", "tflite_jni_binary")
load("//tensorflow/lite:special_rules.bzl", "tflite_hexagon_nn_skel_libraries")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
default_visibility = ["//visibility:private"],
licenses = ["notice"],
)
# See README.md for details about building and executing the benchmark in APK
# format.
APK_VARIANTS = [
# (suffix, extra deps)
("", []),
(
"_plus_flex",
["//tensorflow/lite/delegates/flex:delegate"],
),
]
[android_binary(
name = "benchmark_model%s" % suffix,
srcs = glob([
"src/**/*.java",
]),
custom_package = "org.tensorflow.lite.benchmark",
manifest = "AndroidManifest.xml",
# In some platforms we don't have an Android SDK/NDK and this target
# can't be built. We need to prevent the build system from trying to
# use the target in that case.
tags = ["manual"],
deps = [
":hexagon_libs",
":tensorflowlite_benchmark_native%s" % suffix,
],
) for suffix, _ in APK_VARIANTS]
[tflite_jni_binary(
name = "libtensorflowlite_benchmark%s.so" % suffix,
srcs = glob([
"jni/**/*.cc",
"jni/**/*.h",
]),
deps = [
"//tensorflow/lite/java/jni",
"//tensorflow/lite/tools/benchmark:benchmark_tflite_model_lib",
] + extra_deps,
) for suffix, extra_deps in APK_VARIANTS]
[cc_library(
name = "tensorflowlite_benchmark_native%s" % suffix,
srcs = ["libtensorflowlite_benchmark%s.so" % suffix],
visibility = ["//visibility:private"],
) for suffix, _ in APK_VARIANTS]
cc_library(
name = "hexagon_libs",
srcs = select({
"//tensorflow:android_arm64": [
"//tensorflow/lite/delegates/hexagon/hexagon_nn:libhexagon_interface.so",
] + tflite_hexagon_nn_skel_libraries(),
"//tensorflow:android_arm": [
"//tensorflow/lite/delegates/hexagon/hexagon_nn:libhexagon_interface.so",
] + tflite_hexagon_nn_skel_libraries(),
"//conditions:default": [],
}),
visibility = ["//visibility:private"],
)
tensorflow/lite/tools/benchmark/android/README.md
+157
View File
@@ -0,0 +1,157 @@
# TFLite Model Benchmark Tool with Android Apk
## Description
This Android benchmark app is a simple wrapper around the TensorFlow Lite
[command-line benchmark utility](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark).
Pushing and executing binaries directly on an Android device is a valid approach
to benchmarking, but it can result in subtle (but observable) differences in
performance relative to execution within an actual Android app. In particular,
Android's scheduler tailors behavior based on thread and process priorities,
which differ between a foreground Activity/Application and a regular background
binary executed via `adb shell ...`. This tailored behavior is most evident when
enabling multi-threaded CPU execution with TensorFlow Lite.
To that end, this app offers perhaps a more faithful view of runtime performance
that developers can expect when deploying TensorFlow Lite with their
application.
## To build/install/run
(0) Refer to
https://github.com/tensorflow/tensorflow/tree/master/tensorflow/tools/android/test
to edit the `WORKSPACE` to configure the android NDK/SDK.
(1) Build for your specific platform, e.g.:
```
bazel build -c opt \
--config=android_arm64 \
tensorflow/lite/tools/benchmark/android:benchmark_model
```
(Optional) To enable Hexagon delegate with `--use_hexagon=true` option, you can
download and install the libraries as the guided in [hexagon delegate]
(https://www.tensorflow.org/lite/performance/hexagon_delegate#step_2_add_hexagon_libraries_to_your_android_app)
page. For example, if you installed the libraries at third_party/hexagon_nn_skel
and created third_party/hexagon_nn_skel/BUILD with a build target,
```
filegroup(
name = "libhexagon_nn_skel",
srcs = glob(["*.so"]),
)
```
you need to modify tflite_hexagon_nn_skel_libraries macro in
tensorflow/lite/special_rules.bzl to specify the build target.
```
return ["//third_party/hexagon_nn_skel:libhexagon_nn_skel"]
```
(2) Connect your phone. Install the benchmark APK to your phone with adb:
```
adb install -r -d -g bazel-bin/tensorflow/lite/tools/benchmark/android/benchmark_model.apk
```
Note: Make sure to install with "-g" option to grant the permission for reading
external storage.
(3) Push the compute graph that you need to test.
```
adb push mobilenet_quant_v1_224.tflite /data/local/tmp
```
(4) Run the benchmark. Additional command-line flags are documented
[here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark/README.md)
and can be appended to the `args` string alongside the required `--graph` flag
(note that all args must be nested in the single quoted string that follows the
args key).
```
adb shell am start -S \
-n org.tensorflow.lite.benchmark/.BenchmarkModelActivity \
--es args '"--graph=/data/local/tmp/mobilenet_quant_v1_224.tflite \
--num_threads=4"'
```
(5) The results will be available in Android's logcat, e.g.:
```
adb logcat | grep "Inference timings in us"
... tflite : Inference timings in us: Init: 1007529, First inference: 4098, Warmup (avg): 1686.59, Inference (avg): 1687.92
```
## To trace Tensorflow Lite internals including operator invocation
The steps described here follows the method of
https://developer.android.com/topic/performance/tracing/on-device. Refer to the
page for more detailed information.
(0)-(3) Follow the steps (0)-(3) of [build/install/run](#to-buildinstallrun)
section.
(4) Enable platform tracing.
```
adb shell setprop debug.tflite.trace 1
```
(5) Set up Quick Settings tile for System Tracing app on your device. Follow the
[instruction](https://developer.android.com/topic/performance/tracing/on-device#set-up-tile).
The System Tracing tile will be added to the Quick Settings panel.
Optionally, you can set up other configurations for tracing from the app menu.
Refer to the
[guide](https://developer.android.com/topic/performance/tracing/on-device#app-menu)
for more information.
(6) Tap the System Tracing tile, which has the label "Record trace". The tile
becomes enabled, and a persistent notification appears to notify you that the
system is now recording a trace.
(7) Run the benchmark with platform tracing enabled.
```
adb shell am start -S \
-n org.tensorflow.lite.benchmark/.BenchmarkModelActivity \
--es args '"--graph=/data/local/tmp/mobilenet_quant_v1_224.tflite \
--num_threads=4"'
```
(8) Wait until the benchmark finishes. It can be checked from Android log
messages, e.g.,
```
adb logcat | grep "Inference timings in us"
... tflite : Inference timings in us: Init: 1007529, First inference: 4098, Warmup (avg): 1686.59, Inference (avg): 1687.92
```
(9) Stop tracing by tapping either the System Tracing tile in the Quick Settings
panel or on the System Tracing notification. The system displays a new
notification that contains the message "Saving trace". When saving is complete,
the system dismisses the notification and displays a third notification "Trace
saved", confirming that your trace has been saved and that you're ready to share
the system trace.
(10)
[Share](https://developer.android.com/topic/performance/tracing/on-device#share-trace)
a trace file,
[convert](https://developer.android.com/topic/performance/tracing/on-device#converting_between_trace_formats)
between tracing formats and
[create](https://developer.android.com/topic/performance/tracing/on-device#create-html-report)
an HTML report. Note that, the captured tracing file format is either in
Perfetto format or in Systrace format depending on the Android version of your
device. Select the appropriate method to handle the generated file.
(11) Disable platform tracing.
```
adb shell setprop debug.tflite.trace 0
```
tensorflow/lite/tools/benchmark/android/jni/benchmark_model_jni.cc
+66
View File
@@ -0,0 +1,66 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <jni.h>
#include <sstream>
#include <string>
#include "tensorflow/lite/tools/benchmark/benchmark_tflite_model.h"
#ifdef __ANDROID__
#include <android/log.h>
#endif
namespace tflite {
namespace benchmark {
namespace {
void Run(int argc, char** argv) {
BenchmarkTfLiteModel benchmark;
benchmark.Run(argc, argv);
}
} // namespace
} // namespace benchmark
} // namespace tflite
extern "C" {
JNIEXPORT void JNICALL
Java_org_tensorflow_lite_benchmark_BenchmarkModel_nativeRun(JNIEnv* env,
jclass clazz,
jstring args_obj) {
const char* args_chars = env->GetStringUTFChars(args_obj, nullptr);
// Split the args string into individual arg tokens.
std::istringstream iss(args_chars);
std::vector<std::string> args_split{std::istream_iterator<std::string>(iss),
{}};
// Construct a fake argv command-line object for the benchmark.
std::vector<char*> argv;
std::string arg0 = "(BenchmarkModelAndroid)";
argv.push_back(const_cast<char*>(arg0.data()));
for (auto& arg : args_split) {
argv.push_back(const_cast<char*>(arg.data()));
}
tflite::benchmark::Run(static_cast<int>(argv.size()), argv.data());
env->ReleaseStringUTFChars(args_obj, args_chars);
}
} // extern "C"
tensorflow/lite/tools/benchmark/android/src/org/tensorflow/lite/benchmark/BenchmarkModel.java
+37
View File
@@ -0,0 +1,37 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark;
/** Helper class for running a native TensorFlow Lite benchmark. */
class BenchmarkModel {
static {
// Try loading flex first if available. If not load regular tflite shared library.
try {
System.loadLibrary("tensorflowlite_benchmark_plus_flex");
} catch (UnsatisfiedLinkError e) {
System.loadLibrary("tensorflowlite_benchmark");
}
}
// Executes a standard TensorFlow Lite benchmark according to the provided args.
//
// Note that {@code args} will be split by the native execution code.
public static void run(String args) {
nativeRun(args);
}
private static native void nativeRun(String args);
}
tensorflow/lite/tools/benchmark/android/src/org/tensorflow/lite/benchmark/BenchmarkModelActivity.java
+51
View File
@@ -0,0 +1,51 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark;
import android.app.Activity;
import android.content.Intent;
import android.os.Bundle;
import android.os.Trace;
import android.util.Log;
/** Main {@code Activity} class for the benchmark app. */
public class BenchmarkModelActivity extends Activity {
private static final String TAG = "tflite_BenchmarkModelActivity";
private static final String ARGS_INTENT_KEY_0 = "args";
private static final String ARGS_INTENT_KEY_1 = "--args";
@Override
public void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
Intent intent = getIntent();
Bundle bundle = intent.getExtras();
String args = bundle.getString(ARGS_INTENT_KEY_0, bundle.getString(ARGS_INTENT_KEY_1));
if (args.contains("--use_hexagon=true") || args.contains("--use_hexagon=1")) {
// Users should not specify this argument.
args = args + " --hexagon_lib_path=" + getApplicationInfo().nativeLibraryDir;
}
Log.i(TAG, "Running TensorFlow Lite benchmark with args: " + args);
Trace.beginSection("TFLite Benchmark Model");
BenchmarkModel.run(args);
Trace.endSection();
finish();
}
}
tensorflow/lite/tools/benchmark/benchmark_main.cc
+36
View File
@@ -0,0 +1,36 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <cstdlib>
#include "tensorflow/lite/tools/benchmark/benchmark_tflite_model.h"
#include "tensorflow/lite/tools/logging.h"
namespace tflite {
namespace benchmark {
int Main(int argc, char** argv) {
TFLITE_LOG(INFO) << "STARTING!";
BenchmarkTfLiteModel benchmark;
if (benchmark.Run(argc, argv) != kTfLiteOk) {
TFLITE_LOG(ERROR) << "Benchmarking failed.";
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}
} // namespace benchmark
} // namespace tflite
int main(int argc, char** argv) { return tflite::benchmark::Main(argc, argv); }
tensorflow/lite/tools/benchmark/benchmark_model.cc
+401
View File
@@ -0,0 +1,401 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/lite/tools/benchmark/benchmark_model.h"
#include <cstdint>
#ifdef __linux__
#include <unistd.h>
#endif // __linux__
#include <iostream>
#include <memory>
#include <sstream>
#include <string>
#include "tensorflow/lite/profiling/memory_info.h"
#include "tensorflow/lite/profiling/time.h"
#include "tensorflow/lite/tools/benchmark/benchmark_utils.h"
#include "tensorflow/lite/tools/logging.h"
namespace tflite {
namespace benchmark {
using tensorflow::StatWithPercentiles;
constexpr int kMemoryCheckIntervalMs = 50;
#ifdef __linux__
void GetRssStats(size_t* vsize, size_t* rss, size_t* shared, size_t* code) {
FILE* fp = fopen("/proc/self/statm", "rt");
*vsize = 0;
*rss = 0;
*shared = 0;
*code = 0;
if (fp == nullptr) return;
(void)!fscanf(fp, "%zu %zu %zu %zu", vsize, rss, shared, code);
fclose(fp);
*vsize = *vsize * getpagesize() >> 20;
*rss = *rss * getpagesize() >> 20;
*shared = *shared * getpagesize() >> 20;
*code = *code * getpagesize() >> 20;
}
#endif // __linux__
BenchmarkParams BenchmarkModel::DefaultParams() {
BenchmarkParams params;
params.AddParam("num_runs", BenchmarkParam::Create<int32_t>(50));
params.AddParam("min_secs", BenchmarkParam::Create<float>(1.0f));
params.AddParam("max_secs", BenchmarkParam::Create<float>(150.0f));
params.AddParam("run_delay", BenchmarkParam::Create<float>(-1.0f));
params.AddParam("run_frequency", BenchmarkParam::Create<float>(-1.0f));
params.AddParam("num_threads", BenchmarkParam::Create<int32_t>(-1));
params.AddParam("use_caching", BenchmarkParam::Create<bool>(false));
params.AddParam("benchmark_name", BenchmarkParam::Create<std::string>(""));
params.AddParam("output_prefix", BenchmarkParam::Create<std::string>(""));
params.AddParam("warmup_runs", BenchmarkParam::Create<int32_t>(1));
params.AddParam("warmup_min_secs", BenchmarkParam::Create<float>(0.5f));
params.AddParam("verbose", BenchmarkParam::Create<bool>(false));
params.AddParam("dry_run", BenchmarkParam::Create<bool>(false));
params.AddParam("report_peak_memory_footprint",
BenchmarkParam::Create<bool>(false));
params.AddParam("memory_footprint_check_interval_ms",
BenchmarkParam::Create<int32_t>(kMemoryCheckIntervalMs));
params.AddParam("gpu_invoke_loop_times", BenchmarkParam::Create<int32_t>(1));
return params;
}
BenchmarkModel::BenchmarkModel() : params_(DefaultParams()) {}
void BenchmarkLoggingListener::OnBenchmarkEnd(const BenchmarkResults& results) {
auto inference_us = results.inference_time_us();
auto init_us = results.startup_latency_us();
auto warmup_us = results.warmup_time_us();
auto init_mem_usage = results.init_mem_usage();
auto overall_mem_usage = results.overall_mem_usage();
TFLITE_LOG(INFO) << "Inference timings in us: "
<< "Init: " << init_us << ", "
<< "First inference: " << warmup_us.first() << ", "
<< "Warmup (avg): " << warmup_us.avg() << ", "
<< "Inference (avg): " << inference_us.avg();
if (!init_mem_usage.IsSupported()) return;
TFLITE_LOG(INFO)
<< "Note: as the benchmark tool itself affects memory footprint, the "
"following is only APPROXIMATE to the actual memory footprint of the "
"model at runtime. Take the information at your discretion.";
TFLITE_LOG(INFO) << "Memory footprint delta from the start of the tool (MB): "
<< "init=" << init_mem_usage.mem_footprint_kb / 1024.0
<< " overall="
<< overall_mem_usage.mem_footprint_kb / 1024.0;
auto peak_mem_mb = results.peak_mem_mb();
if (peak_mem_mb > 0) {
TFLITE_LOG(INFO)
<< "Overall peak memory footprint (MB) via periodic monitoring: "
<< peak_mem_mb;
#ifdef __linux__
size_t vsize, rss, shared, code;
GetRssStats(&vsize, &rss, &shared, &code);
TFLITE_LOG(INFO) << "Memory status at the end of exeution:";
TFLITE_LOG(INFO) << "- VmRSS : " << rss << " MB";
TFLITE_LOG(INFO) << "+ RssAnnon : " << rss - shared << " MB";
TFLITE_LOG(INFO) << "+ RssFile + RssShmem : " << shared << " MB";
#endif // __linux_
}
}
std::vector<Flag> BenchmarkModel::GetFlags() {
return {
CreateFlag<int32_t>(
"num_runs", &params_,
"expected number of runs, see also min_secs, max_secs"),
CreateFlag<float>(
"min_secs", &params_,
"minimum number of seconds to rerun for, potentially making the "
"actual number of runs to be greater than num_runs"),
CreateFlag<float>(
"max_secs", &params_,
"maximum number of seconds to rerun for, potentially making the "
"actual number of runs to be less than num_runs. Note if --max-secs "
"is exceeded in the middle of a run, the benchmark will continue to "
"the end of the run but will not start the next run."),
CreateFlag<float>("run_delay", &params_, "delay between runs in seconds"),
CreateFlag<float>(
"run_frequency", &params_,
"Execute at a fixed frequency, instead of a fixed delay."
"Note if the targeted rate per second cannot be reached, the "
"benchmark would start the next run immediately, trying its best to "
"catch up. If set, this will override run_delay."),
CreateFlag<int32_t>("num_threads", &params_, "number of threads"),
CreateFlag<bool>(
"use_caching", &params_,
"Enable caching of prepacked weights matrices in matrix "
"multiplication routines. Currently implies the use of the Ruy "
"library."),
CreateFlag<std::string>("benchmark_name", &params_, "benchmark name"),
CreateFlag<std::string>("output_prefix", &params_,
"benchmark output prefix"),
CreateFlag<int32_t>(
"warmup_runs", &params_,
"minimum number of runs performed on initialization, to "
"allow performance characteristics to settle, see also "
"warmup_min_secs"),
CreateFlag<float>(
"warmup_min_secs", &params_,
"minimum number of seconds to rerun for, potentially making the "
"actual number of warm-up runs to be greater than warmup_runs"),
CreateFlag<bool>("verbose", &params_,
"Whether to log parameters whose values are not set. "
"By default, only log those parameters that are set by "
"parsing their values from the commandline flags."),
CreateFlag<bool>("dry_run", &params_,
"Whether to run the tool just with simply loading the "
"model, allocating tensors etc. but without actually "
"invoking any op kernels."),
CreateFlag<bool>(
"report_peak_memory_footprint", &params_,
"Report the peak memory footprint by periodically checking the "
"memory footprint. Internally, a separate thread will be spawned for "
"this periodic check. Therefore, the performance benchmark result "
"could be affected."),
CreateFlag<int32_t>("memory_footprint_check_interval_ms", &params_,
"The interval in millisecond between two consecutive "
"memory footprint checks. This is only used when "
"--report_peak_memory_footprint is set to true."),
CreateFlag<int32_t>(
"gpu_invoke_loop_times", &params_,
"Number of GPU delegate invoke loop iterations. If > 0 then reported "
"latency is divided by this number. Used only when "
"TFLITE_GPU_ENABLE_INVOKE_LOOP is defined.")};
}
void BenchmarkModel::LogParams() {
const bool verbose = params_.Get<bool>("verbose");
TFLITE_LOG(INFO) << "Log parameter values verbosely: [" << verbose << "]";
LOG_BENCHMARK_PARAM(int32_t, "num_runs", "Min num runs", verbose);
LOG_BENCHMARK_PARAM(float, "min_secs", "Min runs duration (seconds)",
verbose);
LOG_BENCHMARK_PARAM(float, "max_secs", "Max runs duration (seconds)",
verbose);
LOG_BENCHMARK_PARAM(float, "run_delay", "Inter-run delay (seconds)", verbose);
LOG_BENCHMARK_PARAM(float, "run_frequency",
"Number of prorated runs per second", verbose);
LOG_BENCHMARK_PARAM(int32_t, "num_threads", "Num threads", verbose);
LOG_BENCHMARK_PARAM(bool, "use_caching", "Use caching", verbose);
LOG_BENCHMARK_PARAM(std::string, "benchmark_name", "Benchmark name", verbose);
LOG_BENCHMARK_PARAM(std::string, "output_prefix", "Output prefix", verbose);
LOG_BENCHMARK_PARAM(int32_t, "warmup_runs", "Min warmup runs", verbose);
LOG_BENCHMARK_PARAM(float, "warmup_min_secs",
"Min warmup runs duration (seconds)", verbose);
LOG_BENCHMARK_PARAM(bool, "dry_run", "Run w/o invoking kernels", verbose);
LOG_BENCHMARK_PARAM(bool, "report_peak_memory_footprint",
"Report the peak memory footprint", verbose);
LOG_BENCHMARK_PARAM(int32_t, "memory_footprint_check_interval_ms",
"Memory footprint check interval (ms)", verbose);
#ifdef TFLITE_GPU_ENABLE_INVOKE_LOOP
LOG_BENCHMARK_PARAM(int32_t, "gpu_invoke_loop_times",
"Number of GPU delegate invoke loop iterations. Latency "
"will be divided by it.",
verbose);
#endif
}
TfLiteStatus BenchmarkModel::PrepareInputData() { return kTfLiteOk; }
TfLiteStatus BenchmarkModel::ResetInputsAndOutputs() { return kTfLiteOk; }
StatWithPercentiles<int64_t> BenchmarkModel::Run(int min_num_times,
float min_secs, float max_secs,
RunType run_type,
TfLiteStatus* invoke_status) {
StatWithPercentiles<int64_t> run_stats;
TFLITE_LOG(INFO) << "Running benchmark for at least " << min_num_times
<< " iterations and at least " << min_secs << " seconds but"
<< " terminate if exceeding " << max_secs << " seconds.";
int64_t now_us = profiling::time::NowMicros();
int64_t min_finish_us = now_us + static_cast<int64_t>(min_secs * 1.e6f);
int64_t max_finish_us = now_us + static_cast<int64_t>(max_secs * 1.e6f);
*invoke_status = kTfLiteOk;
float inter_run_sleep_time = params_.Get<float>("run_delay");
auto run_frequency = params_.Get<float>("run_frequency");
double manual_inter_run_gap = 1.0 / run_frequency;
// float doesn't have sufficient precision for storing this number
double next_run_finish_time = now_us * 1e-6 + manual_inter_run_gap;
for (int run = 0; (run < min_num_times || now_us < min_finish_us) &&
now_us <= max_finish_us;
run++) {
ResetInputsAndOutputs();
listeners_.OnSingleRunStart(run_type);
int64_t start_us = profiling::time::NowMicros();
TfLiteStatus status = RunImpl();
int64_t end_us = profiling::time::NowMicros();
listeners_.OnSingleRunEnd();
int64_t run_duration_us = end_us - start_us;
#ifdef TFLITE_GPU_ENABLE_INVOKE_LOOP
int32_t gpu_invoke_loop_times = params_.Get<int>("gpu_invoke_loop_times");
if (gpu_invoke_loop_times > 0) {
run_duration_us = static_cast<int64_t>(
static_cast<double>(run_duration_us) / gpu_invoke_loop_times);
}
#endif
run_stats.UpdateStat(run_duration_us);
if (run_frequency > 0) {
inter_run_sleep_time =
next_run_finish_time - profiling::time::NowMicros() * 1e-6;
next_run_finish_time += manual_inter_run_gap;
}
// Note when "inter_run_sleep_time" is negative or 0.0,
// the function will return immediately.
util::SleepForSeconds(inter_run_sleep_time);
now_us = profiling::time::NowMicros();
if (status != kTfLiteOk) {
*invoke_status = status;
}
}
std::stringstream stream;
run_stats.OutputToStream(&stream);
TFLITE_LOG(INFO) << stream.str() << std::endl;
return run_stats;
}
TfLiteStatus BenchmarkModel::ValidateParams() {
if (params_.Get<bool>("report_peak_memory_footprint")) {
const int32_t interval =
params_.Get<int32_t>("memory_footprint_check_interval_ms");
if (interval <= 0) {
TFLITE_LOG(WARN) << "--memory_footprint_check_interval_ms is set to "
<< interval
<< " (ms), This value is invalid, and it will be set to "
"the default value "
<< kMemoryCheckIntervalMs << " (ms).";
params_.Set<int32_t>("memory_footprint_check_interval_ms",
kMemoryCheckIntervalMs);
}
}
return kTfLiteOk;
}
TfLiteStatus BenchmarkModel::Run(int argc, char** argv) {
TF_LITE_ENSURE_STATUS(ParseFlags(argc, argv));
return Run();
}
TfLiteStatus BenchmarkModel::Run() {
TF_LITE_ENSURE_STATUS(ValidateParams());
LogParams();
auto peak_memory_reporter = MayCreateMemoryUsageMonitor();
if (peak_memory_reporter != nullptr) peak_memory_reporter->Start();
const double model_size_mb = MayGetModelFileSize() / 1e6;
const auto start_mem_usage = profiling::memory::GetMemoryUsage();
int64_t initialization_start_us = profiling::time::NowMicros();
TF_LITE_ENSURE_STATUS(Init());
const auto init_end_mem_usage = profiling::memory::GetMemoryUsage();
int64_t initialization_end_us = profiling::time::NowMicros();
int64_t startup_latency_us = initialization_end_us - initialization_start_us;
const auto init_mem_usage = init_end_mem_usage - start_mem_usage;
if (model_size_mb > 0) {
TFLITE_LOG(INFO) << "The input model file size (MB): " << model_size_mb;
} else {
TFLITE_LOG(WARN) << "Failed to get the input model file size.";
}
TFLITE_LOG(INFO) << "Initialized session in " << startup_latency_us / 1e3
<< "ms.";
TF_LITE_ENSURE_STATUS(PrepareInputData());
TfLiteStatus status = kTfLiteOk;
uint64_t input_bytes = ComputeInputBytes();
// Overwrite certain parameters when --dry_run=true is set.
if (params_.Get<bool>("dry_run")) {
params_.Set("warmup_runs", 0);
params_.Set("warmup_min_secs", -1.0f);
params_.Set("num_runs", 0);
params_.Set("min_secs", -1.0f);
}
listeners_.OnBenchmarkStart(params_);
StatWithPercentiles<int64_t> warmup_time_us =
Run(params_.Get<int32_t>("warmup_runs"),
params_.Get<float>("warmup_min_secs"), params_.Get<float>("max_secs"),
WARMUP, &status);
if (status != kTfLiteOk) {
return status;
}
StatWithPercentiles<int64_t> inference_time_us =
Run(params_.Get<int32_t>("num_runs"), params_.Get<float>("min_secs"),
params_.Get<float>("max_secs"), REGULAR, &status);
const auto overall_mem_usage =
profiling::memory::GetMemoryUsage() - start_mem_usage;
float peak_mem_mb = profiling::memory::MemoryUsageMonitor::kInvalidMemUsageMB;
if (peak_memory_reporter != nullptr) {
peak_memory_reporter->Stop();
peak_mem_mb = peak_memory_reporter->GetPeakMemUsageInMB();
}
listeners_.OnBenchmarkEnd({model_size_mb, startup_latency_us, input_bytes,
warmup_time_us, inference_time_us, init_mem_usage,
overall_mem_usage, peak_mem_mb});
return status;
}
TfLiteStatus BenchmarkModel::ParseFlags(int* argc, char** argv) {
auto flag_list = GetFlags();
const bool parse_result =
Flags::Parse(argc, const_cast<const char**>(argv), flag_list);
// "--help" flag is added in tools/delegates/default_execution_provider.cc. As
// this is an optional dependency, we need to check whether "--help" exists or
// not first.
if (!parse_result ||
(params_.HasParam("help") && params_.Get<bool>("help"))) {
std::string usage = Flags::Usage(argv[0], flag_list);
TFLITE_LOG(ERROR) << usage;
// Returning kTfLiteError intentionally when "--help=true" is specified so
// that the caller could check the return value to decide stopping the
// execution.
return kTfLiteError;
}
std::string unconsumed_args =
Flags::ArgsToString(*argc, const_cast<const char**>(argv));
if (!unconsumed_args.empty()) {
TFLITE_LOG(WARN) << "Unconsumed cmdline flags: " << unconsumed_args;
}
return kTfLiteOk;
}
std::unique_ptr<profiling::memory::MemoryUsageMonitor>
BenchmarkModel::MayCreateMemoryUsageMonitor() const {
if (!params_.Get<bool>("report_peak_memory_footprint")) return nullptr;
return std::make_unique<profiling::memory::MemoryUsageMonitor>(
params_.Get<int32_t>("memory_footprint_check_interval_ms"));
}
} // namespace benchmark
} // namespace tflite
tensorflow/lite/tools/benchmark/benchmark_model.h
+243
View File
@@ -0,0 +1,243 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_MODEL_H_
#define TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_MODEL_H_
#include <cmath>
#include <cstdint>
#include <limits>
#include <memory>
#include <ostream>
#include <string>
#include <unordered_set>
#include <utility>
#include <vector>
#include "tensorflow/core/util/stats_calculator.h"
#include "tensorflow/lite/core/c/common.h"
#include "tensorflow/lite/profiling/memory_info.h"
#include "tensorflow/lite/profiling/memory_usage_monitor.h"
#include "tensorflow/lite/tools/benchmark/benchmark_params.h"
#include "tensorflow/lite/tools/command_line_flags.h"
namespace tflite {
namespace benchmark {
enum RunType {
WARMUP,
REGULAR,
};
class BenchmarkResults {
public:
BenchmarkResults() {}
BenchmarkResults(double model_size_mb, int64_t startup_latency_us,
uint64_t input_bytes,
tensorflow::StatWithPercentiles<int64_t> warmup_time_us,
tensorflow::StatWithPercentiles<int64_t> inference_time_us,
const profiling::memory::MemoryUsage& init_mem_usage,
const profiling::memory::MemoryUsage& overall_mem_usage,
float peak_mem_mb)
: model_size_mb_(model_size_mb),
startup_latency_us_(startup_latency_us),
input_bytes_(input_bytes),
warmup_time_us_(warmup_time_us),
inference_time_us_(inference_time_us),
init_mem_usage_(init_mem_usage),
overall_mem_usage_(overall_mem_usage),
peak_mem_mb_(peak_mem_mb) {}
const double model_size_mb() const { return model_size_mb_; }
tensorflow::StatWithPercentiles<int64_t> inference_time_us() const {
return inference_time_us_;
}
tensorflow::StatWithPercentiles<int64_t> warmup_time_us() const {
return warmup_time_us_;
}
int64_t startup_latency_us() const { return startup_latency_us_; }
uint64_t input_bytes() const { return input_bytes_; }
double throughput_MB_per_second() const {
double bytes_per_sec = (input_bytes_ * inference_time_us_.count() * 1e6) /
inference_time_us_.sum();
return bytes_per_sec / (1024.0 * 1024.0);
}
const profiling::memory::MemoryUsage& init_mem_usage() const {
return init_mem_usage_;
}
const profiling::memory::MemoryUsage& overall_mem_usage() const {
return overall_mem_usage_;
}
float peak_mem_mb() const { return peak_mem_mb_; }
private:
double model_size_mb_ = 0.0;
int64_t startup_latency_us_ = 0;
uint64_t input_bytes_ = 0;
tensorflow::StatWithPercentiles<int64_t> warmup_time_us_;
tensorflow::StatWithPercentiles<int64_t> inference_time_us_;
profiling::memory::MemoryUsage init_mem_usage_;
profiling::memory::MemoryUsage overall_mem_usage_;
// An invalid value could happen when we don't monitor memory footprint for
// the inference, or the memory usage info isn't available on the benchmarking
// platform.
float peak_mem_mb_ =
profiling::memory::MemoryUsageMonitor::kInvalidMemUsageMB;
};
class BenchmarkListener {
public:
// Called before the (outer) inference loop begins.
// Note that this is called *after* the interpreter has been initialized, but
// *before* any warmup runs have been executed.
virtual void OnBenchmarkStart(const BenchmarkParams& params) {}
// Called before a single (inner) inference call starts.
virtual void OnSingleRunStart(RunType runType) {}
// Called before a single (inner) inference call ends.
virtual void OnSingleRunEnd() {}
// Called after the (outer) inference loop begins.
virtual void OnBenchmarkEnd(const BenchmarkResults& results) {}
virtual ~BenchmarkListener() {}
};
// A listener that forwards its method calls to a collection of listeners.
class BenchmarkListeners : public BenchmarkListener {
public:
// Added a listener to the listener collection.
// |listener| is not owned by the instance of |BenchmarkListeners|.
// |listener| should not be null and should outlast the instance of
// |BenchmarkListeners|.
void AddListener(BenchmarkListener* listener) {
listeners_.push_back(listener);
}
// Remove all listeners after [index] including the one at 'index'.
void RemoveListeners(int index) {
if (index >= NumListeners()) return;
listeners_.resize(index);
}
int NumListeners() const { return listeners_.size(); }
void OnBenchmarkStart(const BenchmarkParams& params) override {
for (auto listener : listeners_) {
listener->OnBenchmarkStart(params);
}
}
void OnSingleRunStart(RunType runType) override {
for (auto listener : listeners_) {
listener->OnSingleRunStart(runType);
}
}
void OnSingleRunEnd() override {
for (auto listener : listeners_) {
listener->OnSingleRunEnd();
}
}
void OnBenchmarkEnd(const BenchmarkResults& results) override {
for (auto listener : listeners_) {
listener->OnBenchmarkEnd(results);
}
}
~BenchmarkListeners() override {}
private:
// Use vector so listeners are invoked in the order they are added.
std::vector<BenchmarkListener*> listeners_;
};
// Benchmark listener that just logs the results of benchmark run.
class BenchmarkLoggingListener : public BenchmarkListener {
public:
void OnBenchmarkEnd(const BenchmarkResults& results) override;
};
template <typename T>
Flag CreateFlag(const char* name, BenchmarkParams* params,
const std::string& usage) {
return Flag(
name,
[params, name](const T& val, int argv_position) {
params->Set<T>(name, val, argv_position);
},
params->Get<T>(name), usage, Flag::kOptional);
}
// Benchmarks a model.
//
// Subclasses need to implement initialization and running of the model.
// The results can be collected by adding BenchmarkListener(s).
class BenchmarkModel {
public:
static BenchmarkParams DefaultParams();
BenchmarkModel();
explicit BenchmarkModel(BenchmarkParams params)
: params_(std::move(params)) {}
virtual ~BenchmarkModel() {}
virtual TfLiteStatus Init() = 0;
virtual TfLiteStatus Run(int argc, char** argv);
virtual TfLiteStatus Run();
void AddListener(BenchmarkListener* listener) {
listeners_.AddListener(listener);
}
// Remove all listeners after [index] including the one at 'index'.
void RemoveListeners(int index) { listeners_.RemoveListeners(index); }
int NumListeners() const { return listeners_.NumListeners(); }
BenchmarkParams* mutable_params() { return &params_; }
// Unparsable flags will remain in 'argv' in the original order and 'argc'
// will be updated accordingly.
TfLiteStatus ParseFlags(int* argc, char** argv);
protected:
virtual void LogParams();
virtual TfLiteStatus ValidateParams();
TfLiteStatus ParseFlags(int argc, char** argv) {
return ParseFlags(&argc, argv);
}
virtual std::vector<Flag> GetFlags();
// Get the model file size if it's available.
virtual int64_t MayGetModelFileSize() { return -1; }
virtual uint64_t ComputeInputBytes() = 0;
virtual tensorflow::StatWithPercentiles<int64_t> Run(
int min_num_times, float min_secs, float max_secs, RunType run_type,
TfLiteStatus* invoke_status);
// Prepares input data for benchmark. This can be used to initialize input
// data that has non-trivial cost.
virtual TfLiteStatus PrepareInputData();
virtual TfLiteStatus ResetInputsAndOutputs();
virtual TfLiteStatus RunImpl() = 0;
// Create a MemoryUsageMonitor to report peak memory footprint if specified.
virtual std::unique_ptr<profiling::memory::MemoryUsageMonitor>
MayCreateMemoryUsageMonitor() const;
BenchmarkParams params_;
BenchmarkListeners listeners_;
};
} // namespace benchmark
} // namespace tflite
#endif // TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_MODEL_H_
tensorflow/lite/tools/benchmark/benchmark_multirun_stats_recorder.h
+75
View File
@@ -0,0 +1,75 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_MULTIRUN_STATS_RECORDER_H_
#define TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_MULTIRUN_STATS_RECORDER_H_
#include <memory>
#include <string>
#include <vector>
#include "tensorflow/lite/tools/benchmark/benchmark_model.h"
namespace tflite {
namespace benchmark {
class MultiRunStatsRecorder : public BenchmarkListener {
public:
// BenchmarkListener::OnBenchmarkStart is invoked after each run's
// BenchmarkModel::Init. However, some run could fail during Init, e.g.
// delegate fails to be created etc. To still record such run, we will call
// the following function right before a run starts.
void MarkBenchmarkStart(const BenchmarkParams& params) {
results_.emplace_back(EachRunResult());
auto& current = results_.back();
current.completed = false;
current.params = std::make_unique<BenchmarkParams>();
current.params->Merge(params, true /* overwrite*/);
}
void OnBenchmarkEnd(const BenchmarkResults& results) final {
auto& current = results_.back();
current.completed = true;
current.metrics = results;
}
virtual void OutputStats();
protected:
struct EachRunResult {
bool completed = false;
std::unique_ptr<BenchmarkParams> params;
BenchmarkResults metrics;
};
std::vector<EachRunResult> results_;
// Use this to order the runs by the average inference time in increasing
// order (i.e. the fastest run ranks first.). If the run didn't complete,
// we consider it to be slowest.
struct EachRunStatsEntryComparator {
bool operator()(const EachRunResult& i, const EachRunResult& j) {
if (!i.completed) return false;
if (!j.completed) return true;
return i.metrics.inference_time_us().avg() <
j.metrics.inference_time_us().avg();
}
};
virtual std::string PerfOptionName(const BenchmarkParams& params) const;
};
} // namespace benchmark
} // namespace tflite
#endif // TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_MULTIRUN_STATS_RECORDER_H_
tensorflow/lite/tools/benchmark/benchmark_params.h
+31
View File
@@ -0,0 +1,31 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_PARAMS_H_
#define TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_PARAMS_H_
#include "tensorflow/lite/tools/tool_params.h"
namespace tflite {
namespace benchmark {
using BenchmarkParam = tflite::tools::ToolParam;
using BenchmarkParams = tflite::tools::ToolParams;
// To be used in BenchmarkModel::LogParams() and its overrides as we assume
// logging the parameters defined in BenchmarkModel as 'params_'.
#define LOG_BENCHMARK_PARAM(type, name, description, verbose) \
LOG_TOOL_PARAM(params_, type, name, description, verbose)
} // namespace benchmark
} // namespace tflite
#endif // TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_PARAMS_H_
tensorflow/lite/tools/benchmark/benchmark_performance_options.cc
+453
View File
@@ -0,0 +1,453 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/lite/tools/benchmark/benchmark_performance_options.h"
#include <algorithm>
#include <cstdint>
#include <iomanip>
#include <memory>
#include <random>
#include <sstream>
#include <string>
#include <utility>
#include "tensorflow/core/util/stats_calculator.h"
#include "tensorflow/lite/core/c/c_api_types.h"
#include "tensorflow/lite/core/c/common.h"
#if defined(__ANDROID__)
#include "tensorflow/lite/delegates/gpu/delegate.h"
#include "tensorflow/lite/nnapi/nnapi_util.h"
#endif
#include "tensorflow/lite/profiling/time.h"
#include "tensorflow/lite/tools/benchmark/benchmark_params.h"
#include "tensorflow/lite/tools/benchmark/benchmark_utils.h"
#include "tensorflow/lite/tools/command_line_flags.h"
#include "tensorflow/lite/tools/logging.h"
#if defined(__APPLE__)
#include "TargetConditionals.h"
#if (TARGET_OS_IPHONE && !TARGET_IPHONE_SIMULATOR) || \
(TARGET_OS_OSX && TARGET_CPU_ARM64)
// Only enable coreml delegate when using a real iPhone device or Apple Silicon.
#define REAL_IPHONE_DEVICE
#endif
#endif
namespace tflite {
namespace benchmark {
std::string MultiRunStatsRecorder::PerfOptionName(
const BenchmarkParams& params) const {
#if defined(__ANDROID__)
if (params.Get<bool>("use_nnapi")) {
const std::string accelerator =
params.Get<std::string>("nnapi_accelerator_name");
return accelerator.empty() ? "nnapi(w/o accel name)"
: "nnapi(" + accelerator + ")";
}
#endif
bool gpu_enabled = params.Get<bool>("use_gpu");
#if defined(SUPPORTS_GPU_CL_DELEGATE)
gpu_enabled = gpu_enabled || params.Get<bool>("use_gpuv3");
#endif
if (gpu_enabled) {
#if defined(__ANDROID__) || defined(REAL_IPHONE_DEVICE)
if (params.Get<bool>("gpu_precision_loss_allowed")) {
return "gpu-fp16";
} else {
return "gpu-default";
}
#else
return "gpu-default";
#endif
}
#if defined(TFLITE_ENABLE_HEXAGON)
if (params.Get<bool>("use_hexagon")) {
return "dsp w/ hexagon";
}
#endif
#if defined(REAL_IPHONE_DEVICE)
if (params.Get<bool>("use_coreml")) {
return "coreml";
}
#endif
// Handle cases run on CPU
// Note: could use std::to_string to convert an integer to string but it
// requires C++11.
std::stringstream sstm;
sstm << "cpu w/ " << params.Get<int32_t>("num_threads") << " threads";
// Handle cases run on CPU w/ the xnnpack delegate
if (params.Get<bool>("use_xnnpack")) {
sstm << " (xnnpack";
if (params.Get<bool>("xnnpack_force_fp16")) {
sstm << "-fp16";
}
sstm << ")";
}
return sstm.str();
}
void MultiRunStatsRecorder::OutputStats() {
// Make a 80-character-long header.
TFLITE_LOG(INFO) << "\n==============Summary of All Runs w/ Different "
"Performance Options==============";
std::sort(results_.begin(), results_.end(), EachRunStatsEntryComparator());
for (const auto& run_stats : results_) {
const auto perf_option_name = PerfOptionName(*run_stats.params);
std::stringstream stream;
stream << std::setw(26) << perf_option_name << ": ";
if (!run_stats.completed) {
stream << " failed!";
} else {
run_stats.metrics.inference_time_us().OutputToStream(&stream);
// NOTE: As of 2019/11/07, the memory usage is collected in an
// OS-process-wide way and this program performs multiple runs in a single
// OS process, therefore, the memory usage information of each run becomes
// incorrect, hence no output here.
}
TFLITE_LOG(INFO) << stream.str();
}
}
BenchmarkPerformanceOptions::BenchmarkPerformanceOptions(
BenchmarkModel* single_option_run,
std::unique_ptr<MultiRunStatsRecorder> all_run_stats)
: BenchmarkPerformanceOptions(DefaultParams(), single_option_run,
std::move(all_run_stats)) {}
BenchmarkPerformanceOptions::BenchmarkPerformanceOptions(
BenchmarkParams params, BenchmarkModel* single_option_run,
std::unique_ptr<MultiRunStatsRecorder> all_run_stats)
: params_(std::move(params)),
single_option_run_(single_option_run),
single_option_run_params_(single_option_run->mutable_params()),
all_run_stats_(std::move(all_run_stats)) {
single_option_run_->AddListener(all_run_stats_.get());
}
BenchmarkParams BenchmarkPerformanceOptions::DefaultParams() {
BenchmarkParams params;
params.AddParam("perf_options_list",
BenchmarkParam::Create<std::string>("all"));
params.AddParam("option_benchmark_run_delay",
BenchmarkParam::Create<float>(-1.0f));
params.AddParam("random_shuffle_benchmark_runs",
BenchmarkParam::Create<bool>(true));
params.AddParam("gpu_invoke_loop_times", BenchmarkParam::Create<int32_t>(1));
return params;
}
std::vector<Flag> BenchmarkPerformanceOptions::GetFlags() {
return {
CreateFlag<std::string>(
"perf_options_list", &params_,
"A comma-separated list of TFLite performance options to benchmark. "
"By default, all performance options are benchmarked. Note if it's "
"set to 'none', then the tool simply benchmark the model against the "
"specified benchmark parameters."),
CreateFlag<float>("option_benchmark_run_delay", &params_,
"The delay between two consecutive runs of "
"benchmarking performance options in seconds."),
CreateFlag<bool>(
"random_shuffle_benchmark_runs", &params_,
"Whether to perform all benchmark runs, each of which has different "
"performance options, in a random order. It is enabled by default."),
CreateFlag<int32_t>(
"gpu_invoke_loop_times", &params_,
"Number of GPU delegate invoke loop iterations. Used only when "
"TFLITE_GPU_ENABLE_INVOKE_LOOP is defined.")};
}
TfLiteStatus BenchmarkPerformanceOptions::ParseFlags(int* argc, char** argv) {
auto flag_list = GetFlags();
const bool parse_result =
Flags::Parse(argc, const_cast<const char**>(argv), flag_list);
if (!parse_result) {
std::string usage = Flags::Usage(argv[0], flag_list);
TFLITE_LOG(ERROR) << usage;
return kTfLiteError;
}
// Parse the value of --perf_options_list to find performance options to be
// benchmarked.
return ParsePerfOptions();
}
TfLiteStatus BenchmarkPerformanceOptions::ParsePerfOptions() {
const auto& perf_options_list = params_.Get<std::string>("perf_options_list");
if (!util::SplitAndParse(perf_options_list, ',', &perf_options_)) {
TFLITE_LOG(ERROR) << "Cannot parse --perf_options_list: '"
<< perf_options_list
<< "'. Please double-check its value.";
perf_options_.clear();
return kTfLiteError;
}
const auto valid_options = GetValidPerfOptions();
bool is_valid = true;
for (const auto& option : perf_options_) {
if (std::find(valid_options.begin(), valid_options.end(), option) ==
valid_options.end()) {
is_valid = false;
break;
}
}
if (!is_valid) {
std::string valid_options_str;
for (int i = 0; i < valid_options.size() - 1; ++i) {
valid_options_str += (valid_options[i] + ", ");
}
valid_options_str += valid_options.back();
TFLITE_LOG(ERROR)
<< "There are invalid perf options in --perf_options_list: '"
<< perf_options_list << "'. Valid perf options are: ["
<< valid_options_str << "]";
perf_options_.clear();
return kTfLiteError;
}
if (HasOption("none") && perf_options_.size() > 1) {
TFLITE_LOG(ERROR) << "The 'none' option can not be used together with "
"other perf options in --perf_options_list!";
perf_options_.clear();
return kTfLiteError;
}
return kTfLiteOk;
}
std::vector<std::string> BenchmarkPerformanceOptions::GetValidPerfOptions()
const {
std::vector<std::string> valid_options = {"all", "cpu", "gpu",
"nnapi", "coreml", "none"};
#if defined(TFLITE_ENABLE_HEXAGON)
valid_options.emplace_back("dsp");
#endif
return valid_options;
}
bool BenchmarkPerformanceOptions::HasOption(const std::string& option) const {
return std::find(perf_options_.begin(), perf_options_.end(), option) !=
perf_options_.end();
}
void BenchmarkPerformanceOptions::ResetPerformanceOptions() {
single_option_run_params_->Set<int32_t>("num_threads", 1);
single_option_run_params_->Set<bool>("use_gpu", false);
#ifdef TFLITE_GPU_ENABLE_INVOKE_LOOP
single_option_run_params_->Set<int32_t>("gpu_invoke_loop_times", 1);
single_option_run_params_->Set<bool>("require_full_delegation", false);
#endif
#if defined(__ANDROID__)
single_option_run_params_->Set<bool>("gpu_precision_loss_allowed", true);
single_option_run_params_->Set<bool>("use_nnapi", false);
single_option_run_params_->Set<std::string>("nnapi_accelerator_name", "");
single_option_run_params_->Set<bool>("disable_nnapi_cpu", false);
single_option_run_params_->Set<int>("max_delegated_partitions", 0);
single_option_run_params_->Set<bool>("nnapi_allow_fp16", false);
#endif
#if defined(TFLITE_ENABLE_HEXAGON)
single_option_run_params_->Set<bool>("use_hexagon", false);
#endif
#if defined(REAL_IPHONE_DEVICE)
single_option_run_params_->Set<bool>("use_coreml", false);
single_option_run_params_->Set<bool>("gpu_precision_loss_allowed", true);
#endif
single_option_run_params_->Set<bool>("use_xnnpack", false);
single_option_run_params_->Set<bool>("xnnpack_force_fp16", false);
}
void BenchmarkPerformanceOptions::CreatePerformanceOptions() {
TFLITE_LOG(INFO) << "The list of TFLite runtime options to be benchmarked: ["
<< params_.Get<std::string>("perf_options_list") << "]";
if (HasOption("none")) {
// Just add an empty BenchmarkParams instance.
BenchmarkParams params;
all_run_params_.emplace_back(std::move(params));
// As 'none' is exclusive to others, simply return here.
return;
}
const bool benchmark_all = HasOption("all");
if (benchmark_all || HasOption("cpu")) {
const std::vector<int> num_threads = {1, 2, 4};
for (const int count : num_threads) {
BenchmarkParams params;
params.AddParam("num_threads", BenchmarkParam::Create<int32_t>(count));
all_run_params_.emplace_back(std::move(params));
BenchmarkParams xnnpack_params;
xnnpack_params.AddParam("use_xnnpack",
BenchmarkParam::Create<bool>(true));
xnnpack_params.AddParam("xnnpack_force_fp16",
BenchmarkParam::Create<bool>(false));
xnnpack_params.AddParam("num_threads",
BenchmarkParam::Create<int32_t>(count));
all_run_params_.emplace_back(std::move(xnnpack_params));
}
}
if (benchmark_all || HasOption("gpu")) {
#if defined(__ANDROID__)
const std::vector<bool> allow_precision_loss = {true, false};
for (const auto precision_loss : allow_precision_loss) {
BenchmarkParams params;
params.AddParam("use_gpu", BenchmarkParam::Create<bool>(true));
params.AddParam("gpu_precision_loss_allowed",
BenchmarkParam::Create<bool>(precision_loss));
#ifdef TFLITE_GPU_ENABLE_INVOKE_LOOP
int32_t invoke_loop_times = params_.Get<int32_t>("gpu_invoke_loop_times");
params.AddParam("gpu_invoke_loop_times",
BenchmarkParam::Create<int32_t>(invoke_loop_times));
params.AddParam("require_full_delegation",
BenchmarkParam::Create<bool>(true));
#endif
all_run_params_.emplace_back(std::move(params));
}
#else
BenchmarkParams params;
params.AddParam("use_gpu", BenchmarkParam::Create<bool>(true));
#ifdef TFLITE_GPU_ENABLE_INVOKE_LOOP
int32_t invoke_loop_times = params_.Get<int32_t>("gpu_invoke_loop_times");
params.AddParam("gpu_invoke_loop_times",
BenchmarkParam::Create<int32_t>(invoke_loop_times));
params.AddParam("require_full_delegation",
BenchmarkParam::Create<bool>(true));
#endif
all_run_params_.emplace_back(std::move(params));
#endif
}
#if defined(__ANDROID__)
if (benchmark_all || HasOption("nnapi")) {
std::string nnapi_accelerators = nnapi::GetStringDeviceNamesList();
if (!nnapi_accelerators.empty()) {
std::vector<std::string> device_names;
util::SplitAndParse(nnapi_accelerators, ',', &device_names);
for (const auto& name : device_names) {
BenchmarkParams params;
params.AddParam("use_nnapi", BenchmarkParam::Create<bool>(true));
params.AddParam("nnapi_accelerator_name",
BenchmarkParam::Create<std::string>(name));
params.AddParam("disable_nnapi_cpu",
BenchmarkParam::Create<bool>(false));
params.AddParam("max_delegated_partitions",
BenchmarkParam::Create<int>(0));
all_run_params_.emplace_back(std::move(params));
}
}
// Explicitly test the case when there's no "nnapi_accelerator_name"
// parameter as the nnpai execution is different from the case when
// an accelerator name is explicitly specified.
BenchmarkParams params;
params.AddParam("use_nnapi", BenchmarkParam::Create<bool>(true));
all_run_params_.emplace_back(std::move(params));
}
#endif
#if defined(TFLITE_ENABLE_HEXAGON)
if (benchmark_all || HasOption("dsp")) {
BenchmarkParams params;
params.AddParam("use_hexagon", BenchmarkParam::Create<bool>(true));
all_run_params_.emplace_back(std::move(params));
}
#endif
#if defined(REAL_IPHONE_DEVICE)
if (benchmark_all || HasOption("coreml")) {
BenchmarkParams params;
params.AddParam("use_coreml", BenchmarkParam::Create<bool>(true));
all_run_params_.emplace_back(std::move(params));
}
#endif
}
TfLiteStatus BenchmarkPerformanceOptions::Run() {
CreatePerformanceOptions();
if (params_.Get<bool>("random_shuffle_benchmark_runs")) {
std::random_device rd;
std::mt19937 generator(rd());
std::shuffle(all_run_params_.begin(), all_run_params_.end(), generator);
}
// We need to clean *internally* created benchmark listeners, like the
// profiling listener etc. in each Run() invoke because such listeners may be
// reset and become invalid in the next Run(). As a result, we record the
// number of externally-added listeners here to prevent they're cleared later.
const int num_external_listeners = single_option_run_->NumListeners();
// Now perform all runs, each with different performance-affecting parameters.
for (const auto& run_params : all_run_params_) {
// If the run_params is empty, then it means "none" is set for
// --perf_options_list.
if (!run_params.Empty()) {
// Reset all performance-related options before any runs.
ResetPerformanceOptions();
single_option_run_params_->Set(run_params);
}
util::SleepForSeconds(params_.Get<float>("option_benchmark_run_delay"));
// Clear internally created listeners before each run but keep externally
// created ones.
single_option_run_->RemoveListeners(num_external_listeners);
all_run_stats_->MarkBenchmarkStart(*single_option_run_params_);
if (TfLiteStatus status = single_option_run_->Run(); status != kTfLiteOk) {
TFLITE_LOG(ERROR) << "Error while running a single-option run: "
<< status;
return status;
}
}
all_run_stats_->OutputStats();
return kTfLiteOk;
}
TfLiteStatus BenchmarkPerformanceOptions::Run(int argc, char** argv) {
// Parse flags that are supported by this particular binary first.
if (TfLiteStatus status = ParseFlags(&argc, argv); status != kTfLiteOk) {
TFLITE_LOG(ERROR) << "Error while parsing the flags for multi-option runs: "
<< status;
return status;
}
// Then parse flags for single-option runs to get information like parameters
// of the input model etc.
if (TfLiteStatus status = single_option_run_->ParseFlags(&argc, argv);
status != kTfLiteOk) {
TFLITE_LOG(ERROR)
<< "Error while parsing the flags for single-option runs: " << status;
return status;
}
// Now, the remaining are unrecognized flags and we simply print them out.
for (int i = 1; i < argc; ++i) {
TFLITE_LOG(WARN) << "WARNING: unrecognized commandline flag: " << argv[i];
}
return Run();
}
} // namespace benchmark
} // namespace tflite
tensorflow/lite/tools/benchmark/benchmark_performance_options.h
+83
View File
@@ -0,0 +1,83 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_PERFORMANCE_OPTIONS_H_
#define TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_PERFORMANCE_OPTIONS_H_
#include <memory>
#include <string>
#include <vector>
#include "tensorflow/lite/core/c/c_api_types.h"
#include "tensorflow/lite/tools/benchmark/benchmark_model.h"
#include "tensorflow/lite/tools/benchmark/benchmark_multirun_stats_recorder.h"
#include "tensorflow/lite/tools/benchmark/benchmark_params.h"
namespace tflite {
namespace benchmark {
// Benchmarks all performance options on a model by repeatedly invoking the
// single-performance-option run on a passed-in 'BenchmarkModel' object.
class BenchmarkPerformanceOptions {
public:
// Doesn't own the memory of 'single_option_run'.
explicit BenchmarkPerformanceOptions(
BenchmarkModel* single_option_run,
std::unique_ptr<MultiRunStatsRecorder> all_run_stats =
std::make_unique<MultiRunStatsRecorder>());
virtual ~BenchmarkPerformanceOptions() = default;
// Just run the benchmark just w/ default parameter values.
TfLiteStatus Run();
TfLiteStatus Run(int argc, char** argv);
protected:
static BenchmarkParams DefaultParams();
BenchmarkPerformanceOptions(
BenchmarkParams params, BenchmarkModel* single_option_run,
std::unique_ptr<MultiRunStatsRecorder> all_run_stats);
// Unparsable flags will remain in 'argv' in the original order and 'argc'
// will be updated accordingly.
TfLiteStatus ParseFlags(int* argc, char** argv);
virtual std::vector<Flag> GetFlags();
TfLiteStatus ParsePerfOptions();
virtual std::vector<std::string> GetValidPerfOptions() const;
bool HasOption(const std::string& option) const;
virtual void ResetPerformanceOptions();
virtual void CreatePerformanceOptions();
BenchmarkParams params_;
std::vector<std::string> perf_options_;
// The object that drives a single-performance-option run.
BenchmarkModel* const single_option_run_; // Doesn't own the memory.
BenchmarkParams* const single_option_run_params_; // Doesn't own the memory.
// Each element is a set of performance-affecting benchmark parameters to be
// all set for a particular benchmark run.
std::vector<BenchmarkParams> all_run_params_;
std::unique_ptr<MultiRunStatsRecorder> all_run_stats_;
};
} // namespace benchmark
} // namespace tflite
#endif // TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_PERFORMANCE_OPTIONS_H_
tensorflow/lite/tools/benchmark/benchmark_plus_flex_main.cc
+33
View File
@@ -0,0 +1,33 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/lite/testing/init_tensorflow.h"
#include "tensorflow/lite/tools/benchmark/benchmark_tflite_model.h"
#include "tensorflow/lite/tools/logging.h"
namespace tflite {
namespace benchmark {
int Main(int argc, char** argv) {
::tflite::InitTensorFlow();
TFLITE_LOG(INFO) << "STARTING!";
BenchmarkTfLiteModel benchmark;
benchmark.Run(argc, argv);
return 0;
}
} // namespace benchmark
} // namespace tflite
int main(int argc, char** argv) { return tflite::benchmark::Main(argc, argv); }
tensorflow/lite/tools/benchmark/benchmark_test.cc
+761
View File
@@ -0,0 +1,761 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include "absl/log/log.h"
#ifndef _WIN32
#include <fcntl.h>
#endif // !defined(_WIN32)
#include <fstream>
#include <iostream>
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "absl/algorithm/algorithm.h"
#include "absl/strings/str_format.h"
#include "absl/strings/string_view.h"
#include "tensorflow/lite/core/c/c_api_types.h"
#include "tensorflow/lite/core/c/common.h"
#include "tensorflow/lite/interpreter.h"
#include "tensorflow/lite/string_util.h"
#include "tensorflow/lite/testing/util.h"
#include "tensorflow/lite/tools/benchmark/benchmark_performance_options.h"
#include "tensorflow/lite/tools/benchmark/benchmark_tflite_model.h"
#include "tensorflow/lite/tools/benchmark/proto/benchmark_result.pb.h"
#include "tensorflow/lite/tools/command_line_flags.h"
#include "tensorflow/lite/tools/delegates/delegate_provider.h"
#include "tensorflow/lite/tools/logging.h"
namespace {
const std::string* g_fp32_model_path = nullptr;
const std::string* g_int8_model_path = nullptr;
const std::string* g_string_model_path = nullptr;
const std::string* g_string_model_path_no_signature = nullptr;
const std::string* g_multi_signature_model_path = nullptr;
} // namespace
namespace tflite {
namespace benchmark {
namespace {
enum class ModelGraphType { FP32, INT8, STRING };
enum class ModelReadOption { FROM_PATH, FROM_FD };
void InitializeParams(
BenchmarkParams& params, int32_t num_runs, float min_secs, float max_secs,
ModelReadOption model_read_option = ModelReadOption::FROM_PATH,
ModelGraphType graph_type = ModelGraphType::FP32,
absl::string_view signature_key = "",
bool use_legacy_string_model = false) {
params.Set<int32_t>("num_runs", num_runs);
params.Set<float>("min_secs", min_secs);
params.Set<float>("max_secs", max_secs);
// by default, simply use the fp32 one.
std::string graph_path = *g_fp32_model_path;
if (graph_type == ModelGraphType::INT8) {
graph_path = *g_int8_model_path;
} else if (graph_type == ModelGraphType::STRING) {
graph_path = use_legacy_string_model ? *g_string_model_path_no_signature
: *g_string_model_path;
} else if (!signature_key.empty()) {
graph_path = *g_multi_signature_model_path;
}
std::string fd_or_graph_path = graph_path;
#ifndef _WIN32
if (model_read_option == ModelReadOption::FROM_FD) {
int fd = open(graph_path.c_str(), O_RDONLY);
ASSERT_GE(fd, 0);
struct stat stat_buf = {0};
ASSERT_EQ(fstat(fd, &stat_buf), 0);
size_t model_size = stat_buf.st_size;
size_t model_offset = 0;
fd_or_graph_path =
absl::StrFormat("fd:%d:%zu:%zu", fd, model_offset, model_size);
}
#endif // !defined(_WIN32)
params.Set<std::string>("graph", fd_or_graph_path);
if (!signature_key.empty()) {
params.Set<std::string>("signature_to_run_for", std::string(signature_key));
}
}
BenchmarkParams InitializeParams() {
BenchmarkParams params = BenchmarkTfLiteModel::DefaultParams();
InitializeParams(params, /*num_runs=*/2, /*min_secs=*/1.0f,
/*max_secs=*/150.0f);
return params;
}
BenchmarkParams CreateFp32Params() {
BenchmarkParams params = BenchmarkTfLiteModel::DefaultParams();
InitializeParams(
params, /*num_runs=*/2, /*min_secs=*/1.0f, /*max_secs=*/150.0f,
/*model_read_option=*/ModelReadOption::FROM_PATH, ModelGraphType::FP32);
return params;
}
BenchmarkParams CreateInt8Params() {
BenchmarkParams params = BenchmarkTfLiteModel::DefaultParams();
InitializeParams(
params, /*num_runs=*/2, /*min_secs=*/1.0f, /*max_secs=*/150.0f,
/*model_read_option=*/ModelReadOption::FROM_PATH, ModelGraphType::INT8);
return params;
}
BenchmarkParams CreateStringParams() {
BenchmarkParams params = BenchmarkTfLiteModel::DefaultParams();
InitializeParams(
params, /*num_runs=*/2, /*min_secs=*/1.0f, /*max_secs=*/150.0f,
/*model_read_option=*/ModelReadOption::FROM_PATH, ModelGraphType::STRING);
return params;
}
BenchmarkParams CreateLegacyStringParams() {
BenchmarkParams params = BenchmarkTfLiteModel::DefaultParams();
InitializeParams(
params, /*num_runs=*/2, /*min_secs=*/1.0f, /*max_secs=*/150.0f,
/*model_read_option=*/ModelReadOption::FROM_PATH, ModelGraphType::STRING,
/*signature_key=*/"", /*use_legacy_string_model=*/true);
return params;
}
BenchmarkParams CreateStringFdParams() {
BenchmarkParams params = BenchmarkTfLiteModel::DefaultParams();
InitializeParams(
params, /*num_runs=*/2, /*min_secs=*/1.0f, /*max_secs=*/150.0f,
/*model_read_option=*/ModelReadOption::FROM_FD, ModelGraphType::STRING);
return params;
}
BenchmarkParams CreateMultiSignatureParams(std::string signature_key) {
BenchmarkParams params = BenchmarkTfLiteModel::DefaultParams();
InitializeParams(
params, /*num_runs=*/2, /*min_secs=*/1.0f, /*max_secs=*/150.0f,
/*model_read_option=*/ModelReadOption::FROM_PATH, ModelGraphType::FP32,
/*signature_key=*/signature_key);
return params;
}
BenchmarkParams CreateMultiSignatureParamsWithUnspecifiedSignature() {
BenchmarkParams params = BenchmarkTfLiteModel::DefaultParams();
InitializeParams(
params, /*num_runs=*/2, /*min_secs=*/1.0f, /*max_secs=*/150.0f,
/*model_read_option=*/ModelReadOption::FROM_PATH, ModelGraphType::FP32);
params.Set<std::string>("graph", *g_multi_signature_model_path);
return params;
}
std::string CreateFilePath(const std::string& file_name) {
const char* tmp_dir = getenv("TEST_TMPDIR");
return std::string(tmp_dir ? tmp_dir : "./") + file_name;
}
void WriteInputLayerValueFile(const std::string& file_path,
ModelGraphType graph_type, int num_elements,
char file_value = 'a') {
std::ofstream file(file_path);
int bytes = 0;
switch (graph_type) {
case ModelGraphType::FP32:
bytes = 4 * num_elements;
break;
case ModelGraphType::INT8:
bytes = num_elements;
break;
default:
LOG(WARNING) << absl::StrFormat(
"ModelGraphType(enum_value:%d) is not known.", graph_type);
LOG(WARNING) << "The size of the ModelGraphType will be 1 byte in tests.";
bytes = num_elements;
break;
}
std::vector<char> buffer(bytes, file_value);
file.write(buffer.data(), bytes);
}
void CheckInputTensorValue(const TfLiteTensor* input_tensor,
char expected_value) {
ASSERT_THAT(input_tensor, testing::NotNull());
EXPECT_TRUE(std::all_of(
input_tensor->data.raw, input_tensor->data.raw + input_tensor->bytes,
[expected_value](char c) { return c == expected_value; }));
}
void CheckInputTensorValue(const TfLiteTensor* input_tensor,
int tensor_dim_index,
const std::string& expected_value) {
StringRef tensor_value = GetString(input_tensor, tensor_dim_index);
EXPECT_TRUE(std::equal(tensor_value.str, tensor_value.str + tensor_value.len,
expected_value.c_str(),
expected_value.c_str() + expected_value.length()));
}
class TestBenchmark : public BenchmarkTfLiteModel {
public:
explicit TestBenchmark(BenchmarkParams params)
: BenchmarkTfLiteModel(std::move(params)) {}
const tflite::Interpreter* GetInterpreter() { return interpreter_.get(); }
void Prepare() {
PrepareInputData();
ResetInputsAndOutputs();
}
const TfLiteTensor* GetInputTensor(int index) {
return index >= interpreter_runner_->inputs().size()
? nullptr
: interpreter_runner_->tensor(
interpreter_runner_->inputs()[index]);
}
};
TEST(BenchmarkTest, DoesntCrashFp32Model) {
ASSERT_THAT(g_fp32_model_path, testing::NotNull());
TestBenchmark benchmark(CreateFp32Params());
benchmark.Run();
}
TEST(BenchmarkTest, DoesntCrashInt8Model) {
ASSERT_THAT(g_int8_model_path, testing::NotNull());
TestBenchmark benchmark(CreateInt8Params());
benchmark.Run();
}
TEST(BenchmarkTest, DoesntCrashStringModel) {
ASSERT_THAT(g_string_model_path, testing::NotNull());
TestBenchmark benchmark(CreateStringParams());
benchmark.Run();
}
TEST(BenchmarkTest, DoesntCrashStringLegacyModel) {
ASSERT_THAT(g_string_model_path_no_signature, testing::NotNull());
TestBenchmark benchmark(CreateLegacyStringParams());
benchmark.Run();
}
#ifndef _WIN32
TEST(BenchmarkTest, DoesntCrashStringModelWithFd) {
ASSERT_THAT(g_string_model_path, testing::NotNull());
TestBenchmark benchmark(CreateStringFdParams());
benchmark.Run();
}
#endif // !defined(_WIN32)
TEST(BenchmarkTest, DoesntCrashMultiSignatureModel) {
ASSERT_THAT(g_multi_signature_model_path, testing::NotNull());
TestBenchmark benchmark(CreateMultiSignatureParams("add"));
auto status = benchmark.Run();
EXPECT_EQ(kTfLiteOk, status);
TestBenchmark benchmark_sub(CreateMultiSignatureParams("sub"));
auto status_sub = benchmark_sub.Run();
EXPECT_EQ(kTfLiteOk, status_sub);
}
TEST(BenchmarkTest, MultiSignatureModelWithInvalidSignatureKeyFails) {
ASSERT_THAT(g_multi_signature_model_path, testing::NotNull());
TestBenchmark benchmark(CreateMultiSignatureParams("addisabbaba"));
auto status = benchmark.Run();
EXPECT_EQ(kTfLiteError, status);
}
TEST(BenchmarkTest, SingleSignatureModelWithInvalidSignatureKeyFails) {
ASSERT_THAT(g_string_model_path, testing::NotNull());
BenchmarkParams params = CreateStringParams();
params.Set<std::string>("signature_to_run_for", "invalid_signature_key");
TestBenchmark benchmark(std::move(params));
auto status = benchmark.Run();
EXPECT_EQ(kTfLiteError, status);
}
TEST(BenchmarkTest, MultiSignatureModelWithUnspecifiedSignatureKeyFails) {
ASSERT_THAT(g_multi_signature_model_path, testing::NotNull());
TestBenchmark benchmark(CreateMultiSignatureParamsWithUnspecifiedSignature());
auto status = benchmark.Run();
EXPECT_EQ(kTfLiteError, status);
}
TEST(BenchmarkTest, SplitInputLayerNameAndValueFile) {
std::vector<std::string> input_layer_value_files = {
"input:/tmp/input",
"input::0:/tmp/input",
"input::0::0:/tmp/input",
"input::::0:/tmp::input",
};
std::vector<std::pair<std::string, std::string>> expected = {
{"input", "/tmp/input"},
{"input:0", "/tmp/input"},
{"input:0:0", "/tmp/input"},
{"input::0", "/tmp:input"},
};
std::pair<std::string, std::string> name_file_pair;
for (int i = 0; i < input_layer_value_files.size(); ++i) {
SplitInputLayerNameAndValueFile(input_layer_value_files[i], name_file_pair);
EXPECT_EQ(name_file_pair.first, expected[i].first);
EXPECT_EQ(name_file_pair.second, expected[i].second);
}
EXPECT_EQ(SplitInputLayerNameAndValueFile("a:b:c", name_file_pair),
kTfLiteError);
EXPECT_EQ(SplitInputLayerNameAndValueFile("abc", name_file_pair),
kTfLiteError);
}
class TestMultiRunStatsRecorder : public MultiRunStatsRecorder {
public:
void OutputStats() override {
MultiRunStatsRecorder::OutputStats();
// Check results have been sorted according to avg. latency in increasing
// order, and the incomplete runs are at the back of the results.
double pre_avg_latency = -1e6;
bool has_incomplete = false; // ensure complete/incomplete are not mixed.
for (const auto& result : results_) {
const auto current_avg_latency = result.metrics.inference_time_us().avg();
if (result.completed) {
EXPECT_GE(current_avg_latency, pre_avg_latency);
EXPECT_FALSE(has_incomplete);
} else {
EXPECT_EQ(0, result.metrics.inference_time_us().count());
has_incomplete = true;
}
pre_avg_latency = current_avg_latency;
}
}
};
TEST(BenchmarkTest, DoesntCrashMultiPerfOptions) {
ASSERT_THAT(g_fp32_model_path, testing::NotNull());
TestBenchmark benchmark(CreateFp32Params());
BenchmarkPerformanceOptions all_options_benchmark(
&benchmark, std::make_unique<TestMultiRunStatsRecorder>());
all_options_benchmark.Run();
}
TEST(BenchmarkTest, DoesntCrashMultiPerfOptionsWithProfiling) {
ASSERT_THAT(g_fp32_model_path, testing::NotNull());
BenchmarkParams params = CreateFp32Params();
params.Set<bool>("enable_op_profiling", true);
TestBenchmark benchmark(std::move(params));
BenchmarkPerformanceOptions all_options_benchmark(&benchmark);
all_options_benchmark.Run();
}
TEST(BenchmarkTest, DoesntCrashWithExplicitInputFp32Model) {
ASSERT_THAT(g_fp32_model_path, testing::NotNull());
// Note: the following input-related params are *specific* to model
// 'g_fp32_model_path' which is specified as 'lite:testdata/multi_add.bin for
// the test.
BenchmarkParams params = CreateFp32Params();
params.Set<std::string>("input_layer", "a,b,c,d");
params.Set<std::string>("input_layer_shape",
"1,8,8,3:1,8,8,3:1,8,8,3:1,8,8,3");
params.Set<std::string>("input_layer_value_range", "d,1,10:b,0,100");
TestBenchmark benchmark(std::move(params));
benchmark.Run();
}
TEST(BenchmarkTest, DoesntCrashWithExplicitInputInt8Model) {
ASSERT_THAT(g_int8_model_path, testing::NotNull());
// Note: the following input-related params are *specific* to model
// 'g_int8_model_path' which is specified as
// 'lite:testdata/add_quantized_int8.bin for the test.
int a_min = 1;
int a_max = 10;
BenchmarkParams params = CreateInt8Params();
params.Set<std::string>("input_layer", "a");
params.Set<std::string>("input_layer_shape", "1,8,8,3");
params.Set<std::string>("input_layer_value_range",
absl::StrFormat("a,%d,%d", a_min, a_max));
TestBenchmark benchmark(std::move(params));
benchmark.Run();
auto input_tensor = benchmark.GetInputTensor(0);
ASSERT_THAT(input_tensor, testing::NotNull());
EXPECT_TRUE(std::all_of(
input_tensor->data.raw, input_tensor->data.raw + input_tensor->bytes,
[a_min, a_max](int i) { return a_min <= i && i <= a_max; }));
}
TEST(BenchmarkTest, DoesntCrashWithExplicitInputValueFilesFp32Model) {
ASSERT_THAT(g_fp32_model_path, testing::NotNull());
char file_value_b = 'b';
const std::string file_path_b = CreateFilePath("fp32_binary_b");
WriteInputLayerValueFile(file_path_b, ModelGraphType::FP32, 192,
file_value_b);
char file_value_d = 'd';
const std::string file_path_d = CreateFilePath("fp32_binary_d");
WriteInputLayerValueFile(file_path_d, ModelGraphType::FP32, 192,
file_value_d);
// Note: the following input-related params are *specific* to model
// 'g_fp32_model_path' which is specified as 'lite:testdata/multi_add.bin for
// the test.
BenchmarkParams params = CreateFp32Params();
params.Set<std::string>("input_layer", "a,b,c,d");
params.Set<std::string>("input_layer_shape",
"1,8,8,3:1,8,8,3:1,8,8,3:1,8,8,3");
params.Set<std::string>("input_layer_value_files",
"d:" + file_path_d + ",b:" + file_path_b);
TestBenchmark benchmark(std::move(params));
benchmark.Run();
CheckInputTensorValue(benchmark.GetInputTensor(1), file_value_b);
CheckInputTensorValue(benchmark.GetInputTensor(3), file_value_d);
}
TEST(BenchmarkTest, DoesntCrashWithExplicitInputValueFilesInt8Model) {
ASSERT_THAT(g_int8_model_path, testing::NotNull());
const std::string file_path = CreateFilePath("int8_binary");
char file_value = 'a';
WriteInputLayerValueFile(file_path, ModelGraphType::INT8, 192, file_value);
// Note: the following input-related params are *specific* to model
// 'g_int8_model_path' which is specified as
// 'lite:testdata/add_quantized_int8.bin for the test.
BenchmarkParams params = CreateInt8Params();
params.Set<std::string>("input_layer", "a");
params.Set<std::string>("input_layer_shape", "1,8,8,3");
params.Set<std::string>("input_layer_value_files", "a:" + file_path);
TestBenchmark benchmark(std::move(params));
benchmark.Run();
CheckInputTensorValue(benchmark.GetInputTensor(0), file_value);
}
TEST(BenchmarkTest, DoesntCrashWithExplicitInputValueFilesMultiSignatureModel) {
ASSERT_THAT(g_multi_signature_model_path, testing::NotNull());
const std::string file_path_add =
CreateFilePath("multi_signature_binary_add");
char file_value_add = 'a';
WriteInputLayerValueFile(file_path_add, ModelGraphType::FP32, 192,
file_value_add);
// Note: the following input-related params are *specific* to model
// 'g_multi_signature_model_path' which is specified as
// 'lite:testdata/add_quantized_int8.bin for the test.
BenchmarkParams params = CreateMultiSignatureParams("add");
params.Set<std::string>("input_layer", "x");
params.Set<std::string>("input_layer_shape", "192");
params.Set<std::string>("input_layer_value_files", "x:" + file_path_add);
TestBenchmark benchmark(std::move(params));
benchmark.Run();
CheckInputTensorValue(benchmark.GetInputTensor(0), file_value_add);
const std::string file_path_sub =
CreateFilePath("multi_signature_binary_sub");
char file_value_sub = 'z';
WriteInputLayerValueFile(file_path_sub, ModelGraphType::FP32, 192,
file_value_sub);
// Note: the following input-related params are *specific* to model
// 'g_multi_signature_model_path' which is specified as
// 'lite:testdata/add_quantized_int8.bin for the test.
BenchmarkParams params_2 = CreateMultiSignatureParams("sub");
params_2.Set<std::string>("input_layer", "x");
params_2.Set<std::string>("input_layer_shape", "192");
params_2.Set<std::string>("input_layer_value_files", "x:" + file_path_sub);
TestBenchmark benchmark_2(std::move(params_2));
benchmark_2.Run();
CheckInputTensorValue(benchmark_2.GetInputTensor(0), file_value_sub);
}
TEST(BenchmarkTest, DoesntCrashWithExplicitInputValueFilesStringModel) {
ASSERT_THAT(g_string_model_path, testing::NotNull());
const std::string file_path = CreateFilePath("string_binary");
const std::string string_value_0 = "abcd";
const std::string string_value_1 = "12345";
const std::string string_value_2 = "a1b2c3d4e5";
std::ofstream file(file_path);
// Store the terminating null-character ('\0') at the end of the returned
// value by std::string::c_str().
file.write(string_value_0.c_str(), string_value_0.length() + 1);
file.write(string_value_1.c_str(), string_value_1.length() + 1);
file.write(string_value_2.c_str(), string_value_2.length() + 1);
file.close();
// Note: the following input-related params are *specific* to model
// 'g_string_model_path' which is specified as
// 'lite:testdata/string_input_model.bin for the test.
BenchmarkParams params = CreateStringParams();
params.Set<std::string>("input_layer", "a");
params.Set<std::string>("input_layer_shape", "1,3");
params.Set<std::string>("input_layer_value_files", "a:" + file_path);
TestBenchmark benchmark(std::move(params));
benchmark.Run();
auto input_tensor = benchmark.GetInputTensor(0);
ASSERT_THAT(input_tensor, testing::NotNull());
EXPECT_EQ(GetStringCount(input_tensor), 3);
CheckInputTensorValue(input_tensor, 0, string_value_0);
CheckInputTensorValue(input_tensor, 1, string_value_1);
CheckInputTensorValue(input_tensor, 2, string_value_2);
}
class ScopedCommandlineArgs {
public:
explicit ScopedCommandlineArgs(const std::vector<std::string>& actual_args) {
argc_ = actual_args.size() + 1;
argv_ = new char*[argc_];
const std::string program_name = "benchmark_model";
int buffer_size = program_name.length() + 1;
for (const auto& arg : actual_args) buffer_size += arg.length() + 1;
buffer_ = new char[buffer_size];
auto next_start = program_name.copy(buffer_, program_name.length());
buffer_[next_start++] = '\0';
argv_[0] = buffer_;
for (int i = 0; i < actual_args.size(); ++i) {
const auto& arg = actual_args[i];
argv_[i + 1] = buffer_ + next_start;
next_start += arg.copy(argv_[i + 1], arg.length());
buffer_[next_start++] = '\0';
}
}
~ScopedCommandlineArgs() {
delete[] argv_;
delete[] buffer_;
}
int argc() const { return argc_; }
char** argv() const { return argv_; }
private:
char* buffer_; // the buffer for all arguments.
int argc_;
char** argv_; // Each char* element points to each argument.
};
TEST(BenchmarkTest, RunWithCorrectFlags) {
ASSERT_THAT(g_fp32_model_path, testing::NotNull());
TestBenchmark benchmark(CreateFp32Params());
ScopedCommandlineArgs scoped_argv({"--num_threads=4"});
auto status = benchmark.Run(scoped_argv.argc(), scoped_argv.argv());
EXPECT_EQ(kTfLiteOk, status);
}
TEST(BenchmarkTest, RunWithWrongFlags) {
ASSERT_THAT(g_fp32_model_path, testing::NotNull());
TestBenchmark benchmark(CreateFp32Params());
ScopedCommandlineArgs scoped_argv({"--num_threads=str"});
auto status = benchmark.Run(scoped_argv.argc(), scoped_argv.argv());
EXPECT_EQ(kTfLiteError, status);
}
TEST(BenchmarkTest, RunWithUseCaching) {
ASSERT_THAT(g_fp32_model_path, testing::NotNull());
TestBenchmark benchmark(CreateFp32Params());
ScopedCommandlineArgs scoped_argv({"--use_caching=false"});
auto status = benchmark.Run(scoped_argv.argc(), scoped_argv.argv());
EXPECT_EQ(kTfLiteOk, status);
}
class MaxDurationWorksTestListener : public BenchmarkListener {
void OnBenchmarkEnd(const BenchmarkResults& results) override {
const int64_t num_actual_runs = results.inference_time_us().count();
TFLITE_LOG(INFO) << "number of actual runs: " << num_actual_runs;
EXPECT_GE(num_actual_runs, 1);
EXPECT_LT(num_actual_runs, 100000000);
}
};
TEST(BenchmarkTest, MaxDurationWorks) {
ASSERT_THAT(g_fp32_model_path, testing::NotNull());
BenchmarkParams params = BenchmarkTfLiteModel::DefaultParams();
InitializeParams(params, 100000000 /* num_runs */, 1000000.0f /* min_secs */,
0.001f /* max_secs */);
TestBenchmark benchmark(std::move(params));
MaxDurationWorksTestListener listener;
benchmark.AddListener(&listener);
benchmark.Run();
}
TEST(BenchmarkTest, ParametersArePopulatedWhenInputShapeIsNotSpecified) {
ASSERT_THAT(g_fp32_model_path, testing::NotNull());
TestBenchmark benchmark(InitializeParams());
benchmark.Init();
benchmark.Prepare();
auto interpreter = benchmark.GetInterpreter();
auto inputs = interpreter->inputs();
ASSERT_GE(inputs.size(), 1);
auto input_tensor = interpreter->tensor(inputs[0]);
// Copy input tensor to a vector
std::vector<char> input_bytes(input_tensor->data.raw,
input_tensor->data.raw + input_tensor->bytes);
benchmark.Prepare();
// Expect data is not the same.
EXPECT_EQ(input_bytes.size(), input_tensor->bytes);
EXPECT_FALSE(std::equal(input_bytes.begin(), input_bytes.end(),
input_tensor->data.raw,
input_tensor->data.raw + input_tensor->bytes));
}
TEST(BenchmarkTest, InitializationFailedWhenInvalidGraphPathIsProvided) {
BenchmarkParams params = BenchmarkTfLiteModel::DefaultParams();
params.Set<std::string>("graph", "invalid/path");
TestBenchmark benchmark(std::move(params));
EXPECT_EQ(benchmark.Init(), kTfLiteError);
}
TEST(BenchmarkTest, InitializationFailedWhenInvalidGraphFdIsProvided) {
BenchmarkParams params = BenchmarkTfLiteModel::DefaultParams();
params.Set<std::string>("graph", "fd:file:descriptor");
TestBenchmark benchmark(std::move(params));
EXPECT_EQ(benchmark.Init(), kTfLiteError);
}
class TestBenchmarkListener : public BenchmarkListener {
public:
void OnBenchmarkEnd(const BenchmarkResults& results) override {
results_ = results;
}
const BenchmarkResults& results() const { return results_; }
private:
BenchmarkResults results_;
};
TEST(BenchmarkTest, BenchmarkResultFileIsWritten) {
ASSERT_THAT(g_fp32_model_path, testing::NotNull());
BenchmarkParams params = BenchmarkTfLiteModel::DefaultParams();
std::string result_file_path = "/tmp/result.txtproto";
#if defined(__ANDROID__)
result_file_path = "/data/local/tmp/result.txtproto";
#endif
params.Set<std::string>("result_file_path", result_file_path);
params.Set<bool>("report_peak_memory_footprint", true);
params.Set<std::string>("graph", *g_fp32_model_path);
TestBenchmark benchmark(std::move(params));
TestBenchmarkListener listener;
benchmark.AddListener(&listener);
benchmark.Run();
std::ifstream in_file(result_file_path, std::ios::binary | std::ios::in);
tflite::tools::benchmark::BenchmarkResult result;
result.ParseFromIstream(&in_file);
// Verify latency metrics.
EXPECT_FLOAT_EQ(result.latency_metrics().init_ms(),
listener.results().startup_latency_us() / 1000.0);
EXPECT_FLOAT_EQ(result.latency_metrics().first_inference_ms(),
listener.results().warmup_time_us().first() / 1000.0);
EXPECT_FLOAT_EQ(result.latency_metrics().average_warm_up_ms(),
listener.results().warmup_time_us().avg() / 1000.0);
EXPECT_FLOAT_EQ(result.latency_metrics().avg_ms(),
listener.results().inference_time_us().avg() / 1000.0);
EXPECT_FLOAT_EQ(result.latency_metrics().min_ms(),
listener.results().inference_time_us().min() / 1000.0);
EXPECT_FLOAT_EQ(result.latency_metrics().max_ms(),
listener.results().inference_time_us().max() / 1000.0);
EXPECT_FLOAT_EQ(
result.latency_metrics().stddev_ms(),
listener.results().inference_time_us().std_deviation() / 1000.0);
EXPECT_FLOAT_EQ(
result.latency_metrics().median_ms(),
listener.results().inference_time_us().percentile(50) / 1000.0);
EXPECT_FLOAT_EQ(
result.latency_metrics().p95_ms(),
listener.results().inference_time_us().percentile(95) / 1000.0);
EXPECT_FLOAT_EQ(
result.latency_metrics().p5_ms(),
listener.results().inference_time_us().percentile(5) / 1000.0);
// Verify memory metrics.
EXPECT_EQ(result.memory_metrics().init_footprint_kb(),
listener.results().init_mem_usage().mem_footprint_kb);
EXPECT_EQ(result.memory_metrics().overall_footprint_kb(),
listener.results().overall_mem_usage().mem_footprint_kb);
EXPECT_EQ(result.memory_metrics().has_peak_mem_mb(), true);
// Verify misc metrics.
EXPECT_FLOAT_EQ(result.misc_metrics().model_size_mb(),
listener.results().model_size_mb());
EXPECT_EQ(result.misc_metrics().num_runs(),
listener.results().inference_time_us().count());
EXPECT_EQ(result.misc_metrics().num_warmup_runs(),
listener.results().warmup_time_us().count());
EXPECT_FLOAT_EQ(result.misc_metrics().model_throughput_in_mb_per_sec(),
listener.results().throughput_MB_per_second());
}
} // namespace
} // namespace benchmark
} // namespace tflite
int main(int argc, char** argv) {
std::string fp32_model_path, int8_model_path, string_model_path,
string_model_path_with_no_signature, multi_signature_model_path;
std::vector<tflite::Flag> flags = {
tflite::Flag::CreateFlag("fp32_graph", &fp32_model_path,
"Path to a fp32 model file."),
tflite::Flag::CreateFlag("int8_graph", &int8_model_path,
"Path to a int8 model file."),
tflite::Flag::CreateFlag("string_graph_with_signature",
&string_model_path,
"Path to a string model file with a signature."),
tflite::Flag::CreateFlag(
"string_graph_without_signature",
&string_model_path_with_no_signature,
"Path to a string model file without signatures."),
tflite::Flag::CreateFlag("multi_signature_graph",
&multi_signature_model_path,
"Path to a multi-signature model file."),
};
g_fp32_model_path = &fp32_model_path;
g_int8_model_path = &int8_model_path;
g_string_model_path = &string_model_path;
g_multi_signature_model_path = &multi_signature_model_path;
g_string_model_path_no_signature = &string_model_path_with_no_signature;
const bool parse_result =
tflite::Flags::Parse(&argc, const_cast<const char**>(argv), flags);
if (!parse_result) {
std::cerr << tflite::Flags::Usage(argv[0], flags);
return 1;
}
::tflite::LogToStderr();
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc
+1402
View File
File diff suppressed because it is too large Load Diff
tensorflow/lite/tools/benchmark/benchmark_tflite_model.h
+224
View File
@@ -0,0 +1,224 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_TFLITE_MODEL_H_
#define TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_TFLITE_MODEL_H_
#include <algorithm>
#include <cstdint>
#include <map>
#include <memory>
#include <random>
#include <string>
#include <utility>
#include <vector>
#include "tensorflow/lite/core/model.h"
#include "tensorflow/lite/core/subgraph.h"
#include "tensorflow/lite/profiling/profiler.h"
#include "tensorflow/lite/signature_runner.h"
#include "tensorflow/lite/tools/benchmark/benchmark_model.h"
#include "tensorflow/lite/tools/model_loader.h"
#include "tensorflow/lite/tools/utils.h"
namespace tflite {
namespace benchmark {
// Splits the input_layer_name and input_layer_value_files and stores them in
// the name_file_pair. In the case of failures, return an error status, and the
// the state of name_file_pair is unchanged.
//
// BenchmarkTfLiteModel takes --input_layer_value_files flag, which is a comma-
// separated list of input_layer_name:input_value_file_path pairs,
// e.g. input1:/tmp/path.
//
// As TensorFlow allows ':' in the tensor names (e.g. input:0 to denote the
// output index), having ':' as the delimiter can break the benchmark code
// unexpectedly. To avoid this issue, we allow escaping ':' char with '::' for
// this particular flag only. This function handles splitting the name and file
// path that contains escaped colon.
//
// For example, "input::0:/tmp/path" will be divided into input:0 and /tmp/path.
TfLiteStatus SplitInputLayerNameAndValueFile(
const std::string& name_and_value_file,
std::pair<std::string, std::string>& name_file_pair);
// Provides a simplified interface to work with the interpreter and signature
// runner, automatically selecting the appropriate one based on whether a
// signature is specified.
class BenchmarkInterpreterRunner {
public:
BenchmarkInterpreterRunner(tflite::Interpreter* const interpreter,
tflite::SignatureRunner* const signature_runner,
tflite::Subgraph* const subgraph)
: interpreter_(interpreter), subgraph_(subgraph) {
if (signature_runner != nullptr) {
signature_runner_.reset(signature_runner);
}
}
~BenchmarkInterpreterRunner() {
if (signature_runner_ != nullptr) {
signature_runner_.release();
}
}
// Creates a BenchmarkInterpreterRunner for the given interpreter. If a
// signature key is specified, the signature runner is used. Otherwise, the
// interpreter is used.
static std::pair<TfLiteStatus, std::unique_ptr<BenchmarkInterpreterRunner>>
Create(tflite::Interpreter* interpreter, std::string signature_key);
// Updates allocations for all tensors, related to the given signature.
TfLiteStatus AllocateTensors();
// Invokes the interpreter or signature runner (run the graph identified by
// the given signature in dependency order).
TfLiteStatus Invoke();
// Return vector of node indices in the order of execution.
//
// This is a list of node indices (to index into nodes_and_registration).
// This represents a valid topological sort (dependency ordered) execution
// plan. In particular, it is valid for this ordering to contain only a
// subset of the node indices.
// Warning: This is an experimental API and subject to change.
const std::vector<int>& execution_plan() const;
// Read only access to list of inputs.
//
// Array of indices representing the tensors that are inputs to the
// interpreter.
// Warning: This is an experimental API and subject to change.
const std::vector<int>& inputs() const;
// Read only access to list of outputs.
//
// Array of indices representing the tensors that are outputs to the
// interpreter.
// Warning: This is an experimental API and subject to change.
const std::vector<int>& outputs() const;
// Get a mutable tensor data structure via index.
// Warning: This is an experimental API and subject to change.
TfLiteTensor* tensor(int tensor_index);
// Get a pointer to an operation and registration data structure if in
// bounds of the signature subgraph.
// Warning: This is an experimental API and subject to change.
const std::pair<TfLiteNode, TfLiteRegistration>* node_and_registration(
int node_index) const;
// Change the dimensionality of a given tensor. Note, this is only acceptable
// for tensor indices that are inputs or variables.
// Returns status of failure or success. Note that this doesn't actually
// resize any existing buffers. A call to AllocateTensors() is required to
// change the tensor input buffer.
TfLiteStatus ResizeInputTensor(int tensor_index,
const std::vector<int>& new_size);
private:
BenchmarkInterpreterRunner() = delete;
tflite::Interpreter* const interpreter_ = nullptr;
std::unique_ptr<tflite::SignatureRunner> signature_runner_;
tflite::Subgraph* const subgraph_ = nullptr;
};
// Benchmarks a TFLite model by running tflite interpreter.
class BenchmarkTfLiteModel : public BenchmarkModel {
public:
struct InputLayerInfo {
InputLayerInfo() : has_value_range(false), low(0), high(0) {}
std::string name;
std::vector<int> shape;
// The input value is randomly generated when benchmarking the NN model.
// However, the NN model might require the value be limited to a certain
// range [low, high] for this particular input layer. For simplicity,
// support integer value first.
bool has_value_range;
int low;
int high;
// The input value will be loaded from 'input_file_path' INSTEAD OF being
// randomly generated. Note the input file will be opened in binary mode.
std::string input_file_path;
};
explicit BenchmarkTfLiteModel(BenchmarkParams params = DefaultParams());
~BenchmarkTfLiteModel() override;
std::vector<Flag> GetFlags() override;
void LogParams() override;
TfLiteStatus ValidateParams() override;
uint64_t ComputeInputBytes() override;
TfLiteStatus Init() override;
TfLiteStatus RunImpl() override;
static BenchmarkParams DefaultParams();
protected:
TfLiteStatus PrepareInputData() override;
TfLiteStatus ResetInputsAndOutputs() override;
int64_t MayGetModelFileSize() override;
virtual TfLiteStatus LoadModel();
// Allow subclasses to create a customized Op resolver during init.
virtual std::unique_ptr<tflite::OpResolver> GetOpResolver() const;
// Allow subclass to initialize a customized tflite interpreter.
virtual TfLiteStatus InitInterpreter();
// Create a BenchmarkListener that's specifically for TFLite profiling if
// necessary.
virtual std::unique_ptr<BenchmarkListener> MayCreateProfilingListener() const;
void CleanUp();
utils::InputTensorData LoadInputTensorData(
const TfLiteTensor& t, const std::string& input_file_path);
std::vector<InputLayerInfo> inputs_;
std::vector<utils::InputTensorData> inputs_data_;
std::unique_ptr<tflite::FlatBufferModel> model_;
std::unique_ptr<tflite::Interpreter> interpreter_;
std::unique_ptr<BenchmarkInterpreterRunner> interpreter_runner_;
std::unique_ptr<tflite::ExternalCpuBackendContext> external_context_;
private:
utils::InputTensorData CreateRandomTensorData(
const TfLiteTensor& t, const InputLayerInfo* layer_info);
void AddOwnedListener(std::unique_ptr<BenchmarkListener> listener) {
if (listener == nullptr) return;
owned_listeners_.emplace_back(std::move(listener));
AddListener(owned_listeners_.back().get());
}
std::vector<std::unique_ptr<BenchmarkListener>> owned_listeners_;
std::mt19937 random_engine_;
std::vector<Interpreter::TfLiteDelegatePtr> owned_delegates_;
// Always TFLITE_LOG the benchmark result.
BenchmarkLoggingListener log_output_;
std::unique_ptr<tools::ModelLoader> model_loader_;
};
} // namespace benchmark
} // namespace tflite
#endif // TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_TFLITE_MODEL_H_
tensorflow/lite/tools/benchmark/benchmark_tflite_model_test.cc
+96
View File
@@ -0,0 +1,96 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/lite/tools/benchmark/benchmark_tflite_model.h"
#include <fcntl.h>
#include <sys/stat.h>
#include <string>
#include <utility>
#include <gtest/gtest.h>
#include "absl/strings/str_cat.h"
#include "tensorflow/lite/core/c/c_api_types.h"
#include "tensorflow/lite/tools/benchmark/benchmark_model.h"
#include "tensorflow/lite/tools/benchmark/benchmark_params.h"
#include "tensorflow/lite/tools/tool_params.h"
namespace tflite {
namespace benchmark {
namespace {
static constexpr char kModelPath[] =
"../tflite_mobilenet_float/"
"mobilenet_v1_1.0_224.tflite";
class TestBenchmarkListener : public BenchmarkListener {
public:
void OnBenchmarkEnd(const BenchmarkResults& results) override {
results_ = results;
}
BenchmarkResults results_;
};
TEST(BenchmarkTfLiteModelTest, GetModelSizeFromPathSucceeded) {
BenchmarkParams params = BenchmarkTfLiteModel::DefaultParams();
params.Set<std::string>("graph", kModelPath);
params.Set<int>("num_runs", 1);
params.Set<int>("warmup_runs", 0);
BenchmarkTfLiteModel benchmark = BenchmarkTfLiteModel(std::move(params));
TestBenchmarkListener listener;
benchmark.AddListener(&listener);
benchmark.Run();
EXPECT_GE(listener.results_.model_size_mb(), 0);
}
TEST(BenchmarkTfLiteModelTest, GetModelSizeFromFileDescriptorSucceeded) {
BenchmarkParams params = BenchmarkTfLiteModel::DefaultParams();
int fd = open(kModelPath, O_RDONLY);
ASSERT_GE(fd, 0);
int model_offset = 0;
struct stat stat_buf = {0};
ASSERT_EQ(fstat(fd, &stat_buf), 0);
params.Set<std::string>("graph", absl::StrCat("fd:", fd, ":", model_offset,
":", stat_buf.st_size));
params.Set<int>("num_runs", 1);
params.Set<int>("warmup_runs", 0);
BenchmarkTfLiteModel benchmark = BenchmarkTfLiteModel(std::move(params));
TestBenchmarkListener listener;
benchmark.AddListener(&listener);
benchmark.Run();
EXPECT_EQ(listener.results_.model_size_mb(), stat_buf.st_size / 1e6);
}
TEST(BenchmarkTfLiteModelTest, ResizeInputWithDelegate) {
BenchmarkParams params = BenchmarkTfLiteModel::DefaultParams();
params.Set<std::string>("graph", kModelPath);
params.Set<bool>("use_xnnpack", true);
params.Set<std::string>("input_layer", "input_87");
params.Set<std::string>("input_layer_shape", "2,224,224,3");
BenchmarkTfLiteModel benchmark = BenchmarkTfLiteModel(std::move(params));
EXPECT_EQ(benchmark.Run(), kTfLiteOk);
}
} // namespace
} // namespace benchmark
} // namespace tflite
tensorflow/lite/tools/benchmark/benchmark_tflite_performance_options_main.cc
+35
View File
@@ -0,0 +1,35 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <cstdlib>
#include "tensorflow/lite/tools/benchmark/benchmark_performance_options.h"
#include "tensorflow/lite/tools/benchmark/benchmark_tflite_model.h"
#include "tensorflow/lite/tools/logging.h"
namespace tflite {
namespace benchmark {
int Main(int argc, char** argv) {
TFLITE_LOG(INFO) << "STARTING!";
BenchmarkTfLiteModel benchmark;
BenchmarkPerformanceOptions all_options_benchmark(&benchmark);
all_options_benchmark.Run(argc, argv);
return EXIT_SUCCESS;
}
} // namespace benchmark
} // namespace tflite
int main(int argc, char** argv) { return tflite::benchmark::Main(argc, argv); }
tensorflow/lite/tools/benchmark/benchmark_utils.cc
+39
View File
@@ -0,0 +1,39 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/lite/tools/benchmark/benchmark_utils.h"
#include <cstdint>
#include "tensorflow/lite/profiling/time.h"
namespace tflite {
namespace benchmark {
namespace util {
void SleepForSeconds(double sleep_seconds) {
if (sleep_seconds <= 0.0) {
return;
}
// If requested, sleep between runs for an arbitrary amount of time.
// This can be helpful to determine the effect of mobile processor
// scaling and thermal throttling.
tflite::profiling::time::SleepForMicros(
static_cast<uint64_t>(sleep_seconds * 1e6));
}
} // namespace util
} // namespace benchmark
} // namespace tflite
tensorflow/lite/tools/benchmark/benchmark_utils.h
+52
View File
@@ -0,0 +1,52 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_UTILS_H_
#define TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_UTILS_H_
#include <sstream>
#include <string>
#include <vector>
namespace tflite {
namespace benchmark {
namespace util {
// A convenient function that wraps tflite::profiling::time::SleepForMicros and
// simply return if 'sleep_seconds' is negative.
void SleepForSeconds(double sleep_seconds);
// Split the 'str' according to 'delim', and store each splitted element into
// 'values'.
template <typename T>
bool SplitAndParse(const std::string& str, char delim, std::vector<T>* values) {
std::istringstream input(str);
for (std::string line; std::getline(input, line, delim);) {
std::istringstream to_parse(line);
T val;
to_parse >> val;
if (!to_parse.eof() && !to_parse.good()) {
return false;
}
values->emplace_back(val);
}
return true;
}
} // namespace util
} // namespace benchmark
} // namespace tflite
#endif // TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_UTILS_H_
tensorflow/lite/tools/benchmark/benchmark_utils_test.cc
+79
View File
@@ -0,0 +1,79 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/lite/tools/benchmark/benchmark_utils.h"
#include <string>
#include <vector>
#include <gtest/gtest.h>
#include "tensorflow/lite/profiling/time.h"
namespace tflite {
namespace benchmark {
namespace {
TEST(BenchmarkHelpersTest, SleepForNegativeSeconds) {
const auto start_ts = tflite::profiling::time::NowMicros();
// The following should return immediately.
util::SleepForSeconds(-5.0);
const auto end_ts = tflite::profiling::time::NowMicros();
// As we don't have a mocked clock, we simply expect <1 sec has elapsed, which
// is admittedly not quite accurate.
EXPECT_LT(end_ts - start_ts, 1000000);
}
TEST(BenchmarkHelpersTest, SleepForSomeSeconds) {
const auto start_ts = tflite::profiling::time::NowMicros();
// The following should return after 2.0 secs
util::SleepForSeconds(2.0);
const auto end_ts = tflite::profiling::time::NowMicros();
// As we don't have a mocked clock, we simply expect >1.9 sec has elapsed.
EXPECT_GT(end_ts - start_ts, 1900000);
}
TEST(BenchmarkHelpersTest, SplitAndParseFailed) {
std::vector<int> results;
const bool splitted = util::SplitAndParse("hello;world", ';', &results);
EXPECT_FALSE(splitted);
}
TEST(BenchmarkHelpersTest, SplitAndParseString) {
std::vector<std::string> results;
const bool splitted = util::SplitAndParse("hello,world", ',', &results);
EXPECT_TRUE(splitted);
EXPECT_EQ(2, results.size());
EXPECT_EQ("hello", results[0]);
EXPECT_EQ("world", results[1]);
}
TEST(BenchmarkHelpersTest, SplitAndParseInts) {
std::vector<int> results;
const bool splitted = util::SplitAndParse("1,2", ',', &results);
EXPECT_TRUE(splitted);
EXPECT_EQ(2, results.size());
EXPECT_EQ(1, results[0]);
EXPECT_EQ(2, results[1]);
}
} // namespace
} // namespace benchmark
} // namespace tflite
tensorflow/lite/tools/benchmark/experimental/c/BUILD
+40
View File
@@ -0,0 +1,40 @@
load("@rules_cc//cc:cc_library.bzl", "cc_library")
load("//tensorflow/lite:build_def.bzl", "tflite_copts")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
default_visibility = ["benchmark"],
licenses = ["notice"],
)
package_group(
name = "benchmark",
packages = [
"//tensorflow/lite/tools/benchmark/...",
],
)
exports_files(
["benchmark_c_api.h"],
visibility = ["//tensorflow/lite/tools/benchmark/experimental/c:benchmark"],
)
cc_library(
name = "benchmark_c_api",
srcs = ["benchmark_c_api.cc"],
hdrs = [
"benchmark_c_api.h",
],
copts = tflite_copts(),
visibility = [
"benchmark",
],
deps = [
"//tensorflow/lite/c:c_api_types",
"//tensorflow/lite/core/c:c_api_types",
"//tensorflow/lite/tools/benchmark:benchmark_model_lib",
"//tensorflow/lite/tools/benchmark:benchmark_params",
"//tensorflow/lite/tools/benchmark:benchmark_tflite_model_lib",
"@xla//xla/tsl/util:stats_calculator_portable",
],
)
tensorflow/lite/tools/benchmark/experimental/c/benchmark_c_api.cc
+187
View File
@@ -0,0 +1,187 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/lite/tools/benchmark/experimental/c/benchmark_c_api.h"
#include <cstdint>
#include <memory>
#include <utility>
#include "xla/tsl/util/stats_calculator.h"
#include "tensorflow/lite/c/c_api_types.h"
#include "tensorflow/lite/tools/benchmark/benchmark_model.h"
#include "tensorflow/lite/tools/benchmark/benchmark_params.h"
#include "tensorflow/lite/tools/benchmark/benchmark_tflite_model.h"
extern "C" {
// -----------------------------------------------------------------------------
// C APIs corresponding to tflite::benchmark::BenchmarkResults type.
// -----------------------------------------------------------------------------
struct TfLiteBenchmarkResults {
const tflite::benchmark::BenchmarkResults* results;
};
// Converts the given int64_t stat into a TfLiteBenchmarkInt64Stat struct.
TfLiteBenchmarkInt64Stat ConvertStat(const tsl::Stat<int64_t>& stat) {
return {
stat.empty(), stat.first(), stat.newest(), stat.max(),
stat.min(), stat.count(), stat.sum(), stat.squared_sum(),
stat.all_same(), stat.avg(), stat.std_deviation(),
};
}
TfLiteBenchmarkInt64Stat TfLiteBenchmarkResultsGetInferenceTimeMicroseconds(
const TfLiteBenchmarkResults* results) {
return ConvertStat(results->results->inference_time_us());
}
TfLiteBenchmarkInt64Stat TfLiteBenchmarkResultsGetWarmupTimeMicroseconds(
const TfLiteBenchmarkResults* results) {
return ConvertStat(results->results->warmup_time_us());
}
int64_t TfLiteBenchmarkResultsGetStartupLatencyMicroseconds(
const TfLiteBenchmarkResults* results) {
return results->results->startup_latency_us();
}
uint64_t TfLiteBenchmarkResultsGetInputBytes(
const TfLiteBenchmarkResults* results) {
return results->results->input_bytes();
}
double TfLiteBenchmarkResultsGetThroughputMbPerSecond(
const TfLiteBenchmarkResults* results) {
return results->results->throughput_MB_per_second();
}
// -----------------------------------------------------------------------------
// C APIs corresponding to tflite::benchmark::BenchmarkListener type.
// -----------------------------------------------------------------------------
class BenchmarkListenerAdapter : public tflite::benchmark::BenchmarkListener {
public:
void OnBenchmarkStart(
const tflite::benchmark::BenchmarkParams& params) override {
if (on_benchmark_start_fn_ != nullptr) {
on_benchmark_start_fn_(user_data_);
}
}
void OnSingleRunStart(tflite::benchmark::RunType runType) override {
if (on_single_run_start_fn_ != nullptr) {
on_single_run_start_fn_(user_data_, runType == tflite::benchmark::WARMUP
? TfLiteBenchmarkWarmup
: TfLiteBenchmarkRegular);
}
}
void OnSingleRunEnd() override {
if (on_single_run_end_fn_ != nullptr) {
on_single_run_end_fn_(user_data_);
}
}
void OnBenchmarkEnd(
const tflite::benchmark::BenchmarkResults& results) override {
if (on_benchmark_end_fn_ != nullptr) {
TfLiteBenchmarkResults* wrapper = new TfLiteBenchmarkResults{&results};
on_benchmark_end_fn_(user_data_, wrapper);
delete wrapper;
}
}
// Keep the user_data pointer provided when setting the callbacks.
void* user_data_;
// Function pointers set by the TfLiteBenchmarkListenerSetCallbacks call.
// Only non-null callbacks will be actually called.
void (*on_benchmark_start_fn_)(void* user_data);
void (*on_single_run_start_fn_)(void* user_data,
TfLiteBenchmarkRunType runType);
void (*on_single_run_end_fn_)(void* user_data);
void (*on_benchmark_end_fn_)(void* user_data,
TfLiteBenchmarkResults* results);
};
struct TfLiteBenchmarkListener {
std::unique_ptr<BenchmarkListenerAdapter> adapter;
};
TfLiteBenchmarkListener* TfLiteBenchmarkListenerCreate() {
std::unique_ptr<BenchmarkListenerAdapter> adapter =
std::make_unique<BenchmarkListenerAdapter>();
return new TfLiteBenchmarkListener{std::move(adapter)};
}
void TfLiteBenchmarkListenerDelete(TfLiteBenchmarkListener* listener) {
delete listener;
}
void TfLiteBenchmarkListenerSetCallbacks(
TfLiteBenchmarkListener* listener, void* user_data,
void (*on_benchmark_start_fn)(void* user_data),
void (*on_single_run_start_fn)(void* user_data,
TfLiteBenchmarkRunType runType),
void (*on_single_run_end_fn)(void* user_data),
void (*on_benchmark_end_fn)(void* user_data,
TfLiteBenchmarkResults* results)) {
listener->adapter->user_data_ = user_data;
listener->adapter->on_benchmark_start_fn_ = on_benchmark_start_fn;
listener->adapter->on_single_run_start_fn_ = on_single_run_start_fn;
listener->adapter->on_single_run_end_fn_ = on_single_run_end_fn;
listener->adapter->on_benchmark_end_fn_ = on_benchmark_end_fn;
}
// -----------------------------------------------------------------------------
// C APIs corresponding to tflite::benchmark::BenchmarkTfLiteModel type.
// -----------------------------------------------------------------------------
struct TfLiteBenchmarkTfLiteModel {
std::unique_ptr<tflite::benchmark::BenchmarkTfLiteModel> benchmark_model;
};
TfLiteBenchmarkTfLiteModel* TfLiteBenchmarkTfLiteModelCreate() {
std::unique_ptr<tflite::benchmark::BenchmarkTfLiteModel> benchmark_model(
new tflite::benchmark::BenchmarkTfLiteModel());
return new TfLiteBenchmarkTfLiteModel{std::move(benchmark_model)};
}
void TfLiteBenchmarkTfLiteModelDelete(
TfLiteBenchmarkTfLiteModel* benchmark_model) {
delete benchmark_model;
}
TfLiteStatus TfLiteBenchmarkTfLiteModelInit(
TfLiteBenchmarkTfLiteModel* benchmark_model) {
return benchmark_model->benchmark_model->Init();
}
TfLiteStatus TfLiteBenchmarkTfLiteModelRun(
TfLiteBenchmarkTfLiteModel* benchmark_model) {
return benchmark_model->benchmark_model->Run();
}
TfLiteStatus TfLiteBenchmarkTfLiteModelRunWithArgs(
TfLiteBenchmarkTfLiteModel* benchmark_model, int argc, char** argv) {
return benchmark_model->benchmark_model->Run(argc, argv);
}
void TfLiteBenchmarkTfLiteModelAddListener(
TfLiteBenchmarkTfLiteModel* benchmark_model,
const TfLiteBenchmarkListener* listener) {
return benchmark_model->benchmark_model->AddListener(listener->adapter.get());
}
} // extern "C"
tensorflow/lite/tools/benchmark/experimental/c/benchmark_c_api.h
+137
View File
@@ -0,0 +1,137 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_C_BENCHMARK_C_API_H_
#define TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_C_BENCHMARK_C_API_H_
#include <cstdint>
#include "tensorflow/lite/core/c/c_api_types.h"
// -----------------------------------------------------------------------------
// Experimental C APIs for the benchmark tool, mainly intended to be used for
// building a standalone TensorFlow Lite benchmark framework for iOS. This
// header only has a minimal dependency to the C API types, which can be
// included in the framework itself.
// -----------------------------------------------------------------------------
#ifdef __cplusplus
extern "C" {
#endif // __cplusplus
typedef enum {
TfLiteBenchmarkWarmup,
TfLiteBenchmarkRegular,
} TfLiteBenchmarkRunType;
// -----------------------------------------------------------------------------
// C APIs corresponding to tsl::Stat<int64_t> type.
// -----------------------------------------------------------------------------
typedef struct TfLiteBenchmarkInt64Stat {
bool empty;
int64_t first;
int64_t newest;
int64_t max;
int64_t min;
int64_t count;
int64_t sum;
double squared_sum;
bool all_same;
double avg;
int64_t std_deviation;
} TfLiteBenchmarkInt64Stat;
// -----------------------------------------------------------------------------
// C APIs corresponding to tflite::benchmark::BenchmarkResults type.
// -----------------------------------------------------------------------------
typedef struct TfLiteBenchmarkResults TfLiteBenchmarkResults;
extern TfLiteBenchmarkInt64Stat
TfLiteBenchmarkResultsGetInferenceTimeMicroseconds(
const TfLiteBenchmarkResults* results);
extern TfLiteBenchmarkInt64Stat TfLiteBenchmarkResultsGetWarmupTimeMicroseconds(
const TfLiteBenchmarkResults* results);
extern int64_t TfLiteBenchmarkResultsGetStartupLatencyMicroseconds(
const TfLiteBenchmarkResults* results);
extern uint64_t TfLiteBenchmarkResultsGetInputBytes(
const TfLiteBenchmarkResults* results);
extern double TfLiteBenchmarkResultsGetThroughputMbPerSecond(
const TfLiteBenchmarkResults* results);
// -----------------------------------------------------------------------------
// C APIs corresponding to tflite::benchmark::BenchmarkListener type.
// -----------------------------------------------------------------------------
typedef struct TfLiteBenchmarkListener TfLiteBenchmarkListener;
extern TfLiteBenchmarkListener* TfLiteBenchmarkListenerCreate();
extern void TfLiteBenchmarkListenerDelete(TfLiteBenchmarkListener* listener);
// Sets the listener callbacks. Only non-null callback functions will be called
// when the following events occur. The user_data pointer provided by the caller
// will also be forwarded as a parameter of each callback function.
//
// - on_benchmark_start: Called before the (outer) inference loop begins. Note
// that this is called *after* the interpreter has been initialized, but
// *before* any warmup runs have been executed.
// - on_single_run_start: Called before a single (inner) inference call starts.
// - on_single_run_end: Called before a single (inner) inference call ends.
// - on_benchmark_end: Called after the (outer) inference loop ends.
//
// In case of `on_benchmark_end` callback, the passed in `results` pointer is
// only valid during the callback function execution, and will be destroyed
// afterwards.
extern void TfLiteBenchmarkListenerSetCallbacks(
TfLiteBenchmarkListener* listener, void* user_data,
void (*on_benchmark_start_fn)(void* user_data),
void (*on_single_run_start_fn)(void* user_data,
TfLiteBenchmarkRunType runType),
void (*on_single_run_end_fn)(void* user_data),
void (*on_benchmark_end_fn)(void* user_data,
TfLiteBenchmarkResults* results));
// -----------------------------------------------------------------------------
// C APIs corresponding to tflite::benchmark::BenchmarkTfLiteModel type.
// -----------------------------------------------------------------------------
typedef struct TfLiteBenchmarkTfLiteModel TfLiteBenchmarkTfLiteModel;
// TODO(b/144321502): Support BenchmarkParams.
extern TfLiteBenchmarkTfLiteModel* TfLiteBenchmarkTfLiteModelCreate();
extern void TfLiteBenchmarkTfLiteModelDelete(
TfLiteBenchmarkTfLiteModel* benchmark_model);
extern TfLiteStatus TfLiteBenchmarkTfLiteModelInit(
TfLiteBenchmarkTfLiteModel* benchmark_model);
extern TfLiteStatus TfLiteBenchmarkTfLiteModelRun(
TfLiteBenchmarkTfLiteModel* benchmark_model);
extern TfLiteStatus TfLiteBenchmarkTfLiteModelRunWithArgs(
TfLiteBenchmarkTfLiteModel* benchmark_model, int argc, char** argv);
extern void TfLiteBenchmarkTfLiteModelAddListener(
TfLiteBenchmarkTfLiteModel* benchmark_model,
const TfLiteBenchmarkListener* listener);
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
#endif // TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_C_BENCHMARK_C_API_H_
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/AndroidManifest.xml
+55
View File
@@ -0,0 +1,55 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright 2020 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:tools="http://schemas.android.com/tools"
package="org.tensorflow.lite.benchmark.delegateperformance">
<!-- Necessary for loading custom models from disk and writing result to disk. -->
<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE"/>
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE"/>
<!--
Align the minSdkVersion value with that for the TFLite Model Benchmark Tool Android Apk.
-->
<uses-sdk
android:minSdkVersion="23"
android:targetSdkVersion="33" />
<application android:debuggable="true">
<!-- DPB needs to enable debugging to allow developers to copy their stable delegate
libraries into the application's data directory, to enable testing of such stable delegate
libraries. -->
<!-- This Activity runs Delegate Performance Benchmark for latency. -->
<activity
android:name=".BenchmarkLatencyActivity"
android:screenOrientation="portrait"
android:label="TFLite Delegate Performance Benchmark - Latency"
android:exported="true"
android:noHistory="true">
</activity>
<!-- This Activity runs Delegate Performance Benchmark for accuracy. -->
<activity
android:name=".BenchmarkAccuracyActivity"
android:screenOrientation="portrait"
android:label="TFLite Delegate Performance Benchmark - Accuracy"
android:exported="true"
android:noHistory="true">
</activity>
</application>
</manifest>
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/BUILD
+236
View File
@@ -0,0 +1,236 @@
# Description:
# Delegate Performance Benchmark (DPB) Android app.
# This provides model-level latency & accuracy testings for delegates, on Android.
load("@build_bazel_rules_android//android:rules.bzl", "android_library")
load("//tensorflow/lite/core/shims:cc_library_with_tflite.bzl", "android_binary_with_tflite", "android_library_with_tflite")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
default_visibility = ["//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android:__subpackages__"],
licenses = ["notice"],
)
android_library(
name = "benchmark_accuracy_activity",
srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkAccuracyActivity.java"],
deps = [":benchmark_accuracy_impl"],
)
android_library(
name = "benchmark_accuracy_impl",
srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkAccuracyImpl.java"],
deps = [
":benchmark_accuracy",
":benchmark_report",
":csv_writer",
":delegate_performance_benchmark_utils",
":html_writer",
":json_writer",
":model_benchmark_report",
":raw_delegate_metrics_entry",
":tflite_settings_list_entry",
"//tensorflow/lite/acceleration/configuration:configuration_fbs_android",
],
)
android_library(
name = "benchmark_accuracy",
srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkAccuracy.java"],
)
android_library(
name = "benchmark_latency_activity",
srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkLatencyActivity.java"],
deps = [":benchmark_latency_impl"],
)
android_library(
name = "benchmark_latency_impl",
srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkLatencyImpl.java"],
deps = [
":benchmark_report",
":csv_writer",
":delegate_performance_benchmark_utils",
":html_writer",
":json_writer",
":model_benchmark_report",
":model_benchmark_report_interface",
":preconditions",
":raw_delegate_metrics_entry",
":tflite_settings_list_entry",
"//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto:delegate_performance_java_proto_lite",
],
)
android_library_with_tflite(
name = "benchmark_report",
srcs = [
"src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkReport.java",
"src/main/java/org/tensorflow/lite/benchmark/delegateperformance/ReportWriter.java",
],
deps = [
":benchmark_result_type",
":delegate_performance_benchmark_utils",
":model_benchmark_report_interface",
],
)
android_library(
name = "benchmark_result_type",
srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkResultType.java"],
)
android_library_with_tflite(
name = "csv_writer",
srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/CsvWriter.java"],
tflite_deps = [":benchmark_report"],
deps = [
":delegate_metrics_entry",
":metrics_entry",
":model_benchmark_report_interface",
":preconditions",
],
)
android_library(
name = "delegate_metrics_entry",
srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/DelegateMetricsEntry.java"],
deps = [
":benchmark_result_type",
":metrics_entry",
],
)
android_library_with_tflite(
name = "delegate_performance_benchmark_lib",
srcs = [
"src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkAccuracyActivity.java",
"src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkLatencyActivity.java",
],
tflite_deps = [
"//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native:benchmark_native",
],
deps = [
":benchmark_accuracy_impl",
":benchmark_latency_impl",
],
)
android_library(
name = "delegate_performance_benchmark_utils",
srcs = [
"src/main/java/org/tensorflow/lite/benchmark/delegateperformance/DelegatePerformanceBenchmark.java",
],
deps = [
":benchmark_result_type",
":preconditions",
":tflite_settings_list_entry",
"//tensorflow/lite/acceleration/configuration:configuration_fbs_android",
"//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto:delegate_performance_java_proto_lite",
"@flatbuffers//:runtime_android",
],
)
android_library_with_tflite(
name = "html_writer",
srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/HtmlWriter.java"],
tflite_deps = [":benchmark_report"],
deps = [
":benchmark_result_type",
":delegate_metrics_entry",
":metrics_entry",
":model_benchmark_report_interface",
":preconditions",
],
)
android_library_with_tflite(
name = "json_writer",
srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/JsonWriter.java"],
tflite_deps = [":benchmark_report"],
)
android_library(
name = "metrics_entry",
srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/MetricsEntry.java"],
deps = [":benchmark_result_type"],
)
android_library_with_tflite(
name = "model_benchmark_report",
srcs = [
"src/main/java/org/tensorflow/lite/benchmark/delegateperformance/AccuracyBenchmarkReport.java",
"src/main/java/org/tensorflow/lite/benchmark/delegateperformance/LatencyBenchmarkReport.java",
"src/main/java/org/tensorflow/lite/benchmark/delegateperformance/ModelBenchmarkReport.java",
],
deps = [
":benchmark_result_type",
":delegate_metrics_entry",
":delegate_performance_benchmark_utils",
":metrics_entry",
":model_benchmark_report_interface",
":preconditions",
":raw_delegate_metrics_entry",
":tflite_settings_list_entry",
"//tensorflow/lite/acceleration/configuration:configuration_fbs_android",
"//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto:delegate_performance_java_proto_lite",
],
)
android_library(
name = "model_benchmark_report_interface",
srcs = [
"src/main/java/org/tensorflow/lite/benchmark/delegateperformance/ModelBenchmarkReportInterface.java",
],
deps = [
":benchmark_result_type",
":delegate_metrics_entry",
],
)
android_library(
name = "preconditions",
srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/Preconditions.java"],
)
android_library(
name = "raw_delegate_metrics_entry",
srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/RawDelegateMetricsEntry.java"],
deps = ["//tensorflow/lite/acceleration/configuration:configuration_fbs_android"],
)
android_library(
name = "tflite_settings_list_entry",
srcs = [
"src/main/java/org/tensorflow/lite/benchmark/delegateperformance/TfLiteSettingsListEntry.java",
],
deps = [
":preconditions",
"//tensorflow/lite/acceleration/configuration:configuration_fbs_android",
"//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto:delegate_performance_java_proto_lite",
],
)
# The main test app.
android_binary_with_tflite(
name = "delegate_performance_benchmark",
assets = [
"//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/models:accuracy_models",
"//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/models:latency_criteria_files",
"//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/models:latency_models",
"//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto:default_latency_criteria",
],
assets_dir = "assets",
custom_package = "org.tensorflow.lite.benchmark.delegateperformance",
manifest = "AndroidManifest.xml",
nocompress_extensions = [".tflite"],
# In some platforms we don't have an Android SDK/NDK and this target
# can't be built. We need to prevent the build system from trying to
# use the target in that case.
tags = ["manual"],
tflite_deps = [
":delegate_performance_benchmark_lib",
],
visibility = ["//visibility:public"],
)
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/README.md
+352
View File
@@ -0,0 +1,352 @@
# TensorFlow Lite Delegate Performance Benchmark (Android APK)
## Description
This Android Delegate Performance Benchmark (DPB) app is a simple wrapper around
the TensorFlow Lite
[benchmark tool](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark)
and
[MiniBenchmark](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/acceleration/mini_benchmark)
with the focus on testing Tensorflow Lite delegates that implement stable
delegate ABI.
Development of TensorFlow Lite delegates needs both accuracy and latency
evaluations for catching potential performance regressions. Pushing and
executing both latency and accuracy testing binaries directly on an Android
device is a valid approach to benchmarking, but it can result in subtle (but
observable) differences in performance relative to execution within an actual
Android app. In particular, Android's scheduler tailors behavior based on thread
and process priorities, which differ between a foreground Activity/Application
and a regular background binary executed via `adb shell ...`.
In addition to that, having multiple benchmarking apps for different performance
metric evaluations could potentially cost development effort unnecessarily.
To those ends, this app offers a more faithful view of runtime performance
(accuracy and latency) that developers can expect when using TensorFlow Lite
delegates with Android apps, and the app provides a single entrypoint to various
performance metrics to avoid the need to switch between different benchmarking
apps.
## To build/install/run
### Build
1. Clone the TensorFlow repo with
```
git clone --recurse-submodules https://github.com/tensorflow/tensorflow.git
```
Note: --recurse-submodules is necessary to prevent some issues with protobuf
compilation.
1. Refer to
[this page](https://www.tensorflow.org/lite/android/lite_build#set_up_build_environment_without_docker)
for setting up a development environment. Although there are several
practical tips:
- When installing Bazel, for Ubuntu Linux, `sudo apt update && sudo apt
install bazel` may be the easiest way. However sometimes you may need
`sudo apt update && sudo apt install bazel-5.3.0` if prompted.
- When installing Android NDK and SDK, using Android Studio's SDK Manager
may be the easiest way.
- Run the `./configure` script in the root TensorFlow checkout directory,
and answer "Yes" when the script asks to interactively configure the
`./WORKSPACE` for Android builds.
- The versions which we have verified are working:
- Android NDK version: 21.4.7075529
- Provide the value as part of a path when the `./configure`
script asks to specify the home path of the Android NDK.
- Android NDK API level: 26
- Provide the value when the `./configure` script asks to specify
the Android NDK API level.
- Android SDK API level: 33
- Provide the value when the `./configure` script asks to specify
the Android SDK API level.
- Android build tools version: 30.0.0
- Provide the value when the `./configure` script asks to specify
the Android build tools version.
1. Build for your specific platform, e.g.:
```
bazel build -c opt \
--config=android_arm64 \
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android:delegate_performance_benchmark
```
### Install
1. Connect to a physical device. Install the benchmark APK with adb:
```
adb install -r -d -g bazel-bin/tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/delegate_performance_benchmark.apk
```
Note: Make sure to install with "-g" option to grant the permission for
reading external storage.
### Run
#### Benchmarking with stable delegates
The delegate-under-test must implement the
[stable_delegate_interface](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/delegates/utils/experimental/stable_delegate/stable_delegate_interface.h)
API. The stable delegate provider dynamically loads stable delegate symbols from
the provided binary (shared object) file. In order to use Delegate Performance
Benchmark with a stable delegate, you would need to push the shared object file
to the file directory of Delegate Performance Benchmark:
`/data/data/org.tensorflow.lite.benchmark.delegateperformance/files/`.
1. Build and push the stable delegate binary that you want to test. Here we use
the
[sample stable delegate](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/delegates/utils/experimental/sample_stable_delegate)
as an example.
```
bazel build -c opt \
--config=android_arm64 \
tensorflow/lite/delegates/utils/experimental/sample_stable_delegate:tensorflowlite_sample_stable_delegate
# Set the permissions so that we can overwrite a previously installed delegate.
chmod 755 bazel-bin/tensorflow/lite/delegates/utils/experimental/sample_stable_delegate/libtensorflowlite_sample_stable_delegate.so
# Ensure the delegateperformance files path exists.
adb shell run-as org.tensorflow.lite.benchmark.delegateperformance mkdir -p /data/data/org.tensorflow.lite.benchmark.delegateperformance/files
# Install the sample delegate.
adb push \
bazel-bin/tensorflow/lite/delegates/utils/experimental/sample_stable_delegate/libtensorflowlite_sample_stable_delegate.so \
/data/local/tmp/
adb shell run-as org.tensorflow.lite.benchmark.delegateperformance \
cp /data/local/tmp/libtensorflowlite_sample_stable_delegate.so \
/data/data/org.tensorflow.lite.benchmark.delegateperformance/files/
```
1. Dump the test sample delegate settings file on device. Example command:
```
adb shell 'echo "{
\"delegate\": \"NONE\", // Replace NONE with the test target delegate type.
\"stable_delegate_loader_settings\": {
\"delegate_path\": \"/data/data/org.tensorflow.lite.benchmark.delegateperformance/files/libtensorflowlite_sample_stable_delegate.so\"
}
// Add concrete delegate settings for the test target delegate.
}
"> /data/local/tmp/stable_delegate_settings.json'
```
#### Supported models
Currently DPB uses a `mobilenet_v1_1.0_224.tflite` and
`mobilenet_quant_v1_224.tflite` model for latency and accuracy benchmarking. The
TF Lite model files are bundled into the app during the build process. We plan
to expand the supported models based on future use cases.
Note: The sample stable delegate provided here only supports ADD and SUB
operations thus aforementioned mobilenet models would not actually be delegated.
To test your own delegate against the models, please update
`stable_delegate_loader_settings` with your delegate path. To get feedback early
in the development process, e.g. while working towards supporting more OPs, you
can run the `benchmark_model` tool, which supports stable delegates and can be
supplied with arbitrary models via the `--graph` CLI parameter. See
[this document](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/delegates/utils/experimental/sample_stable_delegate/README.md#tf-lite-benchmark-tool)
which shows how to run a model with ADD operations through the sample stable
delegate.
#### Latency benchmarking
##### Options
- `tflite_settings_files`: `str` (required) the comma-delimited paths to the
JSON-encoded delegate `TFLiteSettings` file(s), which is defined in
[configuration.proto](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/acceleration/configuration/configuration.proto).
- Additional optional command-line flags are documented
[here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark/README.md)
and can be appended to the `args` string (note that all args must be nested
in the single quoted string that follows the args key).
##### Recommendation Criteria
The latency benchmark generates a `PASS`, `PASS_WITH_WARNING`, or `FAIL`
recommendation by checking if the regressions of the below metrics for each pair
of the test target delegate and a reference delegate breach the thresholds:
1. Startup overhead latency: the combined overhead start from initialization to
inferences with stable latency. It is calculated as `initialization time +
average warmup time - average inference time`.
1. Average inference latency: average time for the inferences after warmup in
the benchmark run.
When the test target delegate type is the same as the reference delegate, the
checks are more strict. Otherwise, the checks are relaxed. Please see
[BenchmarkResultType.java](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkResultType.java)
for the meanings of `PASS`, `PASS_WITH_WARNING` and `FAIL`.
##### Steps
1. Run the latency benchmark by supplying the settings file via the required
`--tflite_settings_files` flag.
```
adb shell "am start -S \
-n org.tensorflow.lite.benchmark.delegateperformance/org.tensorflow.lite.benchmark.delegateperformance.BenchmarkLatencyActivity \
--esa --tflite_settings_files '/data/local/tmp/stable_delegate_settings.json'"
```
1. The results will be available in Android logcat as well as the app's file
directory, e.g.:
```
adb logcat -c && adb logcat -v color | grep "Inference timings in us"
... tflite : Inference timings in us: Init: 5811, First inference: 67743, Warmup (avg): 65539, Inference (avg): 65155.5
```
The tool also shows overall results.
```
adb logcat -c && adb logcat -v color | grep 'Latency benchmark result'
```
which might show output like the following.
```
... TfLiteLatencyImpl: Latency benchmark result for /data/local/tmp/stable_delegate_settings.json: PASS
```
For a summarized view, run
```
adb shell run-as org.tensorflow.lite.benchmark.delegateperformance "cat /data/user/0/org.tensorflow.lite.benchmark.delegateperformance/files/delegate_performance_result/latency/report.html" > /tmp/dpb-latency.html && xdg-open /tmp/dpb-latency.html
```
It would open a page in the browser like the following:
Summary | FAIL
------- | ----
Model | Metric | Delegate: NONE (/data/local/tmp/stable_delegate_settings.json) | Delegate: NONE (default_delegate) | Change | Status
-------------------------- | ----------------------------------------------------- | -------------------------------------------------------------- | --------------------------------- | --------- | ------
mobilenet_v1_1.0_224 | model_size_megabyte | -1.0E-6 | -1.0E-6 | 0.0% | N/A
mobilenet_v1_1.0_224 | initialization_latency_us | 515667.0 | 844938.0 | -39.0% | N/A
mobilenet_v1_1.0_224 | warmup_latency_average_us | 334263.5 | 1666704.0 | -79.9% | N/A
mobilenet_v1_1.0_224 | warmup_latency_min_us | 318494.0 | 1666704.0 | -80.9% | N/A
mobilenet_v1_1.0_224 | warmup_latency_max_us | 350033.0 | 1666704.0 | -79.0% | N/A
mobilenet_v1_1.0_224 | warmup_latency_standard_deviation_us | 15769.0 | 0.0 | Infinity% | N/A
mobilenet_v1_1.0_224 | inference_latency_average_us | 316702.6 | 630715.5 | -49.8% | PASS
mobilenet_v1_1.0_224 | inference_latency_min_us | 308218.0 | 314117.0 | -1.9% | N/A
mobilenet_v1_1.0_224 | inference_latency_max_us | 338494.0 | 1601144.0 | -78.9% | N/A
mobilenet_v1_1.0_224 | inference_latency_standard_deviation_us | 4896.0 | 347805.0 | -98.6% | N/A
mobilenet_v1_1.0_224 | initialization_memory_max_rss_mebibyte | 0.0 | 34.48828 | -100.0% | N/A
mobilenet_v1_1.0_224 | initialization_memory_total_non_mmapped_heap_mebibyte | 0.0 | 0.0 | 0.0% | N/A
mobilenet_v1_1.0_224 | initialization_memory_in_use_heap_mebibyte | 26.140594 | 21.560455 | 21.2% | N/A
mobilenet_v1_1.0_224 | overall_memory_max_rss_mebibyte | 0.0 | 50.371094 | -100.0% | N/A
mobilenet_v1_1.0_224 | overall_memory_total_non_mmapped_heap_mebibyte | 0.0 | 0.0 | 0.0% | N/A
mobilenet_v1_1.0_224 | overall_memory_in_use_heap_mebibyte | 28.22168 | 23.295578 | 21.1% | N/A
mobilenet_v1_1.0_224 | startup_overhead_latency_us | 533227.9 | 1880926.5 | -71.7% | PASS
mobilenet_v1_1.0_224 | delegate_summary | | | | PASS (strict)
mobilenet_v1_1.0_224 | model_summary | PASS | | |
mobilenet_v1_1.0_224_quant | model_size_megabyte | -1.0E-6 | -1.0E-6 | 0.0% | N/A
mobilenet_v1_1.0_224_quant | initialization_latency_us | 25318.0 | 8271.0 | 206.1% | N/A
mobilenet_v1_1.0_224_quant | warmup_latency_average_us | 189565.0 | 188034.0 | 0.8% | N/A
mobilenet_v1_1.0_224_quant | warmup_latency_min_us | 181333.0 | 175592.0 | 3.3% | N/A
mobilenet_v1_1.0_224_quant | warmup_latency_max_us | 199285.0 | 199388.0 | -0.1% | N/A
mobilenet_v1_1.0_224_quant | warmup_latency_standard_deviation_us | 7404.0 | 9745.0 | -24.0% | N/A
mobilenet_v1_1.0_224_quant | inference_latency_average_us | 178905.2 | 178897.69 | 0.0% | PASS_WITH_WARNING
mobilenet_v1_1.0_224_quant | inference_latency_min_us | 170126.0 | 170102.0 | 0.0% | N/A
mobilenet_v1_1.0_224_quant | inference_latency_max_us | 200089.0 | 193949.0 | 3.2% | N/A
mobilenet_v1_1.0_224_quant | inference_latency_standard_deviation_us | 6355.0 | 6387.0 | -0.5% | N/A
mobilenet_v1_1.0_224_quant | initialization_memory_max_rss_mebibyte | 0.0 | 0.0 | 0.0% | N/A
mobilenet_v1_1.0_224_quant | initialization_memory_total_non_mmapped_heap_mebibyte | 0.0 | 0.0 | 0.0% | N/A
mobilenet_v1_1.0_224_quant | initialization_memory_in_use_heap_mebibyte | 1.4762268 | 1.4715118 | 0.3% | N/A
mobilenet_v1_1.0_224_quant | overall_memory_max_rss_mebibyte | 0.0 | 0.0 | 0.0% | N/A
mobilenet_v1_1.0_224_quant | overall_memory_total_non_mmapped_heap_mebibyte | 0.0 | 0.0 | 0.0% | N/A
mobilenet_v1_1.0_224_quant | overall_memory_in_use_heap_mebibyte | 3.3774261 | 3.38266 | -0.2% | N/A
mobilenet_v1_1.0_224_quant | startup_overhead_latency_us | 35977.797 | 17407.312 | 106.7% | FAIL
mobilenet_v1_1.0_224_quant | delegate_summary | | | | FAIL (strict)
mobilenet_v1_1.0_224_quant | model_summary | FAIL | | |
#### Accuracy benchmarking
##### Options
- `tflite_settings_files`: `str` (required) the comma-delimited paths to the
JSON-encoded delegate `TFLiteSettings` file(s), which is defined in
[configuration.proto](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/acceleration/configuration/configuration.proto).
The first path is the test target delegate and all other paths are treated
as reference delegates. The test target delegate will be compared against
each reference delegate.
##### Recommendation Criteria
The accuracy benchmark delegates the accuracy metric threshold checks to the
metric scripts, which are embedded together with the test input inside the
models. The metric scripts generate an "ok" result by aggregating the outcomes
for every model and every delegate. The accuracy benchmark generates a `PASS`,
`PASS_WITH_WARNING`, or `FAIL` recommendation by aggregating the "ok" results.
When the test target delegate type is the same as the reference delegate, the
checks are more strict. Otherwise, the checks are relaxed. Please see
[BenchmarkResultType.java](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkResultType.java)
for the meanings of `PASS`, `PASS_WITH_WARNING` and `FAIL`.
##### Steps
1. Run the accuracy benchmark by supplying the settings file via the required
`--tflite_settings_files` flag.
```
adb shell "am start -S \
-n org.tensorflow.lite.benchmark.delegateperformance/org.tensorflow.lite.benchmark.delegateperformance.BenchmarkAccuracyActivity \
--esa --tflite_settings_files '/data/local/tmp/stable_delegate_settings.json'"
```
1. The results will be available in Android logcat, e.g.:
```
adb logcat -c && adb logcat -v color | grep "tflite"
... tflite : tflite : accuracy: ok
```
For a summarized view, run
```
adb shell run-as org.tensorflow.lite.benchmark.delegateperformance "cat /data/user/0/org.tensorflow.lite.benchmark.delegateperformance/files/delegate_performance_result/accuracy/report.html" > /tmp/dpb-accuracy.html && xdg-open /tmp/dpb-accuracy.html
```
It would open a page in the browser like the following:
Summary | PASS
------- | ----
Model | Metric | Delegate: NONE (/data/local/tmp/stable_delegate_settings.json) | Delegate: NONE (default_delegate) | Change | Status
------------------------------------------ | -------------------------------- | -------------------------------------------------------------- | --------------------------------- | ------ | ------
mobilenet_v1_1.0_224_quant_with_validation | mse(average) | 1.917638E-6 | 1.917638E-6 | 0.0% | N/A
mobilenet_v1_1.0_224_quant_with_validation | symmetric_kl_divergence(average) | 0.049423933 | 0.049423933 | 0.0% | N/A
mobilenet_v1_1.0_224_quant_with_validation | ok | 0.0 | 0.0 | N/A | PASS
mobilenet_v1_1.0_224_quant_with_validation | max_memory_kb | 0.0 | 0.0 | 0.0% | N/A
mobilenet_v1_1.0_224_quant_with_validation | delegate_summary | | | | PASS (strict)
mobilenet_v1_1.0_224_quant_with_validation | model_summary | PASS | | |
mobilenet_v1_1.0_224_with_validation | mse(average) | 1.0577066E-16 | 1.0577066E-16 | 0.0% | N/A
mobilenet_v1_1.0_224_with_validation | symmetric_kl_divergence(average) | 7.2540787E-9 | 7.2540787E-9 | 0.0% | N/A
mobilenet_v1_1.0_224_with_validation | ok | 0.0 | 0.0 | N/A | PASS
mobilenet_v1_1.0_224_with_validation | max_memory_kb | 0.0 | 0.0 | 0.0% | N/A
mobilenet_v1_1.0_224_with_validation | delegate_summary | | | | PASS (strict)
mobilenet_v1_1.0_224_with_validation | model_summary | PASS | | |
## FAQ
### 1. What does a delegate summary result with a `(strict)` suffix mean?
The `(strict)` suffix is added to reference delegates that have the same
delegate type as the test target delegate. The purpose of the suffix is to let a
user know that the performance metrics are being checked to a higher standard.
The expectation is that the test target delegate is better, or at least not
substantially worse, than the reference delegate in all metrics.
Please see
[BenchmarkResultType.java](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkResultType.java)
for more details.
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/build_defs.bzl
+31
View File
@@ -0,0 +1,31 @@
"""Definitions for targets in DPB (Delegate Performance Benchmark)."""
def latency_benchmark_extra_deps():
"""Defines extra dependencies for latency benchmark. Currently empty."""
return []
def accuracy_benchmark_extra_deps():
"""Defines extra dependencies for accuracy benchmark. Currently empty."""
return []
def latency_benchmark_extra_models():
"""Defines extra models for latency benchmark. Currently empty.
Returns a list of tuples where each tuple has two fields: 1) the model name and 2) the model target label. Example:
[
("model1.tflite", "@repo//package:model1.tflite"),
("model2.tflite", "@repo//package:model2.tflite"),
]
"""
return []
def accuracy_benchmark_extra_models():
"""Defines extra models for accuracy benchmark. Currently empty.
Returns a list of tuples where each tuple has two fields: 1) the model name and 2) the model target label. Example:
[
("model1_with_validation.tflite", "@repo//package:model1_with_validation.tflite"),
("model2_with_validation.tflite", "@repo//package:model2_with_validation.tflite"),
]
"""
return []
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/models/BUILD
+109
View File
@@ -0,0 +1,109 @@
# Description:
# Holds model-specific files. The app will bundle the files into assets.
load("//tensorflow/lite/experimental/acceleration/mini_benchmark:build_defs.bzl", "validation_model")
load("//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android:build_defs.bzl", "accuracy_benchmark_extra_models", "latency_benchmark_extra_models")
load("//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android:proto.bzl", "proto_data")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
default_visibility = ["//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android:__subpackages__"],
licenses = ["notice"],
)
# Embedded models for accuracy benchmarking.
validation_model(
name = "mobilenet_v1_1.0_224_with_validation.tflite",
jpegs = "//tensorflow/lite/experimental/acceleration/mini_benchmark:odt_classifier_testfiles",
main_model = "//tensorflow/lite/experimental/acceleration/mini_benchmark/models:mobilenet_v1_1.0_224.tflite",
metrics_model = "//tensorflow/lite/experimental/acceleration/mini_benchmark/metrics:mobilenet_metrics_tflite",
)
validation_model(
name = "mobilenet_v1_1.0_224_quant_with_validation.tflite",
jpegs = "//tensorflow/lite/experimental/acceleration/mini_benchmark:odt_classifier_testfiles",
main_model = "//tensorflow/lite/experimental/acceleration/mini_benchmark/models:mobilenet_v1_1.0_224_quant.tflite",
metrics_model = "//tensorflow/lite/experimental/acceleration/mini_benchmark/metrics:mobilenet_metrics_tflite",
)
# Migrate the models into assets folder.
ACCURACY_MODELS = [
(
"mobilenet_v1_1.0_224_with_validation.tflite",
":mobilenet_v1_1.0_224_with_validation.tflite",
),
(
"mobilenet_v1_1.0_224_quant_with_validation.tflite",
":mobilenet_v1_1.0_224_quant_with_validation.tflite",
),
] + accuracy_benchmark_extra_models()
BASIC_LATENCY_MODELS = [
(
"mobilenet_v1_1.0_224.tflite",
"@tflite_mobilenet_float//:mobilenet_v1_1.0_224.tflite",
),
(
"mobilenet_v1_1.0_224_quant.tflite",
"@tflite_mobilenet_quant//:mobilenet_v1_1.0_224_quant.tflite",
),
]
LATENCY_MODELS = BASIC_LATENCY_MODELS + latency_benchmark_extra_models()
COPY_CMD = """
srcs=($(SRCS))
outs=($(OUTS))
for ((i = 0; i < $${#srcs[@]}; ++i)); do
echo $${srcs[$$i]};
cp $${srcs[$$i]} $${outs[$$i]};
done
"""
genrule(
name = "accuracy_models",
srcs = [target for _, target in ACCURACY_MODELS],
outs = ["assets/accuracy/%s" % name for name, _ in ACCURACY_MODELS],
cmd = COPY_CMD,
)
genrule(
name = "latency_models",
srcs = [target for _, target in LATENCY_MODELS],
outs = ["assets/latency/%s" % name for name, _ in LATENCY_MODELS],
cmd = COPY_CMD,
)
filegroup(
name = "latency_models_test_only",
testonly = True,
srcs = [
"assets/latency/mobilenet_v1_1.0_224.tflite",
"assets/latency/mobilenet_v1_1.0_224_quant.tflite",
],
)
# Latency criteria for latency benchmarking.
filegroup(
name = "latency_criteria_files",
srcs = [
":mobilenet_v1_1_0_224_latency_criteria",
":mobilenet_v1_1_0_224_quant_latency_criteria",
],
)
proto_data(
name = "mobilenet_v1_1_0_224_latency_criteria",
src = "mobilenet_v1_1.0_224.textproto",
out = "assets/proto/mobilenet_v1_1.0_224.binarypb",
proto_deps = ["//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto:delegate_performance_proto"],
proto_name = "tflite.proto.benchmark.LatencyCriteria",
)
proto_data(
name = "mobilenet_v1_1_0_224_quant_latency_criteria",
src = "mobilenet_v1_1.0_224_quant.textproto",
out = "assets/proto/mobilenet_v1_1.0_224_quant.binarypb",
proto_deps = ["//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto:delegate_performance_proto"],
proto_name = "tflite.proto.benchmark.LatencyCriteria",
)
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/models/mobilenet_v1_1.0_224.textproto
+19
View File
@@ -0,0 +1,19 @@
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License
#
# proto-file: third_party/tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto/delegate_performance.proto
# proto-message: LatencyCriteria
startup_overhead_max_regression_percentage_allowed: 5 # 5%
average_inference_max_regression_percentage_allowed: 5 # 5%
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/models/mobilenet_v1_1.0_224_quant.textproto
+19
View File
@@ -0,0 +1,19 @@
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License
#
# proto-file: third_party/tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto/delegate_performance.proto
# proto-message: LatencyCriteria
startup_overhead_max_regression_percentage_allowed: 5 # 5%
average_inference_max_regression_percentage_allowed: 5 # 5%
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto.bzl
+86
View File
@@ -0,0 +1,86 @@
"""BUILD rules for converting proto text files into binary format."""
load("@bazel_skylib//lib:new_sets.bzl", "sets")
# copybara:uncomment load("@com_google_protobuf//bazel/common:proto_info.bzl", "ProtoInfo")
def _descriptor_set_list(deps):
"""Makes a list of distinct FileDescriptorSet files of deps's transitive dependencies"""
descriptor_set_set = sets.make()
for dep in deps:
if ProtoInfo in dep:
for descriptor_set in dep[ProtoInfo].transitive_descriptor_sets.to_list():
sets.insert(descriptor_set_set, descriptor_set)
return sets.to_list(descriptor_set_set)
def _proto_data_impl(ctx):
output = ctx.actions.declare_file(
ctx.attr.out if ctx.attr.out else "%s.pb" % (ctx.attr.name,),
)
args = []
args.append("--deterministic_output")
args.append("--encode=%s" % (ctx.attr.proto_name,))
# Determine all proto descriptor set dependencies, as well as transitive dependencies. Pass
# them via --descriptor_set_in flag to the protoc tool.
#
# If descriptor_set_in exceeds 20000 characters, this implementation will need to be ported to
# support passing descriptor_set_in as a flagfile argument.
descriptor_set_in = []
descriptor_set_list = _descriptor_set_list(ctx.attr.proto_deps)
for file in ctx.files.proto_deps:
descriptor_set_in.append(file.path)
if descriptor_set_list:
args.append("--descriptor_set_in=%s" % ":".join([d.path for d in descriptor_set_list]))
redirect = [
# textproto input is passed via stdin.
"< '%s\'" % ctx.file.src.path,
# binaryproto output is emitted via stdout.
"> '%s'" % output.path,
]
ctx.actions.run_shell(
outputs = [output],
inputs = [ctx.file.src] + descriptor_set_list,
mnemonic = "ProtoDataEncode",
tools = [ctx.executable._tool],
command = " ".join([ctx.executable._tool.path] + args + redirect),
use_default_shell_env = False,
)
return DefaultInfo(
files = depset([output]),
runfiles = ctx.runfiles(files = [output]),
)
_TOOL = "@com_google_protobuf//:protoc"
# BUILD rule to convert a protocol buffer in text format into the standard binary format.
#
# Args:
# name: The name of the build target.
# src: A text formatted protocol buffer.
# proto_name: The name of the message type in the .proto files that "src" file represents.
# proto_deps: The list of proto_library targets where "proto" is defined.
# Transitive dependencies are pulled in automatically.
# out: (optional) The name of output file. If out is not set then name of
# output file is name + ".binarypb" extension.
proto_data = rule(
implementation = _proto_data_impl,
attrs = {
"src": attr.label(
allow_single_file = True,
),
"proto_name": attr.string(),
"proto_deps": attr.label_list(
providers = [ProtoInfo],
),
"out": attr.string(),
"_tool": attr.label(
executable = True,
cfg = "exec",
allow_files = True,
default = Label(_TOOL),
),
},
)
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto/BUILD
+36
View File
@@ -0,0 +1,36 @@
# Description:
# Holds model-agnostic files and proto definitions. The app will bundle the files into assets.
load("@com_google_protobuf//bazel:java_lite_proto_library.bzl", "java_lite_proto_library")
load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
load("//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android:proto.bzl", "proto_data")
# copybara:uncomment load("//tools/build_defs/proto/cpp:cc_proto_library.bzl", "cc_proto_library")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
default_visibility = ["//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android:__subpackages__"],
licenses = ["notice"],
)
proto_library(
name = "delegate_performance_proto",
srcs = ["delegate_performance.proto"],
)
cc_proto_library(
name = "delegate_performance_cc_proto",
deps = [":delegate_performance_proto"],
)
java_lite_proto_library(
name = "delegate_performance_java_proto_lite",
deps = [":delegate_performance_proto"],
)
proto_data(
name = "default_latency_criteria",
src = "default_latency_criteria.textproto",
out = "assets/proto/default_latency_criteria.binarypb",
proto_deps = [":delegate_performance_proto"],
proto_name = "tflite.proto.benchmark.LatencyCriteria",
)
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto/default_latency_criteria.textproto
+19
View File
@@ -0,0 +1,19 @@
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License
#
# proto-file: third_party/tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto/delegate_performance.proto
# proto-message: LatencyCriteria
startup_overhead_max_regression_percentage_allowed: 5 # 5%
average_inference_max_regression_percentage_allowed: 5 # 5%
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto/delegate_performance.proto
+113
View File
@@ -0,0 +1,113 @@
// Copyright 2022 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// =============================================================================
syntax = "proto3";
package tflite.proto.benchmark;
option java_package = "tflite.proto.benchmark";
// Parameters for latency thresholds of the delegate latency benchmarking
// results. The delegate performance benchmark app generates a
// "PASS"/"PASS_WITH_WARNING"/"FAIL" result for each pair of test target
// delegate with a reference delegate and an overall result by aggregating all
// the results. The result computation involves computing the test target
// delegate performance regressions and checking if the regression thresholds,
// specified by LatencyCriteria, are breached.
//
// Please see
// tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkResultType.java
// for the meanings of "PASS", "PASS_WITH_WARNING" and "FAIL".
//
// The latency criteria is designed to be model specific.
//
// Next ID: 3
message LatencyCriteria {
// The maximum regression (%) of startup overhead time that is allowed.
// If initialization_max_regression_percentage_allow is 5 and the
// initialization time of the test delegate is slower than the reference
// delegate by more than 5%, the threshold is breached.
//
// Startup overhead time is calculated as below:
// initialization time + average warmup time - average inference time
optional float startup_overhead_max_regression_percentage_allowed = 1;
// The maximum regression (%) of inference time that is allowed.
// If average_inference_max_regression_percentage_allowed is 5 and the
// inference time of the test delegate is slower than the reference delegate
// by more than 5%, the threshold is breached.
optional float average_inference_max_regression_percentage_allowed = 2;
}
// Which stage of benchmarking the event is for.
enum BenchmarkEventType {
BENCHMARK_EVENT_TYPE_UNDEFINED = 0;
// Benchmark start. A start without an end can be interpreted as a test that
// has crashed or hung.
BENCHMARK_EVENT_TYPE_START = 1;
// Benchmarking completion. A model was successfully loaded, acceleration
// configured and inference run without errors. There may still be an issue
// with correctness of results, or with performance.
BENCHMARK_EVENT_TYPE_END = 2;
// Benchmark was not completed due to an error. The error may be a handled
// error (e.g., failure in a delegate), or a crash.
BENCHMARK_EVENT_TYPE_ERROR = 3;
}
// A handled error.
//
// Next ID: 2
message ErrorCode {
// What the TF Lite level error is.
// See TfLiteStatus in tensorflow/lite/core/c/c_api_types.h for the meaning of
// the values.
optional int32 tflite_error = 1;
}
// An error that occurred during benchmarking.
//
// Used with event type ERROR.
//
// Next ID: 3
message BenchmarkError {
// Handled tflite error.
optional ErrorCode error_code = 1;
optional string error_message = 2;
}
// A metric from a benchmark, for example an average inference time in
// microseconds.
message BenchmarkMetric {
// Metric name, for example inference_latency_average_us.
optional string name = 1;
// Metric value, for example 180000.
optional float value = 2;
}
// Outcome of a latency benchmark run. If the benchmark run was successfully
// completed, the message contains the latency metrics. The information is
// intended to be compared against with other candidate acceleration
// configurations. If the benchmark run was failed, the message contains the
// handled errors for the callers to investigate further.
//
// Next ID: 4
message LatencyResults {
// Type of the benchmark event.
optional BenchmarkEventType event_type = 1;
// Metrics that are intended to be compared against other acceleration
// configurations, used when type is END.
repeated BenchmarkMetric metrics = 2;
// Error during benchmark, used when type is ERROR.
optional BenchmarkError error = 3;
}
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/AccuracyBenchmarkReport.java
+93
View File
@@ -0,0 +1,93 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark.delegateperformance;
import static org.tensorflow.lite.benchmark.delegateperformance.Preconditions.checkNotNull;
import android.util.Log;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import tflite.BenchmarkEvent;
import tflite.BenchmarkEventType;
import tflite.BenchmarkMetric;
import tflite.BenchmarkResult;
/** Model-level accuracy benchmark report class. */
final class AccuracyBenchmarkReport extends ModelBenchmarkReport {
public static final double PASS = 0;
public static final double FAIL = 1;
private static final String TAG = "AccuracyBenchmarkReport";
private AccuracyBenchmarkReport(
String modelName, List<RawDelegateMetricsEntry> rawDelegateMetricsEntries) {
super(modelName);
Log.i(TAG, "Creating an accuracy benchmark report for " + modelName);
// Adds a regression threshold for "ok" here to make sure that "ok" will be consumed during
// metric computation.
// TODO(b/267313326): replace the mitigation with a proper accuracy benchmark criteria.
maxRegressionPercentageAllowed.put("ok", 0.0);
computeModelReport(rawDelegateMetricsEntries);
}
/**
* Parses {@link BenchmarkEvent} into the unified {@link RawDelegateMetricsEntry} format for
* further processing.
*/
public static RawDelegateMetricsEntry parseResults(
BenchmarkEvent event, TfLiteSettingsListEntry entry) {
Map<String, Double> metrics = new LinkedHashMap<>();
if (event == null || event.eventType() != BenchmarkEventType.END || event.result() == null) {
Log.w(TAG, "The accuracy benchmarking is not completed successfully for " + entry.filePath());
return RawDelegateMetricsEntry.create(
entry.tfliteSettings().delegate(), entry.filePath(), entry.isTestTarget(), metrics);
}
BenchmarkResult accuracyResults = event.result();
for (int i = 0; i < accuracyResults.metricsLength(); i++) {
BenchmarkMetric metric = accuracyResults.metrics(i);
checkNotNull(metric);
if (metric.valuesLength() == 0) {
Log.i(TAG, "The metric " + metric.name() + " is empty. Skipping to the next metric.");
continue;
}
String metricName = metric.name();
double metricValue = metric.values(0);
if (metric.valuesLength() > 1) {
// TODO(b/267765648): consider updating the metric aggregation logic.
metricName += "(average)";
double sum = 0f;
for (int j = 0; j < metric.valuesLength(); j++) {
sum += metric.values(j);
}
metricValue = sum / metric.valuesLength();
}
metrics.put(metricName, metricValue);
}
// The value of {@code ok} is set to {@code 0} when the delegate-under-test has passed the
// accuracy checks performed by MiniBenchmark. Otherwise, the value of {@code ok} is set to
// {@code 1}.
// TODO(b/267313326): replace the mitigation with a proper accuracy benchmark criteria.
metrics.put("ok", accuracyResults.ok() ? PASS : FAIL);
metrics.put("max_memory_kb", (double) accuracyResults.maxMemoryKb());
return RawDelegateMetricsEntry.create(
entry.tfliteSettings().delegate(), entry.filePath(), entry.isTestTarget(), metrics);
}
static AccuracyBenchmarkReport create(
String modelName, List<RawDelegateMetricsEntry> rawDelegateMetricsEntries) {
return new AccuracyBenchmarkReport(modelName, rawDelegateMetricsEntries);
}
}
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkAccuracy.java
+28
View File
@@ -0,0 +1,28 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark.delegateperformance;
import android.content.Context;
/** Interface for Delegate Performance Accuracy Benchmark. */
public interface BenchmarkAccuracy {
/**
* Initializes and runs the accuracy benchmark.
*
* @param context the context to use for finding the test models and exporting reports
* @param tfliteSettingsJsonFiles the list of paths to delegate JSON configurations
*/
void benchmark(Context context, String[] tfliteSettingsJsonFiles);
}
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkAccuracyActivity.java
+46
View File
@@ -0,0 +1,46 @@
/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark.delegateperformance;
import android.app.Activity;
import android.content.Intent;
import android.os.Bundle;
import android.util.Log;
/**
* {@link Activity} class for Delegate Performance Accuracy Benchmark.
*
* <p>This Activity receives test arguments via a command line specified in an intent extra. It
* performs accuracy benchmark tests via TFLite MiniBenchmark based on the input arguments. Please
* check the test example in
* tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/README.md.
*/
public class BenchmarkAccuracyActivity extends Activity {
private static final String TAG = "TfLiteBenchmarkAccuracy";
private static final String TFLITE_SETTINGS_FILES_INTENT_KEY_0 = "--tflite_settings_files";
@Override
public void onCreate(Bundle savedInstanceState) {
Log.i(TAG, "Create benchmark accuracy activity.");
super.onCreate(savedInstanceState);
Intent intent = getIntent();
Bundle bundle = intent.getExtras();
String[] tfliteSettingsJsonFiles = bundle.getStringArray(TFLITE_SETTINGS_FILES_INTENT_KEY_0);
new BenchmarkAccuracyImpl().benchmark(this, tfliteSettingsJsonFiles);
}
}
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkAccuracyImpl.java
+178
View File
@@ -0,0 +1,178 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark.delegateperformance;
import android.app.Activity;
import android.content.Context;
import android.content.res.AssetFileDescriptor;
import android.util.Log;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import tflite.BenchmarkEvent;
/**
* Impl class for Delegate Performance Accuracy Benchmark.
*
* <p>It performs accuracy benchmark tests via TFLite MiniBenchmark based on the input arguments.
* Please check the test example in
* tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/README.md.
*
* <p>Generates a PASS/PASS_WITH_WARNING/FAIL result.
*
* <ul>
* <li>PASS: The test target delegate passed the embedded metric thresholds in all models.
* <li>PASS_WITH_WARNING: Both the test target delegate and the reference delegates breached the
* embedded metric thresholds.
* <li>FAIL: The test target delegate failed at least 1 embedded metric threshold in the models,
* and at least 1 reference delegate passed the embedded metric thresholds in all models.
* </ul>
*
* <p>Generates below list of files to describe the benchmark results under
* delegate_performance_result/accuracy folder in the app files directory.
*
* <ul>
* <li>1. delegate_performance_result/accuracy/report.csv: the performance of each acceleration
* configuration and relative performance differences as percentages in CSV.
* <li>2. delegate_performance_result/accuracy/report.json: detailed performance results. The file
* contains the metric-level, delegate-level and model-level results and the raw metric
* outputs from the native layer in JSON.
* <li>3. delegate_performance_result/accuracy/report.html: the performance of each acceleration
* configuration and relative performance differences as percentages in HTML.
* </ul>
*/
public class BenchmarkAccuracyImpl implements BenchmarkAccuracy {
private static final String TAG = "TfLiteAccuracyImpl";
private static final String ACCURACY_FOLDER_NAME = "accuracy";
private Context context;
private String[] tfliteSettingsJsonFiles;
private BenchmarkReport report;
@Override
public void benchmark(Context context, String[] tfliteSettingsJsonFiles) {
Activity activity = (Activity) context;
if (!benchmarkInternal(context.getApplicationContext(), tfliteSettingsJsonFiles)) {
Log.e(TAG, "Failed to complete accuracy benchmark.");
}
activity.finish();
}
public boolean benchmarkInternal(Context context, String[] tfliteSettingsJsonFiles) {
if (!initialize(context.getApplicationContext(), tfliteSettingsJsonFiles)) {
Log.e(TAG, "Failed to initialize accuracy benchmark.");
return false;
}
return benchmarkDelegatesAndExportReport();
}
private boolean benchmarkDelegatesAndExportReport() {
Log.i(
TAG,
"Running accuracy benchmark with TFLiteSettings JSON files: "
+ Arrays.toString(tfliteSettingsJsonFiles));
List<TfLiteSettingsListEntry> tfliteSettingsList =
DelegatePerformanceBenchmark.loadTfLiteSettingsList(tfliteSettingsJsonFiles);
if (tfliteSettingsList.size() < 2) {
Log.e(TAG, "Failed to load the TFLiteSettings JSON file.");
return false;
}
String[] assets;
try {
assets = context.getAssets().list(ACCURACY_FOLDER_NAME);
} catch (IOException e) {
Log.e(TAG, "Failed to list files from assets folder.", e);
return false;
}
for (String asset : assets) {
if (!asset.endsWith(".tflite")) {
Log.i(TAG, asset + " is not a model file. Skipping.");
continue;
}
String modelResultPath;
String modelName = DelegatePerformanceBenchmark.getModelName(asset);
try {
modelResultPath =
DelegatePerformanceBenchmark.createResultFolder(
context.getFilesDir(), ACCURACY_FOLDER_NAME + "/" + modelName);
} catch (IOException e) {
Log.e(TAG, "Failed to create result folder for " + modelName + ". Exiting application.", e);
return false;
}
try (AssetFileDescriptor modelFileDescriptor =
context.getAssets().openFd(ACCURACY_FOLDER_NAME + "/" + asset)) {
List<RawDelegateMetricsEntry> rawDelegateMetricsEntries = new ArrayList<>();
for (TfLiteSettingsListEntry tfliteSettingsListEntry : tfliteSettingsList) {
BenchmarkEvent benchmarkEvent =
DelegatePerformanceBenchmark.runAccuracyBenchmark(
tfliteSettingsListEntry,
modelFileDescriptor.getParcelFileDescriptor().getFd(),
modelFileDescriptor.getStartOffset(),
modelFileDescriptor.getLength(),
modelResultPath);
rawDelegateMetricsEntries.add(
AccuracyBenchmarkReport.parseResults(benchmarkEvent, tfliteSettingsListEntry));
}
report.addModelBenchmarkReport(
AccuracyBenchmarkReport.create(modelName, rawDelegateMetricsEntries));
} catch (IOException e) {
Log.e(TAG, "Failed to open assets file " + asset, e);
return false;
}
}
// Computes the aggregated results and export the report to local files.
report.export();
TfLiteSettingsListEntry testTarget = tfliteSettingsList.get(tfliteSettingsList.size() - 1);
Log.i(
TAG,
String.format(
"Accuracy benchmark result for %s: %s.", testTarget.filePath(), report.result()));
return true;
}
/**
* Initializes the test environment. Checks the validity of input arguments and creates the result
* folder.
*
* @return {@code true} if the initialization was successful. Otherwise, returns {@code false}.
*/
private boolean initialize(Context context, String[] tfliteSettingsJsonFiles) {
if (tfliteSettingsJsonFiles == null || tfliteSettingsJsonFiles.length == 0) {
Log.e(TAG, "No TFLiteSettings file provided.");
return false;
}
this.context = context;
this.tfliteSettingsJsonFiles = tfliteSettingsJsonFiles;
report = BenchmarkReport.create();
try {
// Creates root result folder.
String resultFolderPath =
DelegatePerformanceBenchmark.createResultFolder(
context.getFilesDir(), ACCURACY_FOLDER_NAME);
report.addWriter(JsonWriter.create(resultFolderPath));
report.addWriter(CsvWriter.create(resultFolderPath));
report.addWriter(HtmlWriter.create(resultFolderPath));
} catch (IOException e) {
Log.e(TAG, "Failed to create result folder", e);
return false;
}
return true;
}
}
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkLatencyActivity.java
+56
View File
@@ -0,0 +1,56 @@
/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark.delegateperformance;
import android.app.Activity;
import android.content.Intent;
import android.os.Bundle;
import android.util.Log;
/**
* {@link Activity} class for Delegate Performance Latency Benchmark.
*
* <p>This Activity receives test arguments via a command line specified in an intent extra. It
* passes the arguments to the {@link BenchmarkLatencyImpl} class to perform latency benchmark tests
* via TFLite Benchmark Tool. Please check the test example in
* tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/README.md.
*/
public class BenchmarkLatencyActivity extends Activity {
private static final String TAG = "TfLiteBenchmarkLatency";
private static final String TFLITE_SETTINGS_FILES_INTENT_KEY_0 = "--tflite_settings_files";
private static final String ARGS_INTENT_KEY_0 = "--args";
@Override
public void onCreate(Bundle savedInstanceState) {
Log.i(TAG, "Create benchmark latency activity.");
super.onCreate(savedInstanceState);
Intent intent = getIntent();
Bundle bundle = intent.getExtras();
String[] tfliteSettingsJsonFiles = bundle.getStringArray(TFLITE_SETTINGS_FILES_INTENT_KEY_0);
String[] args = bundle.getStringArray(ARGS_INTENT_KEY_0);
BenchmarkLatencyImpl impl =
new BenchmarkLatencyImpl(getApplicationContext(), tfliteSettingsJsonFiles, args);
if (impl.initialize()) {
impl.benchmark();
} else {
Log.e(TAG, "Failed to initialize the latency benchmarking.");
}
finish();
}
}
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkLatencyImpl.java
+215
View File
@@ -0,0 +1,215 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark.delegateperformance;
import static org.tensorflow.lite.benchmark.delegateperformance.Preconditions.checkState;
import android.content.Context;
import android.content.res.AssetFileDescriptor;
import android.util.Log;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import tflite.proto.benchmark.DelegatePerformance.LatencyCriteria;
import tflite.proto.benchmark.DelegatePerformance.LatencyResults;
/**
* Impl class for Delegate Performance Latency Benchmark.
*
* <p>It performs latency benchmark tests via TFLite Benchmark Tool based on the input arguments.
* Please check the test example in
* tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/README.md.
*
* <p>Generates below list of files under delegate_performance_result/latency folder to describe the
* benchmark results.
*
* <ul>
* <li>1. delegate_performance_result/latency/report.csv: the performance of each acceleration
* configuration and relative performance differences as percentages in CSV.
* <li>2. delegate_performance_result/latency/report.json: detailed performance results. The file
* contains the metric-level, delegate-level and model-level results, the latency criteria and
* the raw metric outputs from the native layer in JSON.
* <li>3. delegate_performance_result/latency/report.html: the performance of each acceleration
* configuration and relative performance differences as percentages in HTML.
* </ul>
*
* The metrics for generating the Pass/Pass with Warning/Fail decision:
*
* <ul>
* <li>1. startup overhead latency: it is equal to (initialization time + average warmup latency -
* average inference time).
* <li>2. average inference latency: average time for the inferences in the benchmark run.
* </ul>
*/
public final class BenchmarkLatencyImpl {
private static final String LATENCY_FOLDER_NAME = "latency";
private static final String PROTO_FOLDER_NAME = "proto";
private static final String TAG = "TfLiteLatencyImpl";
private static final String DEFAULT_LATENCY_CRITERIA_FILENAME = "default_latency_criteria";
private static final String LATENCY_CRITERIA_FILE_EXT = ".binarypb";
private final Context context;
private final String[] tfliteSettingsJsonFiles;
private final String[] args;
private final BenchmarkReport report;
private LatencyCriteria defaultLatencyCriteria;
public BenchmarkLatencyImpl(Context context, String[] tfliteSettingsJsonFiles, String[] args) {
this.context = context;
this.tfliteSettingsJsonFiles = tfliteSettingsJsonFiles;
if (args == null) {
// The "--args" extra key was not provided.
this.args = new String[0];
} else {
this.args = args;
}
this.report = BenchmarkReport.create();
}
/**
* Initializes the test environment. Creates the result folder and loads the default latency
* criteria file.
*
* <p>Returns {@code true} if the initialization was successful. Otherwise, returns {@code false}.
*/
public boolean initialize() {
if (tfliteSettingsJsonFiles == null || tfliteSettingsJsonFiles.length == 0) {
Log.e(TAG, "No TFLiteSettings file provided.");
return false;
}
try {
// Creates root result folder.
String resultFolderPath =
DelegatePerformanceBenchmark.createResultFolder(
context.getFilesDir(), LATENCY_FOLDER_NAME);
report.addWriter(JsonWriter.create(resultFolderPath));
report.addWriter(CsvWriter.create(resultFolderPath));
report.addWriter(HtmlWriter.create(resultFolderPath));
} catch (IOException e) {
Log.e(
TAG, "Failed to create result folder " + LATENCY_FOLDER_NAME + " in files directory.", e);
return false;
}
try {
// Loads default latency criteria.
defaultLatencyCriteria = loadLatencyCriteria(DEFAULT_LATENCY_CRITERIA_FILENAME);
} catch (IOException e) {
Log.e(TAG, "Failed to load default latency criteria " + DEFAULT_LATENCY_CRITERIA_FILENAME, e);
return false;
}
return true;
}
/** Benchmarks the embedded model files with the input TFLiteSettings JSON files. */
public void benchmark() {
List<TfLiteSettingsListEntry> tfliteSettingsList =
DelegatePerformanceBenchmark.loadTfLiteSettingsList(tfliteSettingsJsonFiles);
if (tfliteSettingsList.size() < 2) {
Log.e(TAG, "Failed to load the TFLiteSettings JSON file.");
return;
}
String[] assets;
try {
assets = context.getAssets().list(LATENCY_FOLDER_NAME);
} catch (IOException e) {
Log.e(TAG, "Failed to list files from assets folder.", e);
return;
}
for (String asset : assets) {
if (asset.endsWith(".tflite")) {
report.addModelBenchmarkReport(benchmarkModel(asset, tfliteSettingsList, args));
}
}
// Computes the aggregated results and export the report to local files.
report.export();
TfLiteSettingsListEntry testTarget = tfliteSettingsList.get(tfliteSettingsList.size() - 1);
checkState(testTarget.isTestTarget());
Log.i(
TAG,
String.format(
"Latency benchmark result for %s: %s", testTarget.filePath(), report.result()));
}
/**
* Benchmarks a model file with the TfLiteSettingsListEntry list.
*
* <p>Returns {@link ModelBenchmarkReportInterface}, which is the model level benchmark report
* that has the delegate information and the raw metric results from the native layer.
*/
private ModelBenchmarkReportInterface benchmarkModel(
String modelFilename, List<TfLiteSettingsListEntry> tfliteSettingsList, String[] args) {
String modelName = DelegatePerformanceBenchmark.getModelName(modelFilename);
LatencyCriteria latencyCriteria = tryLoadLatencyCriteria(modelName);
List<RawDelegateMetricsEntry> rawDelegateMetricsEntries = new ArrayList<>();
try (AssetFileDescriptor modelFileDescriptor =
context.getAssets().openFd(LATENCY_FOLDER_NAME + "/" + modelFilename)) {
for (TfLiteSettingsListEntry tfliteSettingsListEntry : tfliteSettingsList) {
Log.i(
TAG,
"Running latency benchmark with model: "
+ modelName
+ ", settings: "
+ tfliteSettingsListEntry.filePath()
+ ", args: "
+ Arrays.toString(args));
LatencyResults results =
DelegatePerformanceBenchmark.runLatencyBenchmark(
args,
tfliteSettingsListEntry,
modelFileDescriptor.getParcelFileDescriptor().getFd(),
modelFileDescriptor.getStartOffset(),
modelFileDescriptor.getLength());
rawDelegateMetricsEntries.add(
LatencyBenchmarkReport.parseResults(results, tfliteSettingsListEntry));
}
} catch (IOException e) {
Log.e(TAG, "Failed to open asset file " + LATENCY_FOLDER_NAME + "/" + modelFilename);
}
return LatencyBenchmarkReport.create(modelName, rawDelegateMetricsEntries, latencyCriteria);
}
/**
* Tries to load the model-specific latency criteria file by the model name.
*
* <p>Returns the latency criteria for the specific model if the loading was successful.
* Otherwise, returns the default latency criteria.
*/
private LatencyCriteria tryLoadLatencyCriteria(String fileBasename) {
try {
return loadLatencyCriteria(fileBasename);
} catch (IOException e) {
Log.w(
TAG,
"Failed to load the latency criteria of "
+ fileBasename
+ ". Fallback to the default latency criteria.");
}
return defaultLatencyCriteria;
}
/** Loads the latency criteria file from Assets. */
private LatencyCriteria loadLatencyCriteria(String fileBasename) throws IOException {
String latencyCriteriaFileAssetPath =
PROTO_FOLDER_NAME + "/" + fileBasename + LATENCY_CRITERIA_FILE_EXT;
InputStream latencyCriteriaFile = context.getAssets().open(latencyCriteriaFileAssetPath);
return LatencyCriteria.parseFrom(latencyCriteriaFile);
}
}
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkReport.java
+98
View File
@@ -0,0 +1,98 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark.delegateperformance;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
/**
* Benchmark session-level report class to store the model-level {@link
* ModelBenchmarkReportInterface} reports. It allows {@link ReportWriter} subscriptions for
* exporting this report.
*/
final class BenchmarkReport {
private static final String TAG = "TfLiteBenchmarkReport";
private static final String NAME = "report";
private final List<ReportWriter> writers = new ArrayList<>();
private final List<ModelBenchmarkReportInterface> modelBenchmarkReports = new ArrayList<>();
// The benchmark session-level pass status.
// Possible values:
// - UNKNOWN: The metric computation has not started or failed to complete.
// - PASS: All model-level results are "PASS" or "NOT_APPLICABLE".
// - PASS_WITH_WARNING: At least 1 "PASS_WITH_WARNING" model-level result and 0 "FAIL" model-level
// result.
// - FAIL: At least 1 "FAIL" model-level result.
private BenchmarkResultType result = BenchmarkResultType.UNKNOWN;
void addModelBenchmarkReport(ModelBenchmarkReportInterface modelBenchmarkReport) {
modelBenchmarkReports.add(modelBenchmarkReport);
}
void addWriter(ReportWriter writer) {
writers.add(writer);
}
void export() {
if (result == BenchmarkResultType.UNKNOWN) {
// The result is not computed.
computeBenchmarkReport();
}
for (ReportWriter writer : writers) {
writer.writeReport(this);
}
}
// TODO(b/268338967): Use a more informative name for the report.
String name() {
return NAME;
}
List<ModelBenchmarkReportInterface> modelBenchmarkReports() {
return Collections.unmodifiableList(modelBenchmarkReports);
}
BenchmarkResultType result() {
return result;
}
JSONObject toJsonObject() throws JSONException {
JSONObject jsonObject = new JSONObject();
jsonObject.put("name", NAME);
JSONArray jsonArray = new JSONArray();
for (ModelBenchmarkReportInterface modelBenchmarkReport : modelBenchmarkReports) {
jsonArray.put(modelBenchmarkReport.toJsonObject());
}
jsonObject.put("reports", jsonArray);
jsonObject.put("result", result.toString());
return jsonObject;
}
private void computeBenchmarkReport() {
List<BenchmarkResultType> results = new ArrayList<>();
for (ModelBenchmarkReportInterface modelBenchmarkReport : modelBenchmarkReports) {
results.add(modelBenchmarkReport.result());
}
result = DelegatePerformanceBenchmark.aggregateResults(/* strict= */ true, results);
}
static BenchmarkReport create() {
return new BenchmarkReport();
}
}
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkResultType.java
+82
View File
@@ -0,0 +1,82 @@
/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark.delegateperformance;
/** Enumerates the possible benchmark result values. */
public enum BenchmarkResultType {
// Unknown benchmark result, possibly due to internal failures.
UNKNOWN("UNKNOWN"),
// The benchmark results are not involved in the Pass/Pass with Warning/Fail result generation.
NOT_APPLICABLE("N/A"),
// The benchmark activity skips the Pass/Pass with Warning/Fail result generation.
SKIP("SKIP"),
// The meanings of "PASS", "PASS_WITH_WARNING" and "FAIL":
//
// Latency benchmark:
// 1. When the test target delegate type is the same as the reference delegate:
// the test target delegate is expected to be better, or at least not
// substantially worse, than the reference delegate in ALL metrics.
// - PASS: All performance metrics of the test target delegate are better
// than or equal to the reference delegate.
// - PASS_WITH_WARNING: At least 1 metric is worse than the reference
// delegate, but not substantially worse. No regression thresholds are
// breached.
// - FAIL: At least 1 regression threshold is breached.
// 2. When the test target delegate type is different from the reference
// delegate: the test delegate is only expected to be better, or at least not
// substantially worse, than the reference delegate in AT LEAST 1 metric.
// - PASS: All performance metrics of the test target delegate are better than
// or equal to the reference delegate.
// - PASS_WITH_WARNING: At least 1 metric is worse than the reference
// delegate, but, at least 1 regression threshold is not breached.
// - FAIL: All regression thresholds are breached.
// 3. The overall result is generated by aggregating the results from the pairs.
// - PASS: All results are "PASS".
// - PASS_WITH_WARNING: At least 1 "PASS_WITH_WARNING" result and 0 "FAIL"
// result.
// - FAIL: At least 1 "FAIL" result.
//
// Accuracy benchmark:
// The accuracy benchmark delegates the accuracy metric threshold checks to the metric scripts,
// which are embedded together with the test input and the ground truth inside the models. The
// metric scripts are responsible for computing the accuracy metrics (e.g. Mean square error) from
// the model predictions and the ground truth data, checking the accuracy metrics against the
// pre-defined thresholds and aggregating the results into a binarized pass status.
// 1. When the metric scripts check the outcomes for every model and every delegate:
// - PASS: No accuracy threshold in the model is breached by the delegate.
// - FAIL: At least 1 accuracy threshold threshold is breached.
// 2. The overall result is generated by aggregating the results from the delegate results for
// each model.
// - PASS: The test target delegate result is "PASS".
// - PASS_WITH_WARNING: The test delegate and all reference delegates results are "FAIL".
// - FAIL: The test target delegate result is "FAIL" and at least 1 reference delegate result
// is "PASS".
PASS("PASS"),
PASS_WITH_WARNING("PASS_WITH_WARNING"),
FAIL("FAIL");
private final String name;
BenchmarkResultType(String name) {
this.name = name;
}
@Override
public String toString() {
return name;
}
}
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/CsvWriter.java
+113
View File
@@ -0,0 +1,113 @@
/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark.delegateperformance;
import static org.tensorflow.lite.benchmark.delegateperformance.Preconditions.checkState;
import android.util.Log;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.List;
/** Helper class for writing the final report in CSV format. */
final class CsvWriter implements ReportWriter {
private static final String TAG = "TfLiteCsvWriter";
private final String destinationFolderPath;
private CsvWriter(String destinationFolderPath) {
this.destinationFolderPath = destinationFolderPath;
}
/** Writes the benchmark results into a CSV file. */
@Override
public void writeReport(BenchmarkReport report) {
// Example output file:
// Model, Metric, DELEGATE_TYPE (PATH), DELEGATE_TYPE (PATH), Change, Status,...
// model_1, metric_1, 900 , 1000, -10%, PASS, ...
// model_1, metric_2, ...
// model_1, delegate_summary,,,, PASS, ...
// model_1, model_summary, PASS,
// model_2, ...
// ...
// Summary, Summary, PASS
StringBuilder sb = new StringBuilder();
sb.append(destinationFolderPath).append("/").append(report.name()).append(".csv");
String filePath = sb.toString();
Log.i(TAG, "Generating CSV report to " + filePath);
List<ModelBenchmarkReportInterface> modelReports = report.modelBenchmarkReports();
checkState(!modelReports.isEmpty());
sb = new StringBuilder();
// Heading row. It is structured as below:
// Model, Metric, DELEGATE_TYPE (PATH), DELEGATE_TYPE (PATH), Change, Status, ...
sb.append("Model, Metric");
for (DelegateMetricsEntry entry : modelReports.get(0).processedDelegateMetrics()) {
sb.append(", Delegate: ").append(entry.delegateIdentifier());
if (!entry.isTestTarget()) {
sb.append(", Change, Status");
}
}
sb.append('\n');
for (ModelBenchmarkReportInterface modelReport : modelReports) {
writerModelReport(modelReport, sb);
}
sb.append("Summary").append(", Summary,").append(report.result());
sb.append('\n');
try (PrintWriter writer = new PrintWriter(filePath)) {
writer.write(sb.toString());
} catch (IOException e) {
Log.e(TAG, "Failed to open report file " + filePath);
}
}
private void writerModelReport(ModelBenchmarkReportInterface modelReport, StringBuilder sb) {
String modelName = modelReport.modelName();
if (modelReport.processedDelegateMetrics().isEmpty()) {
Log.w(TAG, "The computed metric is empty.");
} else {
for (String metricName : modelReport.processedDelegateMetrics().get(0).metrics().keySet()) {
sb.append(modelName).append(",").append(metricName);
for (DelegateMetricsEntry delegateMetricsEntry : modelReport.processedDelegateMetrics()) {
MetricsEntry metricEntry = delegateMetricsEntry.metrics().get(metricName);
sb.append(",").append(metricEntry.value());
if (!delegateMetricsEntry.isTestTarget()) {
sb.append(",")
.append(metricEntry.regression())
.append(",")
.append(metricEntry.result());
}
}
sb.append('\n');
}
}
sb.append(modelName).append(",delegate_summary");
for (DelegateMetricsEntry delegateMetricsEntry : modelReport.processedDelegateMetrics()) {
sb.append(",");
if (!delegateMetricsEntry.isTestTarget()) {
// Position the delegate-level result correctly.
sb.append(",,").append(delegateMetricsEntry.result());
}
}
sb.append('\n');
sb.append(modelName).append(",model_summary,").append(modelReport.result());
sb.append('\n');
}
static ReportWriter create(String destinationFolderPath) {
return new CsvWriter(destinationFolderPath);
}
}
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/DelegateMetricsEntry.java
+126
View File
@@ -0,0 +1,126 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark.delegateperformance;
import java.util.Collections;
import java.util.Map;
import org.json.JSONException;
import org.json.JSONObject;
/**
* Helper class to store the performance results that were computed by {@link ModelBenchmarkReport}
* from the raw performance metrics.
*
* <p>The computation compares the performance metrics of a reference delegate and the test target
* delegate. Then it checks the performance regressions (if any) with the criteria to generate
* metric-level and delegate-level results.
*
* <p>An instance of {@link DelegateMetricsEntry} corresponds with each tested model and each pair
* reference delegate and the test target delegate.
*/
final class DelegateMetricsEntry {
// Identifies the delegate involved in the computation.
// Format: "DELEGATE_TYPE (PATH_TO_DELEGATE_SETTINGS_FILE)"
// TODO(b/267488243): use the delegate name instead of delegate type.
private final String delegateIdentifier;
/** Map from performance metric names to {@link MetricsEntry}. */
private final Map<String, MetricsEntry> metrics;
// The delegate-level pass status. The value computation involves computing the test target
// delegate performance regressions and checking if the regression thresholds, specified in the
// criteria, are breached.
// Possible values:
// - NOT_APPLICABLE: The reference delegate is the test target delegate.
// 1. When the test target delegate type is the same as the reference delegate.
// - PASS: All performance metrics of the test target delegate are better than
// or equal to the reference delegate.
// - PASS_WITH_WARNING: No regression thresholds are breached.
// - FAIL: At least 1 regression threshold is breached.
// 2. When the test target delegate type is different from the reference delegate.
// - PASS: All performance metrics of the test target delegate are better or
// equal to the reference delegate.
// - PASS_WITH_WARNING: At least 1 regression threshold is not breached.
// - FAIL: All regression thresholds are breached.
private final BenchmarkResultType result;
private final boolean isTestTarget;
/**
* The value is {@code true} when the results are generated from comparing two delegates of the
* same type. Otherwise, the value is {@code false}.
*/
private final boolean isStrictCriteria;
private DelegateMetricsEntry(
String delegateIdentifier,
Map<String, MetricsEntry> metrics,
BenchmarkResultType result,
boolean isTestTarget,
boolean isStrictCriteria) {
this.delegateIdentifier = delegateIdentifier;
this.metrics = metrics;
this.result = result;
this.isTestTarget = isTestTarget;
this.isStrictCriteria = isStrictCriteria;
}
/** Returns an identifer to the delegate involved in the computation. */
String delegateIdentifier() {
return delegateIdentifier;
}
/** Returns an unmodifiable map from performance metric names to {@link MetricsEntry}. */
Map<String, MetricsEntry> metrics() {
return Collections.unmodifiableMap(metrics);
}
/** Returns the delegate-level pass status. */
BenchmarkResultType result() {
return result;
}
boolean isTestTarget() {
return isTestTarget;
}
/**
* Returns {@code true} when the results are generated from comparing two delegates of the same
* type. Otherwise, returns {@code false}.
*/
boolean isStrictCriteria() {
return isStrictCriteria;
}
JSONObject toJsonObject() throws JSONException {
JSONObject jsonObject = new JSONObject();
jsonObject.put("delegate_identifier", delegateIdentifier);
jsonObject.put("result", result.toString());
JSONObject metricsObject = new JSONObject();
for (Map.Entry<String, MetricsEntry> entry : metrics.entrySet()) {
metricsObject.put(entry.getKey(), entry.getValue().toJsonObject());
}
jsonObject.put("metrics", metricsObject);
jsonObject.put("is_test_target", isTestTarget);
jsonObject.put("is_strict_criteria", isStrictCriteria);
return jsonObject;
}
static DelegateMetricsEntry create(
String delegateIdentifier,
Map<String, MetricsEntry> metrics,
BenchmarkResultType result,
boolean isTestTarget,
boolean isStrictCriteria) {
return new DelegateMetricsEntry(
delegateIdentifier, metrics, result, isTestTarget, isStrictCriteria);
}
}
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/DelegatePerformanceBenchmark.java
+258
View File
@@ -0,0 +1,258 @@
/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark.delegateperformance;
import static org.tensorflow.lite.benchmark.delegateperformance.Preconditions.checkArgument;
import static org.tensorflow.lite.benchmark.delegateperformance.Preconditions.checkNotNull;
import static org.tensorflow.lite.benchmark.delegateperformance.Preconditions.checkState;
import android.util.Log;
import com.google.flatbuffers.FlatBufferBuilder;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
import java.util.Set;
import tflite.BenchmarkEvent;
import tflite.TFLiteSettings;
import tflite.proto.benchmark.DelegatePerformance.BenchmarkEventType;
import tflite.proto.benchmark.DelegatePerformance.LatencyResults;
/** Helper class for running delegate performance benchmark. */
class DelegatePerformanceBenchmark {
private static final String DELEGATE_PERFORMANCE_RESULT_FOLDER = "delegate_performance_result";
private static final String TAG = "TfLiteBenchmarkHelper";
private static final String MODEL_EXT = ".tflite";
static {
System.loadLibrary("delegate_performance_benchmark");
}
public static String createResultFolder(File filesDir, String resultFolder) throws IOException {
File resultDir = new File(filesDir, DELEGATE_PERFORMANCE_RESULT_FOLDER + "/" + resultFolder);
String resultPath = resultDir.getAbsolutePath();
if (resultDir.exists() || resultDir.mkdirs()) {
Log.i(TAG, "Logging the result to " + resultPath);
return resultPath;
}
throw new IOException("Failed to create directory for " + resultPath);
}
/**
* Extracts the model name from the model file name.
*
* <p>Strips out the ".tflite" extension from the input. Returns "model" if the input filename is
* "model.tflite".
*/
public static String getModelName(String filename) {
checkNotNull(filename);
checkArgument(filename.endsWith(MODEL_EXT));
return filename.substring(0, filename.length() - MODEL_EXT.length());
}
/**
* Returns a {@code LatencyResults} by parsing the outcome from a TFLite Benchmark Tool execution.
* If it fails to parse the outcome, this method returns a {@code LatencyResults} with an error
* event type.
*/
public static LatencyResults runLatencyBenchmark(
String[] args,
TfLiteSettingsListEntry tfliteSettingslistEntry,
int modelFd,
long modelOffset,
long modelSize) {
byte[] tfliteSettingsByteArray =
new byte[tfliteSettingslistEntry.tfliteSettings().getByteBuffer().remaining()];
tfliteSettingslistEntry.tfliteSettings().getByteBuffer().get(tfliteSettingsByteArray);
tfliteSettingslistEntry.tfliteSettings().getByteBuffer().rewind();
byte[] latencyResultsByteArray =
latencyBenchmarkNativeRun(
args,
tfliteSettingsByteArray,
tfliteSettingslistEntry.filePath(),
modelFd,
modelOffset,
modelSize);
if (latencyResultsByteArray == null || latencyResultsByteArray.length == 0) {
Log.w(
TAG,
String.format(
"Received null response from native for %s. Treating this as error.",
tfliteSettingslistEntry.filePath()));
return LatencyResults.newBuilder()
.setEventType(BenchmarkEventType.BENCHMARK_EVENT_TYPE_ERROR)
.build();
}
try {
return LatencyResults.parseFrom(latencyResultsByteArray);
} catch (IOException e) {
Log.w(
TAG,
String.format(
"Failed to parse the results running %s with exception %s.",
tfliteSettingslistEntry.filePath(), e));
return LatencyResults.newBuilder()
.setEventType(BenchmarkEventType.BENCHMARK_EVENT_TYPE_ERROR)
.build();
}
}
/** Returns a {@code BenchmarkEvent} by parsing the outcome from a MiniBenchmark execution. */
public static BenchmarkEvent runAccuracyBenchmark(
TfLiteSettingsListEntry tfliteSettingslistEntry,
int modelFd,
long modelOffset,
long modelSize,
String resultPath) {
byte[] tfliteSettingsByteArray =
new byte[tfliteSettingslistEntry.tfliteSettings().getByteBuffer().remaining()];
tfliteSettingslistEntry.tfliteSettings().getByteBuffer().get(tfliteSettingsByteArray);
tfliteSettingslistEntry.tfliteSettings().getByteBuffer().rewind();
byte[] accuracyResultsByteArray =
accuracyBenchmarkNativeRun(
tfliteSettingsByteArray, modelFd, modelOffset, modelSize, resultPath);
ByteBuffer byteBuffer = ByteBuffer.wrap(accuracyResultsByteArray);
return BenchmarkEvent.getRootAsBenchmarkEvent(byteBuffer);
}
/**
* Loads the input TFLiteSettings JSON files into TfLiteSettingsListEntry instances.
*
* <p>Adds default entry at the beginning as a reference delegate. The default entry contains a
* dummy TFLiteSettings structure, which lets the interpreter to apply the default acceleration.
* Positions the test target delegate at the end of the entry list.
*/
public static List<TfLiteSettingsListEntry> loadTfLiteSettingsList(String[] jsonFilePaths) {
List<TfLiteSettingsListEntry> tfliteSettingsList = new ArrayList<>();
// Always add the default delegate to compare.
FlatBufferBuilder tfliteSettingsBuilder = new FlatBufferBuilder();
TFLiteSettings.startTFLiteSettings(tfliteSettingsBuilder);
int tfliteSettingsOffset = TFLiteSettings.endTFLiteSettings(tfliteSettingsBuilder);
tfliteSettingsBuilder.finish(tfliteSettingsOffset);
tfliteSettingsList.add(
TfLiteSettingsListEntry.create(
TFLiteSettings.getRootAsTFLiteSettings(tfliteSettingsBuilder.dataBuffer()),
"default_delegate",
/* isTestTarget= */ false));
// To avoid system-level initialization negatively impacting benchmark results, the test target
// delegate is positioned at the end of the list in reverse order. This ensures initialization
// latency is added to the reference delegate that is executed first among the delegates that
// share the same delegate type. Otherwise, it could introduce false positive results when
// comparing delegates that share the same initialization. The order of the input settings
// files will be reinstated when computing the model-level reports.
for (int i = jsonFilePaths.length - 1; i >= 0; i--) {
String jsonFilePath = jsonFilePaths[i];
byte[] tfliteSettingsByteArray = loadTfLiteSettingsJsonNative(jsonFilePath);
if (tfliteSettingsByteArray == null || tfliteSettingsByteArray.length == 0) {
Log.e(TAG, "Failed to load TFLiteSetting from JSON file " + jsonFilePath);
return new ArrayList<>();
}
ByteBuffer byteBuffer = ByteBuffer.wrap(tfliteSettingsByteArray);
tfliteSettingsList.add(
TfLiteSettingsListEntry.create(
TFLiteSettings.getRootAsTFLiteSettings(byteBuffer),
jsonFilePath,
/* isTestTarget= */ i == 0));
}
return tfliteSettingsList;
}
/**
* Aggregates a list of {@link BenchmarkResultType} into a {@link BenchmarkResultType} based on
* the requirements. {@code strict} is set to {@code true} when comparing two delegates with the
* same types or aggregating the results into model-level and session-level. {@code strict} is set
* to {@code false} when comparing two delegates with different types.
*
* <p>If {@code strict} is set to {@code true}, returns:
*
* <ul>
* <li>PASS if all elements in {@code results} are PASS.
* <li>PASS_WITH_WARNING if at least one element in {@code results} is PASS_WITH_WARNING and
* {@code results} doesn't contain FAIL.
* <li>FAIL if at least one element in {@code results} is FAIL.
* </ul>
*
* Otherwise, returns:
*
* <ul>
* <li>PASS if all elements in {@code results} are PASS.
* <li>PASS_WITH_WARNING if at least one element in {@code results} is PASS_WITH_WARNING or
* PASS.
* <li>FAIL if all elements in {@code results} are FAIL.
* </ul>
*/
public static BenchmarkResultType aggregateResults(
boolean strict, List<BenchmarkResultType> results) {
checkState(!results.isEmpty());
checkState(
allMatch(
results,
EnumSet.of(
BenchmarkResultType.PASS,
BenchmarkResultType.PASS_WITH_WARNING,
BenchmarkResultType.FAIL)));
if (strict) {
if (results.contains(BenchmarkResultType.FAIL)) {
return BenchmarkResultType.FAIL;
}
if (results.contains(BenchmarkResultType.PASS_WITH_WARNING)) {
return BenchmarkResultType.PASS_WITH_WARNING;
}
return BenchmarkResultType.PASS;
} else {
if (allMatch(results, EnumSet.of(BenchmarkResultType.PASS))) {
return BenchmarkResultType.PASS;
}
if (allMatch(results, EnumSet.of(BenchmarkResultType.FAIL))) {
return BenchmarkResultType.FAIL;
}
return BenchmarkResultType.PASS_WITH_WARNING;
}
}
/**
* Returns {@code true} when all elements in {@code results} are in the value range specified by
* {@code targets}.
*/
private static boolean allMatch(
List<BenchmarkResultType> results, Set<BenchmarkResultType> targets) {
for (BenchmarkResultType result : results) {
if (!targets.contains(result)) {
return false;
}
}
return true;
}
private static native byte[] latencyBenchmarkNativeRun(
String[] args,
byte[] tfliteSettings,
String tfliteSettingsPath,
int modelFd,
long modelOffset,
long modelSize);
private static native byte[] accuracyBenchmarkNativeRun(
byte[] tfliteSettings, int modelFd, long modelOffset, long modelSize, String resultPath);
private static native byte[] loadTfLiteSettingsJsonNative(String jsonFilePath);
}
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/HtmlWriter.java
+205
View File
@@ -0,0 +1,205 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark.delegateperformance;
import static org.tensorflow.lite.benchmark.delegateperformance.Preconditions.checkState;
import android.util.Log;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.List;
/** Helper class for writing the final report in HTML format. */
final class HtmlWriter implements ReportWriter {
private static final String TAG = "TfLiteHtmlWriter";
private final String destinationFolderPath;
private HtmlWriter(String destinationFolderPath) {
this.destinationFolderPath = destinationFolderPath;
}
/** Writes the benchmark results into an HTML file. */
@Override
public void writeReport(BenchmarkReport report) {
// Example output file contains an overall summary table like:
//
// Summary PASS
//
// And a detailed methods table like:
//
// Model Metric DELEGATE_TYPE (PATH) DELEGATE_TYPE (PATH) Change Status ...
// model_1 metric_1 900 1000 -10% PASS ...
// model_1 metric_2 ...
// model_1 delegate_summary PASS ...
// model_1 model_summary PASS
// model_2 ...
// ...
StringBuilder sb = new StringBuilder();
sb.append(destinationFolderPath).append("/").append(report.name()).append(".html");
String filePath = sb.toString();
Log.i(TAG, "Generating an HTML report to " + filePath);
List<ModelBenchmarkReportInterface> modelReports = report.modelBenchmarkReports();
checkState(!modelReports.isEmpty());
// File header
sb =
new StringBuilder()
.append("<html>\n")
.append("<head>\n")
.append("<title>Delegate Performance Benchmark Report</title>\n")
.append("<style>\n")
.append("body {\n")
.append(" font-family: -apple-system, system-ui, BlinkMacSystemFont, \"Segoe UI\",\n")
.append(" \"Roboto\", \"Helvetica Neue\", Arial, sans-serif;\n")
.append("}\n")
.append("table, th, td {\n")
.append(" border: 1px solid black;\n")
.append(" border-collapse: collapse;\n")
.append(" padding: 15px;\n")
.append(" text-align: center;\n")
.append("}\n")
.append("th {\n")
.append(" background-color:LightSkyBlue\n")
.append("}\n")
.append("tr:nth-child(even) {\n")
.append(" background-color: whitesmoke;\n")
.append("}\n")
.append(".status-PASS {\n")
.append(" background-color: chartreuse;\n")
.append("}\n")
.append(".status-FAIL {\n")
.append(" background-color: red;\n")
.append("}\n")
.append(".status-PASS_WITH_WARNING {\n")
.append(" background-color: yellow;\n")
.append("}\n")
.append(".status-NOT_APPLICABLE {\n")
.append(" background-color: grey;\n")
.append("}\n")
.append(".ls-info-box {\n")
.append(" padding: 5px 10px;\n")
.append(" margin: 10px;\n")
.append(" border-style: solid;\n")
.append(" border-left-width: 5px;\n")
.append(" border-radius: 4px;\n")
.append(" color: #666;\n")
.append(" font-size: 13px;\n")
.append(" line-height: 1.3\n")
.append("}\n")
.append(".ls-info-box {\n")
.append(" background-color: #f0f0f0;\n")
.append(" border-left-color: silver\n")
.append("}\n")
.append("</style>\n")
.append("</head>\n")
.append("<body>\n")
// TODO(b/268338967): Use a more informative name for the report.
.append("<h1>Delegate Performance Benchmark Report</h1>\n");
// Summary table
sb.append("<table>\n").append("<tr>\n").append("<td>Summary</td>\n");
addResultCell(report.result(), /* isStrictCriteria= */ null, sb);
sb.append("</tr>\n").append("</table>\n");
// Heading row for the detailed metric table. It is structured as below:
// Model, Metric, DELEGATE_TYPE (PATH), DELEGATE_TYPE (PATH), Change, Status, ...
sb.append("<table>\n")
.append("<thead>\n")
.append("<tr>\n")
.append("<th>Model</th>\n")
.append("<th>Metric</th>\n");
for (DelegateMetricsEntry entry : modelReports.get(0).processedDelegateMetrics()) {
sb.append("<th>Delegate: ").append(entry.delegateIdentifier()).append("</th>\n");
if (!entry.isTestTarget()) {
sb.append("<th>Change (%)</th>\n").append("<th>Status</th>\n");
}
}
sb.append("</thead>\n").append("<tbody>\n");
for (ModelBenchmarkReportInterface modelReport : modelReports) {
writerModelReport(modelReport, sb);
}
sb.append("</tbody>\n")
.append("</table>\n")
.append("<div class=\"ls-info-box\">\n")
.append("<p>\n")
.append(
"When the test target delegate type is the same as the reference delegate, the checks"
+ " are more strict. Otherwise, the checks are relaxed. Please see \n")
.append(
"<a href=\"https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkResultType.java\">BenchmarkResultType.java</a>\n")
.append(" for the meanings of PASS, PASS_WITH_WARNING and FAIL.\n")
.append("</p>\n")
.append("</div>\n")
.append("</body>\n")
.append("</html>\n");
try (PrintWriter writer = new PrintWriter(filePath)) {
writer.write(sb.toString());
} catch (IOException e) {
Log.e(TAG, "Failed to open report file " + filePath);
}
}
private void writerModelReport(ModelBenchmarkReportInterface modelReport, StringBuilder sb) {
String modelName = modelReport.modelName();
if (modelReport.processedDelegateMetrics().isEmpty()) {
Log.w(TAG, "The computed metric is empty.");
} else {
for (String metricName : modelReport.processedDelegateMetrics().get(0).metrics().keySet()) {
sb.append("<tr>\n<td>")
.append(modelName)
.append("</td>\n<td>")
.append(metricName)
.append("</td>\n");
for (DelegateMetricsEntry delegateMetricsEntry : modelReport.processedDelegateMetrics()) {
MetricsEntry metricEntry = delegateMetricsEntry.metrics().get(metricName);
sb.append("<td>").append(metricEntry.value()).append("</td>\n");
if (!delegateMetricsEntry.isTestTarget()) {
sb.append("<td>").append(metricEntry.regression()).append("</td>\n");
addResultCell(metricEntry.result(), /* isStrictCriteria= */ null, sb);
}
}
sb.append("</tr>\n");
}
sb.append("<tr>\n<td>").append(modelName).append("</td>\n<td>delegate_summary</td>\n");
for (DelegateMetricsEntry delegateMetricsEntry : modelReport.processedDelegateMetrics()) {
// Position the delegate-level result correctly.
sb.append("<td/>\n");
if (!delegateMetricsEntry.isTestTarget()) {
sb.append("<td/>\n");
addResultCell(delegateMetricsEntry.result(), delegateMetricsEntry.isStrictCriteria(), sb);
}
}
sb.append("</tr>\n<tr>\n<td>").append(modelName).append("</td>\n<td>model_summary</td>\n");
addResultCell(modelReport.result(), /* isStrictCriteria= */ null, sb);
sb.append("</tr>\n");
}
}
/** Adds a colored result cell to the table. */
private void addResultCell(
BenchmarkResultType result, Boolean isStrictCriteria, StringBuilder sb) {
sb.append("<td class=\"status-").append(result.name()).append("\">").append(result);
if (isStrictCriteria != null && isStrictCriteria) {
sb.append(" (strict)");
}
sb.append("</td>\n");
}
static ReportWriter create(String destinationFolderPath) {
return new HtmlWriter(destinationFolderPath);
}
}
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/JsonWriter.java
+67
View File
@@ -0,0 +1,67 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark.delegateperformance;
import android.util.Log;
import java.io.IOException;
import java.io.PrintWriter;
import org.json.JSONException;
/** Helper class for writing the final report in JSON format. */
final class JsonWriter implements ReportWriter {
private static final String TAG = "TfLiteJsonWriter";
private final String destinationFolderPath;
private JsonWriter(String destinationFolderPath) {
this.destinationFolderPath = destinationFolderPath;
}
// Writes the benchmark results into a JSON file.
// Example output file:
// {
// "reports": [
// {
// "model": "model_name",
// "metrics": [...],
// "raw_metrics": [... ],
// "max_regression_percentage_allowed": { ... },
// "result": "PASS"
// }
// , ...
// ],
// "name": "report",
// "result": "PASS"
// }
@Override
public void writeReport(BenchmarkReport report) {
StringBuilder sb = new StringBuilder();
sb.append(destinationFolderPath).append("/").append(report.name()).append(".json");
String filePath = sb.toString();
try (PrintWriter writer = new PrintWriter(filePath, "UTF-8")) {
writer.println(report.toJsonObject().toString(/* indentFactor= */ 4));
Log.i(TAG, "Generated report " + filePath + ".");
} catch (IOException e) {
Log.e(TAG, "Failed to open report file " + filePath + "." + e);
} catch (JSONException e) {
Log.e(TAG, "Failed to convert the benchmark report to JSON." + e);
}
}
static ReportWriter create(String destinationFolderPath) {
return new JsonWriter(destinationFolderPath);
}
}
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/LatencyBenchmarkReport.java
+93
View File
@@ -0,0 +1,93 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark.delegateperformance;
import static java.lang.Math.max;
import static org.tensorflow.lite.benchmark.delegateperformance.Preconditions.checkNotNull;
import static org.tensorflow.lite.benchmark.delegateperformance.Preconditions.checkState;
import android.util.Log;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import tflite.proto.benchmark.DelegatePerformance.BenchmarkEventType;
import tflite.proto.benchmark.DelegatePerformance.BenchmarkMetric;
import tflite.proto.benchmark.DelegatePerformance.LatencyCriteria;
import tflite.proto.benchmark.DelegatePerformance.LatencyResults;
/** Model-level latency benchmark report class. */
final class LatencyBenchmarkReport extends ModelBenchmarkReport {
private static final String TAG = "LatencyBenchmarkReport";
private LatencyBenchmarkReport(
String modelName,
List<RawDelegateMetricsEntry> rawDelegateMetricsEntries,
LatencyCriteria criteria) {
super(modelName);
checkNotNull(criteria);
this.maxRegressionPercentageAllowed.put(
"startup_overhead_latency_us",
(double) criteria.getStartupOverheadMaxRegressionPercentageAllowed());
this.maxRegressionPercentageAllowed.put(
"inference_latency_average_us",
(double) criteria.getAverageInferenceMaxRegressionPercentageAllowed());
Log.i(TAG, "Creating a latency benchmark report for " + modelName);
computeModelReport(rawDelegateMetricsEntries);
}
/**
* Parses {@link LatencyResults} into the unified {@link RawDelegateMetricsEntry} format for
* further processing.
*/
public static RawDelegateMetricsEntry parseResults(
LatencyResults results, TfLiteSettingsListEntry entry) {
// Use {@code LinkedHashMap} to keep the metrics insertion order.
Map<String, Double> metrics = new LinkedHashMap<>();
if (results.getEventType() != BenchmarkEventType.BENCHMARK_EVENT_TYPE_END) {
Log.w(TAG, "The latency benchmarking is not completed successfully for " + entry.filePath());
return RawDelegateMetricsEntry.create(
entry.tfliteSettings().delegate(), entry.filePath(), entry.isTestTarget(), metrics);
}
for (BenchmarkMetric metric : results.getMetricsList()) {
if (metric != null) {
metrics.put(metric.getName(), (double) metric.getValue());
}
}
checkState(metrics.containsKey("initialization_latency_us"));
checkState(metrics.containsKey("warmup_latency_average_us"));
checkState(metrics.containsKey("inference_latency_average_us"));
metrics.put(
"startup_overhead_latency_us",
max(
metrics.get("initialization_latency_us")
+ metrics.get("warmup_latency_average_us")
- metrics.get("inference_latency_average_us"),
// The average warmup latency is generally assumed to be greater than the average
// inference latency. Therefore, it is highly unusual for the sum of the initialization
// latency and the average warmup latency to be less than the average inference latency.
// In order to ensure that the regression is computed successfully, we keep the startup
// overhead latency at minimum 0.
0d));
return RawDelegateMetricsEntry.create(
entry.tfliteSettings().delegate(), entry.filePath(), entry.isTestTarget(), metrics);
}
static LatencyBenchmarkReport create(
String modelName,
List<RawDelegateMetricsEntry> rawDelegateMetricsEntries,
LatencyCriteria criteria) {
return new LatencyBenchmarkReport(modelName, rawDelegateMetricsEntries, criteria);
}
}
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/MetricsEntry.java
+72
View File
@@ -0,0 +1,72 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark.delegateperformance;
import org.json.JSONException;
import org.json.JSONObject;
/**
* Helper class to store the metric value, the test target delegate regression compared with the
* reference delegate and the metric-level results based on the criteria.
*
* <p>An instance of {@link MetricsEntry} corresponds with each tested model metric and each pair of
* test target delegate and reference delegate.
*
* <p>TODO(b/267429312): use AutoValue here to simplify the source code.
*/
final class MetricsEntry {
private final double value;
// The test target delegate's performance regression compared against the reference delegate.
// Example value: "5%".
private final String regression;
// The metric pass status after checking the regression with the criteria.
// Possible values:
// - NOT_APPLICABLE: This performance metric is not involved in the criteria.
// - PASS: The performance metric of the test target delegate are better or equal to the reference
// delegate.
// - PASS_WITH_WARNING: The regression doesn't breach the threshold specified in the criteria.
// - FAIL: The regression breaches the threshold specified in the criteria.
private final BenchmarkResultType result;
private MetricsEntry(double value, String regression, BenchmarkResultType result) {
this.value = value;
this.regression = regression;
this.result = result;
}
double value() {
return value;
}
String regression() {
return regression;
}
BenchmarkResultType result() {
return result;
}
JSONObject toJsonObject() throws JSONException {
JSONObject jsonObject = new JSONObject();
jsonObject.put("value", value);
jsonObject.put("regression", regression);
jsonObject.put("result", result.toString());
return jsonObject;
}
static MetricsEntry create(double value, String regression, BenchmarkResultType result) {
return new MetricsEntry(value, regression, result);
}
}
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/ModelBenchmarkReport.java
+203
View File
@@ -0,0 +1,203 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark.delegateperformance;
import static org.tensorflow.lite.benchmark.delegateperformance.Preconditions.checkState;
import android.util.Log;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
/**
* Model-level benchmark report class to be extended by {@link AccuracyBenchmarkReport} and {@link
* LatencyBenchmarkReport}.
*
* <p>This class contains helper functions to convert the raw performance results from the native
* layer into metric-level, delegate-level and model-level pass status values.
*/
public abstract class ModelBenchmarkReport implements ModelBenchmarkReportInterface {
private static final String TAG = "ModelBenchmarkReport";
protected final String modelName;
/* Map from performance metric names to the maximum regression percentage thresholds allowed. */
protected final Map<String, Double> maxRegressionPercentageAllowed = new HashMap<>();
/** List of {@link DelegateMetricsEntry}, which stores delegate-level performance results. */
protected final List<DelegateMetricsEntry> processedDelegateMetrics = new ArrayList<>();
/** Model-level pass status. */
protected BenchmarkResultType result = BenchmarkResultType.UNKNOWN;
protected ModelBenchmarkReport(String modelName) {
this.modelName = modelName;
}
@Override
public String modelName() {
return modelName;
}
@Override
public List<DelegateMetricsEntry> processedDelegateMetrics() {
return Collections.unmodifiableList(processedDelegateMetrics);
}
@Override
public BenchmarkResultType result() {
return result;
}
@Override
public JSONObject toJsonObject() throws JSONException {
JSONObject jsonObject = new JSONObject();
jsonObject.put("model", modelName);
jsonObject.put("result", result.toString());
JSONArray processedDelegateMetricsArray = new JSONArray();
for (DelegateMetricsEntry entry : processedDelegateMetrics) {
processedDelegateMetricsArray.put(entry.toJsonObject());
}
jsonObject.put("metrics", processedDelegateMetricsArray);
JSONObject maxRegressionPercentageAllowedObject = new JSONObject();
for (Map.Entry<String, Double> entry : maxRegressionPercentageAllowed.entrySet()) {
maxRegressionPercentageAllowedObject.put(entry.getKey(), entry.getValue());
}
jsonObject.put("max_regression_percentage_allowed", maxRegressionPercentageAllowedObject);
return jsonObject;
}
/**
* Converts the prepopulated list of {@link RawDelegateMetricsEntry}, the raw performance results
* collected from the native layer, into a list of {@link DelegateMetricsEntry}.
*
* <p>The computation compares the test target delegate with every reference delegate by computing
* the regression of the raw performance results. The list of {@link DelegateMetricsEntry} stores
* the calculated results for all pairs. {@link ModelBenchmarkReport} stores both the raw
* performance results and the processed performance results.
*/
protected void computeModelReport(List<RawDelegateMetricsEntry> rawDelegateMetrics) {
if (!processedDelegateMetrics.isEmpty()) {
// Metrics are computed.
Log.i(TAG, "Delegate metrics are already computed. Skips the computation.");
return;
}
// At least 2 delegates (the default delegate and the test target delegate) are tested.
checkState(rawDelegateMetrics.size() >= 2);
// The test target delegate is the last delegate to benchmark. The order will be guaranteed by
// DelegatePerformanceBenchmark#loadTfLiteSettingsList().
RawDelegateMetricsEntry testTarget = rawDelegateMetrics.get(rawDelegateMetrics.size() - 1);
checkState(testTarget.isTestTarget());
// Use {@code LinkedHashMap} to keep the insertion order.
Map<String, MetricsEntry> metrics = new LinkedHashMap<>();
for (String metricName : testTarget.metrics().keySet()) {
metrics.put(
metricName,
MetricsEntry.create(
testTarget.metrics().get(metricName), "N/A", BenchmarkResultType.NOT_APPLICABLE));
}
processedDelegateMetrics.add(
DelegateMetricsEntry.create(
testTarget.delegateIdentifier(),
metrics,
BenchmarkResultType.NOT_APPLICABLE,
testTarget.isTestTarget(),
/* isStrictCriteria= */ false));
// Processes the reference delegate results. Compute the performance regressions by comparing
// them with the results from the test target delegate.
List<BenchmarkResultType> referenceResults = new ArrayList<>();
// Traverses the list in reverse order, so that the exported order of items matches with the
// input delegate setting files order.
for (int reference = rawDelegateMetrics.size() - 2; reference >= 0; reference--) {
RawDelegateMetricsEntry entry = rawDelegateMetrics.get(reference);
Map<String, MetricsEntry> referenceMetrics = new LinkedHashMap<>();
List<BenchmarkResultType> metricResults = new ArrayList<>();
for (String metricName : testTarget.metrics().keySet()) {
MetricsEntry metricEntry =
computeReferenceMetricEntry(
entry.metrics().get(metricName), testTarget.metrics().get(metricName), metricName);
referenceMetrics.put(metricName, metricEntry);
// Filters for the metrics that are involved in the Pass/Pass with Warning/Fail result
// generation.
if (metricEntry.result() != BenchmarkResultType.NOT_APPLICABLE) {
metricResults.add(metricEntry.result());
}
}
// TODO(b/267488243): Load the delegate name from the native layer.
boolean sameDelegateType = entry.delegateName().equals(testTarget.delegateName());
BenchmarkResultType referenceDelegateResult =
DelegatePerformanceBenchmark.aggregateResults(sameDelegateType, metricResults);
referenceResults.add(referenceDelegateResult);
processedDelegateMetrics.add(
DelegateMetricsEntry.create(
entry.delegateIdentifier(),
referenceMetrics,
referenceDelegateResult,
entry.isTestTarget(),
sameDelegateType));
}
result = DelegatePerformanceBenchmark.aggregateResults(/* strict= */ true, referenceResults);
}
private MetricsEntry computeReferenceMetricEntry(
Double referenceValue, Double testTargetValue, String metricName) {
boolean checkRegression = maxRegressionPercentageAllowed.containsKey(metricName);
String regression = "N/A";
BenchmarkResultType result =
checkRegression ? BenchmarkResultType.FAIL : BenchmarkResultType.NOT_APPLICABLE;
if (referenceValue == null || testTargetValue == null) {
return MetricsEntry.create(referenceValue, regression, result);
}
// Here is a mitigation to the lack of support for criteria operators. "ok" metric is handled
// specifically for the accuracy benchmarking results.
// TODO(b/267313326): remove the mitigation after the criteria operators are added.
if (metricName.equals("ok")) {
if (testTargetValue == AccuracyBenchmarkReport.PASS) {
// The test target delegate passed the accuracy checks.
result = BenchmarkResultType.PASS;
} else if (referenceValue == AccuracyBenchmarkReport.FAIL) {
// Both the test target and the reference delegates failed the accuracy checks. Therefore,
// it is not considered as a regression.
result = BenchmarkResultType.PASS_WITH_WARNING;
}
return MetricsEntry.create(referenceValue, regression, result);
}
// Here we assume that lower values of the metric are better, for all of our metrics.
// TODO(b/267313326): Remove the assumption with the criteria operator support.
double regressionValue = 0.0;
if (!testTargetValue.equals(referenceValue)) {
regressionValue = (testTargetValue - referenceValue) / referenceValue;
}
if (checkRegression) {
if (regressionValue <= 0) {
result = BenchmarkResultType.PASS;
} else if (regressionValue <= maxRegressionPercentageAllowed.get(metricName) / 100f) {
result = BenchmarkResultType.PASS_WITH_WARNING;
}
}
regression = toPercentage(regressionValue);
return MetricsEntry.create(referenceValue, regression, result);
}
private String toPercentage(double n) {
return String.format(Locale.ENGLISH, "%.1f", n * 100) + "%";
}
}
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/ModelBenchmarkReportInterface.java
+48
View File
@@ -0,0 +1,48 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark.delegateperformance;
import java.util.List;
import org.json.JSONException;
import org.json.JSONObject;
/**
* An interface to handle metric computations and exporting results for a benchmark run on a single
* model. It is used in {@link ModelBenchmarkReport}.
*/
public interface ModelBenchmarkReportInterface {
/** Returns the model name. */
String modelName();
/** Returns the list of processed performance results for all delegates. */
List<DelegateMetricsEntry> processedDelegateMetrics();
/**
* Returns the model-level pass status.
*
* <p>Possible return values:
*
* <ul>
* <li>1. PASS: All delegate-level results are "PASS" or "NOT_APPLICABLE".
* <li>2. PASS_WITH_WARNING: At least 1 "PASS_WITH_WARNING" delegate-level result and 0 "FAIL"
* delegate-level result.
* <li>3. FAIL: At least 1 "FAIL" delegate-level result.
* </ul>
*/
BenchmarkResultType result();
/** Serializes the report and returns a {@code JSONObject}. */
JSONObject toJsonObject() throws JSONException;
}
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/Preconditions.java
+62
View File
@@ -0,0 +1,62 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark.delegateperformance;
/** Helper class for checking preconditions. */
final class Preconditions {
/**
* Ensures that an object reference passed as a parameter to the calling method is not null.
*
* <p>TODO(b/250876587): Consider adding proper annotation support.
*
* @param reference an object reference
* @return the non-null reference that was validated
* @throws NullPointerException if {@code reference} is null
*/
public static <T> T checkNotNull(/* @Nullable */ T reference) {
if (reference == null) {
throw new NullPointerException();
}
return reference;
}
/**
* Ensures the truth of an expression involving one or more parameters to the calling method.
*
* @param expression a boolean expression
* @throws IllegalArgumentException if {@code expression} is false
*/
public static void checkArgument(boolean expression) {
if (!expression) {
throw new IllegalArgumentException();
}
}
/**
* Ensures the truth of an expression involving the state of the calling instance, but not
* involving any parameters to the calling method.
*
* @param expression a boolean expression
* @throws IllegalStateException if {@code expression} is false
*/
public static void checkState(boolean expression) {
if (!expression) {
throw new IllegalStateException();
}
}
private Preconditions() {}
}
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/RawDelegateMetricsEntry.java
+78
View File
@@ -0,0 +1,78 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark.delegateperformance;
import java.util.Collections;
import java.util.Map;
import tflite.Delegate;
/**
* Helper class to store the raw performance results from the native layer.
*
* <p>An instance of {@link RawDelegateMetricsEntry} corresponds with each tested model and each
* delegate.
*/
final class RawDelegateMetricsEntry {
// The name of the delegate. The available names are listed in
// tensorflow/tensorflow/lite/acceleration/configuration/configuration.proto
// TODO(b/267431570): consider replacing the field with an Enum value.
private final String delegateName;
// Specifies the path to the delegate settings file.
private final String path;
// If {@link isTestTarget} is set to {@code true}, the metrics are gathered from the benchmark run
// with the test target delegate.
private final boolean isTestTarget;
private final Map<String, Double> metrics;
private RawDelegateMetricsEntry(
int delegate, String path, boolean isTestTarget, Map<String, Double> metrics) {
this.delegateName = Delegate.name(delegate);
this.path = path;
this.isTestTarget = isTestTarget;
this.metrics = metrics;
}
/* Returns the name of the delegate. */
String delegateName() {
return delegateName;
}
/* Returns the path to the delegate settings file. */
String path() {
return path;
}
/* Returns whether the delegate is the test target. */
boolean isTestTarget() {
return isTestTarget;
}
Map<String, Double> metrics() {
return Collections.unmodifiableMap(metrics);
}
/**
* Returns an identifier for the delegate. The idenfier consists of the delegate type and the path
* to the delegate settings file.
*/
String delegateIdentifier() {
return delegateName + " (" + path + ")";
}
static RawDelegateMetricsEntry create(
int delegate, String path, boolean isTestTarget, Map<String, Double> metrics) {
return new RawDelegateMetricsEntry(delegate, path, isTestTarget, metrics);
}
}
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/ReportWriter.java
+24
View File
@@ -0,0 +1,24 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark.delegateperformance;
/**
* An interface to handle exporting {@link BenchmarkReport}. It will be used in {@link CsvWriter},
* {@link JsonWriter} and {@link HtmlWriter}.
*/
public interface ReportWriter {
/** Exports {@link BenchmarkReport} to the target format. */
void writeReport(BenchmarkReport report);
}
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/TfLiteSettingsListEntry.java
+70
View File
@@ -0,0 +1,70 @@
/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark.delegateperformance;
import static org.tensorflow.lite.benchmark.delegateperformance.Preconditions.checkNotNull;
import tflite.TFLiteSettings;
/** Helper class for store the data before and after benchmark runs. */
final class TfLiteSettingsListEntry {
private static final String TAG = "TfLiteSettingsListEntry";
private final TFLiteSettings tfliteSettings;
private final String filePath;
private final boolean isTestTarget;
private TfLiteSettingsListEntry(
TFLiteSettings tfliteSettings, String filePath, boolean isTestTarget) {
checkNotNull(tfliteSettings);
checkNotNull(filePath);
this.tfliteSettings = tfliteSettings;
this.filePath = filePath;
this.isTestTarget = isTestTarget;
}
TFLiteSettings tfliteSettings() {
return tfliteSettings;
}
String filePath() {
return filePath;
}
boolean isTestTarget() {
return isTestTarget;
}
@Override
public String toString() {
return "TfLiteSettingsListEntry{"
// TODO(b/265268620): Dump the entire TFLiteSettings buffer.
+ "delegate="
+ tfliteSettings.delegate()
+ ", "
+ "filePath="
+ filePath
+ ", "
+ "isTestTarget="
+ isTestTarget
+ "}";
}
static TfLiteSettingsListEntry create(
TFLiteSettings tfliteSettings, String filePath, boolean isTestTarget) {
return new TfLiteSettingsListEntry(tfliteSettings, filePath, isTestTarget);
}
}
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/BUILD
+78
View File
@@ -0,0 +1,78 @@
# Description:
# Holds the native layer of the app.
load("@rules_cc//cc:cc_library.bzl", "cc_library")
load("//tensorflow/lite/core/shims:cc_library_with_tflite.bzl", "cc_library_with_tflite", "jni_binary_with_tflite")
load("//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android:build_defs.bzl", "accuracy_benchmark_extra_deps", "latency_benchmark_extra_deps")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
default_visibility = ["//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android:__subpackages__"],
licenses = ["notice"],
)
jni_binary_with_tflite(
name = "libdelegate_performance_benchmark.so",
srcs = ["delegate_performance_benchmark_jni.cc"],
tflite_deps = [":accuracy_benchmark"],
deps = [
":latency_benchmark",
"//tensorflow/lite/acceleration/configuration:configuration_fbs",
"//tensorflow/lite/delegates/utils/experimental/stable_delegate:tflite_settings_json_parser",
"//tensorflow/lite/java/jni",
"//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto:delegate_performance_cc_proto",
"@flatbuffers",
],
)
cc_library(
name = "status_codes",
hdrs = ["status_codes.h"],
)
cc_library(
name = "latency_benchmark",
srcs = ["latency_benchmark.cc"],
hdrs = ["latency_benchmark.h"],
deps = [
"//tensorflow/core/util:stats_calculator_portable",
"//tensorflow/lite:minimal_logging",
"//tensorflow/lite/acceleration/configuration:configuration_fbs",
"//tensorflow/lite/c:c_api_types",
"//tensorflow/lite/profiling:memory_info",
"//tensorflow/lite/tools/benchmark:benchmark_model_lib",
"//tensorflow/lite/tools/benchmark:benchmark_params",
"//tensorflow/lite/tools/benchmark:benchmark_tflite_model_lib",
"//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto:delegate_performance_cc_proto",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
] + latency_benchmark_extra_deps(),
)
cc_library_with_tflite(
name = "accuracy_benchmark",
srcs = ["accuracy_benchmark.cc"],
hdrs = ["accuracy_benchmark.h"],
deps = [
":status_codes",
"//tensorflow/lite:minimal_logging",
"//tensorflow/lite/acceleration/configuration:configuration_fbs",
"//tensorflow/lite/acceleration/configuration:gpu_plugin",
"//tensorflow/lite/acceleration/configuration:stable_delegate_plugin",
"//tensorflow/lite/acceleration/configuration:xnnpack_plugin",
"//tensorflow/lite/core/acceleration/configuration:nnapi_plugin",
"//tensorflow/lite/delegates/utils/experimental/stable_delegate:tflite_settings_json_parser",
"//tensorflow/lite/experimental/acceleration/mini_benchmark:libjpeg_handle_static_link",
"//tensorflow/lite/experimental/acceleration/mini_benchmark:status_codes",
"//tensorflow/lite/experimental/acceleration/mini_benchmark/c:c_api_in_process",
"//tensorflow/lite/kernels/internal:compatibility",
"//tensorflow/lite/tools:command_line_flags",
"//tensorflow/lite/tools:tool_params",
"@flatbuffers",
] + accuracy_benchmark_extra_deps(),
)
cc_library_with_tflite(
name = "benchmark_native",
tflite_jni_binaries = [":libdelegate_performance_benchmark.so"],
)
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/accuracy_benchmark.cc
+137
View File
@@ -0,0 +1,137 @@
/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/accuracy_benchmark.h"
#include <errno.h>
#include <stdio.h>
#include <sys/stat.h>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <string>
#include <vector>
#include "flatbuffers/base.h" // from @flatbuffers
#include "flatbuffers/buffer.h" // from @flatbuffers
#include "flatbuffers/flatbuffer_builder.h" // from @flatbuffers
#include "flatbuffers/flatbuffers.h" // from @flatbuffers
#include "flatbuffers/vector.h" // from @flatbuffers
#include "tensorflow/lite/acceleration/configuration/configuration_generated.h"
#include "tensorflow/lite/experimental/acceleration/mini_benchmark/c/c_api.h"
#include "tensorflow/lite/kernels/internal/compatibility.h"
#include "tensorflow/lite/logger.h"
#include "tensorflow/lite/minimal_logging.h"
#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/status_codes.h"
namespace tflite {
namespace benchmark {
namespace accuracy {
namespace {
std::vector<const tflite::BenchmarkEvent*> ToBenchmarkEvents(uint8_t* data,
size_t size) {
std::vector<const tflite::BenchmarkEvent*> results;
uint8_t* current_root = data;
while (current_root < data + size) {
flatbuffers::uoffset_t current_size =
flatbuffers::GetPrefixedSize(current_root);
results.push_back(
flatbuffers::GetSizePrefixedRoot<tflite::BenchmarkEvent>(current_root));
current_root += current_size + sizeof(flatbuffers::uoffset_t);
}
// Checks the data read is not over the bounds of the buffer.
TFLITE_CHECK_EQ(current_root, data + size);
return results;
}
} // namespace
flatbuffers::Offset<BenchmarkEvent> Benchmark(
flatbuffers::FlatBufferBuilder& fbb, const TFLiteSettings& tflite_settings,
int model_fd, size_t model_offset, size_t model_size,
const char* result_path_chars) {
std::string result_path(result_path_chars);
std::string storage_path = result_path + "/storage_path.fb";
int return_code = std::remove(storage_path.c_str());
if (return_code) {
TFLITE_LOG_PROD(TFLITE_LOG_WARNING,
"Failed to remove storage file (%s): %s.",
storage_path.c_str(), strerror(errno));
}
flatbuffers::FlatBufferBuilder mini_benchmark_fbb;
TFLiteSettingsT tflite_settings_t;
tflite_settings.UnPackTo(&tflite_settings_t);
flatbuffers::Offset<TFLiteSettings> tflite_settings_offset =
CreateTFLiteSettings(mini_benchmark_fbb, &tflite_settings_t);
flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<TFLiteSettings>>>
tflite_settings_vector_offset =
mini_benchmark_fbb.CreateVector({tflite_settings_offset});
ModelFileBuilder model_file_builder(mini_benchmark_fbb);
model_file_builder.add_fd(model_fd);
model_file_builder.add_offset(model_offset);
model_file_builder.add_length(model_size);
flatbuffers::Offset<ModelFile> model_file_offset =
model_file_builder.Finish();
flatbuffers::Offset<BenchmarkStoragePaths> storage_paths_offset =
CreateBenchmarkStoragePaths(mini_benchmark_fbb,
mini_benchmark_fbb.CreateString(storage_path),
mini_benchmark_fbb.CreateString(result_path));
flatbuffers::Offset<ValidationSettings> validation_settings_offset =
CreateValidationSettings(mini_benchmark_fbb,
/*per_test_timeout_ms=*/5000);
mini_benchmark_fbb.Finish(CreateMinibenchmarkSettings(
mini_benchmark_fbb, tflite_settings_vector_offset, model_file_offset,
storage_paths_offset, validation_settings_offset));
TfLiteMiniBenchmarkSettings* settings = TfLiteMiniBenchmarkSettingsCreate();
TfLiteMiniBenchmarkSettingsSetFlatBufferData(
settings, mini_benchmark_fbb.GetBufferPointer(),
mini_benchmark_fbb.GetSize());
TfLiteMiniBenchmarkResult* result =
TfLiteBlockingValidatorRunnerTriggerValidation(settings);
std::vector<const BenchmarkEvent*> events =
ToBenchmarkEvents(TfLiteMiniBenchmarkResultFlatBufferData(result),
TfLiteMiniBenchmarkResultFlatBufferDataSize(result));
TfLiteMiniBenchmarkSettingsFree(settings);
// The settings contains one test only. Therefore, the benchmark checks for
// the first result only.
if (events.size() != 1) {
TfLiteMiniBenchmarkResultFree(result);
TFLITE_LOG_PROD(
TFLITE_LOG_ERROR,
"Number of result events (%zu) doesn't match the expectation (%zu).",
events.size(), 1);
flatbuffers::Offset<BenchmarkError> error =
CreateBenchmarkError(fbb, BenchmarkStage_INFERENCE,
/*exit_code=*/kBenchmarkResultCountMismatch);
BenchmarkEventBuilder builder(fbb);
builder.add_event_type(BenchmarkEventType_ERROR);
builder.add_error(error);
return builder.Finish();
}
BenchmarkEventT benchmark_event;
events[0]->UnPackTo(&benchmark_event);
TfLiteMiniBenchmarkResultFree(result);
return CreateBenchmarkEvent(fbb, &benchmark_event);
}
} // namespace accuracy
} // namespace benchmark
} // namespace tflite
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/accuracy_benchmark.h
+51
View File
@@ -0,0 +1,51 @@
/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_DELEGATE_PERFORMANCE_ANDROID_SRC_MAIN_NATIVE_ACCURACY_BENCHMARK_H_
#define TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_DELEGATE_PERFORMANCE_ANDROID_SRC_MAIN_NATIVE_ACCURACY_BENCHMARK_H_
#include <cstddef>
#include <string>
#include <vector>
#include "flatbuffers/buffer.h" // from @flatbuffers
#include "flatbuffers/flatbuffer_builder.h" // from @flatbuffers
#include "tensorflow/lite/acceleration/configuration/configuration_generated.h"
namespace tflite {
namespace benchmark {
namespace accuracy {
// Triggers MiniBenchmark testings. Uses the arguments passed from the testing
// app to configure MiniBenchmark ValidatorRunner. The tests will access and
// execute the pre-embedded models in the app via the model file descriptor. The
// contents of a model are initialized using model_size bytes starting at
// model_offset position in the file described by model_fd. Any intermediate
// data and results will be dumped to the result path given.
//
// Returns a BenchmarkEvent flatbuffer offset. If the benchmark tests finish
// successfully with a pass from MiniBenchmark, the returned offset contains the
// concrete accuracy metrics and the overall result from MiniBenchmark.
// Otherwise, the returned value contains an error code.
flatbuffers::Offset<BenchmarkEvent> Benchmark(
flatbuffers::FlatBufferBuilder& fbb, const TFLiteSettings& tflite_settings,
int model_fd, size_t model_offset, size_t model_size,
const char* result_path_chars);
} // namespace accuracy
} // namespace benchmark
} // namespace tflite
#endif // TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_DELEGATE_PERFORMANCE_ANDROID_SRC_MAIN_NATIVE_ACCURACY_BENCHMARK_H_
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/delegate_performance_benchmark_jni.cc
+155
View File
@@ -0,0 +1,155 @@
/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <jni.h>
#include <cstdint>
#include <string>
#include <vector>
#include "flatbuffers/flatbuffer_builder.h" // from @flatbuffers
#include "tensorflow/lite/acceleration/configuration/configuration_generated.h"
#include "tensorflow/lite/delegates/utils/experimental/stable_delegate/tflite_settings_json_parser.h"
#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto/delegate_performance.pb.h"
#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/accuracy_benchmark.h"
#ifndef TFLITE_WITH_STABLE_ABI
#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/latency_benchmark.h"
#endif // !TFLITE_WITH_STABLE_ABI
namespace {
// A helper method that converts an array of strings passed from Java to a
// vector of strings in C++.
std::vector<std::string> toStringVector(JNIEnv* env,
jobjectArray string_array) {
int len = env->GetArrayLength(string_array);
std::vector<std::string> vec(len);
for (int i = 0; i < len; ++i) {
jstring str =
static_cast<jstring>(env->GetObjectArrayElement(string_array, i));
const char* chars = env->GetStringUTFChars(str, nullptr);
vec[i] = std::string(chars);
env->ReleaseStringUTFChars(str, chars);
env->DeleteLocalRef(str);
}
return vec;
}
// Serializes the proto message into jbyteArray.
jbyteArray CppProtoToBytes(JNIEnv* env, const google::protobuf::MessageLite& proto) {
jbyteArray array = nullptr;
const int byte_size = proto.ByteSizeLong();
if (byte_size) {
array = env->NewByteArray(byte_size);
void* ptr = env->GetPrimitiveArrayCritical(array, nullptr);
proto.SerializeWithCachedSizesToArray(static_cast<uint8_t*>(ptr));
env->ReleasePrimitiveArrayCritical(array, ptr, 0);
}
return array;
}
} // namespace
extern "C" {
JNIEXPORT jbyteArray JNICALL
Java_org_tensorflow_lite_benchmark_delegateperformance_DelegatePerformanceBenchmark_latencyBenchmarkNativeRun(
JNIEnv* env, jclass clazz, jobjectArray args_obj,
jbyteArray tflite_settings_byte_array, jstring tflite_settings_path_obj,
jint model_fd, jlong model_offset, jlong model_size) {
tflite::proto::benchmark::LatencyResults results;
// The latency benchmark doesn't support TF Lite with the stable ABI path.
#ifndef TFLITE_WITH_STABLE_ABI
std::vector<std::string> args = toStringVector(env, args_obj);
const char* tflite_settings_path_chars =
env->GetStringUTFChars(tflite_settings_path_obj, nullptr);
jbyte* tflite_settings_bytes =
env->GetByteArrayElements(tflite_settings_byte_array, nullptr);
const tflite::TFLiteSettings* tflite_settings =
flatbuffers::GetRoot<tflite::TFLiteSettings>(
reinterpret_cast<const char*>(tflite_settings_bytes));
results = tflite::benchmark::latency::Benchmark(
*tflite_settings, tflite_settings_path_chars, static_cast<int>(model_fd),
static_cast<size_t>(model_offset), static_cast<size_t>(model_size), args);
env->ReleaseByteArrayElements(tflite_settings_byte_array,
tflite_settings_bytes, JNI_ABORT);
env->ReleaseStringUTFChars(tflite_settings_path_obj,
tflite_settings_path_chars);
#endif // !TFLITE_WITH_STABLE_ABI
return CppProtoToBytes(env, results);
}
// TODO(b/262411020): Consider returning jobject directly.
JNIEXPORT jbyteArray JNICALL
Java_org_tensorflow_lite_benchmark_delegateperformance_DelegatePerformanceBenchmark_accuracyBenchmarkNativeRun(
JNIEnv* env, jclass clazz, jbyteArray tflite_settings_byte_array,
jint model_fd, jlong model_offset, jlong model_size,
jstring result_path_obj) {
const char* result_path_chars =
env->GetStringUTFChars(result_path_obj, nullptr);
jbyte* tflite_settings_bytes =
env->GetByteArrayElements(tflite_settings_byte_array, nullptr);
const tflite::TFLiteSettings* tflite_settings =
flatbuffers::GetRoot<tflite::TFLiteSettings>(
reinterpret_cast<const char*>(tflite_settings_bytes));
flatbuffers::FlatBufferBuilder fbb;
flatbuffers::Offset<tflite::BenchmarkEvent> benchmark_event =
tflite::benchmark::accuracy::Benchmark(
fbb, *tflite_settings, static_cast<int>(model_fd),
static_cast<size_t>(model_offset), static_cast<size_t>(model_size),
result_path_chars);
fbb.Finish(benchmark_event);
env->ReleaseByteArrayElements(tflite_settings_byte_array,
tflite_settings_bytes, JNI_ABORT);
env->ReleaseStringUTFChars(result_path_obj, result_path_chars);
jbyteArray byte_array = nullptr;
if (fbb.GetSize() > 0) {
byte_array = env->NewByteArray(fbb.GetSize());
env->SetByteArrayRegion(
byte_array, 0, fbb.GetSize(),
reinterpret_cast<const jbyte*>(fbb.GetBufferPointer()));
}
return byte_array;
}
JNIEXPORT jbyteArray JNICALL
Java_org_tensorflow_lite_benchmark_delegateperformance_DelegatePerformanceBenchmark_loadTfLiteSettingsJsonNative(
JNIEnv* env, jclass clazz, jstring json_file_path_obj) {
const char* json_file_path_chars =
env->GetStringUTFChars(json_file_path_obj, nullptr);
tflite::delegates::utils::TfLiteSettingsJsonParser parser;
parser.Parse(json_file_path_chars);
jbyteArray tflite_settings_byte_array = nullptr;
flatbuffers::uoffset_t tflite_settings_size = parser.GetBufferSize();
if (tflite_settings_size > 0) {
tflite_settings_byte_array = env->NewByteArray(tflite_settings_size);
env->SetByteArrayRegion(
tflite_settings_byte_array, 0, tflite_settings_size,
reinterpret_cast<const jbyte*>(parser.GetBufferPointer()));
}
env->ReleaseStringUTFChars(json_file_path_obj, json_file_path_chars);
return tflite_settings_byte_array;
}
} // extern "C"
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/latency_benchmark.cc
+254
View File
@@ -0,0 +1,254 @@
/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/latency_benchmark.h"
#include <errno.h>
#include <sys/stat.h>
#include <cstddef>
#include <cstdint>
#include <string>
#include <vector>
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "tensorflow/core/util/stats_calculator.h"
#include "tensorflow/lite/acceleration/configuration/configuration_generated.h"
#include "tensorflow/lite/c/c_api_types.h"
#include "tensorflow/lite/logger.h"
#include "tensorflow/lite/minimal_logging.h"
#include "tensorflow/lite/profiling/memory_info.h"
#include "tensorflow/lite/tools/benchmark/benchmark_model.h"
#include "tensorflow/lite/tools/benchmark/benchmark_params.h"
#include "tensorflow/lite/tools/benchmark/benchmark_tflite_model.h"
#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto/delegate_performance.pb.h"
namespace tflite {
namespace benchmark {
namespace latency {
namespace {
static constexpr char kBenchmarkToolName[] = "(BenchmarkModelAndroid)";
// The listener subscribes to the benchmark lifecycle and parses the
// benchmarking results into a LatencyResults proto message. The message will be
// reported later to the app.
class DelegatePerformanceReportingListener : public BenchmarkListener {
public:
void OnBenchmarkStart(const BenchmarkParams& unused) override {
results_proto_.set_event_type(proto::benchmark::BENCHMARK_EVENT_TYPE_START);
}
// TFLite Benchmark Tool triggers this method at the end of a benchmark for
// logging the results.
void OnBenchmarkEnd(const BenchmarkResults& results) override {
ReportResult(results);
}
void ReportFailure(TfLiteStatus status) {
std::string status_msg =
status == kTfLiteError
? "TFLite error"
: (status == kTfLiteDelegateError ? "TFLite delegate error"
: "unexpected TFLite status");
TFLITE_LOG_PROD(TFLITE_LOG_ERROR,
"Benchmark failed due to %s with status code %d.",
status_msg.c_str(), status);
results_proto_.set_event_type(proto::benchmark::BENCHMARK_EVENT_TYPE_ERROR);
results_proto_.mutable_error()->mutable_error_code()->set_tflite_error(
status);
results_proto_.mutable_error()->set_error_message(status_msg);
}
const proto::benchmark::LatencyResults& GetResults() {
return results_proto_;
}
private:
// TODO(b/262399611): Consider putting metric related logic into a separate
// file.
void ReportResult(const BenchmarkResults& results) {
tensorflow::Stat<int64_t> warmup_us = results.warmup_time_us();
tensorflow::Stat<int64_t> inference_us = results.inference_time_us();
profiling::memory::MemoryUsage init_mem_usage = results.init_mem_usage();
profiling::memory::MemoryUsage overall_mem_usage =
results.overall_mem_usage();
if (results.model_size_mb() > 0) {
// Ignores invalid model sizes to avoid confusions.
AddMetric(/*name=*/"model_size_megabyte",
/*value=*/results.model_size_mb());
}
AddMetric(/*name=*/"initialization_latency_us",
/*value=*/results.startup_latency_us());
AddMetric(/*name=*/"warmup_latency_average_us", /*value=*/warmup_us.avg());
AddMetric(/*name=*/"warmup_latency_min_us", /*value=*/warmup_us.min());
AddMetric(/*name=*/"warmup_latency_max_us", /*value=*/warmup_us.max());
AddMetric(/*name=*/"warmup_latency_standard_deviation_us",
/*value=*/warmup_us.std_deviation());
AddMetric(/*name=*/"inference_latency_average_us",
/*value=*/inference_us.avg());
AddMetric(/*name=*/"inference_latency_min_us",
/*value=*/inference_us.min());
AddMetric(/*name=*/"inference_latency_max_us",
/*value=*/inference_us.max());
AddMetric(/*name=*/"inference_latency_standard_deviation_us",
/*value=*/inference_us.std_deviation());
AddMetric(/*name=*/"initialization_memory_max_rss_mebibyte",
/*value=*/init_mem_usage.mem_footprint_kb / 1024.0);
AddMetric(/*name=*/"initialization_memory_total_non_mmapped_heap_mebibyte",
/*value=*/init_mem_usage.total_allocated_bytes / 1024.0 / 1024.0);
AddMetric(
/*name=*/"initialization_memory_in_use_heap_mebibyte",
/*value=*/init_mem_usage.in_use_allocated_bytes / 1024.0 / 1024.0);
AddMetric(/*name=*/"overall_memory_max_rss_mebibyte",
/*value=*/overall_mem_usage.mem_footprint_kb / 1024.0);
AddMetric(
/*name=*/"overall_memory_total_non_mmapped_heap_mebibyte",
/*value=*/overall_mem_usage.total_allocated_bytes / 1024.0 / 1024.0);
AddMetric(
/*name=*/"overall_memory_in_use_heap_mebibyte",
/*value=*/overall_mem_usage.in_use_allocated_bytes / 1024.0 / 1024.0);
results_proto_.set_event_type(proto::benchmark::BENCHMARK_EVENT_TYPE_END);
TFLITE_LOG_PROD(TFLITE_LOG_INFO, "Benchmark finished.");
}
void AddMetric(std::string name, float value) {
proto::benchmark::BenchmarkMetric* metric = results_proto_.add_metrics();
metric->set_name(name);
metric->set_value(value);
}
proto::benchmark::LatencyResults results_proto_;
};
// Converts the input TFLiteSettings into TFLite Benchmark Tool arguments.
// Please see tensorflow/lite/tools/benchmark.
std::vector<std::string> ParseArgumentsFromTfLiteSettings(
const TFLiteSettings& tflite_settings,
const std::string& tflite_settings_path) {
std::vector<std::string> args;
if (tflite_settings_path.empty()) {
return args;
}
if (tflite_settings.stable_delegate_loader_settings()) {
args.push_back(absl::StrFormat("--stable_delegate_settings_file=%s",
tflite_settings_path));
return args;
}
switch (tflite_settings.delegate()) {
case Delegate_XNNPACK: {
args.push_back("--use_xnnpack=true");
if (tflite_settings.xnnpack_settings()) {
if (tflite_settings.xnnpack_settings()->num_threads()) {
args.push_back(absl::StrFormat(
"--num_threads=%d",
tflite_settings.xnnpack_settings()->num_threads()));
}
if (tflite_settings.xnnpack_settings()->flags() ==
XNNPackFlags_TFLITE_XNNPACK_DELEGATE_FLAG_FORCE_FP16) {
args.push_back("--xnnpack_force_fp16=true");
}
}
return args;
}
case Delegate_GPU: {
args.push_back("--use_gpu=true");
const tflite::GPUSettings* gpu_settings = tflite_settings.gpu_settings();
if (gpu_settings) {
if (gpu_settings->is_precision_loss_allowed()) {
args.push_back("--gpu_precision_loss_allowed=true");
}
if (gpu_settings->enable_quantized_inference()) {
args.push_back("--gpu_experimental_enable_quant=true");
}
if (gpu_settings->inference_preference() ==
GPUInferenceUsage_GPU_INFERENCE_PREFERENCE_SUSTAINED_SPEED) {
args.push_back("--gpu_inference_for_sustained_speed=true");
}
if (gpu_settings->force_backend() == GPUBackend_OPENCL) {
args.push_back("--gpu_backend=cl");
} else if (gpu_settings->force_backend() == GPUBackend_OPENGL) {
args.push_back("--gpu_backend=gl");
}
if (gpu_settings->cache_directory()) {
args.push_back(
absl::StrFormat("--delegate_serialize_dir=%s",
gpu_settings->cache_directory()->c_str()));
}
if (gpu_settings->model_token()) {
args.push_back(absl::StrFormat("--delegate_serialize_token=%s",
gpu_settings->model_token()->c_str()));
}
}
break;
}
case Delegate_EDGETPU: {
args.push_back("--use_edgetpu=true");
break;
}
default:
TFLITE_LOG_PROD(TFLITE_LOG_WARNING,
"Delegate type %s is not enabled by the latency module.",
EnumNameDelegate(tflite_settings.delegate()));
break;
}
if (tflite_settings.disable_default_delegates()) {
// Currently TFLite Benchmark Tool doesn't support handling the case with
// applying XNNPack delegate explicitly and disabling the XNNPack delegate
// as the default delegate at the same time. When the
// "disable_default_delegates" configuration is set to true, it only takes
// effect if the delegate is not set to XNNPack. Otherwise, the default
// delegates will still be enabled.
args.push_back("--use_xnnpack=false");
}
return args;
}
} // namespace
proto::benchmark::LatencyResults Benchmark(
const TFLiteSettings& tflite_settings,
const std::string& tflite_settings_path, int model_fd, size_t model_offset,
size_t model_size, const std::vector<std::string>& args) {
// Constructs a fake argv command-line object for the benchmark.
std::vector<char*> argv;
argv.push_back(const_cast<char*>(kBenchmarkToolName));
std::string arg_graph =
absl::StrCat("--graph=fd:", model_fd, ":", model_offset, ":", model_size);
argv.push_back(const_cast<char*>(arg_graph.data()));
std::vector<std::string> args_from_tflite_settings =
ParseArgumentsFromTfLiteSettings(tflite_settings, tflite_settings_path);
for (const std::string& arg : args_from_tflite_settings) {
argv.push_back(const_cast<char*>(arg.data()));
}
// Keep the args here for any additional TFLite Benchmark Tool configurations.
for (const std::string& arg : args) {
argv.push_back(const_cast<char*>(arg.data()));
}
BenchmarkTfLiteModel benchmark;
DelegatePerformanceReportingListener delegatePerformanceReporting;
benchmark.AddListener(&delegatePerformanceReporting);
TfLiteStatus status = benchmark.Run(argv.size(), argv.data());
if (status != kTfLiteOk) {
delegatePerformanceReporting.ReportFailure(status);
}
return delegatePerformanceReporting.GetResults();
}
} // namespace latency
} // namespace benchmark
} // namespace tflite
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/latency_benchmark.h
+49
View File
@@ -0,0 +1,49 @@
/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_DELEGATE_PERFORMANCE_ANDROID_SRC_MAIN_NATIVE_LATENCY_BENCHMARK_H_
#define TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_DELEGATE_PERFORMANCE_ANDROID_SRC_MAIN_NATIVE_LATENCY_BENCHMARK_H_
#include <string>
#include <vector>
#include "tensorflow/lite/acceleration/configuration/configuration_generated.h"
#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto/delegate_performance.pb.h"
namespace tflite {
namespace benchmark {
namespace latency {
// Triggers TFLite Benchmark Tool. Passes the "args" from the testing app to
// directly to TFLite Benchmark Tool. Converts the "tflite_settings" to
// command-line options to configure TFLite Benchmark Tool. If the latency
// benchmarking uses a stable delegate, the "tflite_settings_path" is passed to
// enable the stable delegate provider. The contents of the tested model are
// initialized using model_size bytes starting at model_offset position in the
// file referenced by the file descriptor model_fd.
//
// Returns a LatencyResults proto message. If the benchmark tests finish
// successfully from TFLite Benchmark Tool, the message contains the latency
// metrics. Otherwise, the message contains the corresponding error.
proto::benchmark::LatencyResults Benchmark(
const TFLiteSettings& tflite_settings,
const std::string& tflite_settings_path, int model_fd, size_t model_offset,
size_t model_size, const std::vector<std::string>& args);
} // namespace latency
} // namespace benchmark
} // namespace tflite
#endif // TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_DELEGATE_PERFORMANCE_ANDROID_SRC_MAIN_NATIVE_LATENCY_BENCHMARK_H_
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/status_codes.h
+35
View File
@@ -0,0 +1,35 @@
/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_DELEGATE_PERFORMANCE_ANDROID_SRC_MAIN_NATIVE_STATUS_CODES_H_
#define TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_DELEGATE_PERFORMANCE_ANDROID_SRC_MAIN_NATIVE_STATUS_CODES_H_
namespace tflite {
namespace benchmark {
enum DelegatePerformanceBenchmarkStatus {
kBenchmarkUnknownStatus = 0,
// Set of error codes that are used as the return codes to communicate between
// the native layer and the caller app.
kBenchmarkRunnerInitializationFailed = 1000,
kBenchmarkResultCountMismatch = 1001,
kBenchmarkInvalidTfLiteSettings = 1002,
};
} // namespace benchmark
} // namespace tflite
#endif // TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_DELEGATE_PERFORMANCE_ANDROID_SRC_MAIN_NATIVE_STATUS_CODES_H_
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/test/native/BUILD
+61
View File
@@ -0,0 +1,61 @@
# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# Description:
# Holds the tests for the native layer of the app.
load("@rules_cc//cc:cc_test.bzl", "cc_test")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
licenses = ["notice"],
)
cc_test(
name = "latency_benchmark_test",
srcs = ["latency_benchmark_test.cc"],
data = [
"//tensorflow/lite/delegates/utils/experimental/sample_stable_delegate:tensorflowlite_sample_stable_delegate",
"//tensorflow/lite/tools/delegates/experimental/stable_delegate:test_sample_stable_delegate_settings.json",
"@tflite_mobilenet_float//:mobilenet_v1_1.0_224.tflite",
],
deps = [
"//tensorflow/lite/acceleration/configuration:configuration_fbs",
"//tensorflow/lite/delegates/utils/experimental/stable_delegate:tflite_settings_json_parser",
"//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto:delegate_performance_cc_proto",
"//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native:latency_benchmark",
"@com_google_googletest//:gtest_main",
"@flatbuffers",
],
)
cc_test(
name = "accuracy_benchmark_test",
srcs = ["accuracy_benchmark_test.cc"],
data = [
"//tensorflow/lite/delegates/utils/experimental/sample_stable_delegate:tensorflowlite_sample_stable_delegate",
"//tensorflow/lite/delegates/utils/experimental/stable_delegate:test_xnnpack_settings.json",
"//tensorflow/lite/tools/delegates/experimental/stable_delegate:test_sample_stable_delegate_settings.json",
],
deps = [
"//tensorflow/lite/acceleration/configuration:configuration_fbs",
"//tensorflow/lite/delegates/utils/experimental/stable_delegate:tflite_settings_json_parser",
"//tensorflow/lite/experimental/acceleration/mini_benchmark:embedded_mobilenet_validation_model",
"//tensorflow/lite/experimental/acceleration/mini_benchmark:mini_benchmark_test_helper",
"//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native:accuracy_benchmark",
"//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native:status_codes",
"@com_google_googletest//:gtest_main",
"@flatbuffers",
],
)
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/test/native/accuracy_benchmark_test.cc
+149
View File
@@ -0,0 +1,149 @@
/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/accuracy_benchmark.h"
#include <fcntl.h>
#include <stdio.h>
#include <iostream>
#include <string>
#include <vector>
#include <gtest/gtest.h>
#include "flatbuffers/buffer.h" // from @flatbuffers
#include "flatbuffers/flatbuffer_builder.h" // from @flatbuffers
#include "tensorflow/lite/acceleration/configuration/configuration_generated.h"
#include "tensorflow/lite/delegates/utils/experimental/stable_delegate/tflite_settings_json_parser.h"
#include "tensorflow/lite/experimental/acceleration/mini_benchmark/embedded_mobilenet_validation_model.h"
#include "tensorflow/lite/experimental/acceleration/mini_benchmark/mini_benchmark_test_helper.h"
#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/status_codes.h"
namespace tflite {
namespace benchmark {
namespace accuracy {
namespace {
class AccuracyBenchmarkTest : public ::testing::Test {
protected:
void SetUp() override {
acceleration::MiniBenchmarkTestHelper helper;
should_perform_test_ = helper.should_perform_test();
if (!should_perform_test_) {
return;
}
std::string embedded_model_path = helper.DumpToTempFile(
"mobilenet_quant_with_validation.tflite",
g_tflite_acceleration_embedded_mobilenet_validation_model,
g_tflite_acceleration_embedded_mobilenet_validation_model_len);
ASSERT_FALSE(embedded_model_path.empty());
model_fp_ = fopen(embedded_model_path.c_str(), "rb");
ASSERT_NE(model_fp_, nullptr);
ASSERT_EQ(fseek(model_fp_, 0, SEEK_END), 0);
model_size_ = ftell(model_fp_);
ASSERT_NE(model_size_, -1);
ASSERT_EQ(fseek(model_fp_, 0, SEEK_SET), 0);
result_path_ = ::testing::TempDir();
}
void TearDown() override { fclose(model_fp_); }
std::string result_path_;
size_t model_size_;
FILE* model_fp_;
bool should_perform_test_ = true;
};
TEST_F(AccuracyBenchmarkTest, FailedWithInvalidModelFileDescriptor) {
if (!should_perform_test_) {
std::cerr << "Skipping test";
return;
}
delegates::utils::TfLiteSettingsJsonParser parser;
flatbuffers::FlatBufferBuilder builder;
std::vector<std::string> args;
const TFLiteSettings* tflite_settings = parser.Parse(
"tensorflow/lite/tools/delegates/experimental/"
"stable_delegate/test_sample_stable_delegate_settings.json");
flatbuffers::Offset<BenchmarkEvent> offset =
Benchmark(builder, *tflite_settings, /*model_fd=*/0,
/*model_offset=*/0, /*model_size=*/0, result_path_.c_str());
builder.Finish(offset);
const BenchmarkEvent* event =
flatbuffers::GetRoot<BenchmarkEvent>(builder.GetBufferPointer());
ASSERT_NE(event, nullptr);
EXPECT_EQ(event->event_type(), BenchmarkEventType_ERROR);
ASSERT_NE(event->error(), nullptr);
EXPECT_EQ(event->error()->stage(), BenchmarkStage_INFERENCE);
EXPECT_EQ(event->error()->exit_code(),
DelegatePerformanceBenchmarkStatus::kBenchmarkResultCountMismatch);
}
TEST_F(AccuracyBenchmarkTest, SucceedWithSampleStableDelegate) {
if (!should_perform_test_) {
std::cerr << "Skipping test";
return;
}
delegates::utils::TfLiteSettingsJsonParser parser;
flatbuffers::FlatBufferBuilder builder;
const TFLiteSettings* tflite_settings = parser.Parse(
"tensorflow/lite/tools/delegates/experimental/"
"stable_delegate/test_sample_stable_delegate_settings.json");
flatbuffers::Offset<BenchmarkEvent> offset = Benchmark(
builder, *tflite_settings, /*model_fd=*/fileno(model_fp_),
/*model_offset=*/0, /*model_size=*/model_size_, result_path_.c_str());
builder.Finish(offset);
const BenchmarkEvent* event =
flatbuffers::GetRoot<BenchmarkEvent>(builder.GetBufferPointer());
// TODO(b/253442685): verify that the stable delegate was used.
ASSERT_NE(event, nullptr);
EXPECT_EQ(event->event_type(), BenchmarkEventType_END);
EXPECT_EQ(event->error(), nullptr);
}
TEST_F(AccuracyBenchmarkTest, SucceedWithEmbeddedValidationAndXNNPack) {
if (!should_perform_test_) {
std::cerr << "Skipping test";
return;
}
delegates::utils::TfLiteSettingsJsonParser parser;
flatbuffers::FlatBufferBuilder builder;
const TFLiteSettings* tflite_settings = parser.Parse(
"tensorflow/lite/delegates/utils/experimental/"
"stable_delegate/test_xnnpack_settings.json");
flatbuffers::Offset<BenchmarkEvent> offset = Benchmark(
builder, *tflite_settings, /*model_fd=*/fileno(model_fp_),
/*model_offset=*/0, /*model_size=*/model_size_, result_path_.c_str());
builder.Finish(offset);
const BenchmarkEvent* event =
flatbuffers::GetRoot<BenchmarkEvent>(builder.GetBufferPointer());
// TODO(b/253442685): verify that the XNNPack delegate was used.
ASSERT_NE(event, nullptr);
EXPECT_EQ(event->event_type(), BenchmarkEventType_END);
EXPECT_EQ(event->error(), nullptr);
}
} // namespace
} // namespace accuracy
} // namespace benchmark
} // namespace tflite
tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/test/native/latency_benchmark_test.cc
+153
View File
@@ -0,0 +1,153 @@
/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/latency_benchmark.h"
#include <fcntl.h>
#include <string>
#include <vector>
#include <gtest/gtest.h>
#include "flatbuffers/buffer.h" // from @flatbuffers
#include "flatbuffers/flatbuffer_builder.h" // from @flatbuffers
#include "tensorflow/lite/acceleration/configuration/configuration_generated.h"
#include "tensorflow/lite/delegates/utils/experimental/stable_delegate/tflite_settings_json_parser.h"
#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto/delegate_performance.pb.h"
namespace tflite {
namespace benchmark {
namespace latency {
namespace {
static constexpr char kModelPath[] =
"../tflite_mobilenet_float/"
"mobilenet_v1_1.0_224.tflite";
static constexpr char kSettingsFilePath[] =
"tensorflow/lite/tools/delegates/experimental/stable_delegate/"
"test_sample_stable_delegate_settings.json";
class LatencyBenchmarkTest : public ::testing::Test {
protected:
void SetUp() override {
model_fp_ = fopen(kModelPath, "rb");
ASSERT_TRUE(model_fp_ != nullptr);
ASSERT_EQ(fseek(model_fp_, 0, SEEK_END), 0);
model_size_ = ftell(model_fp_);
ASSERT_NE(model_size_, -1);
ASSERT_EQ(fseek(model_fp_, 0, SEEK_SET), 0);
settings_ = parser_.Parse(kSettingsFilePath);
}
delegates::utils::TfLiteSettingsJsonParser parser_;
const TFLiteSettings* settings_;
size_t model_size_;
FILE* model_fp_;
std::vector<std::string> args_;
};
TEST_F(LatencyBenchmarkTest, FailedWithNullFileDescriptor) {
EXPECT_TRUE(Benchmark(*settings_, kSettingsFilePath,
/*model_fd=*/0, /*model_offset=*/0,
/*model_size=*/0, args_)
.has_error());
}
TEST_F(LatencyBenchmarkTest, FailedWithInvalidNumThreadsSettings) {
flatbuffers::FlatBufferBuilder fbb;
flatbuffers::Offset<tflite::XNNPackSettings> xnnpack_settings =
CreateXNNPackSettings(fbb, /*num_threads=*/-3);
TFLiteSettingsBuilder tflite_settings_builder(fbb);
tflite_settings_builder.add_delegate(Delegate_XNNPACK);
tflite_settings_builder.add_xnnpack_settings(xnnpack_settings);
fbb.Finish(tflite_settings_builder.Finish());
const TFLiteSettings* settings =
flatbuffers::GetRoot<TFLiteSettings>(fbb.GetBufferPointer());
EXPECT_TRUE(Benchmark(*settings,
/*tflite_settings_path=*/"example_path",
fileno(model_fp_),
/*model_offset=*/0, model_size_, args_)
.has_error());
}
TEST_F(LatencyBenchmarkTest, SucceedWithEmptyTfLiteSettings) {
flatbuffers::FlatBufferBuilder fbb;
TFLiteSettingsBuilder tflite_settings_builder(fbb);
fbb.Finish(tflite_settings_builder.Finish());
const TFLiteSettings* settings =
flatbuffers::GetRoot<TFLiteSettings>(fbb.GetBufferPointer());
// TODO(b/253442685): verify that the default delegate was used.
EXPECT_EQ(Benchmark(*settings, /*tflite_settings_path=*/"example_path",
fileno(model_fp_), /*model_offset=*/0, model_size_, args_)
.event_type(),
proto::benchmark::BENCHMARK_EVENT_TYPE_END);
}
TEST_F(LatencyBenchmarkTest, SucceedWithCpuTfLiteSettings) {
flatbuffers::FlatBufferBuilder fbb;
TFLiteSettingsBuilder tflite_settings_builder(fbb);
tflite_settings_builder.add_disable_default_delegates(true);
fbb.Finish(tflite_settings_builder.Finish());
const TFLiteSettings* settings =
flatbuffers::GetRoot<TFLiteSettings>(fbb.GetBufferPointer());
// TODO(b/253442685): verify that no model delegation has occurred.
EXPECT_EQ(Benchmark(*settings, /*tflite_settings_path=*/"example_path",
fileno(model_fp_), /*model_offset=*/0, model_size_, args_)
.event_type(),
proto::benchmark::BENCHMARK_EVENT_TYPE_END);
}
#ifdef __ANDROID__
TEST_F(LatencyBenchmarkTest, SucceedWithGpuTfLiteSettings) {
flatbuffers::FlatBufferBuilder fbb;
TFLiteSettingsBuilder tflite_settings_builder(fbb);
tflite_settings_builder.add_delegate(Delegate_GPU);
fbb.Finish(tflite_settings_builder.Finish());
const TFLiteSettings* settings =
flatbuffers::GetRoot<TFLiteSettings>(fbb.GetBufferPointer());
// TODO(b/253442685): verify that the GPU delegate was used.
EXPECT_EQ(Benchmark(*settings, /*tflite_settings_path=*/"example_path",
fileno(model_fp_), /*model_offset=*/0, model_size_, args_)
.event_type(),
proto::benchmark::BENCHMARK_EVENT_TYPE_END);
}
#endif // __ANDROID__
TEST_F(LatencyBenchmarkTest, SucceedWithSampleStableDelegate) {
// TODO(b/253442685): verify that the stable delegate was used.
EXPECT_EQ(Benchmark(*settings_, kSettingsFilePath, fileno(model_fp_),
/*model_offset=*/0, model_size_, args_)
.event_type(),
proto::benchmark::BENCHMARK_EVENT_TYPE_END);
}
TEST_F(LatencyBenchmarkTest,
SucceedWithSampleStableDelegateAndBenchmarkToolArguments) {
std::vector<std::string> args = {"--warmup_runs=10"};
// TODO(b/253442685): verify that the stable delegate was used.
EXPECT_EQ(Benchmark(*settings_, kSettingsFilePath, fileno(model_fp_),
/*model_offset=*/0, model_size_, args)
.event_type(),
proto::benchmark::BENCHMARK_EVENT_TYPE_END);
}
} // namespace
} // namespace latency
} // namespace benchmark
} // namespace tflite
tensorflow/lite/tools/benchmark/experimental/firebase/android/AndroidManifest.xml
+51
View File
@@ -0,0 +1,51 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright 2020 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="org.tensorflow.lite.benchmark.firebase">
<!-- Necessary for loading custom models from disk and writing result to disk. -->
<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE"/>
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE"/>
<uses-sdk
android:minSdkVersion="23"
android:targetSdkVersion="23" />
<application>
<!-- Number of Firebase Game Loop test scenarios defined in this application. -->
<meta-data
android:name="com.google.test.loops"
android:value="10" />
<!-- This Activity runs on the Firebase Test Lab. -->
<activity
android:name=".BenchmarkModelActivity"
android:screenOrientation="portrait"
android:label="TFLite Benchmark on Firebase Test Lab"
android:theme="@android:style/Theme.NoDisplay"
android:exported="true"
android:noHistory="true">
<!-- Intent filter for the Firebase Game Loop test. -->
<intent-filter>
<action android:name="com.google.intent.action.TEST_LOOP" />
<category android:name="android.intent.category.DEFAULT" />
<data android:mimeType="application/javascript" />
</intent-filter>
</activity>
</application>
</manifest>
tensorflow/lite/tools/benchmark/experimental/firebase/android/BUILD
+62
View File
@@ -0,0 +1,62 @@
# Description:
# BenchmarkModel Android harness for Firebase Test Lab.
load("@build_bazel_rules_android//android:rules.bzl", "android_binary")
load("@rules_cc//cc:cc_library.bzl", "cc_library")
load("//tensorflow/lite:build_def.bzl", "tflite_jni_binary")
load("//tensorflow/lite:special_rules.bzl", "tflite_hexagon_nn_skel_libraries")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
default_visibility = ["//visibility:private"],
licenses = ["notice"],
)
android_binary(
name = "benchmark_model_firebase",
srcs = glob([
"src/**/*.java",
]),
custom_package = "org.tensorflow.lite.benchmark.firebase",
manifest = "AndroidManifest.xml",
# In some platforms we don't have an Android SDK/NDK and this target
# can't be built. We need to prevent the build system from trying to
# use the target in that case.
tags = ["manual"],
deps = [
":hexagon_libs",
":tensorflowlite_benchmark_firebase_native",
],
)
tflite_jni_binary(
name = "libtensorflowlite_benchmark_firebase.so",
srcs = glob([
"jni/**/*.cc",
"jni/**/*.h",
]),
deps = [
"//tensorflow/lite/java/jni",
"//tensorflow/lite/tools/benchmark:benchmark_tflite_model_lib",
],
)
cc_library(
name = "tensorflowlite_benchmark_firebase_native",
srcs = ["libtensorflowlite_benchmark_firebase.so"],
visibility = ["//visibility:private"],
)
cc_library(
name = "hexagon_libs",
srcs = select({
"//tensorflow:android_arm64": [
"//tensorflow/lite/delegates/hexagon/hexagon_nn:libhexagon_interface.so",
] + tflite_hexagon_nn_skel_libraries(),
"//tensorflow:android_arm": [
"//tensorflow/lite/delegates/hexagon/hexagon_nn:libhexagon_interface.so",
] + tflite_hexagon_nn_skel_libraries(),
"//conditions:default": [],
}),
visibility = ["//visibility:private"],
)
tensorflow/lite/tools/benchmark/experimental/firebase/android/jni/benchmark_model_jni.cc
+290
View File
@@ -0,0 +1,290 @@
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <errno.h>
#include <jni.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fstream>
#include <sstream>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
#include "tensorflow/lite/tools/benchmark/benchmark_tflite_model.h"
#ifdef __ANDROID__
#include <android/log.h>
#endif
namespace tflite {
namespace benchmark {
namespace {
const char kOutputDir[] = "/sdcard/benchmark_output";
const char kSerializeDir[] = "/sdcard/serialize";
bool CreateDir(const char* path) {
struct stat st;
if (stat(path, &st) != 0) {
if (mkdir(path, 0777) != 0 && errno != EEXIST) {
return false;
}
} else if (!S_ISDIR(st.st_mode)) {
errno = ENOTDIR;
return false;
}
return true;
}
class FirebaseReportingListener : public BenchmarkListener {
public:
explicit FirebaseReportingListener(std::string tag, int report_fd)
: tag_(tag), report_fd_(report_fd) {
if (report_fd < 0) {
#ifdef __ANDROID__
__android_log_print(
ANDROID_LOG_ERROR, "tflite",
"Report would be streamed only to local log not to Firebase "
"since the Firebase log file is not opened.");
#else
fprintf(stderr,
"Report would be streamed only to local log not to Firebase "
"since the Firebase log file is not opened.");
#endif
}
}
void OnBenchmarkEnd(const BenchmarkResults& results) override {
ReportResult(results);
}
void ReportFailure(TfLiteStatus status) {
std::string status_msg =
status == kTfLiteError
? "TFLite error"
: (status == kTfLiteDelegateError ? "TFLite delegate error"
: "Unknown error code");
Report(status_msg, std::vector<std::pair<std::string, std::string>>());
}
private:
void Report(
const std::string& status,
const std::vector<std::pair<std::string, std::string>>& contents) {
// The output format of Firebase Game Loop test is json.
// https://firebase.google.com/docs/test-lab/android/game-loop#output-example
std::stringstream report;
report << "{\n"
<< " \"name\": \"TFLite benchmark\",\n"
<< " \"benchmark config\": \"" << tag_ << "\",\n"
<< " \"status\": \"" << status << "\"";
for (const auto& content : contents) {
report << ",\n"
<< " \"" << content.first << "\": \"" << content.second << "\"";
}
report << "\n}\n";
auto report_str = report.str();
if (report_fd_ >= 0) {
write(report_fd_, report_str.c_str(), report_str.size());
}
#ifdef __ANDROID__
__android_log_print(ANDROID_LOG_ERROR, "tflite", "%s", report_str.c_str());
#else
fprintf(stderr, "%s", report_str.c_str());
#endif
}
void ReportResult(const BenchmarkResults& results) {
std::vector<std::pair<std::string, std::string>> contents;
std::stringstream avg_time;
avg_time << "init: " << results.startup_latency_us() << ", "
<< "warmup: " << results.warmup_time_us().avg() << ", "
<< "inference: " << results.inference_time_us().avg();
contents.emplace_back("average time in us", avg_time.str());
std::stringstream overall_mem_usage;
overall_mem_usage << results.overall_mem_usage();
contents.emplace_back("overall memory usage", overall_mem_usage.str());
Report("OK", contents);
}
std::string tag_;
int report_fd_;
};
class CsvExportingListener : public BenchmarkListener {
public:
explicit CsvExportingListener(std::string tag) : tag_(tag) {}
void OnBenchmarkEnd(const BenchmarkResults& results) override {
if (!CreateDir(kOutputDir)) {
#ifdef __ANDROID__
__android_log_print(ANDROID_LOG_ERROR, "tflite",
"Failed to create output directory %s.", kOutputDir);
#else
fprintf(stderr, "Failed to create output directory %s.", kOutputDir);
#endif
return;
}
WriteBenchmarkResultCsv(results);
}
private:
void WriteBenchmarkResultCsv(const BenchmarkResults& results) {
auto init_us = results.startup_latency_us();
auto warmup_us = results.warmup_time_us();
auto inference_us = results.inference_time_us();
auto init_mem_usage = results.init_mem_usage();
auto overall_mem_usage = results.overall_mem_usage();
std::stringstream file_name;
file_name << kOutputDir << "/benchmark_result_" << tag_;
std::ofstream file;
file.open(file_name.str().c_str());
file << "config_key,model_size,init_time,"
<< "warmup_avg,warmup_min,warmup_max,warmup_stddev,"
<< "inference_avg,inference_min,inference_max,inference_stddev,"
<< "init_max_rss,init_total_alloc,init_in_use_alloc,"
<< "overall_max_rss,overall_total_alloc,overall_in_use_alloc\n";
file << tag_ << "," << results.model_size_mb() << "," << init_us << ","
<< warmup_us.avg() << "," << warmup_us.min() << "," << warmup_us.max()
<< "," << warmup_us.std_deviation() << "," << inference_us.avg() << ","
<< inference_us.min() << "," << inference_us.max() << ","
<< inference_us.std_deviation() << ","
<< (init_mem_usage.mem_footprint_kb / 1024.0) << ","
<< (init_mem_usage.total_allocated_bytes / 1024.0 / 1024.0) << ","
<< (init_mem_usage.in_use_allocated_bytes / 1024.0 / 1024.0) << ","
<< (overall_mem_usage.mem_footprint_kb / 1024.0) << ","
<< (overall_mem_usage.total_allocated_bytes / 1024.0 / 1024.0) << ","
<< (overall_mem_usage.in_use_allocated_bytes / 1024.0 / 1024.0)
<< "\n";
file.close();
}
std::string tag_;
};
std::string GetScenarioConfig(const std::string& library_dir, int scenario,
std::vector<std::string>& args) {
// The number of scenarios should equal to the value specified in
// AndroidManifest.xml file.
std::unordered_map<int, std::pair<std::string, std::vector<std::string>>>
all_scenarios = {
{1, {"cpu_1thread", {"--num_threads=1"}}},
{2, {"cpu_2threads", {"--num_threads=2"}}},
{3, {"cpu_4threads", {"--num_threads=4"}}},
{4, {"xnnpack_1thread", {"--use_xnnpack=true", "--num_threads=1"}}},
{5, {"xnnpack_2threads", {"--use_xnnpack=true", "--num_threads=2"}}},
{6, {"xnnpack_4threads", {"--use_xnnpack=true", "--num_threads=4"}}},
{7,
{"gpu_default",
{"--use_gpu=true", "--gpu_precision_loss_allowed=false"}}},
{8,
{"gpu_fp16",
{"--use_gpu=true", "--gpu_precision_loss_allowed=true"}}},
{9, {"dsp_hexagon", {"--use_hexagon=true"}}},
{10, {"nnapi", {"--use_nnapi=true"}}},
{11,
{"gpu_default_with_serialization",
{"--use_gpu=true", "--gpu_precision_loss_allowed=false",
"--delegate_serialize_token=dummy_token"}}},
{12,
{"gpu_fp16_with_serialization",
{"--use_gpu=true", "--gpu_precision_loss_allowed=true",
"--delegate_serialize_token=dummy_token"}}},
};
std::string tag;
args.emplace_back("(BenchmarkModelAndroid)");
args.emplace_back("--graph=/data/local/tmp/graph");
auto it = all_scenarios.find(scenario);
if (it != all_scenarios.end()) {
const auto& scenario_info = it->second;
tag = scenario_info.first;
for (const auto& arg : scenario_info.second) {
args.push_back(arg);
}
}
if (scenario == 9) {
std::stringstream hexagon_lib_path;
hexagon_lib_path << "--hexagon_lib_path=" << library_dir;
args.push_back(hexagon_lib_path.str());
}
if (scenario == 11 || scenario == 12) {
if (CreateDir(kSerializeDir)) {
std::stringstream serialize_dir;
serialize_dir << "--delegate_serialize_dir=" << kSerializeDir;
args.push_back(serialize_dir.str());
} else {
#ifdef __ANDROID__
__android_log_print(ANDROID_LOG_ERROR, "tflite",
"Failed to create serialize directory %s.",
kSerializeDir);
#else
fprintf(stderr, "Failed to create serialize directory %s.",
kSerializeDir);
#endif
}
}
return tag;
}
void RunScenario(const std::string& library_dir, int scenario, int report_fd) {
std::vector<std::string> args;
std::string tag = GetScenarioConfig(library_dir, scenario, args);
std::vector<char*> argv;
argv.reserve(args.size());
for (auto& arg : args) {
argv.push_back(const_cast<char*>(arg.data()));
}
BenchmarkTfLiteModel benchmark;
FirebaseReportingListener firebaseReporting(tag, report_fd);
benchmark.AddListener(&firebaseReporting);
CsvExportingListener csvExporting(tag);
benchmark.AddListener(&csvExporting);
auto status = benchmark.Run(static_cast<int>(argv.size()), argv.data());
if (status != kTfLiteOk) {
firebaseReporting.ReportFailure(status);
}
}
} // namespace
} // namespace benchmark
} // namespace tflite
extern "C" {
JNIEXPORT void JNICALL
Java_org_tensorflow_lite_benchmark_firebase_BenchmarkModel_nativeRun(
JNIEnv* env, jclass clazz, jstring library_dir, jint scenario,
jint report_fd) {
const char* lib_dir = env->GetStringUTFChars(library_dir, nullptr);
tflite::benchmark::RunScenario(lib_dir, static_cast<int>(scenario),
static_cast<int>(report_fd));
env->ReleaseStringUTFChars(library_dir, lib_dir);
}
} // extern "C"
tensorflow/lite/tools/benchmark/experimental/firebase/android/src/org/tensorflow/lite/benchmark/firebase/BenchmarkModel.java
+34
View File
@@ -0,0 +1,34 @@
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark.firebase;
import android.content.Context;
/** Helper class for running a native TensorFlow Lite benchmark. */
class BenchmarkModel {
static {
System.loadLibrary("tensorflowlite_benchmark_firebase");
}
// Executes a standard TensorFlow Lite benchmark with predefined args for each scenario.
// Result and status will be reported to the file with reportFd file descriptor.
public static void run(Context context, int scenario, int reportFd) {
String libraryDir = context.getApplicationInfo().nativeLibraryDir;
nativeRun(libraryDir, scenario, reportFd);
}
private static native void nativeRun(String libraryDir, int scenario, int reportFd);
}
tensorflow/lite/tools/benchmark/experimental/firebase/android/src/org/tensorflow/lite/benchmark/firebase/BenchmarkModelActivity.java
+74
View File
@@ -0,0 +1,74 @@
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
package org.tensorflow.lite.benchmark.firebase;
import android.app.Activity;
import android.content.Intent;
import android.net.Uri;
import android.os.Bundle;
import android.os.ParcelFileDescriptor;
import android.util.Log;
import java.io.FileNotFoundException;
import java.io.IOException;
/**
* {@code Activity} class for Firebase Game Loop test.
*
* <p>This Activity receives and handles an {@code Intent} for Firebase Game Loop test. Refer to
* https://firebase.google.com/docs/test-lab/android/game-loop.
*/
public class BenchmarkModelActivity extends Activity {
private static final String TAG = "tflite_BenchmarkModelActivity";
@Override
public void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
Intent intent = getIntent();
if (!intent.getAction().equals("com.google.intent.action.TEST_LOOP")) {
Log.e(TAG, "Received non Firebase Game Loop test intent " + intent.getAction());
finish();
}
int scenario = intent.getIntExtra("scenario", 0);
Log.i(TAG, "Running TensorFlow Lite benchmark with scenario: " + scenario);
ParcelFileDescriptor parcelFileDescriptor = null;
Uri reportFile = intent.getData();
if (reportFile != null) {
Log.i(TAG, "Logging the result to " + reportFile.getEncodedPath());
try {
parcelFileDescriptor =
getContentResolver().openAssetFileDescriptor(reportFile, "w").getParcelFileDescriptor();
} catch (FileNotFoundException | NullPointerException e) {
Log.e(TAG, "Error while opening Firebase Test Lab report file", e);
}
}
int reportFd = parcelFileDescriptor != null ? parcelFileDescriptor.getFd() : -1;
BenchmarkModel.run(this, scenario, reportFd);
if (parcelFileDescriptor != null) {
try {
parcelFileDescriptor.close();
} catch (IOException e) {
Log.e(TAG, "Failed to close Firebase Test Lab result file", e);
}
}
finish();
}
}
tensorflow/lite/tools/benchmark/experimental/ios/BUILD.apple
+45
View File
@@ -0,0 +1,45 @@
load("@bazel_skylib//rules:build_test.bzl", "build_test")
load("@build_bazel_rules_apple//apple:ios.bzl", "ios_static_framework")
load("//tensorflow/lite/ios:ios.bzl", "TFL_MINIMUM_OS_VERSION", "strip_common_include_path_prefix")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
licenses = ["notice"],
)
strip_common_include_path_prefix(
name = "strip_common_include_path_benchmark",
hdr_labels = [
"//tensorflow/lite/core/c:c_api_types.h",
"//tensorflow/lite/tools:logging.h",
"//tensorflow/lite/tools/benchmark/experimental/c:benchmark_c_api.h",
],
)
# Main target for the benchmark tool iOS framework.
# bazel build --config=ios_fat -c opt //tensorflow/lite/tools/benchmark/experimental/ios:TensorFlowLiteBenchmarkC_framework
ios_static_framework(
name = "TensorFlowLiteBenchmarkC_framework",
hdrs = [
":benchmark_c_api.h",
":c_api_types.h",
":logging.h",
],
bundle_name = "TensorFlowLiteBenchmarkC",
minimum_os_version = TFL_MINIMUM_OS_VERSION,
deps = [
"//tensorflow/lite/tools/benchmark/experimental/c:benchmark_c_api",
],
)
# Used for building TensorFlowLiteBenchmarkC_framework framework.
build_test(
name = "framework_build_test",
tags = [
"nomsan", # b/145205324
"notsan", # b/145205324
],
targets = [
":TensorFlowLiteBenchmarkC_framework",
],
)
tensorflow/lite/tools/benchmark/ios/README.md
+46
View File
@@ -0,0 +1,46 @@
# TFLite iOS benchmark app.
## Description
An iOS app to benchmark TFLite models.
The app reads benchmark parameters from a JSON file named
`benchmark_params.json` in its `benchmark_data` directory. Any downloaded models
for benchmarking should also be placed in `benchmark_data` directory.
The JSON file specifies the name of the model file and other benchmarking
parameters like inputs to the model, type of inputs, number of iterations,
number of threads. The default values in the JSON file are for the
Mobilenet_1.0_224 model ([paper][mobilenet-paper],
[tflite&pb][mobilenet-model]).
## Building / running the app
* Follow the [iOS build instructions][build-ios] to configure the Bazel
workspace and `.bazelrc` file correctly.
* Run `build_benchmark_framework.sh` script to build the benchmark framework.
This script will build the benchmark framework targeting iOS arm64 and put
it under `TFLiteBenchmark/TFLiteBenchmark/Frameworks` directory.
* If you want more detailed profiling, run the build script with `-p` option:
`build_benchmark_framework.sh -p`.
* Modify `benchmark_params.json` change the `input_layer`, `input_layer_shape`
and other benchmark parameters.
* Change `Build Phases -> Copy Bundle Resources` and add the model file to the
resources that need to be copied.
* Ensure that `Build Phases -> Link Binary With Library` contains the
`Accelerate framework` and `TensorFlowLiteBenchmarkC.framework`.
* Now try running the app. The app has a single button that runs the benchmark
on the model and displays results in a text view below. You can also see the
console output section in your Xcode to see more detailed benchmark
information.
[build-ios]: https://tensorflow.org/lite/guide/build_ios
[mobilenet-model]: https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_1.0_224.tgz
[mobilenet-paper]: https://arxiv.org/pdf/1704.04861.pdf
tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark/AppDelegate.h
+22
View File
@@ -0,0 +1,22 @@
// Copyright 2018 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import <UIKit/UIKit.h>
@interface AppDelegate : UIResponder <UIApplicationDelegate>
@property(strong, nonatomic) UIWindow *window;
@end
tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark/AppDelegate.m
+27
View File
@@ -0,0 +1,27 @@
// Copyright 2018 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import "AppDelegate.h"
@interface AppDelegate ()
@end
@implementation AppDelegate
- (BOOL)application:(UIApplication *)application
didFinishLaunchingWithOptions:(NSDictionary *)launchOptions {
return YES;
}
@end
tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark/Assets.xcassets/AppIcon.appiconset/Contents.json
+98
View File
@@ -0,0 +1,98 @@
{
"images" : [
{
"idiom" : "iphone",
"size" : "20x20",
"scale" : "2x"
},
{
"idiom" : "iphone",
"size" : "20x20",
"scale" : "3x"
},
{
"idiom" : "iphone",
"size" : "29x29",
"scale" : "2x"
},
{
"idiom" : "iphone",
"size" : "29x29",
"scale" : "3x"
},
{
"idiom" : "iphone",
"size" : "40x40",
"scale" : "2x"
},
{
"idiom" : "iphone",
"size" : "40x40",
"scale" : "3x"
},
{
"idiom" : "iphone",
"size" : "60x60",
"scale" : "2x"
},
{
"idiom" : "iphone",
"size" : "60x60",
"scale" : "3x"
},
{
"idiom" : "ipad",
"size" : "20x20",
"scale" : "1x"
},
{
"idiom" : "ipad",
"size" : "20x20",
"scale" : "2x"
},
{
"idiom" : "ipad",
"size" : "29x29",
"scale" : "1x"
},
{
"idiom" : "ipad",
"size" : "29x29",
"scale" : "2x"
},
{
"idiom" : "ipad",
"size" : "40x40",
"scale" : "1x"
},
{
"idiom" : "ipad",
"size" : "40x40",
"scale" : "2x"
},
{
"idiom" : "ipad",
"size" : "76x76",
"scale" : "1x"
},
{
"idiom" : "ipad",
"size" : "76x76",
"scale" : "2x"
},
{
"idiom" : "ipad",
"size" : "83.5x83.5",
"scale" : "2x"
},
{
"idiom" : "ios-marketing",
"size" : "1024x1024",
"scale" : "1x"
}
],
"info" : {
"version" : 1,
"author" : "xcode"
}
}
tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark/Assets.xcassets/Contents.json
+6
View File
@@ -0,0 +1,6 @@
{
"info" : {
"version" : 1,
"author" : "xcode"
}
}
tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark/Base.lproj/LaunchScreen.storyboard
+26
View File
@@ -0,0 +1,26 @@
<?xml version="1.0" encoding="UTF-8"?>
<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="23727" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="01J-lp-oVM">
<device id="retina6_12" orientation="portrait" appearance="light"/>
<dependencies>
<plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="23721"/>
<capability name="Safe area layout guides" minToolsVersion="9.0"/>
<capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
</dependencies>
<scenes>
<!--View Controller-->
<scene sceneID="EHf-IW-A2E">
<objects>
<viewController id="01J-lp-oVM" sceneMemberID="viewController">
<view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
<rect key="frame" x="0.0" y="0.0" width="393" height="852"/>
<autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
<viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
<color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
</view>
</viewController>
<placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder"/>
</objects>
<point key="canvasLocation" x="53" y="375"/>
</scene>
</scenes>
</document>
tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark/Base.lproj/Main.storyboard
+58
View File
@@ -0,0 +1,58 @@
<?xml version="1.0" encoding="UTF-8"?>
<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="23727" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
<device id="retina4_7" orientation="portrait" appearance="light"/>
<dependencies>
<deployment identifier="iOS"/>
<plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="23721"/>
<capability name="Safe area layout guides" minToolsVersion="9.0"/>
<capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
</dependencies>
<scenes>
<!--Benchmark View Controller-->
<scene sceneID="tne-QT-ifu">
<objects>
<viewController id="BYZ-38-t0r" customClass="BenchmarkViewController" sceneMemberID="viewController">
<view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
<rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
<autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
<subviews>
<textView clipsSubviews="YES" multipleTouchEnabled="YES" contentMode="scaleToFill" textAlignment="natural" translatesAutoresizingMaskIntoConstraints="NO" id="Vd4-Gf-qKO">
<rect key="frame" x="26" y="101" width="333" height="556"/>
<color key="backgroundColor" white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
<fontDescription key="fontDescription" type="system" pointSize="14"/>
<textInputTraits key="textInputTraits" autocapitalizationType="sentences"/>
</textView>
<button opaque="NO" contentMode="scaleToFill" misplaced="YES" contentHorizontalAlignment="center" contentVerticalAlignment="center" buttonType="system" lineBreakMode="middleTruncation" translatesAutoresizingMaskIntoConstraints="NO" id="j0O-Lq-1tJ">
<rect key="frame" x="69" y="30" width="247" height="63"/>
<constraints>
<constraint firstAttribute="height" constant="63" id="8VO-Ln-L2h"/>
</constraints>
<fontDescription key="fontDescription" type="system" pointSize="24"/>
<state key="normal" title="Benchmark model"/>
<connections>
<action selector="onBenchmarkModel:" destination="BYZ-38-t0r" eventType="touchUpInside" id="Rb1-hs-Mub"/>
</connections>
</button>
</subviews>
<viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
<color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
<constraints>
<constraint firstItem="Vd4-Gf-qKO" firstAttribute="top" secondItem="j0O-Lq-1tJ" secondAttribute="bottom" constant="18" id="Kd3-pP-C1k"/>
<constraint firstItem="j0O-Lq-1tJ" firstAttribute="centerX" secondItem="8bC-Xf-vdC" secondAttribute="centerX" id="QJU-cq-L87"/>
<constraint firstItem="Vd4-Gf-qKO" firstAttribute="trailing" secondItem="8bC-Xf-vdC" secondAttribute="trailingMargin" id="Tew-W4-Vq5"/>
<constraint firstItem="j0O-Lq-1tJ" firstAttribute="top" secondItem="6Tk-OE-BBY" secondAttribute="top" id="Uce-n7-kZI"/>
<constraint firstItem="j0O-Lq-1tJ" firstAttribute="leading" secondItem="6Tk-OE-BBY" secondAttribute="leading" constant="64" id="Uhq-Rw-NKT"/>
<constraint firstItem="Vd4-Gf-qKO" firstAttribute="leading" secondItem="6Tk-OE-BBY" secondAttribute="leading" constant="26" id="aXc-6M-kyL"/>
<constraint firstItem="6Tk-OE-BBY" firstAttribute="bottom" secondItem="Vd4-Gf-qKO" secondAttribute="bottom" constant="10" id="tz5-wP-LZs"/>
</constraints>
</view>
<connections>
<outlet property="resultsView" destination="Vd4-Gf-qKO" id="dBT-f6-SYw"/>
</connections>
</viewController>
<placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
</objects>
<point key="canvasLocation" x="140" y="122.78860569715144"/>
</scene>
</scenes>
</document>
tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark/BenchmarkViewController.h
+21
View File
@@ -0,0 +1,21 @@
// Copyright 2018 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import <UIKit/UIKit.h>
@interface BenchmarkViewController : UIViewController
@property(weak, nonatomic) IBOutlet UITextView *resultsView;
@end
tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark/BenchmarkViewController.mm
+152
View File
@@ -0,0 +1,152 @@
// Copyright 2018 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import "BenchmarkViewController.h"
#import <algorithm>
#import <sstream>
#import <string>
#import <vector>
#if defined(USE_TFLITE_BENCHMARK_HEADERS)
#include "tensorflow/lite/tools/benchmark/experimental/c/benchmark_c_api.h"
#include "tensorflow/lite/tools/logging.h"
#else
#import <TensorFlowLiteBenchmarkC/TensorFlowLiteBenchmarkC.h>
#endif
namespace {
NSString* FilePathForResourceName(NSString* filename) {
NSString* name = [filename stringByDeletingPathExtension];
NSString* extension = [filename pathExtension];
NSString* file_path = [[NSBundle mainBundle] pathForResource:name ofType:extension];
if (file_path == NULL) {
TFLITE_LOG(FATAL) << "Couldn't find '" << [name UTF8String] << "." << [extension UTF8String]
<< "' in bundle.";
}
return file_path;
}
NSDictionary* ParseJson() {
NSString* params_json_path = FilePathForResourceName(@"benchmark_params.json");
NSData* data = [NSData dataWithContentsOfFile:params_json_path];
return [NSJSONSerialization JSONObjectWithData:data options:kNilOptions error:nil];
}
std::string FormatCommandLineParam(NSString* key, NSString* value) {
std::ostringstream stream;
stream << "--" << [key UTF8String] << "=" << [value UTF8String];
return stream.str();
}
// Reads the |benchmark_params.json| to read command line parameters and returns them as a vector of
// strings.
void ReadCommandLineParameters(std::vector<std::string>* params) {
NSDictionary* param_dict = ParseJson();
for (NSString* key in param_dict) {
NSString* value = param_dict[key];
if ([key isEqualToString:@"graph"]) {
value = FilePathForResourceName(value);
}
params->push_back(FormatCommandLineParam(key, value));
}
}
std::vector<char*> StringVecToCharPtrVec(const std::vector<std::string>& str_vec) {
std::vector<char*> charptr_vec;
std::transform(str_vec.begin(), str_vec.end(), std::back_inserter(charptr_vec),
[](const std::string& s) -> char* { return const_cast<char*>(s.c_str()); });
return charptr_vec;
}
class ResultsListener {
public:
void OnBenchmarkEnd(TfLiteBenchmarkResults* results);
std::string Results() { return results_; }
private:
std::string results_;
};
void OutputMicrosecondsStatToStream(const TfLiteBenchmarkInt64Stat& time_us,
const std::string& prefix, std::ostringstream* stream) {
*stream << prefix << "Num runs: " << time_us.count << "\n";
*stream << prefix << "Average: " << time_us.avg / 1e3 << " ms\n";
*stream << prefix << "Min: " << time_us.min / 1e3 << " ms \n";
*stream << prefix << "Max: " << time_us.max / 1e3 << " ms \n";
*stream << prefix << "Std deviation: " << time_us.std_deviation / 1e3 << " ms\n";
}
void ResultsListener::OnBenchmarkEnd(TfLiteBenchmarkResults* results) {
std::ostringstream stream;
const std::string prefix = " - ";
TfLiteBenchmarkInt64Stat inference = TfLiteBenchmarkResultsGetInferenceTimeMicroseconds(results);
TfLiteBenchmarkInt64Stat warmup = TfLiteBenchmarkResultsGetWarmupTimeMicroseconds(results);
stream << "Startup latency: ";
stream << TfLiteBenchmarkResultsGetStartupLatencyMicroseconds(results) / 1e3 << " ms\n";
stream << "\nInference:\n";
OutputMicrosecondsStatToStream(inference, prefix, &stream);
stream << "\nWarmup:\n";
OutputMicrosecondsStatToStream(warmup, prefix, &stream);
results_ = stream.str();
}
void OnBenchmarkEnd(void* user_data, TfLiteBenchmarkResults* results) {
if (user_data != nullptr) {
reinterpret_cast<ResultsListener*>(user_data)->OnBenchmarkEnd(results);
}
}
std::string RunBenchmark() {
ResultsListener results_listener;
TfLiteBenchmarkTfLiteModel* benchmark = TfLiteBenchmarkTfLiteModelCreate();
TfLiteBenchmarkListener* listener = TfLiteBenchmarkListenerCreate();
TfLiteBenchmarkListenerSetCallbacks(listener, &results_listener, nullptr, nullptr, nullptr,
OnBenchmarkEnd);
TfLiteBenchmarkTfLiteModelAddListener(benchmark, listener);
// TODO(shashishekhar): Passing arguments like this is brittle, refactor the BenchmarkParams
// so that it contains arguments for BenchmarkTfLiteModel and set parameters using BenchmarkParams
std::vector<std::string> command_line_params;
// Benchmark model expects first arg to be program name.
// push a string for name of program.
command_line_params.push_back("benchmark_tflite_model");
ReadCommandLineParameters(&command_line_params);
std::vector<char*> argv = StringVecToCharPtrVec(command_line_params);
int argc = static_cast<int>(argv.size());
TfLiteBenchmarkTfLiteModelRunWithArgs(benchmark, argc, argv.data());
std::string results = results_listener.Results();
TfLiteBenchmarkListenerDelete(listener);
TfLiteBenchmarkTfLiteModelDelete(benchmark);
return results;
}
} // namespace
@interface BenchmarkViewController ()
@end
@implementation BenchmarkViewController
- (IBAction)onBenchmarkModel:(UIButton*)sender {
std::string results = RunBenchmark();
[_resultsView setText:[NSString stringWithUTF8String:results.c_str()]];
}
@end
tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark/Frameworks/.gitignore
+1
View File
@@ -0,0 +1 @@
*.framework/
tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark/Info.plist
+43
View File
@@ -0,0 +1,43 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleDevelopmentRegion</key>
<string>$(DEVELOPMENT_LANGUAGE)</string>
<key>CFBundleExecutable</key>
<string>$(EXECUTABLE_NAME)</string>
<key>CFBundleIdentifier</key>
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
<key>CFBundleInfoDictionaryVersion</key>
<string>6.0</string>
<key>CFBundleName</key>
<string>$(PRODUCT_NAME)</string>
<key>CFBundlePackageType</key>
<string>APPL</string>
<key>CFBundleShortVersionString</key>
<string>1.0</string>
<key>CFBundleVersion</key>
<string>1</string>
<key>LSRequiresIPhoneOS</key>
<true/>
<key>UILaunchStoryboardName</key>
<string>Main</string>
<key>UIMainStoryboardFile</key>
<string>Main</string>
<key>UIRequiredDeviceCapabilities</key>
<array>
<string>armv7</string>
</array>
<key>UISupportedInterfaceOrientations</key>
<array>
<string>UIInterfaceOrientationPortrait</string>
</array>
<key>UISupportedInterfaceOrientations~ipad</key>
<array>
<string>UIInterfaceOrientationPortrait</string>
<string>UIInterfaceOrientationPortraitUpsideDown</string>
<string>UIInterfaceOrientationLandscapeLeft</string>
<string>UIInterfaceOrientationLandscapeRight</string>
</array>
</dict>
</plist>
tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark/benchmark_data/benchmark_params.json
+10
View File
@@ -0,0 +1,10 @@
{
"benchmark_name" : "mobile_net_benchmark",
"num_threads" : "4",
"num_runs" : "20",
"warmup_runs" : "1",
"graph" : "mobilenet_v1_1.0_224.tflite",
"input_layer" : "input",
"input_layer_shape" : "1,224,224,3",
"run_delay" : "-1"
}
tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark/main.m
+23
View File
@@ -0,0 +1,23 @@
// Copyright 2018 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import <UIKit/UIKit.h>
#import "AppDelegate.h"
int main(int argc, char* argv[]) {
@autoreleasepool {
return UIApplicationMain(argc, argv, nil, NSStringFromClass([AppDelegate class]));
}
}
tensorflow/lite/tools/benchmark/ios/build_benchmark_framework.sh
Executable
+73
View File
@@ -0,0 +1,73 @@
#!/bin/bash
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
set -e
WORKSPACE_ROOT=$(bazel info workspace 2> /dev/null)
BENCHMARK_DIR=tensorflow/lite/tools/benchmark
DEST_DIR="${BENCHMARK_DIR}/ios/TFLiteBenchmark/TFLiteBenchmark/Frameworks"
FRAMEWORK_TARGET=TensorFlowLiteBenchmarkC_framework
function usage() {
echo "Usage: $(basename "$0") [-p]"
echo "-p enable profiling"
exit 1
}
PROFILING_ARGS=""
while getopts "p" opt_name; do
case "$opt_name" in
p) PROFILING_ARGS='--define=ruy_profiler=true';;
*) usage;;
esac
done
shift $(($OPTIND - 1))
function check_ios_configured() {
if [ ! -f "${WORKSPACE_ROOT}/${BENCHMARK_DIR}/experimental/ios/BUILD" ]; then
echo "ERROR: Benchmark framework BUILD file not found."
echo "Please enable iOS support by running the \"./configure\" script" \
"from the workspace root."
exit 1
fi
}
function build_framework() {
set -x
pushd "${WORKSPACE_ROOT}"
# Build the framework.
bazel build --config=ios_arm64 -c opt --cxxopt=-std=c++17 ${PROFILING_ARGS} \
"//${BENCHMARK_DIR}/experimental/ios:${FRAMEWORK_TARGET}"
# Get the generated framework path.
BAZEL_OUTPUT_FILE_PATH=$(bazel cquery "//${BENCHMARK_DIR}/experimental/ios:${FRAMEWORK_TARGET}" --config=ios_arm64 --output=starlark --starlark:expr="' '.join([f.path for f in target.files.to_list()])")
# Copy the framework into the destination and unzip.
mkdir -p "${DEST_DIR}"
cp -f "${BAZEL_OUTPUT_FILE_PATH}" \
"${DEST_DIR}"
pushd "${DEST_DIR}"
unzip -o "${FRAMEWORK_TARGET}.zip"
rm -f "${FRAMEWORK_TARGET}.zip"
popd
popd
}
check_ios_configured
build_framework
tensorflow/lite/tools/benchmark/profiling_listener.cc
+82
View File
@@ -0,0 +1,82 @@
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/lite/tools/benchmark/profiling_listener.h"
#include <cstdint>
#include <memory>
#include <string>
#include "tensorflow/lite/interpreter.h"
#include "tensorflow/lite/profiling/profile_summarizer.h"
#include "tensorflow/lite/profiling/profile_summary_formatter.h"
#include "tensorflow/lite/tools/benchmark/benchmark_model.h"
#include "tensorflow/lite/tools/benchmark/benchmark_params.h"
#include "tensorflow/lite/tools/logging.h"
namespace tflite {
namespace benchmark {
ProfilingListener::ProfilingListener(
Interpreter* interpreter, uint32_t max_num_initial_entries,
bool allow_dynamic_buffer_increase, const std::string& output_file_path,
std::shared_ptr<profiling::ProfileSummaryFormatter> summarizer_formatter)
: run_summarizer_(summarizer_formatter),
init_summarizer_(summarizer_formatter),
output_file_path_(output_file_path),
interpreter_(interpreter),
profiler_(max_num_initial_entries, allow_dynamic_buffer_increase),
summarizer_formatter_(summarizer_formatter) {
TFLITE_TOOLS_CHECK(interpreter);
interpreter_->SetProfiler(&profiler_);
// We start profiling here in order to catch events that are recorded during
// the benchmark run preparation stage where TFLite interpreter is
// initialized and model graph is prepared.
profiler_.Reset();
profiler_.StartProfiling();
}
void ProfilingListener::OnBenchmarkStart(const BenchmarkParams& params) {
// At this point, we have completed the preparation for benchmark runs
// including TFLite interpreter initialization etc. So we are going to process
// profiling events recorded during this stage.
profiler_.StopProfiling();
auto profile_events = profiler_.GetProfileEvents();
init_summarizer_.ProcessProfiles(profile_events, *interpreter_);
profiler_.Reset();
}
void ProfilingListener::OnSingleRunStart(RunType run_type) {
if (run_type == REGULAR) {
profiler_.Reset();
profiler_.StartProfiling();
}
}
void ProfilingListener::OnSingleRunEnd() {
profiler_.StopProfiling();
auto profile_events = profiler_.GetProfileEvents();
run_summarizer_.ProcessProfiles(profile_events, *interpreter_);
}
void ProfilingListener::OnBenchmarkEnd(const BenchmarkResults& results) {
summarizer_formatter_->HandleOutput(init_summarizer_.GetOutputString(),
run_summarizer_.GetOutputString(),
output_file_path_);
}
} // namespace benchmark
} // namespace tflite
tensorflow/lite/tools/benchmark/profiling_listener.h
+65
View File
@@ -0,0 +1,65 @@
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_TOOLS_BENCHMARK_PROFILING_LISTENER_H_
#define TENSORFLOW_LITE_TOOLS_BENCHMARK_PROFILING_LISTENER_H_
#include <cstdint>
#include <memory>
#include <string>
#include "tensorflow/lite/interpreter.h"
#include "tensorflow/lite/profiling/buffered_profiler.h"
#include "tensorflow/lite/profiling/profile_summarizer.h"
#include "tensorflow/lite/profiling/profile_summary_formatter.h"
#include "tensorflow/lite/tools/benchmark/benchmark_model.h"
#include "tensorflow/lite/tools/benchmark/benchmark_params.h"
namespace tflite {
namespace benchmark {
// Dumps profiling events if profiling is enabled.
class ProfilingListener : public BenchmarkListener {
public:
ProfilingListener(
Interpreter* interpreter, uint32_t max_num_initial_entries,
bool allow_dynamic_buffer_increase,
const std::string& output_file_path = "",
std::shared_ptr<profiling::ProfileSummaryFormatter> summarizer_formatter =
std::make_shared<profiling::ProfileSummaryDefaultFormatter>());
void OnBenchmarkStart(const BenchmarkParams& params) override;
void OnSingleRunStart(RunType run_type) override;
void OnSingleRunEnd() override;
void OnBenchmarkEnd(const BenchmarkResults& results) override;
protected:
profiling::ProfileSummarizer run_summarizer_;
profiling::ProfileSummarizer init_summarizer_;
std::string output_file_path_;
private:
Interpreter* interpreter_;
profiling::BufferedProfiler profiler_;
std::shared_ptr<profiling::ProfileSummaryFormatter> summarizer_formatter_;
};
} // namespace benchmark
} // namespace tflite
#endif // TENSORFLOW_LITE_TOOLS_BENCHMARK_PROFILING_LISTENER_H_
tensorflow/lite/tools/benchmark/proto/BUILD
+34
View File
@@ -0,0 +1,34 @@
load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
load("@rules_python//python:proto.bzl", "py_proto_library")
load("//tensorflow:tensorflow.default.bzl", "get_compatible_with_portable")
load(
"//tensorflow/core/platform:build_config.bzl",
"tf_proto_library",
)
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
default_visibility = ["//visibility:public"],
licenses = ["notice"],
)
proto_library(
name = "benchmark_result_proto",
srcs = ["benchmark_result.proto"],
compatible_with = get_compatible_with_portable(),
visibility = ["//visibility:public"],
)
tf_proto_library(
name = "benchmark_result", # bzl adds _py for python proto library
srcs = ["benchmark_result.proto"],
visibility = ["//visibility:public"],
)
# copybara:uncomment_begin(google-only)
# py_proto_library(
# name = "benchmark_result_py_pb2",
# compatible_with = get_compatible_with_portable(),
# deps = [":benchmark_result_proto"],
# )
# copybara:uncomment_end
tensorflow/lite/tools/benchmark/proto/CMakeLists.txt
+36
View File
@@ -0,0 +1,36 @@
#
# Copyright 2025 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
find_package(Protobuf REQUIRED)
add_library(benchmark_result_proto benchmark_result.proto)
list(APPEND benchmark_result_generated_files
${CMAKE_BINARY_DIR}/tensorflow/lite/tools/benchmark/proto/benchmark_result.pb.cc
${CMAKE_BINARY_DIR}/tensorflow/lite/tools/benchmark/proto/benchmark_result.pb.h)
# Generate benchmark_result.pb.cc and benchmark_result.pb.h from
# benchmark_result.proto using protoc. Once the protobuf package version is
# upgraded, we can use protobuf_generate_cpp/protobuf_generate here directly.
add_custom_command(
OUTPUT ${benchmark_result_generated_files}
COMMAND ${Protobuf_PROTOC_EXECUTABLE}
ARGS --cpp_out=${CMAKE_BINARY_DIR} --proto_path=${TENSORFLOW_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/benchmark_result.proto
DEPENDS ${Protobuf_PROTOC_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/benchmark_result.proto
)
set_source_files_properties(${benchmark_result_generated_files} PROPERTIES GENERATED TRUE)
target_sources(benchmark_result_proto PRIVATE ${benchmark_result_generated_files})
target_link_libraries(benchmark_result_proto protobuf::libprotobuf)
target_include_directories(benchmark_result_proto PUBLIC ${CMAKE_BINARY_DIR})
tensorflow/lite/tools/benchmark/proto/benchmark_result.proto
+73
View File
@@ -0,0 +1,73 @@
/* Copyright 2025 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
syntax = "proto2";
package tflite.tools.benchmark;
option java_multiple_files = true;
option java_package = "tflite.tools.benchmark";
// Next ID: 11
message LatencyMetrics {
// Note all fields defined here refers to wall time by default.
optional float avg_ms = 1;
optional float min_ms = 2;
optional float max_ms = 3;
optional float stddev_ms = 4;
optional float median_ms = 5;
optional float p5_ms = 6;
optional float p95_ms = 7;
optional float init_ms = 8;
optional float first_inference_ms = 9;
optional float average_warm_up_ms = 10;
}
// Next ID: 4
message MemoryMetrics {
// The amount of memory allocated during model initialization. This is the
// delta of memory footprint after the model has been loaded and interpreter
// has been created, compared to the memory footprint at the beginning of the
// benchmark tool.
optional int64 init_footprint_kb = 1;
// The memory allocated during the model initialization and execution.
// This is the delta of memory footprint after the model has been intialized
// and executed, compared to the memory footprint at the beginning of the
// benchmark tool.
optional int64 overall_footprint_kb = 2;
// Peak memory usage (in megabytes).
optional float peak_mem_mb = 3;
}
// Next ID: 5
message MiscMetrics {
optional float model_size_mb = 1;
optional int32 num_runs = 2;
optional int32 num_warmup_runs = 3;
// Model throughput in megabytes per second. This is the average throughput
// of the model over all the inferences.
optional float model_throughput_in_mb_per_sec = 4;
}
// Next ID: 5
message BenchmarkResult {
// The name representing the configuration of the benchmark run.
optional string name = 1;
optional LatencyMetrics latency_metrics = 2;
optional MemoryMetrics memory_metrics = 3;
optional MiscMetrics misc_metrics = 4;
}
tensorflow/lite/tools/benchmark/register_custom_op.cc
+23
View File
@@ -0,0 +1,23 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "absl/base/attributes.h"
#include "tensorflow/lite/op_resolver.h"
// Version with Weak linker attribute doing nothing: if someone links this
// library with another definition of this function (presumably to actually
// register custom ops), that version will be used instead.
void ABSL_ATTRIBUTE_WEAK
RegisterSelectedOps(::tflite::MutableOpResolver* resolver) {}
tensorflow/lite/tools/benchmark/register_custom_op.h
+23
View File
@@ -0,0 +1,23 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_LITE_TOOLS_BENCHMARK_REGISTER_CUSTOM_OP_H_
#define TENSORFLOW_LITE_TOOLS_BENCHMARK_REGISTER_CUSTOM_OP_H_
#include "tensorflow/lite/op_resolver.h"
void RegisterSelectedOps(::tflite::MutableOpResolver* resolver);
#endif // TENSORFLOW_LITE_TOOLS_BENCHMARK_REGISTER_CUSTOM_OP_H_
tensorflow/lite/tools/build_aar.sh
Executable
+229
View File
@@ -0,0 +1,229 @@
#!/bin/bash
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
set -e
set -x
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT_DIR="$(cd "${SCRIPT_DIR}/../../../" && pwd)"
function print_usage {
echo "Usage:"
echo " $(basename ${BASH_SOURCE}) \\"
echo " --input_models=model1.tflite,model2.tflite \\"
echo " --target_archs=x86,x86_64,arm64-v8a,armeabi-v7a \\"
echo " --tflite_custom_ops_srcs=file1.cc,file2.h \\"
echo " --tflite_custom_ops_deps=dep1,dep2"
echo ""
echo "Where: "
echo " --input_models: Supported TFLite models. "
echo " --target_archs: Supported arches included in the aar file."
echo " --tflite_custom_ops_srcs: The src files for building additional TFLite custom ops if any."
echo " --tflite_custom_ops_deps: Dependencies for building additional TFLite custom ops if any."
echo ""
exit 1
}
function generate_list_field {
local name="$1"
local list_string="$2"
local list=(${list_string//,/ })
local message=("$name=[")
for item in "${list[@]}"
do
message+=("\"$item\",")
done
message+=('],')
printf '%s' "${message[@]}"
}
function print_output {
if [ -z OMIT_PRINTING_OUTPUT_PATHS ]; then
echo "Output can be found here:"
for i in "$@"
do
# Check if the file exist.
ls -1a ${ROOT_DIR}/$i
done
fi
}
function generate_tflite_aar {
pushd ${TMP_DIR} > /dev/null
# Generate the BUILD file.
message=(
'load("//tensorflow/lite:build_def.bzl", "tflite_custom_android_library")'
'load("//tensorflow/lite/java:aar_with_jni.bzl", "aar_with_jni")'
''
'tflite_custom_android_library('
' name = "custom_tensorflowlite",'
)
message+=(' '$(generate_list_field "models" $MODEL_NAMES))
message+=(' '$(generate_list_field "srcs" $TFLITE_OPS_SRCS))
message+=(' '$(generate_list_field "deps" $FLAG_TFLITE_OPS_DEPS))
message+=(
')'
''
'aar_with_jni('
' name = "tensorflow-lite",'
' android_library = ":custom_tensorflowlite",'
')'
''
)
printf '%s\n' "${message[@]}" >> BUILD
# Build the aar package.
popd > /dev/null
bazel ${CACHE_DIR_FLAG} build -c opt --config=opt --cxxopt='--std=c++17' \
--fat_apk_cpu=${TARGET_ARCHS} \
--define=android_dexmerger_tool=d8_dexmerger \
--define=android_incremental_dexing_tool=d8_dexbuilder\
--host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
//tmp:tensorflow-lite
OUT_FILES="${OUT_FILES} bazel-bin/tmp/tensorflow-lite.aar"
}
function generate_flex_aar {
pushd ${TMP_DIR}
# Generating the BUILD file.
message=(
'load("//tensorflow/lite/delegates/flex:build_def.bzl", "tflite_flex_android_library")'
'load("//tensorflow/lite/java:aar_with_jni.bzl", "aar_with_jni")'
''
'tflite_flex_android_library('
' name = "custom_tensorflowlite_flex",'
)
message+=(' '$(generate_list_field "models" $MODEL_NAMES))
message+=(
')'
''
'aar_with_jni('
' name = "tensorflow-lite-select-tf-ops",'
' android_library = ":custom_tensorflowlite_flex",'
')'
)
printf '%s\n' "${message[@]}" >> BUILD
cp ${ROOT_DIR}/tensorflow/lite/java/AndroidManifest.xml .
cp ${ROOT_DIR}/tensorflow/lite/java/proguard.flags .
popd
bazel ${CACHE_DIR_FLAG} build -c opt --config=opt --cxxopt='--std=c++17' \
--fat_apk_cpu=${TARGET_ARCHS} \
--define=android_dexmerger_tool=d8_dexmerger \
--define=android_incremental_dexing_tool=d8_dexbuilder\
--host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
//tmp:tensorflow-lite-select-tf-ops
OUT_FILES="${OUT_FILES} bazel-bin/tmp/tensorflow-lite-select-tf-ops.aar"
}
# Check command line flags.
TARGET_ARCHS=x86,x86_64,arm64-v8a,armeabi-v7a
# If the environmant variable BAZEL_CACHE_DIR is set, use it as the user root
# directory of bazel.
if [ ! -z ${BAZEL_CACHE_DIR} ]; then
CACHE_DIR_FLAG="--output_user_root=${BAZEL_CACHE_DIR}/cache"
fi
if [ "$#" -gt 4 ]; then
echo "ERROR: Too many arguments."
print_usage
fi
for i in "$@"
do
case $i in
--input_models=*)
FLAG_MODELS="${i#*=}"
shift;;
--target_archs=*)
TARGET_ARCHS="${i#*=}"
shift;;
--tflite_custom_ops_srcs=*)
FLAG_TFLITE_OPS_SRCS="${i#*=}"
shift;;
--tflite_custom_ops_deps=*)
FLAG_TFLITE_OPS_DEPS="${i#*=}"
shift;;
*)
echo "ERROR: Unrecognized argument: ${i}"
print_usage;;
esac
done
# Check if users already run configure
cd $ROOT_DIR
if [ ! -f "$ROOT_DIR/.tf_configure.bazelrc" ]; then
echo "ERROR: Please run ./configure first."
exit 1
else
if ! grep -q ANDROID_SDK_HOME "$ROOT_DIR/.tf_configure.bazelrc"; then
echo "ERROR: Please run ./configure with Android config."
exit 1
fi
fi
if [ -z ${FLAG_MODELS} ]; then
bazel ${CACHE_DIR_FLAG} build -c opt --config=opt --cxxopt='--std=c++17' \
--config=monolithic \
--fat_apk_cpu=${TARGET_ARCHS} \
--define=android_dexmerger_tool=d8_dexmerger \
--define=android_incremental_dexing_tool=d8_dexbuilder\
--host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
//tensorflow/lite/java:tensorflow-lite
print_output bazel-bin/tensorflow/lite/java/tensorflow-lite.aar
exit 0
fi
# Prepare the tmp directory.
TMP_DIR="${ROOT_DIR}/tmp/"
rm -rf ${TMP_DIR} && mkdir -p ${TMP_DIR}
# Copy models to tmp directory.
MODEL_NAMES=""
for model in $(echo ${FLAG_MODELS} | sed "s/,/ /g")
do
cp ${model} ${TMP_DIR}
MODEL_NAMES="${MODEL_NAMES},$(basename ${model})"
done
# Copy srcs of additional tflite ops to tmp directory.
TFLITE_OPS_SRCS=""
for src_file in $(echo ${FLAG_TFLITE_OPS_SRCS} | sed "s/,/ /g")
do
cp ${src_file} ${TMP_DIR}
TFLITE_OPS_SRCS="${TFLITE_OPS_SRCS},$(basename ${src_file})"
done
# Build the custom aar package.
generate_tflite_aar
# Build flex aar if one of the models contain flex ops.
bazel ${CACHE_DIR_FLAG} build -c opt --config=monolithic \
//tensorflow/lite/tools:list_flex_ops_no_kernel_main
bazel-bin/tensorflow/lite/tools/list_flex_ops_no_kernel_main \
--graphs=${FLAG_MODELS} > ${TMP_DIR}/ops_list.txt
if [[ `cat ${TMP_DIR}/ops_list.txt` != "[]" ]]; then
generate_flex_aar
fi
# List the output files.
rm -rf ${TMP_DIR}
print_output ${OUT_FILES}
tensorflow/lite/tools/build_aar_with_docker.sh
Executable
+149
View File
@@ -0,0 +1,149 @@
#!/bin/bash
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
set -e
set -x
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
function print_usage {
echo "Usage:"
echo " $(basename ${BASH_SOURCE}) \\"
echo " --input_models=model1.tflite,model2.tflite \\"
echo " --target_archs=x86,x86_64,arm64-v8a,armeabi-v7a \\"
echo " --src_dir=$PWD \\"
echo " [--cache_dir=<path to cache directory>]"
echo ""
echo "Where: "
echo " --input_models: Supported TFLite models. "
echo " --target_archs: Supported arches included in the aar file."
echo " --src_dir: Directory of tensorflow source code."
echo " --cache_dir: Path to the directory to store bazel cache. If not "
echo " provided, a directory name bazel-build-cache will be created."
echo ""
exit 1
}
# Check command line flags.
ARGUMENTS=$@
BUILD_FLAGS=""
TARGET_ARCHS=x86,x86_64,arm64-v8a,armeabi-v7a
FLAG_SRC_DIR=""
if [ "$#" -gt 5 ]; then
echo "ERROR: Too many arguments."
print_usage
fi
for i in "$@"
do
case $i in
--input_models=*)
FLAG_MODELS="${i#*=}"
BUILD_FLAGS="${BUILD_FLAGS} ${i}"
shift;;
--target_archs=*)
TARGET_ARCHS="${i#*=}"
BUILD_FLAGS="${BUILD_FLAGS} ${i}"
shift;;
--src_dir=*)
FLAG_SRC_DIR="${i#*=}"
shift;;
--cache_dir=*)
BAZEL_CACHE_DIR="${i#*=}"
shift;;
--debug)
DEBUG_MODE=true
shift;;
*)
echo "ERROR: Unrecognized argument: ${i}"
print_usage;;
esac
done
if [ ! -d /tensorflow_src ]; then
# Running on host.
for model in $(echo ${FLAG_MODELS} | sed "s/,/ /g")
do
FLAG_DIR="${FLAG_DIR} -v ${model}:${model}"
done
if [ -z ${BAZEL_CACHE_DIR} ]; then
mkdir -p "bazel-build-cache"
BAZEL_CACHE_DIR="$PWD/bazel-build-cache"
ARGUMENTS="${ARGUMENTS} --cache_dir=${BAZEL_CACHE_DIR}"
fi
FLAG_DIR="${FLAG_DIR} -v ${BAZEL_CACHE_DIR}:${BAZEL_CACHE_DIR}"
docker run --rm -it -v ${FLAG_SRC_DIR}:/tensorflow_src -v $PWD:/host_dir \
-v ${SCRIPT_DIR}:/script_dir ${FLAG_DIR} \
--entrypoint /script_dir/build_aar_with_docker.sh tflite-builder \
${ARGUMENTS}
exit 0
else
# Running inside docker container, download the SDK first.
sdkmanager --licenses
sdkmanager \
"build-tools;${ANDROID_BUILD_TOOLS_VERSION}" \
"platform-tools" \
"platforms;android-${ANDROID_API_LEVEL}"
cd /tensorflow_src
# Run configure.
# -Wno-c++20-designator can be removed once tf supports C++20.
# -Wno-gnu-inline-cpp-without-extern is needed for NEON2SSE. Can remove after
# https://github.com/intel/ARM_NEON_2_x86_SSE/issues/57 is resolved.
configs=(
'/usr/bin/python3'
'/usr/lib/python3/dist-packages'
'N'
'N'
'Y'
'/usr/lib/llvm-18/bin/clang'
'-Wno-sign-compare -Wno-c++20-designator -Wno-gnu-inline-cpp-without-extern'
'y'
'/android/sdk'
)
printf '%s\n' "${configs[@]}" | ./configure
# Configure Bazel.
source tensorflow/tools/ci_build/release/common.sh
install_bazelisk
# Building with bazel.
export BAZEL_CACHE_DIR=${BAZEL_CACHE_DIR}
export OMIT_PRINTING_OUTPUT_PATHS=YES
if [ "${DEBUG_MODE}" = true ]; then
echo "### Run /tensorflow_src/tensorflow/lite/tools/build_aar.sh ${BUILD_FLAGS}"
bash -i
exit 0
else
bash /tensorflow_src/tensorflow/lite/tools/build_aar.sh ${BUILD_FLAGS}
fi
# Copy the output files from docker container.
OUT_FILES="/tensorflow_src/bazel-bin/tmp/tensorflow-lite.aar"
OUT_FILES="${OUT_FILES} /tensorflow_src/bazel-bin/tmp/tensorflow-lite-select-tf-ops.aar"
echo "Output can be found here:"
for i in ${OUT_FILES}
do
if [ -f $i ]; then
cp $i /host_dir
basename $i
fi
done
fi

Some files were not shown because too many files have changed in this diff Show More