chore: import upstream snapshot with attribution

tensorflow/lite/tools/evaluation/BUILD

@@ -0,0 +1,137 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
+load("@rules_cc//cc:cc_test.bzl", "cc_test")
+load("//tensorflow/lite:build_def.bzl", "tflite_copts", "tflite_linkopts")
+load("//tensorflow/lite/core/shims:cc_library_with_tflite.bzl", "cc_library_with_stable_tflite_abi", "cc_test_with_tflite")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_visibility = ["//visibility:public"],
+    licenses = ["notice"],
+)
+
+exports_files(glob([
+    "testdata/**",
+]))
+
+cc_library(
+    name = "evaluation_stage",
+    hdrs = ["evaluation_stage.h"],
+    copts = tflite_copts(),
+    deps = [
+        "//tensorflow/lite/core/c:common",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
+    ],
+)
+
+cc_library_with_stable_tflite_abi(
+    name = "utils",
+    srcs = ["utils.cc"],
+    hdrs = ["utils.h"],
+    copts = tflite_copts(),
+    non_stable_abi_deps = [
+        "//tensorflow/lite/delegates/nnapi:nnapi_delegate",
+    ] + select({
+        "//tensorflow:ios": [
+            "//tensorflow/lite/delegates/coreml:coreml_delegate",
+        ],
+        "//tensorflow:macos_arm64": [
+            "//tensorflow/lite/delegates/coreml:coreml_delegate",
+        ],
+        "//conditions:default": [],
+    }) + select({
+        "//tensorflow/lite/delegates/gpu:supports_gpu_delegate": [
+            "//tensorflow/lite/delegates/gpu:delegate",
+        ],
+        "//conditions:default": [],
+    }) + select({
+        "//tensorflow:arm_any": [
+            "//tensorflow/lite/delegates/hexagon:hexagon_delegate",
+        ],
+        "//conditions:default": [],
+    }),
+    tflite_deps = [
+        "//tensorflow/lite/c:c_api",
+        "//tensorflow/lite/c:common",
+        "//tensorflow/lite/acceleration/configuration/c:delegate_plugin",
+        "//tensorflow/lite/tools/delegates:delegate_provider_hdr",
+        "//tensorflow/lite/tools/delegates:delegate_provider_lib",
+    ],
+    tflite_deps_selects = [{
+        "//tensorflow/lite:tflite_with_xnnpack_explicit_false": [],
+        "//conditions:default": [
+            "//tensorflow/lite/acceleration/configuration/c:xnnpack_plugin",
+        ],
+    }],
+    deps = [
+        "//tensorflow/lite/tools:logging",
+        "@flatbuffers",
+    ] + select({
+        "//tensorflow/lite:tflite_with_xnnpack_explicit_false": [],
+        "//conditions:default": [
+            "//tensorflow/lite/acceleration/configuration:configuration_fbs",
+            "//tensorflow/lite/delegates/xnnpack:xnnpack_delegate_hdrs_only",
+        ],
+    }),
+)
+
+cc_library(
+    name = "evaluation_delegate_provider",
+    srcs = ["evaluation_delegate_provider.cc"],
+    hdrs = ["evaluation_delegate_provider.h"],
+    copts = tflite_copts(),
+    deps = [
+        ":utils",
+        "//tensorflow/lite/c:c_api_types",
+        "//tensorflow/lite/tools:command_line_flags",
+        "//tensorflow/lite/tools:logging",
+        "//tensorflow/lite/tools:tool_params",
+        "//tensorflow/lite/tools/delegates:delegate_provider_hdr",
+        "//tensorflow/lite/tools/delegates:tflite_execution_providers",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
+    ],
+)
+
+cc_test_with_tflite(
+    name = "utils_test",
+    srcs = ["utils_test.cc"],
+    data = [
+        "testdata/empty.txt",
+        "testdata/labels.txt",
+    ],
+    linkopts = tflite_linkopts(),
+    linkstatic = 1,
+    tflite_deps = [
+        ":utils",
+    ],
+    deps = [
+        "//tensorflow/lite:context",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
+cc_test(
+    name = "evaluation_delegate_provider_test",
+    srcs = ["evaluation_delegate_provider_test.cc"],
+    linkopts = tflite_linkopts(),
+    deps = [
+        ":evaluation_delegate_provider",
+        "//tensorflow/lite/tools:tool_params",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
+        "@com_google_googletest//:gtest_main",
+    ],
+)

tensorflow/lite/tools/evaluation/evaluation_delegate_provider.cc

@@ -0,0 +1,184 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/tools/evaluation/evaluation_delegate_provider.h"
+
+#include <cstdint>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "tensorflow/lite/c/c_api_types.h"
+#include "tensorflow/lite/tools/command_line_flags.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+#include "tensorflow/lite/tools/evaluation/utils.h"
+#include "tensorflow/lite/tools/logging.h"
+#include "tensorflow/lite/tools/tool_params.h"
+
+namespace tflite {
+namespace evaluation {
+namespace {
+constexpr char kNnapiDelegate[] = "nnapi";
+constexpr char kGpuDelegate[] = "gpu";
+constexpr char kHexagonDelegate[] = "hexagon";
+constexpr char kXnnpackDelegate[] = "xnnpack";
+constexpr char kCoremlDelegate[] = "coreml";
+}  // namespace
+
+TfliteInferenceParams::Delegate ParseStringToDelegateType(
+    const std::string& val) {
+  if (val == kNnapiDelegate) return TfliteInferenceParams::NNAPI;
+  if (val == kGpuDelegate) return TfliteInferenceParams::GPU;
+  if (val == kHexagonDelegate) return TfliteInferenceParams::HEXAGON;
+  if (val == kXnnpackDelegate) return TfliteInferenceParams::XNNPACK;
+  if (val == kCoremlDelegate) return TfliteInferenceParams::COREML;
+  return TfliteInferenceParams::NONE;
+}
+
+TfLiteDelegatePtr CreateTfLiteDelegate(const TfliteInferenceParams& params,
+                                       std::string* error_msg) {
+  const auto type = params.delegate();
+  switch (type) {
+    case TfliteInferenceParams::NNAPI: {
+      auto p = CreateNNAPIDelegate();
+      if (!p && error_msg) *error_msg = "NNAPI not supported";
+      return p;
+    }
+    case TfliteInferenceParams::GPU: {
+      auto p = CreateGPUDelegate();
+      if (!p && error_msg) *error_msg = "GPU delegate not supported.";
+      return p;
+    }
+    case TfliteInferenceParams::HEXAGON: {
+      auto p = CreateHexagonDelegate(/*library_directory_path=*/"",
+                                     /*profiling=*/false);
+      if (!p && error_msg) {
+        *error_msg =
+            "Hexagon delegate is not supported on the platform or required "
+            "libraries are missing.";
+      }
+      return p;
+    }
+    case TfliteInferenceParams::XNNPACK: {
+      auto p = CreateXNNPACKDelegate(params.num_threads(), false);
+      if (!p && error_msg) *error_msg = "XNNPACK delegate not supported.";
+      return p;
+    }
+    case TfliteInferenceParams::COREML: {
+      auto p = CreateCoreMlDelegate();
+      if (!p && error_msg) *error_msg = "CoreML delegate not supported.";
+      return p;
+    }
+    case TfliteInferenceParams::NONE:
+      return TfLiteDelegatePtr(nullptr, [](TfLiteDelegate*) {});
+    default:
+      if (error_msg) {
+        *error_msg = "Creation of delegate type: " +
+                     TfliteInferenceParams::Delegate_Name(type) +
+                     " not supported yet.";
+      }
+      return TfLiteDelegatePtr(nullptr, [](TfLiteDelegate*) {});
+  }
+}
+
+DelegateProviders::DelegateProviders()
+    : delegate_list_util_(&params_),
+      delegates_map_([=]() -> std::unordered_map<std::string, int> {
+        std::unordered_map<std::string, int> delegates_map;
+        const auto& providers = delegate_list_util_.providers();
+        for (int i = 0; i < providers.size(); ++i) {
+          delegates_map[providers[i]->GetName()] = i;
+        }
+        return delegates_map;
+      }()) {
+  delegate_list_util_.AddAllDelegateParams();
+}
+
+std::vector<Flag> DelegateProviders::GetFlags() {
+  std::vector<Flag> flags;
+  delegate_list_util_.AppendCmdlineFlags(flags);
+  return flags;
+}
+
+bool DelegateProviders::InitFromCmdlineArgs(int* argc, const char** argv) {
+  std::vector<Flag> flags = GetFlags();
+  bool parse_result = Flags::Parse(argc, argv, flags);
+  if (!parse_result || params_.Get<bool>("help")) {
+    std::string usage = Flags::Usage(argv[0], flags);
+    TFLITE_LOG(ERROR) << usage;
+    // Returning false intentionally when "--help=true" is specified so that
+    // the caller could check the return value to decide stopping the execution.
+    parse_result = false;
+  }
+  return parse_result;
+}
+
+TfLiteDelegatePtr DelegateProviders::CreateDelegate(
+    const std::string& name) const {
+  const auto it = delegates_map_.find(name);
+  if (it == delegates_map_.end()) {
+    return TfLiteDelegatePtr(nullptr, [](TfLiteDelegate*) {});
+  }
+  const auto& providers = delegate_list_util_.providers();
+  return providers[it->second]->CreateTfLiteDelegate(params_);
+}
+
+tools::ToolParams DelegateProviders::GetAllParams(
+    const TfliteInferenceParams& params) const {
+  tools::ToolParams tool_params;
+  tool_params.Merge(params_, /*overwrite*/ false);
+
+  if (params.has_num_threads()) {
+    tool_params.Set<int32_t>("num_threads", params.num_threads());
+  }
+
+  const auto type = params.delegate();
+  switch (type) {
+    case TfliteInferenceParams::NNAPI:
+      if (tool_params.HasParam("use_nnapi")) {
+        tool_params.Set<bool>("use_nnapi", true);
+      }
+      break;
+    case TfliteInferenceParams::GPU:
+      if (tool_params.HasParam("use_gpu")) {
+        tool_params.Set<bool>("use_gpu", true);
+      }
+      break;
+    case TfliteInferenceParams::HEXAGON:
+      if (tool_params.HasParam("use_hexagon")) {
+        tool_params.Set<bool>("use_hexagon", true);
+      }
+      break;
+    case TfliteInferenceParams::XNNPACK:
+      if (tool_params.HasParam("use_xnnpack")) {
+        tool_params.Set<bool>("use_xnnpack", true);
+      }
+      if (tool_params.HasParam("xnnpack_force_fp16")) {
+        tool_params.Set<bool>("xnnpack_force_fp16", true);
+      }
+      break;
+    case TfliteInferenceParams::COREML:
+      if (tool_params.HasParam("use_coreml")) {
+        tool_params.Set<bool>("use_coreml", true);
+      }
+      break;
+    default:
+      break;
+  }
+  return tool_params;
+}
+
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/evaluation_delegate_provider.h

@@ -0,0 +1,101 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_TOOLS_EVALUATION_EVALUATION_DELEGATE_PROVIDER_H_
+#define TENSORFLOW_LITE_TOOLS_EVALUATION_EVALUATION_DELEGATE_PROVIDER_H_
+
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "tensorflow/lite/tools/command_line_flags.h"
+#include "tensorflow/lite/tools/delegates/delegate_provider.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+#include "tensorflow/lite/tools/evaluation/utils.h"
+#include "tensorflow/lite/tools/tool_params.h"
+
+namespace tflite {
+namespace evaluation {
+
+using ProvidedDelegateList = tflite::tools::ProvidedDelegateList;
+class DelegateProviders {
+ public:
+  DelegateProviders();
+
+  // Returns a list of commandline flags that delegate providers define.
+  std::vector<Flag> GetFlags();
+
+  // Initialize delegate-related parameters from commandline arguments and
+  // returns true if successful.
+  bool InitFromCmdlineArgs(int* argc, const char** argv);
+
+  // Get all parameters from all registered delegate providers.
+  const tools::ToolParams& GetAllParams() const { return params_; }
+
+  // Get a new set of parameters based on the given TfliteInferenceParams
+  // 'params' but considering what have been initialized (i.e. 'params_').
+  // Note the same-meaning parameter (e.g. number of TfLite interpreter threads)
+  // in TfliteInferenceParams will take precedence over the parameter of the
+  // same meaning in 'params_'.
+  tools::ToolParams GetAllParams(const TfliteInferenceParams& params) const;
+
+  // Create the a TfLite delegate instance based on the provided delegate
+  // 'name'. If the specified one isn't found, an empty TfLiteDelegatePtr is
+  // returned.
+  TfLiteDelegatePtr CreateDelegate(const std::string& name) const;
+
+  // Create a list of TfLite delegates based on what have been initialized (i.e.
+  // 'params_').
+  std::vector<ProvidedDelegateList::ProvidedDelegate> CreateAllDelegates()
+      const {
+    return delegate_list_util_.CreateAllRankedDelegates();
+  }
+
+  // Create a list of TfLite delegates based on the given TfliteInferenceParams
+  // 'params' but considering what have been initialized (i.e. 'params_').
+  std::vector<ProvidedDelegateList::ProvidedDelegate> CreateAllDelegates(
+      const TfliteInferenceParams& params) const {
+    auto converted = GetAllParams(params);
+    ProvidedDelegateList util(&converted);
+    return util.CreateAllRankedDelegates();
+  }
+
+ private:
+  // Contain delegate-related parameters that are initialized from command-line
+  // flags.
+  tools::ToolParams params_;
+
+  // A helper to create TfLite delegates.
+  ProvidedDelegateList delegate_list_util_;
+
+  // Key is the delegate name, and the value is the index to a TfLite delegate
+  // provider in the "delegate_list_util_.providers()" list.
+  const std::unordered_map<std::string, int> delegates_map_;
+};
+
+// Parse a string 'val' to the corresponding delegate type defined by
+// TfliteInferenceParams::Delegate.
+TfliteInferenceParams::Delegate ParseStringToDelegateType(
+    const std::string& val);
+
+// Create a TfLite delegate based on the given TfliteInferenceParams 'params'.
+// If there's an error during the creation, an error message will be recorded to
+// 'error_msg' if provided.
+TfLiteDelegatePtr CreateTfLiteDelegate(const TfliteInferenceParams& params,
+                                       std::string* error_msg = nullptr);
+}  // namespace evaluation
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_EVALUATION_EVALUATION_DELEGATE_PROVIDER_H_

tensorflow/lite/tools/evaluation/evaluation_delegate_provider_test.cc

@@ -0,0 +1,77 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/evaluation/evaluation_delegate_provider.h"
+
+#include <gtest/gtest.h>
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+#include "tensorflow/lite/tools/tool_params.h"
+
+namespace tflite {
+namespace evaluation {
+namespace {
+TEST(EvaluationDelegateProviderTest, ParseStringToDelegateType) {
+  EXPECT_EQ(TfliteInferenceParams::NNAPI, ParseStringToDelegateType("nnapi"));
+  EXPECT_EQ(TfliteInferenceParams::GPU, ParseStringToDelegateType("gpu"));
+  EXPECT_EQ(TfliteInferenceParams::HEXAGON,
+            ParseStringToDelegateType("hexagon"));
+  EXPECT_EQ(TfliteInferenceParams::XNNPACK,
+            ParseStringToDelegateType("xnnpack"));
+
+  EXPECT_EQ(TfliteInferenceParams::NONE, ParseStringToDelegateType("Gpu"));
+  EXPECT_EQ(TfliteInferenceParams::NONE, ParseStringToDelegateType("Testing"));
+}
+
+TEST(EvaluationDelegateProviderTest, CreateTfLiteDelegate) {
+  TfliteInferenceParams params;
+  params.set_delegate(TfliteInferenceParams::NONE);
+  // A NONE delegate type will return a nullptr TfLite delegate ptr.
+  EXPECT_TRUE(!CreateTfLiteDelegate(params));
+}
+
+TEST(EvaluationDelegateProviderTest, DelegateProvidersParams) {
+  DelegateProviders providers;
+  const auto& params = providers.GetAllParams();
+  EXPECT_TRUE(params.HasParam("use_nnapi"));
+  EXPECT_TRUE(params.HasParam("use_gpu"));
+
+  int argc = 3;
+  const char* argv[] = {"program_name", "--use_gpu=true",
+                        "--other_undefined_flag=1"};
+  EXPECT_TRUE(providers.InitFromCmdlineArgs(&argc, argv));
+  EXPECT_TRUE(params.Get<bool>("use_gpu"));
+  EXPECT_EQ(2, argc);
+  EXPECT_EQ("--other_undefined_flag=1", argv[1]);
+}
+
+TEST(EvaluationDelegateProviderTest, GetAllParamsWithTfliteInferenceParams) {
+  DelegateProviders providers;
+  int argc = 2;
+  const char* argv[] = {"program_name", "--num_threads=1"};
+  EXPECT_TRUE(providers.InitFromCmdlineArgs(&argc, argv));
+  const auto& default_params = providers.GetAllParams();
+  EXPECT_EQ(1, default_params.Get<int>("num_threads"));
+
+  TfliteInferenceParams params;
+  params.set_delegate(TfliteInferenceParams::NONE);
+  params.set_num_threads(4);
+  // The same-meaning parameter in TfliteInferenceParams takes precedence.
+  tools::ToolParams tool_params = providers.GetAllParams(params);
+  EXPECT_EQ(4, tool_params.Get<int>("num_threads"));
+  EXPECT_EQ(1, argc);
+}
+
+}  // namespace
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/evaluation_stage.h

@@ -0,0 +1,64 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_EVALUATION_EVALUATION_STAGE_H_
+#define TENSORFLOW_LITE_TOOLS_EVALUATION_EVALUATION_STAGE_H_
+
+#include "tensorflow/lite/core/c/common.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+
+namespace tflite {
+namespace evaluation {
+
+// Superclass for a single stage of an EvaluationPipeline.
+// Defines basic skeleton for sub-classes to implement.
+//
+// Ideally EvaluationStages should obtain access to initializer/input objects
+// via Get/Set methods on pointers, and not take ownership unless necessary.
+class EvaluationStage {
+ public:
+  // Initializes an EvaluationStage, including verifying the
+  // EvaluationStageConfig. Returns kTfLiteError if initialization failed,
+  // kTfLiteOk otherwise.
+  //
+  // Sub-classes are responsible for ensuring required class members are defined
+  // via Get/Set methods.
+  virtual TfLiteStatus Init() = 0;
+
+  // An individual run of the EvaluationStage. This is where the task to be
+  // evaluated takes place. Returns kTfLiteError if there was a failure,
+  // kTfLiteOk otherwise.
+  //
+  // Sub-classes are responsible for ensuring they have access to required
+  // inputs via Get/Set methods.
+  virtual TfLiteStatus Run() = 0;
+
+  // Returns the latest metrics based on all Run() calls made so far.
+  virtual EvaluationStageMetrics LatestMetrics() = 0;
+
+  virtual ~EvaluationStage() = default;
+
+ protected:
+  // Constructs an EvaluationStage.
+  // Each subclass constructor must invoke this constructor.
+  explicit EvaluationStage(const EvaluationStageConfig& config)
+      : config_(config) {}
+
+  EvaluationStageConfig config_;
+};
+
+}  // namespace evaluation
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_EVALUATION_EVALUATION_STAGE_H_

tensorflow/lite/tools/evaluation/proto/BUILD

@@ -0,0 +1,100 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+load("@com_google_protobuf//bazel:java_proto_library.bzl", "java_proto_library")
+load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
+load("@rules_python//python:proto.bzl", "py_proto_library")
+load(
+    "//tensorflow/core/platform:build_config.bzl",
+    "tf_proto_library",
+)
+# copybara:uncomment load("//tools/build_defs/proto/cpp:cc_proto_library.bzl", "cc_proto_library")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_visibility = ["//visibility:public"],
+    licenses = ["notice"],
+)
+
+proto_library(
+    name = "evaluation_stages_proto",
+    srcs = [
+        "evaluation_stages.proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":preprocessing_steps_proto"],
+)
+
+cc_proto_library(
+    name = "evaluation_stages_cc_proto",
+    deps = ["evaluation_stages_proto"],
+)
+
+tf_proto_library(
+    name = "evaluation_stages",  # bzl adds _py
+    srcs = [
+        "evaluation_stages.proto",
+    ],
+    protodeps = [":preprocessing_steps"],
+    visibility = ["//visibility:public"],
+)
+
+tf_proto_library(
+    name = "preprocessing_steps",
+    srcs = [
+        "preprocessing_steps.proto",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+proto_library(
+    name = "evaluation_config_proto",
+    srcs = [
+        "evaluation_config.proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":evaluation_stages_proto"],
+)
+
+cc_proto_library(
+    name = "evaluation_config_cc_proto",
+    deps = ["evaluation_config_proto"],
+)
+
+java_proto_library(
+    name = "evaluation_config_java_proto",
+    deps = ["evaluation_config_proto"],
+)
+
+proto_library(
+    name = "preprocessing_steps_proto",
+    srcs = [
+        "preprocessing_steps.proto",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+cc_proto_library(
+    name = "preprocessing_steps_cc_proto",
+    deps = ["preprocessing_steps_proto"],
+)
+
+# copybara:uncomment_begin(google-only)
+# py_proto_library(
+#     name = "evaluation_stages_py_pb2",
+#     visibility = ["//visibility:public"],
+#     deps = [":evaluation_stages_proto"],
+# )
+# copybara:uncomment_end

tensorflow/lite/tools/evaluation/proto/evaluation_config.proto

@@ -0,0 +1,46 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+syntax = "proto2";
+
+package tflite.evaluation;
+
+import "tensorflow/lite/tools/evaluation/proto/evaluation_stages.proto";
+
+option cc_enable_arenas = true;
+option java_multiple_files = true;
+option java_package = "tflite.evaluation";
+
+// Contains parameters that define how an EvaluationStage will be executed.
+// This would typically be validated only once during initialization, so should
+// not contain any variables that change with each run.
+//
+// Next ID: 3
+message EvaluationStageConfig {
+  optional string name = 1;
+
+  // Specification defining what this stage does, and any required parameters.
+  optional ProcessSpecification specification = 2;
+}
+
+// Metrics returned from EvaluationStage.LatestMetrics() need not have all
+// fields set.
+message EvaluationStageMetrics {
+  // Total number of times the EvaluationStage is run.
+  optional int32 num_runs = 1;
+
+  // Process-specific numbers such as latencies, accuracy, etc.
+  optional ProcessMetrics process_metrics = 2;
+}

tensorflow/lite/tools/evaluation/proto/evaluation_stages.proto

@@ -0,0 +1,314 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+syntax = "proto2";
+
+package tflite.evaluation;
+
+import "tensorflow/lite/tools/evaluation/proto/preprocessing_steps.proto";
+
+option cc_enable_arenas = true;
+option java_multiple_files = true;
+option java_package = "tflite.evaluation";
+
+// Defines the functionality executed by an EvaluationStage.
+//
+// Next ID: 7
+message ProcessSpecification {
+  oneof params {
+    ImagePreprocessingParams image_preprocessing_params = 1;
+    TopkAccuracyEvalParams topk_accuracy_eval_params = 2;
+    TfliteInferenceParams tflite_inference_params = 3;
+    ImageClassificationParams image_classification_params = 4;
+    ObjectDetectionAveragePrecisionParams
+        object_detection_average_precision_params = 5;
+    ObjectDetectionParams object_detection_params = 6;
+  }
+}
+
+// Latency numbers in microseconds, based on all EvaluationStage::Run() calls so
+// far.
+//
+// Next ID: 7
+message LatencyMetrics {
+  // Latency for the last Run.
+  optional int64 last_us = 1;
+  // Maximum latency observed for any Run.
+  optional int64 max_us = 2;
+  // Minimum latency observed for any Run.
+  optional int64 min_us = 3;
+  // Sum of all Run latencies.
+  optional int64 sum_us = 4;
+  // Average latency across all Runs.
+  optional double avg_us = 5;
+  // Standard deviation for latency across all Runs.
+  optional int64 std_deviation_us = 6;
+}
+
+// Statistics for an accuracy value over multiple runs of evaluation.
+//
+// Next ID: 5
+message AccuracyMetrics {
+  // Maximum value observed for any Run.
+  optional float max_value = 1;
+  // Minimum value observed for any Run.
+  optional float min_value = 2;
+  // Average value across all Runs.
+  optional double avg_value = 3;
+  // Standard deviation across all Runs.
+  optional float std_deviation = 4;
+}
+
+// Contains process-specific metrics, which may differ based on what an
+// EvaluationStage does.
+//
+// Next ID: 8
+message ProcessMetrics {
+  optional LatencyMetrics total_latency = 1;
+
+  oneof stage_metrics {
+    TopkAccuracyEvalMetrics topk_accuracy_metrics = 2;
+    TfliteInferenceMetrics tflite_inference_metrics = 3;
+    ImageClassificationMetrics image_classification_metrics = 4;
+    InferenceProfilerMetrics inference_profiler_metrics = 5;
+    ObjectDetectionAveragePrecisionMetrics
+        object_detection_average_precision_metrics = 6;
+    ObjectDetectionMetrics object_detection_metrics = 7;
+  }
+}
+
+// Parameters that define how images are preprocessed.
+//
+// Next ID: 3
+message ImagePreprocessingParams {
+  // Required.
+  repeated ImagePreprocessingStepParams steps = 1;
+  // Same as tflite::TfLiteType.
+  required int32 output_type = 2;
+}
+
+// Parameters that control TFLite inference.
+//
+// Next ID: 5
+message TfliteInferenceParams {
+  // Required
+  optional string model_file_path = 1;
+
+  enum Delegate {
+    NONE = 0;
+    NNAPI = 1;
+    GPU = 2;
+    HEXAGON = 3;
+    XNNPACK = 4;
+    COREML = 5;
+  }
+  optional Delegate delegate = 2;
+  // Number of threads available to the TFLite Interpreter.
+  optional int32 num_threads = 3 [default = 1];
+
+  // Defines how many times the TFLite Interpreter is invoked for every input.
+  // This helps benchmark cases where extensive pre-processing might not be
+  // required for every input.
+  optional int32 invocations_per_run = 4 [default = 1];
+}
+
+// Metrics specific to TFLite inference.
+//
+// Next ID: 2
+message TfliteInferenceMetrics {
+  // Number of times the interpreter is invoked.
+  optional int32 num_inferences = 1;
+}
+
+// Parameters that define how top-K accuracy is evaluated.
+//
+// Next ID: 2
+message TopkAccuracyEvalParams {
+  // Required.
+  optional int32 k = 1;
+}
+
+// Metrics from top-K accuracy evaluation.
+//
+// Next ID: 2
+message TopkAccuracyEvalMetrics {
+  // A repeated field of size |k| where the ith element denotes the fraction of
+  // samples for which the correct label was present in the top (i + 1) model
+  // outputs.
+  // For example, topk_accuracies(1) will contain the fraction of
+  // samples for which the model returned the correct label as the top first or
+  // second output.
+  repeated float topk_accuracies = 1;
+}
+
+// Parameters that define how the Image Classification task is evaluated
+// end-to-end.
+//
+// Next ID: 3
+message ImageClassificationParams {
+  // Required.
+  // TfLite model should have 1 input & 1 output tensor.
+  // Input shape: {1, image_height, image_width, 3}
+  // Output shape: {1, num_total_labels}
+  optional TfliteInferenceParams inference_params = 1;
+
+  // Optional.
+  // If not set, accuracy evaluation is not performed.
+  optional TopkAccuracyEvalParams topk_accuracy_eval_params = 2;
+}
+
+// Metrics from evaluation of the image classification task.
+//
+// Next ID: 5
+message ImageClassificationMetrics {
+  optional LatencyMetrics pre_processing_latency = 1;
+  optional LatencyMetrics inference_latency = 2;
+  optional TfliteInferenceMetrics inference_metrics = 3;
+  // Not set if topk_accuracy_eval_params was not populated in
+  // ImageClassificationParams.
+  optional TopkAccuracyEvalMetrics topk_accuracy_metrics = 4;
+}
+
+// Metrics computed from comparing TFLite execution in two settings:
+// 1. User-defined TfliteInferenceParams (The 'test' setting)
+// 2. Default TfliteInferenceParams (The 'reference' setting)
+//
+// Next ID: 4
+message InferenceProfilerMetrics {
+  // Latency metrics from Single-thread CPU inference.
+  optional LatencyMetrics reference_latency = 1;
+  // Latency from TfliteInferenceParams under test.
+  optional LatencyMetrics test_latency = 2;
+  // For reference & test output vectors {R, T}, the error is computed as:
+  // Mean([Abs(R[i] - T[i]) for i in num_elements])
+  // output_errors[v] : statistics for the error value of the vth output vector
+  //   across all Runs.
+  repeated AccuracyMetrics output_errors = 3;
+}
+
+// Proto containing information about all the objects (predicted or
+// ground-truth) contained in an image.
+//
+// Next ID: 4
+message ObjectDetectionResult {
+  // One instance of an object detected in an image.
+  // Next ID: 4
+  message ObjectInstance {
+    // Defines the bounding box for a detected object.
+    // Next ID: 5
+    message NormalizedBoundingBox {
+      // All boundaries defined below are required.
+      // Each boundary value should be normalized with respect to the image
+      // dimensions. This helps evaluate detections independent of image size.
+      // For example, normalized_top = top_boundary / image_height.
+      optional float normalized_top = 1;
+      optional float normalized_bottom = 2;
+      optional float normalized_left = 3;
+      optional float normalized_right = 4;
+    }
+
+    // Required.
+    optional int32 class_id = 1;
+    // Required
+    optional NormalizedBoundingBox bounding_box = 2;
+    // Value in (0, 1.0] denoting confidence in this prediction.
+    // Default value of 1.0 for ground-truth data.
+    optional float score = 3 [default = 1.0];
+  }
+
+  repeated ObjectInstance objects = 1;
+  // Filename of the image.
+  optional string image_name = 2;
+  // Unique id for the image.
+  optional int64 image_id = 3;
+}
+
+// Proto containing ground-truth ObjectsSets for all images in a COCO validation
+// set.
+//
+// Next ID: 2
+message ObjectDetectionGroundTruth {
+  repeated ObjectDetectionResult detection_results = 1;
+}
+
+// Parameters that define how Average Precision is computed for Object Detection
+// task.
+// Refer for details: http://cocodataset.org/#detection-eval
+//
+// Next ID: 4
+message ObjectDetectionAveragePrecisionParams {
+  // Total object classes. The AP value returned for each IoU threshold is an
+  // average over all classes encountered in predicted/ground truth sets.
+  optional int32 num_classes = 1;
+  // A predicted box matches a ground truth box if and only if
+  // IoU between these two are larger than an IoU threshold.
+  // AP is computed for all relevant {IoU threshold, class} combinations and
+  // averaged to get mAP.
+  // If left empty, evaluation is done for all IoU threshods in the range
+  // 0.5:0.05:0.95 (min:increment:max).
+  repeated float iou_thresholds = 2;
+  // AP is computed as the average of maximum precision at (1
+  // + num_recall_points) recall levels. E.g., if num_recall_points is 10,
+  // recall levels are 0., 0.1, 0.2, ..., 0.9, 1.0.
+  // Default: 100
+  optional int32 num_recall_points = 3 [default = 100];
+}
+
+// Average Precision metrics from Object Detection task.
+//
+// Next ID: 3
+message ObjectDetectionAveragePrecisionMetrics {
+  // Average Precision value for a particular IoU threshold.
+  // Next ID: 3
+  message AveragePrecision {
+    optional float iou_threshold = 1;
+    optional float average_precision = 2;
+  }
+
+  // One entry for each in
+  // ObjectDetectionAveragePrecisionParams::iou_thresholds, averaged over all
+  // classes.
+  repeated AveragePrecision individual_average_precisions = 1;
+  // Average of Average Precision across all IoU thresholds.
+  optional float overall_mean_average_precision = 2;
+}
+
+// Parameters that define how the Object Detection task is evaluated
+// end-to-end.
+//
+// Next ID: 4
+message ObjectDetectionParams {
+  // Required.
+  // Model's outputs should be same as a TFLite-compatible SSD model.
+  // Refer:
+  // https://www.tensorflow.org/lite/examples/object_detection/overview#output_signature
+  optional TfliteInferenceParams inference_params = 1;
+  // Optional. Used to match ground-truth categories with model output.
+  // SSD Mobilenet V1 Model trained on COCO assumes class 0 is background class
+  // in the label file and class labels start from 1 to number_of_classes+1.
+  // Therefore, default value is set as 1.
+  optional int32 class_offset = 2 [default = 1];
+  optional ObjectDetectionAveragePrecisionParams ap_params = 3;
+}
+
+// Metrics from evaluation of the object detection task.
+//
+// Next ID: 5
+message ObjectDetectionMetrics {
+  optional LatencyMetrics pre_processing_latency = 1;
+  optional LatencyMetrics inference_latency = 2;
+  optional TfliteInferenceMetrics inference_metrics = 3;
+  optional ObjectDetectionAveragePrecisionMetrics average_precision_metrics = 4;
+}

tensorflow/lite/tools/evaluation/proto/preprocessing_steps.proto

@@ -0,0 +1,111 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+syntax = "proto2";
+
+package tflite.evaluation;
+
+option cc_enable_arenas = true;
+option java_multiple_files = true;
+option java_package = "tflite.evaluation";
+
+// Defines the preprocesing steps available.
+//
+// Next ID: 5
+message ImagePreprocessingStepParams {
+  oneof params {
+    CroppingParams cropping_params = 1;
+    ResizingParams resizing_params = 2;
+    PaddingParams padding_params = 3;
+    NormalizationParams normalization_params = 4;
+  }
+}
+
+// Defines the size of an image.
+//
+// Next ID: 3
+message ImageSize {
+  // Width of the image.
+  required uint32 width = 1;
+  // Height of the image.
+  required uint32 height = 2;
+}
+
+// Defines parameters for central-cropping.
+//
+// Next ID: 4
+message CroppingParams {
+  oneof params {
+    // Fraction for central-cropping.
+    // A central cropping-fraction of 0.875 is considered best for Inception
+    // models, hence the default value. See:
+    // https://github.com/tensorflow/tpu/blob/master/models/experimental/inception/inception_preprocessing.py#L296
+    // Set to 0 to disable cropping.
+    float cropping_fraction = 1 [default = 0.875];
+    // The target size after cropping.
+    ImageSize target_size = 2;
+  }
+  // Crops to a square image.
+  optional bool square_cropping = 3;
+}
+
+// Defines parameters for bilinear central-resizing.
+//
+// Next ID: 3
+message ResizingParams {
+  // Size of the image after resizing.
+  required ImageSize target_size = 1;
+  // If this flag is true, the resize function will preserve the image's aspect
+  // ratio. Note that in this case, the size of output image may not equal to
+  // the target size defined above.
+  required bool aspect_preserving = 2;
+}
+
+// Defines parameters for central-padding.
+//
+// Next ID: 4
+message PaddingParams {
+  oneof params {
+    // Size of the image after padding.
+    ImageSize target_size = 1;
+    // Pads to a square image.
+    bool square_padding = 2;
+  }
+  // Padding value.
+  required int32 padding_value = 3;
+}
+
+// Defines parameters for normalization.
+// The normalization formula is: output = (input - mean) * scale.
+//
+// Next ID: 4
+message NormalizationParams {
+  message PerChannelMeanValues {
+    // The mean values of r channel.
+    required float r_mean = 1;
+    // The mean values of g channel.
+    required float g_mean = 2;
+    // The mean values of b channel.
+    required float b_mean = 3;
+  }
+  oneof mean {
+    // Channelwise mean value.
+    float channelwise_mean = 1;
+    // Per-Channel mean values.
+    PerChannelMeanValues means = 2;
+  }
+  // Scale value in the normalization.
+  required float scale = 3 [default = 1.0];
+}

tensorflow/lite/tools/evaluation/stages/BUILD

@@ -0,0 +1,265 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
+load("@rules_cc//cc:cc_test.bzl", "cc_test")
+load("//tensorflow/lite:build_def.bzl", "tflite_copts", "tflite_linkopts")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_visibility = ["//visibility:public"],
+    licenses = ["notice"],
+)
+
+common_linkopts = tflite_linkopts() + select({
+    "//conditions:default": [],
+    "//tensorflow:android": [
+        "-pie",
+        "-llog",
+    ],
+})
+
+exports_files(glob([
+    "testdata/*.jpg",
+]))
+
+cc_library(
+    name = "image_preprocessing_stage",
+    srcs = ["image_preprocessing_stage.cc"],
+    hdrs = ["image_preprocessing_stage.h"],
+    copts = tflite_copts(),
+    deps = [
+        "//tensorflow/core:tflite_portable_logging",
+        "//tensorflow/lite:string",
+        "//tensorflow/lite/c:c_api_types",
+        "//tensorflow/lite/c:common",
+        "//tensorflow/lite/kernels/internal:reference_base",
+        "//tensorflow/lite/kernels/internal:types",
+        "//tensorflow/lite/profiling:time",
+        "//tensorflow/lite/tools/evaluation:evaluation_stage",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
+        "//tensorflow/lite/tools/evaluation/proto:preprocessing_steps_cc_proto",
+        "@com_google_absl//absl/base",
+        "@com_google_absl//absl/log",
+        "@com_google_absl//absl/log:absl_log",
+        "@com_google_absl//absl/strings",
+        "@libjpeg_turbo//:jpeg",
+        "@xla//xla/tsl/util:stats_calculator_portable",
+    ] + select({
+        "//tensorflow:android": [
+            "//tensorflow/core:portable_jpeg_internal",
+        ],
+        "//conditions:default": [
+            "//tensorflow/core:jpeg_internal",
+        ],
+    }),
+)
+
+cc_test(
+    name = "image_preprocessing_stage_test",
+    srcs = ["image_preprocessing_stage_test.cc"],
+    data = ["testdata/grace_hopper.jpg"],
+    linkopts = common_linkopts,
+    linkstatic = 1,
+    deps = [
+        ":image_preprocessing_stage",
+        "//tensorflow/lite/c:c_api_types",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
+        "@com_google_absl//absl/strings",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
+cc_library(
+    name = "topk_accuracy_eval_stage",
+    srcs = ["topk_accuracy_eval_stage.cc"],
+    hdrs = ["topk_accuracy_eval_stage.h"],
+    copts = tflite_copts(),
+    deps = [
+        "//tensorflow/core:tflite_portable_logging",
+        "//tensorflow/lite/c:c_api_types",
+        "//tensorflow/lite/c:common",
+        "//tensorflow/lite/tools/evaluation:evaluation_stage",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
+        "@com_google_absl//absl/log",
+    ],
+)
+
+cc_test(
+    name = "topk_accuracy_eval_stage_test",
+    srcs = ["topk_accuracy_eval_stage_test.cc"],
+    linkopts = common_linkopts,
+    linkstatic = 1,
+    deps = [
+        ":topk_accuracy_eval_stage",
+        "//tensorflow/lite/c:c_api_types",
+        "//tensorflow/lite/c:common",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
+cc_library(
+    name = "tflite_inference_stage",
+    srcs = ["tflite_inference_stage.cc"],
+    hdrs = ["tflite_inference_stage.h"],
+    copts = tflite_copts(),
+    deps = [
+        "//tensorflow/core:tflite_portable_logging",
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/c:common",
+        "//tensorflow/lite/kernels:builtin_ops",
+        "//tensorflow/lite/profiling:time",
+        "//tensorflow/lite/tools/evaluation:evaluation_delegate_provider",
+        "//tensorflow/lite/tools/evaluation:evaluation_stage",
+        "//tensorflow/lite/tools/evaluation:utils",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
+        "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/log:absl_log",
+        "@xla//xla/tsl/util:stats_calculator_portable",
+    ],
+)
+
+cc_test(
+    name = "tflite_inference_stage_test",
+    srcs = ["tflite_inference_stage_test.cc"],
+    data = ["//tensorflow/lite:testdata/add_quantized.bin"],
+    linkopts = common_linkopts,
+    linkstatic = 1,
+    deps = [
+        ":tflite_inference_stage",
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/c:common",
+        "//tensorflow/lite/tools/evaluation:utils",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:span",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
+cc_library(
+    name = "image_classification_stage",
+    srcs = ["image_classification_stage.cc"],
+    hdrs = ["image_classification_stage.h"],
+    copts = tflite_copts(),
+    deps = [
+        ":image_preprocessing_stage",
+        ":tflite_inference_stage",
+        ":topk_accuracy_eval_stage",
+        "//tensorflow/core:tflite_portable_logging",
+        "//tensorflow/lite/c:c_api_types",
+        "//tensorflow/lite/tools/evaluation:evaluation_delegate_provider",
+        "//tensorflow/lite/tools/evaluation:evaluation_stage",
+        "//tensorflow/lite/tools/evaluation:utils",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
+        "@com_google_absl//absl/log",
+    ],
+)
+
+cc_library(
+    name = "inference_profiler_stage",
+    srcs = ["inference_profiler_stage.cc"],
+    hdrs = ["inference_profiler_stage.h"],
+    copts = tflite_copts(),
+    deps = [
+        ":tflite_inference_stage",
+        "//tensorflow/core:tflite_portable_logging",
+        "//tensorflow/lite/c:c_api_types",
+        "//tensorflow/lite/tools/evaluation:evaluation_delegate_provider",
+        "//tensorflow/lite/tools/evaluation:evaluation_stage",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
+        "@FP16",
+        "@com_google_absl//absl/log",
+        "@xla//xla/tsl/util:stats_calculator_portable",
+    ],
+)
+
+cc_test(
+    name = "inference_profiler_stage_test",
+    srcs = ["inference_profiler_stage_test.cc"],
+    data = ["//tensorflow/lite:testdata/add_quantized.bin"],
+    linkopts = common_linkopts,
+    linkstatic = 1,
+    deps = [
+        ":inference_profiler_stage",
+        "//tensorflow/lite/core/c:common",
+        "//tensorflow/lite/delegates/nnapi:nnapi_delegate",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
+cc_library(
+    name = "object_detection_average_precision_stage",
+    srcs = ["object_detection_average_precision_stage.cc"],
+    hdrs = ["object_detection_average_precision_stage.h"],
+    copts = tflite_copts(),
+    deps = [
+        "//tensorflow/core:tflite_portable_logging",
+        "//tensorflow/lite/c:c_api_types",
+        "//tensorflow/lite/core/c:common",
+        "//tensorflow/lite/tools/evaluation:evaluation_stage",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
+        "//tensorflow/lite/tools/evaluation/stages/utils:image_metrics",
+        "@com_google_absl//absl/log",
+    ],
+)
+
+cc_test(
+    name = "object_detection_average_precision_stage_test",
+    srcs = ["object_detection_average_precision_stage_test.cc"],
+    linkopts = common_linkopts,
+    linkstatic = 1,
+    deps = [
+        ":object_detection_average_precision_stage",
+        "//tensorflow/lite/c:c_api_types",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
+cc_library(
+    name = "object_detection_stage",
+    srcs = ["object_detection_stage.cc"],
+    hdrs = ["object_detection_stage.h"],
+    copts = tflite_copts(),
+    deps = [
+        ":image_preprocessing_stage",
+        ":object_detection_average_precision_stage",
+        ":tflite_inference_stage",
+        "//tensorflow/core:tflite_portable_logging",
+        "//tensorflow/lite/c:c_api_types",
+        "//tensorflow/lite/core/c:common",
+        "//tensorflow/lite/tools/evaluation:evaluation_delegate_provider",
+        "//tensorflow/lite/tools/evaluation:evaluation_stage",
+        "//tensorflow/lite/tools/evaluation:utils",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/log",
+    ],
+)

tensorflow/lite/tools/evaluation/stages/image_classification_stage.cc

@@ -0,0 +1,213 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/evaluation/stages/image_classification_stage.h"
+
+#include <algorithm>
+#include <cstddef>
+#include <iterator>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/log/log.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/lite/c/c_api_types.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_delegate_provider.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+#include "tensorflow/lite/tools/evaluation/stages/image_preprocessing_stage.h"
+#include "tensorflow/lite/tools/evaluation/stages/tflite_inference_stage.h"
+#include "tensorflow/lite/tools/evaluation/stages/topk_accuracy_eval_stage.h"
+#include "tensorflow/lite/tools/evaluation/utils.h"
+
+namespace tflite {
+namespace evaluation {
+namespace {
+// Default cropping fraction value.
+const float kCroppingFraction = 0.875;
+}  // namespace
+
+TfLiteStatus ImageClassificationStage::Init(
+    const DelegateProviders* delegate_providers) {
+  // Ensure inference params are provided.
+  if (!config_.specification().has_image_classification_params()) {
+    LOG(ERROR) << "ImageClassificationParams not provided";
+    return kTfLiteError;
+  }
+  auto& params = config_.specification().image_classification_params();
+  if (!params.has_inference_params()) {
+    LOG(ERROR) << "Inference_params not provided";
+    return kTfLiteError;
+  }
+
+  // TfliteInferenceStage.
+  EvaluationStageConfig tflite_inference_config;
+  tflite_inference_config.set_name("tflite_inference");
+  *tflite_inference_config.mutable_specification()
+       ->mutable_tflite_inference_params() = params.inference_params();
+  inference_stage_ =
+      std::make_unique<TfliteInferenceStage>(tflite_inference_config);
+  if (inference_stage_->Init(delegate_providers) != kTfLiteOk)
+    return kTfLiteError;
+
+  // Validate model inputs.
+  const TfLiteModelInfo* model_info = inference_stage_->GetModelInfo();
+  if (model_info->inputs.size() != 1 || model_info->outputs.size() != 1) {
+    LOG(ERROR) << "Model must have 1 input & 1 output";
+    return kTfLiteError;
+  }
+  TfLiteType input_type = model_info->inputs[0]->type;
+  auto* input_shape = model_info->inputs[0]->dims;
+  // Input should be of the shape {1, height, width, 3}
+  if (input_shape->size != 4 || input_shape->data[0] != 1 ||
+      input_shape->data[3] != 3) {
+    LOG(ERROR) << "Invalid input shape for model";
+    return kTfLiteError;
+  }
+
+  // ImagePreprocessingStage
+  if (!config_.specification().has_image_preprocessing_params()) {
+    tflite::evaluation::ImagePreprocessingConfigBuilder builder(
+        "image_preprocessing", input_type);
+    builder.AddCroppingStep(kCroppingFraction, true /*square*/);
+    builder.AddResizingStep(input_shape->data[2], input_shape->data[1], false);
+    builder.AddDefaultNormalizationStep();
+    preprocessing_stage_ =
+        std::make_unique<ImagePreprocessingStage>(builder.build());
+  } else {
+    preprocessing_stage_ = std::make_unique<ImagePreprocessingStage>(config_);
+  }
+  if (preprocessing_stage_->Init() != kTfLiteOk) return kTfLiteError;
+
+  // TopkAccuracyEvalStage.
+  if (params.has_topk_accuracy_eval_params()) {
+    EvaluationStageConfig topk_accuracy_eval_config;
+    topk_accuracy_eval_config.set_name("topk_accuracy");
+    *topk_accuracy_eval_config.mutable_specification()
+         ->mutable_topk_accuracy_eval_params() =
+        params.topk_accuracy_eval_params();
+    if (!all_labels_) {
+      LOG(ERROR) << "all_labels not set for TopkAccuracyEvalStage";
+      return kTfLiteError;
+    }
+    accuracy_eval_stage_ =
+        std::make_unique<TopkAccuracyEvalStage>(topk_accuracy_eval_config);
+    accuracy_eval_stage_->SetTaskInfo(*all_labels_, input_type,
+                                      model_info->outputs[0]->dims);
+    if (accuracy_eval_stage_->Init() != kTfLiteOk) return kTfLiteError;
+  }
+
+  return kTfLiteOk;
+}
+
+TfLiteStatus ImageClassificationStage::Run() {
+  if (image_path_.empty()) {
+    LOG(ERROR) << "Input image not set";
+    return kTfLiteError;
+  }
+
+  // Preprocessing.
+  preprocessing_stage_->SetImagePath(&image_path_);
+  if (preprocessing_stage_->Run() != kTfLiteOk) return kTfLiteError;
+  if (preprocessing_stage_->GetPreprocessedImageBytes() <
+      inference_stage_->GetModelInfo()->inputs[0]->bytes) {
+    LOG(ERROR)
+        << "Preprocessed image buffer is smaller than model input tensor size";
+    return kTfLiteError;
+  }
+  // Inference.
+  std::vector<void*> data_ptrs = {};
+  data_ptrs.push_back(preprocessing_stage_->GetPreprocessedImageData());
+  std::vector<size_t> data_sizes = {
+      preprocessing_stage_->GetPreprocessedImageBytes()};
+  inference_stage_->SetInputs(data_ptrs, data_sizes);
+  if (inference_stage_->Run() != kTfLiteOk) return kTfLiteError;
+  // Accuracy Eval.
+  if (accuracy_eval_stage_) {
+    if (ground_truth_label_.empty()) {
+      LOG(ERROR) << "Ground truth label not provided";
+      return kTfLiteError;
+    }
+    accuracy_eval_stage_->SetEvalInputs(inference_stage_->GetOutputs()->at(0),
+                                        &ground_truth_label_);
+    if (accuracy_eval_stage_->Run() != kTfLiteOk) return kTfLiteError;
+  }
+
+  return kTfLiteOk;
+}
+
+EvaluationStageMetrics ImageClassificationStage::LatestMetrics() {
+  EvaluationStageMetrics metrics;
+  auto* classification_metrics =
+      metrics.mutable_process_metrics()->mutable_image_classification_metrics();
+
+  *classification_metrics->mutable_pre_processing_latency() =
+      preprocessing_stage_->LatestMetrics().process_metrics().total_latency();
+  EvaluationStageMetrics inference_metrics = inference_stage_->LatestMetrics();
+  *classification_metrics->mutable_inference_latency() =
+      inference_metrics.process_metrics().total_latency();
+  *classification_metrics->mutable_inference_metrics() =
+      inference_metrics.process_metrics().tflite_inference_metrics();
+  if (accuracy_eval_stage_) {
+    *classification_metrics->mutable_topk_accuracy_metrics() =
+        accuracy_eval_stage_->LatestMetrics()
+            .process_metrics()
+            .topk_accuracy_metrics();
+  }
+  metrics.set_num_runs(inference_metrics.num_runs());
+  return metrics;
+}
+
+TfLiteStatus FilterDenyListedImages(const std::string& denylist_file_path,
+                                    std::vector<ImageLabel>* image_labels) {
+  if (!denylist_file_path.empty()) {
+    std::vector<std::string> lines;
+    if (!tflite::evaluation::ReadFileLines(denylist_file_path, &lines)) {
+      LOG(ERROR) << "Could not read: " << denylist_file_path;
+      return kTfLiteError;
+    }
+    std::vector<int> denylist_ids;
+    denylist_ids.reserve(lines.size());
+    // Populate denylist_ids with indices of images.
+    std::transform(lines.begin(), lines.end(), std::back_inserter(denylist_ids),
+                   [](const std::string& val) { return std::stoi(val) - 1; });
+
+    std::vector<ImageLabel> filtered_images;
+    std::sort(denylist_ids.begin(), denylist_ids.end());
+    const size_t size_post_filtering =
+        image_labels->size() - denylist_ids.size();
+    filtered_images.reserve(size_post_filtering);
+    int denylist_index = 0;
+    for (int image_index = 0; image_index < image_labels->size();
+         image_index++) {
+      if (denylist_index < denylist_ids.size() &&
+          denylist_ids[denylist_index] == image_index) {
+        denylist_index++;
+        continue;
+      }
+      filtered_images.push_back((*image_labels)[image_index]);
+    }
+
+    if (filtered_images.size() != size_post_filtering) {
+      LOG(ERROR) << "Invalid number of filtered images";
+      return kTfLiteError;
+    }
+    *image_labels = filtered_images;
+  }
+  return kTfLiteOk;
+}
+
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/stages/image_classification_stage.h

@@ -0,0 +1,92 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_IMAGE_CLASSIFICATION_STAGE_H_
+#define TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_IMAGE_CLASSIFICATION_STAGE_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "tensorflow/lite/c/c_api_types.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_delegate_provider.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_stage.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/stages/image_preprocessing_stage.h"
+#include "tensorflow/lite/tools/evaluation/stages/tflite_inference_stage.h"
+#include "tensorflow/lite/tools/evaluation/stages/topk_accuracy_eval_stage.h"
+
+namespace tflite {
+namespace evaluation {
+
+// An EvaluationStage to encapsulate the complete Image Classification task.
+// Utilizes ImagePreprocessingStage, TfLiteInferenceStage &
+// TopkAccuracyEvalStage for individual sub-tasks.
+class ImageClassificationStage : public EvaluationStage {
+ public:
+  explicit ImageClassificationStage(const EvaluationStageConfig& config)
+      : EvaluationStage(config) {}
+
+  TfLiteStatus Init() override { return Init(nullptr); }
+  TfLiteStatus Init(const DelegateProviders* delegate_providers);
+
+  TfLiteStatus Run() override;
+
+  EvaluationStageMetrics LatestMetrics() override;
+
+  // Call before Init(), if topk_accuracy_eval_params is set in
+  // ImageClassificationParams. all_labels should contain the labels
+  // corresponding to model's output, in the same order. all_labels should
+  // outlive the call to Init().
+  void SetAllLabels(const std::vector<std::string>& all_labels) {
+    all_labels_ = &all_labels;
+  }
+
+  // Call before Run().
+  // If accuracy eval is not being performed, ground_truth_label is ignored.
+  void SetInputs(const std::string& image_path,
+                 const std::string& ground_truth_label) {
+    image_path_ = image_path;
+    ground_truth_label_ = ground_truth_label;
+  }
+
+  // Provides a pointer to the underlying TfLiteInferenceStage.
+  // Returns non-null value only if this stage has been initialized.
+  TfliteInferenceStage* const GetInferenceStage() {
+    return inference_stage_.get();
+  }
+
+ private:
+  const std::vector<std::string>* all_labels_ = nullptr;
+  std::unique_ptr<ImagePreprocessingStage> preprocessing_stage_;
+  std::unique_ptr<TfliteInferenceStage> inference_stage_;
+  std::unique_ptr<TopkAccuracyEvalStage> accuracy_eval_stage_;
+  std::string image_path_;
+  std::string ground_truth_label_;
+};
+
+struct ImageLabel {
+  std::string image;
+  std::string label;
+};
+
+// Reads a file containing newline-separated denylisted image indices and
+// filters them out from image_labels.
+TfLiteStatus FilterDenyListedImages(const std::string& denylist_file_path,
+                                    std::vector<ImageLabel>* image_labels);
+
+}  // namespace evaluation
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_IMAGE_CLASSIFICATION_STAGE_H_

tensorflow/lite/tools/evaluation/stages/image_preprocessing_stage.cc

@@ -0,0 +1,563 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/evaluation/stages/image_preprocessing_stage.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstddef>
+#include <cstdint>
+#include <fstream>
+#include <ios>
+#include <iterator>
+#include <limits>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "absl/base/casts.h"
+#include "absl/log/absl_log.h"
+#include "absl/log/log.h"
+#include "absl/strings/ascii.h"
+#include "absl/strings/string_view.h"
+#include "jpeglib.h"  // from @libjpeg_turbo
+#include "tensorflow/core/lib/jpeg/jpeg_mem.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/lite/c/c_api_types.h"
+#include "tensorflow/lite/c/common.h"
+#include "tensorflow/lite/kernels/internal/reference/pad.h"
+#include "tensorflow/lite/kernels/internal/reference/resize_bilinear.h"
+#include "tensorflow/lite/kernels/internal/runtime_shape.h"
+#include "tensorflow/lite/kernels/internal/types.h"
+#include "tensorflow/lite/profiling/time.h"
+#include "tensorflow/lite/string_type.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/preprocessing_steps.pb.h"
+
+namespace tflite {
+namespace evaluation {
+namespace {
+
+// We assume 3-channel RGB images.
+constexpr int kNumChannels = 3;
+
+// Returns the offset for the element in the raw image array based on the image
+// height/weight & coordinates of a pixel (h, w, c).
+inline int ImageArrayOffset(int height, int width, int h, int w, int c) {
+  return (h * width + w) * kNumChannels + c;
+}
+
+// Stores data and size information of an image.
+struct ImageData {
+  uint32_t width;
+  uint32_t height;
+  std::vector<float> data;
+
+  // GetData performs no checks.
+  float GetData(int h, int w, int c) const {
+    return data[ImageArrayOffset(height, width, h, w, c)];
+  }
+};
+
+// Loads the raw image.
+inline TfLiteStatus LoadImageRaw(absl::string_view filename,
+                                 ImageData* image_data) {
+  std::ifstream stream(std::string(filename).c_str(),
+                       std::ios::in | std::ios::binary);
+  if (!stream.is_open()) {
+    ABSL_LOG(ERROR) << "Failed to open file: " << filename;
+    return kTfLiteError;
+  }
+  std::vector<uint8_t> raw_data((std::istreambuf_iterator<char>(stream)),
+                                std::istreambuf_iterator<char>());
+  if (raw_data.size() % kNumChannels != 0) {
+    ABSL_LOG(ERROR) << "Raw image size is not a multiple of " << kNumChannels;
+    return kTfLiteError;
+  }
+  image_data->data.clear();
+  image_data->data.reserve(raw_data.size());
+  for (uint8_t val : raw_data) {
+    image_data->data.push_back(static_cast<float>(val));
+  }
+  return kTfLiteOk;
+}
+
+// Loads the jpeg image.
+inline TfLiteStatus LoadImageJpeg(absl::string_view filename,
+                                  ImageData* image_data) {
+  // Reads image.
+  std::ifstream t(std::string(filename).c_str());
+  if (!t.is_open()) {
+    ABSL_LOG(ERROR) << "Failed to open file: " << filename;
+    return kTfLiteError;
+  }
+  std::string image_str((std::istreambuf_iterator<char>(t)),
+                        std::istreambuf_iterator<char>());
+  const int fsize = image_str.size();
+  auto temp = absl::bit_cast<const uint8_t*>(image_str.data());
+  std::unique_ptr<uint8_t[]> original_image;
+  int original_width, original_height, original_channels;
+  tensorflow::jpeg::UncompressFlags flags;
+  // JDCT_ISLOW performs slower but more accurate pre-processing.
+  // This isn't always obvious in unit tests, but makes a difference during
+  // accuracy testing with ILSVRC dataset.
+  flags.dct_method = JDCT_ISLOW;
+  // We necessarily require a 3-channel image as the output.
+  flags.components = kNumChannels;
+  original_image.reset(Uncompress(temp, fsize, flags, &original_width,
+                                  &original_height, &original_channels,
+                                  nullptr));
+  if (!original_image) {
+    ABSL_LOG(ERROR) << "Failed to decompress JPEG image: " << filename;
+    return kTfLiteError;
+  }
+  // Copies the image data.
+  image_data->width = original_width;
+  image_data->height = original_height;
+  int original_size = original_height * original_width * original_channels;
+  image_data->data.clear();
+  image_data->data.reserve(original_size);
+  for (int i = 0; i < original_size; ++i) {
+    image_data->data.push_back(static_cast<float>(original_image[i]));
+  }
+  return kTfLiteOk;
+}
+
+// Central-cropping.
+inline TfLiteStatus Crop(ImageData* image_data,
+                         const CroppingParams& crop_params) {
+  int crop_height = 0;
+  int crop_width = 0;
+  int input_width = image_data->width;
+  int input_height = image_data->height;
+  if (crop_params.has_cropping_fraction()) {
+    crop_height =
+        static_cast<int>(round(crop_params.cropping_fraction() * input_height));
+    crop_width =
+        static_cast<int>(round(crop_params.cropping_fraction() * input_width));
+  } else if (crop_params.has_target_size()) {
+    crop_height = crop_params.target_size().height();
+    crop_width = crop_params.target_size().width();
+  }
+  if (crop_params.has_cropping_fraction() && crop_params.square_cropping()) {
+    crop_height = std::min(crop_height, crop_width);
+    crop_width = crop_height;
+  }
+  // The crop region must fit within the image. A larger crop than the image
+  // (possible with target_size, since the image dimensions come from the
+  // decoded input) would make the start offsets negative and cause
+  // GetData to read out of bounds.
+  if (crop_width <= 0 || crop_height <= 0 || crop_width > input_width ||
+      crop_height > input_height) {
+    ABSL_LOG(ERROR) << "Cropping size is invalid or larger than the image size";
+    return kTfLiteError;
+  }
+  int start_w = static_cast<int>(round((input_width - crop_width) / 2.0));
+  int start_h = static_cast<int>(round((input_height - crop_height) / 2.0));
+  std::vector<float> cropped_image;
+  cropped_image.reserve(crop_height * crop_width * kNumChannels);
+  for (int in_h = start_h; in_h < start_h + crop_height; ++in_h) {
+    for (int in_w = start_w; in_w < start_w + crop_width; ++in_w) {
+      for (int c = 0; c < kNumChannels; ++c) {
+        cropped_image.push_back(image_data->GetData(in_h, in_w, c));
+      }
+    }
+  }
+  image_data->height = crop_height;
+  image_data->width = crop_width;
+  image_data->data = std::move(cropped_image);
+  return kTfLiteOk;
+}
+
+// Performs billinear interpolation for 3-channel RGB image.
+// See: https://en.wikipedia.org/wiki/Bilinear_interpolation
+inline TfLiteStatus ResizeBilinear(ImageData* image_data,
+                                   const ResizingParams& params) {
+  tflite::ResizeBilinearParams resize_params;
+  resize_params.align_corners = false;
+  // TODO(b/143292772): Set this to true for more accurate behavior?
+  resize_params.half_pixel_centers = false;
+  tflite::RuntimeShape input_shape({1, static_cast<int>(image_data->height),
+                                    static_cast<int>(image_data->width),
+                                    kNumChannels});
+  // Calculates output size.
+  int output_height = 0;
+  int output_width = 0;
+  if (params.aspect_preserving()) {
+    float ratio_w =
+        params.target_size().width() / static_cast<float>(image_data->width);
+    float ratio_h =
+        params.target_size().height() / static_cast<float>(image_data->height);
+    if (ratio_w >= ratio_h) {
+      output_width = params.target_size().width();
+      output_height = static_cast<int>(round(image_data->height * ratio_w));
+    } else {
+      output_width = static_cast<int>(round(image_data->width * ratio_h));
+      output_height = params.target_size().height();
+    }
+  } else {
+    output_height = params.target_size().height();
+    output_width = params.target_size().width();
+  }
+  tflite::RuntimeShape output_size_dims({1, 1, 1, 2});
+  std::vector<int32_t> output_size_data = {output_height, output_width};
+  tflite::RuntimeShape output_shape(
+      {1, output_height, output_width, kNumChannels});
+  // Guards against integer overflow when computing the output buffer size for
+  // a very large target_size.
+  if (output_width <= 0 || output_height <= 0 ||
+      output_height > std::numeric_limits<int>::max() / output_width ||
+      output_width * output_height >
+          std::numeric_limits<int>::max() / kNumChannels) {
+    ABSL_LOG(ERROR) << "Resizing output size is invalid or too large";
+    return kTfLiteError;
+  }
+  int output_size = output_width * output_height * kNumChannels;
+  std::vector<float> output_data(output_size, 0);
+  tflite::reference_ops::ResizeBilinear(
+      resize_params, input_shape, image_data->data.data(), output_size_dims,
+      output_size_data.data(), output_shape, output_data.data());
+  image_data->height = output_height;
+  image_data->width = output_width;
+  image_data->data = std::move(output_data);
+  return kTfLiteOk;
+}
+
+// Pads the image to a pre-defined size.
+inline TfLiteStatus Pad(ImageData* image_data, const PaddingParams& params) {
+  int output_width = params.target_size().width();
+  int output_height = params.target_size().height();
+  // The padded output must be at least as large as the image. A smaller
+  // target than the image would make the padding counts negative and cause
+  // the pad kernel to read/write out of bounds.
+  if (output_width < static_cast<int>(image_data->width) ||
+      output_height < static_cast<int>(image_data->height)) {
+    ABSL_LOG(ERROR) << "Padding size is smaller than the image size";
+    return kTfLiteError;
+  }
+  int pad_value = params.padding_value();
+  tflite::PadParams pad_params{};
+  pad_params.left_padding_count = 4;
+  pad_params.left_padding[1] =
+      static_cast<int>(round((output_height - image_data->height) / 2.0));
+  pad_params.left_padding[2] =
+      static_cast<int>(round((output_width - image_data->width) / 2.0));
+  pad_params.right_padding_count = 4;
+  pad_params.right_padding[1] =
+      output_height - pad_params.left_padding[1] - image_data->height;
+  pad_params.right_padding[2] =
+      output_width - pad_params.left_padding[2] - image_data->width;
+  tflite::RuntimeShape input_shape({1, static_cast<int>(image_data->height),
+                                    static_cast<int>(image_data->width),
+                                    kNumChannels});
+  tflite::RuntimeShape output_shape(
+      {1, output_height, output_width, kNumChannels});
+  // Guards against integer overflow when computing the output buffer size for
+  // a very large target_size. The explicit > 0 checks keep the divisions safe
+  // and self-contained, matching ResizeBilinear.
+  if (output_width <= 0 || output_height <= 0 ||
+      output_height > std::numeric_limits<int>::max() / output_width ||
+      output_width * output_height >
+          std::numeric_limits<int>::max() / kNumChannels) {
+    ABSL_LOG(ERROR) << "Padding output size is invalid or too large";
+    return kTfLiteError;
+  }
+  int output_size = output_width * output_height * kNumChannels;
+  std::vector<float> output_data(output_size, 0);
+  tflite::reference_ops::Pad(pad_params, input_shape, image_data->data.data(),
+                             &pad_value, output_shape, output_data.data());
+  image_data->height = output_height;
+  image_data->width = output_width;
+  image_data->data = std::move(output_data);
+  return kTfLiteOk;
+}
+
+// Normalizes the image data to a specific range with mean and scale.
+inline TfLiteStatus Normalize(ImageData* image_data,
+                              const NormalizationParams& params) {
+  if (image_data->data.size() % kNumChannels != 0) {
+    ABSL_LOG(ERROR) << "Image data size is not a multiple of " << kNumChannels;
+    return kTfLiteError;
+  }
+  float scale = params.scale();
+  float* data_end = image_data->data.data() + image_data->data.size();
+  if (params.has_channelwise_mean()) {
+    float mean = params.channelwise_mean();
+    for (float* data = image_data->data.data(); data < data_end; ++data) {
+      *data = (*data - mean) * scale;
+    }
+  } else {
+    float r_mean = params.means().r_mean();
+    float g_mean = params.means().g_mean();
+    float b_mean = params.means().b_mean();
+    for (float* data = image_data->data.data(); data < data_end;) {
+      *data = (*data - r_mean) * scale;
+      ++data;
+      *data = (*data - g_mean) * scale;
+      ++data;
+      *data = (*data - b_mean) * scale;
+      ++data;
+    }
+  }
+  return kTfLiteOk;
+}
+}  // namespace
+
+TfLiteStatus ImagePreprocessingStage::Init() {
+  if (!config_.has_specification() ||
+      !config_.specification().has_image_preprocessing_params()) {
+    LOG(ERROR) << "No preprocessing params";
+    return kTfLiteError;
+  }
+  const ImagePreprocessingParams& params =
+      config_.specification().image_preprocessing_params();
+  for (const ImagePreprocessingStepParams& param : params.steps()) {
+    if (param.has_cropping_params()) {
+      const CroppingParams& crop_params = param.cropping_params();
+      if (!crop_params.has_cropping_fraction() &&
+          !crop_params.has_target_size()) {
+        ABSL_LOG(ERROR)
+            << "Cropping params must have either cropping_fraction or "
+               "target_size";
+        return kTfLiteError;
+      }
+      if (crop_params.has_cropping_fraction() &&
+          (crop_params.cropping_fraction() <= 0 ||
+           crop_params.cropping_fraction() > 1.0)) {
+        LOG(ERROR) << "Invalid cropping fraction";
+        return kTfLiteError;
+      }
+      if (crop_params.has_target_size() &&
+          (crop_params.target_size().width() <= 0 ||
+           crop_params.target_size().height() <= 0)) {
+        ABSL_LOG(ERROR) << "Invalid cropping target size";
+        return kTfLiteError;
+      }
+    } else if (param.has_resizing_params()) {
+      const ResizingParams& resizing_params = param.resizing_params();
+      if (!resizing_params.has_target_size() ||
+          resizing_params.target_size().width() <= 0 ||
+          resizing_params.target_size().height() <= 0) {
+        ABSL_LOG(ERROR) << "Invalid resizing target size";
+        return kTfLiteError;
+      }
+    } else if (param.has_padding_params()) {
+      const PaddingParams& padding_params = param.padding_params();
+      if (!padding_params.has_target_size() ||
+          padding_params.target_size().width() <= 0 ||
+          padding_params.target_size().height() <= 0) {
+        ABSL_LOG(ERROR) << "Invalid padding target size";
+        return kTfLiteError;
+      }
+    }
+  }
+  output_type_ = static_cast<TfLiteType>(params.output_type());
+  return kTfLiteOk;
+}
+
+TfLiteStatus ImagePreprocessingStage::Run() {
+  if (!image_path_) {
+    LOG(ERROR) << "Image path not set";
+    return kTfLiteError;
+  }
+
+  ImageData image_data;
+  const ImagePreprocessingParams& params =
+      config_.specification().image_preprocessing_params();
+  int64_t start_us = profiling::time::NowMicros();
+  // Loads the image from file.
+  std::string image_ext;
+  size_t dot_pos = image_path_->find_last_of('.');
+  if (dot_pos != std::string::npos) {
+    image_ext = image_path_->substr(dot_pos);
+  }
+  absl::AsciiStrToLower(&image_ext);
+  bool is_raw_image = (image_ext == ".rgb8");
+  if (is_raw_image) {
+    if (LoadImageRaw(*image_path_, &image_data) != kTfLiteOk) {
+      return kTfLiteError;
+    }
+  } else if (image_ext == ".jpg" || image_ext == ".jpeg") {
+    if (LoadImageJpeg(*image_path_, &image_data) != kTfLiteOk) {
+      return kTfLiteError;
+    }
+  } else {
+    LOG(ERROR) << "Extension " << image_ext << " is not supported";
+    return kTfLiteError;
+  }
+
+  // Find the index of the last sizing step with a target size.
+  int last_sizing_step_index = -1;
+  for (int i = params.steps_size() - 1; i >= 0; --i) {
+    const auto& param = params.steps(i);
+    bool is_sizing_step_with_target = false;
+    if (param.has_cropping_params() &&
+        param.cropping_params().has_target_size()) {
+      is_sizing_step_with_target = true;
+    } else if (param.has_resizing_params() &&
+               param.resizing_params().has_target_size()) {
+      is_sizing_step_with_target = true;
+    } else if (param.has_padding_params() &&
+               param.padding_params().has_target_size()) {
+      is_sizing_step_with_target = true;
+    }
+    if (is_sizing_step_with_target) {
+      last_sizing_step_index = i;
+      break;
+    }
+  }
+
+  // Cropping, padding and resizing are not supported with raw images since raw
+  // images do not contain image size information. Those steps are assumed to
+  // be done before raw images are generated.
+  for (int i = 0; i < params.steps_size(); ++i) {
+    const ImagePreprocessingStepParams& param = params.steps(i);
+    if (param.has_cropping_params()) {
+      if (is_raw_image) {
+        LOG(WARNING) << "Image cropping will not be performed on raw images";
+        if (param.cropping_params().has_target_size() &&
+            i == last_sizing_step_index) {
+          // Only validate against the target size if this is the last sizing
+          // step in the preprocessing chain.
+          if (image_data.data.size() !=
+              param.cropping_params().target_size().width() *
+                  param.cropping_params().target_size().height() *
+                  kNumChannels) {
+            LOG(ERROR)
+                << "Raw image size does not match the final expected size "
+                   "from cropping params.";
+            return kTfLiteError;
+          }
+        }
+        continue;
+      }
+      TF_LITE_ENSURE_STATUS(Crop(&image_data, param.cropping_params()));
+    } else if (param.has_resizing_params()) {
+      if (is_raw_image) {
+        LOG(WARNING) << "Image resizing will not be performed on raw images";
+        if (param.resizing_params().has_target_size() &&
+            i == last_sizing_step_index) {
+          // Only validate against the target size if this is the last sizing
+          // step in the preprocessing chain.
+          if (image_data.data.size() !=
+              param.resizing_params().target_size().width() *
+                  param.resizing_params().target_size().height() *
+                  kNumChannels) {
+            LOG(ERROR)
+                << "Raw image size does not match the final expected size "
+                   "from resizing params.";
+            return kTfLiteError;
+          }
+        }
+        continue;
+      }
+      TF_LITE_ENSURE_STATUS(
+          ResizeBilinear(&image_data, param.resizing_params()));
+    } else if (param.has_padding_params()) {
+      if (is_raw_image) {
+        LOG(WARNING) << "Image padding will not be performed on raw images";
+        if (param.padding_params().has_target_size() &&
+            i == last_sizing_step_index) {
+          // Only validate against the target size if this is the last sizing
+          // step in the preprocessing chain.
+          if (image_data.data.size() !=
+              param.padding_params().target_size().width() *
+                  param.padding_params().target_size().height() *
+                  kNumChannels) {
+            LOG(ERROR)
+                << "Raw image size does not match the final expected size "
+                   "from padding params.";
+            return kTfLiteError;
+          }
+        }
+        continue;
+      }
+      TF_LITE_ENSURE_STATUS(Pad(&image_data, param.padding_params()));
+    } else if (param.has_normalization_params()) {
+      TF_LITE_ENSURE_STATUS(
+          Normalize(&image_data, param.normalization_params()));
+    }
+  }
+  // Converts data to output type.
+  if (output_type_ == kTfLiteUInt8) {
+    uint8_preprocessed_image_.clear();
+    uint8_preprocessed_image_.resize(image_data.data.size() +
+                                     /*XNN_EXTRA_BYTES=*/16);
+    for (size_t i = 0; i < image_data.data.size(); ++i) {
+      uint8_preprocessed_image_[i] = static_cast<uint8_t>(image_data.data[i]);
+    }
+  } else if (output_type_ == kTfLiteInt8) {
+    int8_preprocessed_image_.clear();
+    int8_preprocessed_image_.resize(image_data.data.size() +
+                                    /*XNN_EXTRA_BYTES=*/16);
+    for (size_t i = 0; i < image_data.data.size(); ++i) {
+      int8_preprocessed_image_[i] = static_cast<int8_t>(image_data.data[i]);
+    }
+  } else if (output_type_ == kTfLiteFloat32) {
+    float_preprocessed_image_ = std::move(image_data.data);
+  }
+
+  latency_stats_.UpdateStat(profiling::time::NowMicros() - start_us);
+  return kTfLiteOk;
+}
+
+void* ImagePreprocessingStage::GetPreprocessedImageData() {
+  if (latency_stats_.count() == 0) return nullptr;
+
+  if (output_type_ == kTfLiteUInt8) {
+    return uint8_preprocessed_image_.data();
+  } else if (output_type_ == kTfLiteInt8) {
+    return int8_preprocessed_image_.data();
+  } else if (output_type_ == kTfLiteFloat32) {
+    return float_preprocessed_image_.data();
+  }
+  return nullptr;
+}
+
+size_t ImagePreprocessingStage::GetPreprocessedImageBytes() {
+  if (latency_stats_.count() == 0) return 0;
+
+  if (output_type_ == kTfLiteUInt8) {
+    return (uint8_preprocessed_image_.size() >= 16
+                ? uint8_preprocessed_image_.size() - 16
+                : 0) *
+           sizeof(uint8_t);
+  } else if (output_type_ == kTfLiteInt8) {
+    return (int8_preprocessed_image_.size() >= 16
+                ? int8_preprocessed_image_.size() - 16
+                : 0) *
+           sizeof(int8_t);
+  } else if (output_type_ == kTfLiteFloat32) {
+    return float_preprocessed_image_.size() * sizeof(float);
+  }
+  return 0;
+}
+
+EvaluationStageMetrics ImagePreprocessingStage::LatestMetrics() {
+  EvaluationStageMetrics metrics;
+  auto* latency_metrics =
+      metrics.mutable_process_metrics()->mutable_total_latency();
+  latency_metrics->set_last_us(latency_stats_.newest());
+  latency_metrics->set_max_us(latency_stats_.max());
+  latency_metrics->set_min_us(latency_stats_.min());
+  latency_metrics->set_sum_us(latency_stats_.sum());
+  latency_metrics->set_avg_us(latency_stats_.avg());
+  metrics.set_num_runs(static_cast<int>(latency_stats_.count()));
+  return metrics;
+}
+
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/stages/image_preprocessing_stage.h

@@ -0,0 +1,193 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_IMAGE_PREPROCESSING_STAGE_H_
+#define TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_IMAGE_PREPROCESSING_STAGE_H_
+
+#include <stdint.h>
+
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "absl/log/log.h"
+#include "xla/tsl/util/stats_calculator.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/lite/c/c_api_types.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_stage.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/preprocessing_steps.pb.h"
+
+namespace tflite {
+namespace evaluation {
+
+// EvaluationStage to read contents of an image and preprocess it for inference.
+// Currently only supports JPEGs.
+class ImagePreprocessingStage : public EvaluationStage {
+ public:
+  explicit ImagePreprocessingStage(const EvaluationStageConfig& config)
+      : EvaluationStage(config) {}
+
+  TfLiteStatus Init() override;
+
+  TfLiteStatus Run() override;
+
+  EvaluationStageMetrics LatestMetrics() override;
+
+  ~ImagePreprocessingStage() override {}
+
+  // Call before Run().
+  void SetImagePath(std::string* image_path) { image_path_ = image_path; }
+
+  // Provides preprocessing output.
+  void* GetPreprocessedImageData();
+  size_t GetPreprocessedImageBytes();
+
+ private:
+  std::string* image_path_ = nullptr;
+  TfLiteType output_type_;
+  tsl::Stat<int64_t> latency_stats_;
+
+  // One of the following 3 vectors will be populated based on output_type_.
+  std::vector<float> float_preprocessed_image_;
+  std::vector<int8_t> int8_preprocessed_image_;
+  std::vector<uint8_t> uint8_preprocessed_image_;
+};
+
+// Helper class to build a new ImagePreprocessingParams.
+class ImagePreprocessingConfigBuilder {
+ public:
+  ImagePreprocessingConfigBuilder(const std::string& name,
+                                  TfLiteType output_type) {
+    config_.set_name(name);
+    config_.mutable_specification()
+        ->mutable_image_preprocessing_params()
+        ->set_output_type(static_cast<int>(output_type));
+  }
+
+  // Adds a cropping step with cropping fraction.
+  void AddCroppingStep(float cropping_fraction,
+                       bool use_square_cropping = false) {
+    ImagePreprocessingStepParams params;
+    params.mutable_cropping_params()->set_cropping_fraction(cropping_fraction);
+    params.mutable_cropping_params()->set_square_cropping(use_square_cropping);
+    config_.mutable_specification()
+        ->mutable_image_preprocessing_params()
+        ->mutable_steps()
+        ->Add(std::move(params));
+  }
+
+  // Adds a cropping step with target size.
+  void AddCroppingStep(uint32_t width, uint32_t height,
+                       bool use_square_cropping = false) {
+    ImagePreprocessingStepParams params;
+    params.mutable_cropping_params()->mutable_target_size()->set_height(height);
+    params.mutable_cropping_params()->mutable_target_size()->set_width(width);
+    params.mutable_cropping_params()->set_square_cropping(use_square_cropping);
+    config_.mutable_specification()
+        ->mutable_image_preprocessing_params()
+        ->mutable_steps()
+        ->Add(std::move(params));
+  }
+
+  // Adds a resizing step.
+  void AddResizingStep(uint32_t width, uint32_t height,
+                       bool aspect_preserving) {
+    ImagePreprocessingStepParams params;
+    params.mutable_resizing_params()->set_aspect_preserving(aspect_preserving);
+    params.mutable_resizing_params()->mutable_target_size()->set_height(height);
+    params.mutable_resizing_params()->mutable_target_size()->set_width(width);
+    config_.mutable_specification()
+        ->mutable_image_preprocessing_params()
+        ->mutable_steps()
+        ->Add(std::move(params));
+  }
+
+  // Adds a padding step.
+  void AddPaddingStep(uint32_t width, uint32_t height, int value) {
+    ImagePreprocessingStepParams params;
+    params.mutable_padding_params()->mutable_target_size()->set_height(height);
+    params.mutable_padding_params()->mutable_target_size()->set_width(width);
+    params.mutable_padding_params()->set_padding_value(value);
+    config_.mutable_specification()
+        ->mutable_image_preprocessing_params()
+        ->mutable_steps()
+        ->Add(std::move(params));
+  }
+
+  // Adds a square padding step.
+  void AddSquarePaddingStep(int value) {
+    ImagePreprocessingStepParams params;
+    params.mutable_padding_params()->set_square_padding(true);
+    params.mutable_padding_params()->set_padding_value(value);
+    config_.mutable_specification()
+        ->mutable_image_preprocessing_params()
+        ->mutable_steps()
+        ->Add(std::move(params));
+  }
+
+  // Adds a subtracting means step.
+  void AddPerChannelNormalizationStep(float r_mean, float g_mean, float b_mean,
+                                      float scale) {
+    ImagePreprocessingStepParams params;
+    params.mutable_normalization_params()->mutable_means()->set_r_mean(r_mean);
+    params.mutable_normalization_params()->mutable_means()->set_g_mean(g_mean);
+    params.mutable_normalization_params()->mutable_means()->set_b_mean(b_mean);
+    params.mutable_normalization_params()->set_scale(scale);
+    config_.mutable_specification()
+        ->mutable_image_preprocessing_params()
+        ->mutable_steps()
+        ->Add(std::move(params));
+  }
+
+  // Adds a normalization step.
+  void AddNormalizationStep(float mean, float scale) {
+    ImagePreprocessingStepParams params;
+    params.mutable_normalization_params()->set_channelwise_mean(mean);
+    params.mutable_normalization_params()->set_scale(scale);
+    config_.mutable_specification()
+        ->mutable_image_preprocessing_params()
+        ->mutable_steps()
+        ->Add(std::move(params));
+  }
+
+  // Adds a normalization step with default value.
+  void AddDefaultNormalizationStep() {
+    switch (
+        config_.specification().image_preprocessing_params().output_type()) {
+      case kTfLiteFloat32:
+        AddNormalizationStep(127.5, 1.0 / 127.5);
+        break;
+      case kTfLiteUInt8:
+        break;
+      case kTfLiteInt8:
+        AddNormalizationStep(128.0, 1.0);
+        break;
+      default:
+        LOG(ERROR) << "Type not supported";
+        break;
+    }
+  }
+
+  EvaluationStageConfig build() { return std::move(config_); }
+
+ private:
+  EvaluationStageConfig config_;
+};
+
+}  // namespace evaluation
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_IMAGE_PREPROCESSING_STAGE_H_

tensorflow/lite/tools/evaluation/stages/image_preprocessing_stage_test.cc

@@ -0,0 +1,363 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/evaluation/stages/image_preprocessing_stage.h"
+
+#include <cstdint>
+#include <fstream>
+#include <ios>
+#include <string>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+#include "absl/strings/string_view.h"
+#include "tensorflow/lite/c/c_api_types.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+
+namespace tflite {
+namespace evaluation {
+namespace {
+
+constexpr absl::string_view kImagePreprocessingStageName =
+    "inception_preprocessing_stage";
+constexpr absl::string_view kTestImage =
+    "tensorflow/lite/tools/evaluation/stages/testdata/"
+    "grace_hopper.jpg";
+
+MATCHER(HasValidLatencyMetrics, "") {
+  if (arg.num_runs() != 1) {
+    *result_listener << "num_runs is " << arg.num_runs() << " (expected 1)";
+    return false;
+  }
+  const auto& total_latency = arg.process_metrics().total_latency();
+  int64_t last_latency = total_latency.last_us();
+  if (last_latency <= 0 || last_latency >= 1e7) {
+    *result_listener << "last_latency is " << last_latency
+                     << " (expected to be in range (0, 1e7))";
+    return false;
+  }
+  if (total_latency.max_us() != last_latency) {
+    *result_listener << "max_us (" << total_latency.max_us()
+                     << ") != last_latency (" << last_latency << ")";
+    return false;
+  }
+  if (total_latency.min_us() != last_latency) {
+    *result_listener << "min_us (" << total_latency.min_us()
+                     << ") != last_latency (" << last_latency << ")";
+    return false;
+  }
+  if (total_latency.sum_us() != last_latency) {
+    *result_listener << "sum_us (" << total_latency.sum_us()
+                     << ") != last_latency (" << last_latency << ")";
+    return false;
+  }
+  if (total_latency.avg_us() != last_latency) {
+    *result_listener << "avg_us (" << total_latency.avg_us()
+                     << ") != last_latency (" << last_latency << ")";
+    return false;
+  }
+  return true;
+}
+
+TEST(ImagePreprocessingStage, NoParams) {
+  ImagePreprocessingConfigBuilder builder(
+      std::string(kImagePreprocessingStageName), kTfLiteFloat32);
+  EvaluationStageConfig config = builder.build();
+  config.mutable_specification()->clear_image_preprocessing_params();
+  ImagePreprocessingStage stage(config);
+  EXPECT_EQ(stage.Init(), kTfLiteError);
+}
+
+TEST(ImagePreprocessingStage, InvalidCroppingFraction) {
+  ImagePreprocessingConfigBuilder builder(
+      std::string(kImagePreprocessingStageName), kTfLiteFloat32);
+  builder.AddCroppingStep(-0.8);
+  ImagePreprocessingStage stage(builder.build());
+  EXPECT_EQ(stage.Init(), kTfLiteError);
+}
+
+TEST(ImagePreprocessingStage, ImagePathNotSet) {
+  ImagePreprocessingConfigBuilder builder(
+      std::string(kImagePreprocessingStageName), kTfLiteFloat32);
+  ImagePreprocessingStage stage(builder.build());
+  EXPECT_EQ(stage.Init(), kTfLiteOk);
+
+  EXPECT_EQ(stage.Run(), kTfLiteError);
+  EXPECT_EQ(stage.GetPreprocessedImageData(), nullptr);
+}
+
+TEST(ImagePreprocessingStage, TestImagePreprocessingFloat) {
+  std::string image_path(kTestImage);
+
+  ImagePreprocessingConfigBuilder builder(
+      std::string(kImagePreprocessingStageName), kTfLiteFloat32);
+  builder.AddCroppingStep(0.875);
+  builder.AddResizingStep(224, 224, false);
+  builder.AddNormalizationStep(127.5, 1.0 / 127.5);
+  ImagePreprocessingStage stage(builder.build());
+  EXPECT_EQ(stage.Init(), kTfLiteOk);
+
+  // Pre-run.
+  EXPECT_EQ(stage.GetPreprocessedImageData(), nullptr);
+
+  stage.SetImagePath(&image_path);
+  EXPECT_EQ(stage.Run(), kTfLiteOk);
+  EvaluationStageMetrics metrics = stage.LatestMetrics();
+
+  float* preprocessed_image_ptr =
+      static_cast<float*>(stage.GetPreprocessedImageData());
+  EXPECT_NE(preprocessed_image_ptr, nullptr);
+  // We check raw values computed from central-cropping & bilinear interpolation
+  // on the test image. The interpolation math is similar to Unit Square formula
+  // here: https://en.wikipedia.org/wiki/Bilinear_interpolation#Unit_square
+  // These values were verified by running entire image classification pipeline
+  // & ensuring output is accurate. We test 3 values, one for each of R/G/B
+  // channels.
+  EXPECT_FLOAT_EQ(preprocessed_image_ptr[0], -0.74901962);
+  EXPECT_FLOAT_EQ(preprocessed_image_ptr[1], -0.74901962);
+  EXPECT_FLOAT_EQ(preprocessed_image_ptr[2], -0.68627453);
+  EXPECT_THAT(metrics, HasValidLatencyMetrics());
+}
+
+TEST(ImagePreprocessingStage, TestImagePreprocessingFloat_NoCrop) {
+  std::string image_path(kTestImage);
+
+  ImagePreprocessingConfigBuilder builder(
+      std::string(kImagePreprocessingStageName), kTfLiteFloat32);
+  builder.AddResizingStep(224, 224, false);
+  builder.AddNormalizationStep(127.5, 1.0 / 127.5);
+  ImagePreprocessingStage stage(builder.build());
+  EXPECT_EQ(stage.Init(), kTfLiteOk);
+
+  // Pre-run.
+  EXPECT_EQ(stage.GetPreprocessedImageData(), nullptr);
+
+  stage.SetImagePath(&image_path);
+  EXPECT_EQ(stage.Run(), kTfLiteOk);
+  EvaluationStageMetrics metrics = stage.LatestMetrics();
+
+  float* preprocessed_image_ptr =
+      static_cast<float*>(stage.GetPreprocessedImageData());
+  EXPECT_NE(preprocessed_image_ptr, nullptr);
+  // We check raw values computed from central-cropping & bilinear interpolation
+  // on the test image. The interpolation math is similar to Unit Square formula
+  // here: https://en.wikipedia.org/wiki/Bilinear_interpolation#Unit_square
+  // These values were verified by running entire image classification pipeline
+  // & ensuring output is accurate. We test 3 values, one for each of R/G/B
+  // channels.
+  EXPECT_FLOAT_EQ(preprocessed_image_ptr[0], -0.83529419);
+  EXPECT_FLOAT_EQ(preprocessed_image_ptr[1], -0.7960785);
+  EXPECT_FLOAT_EQ(preprocessed_image_ptr[2], -0.35686275);
+  EXPECT_THAT(metrics, HasValidLatencyMetrics());
+}
+
+TEST(ImagePreprocessingStage, TestImagePreprocessingUInt8Quantized) {
+  std::string image_path(kTestImage);
+
+  ImagePreprocessingConfigBuilder builder(
+      std::string(kImagePreprocessingStageName), kTfLiteUInt8);
+  builder.AddCroppingStep(0.875);
+  builder.AddResizingStep(224, 224, false);
+  ImagePreprocessingStage stage(builder.build());
+  EXPECT_EQ(stage.Init(), kTfLiteOk);
+
+  // Pre-run.
+  EXPECT_EQ(stage.GetPreprocessedImageData(), nullptr);
+
+  stage.SetImagePath(&image_path);
+  EXPECT_EQ(stage.Run(), kTfLiteOk);
+  EvaluationStageMetrics metrics = stage.LatestMetrics();
+
+  uint8_t* preprocessed_image_ptr =
+      static_cast<uint8_t*>(stage.GetPreprocessedImageData());
+  EXPECT_NE(preprocessed_image_ptr, nullptr);
+  // We check raw values computed from central-cropping & bilinear interpolation
+  // on the test image. The interpolation math is similar to Unit Square formula
+  // here: https://en.wikipedia.org/wiki/Bilinear_interpolation#Unit_square
+  // These values were verified by running entire image classification pipeline
+  // & ensuring output is accurate. We test 3 values, one for each of R/G/B
+  // channels.
+  EXPECT_EQ(preprocessed_image_ptr[0], 32);
+  EXPECT_EQ(preprocessed_image_ptr[1], 32);
+  EXPECT_EQ(preprocessed_image_ptr[2], 40);
+  EXPECT_THAT(metrics, HasValidLatencyMetrics());
+}
+
+TEST(ImagePreprocessingStage, TestImagePreprocessingInt8Quantized) {
+  std::string image_path(kTestImage);
+
+  ImagePreprocessingConfigBuilder builder(
+      std::string(kImagePreprocessingStageName), kTfLiteInt8);
+  builder.AddCroppingStep(0.875);
+  builder.AddResizingStep(224, 224, false);
+  builder.AddNormalizationStep(128.0, 1.0);
+  ImagePreprocessingStage stage(builder.build());
+  EXPECT_EQ(stage.Init(), kTfLiteOk);
+
+  // Pre-run.
+  EXPECT_EQ(stage.GetPreprocessedImageData(), nullptr);
+
+  stage.SetImagePath(&image_path);
+  EXPECT_EQ(stage.Run(), kTfLiteOk);
+  EvaluationStageMetrics metrics = stage.LatestMetrics();
+
+  int8_t* preprocessed_image_ptr =
+      static_cast<int8_t*>(stage.GetPreprocessedImageData());
+  EXPECT_NE(preprocessed_image_ptr, nullptr);
+  // We check raw values computed from central-cropping & bilinear interpolation
+  // on the test image. The interpolation math is similar to Unit Square formula
+  // here: https://en.wikipedia.org/wiki/Bilinear_interpolation#Unit_square
+  // These values were verified by running entire image classification pipeline
+  // & ensuring output is accurate. We test 3 values, one for each of R/G/B
+  // channels.
+  EXPECT_EQ(preprocessed_image_ptr[0], -96);
+  EXPECT_EQ(preprocessed_image_ptr[1], -96);
+  EXPECT_EQ(preprocessed_image_ptr[2], -88);
+  EXPECT_THAT(metrics, HasValidLatencyMetrics());
+}
+
+TEST(ImagePreprocessingStage, TestImagePreprocessingPadding) {
+  std::string image_path(kTestImage);
+
+  ImagePreprocessingConfigBuilder builder(
+      std::string(kImagePreprocessingStageName), kTfLiteInt8);
+  builder.AddCroppingStep(0.875);
+  builder.AddResizingStep(224, 224, false);
+  builder.AddPaddingStep(225, 225, 0);
+  builder.AddNormalizationStep(128.0, 1.0);
+  ImagePreprocessingStage stage(builder.build());
+  EXPECT_EQ(stage.Init(), kTfLiteOk);
+
+  // Pre-run.
+  EXPECT_EQ(stage.GetPreprocessedImageData(), nullptr);
+
+  stage.SetImagePath(&image_path);
+  EXPECT_EQ(stage.Run(), kTfLiteOk);
+  EvaluationStageMetrics metrics = stage.LatestMetrics();
+
+  int8_t* preprocessed_image_ptr =
+      static_cast<int8_t*>(stage.GetPreprocessedImageData());
+  EXPECT_NE(preprocessed_image_ptr, nullptr);
+  // We check raw values computed from central-cropping & bilinear interpolation
+  // on the test image. The interpolation math is similar to Unit Square formula
+  // here: https://en.wikipedia.org/wiki/Bilinear_interpolation#Unit_square
+  // These values were verified by running entire image classification pipeline
+  // & ensuring output is accurate. We test 3 values, one for each of R/G/B
+  // channels.
+  EXPECT_EQ(preprocessed_image_ptr[0], -128);
+  EXPECT_EQ(preprocessed_image_ptr[224], -128);
+  EXPECT_EQ(preprocessed_image_ptr[225 * 3], -128);
+  EXPECT_EQ(preprocessed_image_ptr[225 * 3 + 3], -96);
+  EXPECT_EQ(preprocessed_image_ptr[225 * 3 + 4], -96);
+  EXPECT_EQ(preprocessed_image_ptr[225 * 3 + 5], -88);
+  EXPECT_THAT(metrics, HasValidLatencyMetrics());
+}
+
+TEST(ImagePreprocessingStage, TestImagePreprocessingSubtractMean) {
+  std::string image_path(kTestImage);
+
+  ImagePreprocessingConfigBuilder builder(
+      std::string(kImagePreprocessingStageName), kTfLiteFloat32);
+  builder.AddCroppingStep(0.875);
+  builder.AddResizingStep(224, 224, false);
+  builder.AddPerChannelNormalizationStep(110.0, 120.0, 123.0, 1.0);
+  ImagePreprocessingStage stage(builder.build());
+  EXPECT_EQ(stage.Init(), kTfLiteOk);
+
+  // Pre-run.
+  EXPECT_EQ(stage.GetPreprocessedImageData(), nullptr);
+
+  stage.SetImagePath(&image_path);
+  EXPECT_EQ(stage.Run(), kTfLiteOk);
+  EvaluationStageMetrics metrics = stage.LatestMetrics();
+
+  float* preprocessed_image_ptr =
+      static_cast<float*>(stage.GetPreprocessedImageData());
+  EXPECT_NE(preprocessed_image_ptr, nullptr);
+  // We check raw values computed from central-cropping & bilinear interpolation
+  // on the test image. The interpolation math is similar to Unit Square formula
+  // here: https://en.wikipedia.org/wiki/Bilinear_interpolation#Unit_square
+  // These values were verified by running entire image classification pipeline
+  // & ensuring output is accurate. We test 3 values, one for each of R/G/B
+  // channels.
+  EXPECT_EQ(preprocessed_image_ptr[0], -78);
+  EXPECT_EQ(preprocessed_image_ptr[1], -88);
+  EXPECT_EQ(preprocessed_image_ptr[2], -83);
+  EXPECT_THAT(metrics, HasValidLatencyMetrics());
+}
+
+TEST(ImagePreprocessingStage, TestCropTargetLargerThanImage) {
+  std::string image_path(kTestImage);
+
+  ImagePreprocessingConfigBuilder builder(
+      std::string(kImagePreprocessingStageName), kTfLiteFloat32);
+  // A crop target larger than the image would produce negative start offsets
+  // and read out of bounds. It must be rejected at run time.
+  builder.AddCroppingStep(/*width=*/100000U, /*height=*/100000U);
+  builder.AddNormalizationStep(127.5, 1.0 / 127.5);
+  ImagePreprocessingStage stage(builder.build());
+  EXPECT_EQ(stage.Init(), kTfLiteOk);
+  stage.SetImagePath(&image_path);
+  EXPECT_EQ(stage.Run(), kTfLiteError);
+}
+
+TEST(ImagePreprocessingStage, TestPaddingTargetSmallerThanImage) {
+  std::string image_path(kTestImage);
+
+  ImagePreprocessingConfigBuilder builder(
+      std::string(kImagePreprocessingStageName), kTfLiteFloat32);
+  // A padding target smaller than the image would produce negative padding
+  // counts and write out of bounds. It must be rejected at run time.
+  builder.AddPaddingStep(/*width=*/1, /*height=*/1, 0);
+  builder.AddNormalizationStep(127.5, 1.0 / 127.5);
+  ImagePreprocessingStage stage(builder.build());
+  EXPECT_EQ(stage.Init(), kTfLiteOk);
+  stage.SetImagePath(&image_path);
+  EXPECT_EQ(stage.Run(), kTfLiteError);
+}
+
+TEST(ImagePreprocessingStage, TestPaddingTargetTooLarge) {
+  std::string image_path(kTestImage);
+
+  ImagePreprocessingConfigBuilder builder(
+      std::string(kImagePreprocessingStageName), kTfLiteFloat32);
+  // A padding target large enough to overflow the output buffer size
+  // computation must be rejected at run time.
+  builder.AddPaddingStep(/*width=*/100000, /*height=*/100000, 0);
+  builder.AddNormalizationStep(127.5, 1.0 / 127.5);
+  ImagePreprocessingStage stage(builder.build());
+  EXPECT_EQ(stage.Init(), kTfLiteOk);
+  stage.SetImagePath(&image_path);
+  EXPECT_EQ(stage.Run(), kTfLiteError);
+}
+
+TEST(ImagePreprocessingStage, TestInvalidRawImageSize) {
+  std::string image_path = testing::TempDir() + "/invalid.rgb8";
+  std::ofstream stream(image_path, std::ios::out | std::ios::binary);
+  stream.write("1234", 4);
+  stream.close();
+
+  ImagePreprocessingConfigBuilder builder(
+      std::string(kImagePreprocessingStageName), kTfLiteFloat32);
+  builder.AddNormalizationStep(127.5, 1.0 / 127.5);
+  ImagePreprocessingStage stage(builder.build());
+  EXPECT_EQ(stage.Init(), kTfLiteOk);
+  stage.SetImagePath(&image_path);
+  EXPECT_EQ(stage.Run(), kTfLiteError);
+}
+
+}  // namespace
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/stages/inference_profiler_stage.cc

@@ -0,0 +1,275 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/evaluation/stages/inference_profiler_stage.h"
+
+#include <cmath>
+#include <cstddef>
+#include <cstdint>
+#include <limits>
+#include <memory>
+#include <random>
+#include <vector>
+
+#include "fp16.h"  // from @FP16
+#include "absl/log/log.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/lite/c/c_api_types.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_delegate_provider.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+#include "tensorflow/lite/tools/evaluation/stages/tflite_inference_stage.h"
+
+namespace tflite {
+namespace evaluation {
+namespace {
+
+// Parameters for a simple Gaussian distribution to generate values roughly in
+// [0, 1).
+constexpr float kGaussianFloatMean = 0.5;
+constexpr float kGaussianStdDev = 1.0 / 3;
+
+// TODO(b/131420973): Reconcile these with the functionality in
+// testing/kernel_test.
+template <typename T>
+void GenerateRandomGaussianData(int64_t num_elements, float min, float max,
+                                std::vector<T>* data) {
+  data->clear();
+  data->reserve(num_elements);
+
+  static std::normal_distribution<double> distribution(kGaussianFloatMean,
+                                                       kGaussianStdDev);
+  static std::default_random_engine generator;
+  for (int i = 0; i < num_elements; ++i) {
+    auto rand_n = distribution(generator);
+    while (rand_n < 0 || rand_n >= 1) {
+      rand_n = distribution(generator);
+    }
+    auto rand_float = min + (max - min) * static_cast<float>(rand_n);
+    data->push_back(static_cast<T>(rand_float));
+  }
+}
+
+template <typename T>
+float CalculateAverageError(T* reference, T* test, int64_t num_elements) {
+  float error = 0;
+
+  for (int i = 0; i < num_elements; i++) {
+    float test_value = static_cast<float>(test[i]);
+    float reference_value = static_cast<float>(reference[i]);
+    error += std::abs(test_value - reference_value);
+  }
+  error /= num_elements;
+
+  return error;
+}
+
+}  // namespace
+
+TfLiteStatus InferenceProfilerStage::Init(
+    const DelegateProviders* delegate_providers) {
+  // Initialize TfliteInferenceStage with the user-provided
+  // TfliteInferenceParams.
+  test_stage_ = std::make_unique<TfliteInferenceStage>(config_);
+  if (test_stage_->Init(delegate_providers) != kTfLiteOk) return kTfLiteError;
+  LOG(INFO) << "Test interpreter has been initialized.";
+
+  // Initialize a reference TfliteInferenceStage that uses the given model &
+  // num_runs, but maintains the rest of TfliteInferenceParams to default.
+  EvaluationStageConfig reference_config;
+  reference_config.set_name("reference_inference");
+  auto* params = reference_config.mutable_specification()
+                     ->mutable_tflite_inference_params();
+  params->set_model_file_path(
+      config_.specification().tflite_inference_params().model_file_path());
+  params->set_invocations_per_run(
+      config_.specification().tflite_inference_params().invocations_per_run());
+  reference_stage_ = std::make_unique<TfliteInferenceStage>(reference_config);
+  if (reference_stage_->Init() != kTfLiteOk) return kTfLiteError;
+  LOG(INFO) << "Reference interpreter (1 thread on CPU) has been initialized.";
+
+  model_info_ = reference_stage_->GetModelInfo();
+
+  // Preprocess model input metadata for generating random data later.
+  for (int i = 0; i < model_info_->inputs.size(); ++i) {
+    const TfLiteType model_input_type = model_info_->inputs[i]->type;
+    if (model_input_type == kTfLiteUInt8 || model_input_type == kTfLiteInt8 ||
+        model_input_type == kTfLiteInt32 || model_input_type == kTfLiteInt64 ||
+        model_input_type == kTfLiteBool || model_input_type == kTfLiteFloat32 ||
+        model_input_type == kTfLiteFloat16) {
+    } else {
+      LOG(ERROR) << "InferenceProfilerStage only supports "
+                    "float16/float32/int8/uint8/int32/int64/bool "
+                    "input types";
+      return kTfLiteError;
+    }
+    auto* input_shape = model_info_->inputs[i]->dims;
+    int64_t total_num_elements = 1;
+    for (int i = 0; i < input_shape->size; i++) {
+      total_num_elements *= input_shape->data[i];
+    }
+    input_num_elements_.push_back(total_num_elements);
+    float_tensors_.emplace_back();
+    uint8_tensors_.emplace_back();
+    int8_tensors_.emplace_back();
+    float16_tensors_.emplace_back();
+    int64_tensors_.emplace_back();
+    int32_tensors_.emplace_back();
+    bool_tensors_.emplace_back();
+  }
+  // Preprocess output metadata for calculating diffs later.
+  for (int i = 0; i < model_info_->outputs.size(); ++i) {
+    const TfLiteType model_output_type = model_info_->outputs[i]->type;
+    if (model_output_type == kTfLiteUInt8 || model_output_type == kTfLiteInt8 ||
+        model_output_type == kTfLiteInt32 || model_output_type == kTfLiteBool ||
+        model_output_type == kTfLiteFloat32) {
+    } else {
+      LOG(ERROR) << "InferenceProfilerStage only supports "
+                    "float32/int8/uint8/int32/bool "
+                    "output types";
+      return kTfLiteError;
+    }
+    auto* output_shape = model_info_->outputs[i]->dims;
+    int64_t total_num_elements = 1;
+    for (int i = 0; i < output_shape->size; i++) {
+      total_num_elements *= output_shape->data[i];
+    }
+    output_num_elements_.push_back(total_num_elements);
+
+    error_stats_.emplace_back();
+  }
+
+  return kTfLiteOk;
+}
+
+TfLiteStatus InferenceProfilerStage::Run() {
+  // Generate random inputs.
+  std::vector<void*> input_ptrs;
+  for (int i = 0; i < model_info_->inputs.size(); ++i) {
+    const TfLiteType model_input_type = model_info_->inputs[i]->type;
+    if (model_input_type == kTfLiteUInt8) {
+      GenerateRandomGaussianData(
+          input_num_elements_[i], std::numeric_limits<uint8_t>::min(),
+          std::numeric_limits<uint8_t>::max(), &uint8_tensors_[i]);
+      input_ptrs.push_back(uint8_tensors_[i].data());
+    } else if (model_input_type == kTfLiteInt8) {
+      GenerateRandomGaussianData(
+          input_num_elements_[i], std::numeric_limits<int8_t>::min(),
+          std::numeric_limits<int8_t>::max(), &int8_tensors_[i]);
+      input_ptrs.push_back(int8_tensors_[i].data());
+    } else if (model_input_type == kTfLiteInt32) {
+      GenerateRandomGaussianData(
+          input_num_elements_[i], std::numeric_limits<int32_t>::min(),
+          std::numeric_limits<int32_t>::max(), &int32_tensors_[i]);
+      input_ptrs.push_back(int32_tensors_[i].data());
+    } else if (model_input_type == kTfLiteInt64) {
+      GenerateRandomGaussianData(
+          input_num_elements_[i], std::numeric_limits<int64_t>::min(),
+          std::numeric_limits<int64_t>::max(), &int64_tensors_[i]);
+      input_ptrs.push_back(int64_tensors_[i].data());
+    } else if (model_input_type == kTfLiteBool) {
+      GenerateRandomGaussianData(input_num_elements_[i], 0, 1,
+                                 &bool_tensors_[i]);
+      input_ptrs.push_back(bool_tensors_[i].data());
+    } else if (model_input_type == kTfLiteFloat32) {
+      GenerateRandomGaussianData(input_num_elements_[i], -1, 1,
+                                 &(float_tensors_[i]));
+      input_ptrs.push_back(float_tensors_[i].data());
+    } else if (model_input_type == kTfLiteFloat16) {
+      GenerateRandomGaussianData(input_num_elements_[i], -1, 1,
+                                 &(float_tensors_[i]));
+      for (size_t j = 0; j < float_tensors_[i].size(); j++) {
+        float16_tensors_[i][j] =
+            fp16_ieee_from_fp32_value(float_tensors_[i][j]);
+      }
+      input_ptrs.push_back(float16_tensors_[i].data());
+    } else {
+      LOG(ERROR) << "InferenceProfilerStage only supports "
+                    "float16/float32/int8/uint8/int32/int64/bool "
+                    "input types";
+      return kTfLiteError;
+    }
+  }
+
+  // Run both inference stages.
+  test_stage_->SetInputs(input_ptrs);
+  reference_stage_->SetInputs(input_ptrs);
+  if (test_stage_->Run() != kTfLiteOk) return kTfLiteError;
+  if (reference_stage_->Run() != kTfLiteOk) return kTfLiteError;
+
+  // Calculate errors per output vector.
+  for (int i = 0; i < model_info_->outputs.size(); ++i) {
+    const TfLiteType model_output_type = model_info_->outputs[i]->type;
+    void* reference_ptr = reference_stage_->GetOutputs()->at(i);
+    void* test_ptr = test_stage_->GetOutputs()->at(i);
+    float output_diff = 0;
+    if (model_output_type == kTfLiteUInt8) {
+      output_diff = CalculateAverageError(static_cast<uint8_t*>(reference_ptr),
+                                          static_cast<uint8_t*>(test_ptr),
+                                          output_num_elements_[i]);
+    } else if (model_output_type == kTfLiteInt8) {
+      output_diff = CalculateAverageError(static_cast<int8_t*>(reference_ptr),
+                                          static_cast<int8_t*>(test_ptr),
+                                          output_num_elements_[i]);
+    } else if (model_output_type == kTfLiteInt32) {
+      output_diff = CalculateAverageError(static_cast<int32_t*>(reference_ptr),
+                                          static_cast<int32_t*>(test_ptr),
+                                          output_num_elements_[i]);
+    } else if (model_output_type == kTfLiteBool) {
+      // Use int8_t* for bool tensors to use void* casting.
+      output_diff = CalculateAverageError(static_cast<int8_t*>(reference_ptr),
+                                          static_cast<int8_t*>(test_ptr),
+                                          output_num_elements_[i]);
+    } else if (model_output_type == kTfLiteFloat32) {
+      output_diff = CalculateAverageError(static_cast<float*>(reference_ptr),
+                                          static_cast<float*>(test_ptr),
+                                          output_num_elements_[i]);
+    }
+    error_stats_[i].UpdateStat(output_diff);
+  }
+
+  return kTfLiteOk;
+}
+
+EvaluationStageMetrics InferenceProfilerStage::LatestMetrics() {
+  EvaluationStageMetrics metrics;
+  const auto& reference_metrics = reference_stage_->LatestMetrics();
+  metrics.set_num_runs(reference_metrics.num_runs());
+  auto* inference_profiler_metrics =
+      metrics.mutable_process_metrics()->mutable_inference_profiler_metrics();
+
+  *inference_profiler_metrics->mutable_reference_latency() =
+      reference_metrics.process_metrics().total_latency();
+  *inference_profiler_metrics->mutable_test_latency() =
+      test_stage_->LatestMetrics().process_metrics().total_latency();
+
+  for (int i = 0; i < error_stats_.size(); ++i) {
+    AccuracyMetrics* diff = inference_profiler_metrics->add_output_errors();
+    diff->set_avg_value(error_stats_[i].avg());
+    diff->set_std_deviation(error_stats_[i].std_deviation());
+    diff->set_min_value(error_stats_[i].min());
+    // Avoiding the small positive values contained in max() even when avg() ==
+    // 0.
+    if (error_stats_[i].avg() != 0) {
+      diff->set_max_value(error_stats_[i].max());
+    } else {
+      diff->set_max_value(0);
+    }
+  }
+
+  return metrics;
+}
+
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/stages/inference_profiler_stage.h

@@ -0,0 +1,78 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_INFERENCE_PROFILER_STAGE_H_
+#define TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_INFERENCE_PROFILER_STAGE_H_
+
+#include <stdint.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "xla/tsl/util/stats_calculator.h"
+#include "tensorflow/lite/c/c_api_types.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_delegate_provider.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_stage.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/stages/tflite_inference_stage.h"
+
+namespace tflite {
+namespace evaluation {
+
+// An EvaluationStage to profile a custom TFLite inference config by comparing
+// performance in two settings:
+// 1. User-defined TfliteInferenceParams (The 'test' setting)
+// 2. Default TfliteInferenceParams (The 'reference' setting)
+// The latter essentially implies single-threaded CPU execution.
+class InferenceProfilerStage : public EvaluationStage {
+ public:
+  explicit InferenceProfilerStage(const EvaluationStageConfig& config)
+      : EvaluationStage(config) {}
+
+  TfLiteStatus Init() override { return Init(nullptr); }
+  TfLiteStatus Init(const DelegateProviders* delegate_providers);
+
+  // New Gaussian random data is used as input for each Run.
+  TfLiteStatus Run() override;
+
+  EvaluationStageMetrics LatestMetrics() override;
+
+ private:
+  std::unique_ptr<TfliteInferenceStage> reference_stage_;
+  std::unique_ptr<TfliteInferenceStage> test_stage_;
+
+  const TfLiteModelInfo* model_info_;
+  std::vector<int64_t> input_num_elements_;
+  std::vector<int64_t> output_num_elements_;
+
+  // One Stat for each model output.
+  std::vector<tsl::Stat<float>> error_stats_;
+
+  // One of the following vectors will be populated based on model_input_type_,
+  // and used as the input for the underlying model.
+  std::vector<std::vector<float>> float_tensors_;
+  std::vector<std::vector<int8_t>> int8_tensors_;
+  std::vector<std::vector<uint8_t>> uint8_tensors_;
+  std::vector<std::vector<uint16_t>> float16_tensors_;
+  std::vector<std::vector<int32_t>> int32_tensors_;
+  std::vector<std::vector<int64_t>> int64_tensors_;
+  // Use uint8_t for bool tensors to use void* casting.
+  std::vector<std::vector<uint8_t>> bool_tensors_;
+};
+
+}  // namespace evaluation
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_INFERENCE_PROFILER_STAGE_H_

tensorflow/lite/tools/evaluation/stages/inference_profiler_stage_test.cc

@@ -0,0 +1,82 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/evaluation/stages/inference_profiler_stage.h"
+
+#include <stdint.h>
+
+#include <gtest/gtest.h>
+#include "tensorflow/lite/core/c/common.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+
+namespace tflite {
+namespace evaluation {
+namespace {
+
+constexpr char kInferenceProfilerStageName[] = "inference_profiler_stage";
+constexpr char kModelPath[] =
+    "tensorflow/lite/testdata/add_quantized.bin";
+
+EvaluationStageConfig GetInferenceProfilerStageConfig(int num_threads = 1) {
+  EvaluationStageConfig config;
+  config.set_name(kInferenceProfilerStageName);
+  auto* params =
+      config.mutable_specification()->mutable_tflite_inference_params();
+  params->set_model_file_path(kModelPath);
+  params->set_invocations_per_run(2);
+  params->set_num_threads(num_threads);
+  return config;
+}
+
+TEST(InferenceProfilerStage, NoParams) {
+  EvaluationStageConfig config = GetInferenceProfilerStageConfig();
+  config.mutable_specification()->clear_tflite_inference_params();
+  InferenceProfilerStage stage(config);
+  EXPECT_EQ(stage.Init(), kTfLiteError);
+}
+
+TEST(InferenceProfilerStage, NoModelPath) {
+  EvaluationStageConfig config = GetInferenceProfilerStageConfig();
+  config.mutable_specification()
+      ->mutable_tflite_inference_params()
+      ->clear_model_file_path();
+  InferenceProfilerStage stage(config);
+  EXPECT_EQ(stage.Init(), kTfLiteError);
+}
+
+TEST(InferenceProfilerStage, NoOutputDiffForDefaultConfig) {
+  // Create stage.
+  EvaluationStageConfig config = GetInferenceProfilerStageConfig();
+  InferenceProfilerStage stage(config);
+
+  // Initialize.
+  EXPECT_EQ(stage.Init(), kTfLiteOk);
+
+  for (int i = 0; i < 5; ++i) {
+    EXPECT_EQ(stage.Run(), kTfLiteOk);
+  }
+  EvaluationStageMetrics metrics = stage.LatestMetrics();
+  EXPECT_TRUE(metrics.process_metrics().has_inference_profiler_metrics());
+  auto profiler_metrics =
+      metrics.process_metrics().inference_profiler_metrics();
+  EXPECT_TRUE(profiler_metrics.has_reference_latency());
+  EXPECT_TRUE(profiler_metrics.has_test_latency());
+  EXPECT_EQ(profiler_metrics.output_errors_size(), 1);
+  EXPECT_EQ(profiler_metrics.output_errors(0).avg_value(), 0);
+}
+
+}  // namespace
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/stages/object_detection_average_precision_stage.cc

@@ -0,0 +1,165 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/evaluation/stages/object_detection_average_precision_stage.h"
+
+#include <vector>
+
+#include "absl/log/log.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/lite/c/c_api_types.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+#include "tensorflow/lite/tools/evaluation/stages/utils/image_metrics.h"
+
+namespace tflite {
+namespace evaluation {
+namespace {
+
+image::Detection ConvertProtoToDetection(
+    const ObjectDetectionResult::ObjectInstance& input, int image_id) {
+  image::Detection detection;
+  detection.box.x.min = input.bounding_box().normalized_left();
+  detection.box.x.max = input.bounding_box().normalized_right();
+  detection.box.y.min = input.bounding_box().normalized_top();
+  detection.box.y.max = input.bounding_box().normalized_bottom();
+  detection.imgid = image_id;
+  detection.score = input.score();
+  return detection;
+}
+
+}  // namespace
+
+TfLiteStatus ObjectDetectionAveragePrecisionStage::Init() {
+  num_classes_ = config_.specification()
+                     .object_detection_average_precision_params()
+                     .num_classes();
+  if (num_classes_ <= 0) {
+    LOG(ERROR) << "num_classes cannot be <= 0";
+    return kTfLiteError;
+  }
+
+  ground_truth_object_vectors_.clear();
+  predicted_object_vectors_.clear();
+
+  // Initialize per-class data structures.
+  for (int i = 0; i < num_classes_; ++i) {
+    ground_truth_object_vectors_.emplace_back();
+    predicted_object_vectors_.emplace_back();
+  }
+  return kTfLiteOk;
+}
+
+TfLiteStatus ObjectDetectionAveragePrecisionStage::Run() {
+  for (const auto& object : ground_truth_objects_.objects()) {
+    const int class_id = object.class_id();
+    if (class_id < 0 || class_id >= num_classes_) {
+      LOG(ERROR) << "Encountered invalid class ID: " << class_id;
+      return kTfLiteError;
+    }
+  }
+
+  for (const auto& object : predicted_objects_.objects()) {
+    const int class_id = object.class_id();
+    if (class_id < 0 || class_id >= num_classes_) {
+      LOG(ERROR) << "Encountered invalid class ID: " << class_id;
+      return kTfLiteError;
+    }
+  }
+
+  for (const auto& object : ground_truth_objects_.objects()) {
+    const int class_id = object.class_id();
+    ground_truth_object_vectors_[class_id].push_back(
+        ConvertProtoToDetection(object, current_image_index_));
+  }
+
+  for (const auto& object : predicted_objects_.objects()) {
+    const int class_id = object.class_id();
+    predicted_object_vectors_[class_id].push_back(
+        ConvertProtoToDetection(object, current_image_index_));
+  }
+
+  current_image_index_++;
+  return kTfLiteOk;
+}
+
+EvaluationStageMetrics ObjectDetectionAveragePrecisionStage::LatestMetrics() {
+  EvaluationStageMetrics metrics;
+  if (current_image_index_ == 0) return metrics;
+
+  metrics.set_num_runs(current_image_index_);
+  auto* ap_metrics = metrics.mutable_process_metrics()
+                         ->mutable_object_detection_average_precision_metrics();
+  auto& ap_params =
+      config_.specification().object_detection_average_precision_params();
+
+  std::vector<float> iou_thresholds;
+  if (ap_params.iou_thresholds_size() == 0) {
+    // Default IoU thresholds as defined by COCO evaluation.
+    // Refer: http://cocodataset.org/#detection-eval
+    float threshold = 0.5;
+    for (int i = 0; i < 10; ++i) {
+      iou_thresholds.push_back(threshold + i * 0.05f);
+    }
+  } else {
+    for (const float threshold : ap_params.iou_thresholds()) {
+      iou_thresholds.push_back(threshold);
+    }
+  }
+
+  image::AveragePrecision::Options opts;
+  opts.num_recall_points = ap_params.num_recall_points();
+
+  float ap_sum = 0;
+  int num_total_aps = 0;
+  for (float threshold : iou_thresholds) {
+    float threshold_ap_sum = 0;
+    int num_counted_classes = 0;
+
+    for (int i = 0; i < num_classes_; ++i) {
+      // Skip if this class wasn't encountered at all.
+      // TODO(b/133772912): Investigate the validity of this snippet when a
+      // subset of the classes is encountered in datasets.
+      if (ground_truth_object_vectors_[i].empty() &&
+          predicted_object_vectors_[i].empty())
+        continue;
+
+      // Output is NaN if there are no ground truth objects.
+      // So we assume 0.
+      float ap_value = 0.0;
+      if (!ground_truth_object_vectors_[i].empty()) {
+        opts.iou_threshold = threshold;
+        ap_value = image::AveragePrecision(opts).FromBoxes(
+            ground_truth_object_vectors_[i], predicted_object_vectors_[i]);
+      }
+
+      ap_sum += ap_value;
+      num_total_aps += 1;
+      threshold_ap_sum += ap_value;
+      num_counted_classes += 1;
+    }
+
+    if (num_counted_classes == 0) continue;
+    auto* threshold_ap = ap_metrics->add_individual_average_precisions();
+    threshold_ap->set_average_precision(threshold_ap_sum / num_counted_classes);
+    threshold_ap->set_iou_threshold(threshold);
+  }
+
+  if (num_total_aps == 0) return metrics;
+  ap_metrics->set_overall_mean_average_precision(ap_sum / num_total_aps);
+  return metrics;
+}
+
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/stages/object_detection_average_precision_stage.h

@@ -0,0 +1,67 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_OBJECT_DETECTION_AVERAGE_PRECISION_STAGE_H_
+#define TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_OBJECT_DETECTION_AVERAGE_PRECISION_STAGE_H_
+
+#include <vector>
+
+#include "tensorflow/lite/c/c_api_types.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_stage.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+#include "tensorflow/lite/tools/evaluation/stages/utils/image_metrics.h"
+
+namespace tflite {
+namespace evaluation {
+
+// EvaluationStage to compute Average Precision for Object Detection Task.
+// Computes Average Precision per-IoU threshold (averaged across all classes),
+// and then mean Average Precision (mAP) as the average AP value across all
+// thresholds.
+class ObjectDetectionAveragePrecisionStage : public EvaluationStage {
+ public:
+  explicit ObjectDetectionAveragePrecisionStage(
+      const EvaluationStageConfig& config)
+      : EvaluationStage(config) {}
+
+  TfLiteStatus Init() override;
+
+  TfLiteStatus Run() override;
+
+  EvaluationStageMetrics LatestMetrics() override;
+
+  // Call before Run().
+  void SetEvalInputs(const ObjectDetectionResult& predicted_objects,
+                     const ObjectDetectionResult& ground_truth_objects) {
+    predicted_objects_ = predicted_objects;
+    ground_truth_objects_ = ground_truth_objects;
+  }
+
+ private:
+  int num_classes_ = -1;
+  ObjectDetectionResult predicted_objects_;
+  ObjectDetectionResult ground_truth_objects_;
+  int current_image_index_ = 0;
+
+  // One inner vector per class for ground truth objects.
+  std::vector<std::vector<image::Detection>> ground_truth_object_vectors_;
+  // One inner vector per class for predicted objects.
+  std::vector<std::vector<image::Detection>> predicted_object_vectors_;
+};
+
+}  // namespace evaluation
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_OBJECT_DETECTION_AVERAGE_PRECISION_STAGE_H_

tensorflow/lite/tools/evaluation/stages/object_detection_average_precision_stage_test.cc

@@ -0,0 +1,243 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/evaluation/stages/object_detection_average_precision_stage.h"
+
+#include <gtest/gtest.h>
+#include "tensorflow/lite/c/c_api_types.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+
+namespace tflite {
+namespace evaluation {
+namespace {
+
+constexpr char kAveragePrecisionStageName[] =
+    "object_detection_average_precision";
+
+EvaluationStageConfig GetAveragePrecisionStageConfig(int num_classes) {
+  EvaluationStageConfig config;
+  config.set_name(kAveragePrecisionStageName);
+  auto* params = config.mutable_specification()
+                     ->mutable_object_detection_average_precision_params();
+  params->add_iou_thresholds(0.5);
+  params->add_iou_thresholds(0.999);
+  params->set_num_classes(num_classes);
+  return config;
+}
+
+ObjectDetectionResult GetGroundTruthDetectionResult() {
+  ObjectDetectionResult ground_truth;
+  ground_truth.set_image_name("some_image.jpg");
+
+  auto* object_1 = ground_truth.add_objects();
+  object_1->set_class_id(1);
+  auto* object_1_bbox = object_1->mutable_bounding_box();
+  object_1_bbox->set_normalized_top(0.5);
+  object_1_bbox->set_normalized_bottom(1.0);
+  object_1_bbox->set_normalized_left(0.5);
+  object_1_bbox->set_normalized_right(1.0);
+
+  auto* object_2 = ground_truth.add_objects();
+  object_2->set_class_id(1);
+  auto* object_2_bbox = object_2->mutable_bounding_box();
+  object_2_bbox->set_normalized_top(0);
+  object_2_bbox->set_normalized_bottom(1.0);
+  object_2_bbox->set_normalized_left(0);
+  object_2_bbox->set_normalized_right(1.0);
+
+  auto* object_3 = ground_truth.add_objects();
+  object_3->set_class_id(2);
+  auto* object_3_bbox = object_3->mutable_bounding_box();
+  object_3_bbox->set_normalized_top(0.5);
+  object_3_bbox->set_normalized_bottom(1.0);
+  object_3_bbox->set_normalized_left(0.5);
+  object_3_bbox->set_normalized_right(1.0);
+
+  return ground_truth;
+}
+
+ObjectDetectionResult GetPredictedDetectionResult() {
+  ObjectDetectionResult predicted;
+
+  auto* object_1 = predicted.add_objects();
+  object_1->set_class_id(1);
+  object_1->set_score(0.8);
+  auto* object_1_bbox = object_1->mutable_bounding_box();
+  object_1_bbox->set_normalized_top(0.091);
+  object_1_bbox->set_normalized_bottom(1.0);
+  object_1_bbox->set_normalized_left(0.091);
+  object_1_bbox->set_normalized_right(1.0);
+
+  auto* object_2 = predicted.add_objects();
+  object_2->set_class_id(1);
+  object_2->set_score(0.9);
+  auto* object_2_bbox = object_2->mutable_bounding_box();
+  object_2_bbox->set_normalized_top(0.474);
+  object_2_bbox->set_normalized_bottom(1.0);
+  object_2_bbox->set_normalized_left(0.474);
+  object_2_bbox->set_normalized_right(1.0);
+
+  auto* object_3 = predicted.add_objects();
+  object_3->set_class_id(1);
+  object_3->set_score(0.95);
+  auto* object_3_bbox = object_3->mutable_bounding_box();
+  object_3_bbox->set_normalized_top(0.474);
+  object_3_bbox->set_normalized_bottom(1.0);
+  object_3_bbox->set_normalized_left(0.474);
+  object_3_bbox->set_normalized_right(1.0);
+  return predicted;
+}
+
+TEST(ObjectDetectionAveragePrecisionStage, ZeroClasses) {
+  // Create stage.
+  EvaluationStageConfig config = GetAveragePrecisionStageConfig(0);
+  ObjectDetectionAveragePrecisionStage stage =
+      ObjectDetectionAveragePrecisionStage(config);
+
+  EXPECT_EQ(stage.Init(), kTfLiteError);
+}
+
+// Tests ObjectDetectionAveragePrecisionStage with sample inputs & outputs.
+// The underlying library is tested extensively in utils/image_metrics_test.
+TEST(ObjectDetectionAveragePrecisionStage, SampleInputs) {
+  // Create & initialize stage.
+  EvaluationStageConfig config = GetAveragePrecisionStageConfig(3);
+  ObjectDetectionAveragePrecisionStage stage =
+      ObjectDetectionAveragePrecisionStage(config);
+  EXPECT_EQ(stage.Init(), kTfLiteOk);
+
+  const ObjectDetectionResult ground_truth = GetGroundTruthDetectionResult();
+  const ObjectDetectionResult predicted = GetPredictedDetectionResult();
+
+  // Run with no predictions.
+  stage.SetEvalInputs(ObjectDetectionResult(), ground_truth);
+  EXPECT_EQ(stage.Run(), kTfLiteOk);
+
+  EvaluationStageMetrics metrics = stage.LatestMetrics();
+  ObjectDetectionAveragePrecisionMetrics detection_metrics =
+      metrics.process_metrics().object_detection_average_precision_metrics();
+  EXPECT_FLOAT_EQ(detection_metrics.overall_mean_average_precision(), 0.0);
+  EXPECT_EQ(detection_metrics.individual_average_precisions_size(), 2);
+
+  // Run with matching predictions.
+  stage.SetEvalInputs(ground_truth, ground_truth);
+  EXPECT_EQ(stage.Run(), kTfLiteOk);
+
+  metrics = stage.LatestMetrics();
+  detection_metrics =
+      metrics.process_metrics().object_detection_average_precision_metrics();
+  EXPECT_FLOAT_EQ(detection_metrics.overall_mean_average_precision(),
+                  0.50495052);
+  EXPECT_EQ(metrics.num_runs(), 2);
+
+  // Run.
+  stage.SetEvalInputs(predicted, ground_truth);
+  EXPECT_EQ(stage.Run(), kTfLiteOk);
+
+  metrics = stage.LatestMetrics();
+  detection_metrics =
+      metrics.process_metrics().object_detection_average_precision_metrics();
+  EXPECT_FLOAT_EQ(
+      detection_metrics.individual_average_precisions(0).iou_threshold(), 0.5);
+  EXPECT_FLOAT_EQ(
+      detection_metrics.individual_average_precisions(0).average_precision(),
+      0.4841584);
+  EXPECT_FLOAT_EQ(
+      detection_metrics.individual_average_precisions(1).iou_threshold(),
+      0.999);
+  EXPECT_FLOAT_EQ(
+      detection_metrics.individual_average_precisions(1).average_precision(),
+      0.33663365);
+  // Should be average of above two values.
+  EXPECT_FLOAT_EQ(detection_metrics.overall_mean_average_precision(),
+                  0.41039604);
+}
+
+TEST(ObjectDetectionAveragePrecisionStage, DefaultIoUThresholds) {
+  // Create & initialize stage.
+  EvaluationStageConfig config = GetAveragePrecisionStageConfig(3);
+  auto* params = config.mutable_specification()
+                     ->mutable_object_detection_average_precision_params();
+  params->clear_iou_thresholds();
+  ObjectDetectionAveragePrecisionStage stage =
+      ObjectDetectionAveragePrecisionStage(config);
+  EXPECT_EQ(stage.Init(), kTfLiteOk);
+
+  const ObjectDetectionResult ground_truth = GetGroundTruthDetectionResult();
+  const ObjectDetectionResult predicted = GetPredictedDetectionResult();
+
+  // Run with matching predictions.
+  stage.SetEvalInputs(ground_truth, ground_truth);
+  EXPECT_EQ(stage.Run(), kTfLiteOk);
+
+  EvaluationStageMetrics metrics = stage.LatestMetrics();
+  ObjectDetectionAveragePrecisionMetrics detection_metrics =
+      metrics.process_metrics().object_detection_average_precision_metrics();
+  // Full AP, since ground-truth & predictions match.
+  EXPECT_FLOAT_EQ(detection_metrics.overall_mean_average_precision(), 1.0);
+  // Should be 10 IoU thresholds.
+  EXPECT_EQ(detection_metrics.individual_average_precisions_size(), 10);
+  EXPECT_FLOAT_EQ(
+      detection_metrics.individual_average_precisions(0).iou_threshold(), 0.5);
+  EXPECT_FLOAT_EQ(
+      detection_metrics.individual_average_precisions(9).iou_threshold(), 0.95);
+}
+
+TEST(ObjectDetectionAveragePrecisionStage, NegativeClassId) {
+  EvaluationStageConfig config = GetAveragePrecisionStageConfig(3);
+  ObjectDetectionAveragePrecisionStage stage =
+      ObjectDetectionAveragePrecisionStage(config);
+  EXPECT_EQ(stage.Init(), kTfLiteOk);
+
+  ObjectDetectionResult ground_truth;
+  ground_truth.set_image_name("some_image.jpg");
+  auto* object = ground_truth.add_objects();
+  object->set_class_id(-1);
+  auto* bbox = object->mutable_bounding_box();
+  bbox->set_normalized_top(0.5);
+  bbox->set_normalized_bottom(1.0);
+  bbox->set_normalized_left(0.5);
+  bbox->set_normalized_right(1.0);
+
+  stage.SetEvalInputs(ObjectDetectionResult(), ground_truth);
+  EXPECT_EQ(stage.Run(), kTfLiteError);
+
+  ObjectDetectionResult predicted;
+  auto* pred_object = predicted.add_objects();
+  pred_object->set_class_id(-1);
+  pred_object->set_score(0.8);
+  auto* pred_bbox = pred_object->mutable_bounding_box();
+  pred_bbox->set_normalized_top(0.5);
+  pred_bbox->set_normalized_bottom(1.0);
+  pred_bbox->set_normalized_left(0.5);
+  pred_bbox->set_normalized_right(1.0);
+
+  stage.SetEvalInputs(predicted, GetGroundTruthDetectionResult());
+  EXPECT_EQ(stage.Run(), kTfLiteError);
+
+  // Verify internal state remains unmutated and fully functional after error
+  stage.SetEvalInputs(GetGroundTruthDetectionResult(),
+                      GetGroundTruthDetectionResult());
+  EXPECT_EQ(stage.Run(), kTfLiteOk);
+  EvaluationStageMetrics metrics = stage.LatestMetrics();
+  EXPECT_FLOAT_EQ(metrics.process_metrics()
+                      .object_detection_average_precision_metrics()
+                      .overall_mean_average_precision(),
+                  1.0);
+}
+
+}  // namespace
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/stages/object_detection_stage.cc

@@ -0,0 +1,206 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/evaluation/stages/object_detection_stage.h"
+
+#include <cstddef>
+#include <fstream>
+#include <iterator>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/container/flat_hash_map.h"
+#include "absl/log/log.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/lite/core/c/common.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_delegate_provider.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+#include "tensorflow/lite/tools/evaluation/stages/image_preprocessing_stage.h"
+#include "tensorflow/lite/tools/evaluation/stages/object_detection_average_precision_stage.h"
+#include "tensorflow/lite/tools/evaluation/stages/tflite_inference_stage.h"
+
+namespace tflite {
+namespace evaluation {
+
+TfLiteStatus ObjectDetectionStage::Init(
+    const DelegateProviders* delegate_providers) {
+  // Ensure inference params are provided.
+  if (!config_.specification().has_object_detection_params()) {
+    LOG(ERROR) << "ObjectDetectionParams not provided";
+    return kTfLiteError;
+  }
+  auto& params = config_.specification().object_detection_params();
+  if (!params.has_inference_params()) {
+    LOG(ERROR) << "inference_params not provided";
+    return kTfLiteError;
+  }
+  if (all_labels_ == nullptr) {
+    LOG(ERROR) << "Detection output labels not provided";
+    return kTfLiteError;
+  }
+
+  // TfliteInferenceStage.
+  EvaluationStageConfig tflite_inference_config;
+  tflite_inference_config.set_name("tflite_inference");
+  *tflite_inference_config.mutable_specification()
+       ->mutable_tflite_inference_params() = params.inference_params();
+  inference_stage_ =
+      std::make_unique<TfliteInferenceStage>(tflite_inference_config);
+  TF_LITE_ENSURE_STATUS(inference_stage_->Init(delegate_providers));
+
+  // Validate model inputs.
+  const TfLiteModelInfo* model_info = inference_stage_->GetModelInfo();
+  if (model_info->inputs.size() != 1 || model_info->outputs.size() != 4) {
+    LOG(ERROR) << "Object detection model must have 1 input & 4 outputs";
+    return kTfLiteError;
+  }
+  TfLiteType input_type = model_info->inputs[0]->type;
+  auto* input_shape = model_info->inputs[0]->dims;
+  // Input should be of the shape {1, height, width, 3}
+  if (input_shape->size != 4 || input_shape->data[0] != 1 ||
+      input_shape->data[3] != 3) {
+    LOG(ERROR) << "Invalid input shape for model";
+    return kTfLiteError;
+  }
+
+  // ImagePreprocessingStage
+  tflite::evaluation::ImagePreprocessingConfigBuilder builder(
+      "image_preprocessing", input_type);
+  builder.AddResizingStep(input_shape->data[2], input_shape->data[1], false);
+  builder.AddDefaultNormalizationStep();
+  preprocessing_stage_ =
+      std::make_unique<ImagePreprocessingStage>(builder.build());
+  TF_LITE_ENSURE_STATUS(preprocessing_stage_->Init());
+
+  // ObjectDetectionAveragePrecisionStage
+  EvaluationStageConfig eval_config;
+  eval_config.set_name("average_precision");
+  *eval_config.mutable_specification()
+       ->mutable_object_detection_average_precision_params() =
+      params.ap_params();
+  eval_config.mutable_specification()
+      ->mutable_object_detection_average_precision_params()
+      ->set_num_classes(all_labels_->size());
+  eval_stage_ =
+      std::make_unique<ObjectDetectionAveragePrecisionStage>(eval_config);
+  TF_LITE_ENSURE_STATUS(eval_stage_->Init());
+
+  return kTfLiteOk;
+}
+
+TfLiteStatus ObjectDetectionStage::Run() {
+  if (image_path_.empty()) {
+    LOG(ERROR) << "Input image not set";
+    return kTfLiteError;
+  }
+
+  // Preprocessing.
+  preprocessing_stage_->SetImagePath(&image_path_);
+  TF_LITE_ENSURE_STATUS(preprocessing_stage_->Run());
+  if (preprocessing_stage_->GetPreprocessedImageBytes() <
+      inference_stage_->GetModelInfo()->inputs[0]->bytes) {
+    LOG(ERROR)
+        << "Preprocessed image buffer is smaller than model input tensor size";
+    return kTfLiteError;
+  }
+
+  // Inference.
+  std::vector<void*> data_ptrs = {};
+  data_ptrs.push_back(preprocessing_stage_->GetPreprocessedImageData());
+  std::vector<size_t> data_sizes = {
+      preprocessing_stage_->GetPreprocessedImageBytes()};
+  inference_stage_->SetInputs(data_ptrs, data_sizes);
+  TF_LITE_ENSURE_STATUS(inference_stage_->Run());
+
+  // Convert model output to ObjectsSet.
+  predicted_objects_.Clear();
+  const int class_offset =
+      config_.specification().object_detection_params().class_offset();
+  const std::vector<void*>* outputs = inference_stage_->GetOutputs();
+  int num_detections = static_cast<int>(*static_cast<float*>(outputs->at(3)));
+  float* detected_label_boxes = static_cast<float*>(outputs->at(0));
+  float* detected_label_indices = static_cast<float*>(outputs->at(1));
+  float* detected_label_probabilities = static_cast<float*>(outputs->at(2));
+  for (int i = 0; i < num_detections; ++i) {
+    const int bounding_box_offset = i * 4;
+    auto* object = predicted_objects_.add_objects();
+    // Bounding box
+    auto* bbox = object->mutable_bounding_box();
+    bbox->set_normalized_top(detected_label_boxes[bounding_box_offset + 0]);
+    bbox->set_normalized_left(detected_label_boxes[bounding_box_offset + 1]);
+    bbox->set_normalized_bottom(detected_label_boxes[bounding_box_offset + 2]);
+    bbox->set_normalized_right(detected_label_boxes[bounding_box_offset + 3]);
+    // Class.
+    object->set_class_id(static_cast<int>(detected_label_indices[i]) +
+                         class_offset);
+    // Score
+    object->set_score(detected_label_probabilities[i]);
+  }
+
+  // AP Evaluation.
+  eval_stage_->SetEvalInputs(predicted_objects_, *ground_truth_objects_);
+  TF_LITE_ENSURE_STATUS(eval_stage_->Run());
+
+  return kTfLiteOk;
+}
+
+EvaluationStageMetrics ObjectDetectionStage::LatestMetrics() {
+  EvaluationStageMetrics metrics;
+  auto* detection_metrics =
+      metrics.mutable_process_metrics()->mutable_object_detection_metrics();
+
+  *detection_metrics->mutable_pre_processing_latency() =
+      preprocessing_stage_->LatestMetrics().process_metrics().total_latency();
+  EvaluationStageMetrics inference_metrics = inference_stage_->LatestMetrics();
+  *detection_metrics->mutable_inference_latency() =
+      inference_metrics.process_metrics().total_latency();
+  *detection_metrics->mutable_inference_metrics() =
+      inference_metrics.process_metrics().tflite_inference_metrics();
+  *detection_metrics->mutable_average_precision_metrics() =
+      eval_stage_->LatestMetrics()
+          .process_metrics()
+          .object_detection_average_precision_metrics();
+  metrics.set_num_runs(inference_metrics.num_runs());
+  return metrics;
+}
+
+TfLiteStatus PopulateGroundTruth(
+    const std::string& grouth_truth_proto_file,
+    absl::flat_hash_map<std::string, ObjectDetectionResult>*
+        ground_truth_mapping) {
+  if (ground_truth_mapping == nullptr) {
+    return kTfLiteError;
+  }
+  ground_truth_mapping->clear();
+
+  // Read the ground truth dump.
+  std::ifstream t(grouth_truth_proto_file);
+  std::string proto_str((std::istreambuf_iterator<char>(t)),
+                        std::istreambuf_iterator<char>());
+  ObjectDetectionGroundTruth ground_truth_proto;
+  ground_truth_proto.ParseFromString(proto_str);
+
+  for (const auto& image_ground_truth :
+       ground_truth_proto.detection_results()) {
+    (*ground_truth_mapping)[image_ground_truth.image_name()] =
+        image_ground_truth;
+  }
+
+  return kTfLiteOk;
+}
+
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/stages/object_detection_stage.h

@@ -0,0 +1,109 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_OBJECT_DETECTION_STAGE_H_
+#define TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_OBJECT_DETECTION_STAGE_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/container/flat_hash_map.h"
+#include "tensorflow/lite/c/c_api_types.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_delegate_provider.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_stage.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+#include "tensorflow/lite/tools/evaluation/stages/image_preprocessing_stage.h"
+#include "tensorflow/lite/tools/evaluation/stages/object_detection_average_precision_stage.h"
+#include "tensorflow/lite/tools/evaluation/stages/tflite_inference_stage.h"
+
+namespace tflite {
+namespace evaluation {
+
+// An EvaluationStage to encapsulate the complete Object Detection task.
+// Assumes that the object detection model's signature (number of
+// inputs/outputs, ordering of outputs & what they denote) is same as the
+// MobileNet SSD model:
+// https://www.tensorflow.org/lite/examples/object_detection/overview#output_signature.
+// Input size/type & number of detections could be different.
+//
+// This class will be extended to support other types of detection models, if
+// required in the future.
+class ObjectDetectionStage : public EvaluationStage {
+ public:
+  explicit ObjectDetectionStage(const EvaluationStageConfig& config)
+      : EvaluationStage(config) {}
+
+  TfLiteStatus Init() override { return Init(nullptr); }
+  TfLiteStatus Init(const DelegateProviders* delegate_providers);
+
+  TfLiteStatus Run() override;
+
+  EvaluationStageMetrics LatestMetrics() override;
+
+  // Call before Init(). all_labels should contain all possible object labels
+  // that can be detected by the model, in the correct order. all_labels should
+  // outlive the call to Init().
+  void SetAllLabels(const std::vector<std::string>& all_labels) {
+    all_labels_ = &all_labels;
+  }
+
+  // Call before Run().
+  // ground_truth_objects instance should outlive the call to Run().
+  void SetInputs(const std::string& image_path,
+                 const ObjectDetectionResult& ground_truth_objects) {
+    image_path_ = image_path;
+    ground_truth_objects_ = &ground_truth_objects;
+  }
+
+  // Provides a pointer to the underlying TfLiteInferenceStage.
+  // Returns non-null value only if this stage has been initialized.
+  TfliteInferenceStage* const GetInferenceStage() {
+    return inference_stage_.get();
+  }
+
+  // Returns a const pointer to the latest inference output.
+  const ObjectDetectionResult* GetLatestPrediction() {
+    return &predicted_objects_;
+  }
+
+ private:
+  const std::vector<std::string>* all_labels_ = nullptr;
+  std::unique_ptr<ImagePreprocessingStage> preprocessing_stage_;
+  std::unique_ptr<TfliteInferenceStage> inference_stage_;
+  std::unique_ptr<ObjectDetectionAveragePrecisionStage> eval_stage_;
+  std::string image_path_;
+
+  // Obtained from SetInputs(...).
+  const ObjectDetectionResult* ground_truth_objects_;
+  // Reflects the outputs generated from the latest call to Run().
+  ObjectDetectionResult predicted_objects_;
+};
+
+// Reads a tflite::evaluation::ObjectDetectionGroundTruth instance from a
+// textproto file and populates a mapping of image name to
+// ObjectDetectionResult.
+// File with ObjectDetectionGroundTruth can be generated using the
+// preprocess_coco_minival.py script in evaluation/tasks/coco_object_detection.
+// Useful for wrappers/scripts that use ObjectDetectionStage.
+TfLiteStatus PopulateGroundTruth(
+    const std::string& grouth_truth_proto_file,
+    absl::flat_hash_map<std::string, ObjectDetectionResult>*
+        ground_truth_mapping);
+
+}  // namespace evaluation
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_OBJECT_DETECTION_STAGE_H_

tensorflow/lite/tools/evaluation/stages/tflite_inference_stage.cc

@@ -0,0 +1,290 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/evaluation/stages/tflite_inference_stage.h"
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <fstream>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "absl/base/attributes.h"
+#include "absl/log/absl_log.h"
+#include "tensorflow/lite/c/common.h"
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/interpreter_builder.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/model_builder.h"
+#include "tensorflow/lite/mutable_op_resolver.h"
+#include "tensorflow/lite/profiling/time.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_delegate_provider.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+
+void RegisterSelectedOps(::tflite::MutableOpResolver* resolver);
+
+// Version with Weak linker attribute doing nothing: if someone links this
+// library with another definition of this function (presumably to actually
+// register custom ops), that version will be used instead.
+void ABSL_ATTRIBUTE_WEAK
+RegisterSelectedOps(::tflite::MutableOpResolver* resolver) {}
+
+namespace tflite {
+namespace evaluation {
+namespace {
+
+TfLiteModelInfo GetTfliteModelInfo(const Interpreter& interpreter) {
+  TfLiteModelInfo model_info;
+  for (int i : interpreter.inputs()) {
+    model_info.inputs.push_back(interpreter.tensor(i));
+  }
+  for (int i : interpreter.outputs()) {
+    model_info.outputs.push_back(interpreter.tensor(i));
+  }
+  return model_info;
+}
+
+}  // namespace
+
+void TfliteInferenceStage::UpdateModelInfo() {
+  model_info_ = GetTfliteModelInfo(*interpreter_);
+}
+
+const std::vector<void*>* TfliteInferenceStage::GetOutputs() const {
+  outputs_.clear();
+  if (interpreter_) {
+    outputs_.reserve(interpreter_->outputs().size());
+    for (int i : interpreter_->outputs()) {
+      TfLiteTensor* tensor = interpreter_->tensor(i);
+      outputs_.push_back(tensor->data.raw);
+    }
+  }
+  return &outputs_;
+}
+
+TfLiteStatus TfliteInferenceStage::ResizeInputs(
+    const std::vector<std::vector<int>>& shapes) {
+  if (!interpreter_) {
+    ABSL_LOG(ERROR) << "Stage not initialized before calling ResizeInputs";
+    return kTfLiteError;
+  }
+  const std::vector<int>& interpreter_inputs = interpreter_->inputs();
+  if (interpreter_inputs.size() != shapes.size()) {
+    ABSL_LOG(ERROR) << "New shape is not compatible";
+    return kTfLiteError;
+  }
+
+  for (size_t j = 0; j < shapes.size(); ++j) {
+    int i = interpreter_inputs[j];
+    TfLiteTensor* t = interpreter_->tensor(i);
+    if (t->type != kTfLiteString) {
+      TF_LITE_ENSURE_STATUS(interpreter_->ResizeInputTensor(i, shapes[j]));
+    }
+  }
+
+  TF_LITE_ENSURE_STATUS(interpreter_->AllocateTensors());
+  UpdateModelInfo();
+  return kTfLiteOk;
+}
+
+TfLiteStatus TfliteInferenceStage::ApplyCustomDelegate(
+    Interpreter::TfLiteDelegatePtr delegate) {
+  if (!interpreter_) {
+    ABSL_LOG(ERROR)
+        << "Stage not initialized before calling ApplyCustomDelegate";
+    return kTfLiteError;
+  }
+  // Skip if delegate is a nullptr.
+  if (!delegate) {
+    ABSL_LOG(WARNING)
+        << "Tried to apply null TfLiteDelegatePtr to TfliteInferenceStage";
+    return kTfLiteOk;
+  }
+  delegates_.push_back(std::move(delegate));
+  TF_LITE_ENSURE_STATUS(
+      interpreter_->ModifyGraphWithDelegate(delegates_.back().get()));
+  UpdateModelInfo();
+  return kTfLiteOk;
+}
+
+TfLiteStatus TfliteInferenceStage::Init(
+    const DelegateProviders* delegate_providers) {
+  if (!config_.specification().has_tflite_inference_params()) {
+    ABSL_LOG(ERROR) << "TfliteInferenceParams not provided";
+    return kTfLiteError;
+  }
+  const TfliteInferenceParams& params =
+      config_.specification().tflite_inference_params();
+  if (params.invocations_per_run() <= 0) {
+    ABSL_LOG(ERROR) << "Invalid invocations_per_run: "
+                    << params.invocations_per_run();
+    return kTfLiteError;
+  }
+  if (!params.has_model_file_path()) {
+    ABSL_LOG(ERROR) << "Model path not provided";
+    return kTfLiteError;
+  }
+  std::ifstream model_check(params.model_file_path());
+  if (!model_check.good()) {
+    ABSL_LOG(ERROR) << "Model file not found";
+    return kTfLiteError;
+  }
+
+  model_ = FlatBufferModel::BuildFromFile(params.model_file_path().c_str());
+  if (!model_) {
+    ABSL_LOG(ERROR) << "Failed to build FlatBufferModel from file: "
+                    << params.model_file_path();
+    return kTfLiteError;
+  }
+
+  bool apply_default_delegates = true;
+  if (delegate_providers != nullptr) {
+    const auto& provider_params = delegate_providers->GetAllParams();
+    // When --use_xnnpack is explicitly set to false, to honor this, skip
+    // applying the XNNPACK delegate by default in TfLite runtime.
+    if (provider_params.HasParam("use_xnnpack") &&
+        provider_params.HasValueSet<bool>("use_xnnpack") &&
+        !provider_params.Get<bool>("use_xnnpack")) {
+      apply_default_delegates = false;
+    }
+  }
+
+  if (apply_default_delegates) {
+    resolver_ = std::make_unique<ops::builtin::BuiltinOpResolver>();
+  } else {
+    resolver_ = std::make_unique<
+        ops::builtin::BuiltinOpResolverWithoutDefaultDelegates>();
+  }
+  RegisterSelectedOps(resolver_.get());
+  InterpreterBuilder(*model_, *resolver_)(&interpreter_);
+  if (!interpreter_) {
+    ABSL_LOG(ERROR) << "Could not build interpreter";
+    return kTfLiteError;
+  }
+  interpreter_->SetNumThreads(params.num_threads());
+
+  delegates_.clear();
+  if (!delegate_providers) {
+    std::string error_message;
+    Interpreter::TfLiteDelegatePtr delegate =
+        CreateTfLiteDelegate(params, &error_message);
+    if (delegate) {
+      delegates_.push_back(std::move(delegate));
+      ABSL_LOG(INFO) << "Successfully created "
+                     << params.Delegate_Name(params.delegate()) << " delegate.";
+    } else {
+      ABSL_LOG(WARNING) << error_message;
+    }
+  } else {
+    std::vector<ProvidedDelegateList::ProvidedDelegate> delegates =
+        delegate_providers->CreateAllDelegates(params);
+    for (ProvidedDelegateList::ProvidedDelegate& one : delegates) {
+      if (one.delegate != nullptr) {
+        delegates_.push_back(std::move(one.delegate));
+      }
+    }
+  }
+
+  for (size_t i = 0; i < delegates_.size(); ++i) {
+    if (interpreter_->ModifyGraphWithDelegate(delegates_[i].get()) !=
+        kTfLiteOk) {
+      ABSL_LOG(ERROR) << "Failed to apply delegate " << i;
+      return kTfLiteError;
+    }
+  }
+  TF_LITE_ENSURE_STATUS(interpreter_->AllocateTensors());
+  UpdateModelInfo();
+
+  return kTfLiteOk;
+}
+
+TfLiteStatus TfliteInferenceStage::Run() {
+  if (!interpreter_) {
+    ABSL_LOG(ERROR) << "Stage not initialized before calling Run";
+    return kTfLiteError;
+  }
+  if (!inputs_) {
+    ABSL_LOG(ERROR) << "Input data not set";
+    return kTfLiteError;
+  }
+  if (inputs_->size() != interpreter_->inputs().size()) {
+    ABSL_LOG(ERROR) << "Input data size does not match interpreter input size";
+    return kTfLiteError;
+  }
+
+  // Copy input data.
+  for (size_t i = 0; i < interpreter_->inputs().size(); ++i) {
+    TfLiteTensor* tensor = interpreter_->tensor(interpreter_->inputs()[i]);
+    if (tensor->type != kTfLiteString) {
+      size_t copy_bytes = tensor->bytes;
+      if (input_buffer_sizes_) {
+        if (i < input_buffer_sizes_->size()) {
+          copy_bytes = std::min(tensor->bytes, input_buffer_sizes_->at(i));
+        } else {
+          ABSL_LOG(ERROR) << "Input buffer size not provided for tensor index "
+                          << i;
+          return kTfLiteError;
+        }
+      }
+      std::memcpy(tensor->data.raw, inputs_->at(i), copy_bytes);
+    } else {
+      ABSL_LOG(WARNING)
+          << "Skipping input tensor of type kTfLiteString. String inputs are "
+             "not copied in this stage.";
+    }
+  }
+
+  // Invoke.
+  const TfliteInferenceParams& params =
+      config_.specification().tflite_inference_params();
+  for (int i = 0; i < params.invocations_per_run(); ++i) {
+    int64_t start_us = profiling::time::NowMicros();
+    if (interpreter_->Invoke() != kTfLiteOk) {
+      ABSL_LOG(ERROR) << "TFLite interpreter failed to invoke at run " << i;
+      return kTfLiteError;
+    }
+    latency_stats_.UpdateStat(profiling::time::NowMicros() - start_us);
+  }
+  UpdateModelInfo();
+
+  return kTfLiteOk;
+}
+
+EvaluationStageMetrics TfliteInferenceStage::LatestMetrics() {
+  const TfliteInferenceParams& params =
+      config_.specification().tflite_inference_params();
+  EvaluationStageMetrics metrics;
+  auto* latency_metrics =
+      metrics.mutable_process_metrics()->mutable_total_latency();
+  latency_metrics->set_last_us(latency_stats_.newest());
+  latency_metrics->set_max_us(latency_stats_.max());
+  latency_metrics->set_min_us(latency_stats_.min());
+  latency_metrics->set_sum_us(latency_stats_.sum());
+  latency_metrics->set_avg_us(latency_stats_.avg());
+  latency_metrics->set_std_deviation_us(latency_stats_.std_deviation());
+  metrics.set_num_runs(
+      static_cast<int>(latency_stats_.count() / params.invocations_per_run()));
+  auto* inference_metrics =
+      metrics.mutable_process_metrics()->mutable_tflite_inference_metrics();
+  inference_metrics->set_num_inferences(latency_stats_.count());
+  return metrics;
+}
+
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/stages/tflite_inference_stage.h

@@ -0,0 +1,106 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_TFLITE_INFERENCE_STAGE_H_
+#define TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_TFLITE_INFERENCE_STAGE_H_
+
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+#include "xla/tsl/util/stats_calculator.h"
+#include "tensorflow/lite/c/common.h"
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/model.h"
+#include "tensorflow/lite/model_builder.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_delegate_provider.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_stage.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+
+namespace tflite {
+namespace evaluation {
+
+struct TfLiteModelInfo {
+  std::vector<const TfLiteTensor*> inputs;
+  std::vector<const TfLiteTensor*> outputs;
+};
+
+// EvaluationStage to run inference using TFLite.
+class TfliteInferenceStage : public EvaluationStage {
+ public:
+  explicit TfliteInferenceStage(const EvaluationStageConfig& config)
+      : EvaluationStage(config) {}
+
+  TfLiteStatus Init() override { return Init(nullptr); }
+  TfLiteStatus Init(const DelegateProviders* delegate_providers);
+
+  TfLiteStatus Run() override;
+
+  // EvaluationStageMetrics.num_runs denotes the number of inferences run.
+  EvaluationStageMetrics LatestMetrics() override;
+
+  ~TfliteInferenceStage() override {}
+
+  // Call before Run().
+  // This class does not take ownership of raw_input_ptrs.
+  void SetInputs(const std::vector<void*>& raw_input_ptrs) {
+    inputs_ = &raw_input_ptrs;
+    input_buffer_sizes_ = nullptr;
+  }
+
+  // Call before Run().
+  // This class does not take ownership of raw_input_ptrs or input_buffer_sizes.
+  void SetInputs(const std::vector<void*>& raw_input_ptrs,
+                 const std::vector<size_t>& input_buffer_sizes) {
+    inputs_ = &raw_input_ptrs;
+    input_buffer_sizes_ = &input_buffer_sizes;
+  }
+
+  // Resize input tensors with given shapes.
+  TfLiteStatus ResizeInputs(const std::vector<std::vector<int>>& shapes);
+
+  // Applies provided delegate to the underlying TFLite Interpreter.
+  TfLiteStatus ApplyCustomDelegate(Interpreter::TfLiteDelegatePtr delegate);
+
+  // Read-only view of a TfliteModelInfo. TfliteInferenceStage retains
+  // ownership.
+  // Only available after Init is done.
+  const TfLiteModelInfo* GetModelInfo() const { return &model_info_; }
+
+  // Provides a read-only view to the model's output tensor(s). Retains
+  // ownership of object.
+  const std::vector<void*>* GetOutputs() const;
+
+ private:
+  // Sets model_info_ after interpreter tensors are (re)allocated.
+  void UpdateModelInfo();
+
+  std::unique_ptr<FlatBufferModel> model_;
+  std::unique_ptr<ops::builtin::BuiltinOpResolver> resolver_;
+  std::unique_ptr<Interpreter> interpreter_;
+  std::vector<Interpreter::TfLiteDelegatePtr> delegates_;
+
+  TfLiteModelInfo model_info_;
+  const std::vector<void*>* inputs_ = nullptr;
+  const std::vector<size_t>* input_buffer_sizes_ = nullptr;
+  mutable std::vector<void*> outputs_;
+
+  tsl::Stat<int64_t> latency_stats_;
+};
+
+}  // namespace evaluation
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_TFLITE_INFERENCE_STAGE_H_

tensorflow/lite/tools/evaluation/stages/tflite_inference_stage_test.cc

@@ -0,0 +1,216 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/evaluation/stages/tflite_inference_stage.h"
+
+#include <cstdint>
+#include <utility>
+#include <vector>
+
+#include <gtest/gtest.h>
+#include "absl/strings/string_view.h"
+#include "absl/types/span.h"
+#include "tensorflow/lite/c/common.h"
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+#include "tensorflow/lite/tools/evaluation/utils.h"
+
+namespace tflite {
+namespace evaluation {
+namespace {
+
+constexpr absl::string_view kTfliteInferenceStageName =
+    "tflite_inference_stage";
+constexpr absl::string_view kModelPath =
+    "tensorflow/lite/testdata/add_quantized.bin";
+constexpr int kTotalElements = 1 * 8 * 8 * 3;
+
+template <typename T>
+void SetValues(absl::Span<T> array, T value) {
+  for (T& element : array) {
+    element = value;
+  }
+}
+
+EvaluationStageConfig GetTfliteInferenceStageConfig() {
+  EvaluationStageConfig config;
+  config.set_name(kTfliteInferenceStageName);
+  auto* params =
+      config.mutable_specification()->mutable_tflite_inference_params();
+  params->set_model_file_path(kModelPath);
+  params->set_invocations_per_run(2);
+  return config;
+}
+
+TEST(TfliteInferenceStage, NoParams) {
+  EvaluationStageConfig config = GetTfliteInferenceStageConfig();
+  config.mutable_specification()->clear_tflite_inference_params();
+  TfliteInferenceStage stage(config);
+  EXPECT_EQ(stage.Init(), kTfLiteError);
+}
+
+TEST(TfliteInferenceStage, NoModelPath) {
+  EvaluationStageConfig config = GetTfliteInferenceStageConfig();
+  config.mutable_specification()
+      ->mutable_tflite_inference_params()
+      ->clear_model_file_path();
+  TfliteInferenceStage stage(config);
+  EXPECT_EQ(stage.Init(), kTfLiteError);
+}
+
+TEST(TfliteInferenceStage, IncorrectModelPath) {
+  EvaluationStageConfig config = GetTfliteInferenceStageConfig();
+  config.mutable_specification()
+      ->mutable_tflite_inference_params()
+      ->set_model_file_path("xyz.tflite");
+  TfliteInferenceStage stage(config);
+  EXPECT_EQ(stage.Init(), kTfLiteError);
+}
+
+TEST(TfliteInferenceStage, NoInputData) {
+  // Create stage.
+  EvaluationStageConfig config = GetTfliteInferenceStageConfig();
+  TfliteInferenceStage stage(config);
+
+  // Initialize.
+  EXPECT_EQ(stage.Init(), kTfLiteOk);
+
+  // Run.
+  EXPECT_EQ(stage.Run(), kTfLiteError);
+}
+
+TEST(TfliteInferenceStage, CorrectModelInfo) {
+  // Create stage.
+  EvaluationStageConfig config = GetTfliteInferenceStageConfig();
+  TfliteInferenceStage stage(config);
+
+  // Initialize.
+  EXPECT_EQ(stage.Init(), kTfLiteOk);
+
+  const TfLiteModelInfo* model_info = stage.GetModelInfo();
+  // Verify Input
+  EXPECT_EQ(model_info->inputs.size(), 1);
+  const TfLiteTensor* tensor = model_info->inputs[0];
+  EXPECT_EQ(tensor->type, kTfLiteUInt8);
+  EXPECT_EQ(tensor->bytes, kTotalElements);
+  const TfLiteIntArray* input_shape = tensor->dims;
+  EXPECT_EQ(input_shape->data[0], 1);
+  EXPECT_EQ(input_shape->data[1], 8);
+  EXPECT_EQ(input_shape->data[2], 8);
+  EXPECT_EQ(input_shape->data[3], 3);
+  // Verify Output
+  EXPECT_EQ(model_info->outputs.size(), 1);
+  tensor = model_info->outputs[0];
+  EXPECT_EQ(tensor->type, kTfLiteUInt8);
+  EXPECT_EQ(tensor->bytes, kTotalElements);
+  const TfLiteIntArray* output_shape = tensor->dims;
+  EXPECT_EQ(output_shape->data[0], 1);
+  EXPECT_EQ(output_shape->data[1], 8);
+  EXPECT_EQ(output_shape->data[2], 8);
+  EXPECT_EQ(output_shape->data[3], 3);
+}
+
+TEST(TfliteInferenceStage, TestResizeModel) {
+  // Create stage.
+  EvaluationStageConfig config = GetTfliteInferenceStageConfig();
+  TfliteInferenceStage stage(config);
+
+  // Initialize.
+  EXPECT_EQ(stage.Init(), kTfLiteOk);
+
+  // Resize.
+  EXPECT_EQ(stage.ResizeInputs({{3, 8, 8, 3}}), kTfLiteOk);
+
+  const TfLiteModelInfo* model_info = stage.GetModelInfo();
+  // Verify Input
+  EXPECT_EQ(model_info->inputs.size(), 1);
+  const TfLiteTensor* tensor = model_info->inputs[0];
+  EXPECT_EQ(tensor->type, kTfLiteUInt8);
+  EXPECT_EQ(tensor->bytes, 3 * kTotalElements);
+  const TfLiteIntArray* input_shape = tensor->dims;
+  EXPECT_EQ(input_shape->data[0], 3);
+  EXPECT_EQ(input_shape->data[1], 8);
+  EXPECT_EQ(input_shape->data[2], 8);
+  EXPECT_EQ(input_shape->data[3], 3);
+  // Verify Output
+  EXPECT_EQ(model_info->outputs.size(), 1);
+  tensor = model_info->outputs[0];
+  EXPECT_EQ(tensor->type, kTfLiteUInt8);
+  EXPECT_EQ(tensor->bytes, 3 * kTotalElements);
+  const TfLiteIntArray* output_shape = tensor->dims;
+  EXPECT_EQ(output_shape->data[0], 3);
+  EXPECT_EQ(output_shape->data[1], 8);
+  EXPECT_EQ(output_shape->data[2], 8);
+  EXPECT_EQ(output_shape->data[3], 3);
+}
+
+TEST(TfliteInferenceStage, CorrectOutput) {
+  // Create stage.
+  EvaluationStageConfig config = GetTfliteInferenceStageConfig();
+  TfliteInferenceStage stage(config);
+
+  // Initialize.
+  EXPECT_EQ(stage.Init(), kTfLiteOk);
+
+  // Set input data.
+  uint8_t input_tensor[kTotalElements];
+  SetValues<uint8_t>(input_tensor, static_cast<uint8_t>(2));
+  std::vector<void*> inputs;
+  inputs.push_back(input_tensor);
+  stage.SetInputs(inputs);
+
+  // Run.
+  EXPECT_EQ(stage.Run(), kTfLiteOk);
+
+  // Verify outputs.
+  uint8_t* output_tensor = static_cast<uint8_t*>(stage.GetOutputs()->at(0));
+  for (int i = 0; i < kTotalElements; i++) {
+    EXPECT_EQ(output_tensor[i], static_cast<uint8_t>(6));
+  }
+
+  // Verify metrics.
+  EvaluationStageMetrics metrics = stage.LatestMetrics();
+  EXPECT_EQ(metrics.num_runs(), 1);
+  const tflite::evaluation::LatencyMetrics& latency =
+      metrics.process_metrics().total_latency();
+  const int64_t max_latency = latency.max_us();
+  EXPECT_GT(max_latency, 0);
+  EXPECT_LT(max_latency, 1e7);
+  EXPECT_LE(latency.last_us(), max_latency);
+  EXPECT_LE(latency.min_us(), max_latency);
+  EXPECT_GE(latency.sum_us(), max_latency);
+  EXPECT_LE(latency.avg_us(), max_latency);
+  EXPECT_TRUE(latency.has_std_deviation_us());
+  EXPECT_EQ(
+      metrics.process_metrics().tflite_inference_metrics().num_inferences(), 2);
+}
+
+TEST(TfliteInferenceStage, CustomDelegate) {
+  // Create stage.
+  EvaluationStageConfig config = GetTfliteInferenceStageConfig();
+  TfliteInferenceStage stage(config);
+
+  Interpreter::TfLiteDelegatePtr test_delegate = CreateNNAPIDelegate();
+
+  // Delegate application should only work after initialization of stage.
+  EXPECT_NE(stage.ApplyCustomDelegate(std::move(test_delegate)), kTfLiteOk);
+  EXPECT_EQ(stage.Init(), kTfLiteOk);
+  Interpreter::TfLiteDelegatePtr test_delegate2 = CreateNNAPIDelegate();
+  EXPECT_EQ(stage.ApplyCustomDelegate(std::move(test_delegate2)), kTfLiteOk);
+}
+
+}  // namespace
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/stages/topk_accuracy_eval_stage.cc

@@ -0,0 +1,144 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/evaluation/stages/topk_accuracy_eval_stage.h"
+
+#include <stdint.h>
+
+#include <algorithm>
+#include <cstddef>
+#include <numeric>
+#include <vector>
+
+#include "absl/log/log.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/lite/c/c_api_types.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+
+namespace tflite {
+namespace evaluation {
+namespace {
+
+std::vector<int> GetTopKIndices(const std::vector<float>& values, int k) {
+  std::vector<int> indices(values.size());
+  std::iota(indices.begin(), indices.end(), 0);
+  std::stable_sort(indices.begin(), indices.end(),
+                   [&values](int a, int b) { return values[a] > values[b]; });
+  indices.resize(k);
+  return indices;
+}
+
+}  // namespace
+
+TfLiteStatus TopkAccuracyEvalStage::Init() {
+  num_runs_ = 0;
+  auto& params = config_.specification().topk_accuracy_eval_params();
+  if (!params.has_k()) {
+    LOG(ERROR) << "Value of k not provided for TopkAccuracyEvalStage";
+    return kTfLiteError;
+  }
+  accuracy_counts_ = std::vector<int>(params.k(), 0);
+
+  if (ground_truth_labels_.empty()) {
+    LOG(ERROR) << "Ground-truth labels are empty";
+    return kTfLiteError;
+  }
+  num_total_labels_ = ground_truth_labels_.size();
+  if (params.k() > num_total_labels_) {
+    LOG(ERROR) << "k is too large";
+    return kTfLiteError;
+  }
+
+  if (!model_output_shape_) {
+    LOG(ERROR) << "Model output details not correctly set";
+    return kTfLiteError;
+  }
+  // Ensure model output is of shape (1, num_total_labels_).
+  if (!(model_output_shape_->size == 2) ||
+      !(model_output_shape_->data[0] == 1) ||
+      !(model_output_shape_->data[1] == num_total_labels_)) {
+    LOG(ERROR) << "Invalid model_output_shape_";
+    return kTfLiteError;
+  }
+  if (model_output_type_ != kTfLiteFloat32 &&
+      model_output_type_ != kTfLiteUInt8 && model_output_type_ != kTfLiteInt8) {
+    LOG(ERROR) << "model_output_type_ not supported";
+    return kTfLiteError;
+  }
+  return kTfLiteOk;
+}
+
+TfLiteStatus TopkAccuracyEvalStage::Run() {
+  if (!model_output_) {
+    LOG(ERROR) << "model_output_ not set correctly";
+    return kTfLiteError;
+  }
+  if (!ground_truth_label_) {
+    LOG(ERROR) << "ground_truth_label_ not provided";
+    return kTfLiteError;
+  }
+  auto& params = config_.specification().topk_accuracy_eval_params();
+
+  std::vector<float> probabilities;
+  probabilities.reserve(num_total_labels_);
+  if (model_output_type_ == kTfLiteFloat32) {
+    auto probs = static_cast<float*>(model_output_);
+    for (size_t i = 0; i < num_total_labels_; i++) {
+      probabilities.push_back(probs[i]);
+    }
+  } else if (model_output_type_ == kTfLiteUInt8) {
+    auto probs = static_cast<uint8_t*>(model_output_);
+    for (size_t i = 0; i < num_total_labels_; i++) {
+      probabilities.push_back(probs[i]);
+    }
+  } else if (model_output_type_ == kTfLiteInt8) {
+    auto probs = static_cast<int8_t*>(model_output_);
+    for (size_t i = 0; i < num_total_labels_; i++) {
+      probabilities.push_back(probs[i]);
+    }
+  }
+
+  std::vector<int> top_k = GetTopKIndices(probabilities, params.k());
+  UpdateCounts(top_k);
+  return kTfLiteOk;
+}
+
+EvaluationStageMetrics TopkAccuracyEvalStage::LatestMetrics() {
+  EvaluationStageMetrics metrics;
+  if (num_runs_ == 0) return metrics;
+
+  metrics.set_num_runs(num_runs_);
+  auto* topk_metrics =
+      metrics.mutable_process_metrics()->mutable_topk_accuracy_metrics();
+  for (const auto& count : accuracy_counts_) {
+    topk_metrics->add_topk_accuracies(static_cast<float>(count) / num_runs_);
+  }
+  return metrics;
+}
+
+void TopkAccuracyEvalStage::UpdateCounts(const std::vector<int>& topk_indices) {
+  for (size_t i = 0; i < topk_indices.size(); ++i) {
+    if (*ground_truth_label_ == ground_truth_labels_[topk_indices[i]]) {
+      for (size_t j = i; j < topk_indices.size(); j++) {
+        accuracy_counts_[j] += 1;
+      }
+      break;
+    }
+  }
+  num_runs_++;
+}
+
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/stages/topk_accuracy_eval_stage.h

@@ -0,0 +1,89 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_TOPK_ACCURACY_EVAL_STAGE_H_
+#define TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_TOPK_ACCURACY_EVAL_STAGE_H_
+
+#include <string>
+#include <vector>
+
+#include "tensorflow/lite/c/c_api_types.h"
+#include "tensorflow/lite/c/common.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_stage.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+
+namespace tflite {
+namespace evaluation {
+
+// EvaluationStage to compute top-K accuracy of a classification model.
+// The computed weights in the model output should be in the same order
+// as the vector provided during SetAllLabels
+// Ground truth label must be one of provided labels.
+// Current accuracies can be obtained with GetLatestMetrics().
+class TopkAccuracyEvalStage : public EvaluationStage {
+ public:
+  explicit TopkAccuracyEvalStage(const EvaluationStageConfig& config)
+      : EvaluationStage(config) {}
+
+  TfLiteStatus Init() override;
+
+  TfLiteStatus Run() override;
+
+  EvaluationStageMetrics LatestMetrics() override;
+
+  ~TopkAccuracyEvalStage() override {}
+
+  // Call before Init().
+  // model_output_shape is not owned, so this class does not free the
+  // TfLiteIntArray.
+  void SetTaskInfo(const std::vector<std::string>& all_labels,
+                   TfLiteType model_output_type,
+                   TfLiteIntArray* model_output_shape) {
+    // We copy ground_truth_labels to ensure we can access the data throughout
+    // the lifetime of this evaluation stage.
+    ground_truth_labels_ = all_labels;
+    model_output_type_ = model_output_type;
+    model_output_shape_ = model_output_shape;
+  }
+
+  // Call before Run().
+  void SetEvalInputs(void* model_raw_output, std::string* ground_truth_label) {
+    model_output_ = model_raw_output;
+    ground_truth_label_ = ground_truth_label;
+  }
+
+ private:
+  // Updates accuracy_counts_ based on comparing top k labels and the
+  // groundtruth one. Using string comparison since there are some duplicate
+  // labels in the imagenet dataset.
+  void UpdateCounts(const std::vector<int>& topk_indices);
+
+  std::vector<std::string> ground_truth_labels_;
+  TfLiteType model_output_type_ = kTfLiteNoType;
+  TfLiteIntArray* model_output_shape_ = nullptr;
+  int num_total_labels_;
+  void* model_output_ = nullptr;
+  std::string* ground_truth_label_ = nullptr;
+
+  // Equal to number of samples evaluated so far.
+  int num_runs_;
+  // Stores |k_| values, where the ith value denotes number of samples (out of
+  // num_runs_) for which correct label appears in the top (i+1) model outputs.
+  std::vector<int> accuracy_counts_;
+};
+
+}  // namespace evaluation
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_TOPK_ACCURACY_EVAL_STAGE_H_

tensorflow/lite/tools/evaluation/stages/topk_accuracy_eval_stage_test.cc

@@ -0,0 +1,316 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/evaluation/stages/topk_accuracy_eval_stage.h"
+
+#include <stdint.h>
+
+#include <string>
+#include <vector>
+
+#include <gtest/gtest.h>
+#include "tensorflow/lite/c/c_api_types.h"
+#include "tensorflow/lite/c/common.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+
+namespace tflite {
+namespace evaluation {
+namespace {
+
+constexpr char kTopkAccuracyEvalStageName[] = "topk_accuracy_eval_stage";
+constexpr int kNumCategories = 1001;
+
+EvaluationStageConfig GetTopkAccuracyEvalStageConfig() {
+  EvaluationStageConfig config;
+  config.set_name(kTopkAccuracyEvalStageName);
+  auto* params =
+      config.mutable_specification()->mutable_topk_accuracy_eval_params();
+  params->set_k(5);
+  return config;
+}
+
+template <typename T>
+T* ResetOutputArray(T array[]) {
+  for (int i = 0; i < kNumCategories; i++) {
+    array[i] = 0;
+  }
+  return array;
+}
+
+std::vector<std::string> CreateGroundTruthLabels() {
+  std::vector<std::string> ground_truth_labels;
+  ground_truth_labels.reserve(kNumCategories);
+  for (int i = 0; i < kNumCategories; i++) {
+    ground_truth_labels.push_back(std::to_string(i));
+  }
+  return ground_truth_labels;
+}
+
+TEST(TopkAccuracyEvalStage, NoInitializers) {
+  // Create stage.
+  EvaluationStageConfig config = GetTopkAccuracyEvalStageConfig();
+  TopkAccuracyEvalStage stage = TopkAccuracyEvalStage(config);
+
+  EXPECT_EQ(stage.Init(), kTfLiteError);
+}
+
+TEST(TopkAccuracyEvalStage, NoK) {
+  // Create stage.
+  EvaluationStageConfig config = GetTopkAccuracyEvalStageConfig();
+  config.mutable_specification()
+      ->mutable_topk_accuracy_eval_params()
+      ->clear_k();
+  TopkAccuracyEvalStage stage = TopkAccuracyEvalStage(config);
+
+  // Initialize.
+  std::vector<std::string> ground_truth_labels = CreateGroundTruthLabels();
+  TfLiteIntArray* model_output_shape = TfLiteIntArrayCreate(2);
+  model_output_shape->data[0] = 1;
+  model_output_shape->data[1] = kNumCategories;
+  TfLiteType model_output_type = kTfLiteFloat32;
+  stage.SetTaskInfo(ground_truth_labels, model_output_type, model_output_shape);
+  EXPECT_EQ(stage.Init(), kTfLiteError);
+  TfLiteIntArrayFree(model_output_shape);
+}
+
+TEST(TopkAccuracyEvalStage, NoGroundTruthLabels) {
+  // Create stage.
+  EvaluationStageConfig config = GetTopkAccuracyEvalStageConfig();
+  TopkAccuracyEvalStage stage = TopkAccuracyEvalStage(config);
+
+  // Initialize.
+  std::vector<std::string> ground_truth_labels = {};
+  TfLiteIntArray* model_output_shape = TfLiteIntArrayCreate(2);
+  model_output_shape->data[0] = 1;
+  model_output_shape->data[1] = kNumCategories;
+  TfLiteType model_output_type = kTfLiteFloat32;
+  stage.SetTaskInfo(ground_truth_labels, model_output_type, model_output_shape);
+  EXPECT_EQ(stage.Init(), kTfLiteError);
+  TfLiteIntArrayFree(model_output_shape);
+}
+
+TEST(TopkAccuracyEvalStage, KTooLarge) {
+  // Create stage.
+  EvaluationStageConfig config = GetTopkAccuracyEvalStageConfig();
+  config.mutable_specification()->mutable_topk_accuracy_eval_params()->set_k(
+      10000);
+  TopkAccuracyEvalStage stage = TopkAccuracyEvalStage(config);
+
+  // Initialize.
+  std::vector<std::string> ground_truth_labels = CreateGroundTruthLabels();
+  TfLiteIntArray* model_output_shape = TfLiteIntArrayCreate(2);
+  model_output_shape->data[0] = 1;
+  model_output_shape->data[1] = kNumCategories;
+  TfLiteType model_output_type = kTfLiteFloat32;
+  stage.SetTaskInfo(ground_truth_labels, model_output_type, model_output_shape);
+  EXPECT_EQ(stage.Init(), kTfLiteError);
+  TfLiteIntArrayFree(model_output_shape);
+}
+
+TEST(TopkAccuracyEvalStage, WeirdModelOutputShape) {
+  // Create stage.
+  EvaluationStageConfig config = GetTopkAccuracyEvalStageConfig();
+  TopkAccuracyEvalStage stage = TopkAccuracyEvalStage(config);
+
+  // Initialize.
+  std::vector<std::string> ground_truth_labels = CreateGroundTruthLabels();
+  TfLiteIntArray* model_output_shape = TfLiteIntArrayCreate(2);
+  model_output_shape->data[0] = 1;
+  model_output_shape->data[1] = kNumCategories + 1;
+  TfLiteType model_output_type = kTfLiteFloat32;
+  stage.SetTaskInfo(ground_truth_labels, model_output_type, model_output_shape);
+  EXPECT_EQ(stage.Init(), kTfLiteError);
+  TfLiteIntArrayFree(model_output_shape);
+}
+
+TEST(TopkAccuracyEvalStage, UnsupportedModelOutputType) {
+  // Create stage.
+  EvaluationStageConfig config = GetTopkAccuracyEvalStageConfig();
+  TopkAccuracyEvalStage stage = TopkAccuracyEvalStage(config);
+
+  // Initialize.
+  std::vector<std::string> ground_truth_labels = CreateGroundTruthLabels();
+  TfLiteIntArray* model_output_shape = TfLiteIntArrayCreate(2);
+  model_output_shape->data[0] = 1;
+  model_output_shape->data[1] = kNumCategories + 1;
+  TfLiteType model_output_type = kTfLiteComplex64;
+  stage.SetTaskInfo(ground_truth_labels, model_output_type, model_output_shape);
+  EXPECT_EQ(stage.Init(), kTfLiteError);
+  TfLiteIntArrayFree(model_output_shape);
+}
+
+TEST(TopkAccuracyEvalStage, NoInputs) {
+  // Create stage.
+  EvaluationStageConfig config = GetTopkAccuracyEvalStageConfig();
+  TopkAccuracyEvalStage stage = TopkAccuracyEvalStage(config);
+  // Initialize.
+  std::vector<std::string> ground_truth_labels = CreateGroundTruthLabels();
+  TfLiteIntArray* model_output_shape = TfLiteIntArrayCreate(2);
+  model_output_shape->data[0] = 1;
+  model_output_shape->data[1] = kNumCategories;
+  TfLiteType model_output_type = kTfLiteFloat32;
+  stage.SetTaskInfo(ground_truth_labels, model_output_type, model_output_shape);
+  EXPECT_EQ(stage.Init(), kTfLiteOk);
+  TfLiteIntArrayFree(model_output_shape);
+
+  EXPECT_EQ(stage.Run(), kTfLiteError);
+}
+
+TEST(TopkAccuracyEvalStage, InvalidGroundTruth) {
+  // Create stage.
+  EvaluationStageConfig config = GetTopkAccuracyEvalStageConfig();
+  TopkAccuracyEvalStage stage = TopkAccuracyEvalStage(config);
+  // Initialize.
+  std::vector<std::string> ground_truth_labels = CreateGroundTruthLabels();
+  TfLiteIntArray* model_output_shape = TfLiteIntArrayCreate(2);
+  model_output_shape->data[0] = 1;
+  model_output_shape->data[1] = kNumCategories;
+  TfLiteType model_output_type = kTfLiteFloat32;
+  stage.SetTaskInfo(ground_truth_labels, model_output_type, model_output_shape);
+  EXPECT_EQ(stage.Init(), kTfLiteOk);
+  TfLiteIntArrayFree(model_output_shape);
+
+  float array[kNumCategories];
+  float* tensor = ResetOutputArray(array);
+  tensor[0] = 0.8;
+  stage.SetEvalInputs(tensor, /*ground_truth_label=*/nullptr);
+  EXPECT_EQ(stage.Run(), kTfLiteError);
+}
+
+TEST(TopkAccuracyEvalStage, FloatTest_CorrectLabelsAtLastIndices) {
+  // Create stage.
+  EvaluationStageConfig config = GetTopkAccuracyEvalStageConfig();
+  TopkAccuracyEvalStage stage = TopkAccuracyEvalStage(config);
+  // Initialize.
+  std::vector<std::string> ground_truth_labels = CreateGroundTruthLabels();
+  TfLiteIntArray* model_output_shape = TfLiteIntArrayCreate(2);
+  model_output_shape->data[0] = 1;
+  model_output_shape->data[1] = kNumCategories;
+  TfLiteType model_output_type = kTfLiteFloat32;
+  stage.SetTaskInfo(ground_truth_labels, model_output_type, model_output_shape);
+  EXPECT_EQ(stage.Init(), kTfLiteOk);
+  TfLiteIntArrayFree(model_output_shape);
+
+  float array[kNumCategories];
+
+  // The ground truth is index 0, but it is 5th most likely based on model's
+  // output.
+  float* tensor = ResetOutputArray(array);
+  tensor[4] = 0.9;
+  tensor[3] = 0.8;
+  tensor[2] = 0.7;
+  tensor[1] = 0.6;
+  tensor[0] = 0.5;
+  std::string ground_truth = "0";
+  stage.SetEvalInputs(tensor, &ground_truth);
+  EXPECT_EQ(stage.Run(), kTfLiteOk);
+  EvaluationStageMetrics metrics = stage.LatestMetrics();
+  EXPECT_EQ(1, metrics.num_runs());
+  auto accuracy_metrics = metrics.process_metrics().topk_accuracy_metrics();
+  // Only top-5 count is 1.0, rest are 0.0
+  EXPECT_FLOAT_EQ(1.0, accuracy_metrics.topk_accuracies(4));
+  for (int i = 0; i < 4; ++i) {
+    EXPECT_FLOAT_EQ(0.0, accuracy_metrics.topk_accuracies(i));
+  }
+
+  // The ground truth is index 1, but it is 4th highest based on model's output.
+  ground_truth = "1";
+  stage.SetEvalInputs(tensor, &ground_truth);
+  EXPECT_EQ(stage.Run(), kTfLiteOk);
+  metrics = stage.LatestMetrics();
+  EXPECT_EQ(2, metrics.num_runs());
+  accuracy_metrics = metrics.process_metrics().topk_accuracy_metrics();
+  // 1/2 images had the currect output in top-4, 2/2 has currect output in
+  // top-5.
+  EXPECT_FLOAT_EQ(1.0, accuracy_metrics.topk_accuracies(4));
+  EXPECT_FLOAT_EQ(0.5, accuracy_metrics.topk_accuracies(3));
+  for (int i = 0; i < 3; ++i) {
+    EXPECT_FLOAT_EQ(0.0, accuracy_metrics.topk_accuracies(i));
+  }
+}
+
+class CorrectTopkAccuracyEvalTest : public ::testing::Test {
+ protected:
+  template <typename T>
+  void VerifyCorrectBehaviorForType(T ground_truth_0_value,
+                                    T ground_truth_1_value,
+                                    TfLiteType model_output_type) {
+    // Create stage.
+    EvaluationStageConfig config = GetTopkAccuracyEvalStageConfig();
+    TopkAccuracyEvalStage stage = TopkAccuracyEvalStage(config);
+    // Initialize.
+    std::vector<std::string> ground_truth_labels = CreateGroundTruthLabels();
+    TfLiteIntArray* model_output_shape = TfLiteIntArrayCreate(2);
+    model_output_shape->data[0] = 1;
+    model_output_shape->data[1] = kNumCategories;
+    stage.SetTaskInfo(ground_truth_labels, model_output_type,
+                      model_output_shape);
+    EXPECT_EQ(stage.Init(), kTfLiteOk);
+    TfLiteIntArrayFree(model_output_shape);
+
+    // Pre-run state.
+    EvaluationStageMetrics metrics = stage.LatestMetrics();
+    EXPECT_EQ(0, metrics.num_runs());
+    auto accuracy_metrics = metrics.process_metrics().topk_accuracy_metrics();
+    EXPECT_EQ(0, accuracy_metrics.topk_accuracies_size());
+
+    T array[kNumCategories];
+
+    // First image was correctly identified as "0".
+    T* tensor = ResetOutputArray(array);
+    tensor[0] = ground_truth_0_value;
+    std::string ground_truth = "0";
+    stage.SetEvalInputs(tensor, &ground_truth);
+    EXPECT_EQ(stage.Run(), kTfLiteOk);
+    metrics = stage.LatestMetrics();
+    EXPECT_EQ(1, metrics.num_runs());
+    accuracy_metrics = metrics.process_metrics().topk_accuracy_metrics();
+    for (int i = 0; i < accuracy_metrics.topk_accuracies_size(); ++i) {
+      EXPECT_FLOAT_EQ(1.0, accuracy_metrics.topk_accuracies(i));
+    }
+
+    // Second image was also correctly identified as "1".
+    // Hence, for the second image as well, the top output ("1") was correct.
+    tensor[1] = ground_truth_1_value;
+    ground_truth = "1";
+    stage.SetEvalInputs(tensor, &ground_truth);
+    EXPECT_EQ(stage.Run(), kTfLiteOk);
+    metrics = stage.LatestMetrics();
+    EXPECT_EQ(2, metrics.num_runs());
+    accuracy_metrics = metrics.process_metrics().topk_accuracy_metrics();
+    for (int i = 0; i < accuracy_metrics.topk_accuracies_size(); ++i) {
+      EXPECT_FLOAT_EQ(1.0, accuracy_metrics.topk_accuracies(i));
+    }
+  }
+};
+
+TEST_F(CorrectTopkAccuracyEvalTest, FloatTest) {
+  VerifyCorrectBehaviorForType(static_cast<float>(0.8), static_cast<float>(0.9),
+                               kTfLiteFloat32);
+}
+
+TEST_F(CorrectTopkAccuracyEvalTest, Int8Test) {
+  VerifyCorrectBehaviorForType(static_cast<int8_t>(1), static_cast<int8_t>(2),
+                               kTfLiteInt8);
+}
+
+TEST_F(CorrectTopkAccuracyEvalTest, UInt8Test) {
+  VerifyCorrectBehaviorForType(static_cast<uint8_t>(1), static_cast<uint8_t>(2),
+                               kTfLiteUInt8);
+}
+
+}  // namespace
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/stages/utils/BUILD

@@ -0,0 +1,47 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
+load("@rules_cc//cc:cc_test.bzl", "cc_test")
+load("//tensorflow/lite:build_def.bzl", "tflite_copts", "tflite_linkopts")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_visibility = ["//tensorflow/lite:__subpackages__"],
+    licenses = ["notice"],
+)
+
+cc_library(
+    name = "image_metrics",
+    srcs = ["image_metrics.cc"],
+    hdrs = ["image_metrics.h"],
+    copts = tflite_copts(),
+    deps = [
+        "//tensorflow/core:tflite_portable_logging",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/log",
+    ],
+)
+
+cc_test(
+    name = "image_metrics_test",
+    srcs = ["image_metrics_test.cc"],
+    linkopts = tflite_linkopts(),
+    linkstatic = 1,
+    deps = [
+        ":image_metrics",
+        "@com_google_googletest//:gtest_main",
+    ],
+)

tensorflow/lite/tools/evaluation/stages/utils/image_metrics.cc

@@ -0,0 +1,181 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/evaluation/stages/utils/image_metrics.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <list>
+#include <vector>
+
+#include "absl/container/flat_hash_map.h"
+#include "absl/log/log.h"
+#include "tensorflow/core/platform/logging.h"
+
+namespace tflite {
+namespace evaluation {
+namespace image {
+
+float Box2D::Length(const Box2D::Interval& a) {
+  return std::max(0.f, a.max - a.min);
+}
+
+float Box2D::Intersection(const Box2D::Interval& a, const Box2D::Interval& b) {
+  return Length(Interval{std::max(a.min, b.min), std::min(a.max, b.max)});
+}
+
+float Box2D::Area() const { return Length(x) * Length(y); }
+
+float Box2D::Intersection(const Box2D& other) const {
+  return Intersection(x, other.x) * Intersection(y, other.y);
+}
+
+float Box2D::Union(const Box2D& other) const {
+  return Area() + other.Area() - Intersection(other);
+}
+
+float Box2D::IoU(const Box2D& other) const {
+  const float total = Union(other);
+  if (total > 0) {
+    return Intersection(other) / total;
+  } else {
+    return 0.0;
+  }
+}
+
+float Box2D::Overlap(const Box2D& other) const {
+  const float intersection = Intersection(other);
+  return intersection > 0 ? intersection / Area() : 0.0;
+}
+
+float AveragePrecision::FromPRCurve(const std::vector<PR>& pr,
+                                    std::vector<PR>* pr_out) {
+  // Because pr[...] are ordered by recall, iterate backward to compute max
+  // precision. p(r) = max_{r' >= r} p(r') for r in 0.0, 0.1, 0.2, ..., 0.9,
+  // 1.0. Then, take the average of (num_recal_points) quantities.
+  float p = 0;
+  float sum = 0;
+  int r_level = opts_.num_recall_points;
+  for (int i = pr.size() - 1; i >= 0; --i) {
+    const PR& item = pr[i];
+    if (i > 0) {
+      if (item.r < pr[i - 1].r) {
+        LOG(ERROR) << "recall points are not in order: " << pr[i - 1].r << ", "
+                   << item.r;
+        return 0;
+      }
+    }
+
+    // Because r takes values opts_.num_recall_points, opts_.num_recall_points -
+    // 1, ..., 0, the following condition is checking whether item.r crosses r /
+    // opts_.num_recall_points. I.e., 1.0, 0.90, ..., 0.01, 0.0.  We don't use
+    // float to represent r because 0.01 is not representable precisely.
+    while (item.r * opts_.num_recall_points < r_level) {
+      const float recall =
+          static_cast<float>(r_level) / opts_.num_recall_points;
+      if (r_level < 0) {
+        LOG(ERROR) << "Number of recall points should be > 0";
+        return 0;
+      }
+      sum += p;
+      r_level -= 1;
+      if (pr_out != nullptr) {
+        pr_out->emplace_back(p, recall);
+      }
+    }
+    p = std::max(p, item.p);
+  }
+  for (; r_level >= 0; --r_level) {
+    const float recall = static_cast<float>(r_level) / opts_.num_recall_points;
+    sum += p;
+    if (pr_out != nullptr) {
+      pr_out->emplace_back(p, recall);
+    }
+  }
+  return sum / (1 + opts_.num_recall_points);
+}
+
+float AveragePrecision::FromBoxes(const std::vector<Detection>& groundtruth,
+                                  const std::vector<Detection>& prediction,
+                                  std::vector<PR>* pr_out) {
+  // Index ground truth boxes based on imageid.
+  absl::flat_hash_map<int64_t, std::list<Detection>> gt;
+  int num_gt = 0;
+  for (auto& box : groundtruth) {
+    gt[box.imgid].push_back(box);
+    if (!box.difficult && box.ignore == kDontIgnore) {
+      ++num_gt;
+    }
+  }
+
+  if (num_gt == 0) {
+    return NAN;
+  }
+
+  // Sort all predicted boxes by their scores in a non-ascending order.
+  std::vector<Detection> pd = prediction;
+  std::sort(pd.begin(), pd.end(), [](const Detection& a, const Detection& b) {
+    return a.score > b.score;
+  });
+
+  // Computes p-r for every prediction.
+  std::vector<PR> pr;
+  int correct = 0;
+  int num_pd = 0;
+  for (int i = 0; i < pd.size(); ++i) {
+    const Detection& b = pd[i];
+    auto* g = &gt[b.imgid];
+    auto best = g->end();
+    float best_iou = -INFINITY;
+    for (auto it = g->begin(); it != g->end(); ++it) {
+      const auto iou = b.box.IoU(it->box);
+      if (iou > best_iou) {
+        best = it;
+        best_iou = iou;
+      }
+    }
+    if ((best != g->end()) && (best_iou >= opts_.iou_threshold)) {
+      if (best->difficult) {
+        continue;
+      }
+      switch (best->ignore) {
+        case kDontIgnore: {
+          ++correct;
+          ++num_pd;
+          g->erase(best);
+          pr.push_back({static_cast<float>(correct) / num_pd,
+                        static_cast<float>(correct) / num_gt});
+          break;
+        }
+        case kIgnoreOneMatch: {
+          g->erase(best);
+          break;
+        }
+        case kIgnoreAllMatches: {
+          break;
+        }
+      }
+    } else {
+      ++num_pd;
+      pr.push_back({static_cast<float>(correct) / num_pd,
+                    static_cast<float>(correct) / num_gt});
+    }
+  }
+  return FromPRCurve(pr, pr_out);
+}
+
+}  // namespace image
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/stages/utils/image_metrics.h

@@ -0,0 +1,132 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_UTILS_IMAGE_METRICS_H_
+#define TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_UTILS_IMAGE_METRICS_H_
+
+#include <stdint.h>
+
+#include <vector>
+
+namespace tflite {
+namespace evaluation {
+namespace image {
+
+struct Box2D {
+  struct Interval {
+    float min = 0;
+    float max = 0;
+    Interval(float x, float y) {
+      min = x;
+      max = y;
+    }
+    Interval() {}
+  };
+
+  Interval x;
+  Interval y;
+  static float Length(const Interval& a);
+  static float Intersection(const Interval& a, const Interval& b);
+  float Area() const;
+  float Intersection(const Box2D& other) const;
+  float Union(const Box2D& other) const;
+  // Intersection of this box and the given box normalized over the union of
+  // this box and the given box.
+  float IoU(const Box2D& other) const;
+  // Intersection of this box and the given box normalized over the area of
+  // this box.
+  float Overlap(const Box2D& other) const;
+};
+
+// If the value is:
+//   - kDontIgnore: The object is included in this evaluation.
+//   - kIgnoreOneMatch: the first matched prediction bbox will be ignored. This
+//      is useful when this groundtruth object is not intended to be evaluated.
+//   - kIgnoreAllMatches: all matched prediction bbox will be ignored. Typically
+//      it is used to mark an area that has not been labeled.
+enum IgnoreType {
+  kDontIgnore = 0,
+  kIgnoreOneMatch = 1,
+  kIgnoreAllMatches = 2,
+};
+
+struct Detection {
+ public:
+  bool difficult = false;
+  int64_t imgid = 0;
+  float score = 0;
+  Box2D box;
+  IgnoreType ignore = IgnoreType::kDontIgnore;
+
+  Detection() {}
+  Detection(bool d, int64_t id, float s, Box2D b)
+      : difficult(d), imgid(id), score(s), box(b) {}
+  Detection(bool d, int64_t id, float s, Box2D b, IgnoreType i)
+      : difficult(d), imgid(id), score(s), box(b), ignore(i) {}
+};
+
+// Precision and recall.
+struct PR {
+  float p = 0;
+  float r = 0;
+  PR(const float p_, const float r_) : p(p_), r(r_) {}
+};
+
+class AveragePrecision {
+ public:
+  // iou_threshold: A predicted box matches a ground truth box if and only if
+  //   IoU between these two are larger than this iou_threshold. Default: 0.5.
+  // num_recall_points: AP is computed as the average of maximum precision at (1
+  //   + num_recall_points) recall levels. E.g., if num_recall_points is 10,
+  //   recall levels are 0., 0.1, 0.2, ..., 0.9, 1.0.
+  // Default: 100. If num_recall_points < 0, AveragePrecision of 0 is returned.
+  struct Options {
+    float iou_threshold = 0.5;
+    int num_recall_points = 100;
+  };
+  AveragePrecision() : AveragePrecision(Options()) {}
+  explicit AveragePrecision(const Options& opts) : opts_(opts) {}
+
+  // Given a sequence of precision-recall points ordered by the recall in
+  // non-increasing order, returns the average of maximum precisions at
+  // different recall values (0.0, 0.1, 0.2, ..., 0.9, 1.0).
+  // The p-r pairs at these fixed recall points will be written to pr_out, if
+  // it is not null_ptr.
+  float FromPRCurve(const std::vector<PR>& pr,
+                    std::vector<PR>* pr_out = nullptr);
+
+  // An axis aligned bounding box for an image with id 'imageid'.  Score
+  // indicates its confidence.
+  //
+  // 'difficult' is a special bit specific to Pascal VOC dataset and tasks using
+  // the data. If 'difficult' is true, by convention, the box is often ignored
+  // during the AP calculation. I.e., if a predicted box matches a 'difficult'
+  // ground box, this predicted box is ignored as if the model does not make
+  // such a prediction.
+
+  // Given the set of ground truth boxes and a set of predicted boxes, returns
+  // the average of the maximum precisions at different recall values.
+  float FromBoxes(const std::vector<Detection>& groundtruth,
+                  const std::vector<Detection>& prediction,
+                  std::vector<PR>* pr_out = nullptr);
+
+ private:
+  Options opts_;
+};
+
+}  // namespace image
+}  // namespace evaluation
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_UTILS_IMAGE_METRICS_H_

tensorflow/lite/tools/evaluation/stages/utils/image_metrics_test.cc

@@ -0,0 +1,160 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/evaluation/stages/utils/image_metrics.h"
+
+#include <stdint.h>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdlib>
+#include <iterator>
+#include <vector>
+
+#include <gtest/gtest.h>
+
+namespace tflite {
+namespace evaluation {
+namespace image {
+
+// Find the max precision with the minimum recall.
+float MaxP(float minr, const std::vector<PR>& prs) {
+  float p = 0;
+  for (auto& pr : prs) {
+    if (pr.r >= minr) p = std::max(p, pr.p);
+  }
+  return p;
+}
+
+float ExpectedAP(const std::vector<PR>& prs) {
+  float sum = 0;
+  for (float r = 0; r <= 1.0; r += 0.01) {
+    sum += MaxP(r, prs);
+  }
+  return sum / 101;
+}
+
+float GenerateRandomFraction() {
+  return static_cast<float>(std::rand()) / RAND_MAX;
+}
+
+TEST(ImageMetricsTest, APBasic) {
+  std::vector<PR> prs;
+
+  prs = {{1., 1.}, {0.5, 1.0}, {1 / 3, 1.0}};
+  EXPECT_NEAR(ExpectedAP(prs), AveragePrecision().FromPRCurve(prs), 1e-6);
+
+  prs = {{1.0, 0.01}};
+  EXPECT_NEAR(ExpectedAP(prs), AveragePrecision().FromPRCurve(prs), 1e-6);
+
+  prs = {{1.0, 0.2}, {1.0, 0.4},  {0.67, 0.4}, {0.5, 0.4},  {0.4, 0.4},
+         {0.5, 0.6}, {0.57, 0.8}, {0.5, 0.8},  {0.44, 0.8}, {0.5, 1.0}};
+  EXPECT_NEAR(ExpectedAP(prs), AveragePrecision().FromPRCurve(prs), 1e-6);
+}
+
+TEST(ImageMetricsTest, APRandom) {
+  // Generates a set of p-r points.
+  std::vector<PR> prs;
+  for (int i = 0; i < 5000; ++i) {
+    float p = GenerateRandomFraction();
+    float r = GenerateRandomFraction();
+    prs.push_back({p, r});
+  }
+
+  const float expected = ExpectedAP(prs);
+
+  // Sort them w/ recall non-decreasing.
+  std::sort(std::begin(prs), std::end(prs),
+            [](const PR& a, const PR& b) { return a.r < b.r; });
+  const float actual = AveragePrecision().FromPRCurve(prs);
+
+  EXPECT_NEAR(expected, actual, 1e-5);
+}
+
+TEST(ImageMetricsTest, BBoxAPBasic) {
+  std::vector<Detection> gt;
+  gt.push_back(Detection({false, 100, 1, {{0, 1}, {0, 1}}}));
+  gt.push_back(Detection({false, 200, 1, {{1, 2}, {1, 2}}}));
+  std::vector<Detection> pd;
+  pd.push_back(Detection({false, 100, 0.8, {{0.1, 1.1}, {0.1, 1.1}}}));
+  pd.push_back(Detection({false, 200, 0.8, {{0.9, 1.9}, {0.9, 1.9}}}));
+  EXPECT_NEAR(1.0, AveragePrecision().FromBoxes(gt, pd), 1e-6);
+  AveragePrecision::Options opts;
+  opts.iou_threshold = 0.85;
+  EXPECT_NEAR(0.0, AveragePrecision(opts).FromBoxes(gt, pd), 1e-6);
+}
+
+TEST(ImageMetricsTest, Box2DOverlap) {
+  Box2D a({{0, 1}, {0, 1}});
+  Box2D b({{0.5, 2.5}, {0.5, 2.5}});
+  // Upper right quarter of box a overlaps.
+  EXPECT_NEAR(0.25, a.Overlap(b), 1e-6);
+
+  // Not symmetric if a and b have different areas.  Only lower left 0.5
+  // of b overlaps, so a total of 0.25 over an area of 4 overlaps.
+  EXPECT_NEAR(0.0625, b.Overlap(a), 1e-6);
+}
+
+TEST(ImageMetricsTest, BBoxAPwithIgnoredGroundTruth) {
+  std::vector<Detection> gt;
+  std::vector<Detection> pd;
+  gt.push_back(Detection({false, 100, 1, {{1, 2}, {1, 2}}, kIgnoreOneMatch}));
+  pd.push_back(Detection({false, 100, 0.8, {{0.1, 1.1}, {0.1, 1.1}}}));
+  // All gt box are ignored, expect NaN.
+  EXPECT_TRUE(std::isnan(AveragePrecision().FromBoxes(gt, pd)));
+
+  gt.push_back({false, 100, 1, {{0, 1}, {0, 1}}});
+  // Two gt and one pd, ap=1 because the unmatched gt is ignored.
+  EXPECT_NEAR(1.0, AveragePrecision().FromBoxes(gt, pd), 1e-6);
+
+  // Two gt and two pd, ap=1.
+  pd.push_back({false, 100, 0.9, {{0.9, 1.9}, {0.9, 1.9}}});
+  EXPECT_NEAR(1.0, AveragePrecision().FromBoxes(gt, pd), 1e-6);
+
+  pd.push_back({false, 100, 0.95, {{0.9, 1.9}, {0.9, 1.9}}});
+
+  // Two gt and three pd, one pair get ignored. So it's actually one gt with
+  // two pd.
+  EXPECT_NEAR(0.5, AveragePrecision().FromBoxes(gt, pd), 1e-6);
+  gt[0].ignore = kIgnoreAllMatches;
+  EXPECT_NEAR(1.0, AveragePrecision().FromBoxes(gt, pd), 1e-6);
+}
+
+TEST(ImageMetricsTest, BBoxAPRandom) {
+  auto rand = [](int64_t id) {
+    auto xmin = GenerateRandomFraction();
+    auto xmax = xmin + GenerateRandomFraction();
+    auto ymin = GenerateRandomFraction();
+    auto ymax = ymin + GenerateRandomFraction();
+    return Detection(
+        {false, id, GenerateRandomFraction(), {{xmin, xmax}, {ymin, ymax}}});
+  };
+  std::vector<Detection> gt;
+  for (int i = 0; i < 100; ++i) {
+    gt.push_back(rand(i % 10));
+  }
+  std::vector<Detection> pd = gt;
+  for (int i = 0; i < 10000; ++i) {
+    pd.push_back(rand(i % 10));
+  }
+  std::vector<PR> pr;
+  // Test pr curve output.
+  AveragePrecision().FromBoxes(gt, pd, &pr);
+  // Default num_recall_points=100, so there are p-r pairs of 101 levels.
+  EXPECT_EQ(101, pr.size());
+}
+
+}  // namespace image
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/tasks/BUILD

@@ -0,0 +1,61 @@
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
+load("//tensorflow/lite:build_def.bzl", "tflite_copts")
+load("//tensorflow/lite/tools/evaluation/tasks:build_def.bzl", "task_linkopts")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_visibility = [
+        "//visibility:public",
+    ],
+    licenses = ["notice"],
+)
+
+exports_files(
+    ["task_executor_c_api.h"],
+    visibility = ["//tensorflow/lite/tools/evaluation/tasks:__subpackages__"],
+)
+
+cc_library(
+    name = "task_executor",
+    srcs = ["task_executor.cc"],
+    hdrs = ["task_executor.h"],
+    copts = tflite_copts(),
+    linkopts = task_linkopts(),
+    deps = [
+        "//tensorflow/lite/tools:command_line_flags",
+        "//tensorflow/lite/tools:logging",
+        "//tensorflow/lite/tools/evaluation:evaluation_delegate_provider",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
+        "@com_google_absl//absl/types:optional",
+    ],
+)
+
+cc_library(
+    name = "task_executor_main",
+    srcs = ["task_executor_main.cc"],
+    copts = tflite_copts(),
+    linkopts = task_linkopts(),
+    deps = [
+        ":task_executor",
+        "//tensorflow/lite/tools:logging",
+        "@com_google_absl//absl/types:optional",
+    ],
+)
+
+cc_library(
+    name = "task_executor_c_api",
+    srcs = ["task_executor_c_api.cc"],
+    hdrs = [
+        "task_executor_c_api.h",
+    ],
+    copts = tflite_copts(),
+    visibility = [
+        "//tensorflow/lite/tools/evaluation/tasks:__subpackages__",
+    ],
+    deps = [
+        ":task_executor",
+        "//tensorflow/lite/tools:logging",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
+    ],
+)

tensorflow/lite/tools/evaluation/tasks/README.md

@@ -0,0 +1,45 @@
+# TFLite Model Task Evaluation
+
+This page describes how you can check the accuracy of quantized models to verify
+that any degradation in accuracy is within acceptable limits.
+
+## Accuracy & correctness
+
+TensorFlow Lite has two types of tooling to measure how accurately a delegate
+behaves for a given model: Task-Based and Task-Agnostic.
+
+**Task-Based Evaluation** TFLite has two tools to evaluate correctness on two
+image-based tasks: - [ILSVRC 2012](http://image-net.org/challenges/LSVRC/2012/)
+(Image Classification) with top-K accuracy -
+[COCO Object Detection](https://cocodataset.org/#detection-2020) (w/ bounding
+boxes) with mean Average Precision (mAP)
+
+**Task-Agnostic Evaluation** For tasks where there isn't an established
+on-device evaluation tool, or if you are experimenting with custom models,
+TensorFlow Lite has the Inference Diff tool.
+
+## Tools
+
+There are three different binaries which are supported. A brief description of
+each is provided below.
+
+### [Inference Diff Tool](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/evaluation/tasks/inference_diff#inference-diff-tool)
+
+This binary compares TensorFlow Lite execution in single-threaded CPU inference
+and user-defined inference.
+
+### [Image Classification Evaluation](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/evaluation/tasks/imagenet_image_classification#image-classification-evaluation-based-on-ilsvrc-2012-task)
+
+This binary evaluates TensorFlow Lite models trained for the
+[ILSVRC 2012 image classification task.](http://www.image-net.org/challenges/LSVRC/2012/)
+
+### [Object Detection Evaluation](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/evaluation/tasks/coco_object_detection#object-detection-evaluation-using-the-2014-coco-minival-dataset)
+
+This binary evaluates TensorFlow Lite models trained for the bounding box-based
+[COCO Object Detection](https://cocodataset.org/#detection-eval) task.
+
+********************************************************************************
+
+For more information visit the TensorFlow Lite guide on
+[Accuracy & correctness](https://www.tensorflow.org/lite/performance/delegates#accuracy_correctness)
+page.

tensorflow/lite/tools/evaluation/tasks/build_def.bzl

@@ -0,0 +1,14 @@
+"""Common BUILD-related definitions across different tasks"""
+
+load("//tensorflow/lite:build_def.bzl", "tflite_linkopts")
+
+def task_linkopts():
+    return tflite_linkopts() + select({
+        "//tensorflow:android": [
+            "-pie",  # Android 5.0 and later supports only PIE
+            "-lm",  # some builtin ops, e.g., tanh, need -lm
+            # Hexagon delegate libraries should be in /data/local/tmp
+            "-Wl,--rpath=/data/local/tmp/",
+        ],
+        "//conditions:default": [],
+    })

tensorflow/lite/tools/evaluation/tasks/coco_object_detection/BUILD

@@ -0,0 +1,56 @@
+load("@rules_cc//cc:cc_binary.bzl", "cc_binary")
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
+load("@xla//third_party/rules_python/python:py_binary.bzl", "py_binary")
+load("//tensorflow/lite:build_def.bzl", "tflite_copts")
+load("//tensorflow/lite/tools/evaluation/tasks:build_def.bzl", "task_linkopts")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_visibility = [
+        "//visibility:public",
+    ],
+    licenses = ["notice"],
+)
+
+py_binary(
+    name = "preprocess_coco_minival",
+    srcs = ["preprocess_coco_minival.py"],
+    strict_deps = True,
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_py",
+        "@absl_py//absl/logging",
+    ],
+)
+
+cc_library(
+    name = "run_eval_lib",
+    srcs = ["run_eval.cc"],
+    copts = tflite_copts(),
+    linkopts = task_linkopts(),
+    deps = [
+        "//tensorflow/lite/core/c:common",
+        "//tensorflow/lite/tools:command_line_flags",
+        "//tensorflow/lite/tools:logging",
+        "//tensorflow/lite/tools/evaluation:evaluation_delegate_provider",
+        "//tensorflow/lite/tools/evaluation:evaluation_stage",
+        "//tensorflow/lite/tools/evaluation:utils",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
+        "//tensorflow/lite/tools/evaluation/stages:object_detection_stage",
+        "//tensorflow/lite/tools/evaluation/tasks:task_executor",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/types:optional",
+    ],
+    alwayslink = 1,
+)
+
+cc_binary(
+    name = "run_eval",
+    copts = tflite_copts(),
+    linkopts = task_linkopts(),
+    deps = [
+        ":run_eval_lib",
+        "//tensorflow/lite/tools/evaluation/tasks:task_executor_main",
+    ],
+)

tensorflow/lite/tools/evaluation/tasks/coco_object_detection/README.md

@@ -0,0 +1,264 @@
+# Object Detection evaluation using the 2014 COCO minival dataset.
+
+This binary evaluates the following parameters of TFLite models trained for the
+**bounding box-based**
+[COCO Object Detection](http://cocodataset.org/#detection-eval) task:
+
+*   Native pre-processing latency
+*   Inference latency
+*   mean Average Precision (mAP) averaged across IoU thresholds from 0.5 to 0.95
+    (in increments of 0.05) and all object categories.
+
+The binary takes the path to validation images and a ground truth proto file as
+inputs, along with the model and inference-specific parameters such as delegate
+and number of threads. It outputs the metrics to std-out as follows:
+
+```
+Num evaluation runs: 8059
+Preprocessing latency: avg=16589.9(us), std_dev=0(us)
+Inference latency: avg=85169.7(us), std_dev=505(us)
+Average Precision [IOU Threshold=0.5]: 0.349581
+Average Precision [IOU Threshold=0.55]: 0.330213
+Average Precision [IOU Threshold=0.6]: 0.307694
+Average Precision [IOU Threshold=0.65]: 0.281025
+Average Precision [IOU Threshold=0.7]: 0.248507
+Average Precision [IOU Threshold=0.75]: 0.210295
+Average Precision [IOU Threshold=0.8]: 0.165011
+Average Precision [IOU Threshold=0.85]: 0.116215
+Average Precision [IOU Threshold=0.9]: 0.0507883
+Average Precision [IOU Threshold=0.95]: 0.0064338
+Overall mAP: 0.206576
+```
+
+To run the binary, please follow the
+[Preprocessing section](#preprocessing-the-minival-dataset) to prepare the data,
+and then execute the commands in the
+[Running the binary section](#running-the-binary).
+
+## Parameters
+
+The binary takes the following parameters:
+
+*   `model_file` : `string` \
+    Path to the TFlite model file. It should accept images preprocessed in the
+    Inception format, and the output signature should be similar to the
+    [SSD MobileNet model](https://www.tensorflow.org/lite/examples/object_detection/overview#output_signature.):
+
+*   `model_output_labels`: `string` \
+    Path to labels that correspond to output of model. E.g. in case of
+    COCO-trained SSD model, this is the path to a file where each line contains
+    a class detected by the model in correct order, starting from 'background'.
+
+A sample model & label-list combination for COCO can be downloaded from the
+TFLite
+[Hosted models page](https://www.tensorflow.org/lite/guide/hosted_models#object_detection).
+
+*   `ground_truth_images_path`: `string` \
+    The path to the directory containing ground truth images.
+
+*   `ground_truth_proto`: `string` \
+    Path to file containing tflite::evaluation::ObjectDetectionGroundTruth proto
+    in text format. If left empty, mAP numbers are not provided.
+
+The above two parameters can be prepared using the `preprocess_coco_minival`
+script included in this folder.
+
+*   `output_file_path`: `string` \
+    The final metrics are dumped into `output_file_path` as a string-serialized
+    instance of `tflite::evaluation::EvaluationStageMetrics`.
+
+The following optional parameters can be used to modify the inference runtime:
+
+*   `num_interpreter_threads`: `int` (default=1) \
+    This modifies the number of threads used by the TFLite Interpreter for
+    inference.
+
+*   `delegate`: `string` \
+    If provided, tries to use the specified delegate for accuracy evaluation.
+    Valid values: "nnapi", "gpu", "hexagon".
+
+    NOTE: Please refer to the
+    [Hexagon delegate documentation](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/performance/hexagon_delegate.md)
+    for instructions on how to set it up for the Hexagon delegate. The tool
+    assumes that `libhexagon_interface.so` and Qualcomm libraries lie in
+    `/data/local/tmp`.
+
+This script also supports runtime/delegate arguments introduced by the
+[delegate registrar](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/delegates).
+If there is any conflict (for example, `num_threads` vs
+`num_interpreter_threads` here), the parameters of this
+script are given precedence.
+
+When **multiple delegates** are specified to be used in the commandline flags
+via the support of delegate registrar, the order of delegates applied to the
+TfLite runtime will be same as their enabling commandline flag is specified. For
+example, "--use_xnnpack=true --use_gpu=true" means applying the XNNPACK delegate
+first, and then the GPU delegate secondly. In comparison,
+"--use_gpu=true --use_xnnpack=true" means applying the GPU delegate first, and
+then the XNNPACK delegate secondly.
+
+Note, one could specify `--help` when launching the binary to see the full list
+of supported arguments.
+
+### Debug Mode
+
+The script also supports a debug mode with the following parameter:
+
+*   `debug_mode`: `boolean` \
+    Whether to enable debug mode. Per-image predictions are written to std-out
+    along with metrics.
+
+Image-wise predictions are output as follows:
+
+```
+======================================================
+
+Image: image_1.jpg
+
+Object [0]
+  Score: 0.585938
+  Class-ID: 5
+  Bounding Box:
+    Normalized Top: 0.23103
+    Normalized Bottom: 0.388524
+    Normalized Left: 0.559144
+    Normalized Right: 0.763928
+Object [1]
+  Score: 0.574219
+  Class-ID: 5
+  Bounding Box:
+    Normalized Top: 0.269571
+    Normalized Bottom: 0.373971
+    Normalized Left: 0.613175
+    Normalized Right: 0.760507
+======================================================
+
+Image: image_2.jpg
+...
+```
+
+This mode lets you debug the output of an object detection model that isn't
+necessarily trained on the COCO dataset (by leaving `ground_truth_proto` empty).
+The model output signature would still need to follow the convention mentioned
+above, and you we still need an output labels file.
+
+## Preprocessing the minival dataset
+
+To compute mAP in a consistent and interpretable way, we utilize the same 2014
+COCO 'minival' dataset that is mentioned in the
+[Tensorflow detection model zoo](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf1_detection_zoo.md).
+
+The links to download the components of the validation set are:
+
+*   [2014 COCO Validation Images](http://images.cocodataset.org/zips/val2014.zip)
+*   [2014 COCO Train/Val annotations](http://images.cocodataset.org/annotations/annotations_trainval2014.zip):
+    Out of the files from this zip, we only require `instances_val2014.json`.
+*   [minival Image IDs](https://github.com/tensorflow/models/blob/master/research/object_detection/data/mscoco_minival_ids.txt) :
+    Only applies to the 2014 validation set. You would need to copy the contents
+    into a text file.
+
+Since evaluation has to be performed on-device, we first filter the above data
+and extract a subset that only contains the images & ground-truth bounding boxes
+we need.
+
+To do so, we utilize the `preprocess_coco_minival` Python binary as follows:
+
+```
+bazel run //tensorflow/lite/tools/evaluation/tasks/coco_object_detection:preprocess_coco_minival -- \
+  --images_folder=/path/to/val2014 \
+  --instances_file=/path/to/instances_val2014.json \
+  --allowlist_file=/path/to/minival_allowlist.txt \
+  --output_folder=/path/to/output/folder
+
+```
+
+Optionally, you can specify a `--num_images=N` argument, to preprocess the first
+`N` image files (based on sorted list of filenames).
+
+The script generates the following within the output folder:
+
+*   `images/`: the resulting subset of the 2014 COCO Validation images.
+
+*   `ground_truth.pb`: a `.pb` (binary-format proto) file holding
+    `tflite::evaluation::ObjectDetectionGroundTruth` corresponding to image
+    subset.
+
+## Running the binary
+
+### On Android
+
+(0) Refer to
+https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android
+for configuring NDK and SDK.
+
+(1) Build using the following command:
+
+```
+bazel build -c opt \
+  --config=android_arm64 \
+  --cxxopt='--std=c++17' \
+  //tensorflow/lite/tools/evaluation/tasks/coco_object_detection:run_eval
+```
+
+(2) Connect your phone. Push the binary to your phone with adb push (make the
+directory if required):
+
+```
+adb push bazel-bin/third_party/tensorflow/lite/tools/evaluation/tasks/coco_object_detection/run_eval /data/local/tmp
+```
+
+(3) Make the binary executable.
+
+```
+adb shell chmod +x /data/local/tmp/run_eval
+```
+
+(4) Push the TFLite model that you need to test:
+
+```
+adb push ssd_mobilenet_v1_float.tflite /data/local/tmp
+```
+
+(5) Push the model labels text file to device.
+
+```
+adb push /path/to/labelmap.txt /data/local/tmp/labelmap.txt
+```
+
+(6) Preprocess the dataset using the instructions given in the
+[Preprocessing section](#preprocessing-the-minival-dataset) and push the data
+(folder containing images & ground truth proto) to the device:
+
+```
+adb shell mkdir /data/local/tmp/coco_validation && \
+adb push /path/to/output/folder /data/local/tmp/coco_validation
+```
+
+(7) Run the binary.
+
+```
+adb shell /data/local/tmp/run_eval \
+  --model_file=/data/local/tmp/ssd_mobilenet_v1_float.tflite \
+  --ground_truth_images_path=/data/local/tmp/coco_validation/images \
+  --ground_truth_proto=/data/local/tmp/coco_validation/ground_truth.pb \
+  --model_output_labels=/data/local/tmp/labelmap.txt \
+  --output_file_path=/data/local/tmp/coco_output.txt
+```
+
+Optionally, you could also pass in the `--num_interpreter_threads` &
+`--delegate` arguments to run with different configurations.
+
+### On Desktop
+
+(1) Build and run using the following command:
+
+```
+bazel run -c opt \
+  -- \
+  //tensorflow/lite/tools/evaluation/tasks/coco_object_detection:run_eval \
+  --model_file=/path/to/ssd_mobilenet_v1_float.tflite \
+  --ground_truth_images_path=/path/to/images \
+  --ground_truth_proto=/path/to/ground_truth.pb \
+  --model_output_labels=/path/to/labelmap.txt \
+  --output_file_path=/path/to/coco_output.txt
+```

tensorflow/lite/tools/evaluation/tasks/coco_object_detection/preprocess_coco_minival.py

@@ -0,0 +1,227 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Preprocesses COCO minival data for Object Detection evaluation using mean Average Precision.
+
+The 2014 validation images & annotations can be downloaded from:
+http://cocodataset.org/#download
+The minival image ID allowlist, a subset of the 2014 validation set, can be
+found here:
+https://github.com/tensorflow/models/blob/master/research/object_detection/data/mscoco_minival_ids.txt.
+
+This script takes in the original images folder, instances JSON file and
+image ID allowlist and produces the following in the specified output folder:
+A subfolder for allowlisted images (images/), and a file (ground_truth.pbtxt)
+containing an instance of tflite::evaluation::ObjectDetectionGroundTruth.
+"""
+
+import argparse
+import ast
+import collections
+import os
+import shutil
+import sys
+
+from absl import logging
+from tensorflow.lite.tools.evaluation.proto import evaluation_stages_pb2
+
+
+def _get_ground_truth_detections(instances_file,
+                                 allowlist_file=None,
+                                 num_images=None):
+  """Processes the annotations JSON file and returns ground truth data corresponding to allowlisted image IDs.
+
+  Args:
+    instances_file: COCO instances JSON file, usually named as
+      instances_val20xx.json.
+    allowlist_file: File containing COCO minival image IDs to allowlist for
+      evaluation, one per line.
+    num_images: Number of allowlisted images to pre-process. First num_images
+      are chosen based on sorted list of filenames. If None, all allowlisted
+      files are preprocessed.
+
+  Returns:
+    A dict mapping image id (int) to a per-image dict that contains:
+      'filename', 'image' & 'height' mapped to filename & image dimensions
+      respectively
+      AND
+      'detections' to a list of detection dicts, with each mapping:
+        'category_id' to COCO category id (starting with 1) &
+        'bbox' to a list of dimension-normalized [top, left, bottom, right]
+        bounding-box values.
+  """
+  # Read JSON data into a dict.
+  with open(instances_file, 'r') as annotation_dump:
+    data_dict = ast.literal_eval(annotation_dump.readline())
+
+  image_data = collections.OrderedDict()
+
+  # Read allowlist.
+  if allowlist_file is not None:
+    with open(allowlist_file, 'r') as allowlist:
+      image_id_allowlist = set([int(x) for x in allowlist.readlines()])
+  else:
+    image_id_allowlist = [image['id'] for image in data_dict['images']]
+
+  # Get image names and dimensions.
+  for image_dict in data_dict['images']:
+    image_id = image_dict['id']
+    if image_id not in image_id_allowlist:
+      continue
+    image_data_dict = {}
+    image_data_dict['id'] = image_dict['id']
+    image_data_dict['file_name'] = image_dict['file_name']
+    image_data_dict['height'] = image_dict['height']
+    image_data_dict['width'] = image_dict['width']
+    image_data_dict['detections'] = []
+    image_data[image_id] = image_data_dict
+
+  shared_image_ids = set()
+  for annotation_dict in data_dict['annotations']:
+    image_id = annotation_dict['image_id']
+    if image_id in image_data:
+      shared_image_ids.add(image_id)
+
+  output_image_ids = sorted(shared_image_ids)
+  if num_images:
+    if num_images <= 0:
+      logging.warning(
+          '--num_images is %d, hence outputing all annotated images.',
+          num_images)
+    elif num_images > len(shared_image_ids):
+      logging.warning(
+          '--num_images (%d) is larger than the number of annotated images.',
+          num_images)
+    else:
+      output_image_ids = output_image_ids[:num_images]
+
+  for image_id in list(image_data):
+    if image_id not in output_image_ids:
+      del image_data[image_id]
+
+  # Get detected object annotations per image.
+  for annotation_dict in data_dict['annotations']:
+    image_id = annotation_dict['image_id']
+    if image_id not in output_image_ids:
+      continue
+
+    image_data_dict = image_data[image_id]
+    bbox = annotation_dict['bbox']
+    # bbox format is [x, y, width, height]
+    # Refer: http://cocodataset.org/#format-data
+    top = bbox[1]
+    left = bbox[0]
+    bottom = top + bbox[3]
+    right = left + bbox[2]
+    if (top > image_data_dict['height'] or left > image_data_dict['width'] or
+        bottom > image_data_dict['height'] or right > image_data_dict['width']):
+      continue
+    object_d = {}
+    object_d['bbox'] = [
+        top / image_data_dict['height'], left / image_data_dict['width'],
+        bottom / image_data_dict['height'], right / image_data_dict['width']
+    ]
+    object_d['category_id'] = annotation_dict['category_id']
+    image_data_dict['detections'].append(object_d)
+
+  return image_data
+
+
+def _dump_data(ground_truth_detections, images_folder_path, output_folder_path):
+  """Dumps images & data from ground-truth objects into output_folder_path.
+
+  The following are created in output_folder_path:
+    images/: sub-folder for allowlisted validation images.
+    ground_truth.pb: A binary proto file containing all ground-truth
+    object-sets.
+
+  Args:
+    ground_truth_detections: A dict mapping image id to ground truth data.
+      Output of _get_ground_truth_detections.
+    images_folder_path: Validation images folder
+    output_folder_path: folder to output files to.
+  """
+  # Ensure output folders exist.
+  if not os.path.exists(output_folder_path):
+    os.makedirs(output_folder_path)
+  output_images_folder = os.path.join(output_folder_path, 'images')
+  if not os.path.exists(output_images_folder):
+    os.makedirs(output_images_folder)
+  output_proto_file = os.path.join(output_folder_path, 'ground_truth.pb')
+
+  ground_truth_data = evaluation_stages_pb2.ObjectDetectionGroundTruth()
+  for image_dict in ground_truth_detections.values():
+    # Create an ObjectsSet proto for this file's ground truth.
+    detection_result = ground_truth_data.detection_results.add()
+    detection_result.image_id = image_dict['id']
+    detection_result.image_name = image_dict['file_name']
+    for detection_dict in image_dict['detections']:
+      object_instance = detection_result.objects.add()
+      object_instance.bounding_box.normalized_top = detection_dict['bbox'][0]
+      object_instance.bounding_box.normalized_left = detection_dict['bbox'][1]
+      object_instance.bounding_box.normalized_bottom = detection_dict['bbox'][2]
+      object_instance.bounding_box.normalized_right = detection_dict['bbox'][3]
+      object_instance.class_id = detection_dict['category_id']
+    # Copy image.
+    shutil.copy2(
+        os.path.join(images_folder_path, image_dict['file_name']),
+        output_images_folder)
+
+  # Dump proto.
+  with open(output_proto_file, 'wb') as proto_file:
+    proto_file.write(ground_truth_data.SerializeToString())
+
+
+def _parse_args():
+  """Creates a parser that parse the command line arguments.
+
+  Returns:
+    A namespace parsed from command line arguments.
+  """
+  parser = argparse.ArgumentParser(
+      description='preprocess_coco_minival: Preprocess COCO minival dataset')
+  parser.add_argument(
+      '--images_folder',
+      type=str,
+      help='Full path of the validation images folder.',
+      required=True)
+  parser.add_argument(
+      '--instances_file',
+      type=str,
+      help='Full path of the input JSON file, like instances_val20xx.json.',
+      required=True)
+  parser.add_argument(
+      '--allowlist_file',
+      type=str,
+      help='File with COCO image ids to preprocess, one on each line.',
+      required=False)
+  parser.add_argument(
+      '--num_images',
+      type=int,
+      help='Number of allowlisted images to preprocess into the output folder.',
+      required=False)
+  parser.add_argument(
+      '--output_folder',
+      type=str,
+      help='Full path to output images & text proto files into.',
+      required=True)
+  return parser.parse_known_args(args=sys.argv[1:])[0]
+
+
+if __name__ == '__main__':
+  args = _parse_args()
+  ground_truths = _get_ground_truth_detections(args.instances_file,
+                                               args.allowlist_file,
+                                               args.num_images)
+  _dump_data(ground_truths, args.images_folder, args.output_folder)

tensorflow/lite/tools/evaluation/tasks/coco_object_detection/run_eval.cc

@@ -0,0 +1,232 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <fstream>
+#include <ios>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "absl/container/flat_hash_map.h"
+#include "tensorflow/lite/core/c/common.h"
+#include "tensorflow/lite/tools/command_line_flags.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_delegate_provider.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+#include "tensorflow/lite/tools/evaluation/stages/object_detection_stage.h"
+#include "tensorflow/lite/tools/evaluation/tasks/task_executor.h"
+#include "tensorflow/lite/tools/evaluation/utils.h"
+#include "tensorflow/lite/tools/logging.h"
+
+namespace tflite {
+namespace evaluation {
+
+constexpr char kModelFileFlag[] = "model_file";
+constexpr char kGroundTruthImagesPathFlag[] = "ground_truth_images_path";
+constexpr char kModelOutputLabelsFlag[] = "model_output_labels";
+constexpr char kOutputFilePathFlag[] = "output_file_path";
+constexpr char kGroundTruthProtoFileFlag[] = "ground_truth_proto";
+constexpr char kInterpreterThreadsFlag[] = "num_interpreter_threads";
+constexpr char kDebugModeFlag[] = "debug_mode";
+constexpr char kDelegateFlag[] = "delegate";
+
+std::string GetNameFromPath(const std::string& str) {
+  int pos = str.find_last_of("/\\");
+  if (pos == std::string::npos) return "";
+  return str.substr(pos + 1);
+}
+
+class CocoObjectDetection : public TaskExecutor {
+ public:
+  CocoObjectDetection() : debug_mode_(false), num_interpreter_threads_(1) {}
+  ~CocoObjectDetection() override {}
+
+ protected:
+  std::vector<Flag> GetFlags() final;
+
+  // If the run is successful, the latest metrics will be returned.
+  std::optional<EvaluationStageMetrics> RunImpl() final;
+
+ private:
+  void OutputResult(const EvaluationStageMetrics& latest_metrics) const;
+  std::string model_file_path_;
+  std::string model_output_labels_path_;
+  std::string ground_truth_images_path_;
+  std::string ground_truth_proto_file_;
+  std::string output_file_path_;
+  bool debug_mode_;
+  std::string delegate_;
+  int num_interpreter_threads_;
+};
+
+std::vector<Flag> CocoObjectDetection::GetFlags() {
+  std::vector<tflite::Flag> flag_list = {
+      tflite::Flag::CreateFlag(kModelFileFlag, &model_file_path_,
+                               "Path to test tflite model file."),
+      tflite::Flag::CreateFlag(
+          kModelOutputLabelsFlag, &model_output_labels_path_,
+          "Path to labels that correspond to output of model."
+          " E.g. in case of COCO-trained SSD model, this is the path to file "
+          "where each line contains a class detected by the model in correct "
+          "order, starting from background."),
+      tflite::Flag::CreateFlag(
+          kGroundTruthImagesPathFlag, &ground_truth_images_path_,
+          "Path to ground truth images. These will be evaluated in "
+          "alphabetical order of filenames"),
+      tflite::Flag::CreateFlag(kGroundTruthProtoFileFlag,
+                               &ground_truth_proto_file_,
+                               "Path to file containing "
+                               "tflite::evaluation::ObjectDetectionGroundTruth "
+                               "proto in binary serialized format. If left "
+                               "empty, mAP numbers are not output."),
+      tflite::Flag::CreateFlag(
+          kOutputFilePathFlag, &output_file_path_,
+          "File to output to. Contains only metrics proto if debug_mode is "
+          "off, and per-image predictions also otherwise."),
+      tflite::Flag::CreateFlag(kDebugModeFlag, &debug_mode_,
+                               "Whether to enable debug mode. Per-image "
+                               "predictions are written to the output file "
+                               "along with metrics."),
+      tflite::Flag::CreateFlag(
+          kInterpreterThreadsFlag, &num_interpreter_threads_,
+          "Number of interpreter threads to use for inference."),
+      tflite::Flag::CreateFlag(
+          kDelegateFlag, &delegate_,
+          "Delegate to use for inference, if available. "
+          "Must be one of {'nnapi', 'gpu', 'xnnpack', 'hexagon'}"),
+  };
+  return flag_list;
+}
+
+std::optional<EvaluationStageMetrics> CocoObjectDetection::RunImpl() {
+  // Process images in filename-sorted order.
+  std::vector<std::string> image_paths;
+  if (GetSortedFileNames(StripTrailingSlashes(ground_truth_images_path_),
+                         &image_paths) != kTfLiteOk) {
+    return std::nullopt;
+  }
+
+  std::vector<std::string> model_labels;
+  if (!ReadFileLines(model_output_labels_path_, &model_labels)) {
+    TFLITE_LOG(ERROR) << "Could not read model output labels file";
+    return std::nullopt;
+  }
+
+  EvaluationStageConfig eval_config;
+  eval_config.set_name("object_detection");
+  auto* detection_params =
+      eval_config.mutable_specification()->mutable_object_detection_params();
+  auto* inference_params = detection_params->mutable_inference_params();
+  inference_params->set_model_file_path(model_file_path_);
+  inference_params->set_num_threads(num_interpreter_threads_);
+  inference_params->set_delegate(ParseStringToDelegateType(delegate_));
+
+  // Get ground truth data.
+  absl::flat_hash_map<std::string, ObjectDetectionResult> ground_truth_map;
+  if (!ground_truth_proto_file_.empty()) {
+    PopulateGroundTruth(ground_truth_proto_file_, &ground_truth_map);
+  }
+
+  ObjectDetectionStage eval(eval_config);
+
+  eval.SetAllLabels(model_labels);
+  if (eval.Init(&delegate_providers_) != kTfLiteOk) return std::nullopt;
+
+  const int step = image_paths.size() / 100;
+  for (int i = 0; i < image_paths.size(); ++i) {
+    if (step > 1 && i % step == 0) {
+      TFLITE_LOG(INFO) << "Finished: " << i / step << "%";
+    }
+
+    const std::string image_name = GetNameFromPath(image_paths[i]);
+    eval.SetInputs(image_paths[i], ground_truth_map[image_name]);
+    if (eval.Run() != kTfLiteOk) return std::nullopt;
+
+    if (debug_mode_) {
+      ObjectDetectionResult prediction = *eval.GetLatestPrediction();
+      TFLITE_LOG(INFO) << "Image: " << image_name << "\n";
+      for (int i = 0; i < prediction.objects_size(); ++i) {
+        const auto& object = prediction.objects(i);
+        TFLITE_LOG(INFO) << "Object [" << i << "]";
+        TFLITE_LOG(INFO) << "  Score: " << object.score();
+        TFLITE_LOG(INFO) << "  Class-ID: " << object.class_id();
+        TFLITE_LOG(INFO) << "  Bounding Box:";
+        const auto& bounding_box = object.bounding_box();
+        TFLITE_LOG(INFO) << "    Normalized Top: "
+                         << bounding_box.normalized_top();
+        TFLITE_LOG(INFO) << "    Normalized Bottom: "
+                         << bounding_box.normalized_bottom();
+        TFLITE_LOG(INFO) << "    Normalized Left: "
+                         << bounding_box.normalized_left();
+        TFLITE_LOG(INFO) << "    Normalized Right: "
+                         << bounding_box.normalized_right();
+      }
+      TFLITE_LOG(INFO)
+          << "======================================================\n";
+    }
+  }
+
+  // Write metrics to file.
+  EvaluationStageMetrics latest_metrics = eval.LatestMetrics();
+  if (ground_truth_proto_file_.empty()) {
+    TFLITE_LOG(WARN) << "mAP metrics are meaningless w/o ground truth.";
+    latest_metrics.mutable_process_metrics()
+        ->mutable_object_detection_metrics()
+        ->clear_average_precision_metrics();
+  }
+
+  OutputResult(latest_metrics);
+  return std::make_optional(latest_metrics);
+}
+
+void CocoObjectDetection::OutputResult(
+    const EvaluationStageMetrics& latest_metrics) const {
+  if (!output_file_path_.empty()) {
+    std::ofstream metrics_ofile;
+    metrics_ofile.open(output_file_path_, std::ios::out);
+    metrics_ofile << latest_metrics.SerializeAsString();
+    metrics_ofile.close();
+  }
+  TFLITE_LOG(INFO) << "Num evaluation runs: " << latest_metrics.num_runs();
+  const auto object_detection_metrics =
+      latest_metrics.process_metrics().object_detection_metrics();
+  const auto& preprocessing_latency =
+      object_detection_metrics.pre_processing_latency();
+  TFLITE_LOG(INFO) << "Preprocessing latency: avg="
+                   << preprocessing_latency.avg_us() << "(us), std_dev="
+                   << preprocessing_latency.std_deviation_us() << "(us)";
+  const auto& inference_latency = object_detection_metrics.inference_latency();
+  TFLITE_LOG(INFO) << "Inference latency: avg=" << inference_latency.avg_us()
+                   << "(us), std_dev=" << inference_latency.std_deviation_us()
+                   << "(us)";
+  const auto& precision_metrics =
+      object_detection_metrics.average_precision_metrics();
+  for (int i = 0; i < precision_metrics.individual_average_precisions_size();
+       ++i) {
+    const auto ap_metric = precision_metrics.individual_average_precisions(i);
+    TFLITE_LOG(INFO) << "Average Precision [IOU Threshold="
+                     << ap_metric.iou_threshold()
+                     << "]: " << ap_metric.average_precision();
+  }
+  TFLITE_LOG(INFO) << "Overall mAP: "
+                   << precision_metrics.overall_mean_average_precision();
+}
+
+std::unique_ptr<TaskExecutor> CreateTaskExecutor() {
+  return std::unique_ptr<TaskExecutor>(new CocoObjectDetection());
+}
+
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/tasks/imagenet_image_classification/BUILD

@@ -0,0 +1,43 @@
+load("@rules_cc//cc:cc_binary.bzl", "cc_binary")
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
+load("//tensorflow/lite:build_def.bzl", "tflite_copts")
+load("//tensorflow/lite/tools/evaluation/tasks:build_def.bzl", "task_linkopts")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_visibility = [
+        "//visibility:public",
+    ],
+    licenses = ["notice"],
+)
+
+cc_library(
+    name = "run_eval_lib",
+    srcs = ["run_eval.cc"],
+    copts = tflite_copts(),
+    linkopts = task_linkopts(),
+    deps = [
+        "//tensorflow/lite/core/c:common",
+        "//tensorflow/lite/tools:command_line_flags",
+        "//tensorflow/lite/tools:logging",
+        "//tensorflow/lite/tools/evaluation:evaluation_delegate_provider",
+        "//tensorflow/lite/tools/evaluation:evaluation_stage",
+        "//tensorflow/lite/tools/evaluation:utils",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
+        "//tensorflow/lite/tools/evaluation/stages:image_classification_stage",
+        "//tensorflow/lite/tools/evaluation/tasks:task_executor",
+        "@com_google_absl//absl/types:optional",
+    ],
+    alwayslink = 1,
+)
+
+cc_binary(
+    name = "run_eval",
+    copts = tflite_copts(),
+    linkopts = task_linkopts(),
+    deps = [
+        ":run_eval_lib",
+        "//tensorflow/lite/tools/evaluation/tasks:task_executor_main",
+    ],
+)

tensorflow/lite/tools/evaluation/tasks/imagenet_image_classification/README.md

@@ -0,0 +1,216 @@
+## Image Classification evaluation based on ILSVRC 2012 task
+
+This binary evaluates the following parameters of TFLite models trained for the
+[ILSVRC 2012 image classification task](http://www.image-net.org/challenges/LSVRC/2012/):
+
+*   Native pre-processing latency
+*   Inference latency
+*   Top-K (1 to 10) accuracy values
+
+The binary takes the path to validation images and labels as inputs, along with
+the model and inference-specific parameters such as delegate and number of
+threads. It outputs the metrics to std-out as follows:
+
+```
+Num evaluation runs: 300 # Total images evaluated
+Preprocessing latency: avg=13772.5(us), std_dev=0(us)
+Inference latency: avg=76578.4(us), std_dev=600(us)
+Top-1 Accuracy: 0.733333
+Top-2 Accuracy: 0.826667
+Top-3 Accuracy: 0.856667
+Top-4 Accuracy: 0.87
+Top-5 Accuracy: 0.89
+Top-6 Accuracy: 0.903333
+Top-7 Accuracy: 0.906667
+Top-8 Accuracy: 0.913333
+Top-9 Accuracy: 0.92
+Top-10 Accuracy: 0.923333
+```
+
+To run the binary download the ILSVRC 2012 devkit
+[see instructions](#downloading-ilsvrc) and run the
+[`generate_validation_ground_truth` script](#ground-truth-label-generation) to
+generate the ground truth labels.
+
+## Parameters
+
+The binary takes the following parameters:
+
+*   `model_file` : `string` \
+    Path to the TFlite model file.
+
+*   `ground_truth_images_path`: `string` \
+    The path to the directory containing ground truth images.
+
+*   `ground_truth_labels`: `string` \
+    Path to ground truth labels file. This file should contain the same number
+    of labels as the number images in the ground truth directory. The labels are
+    assumed to be in the same order as the sorted filename of images. See
+    [ground truth label generation](#ground-truth-label-generation) section for
+    more information about how to generate labels for images.
+
+*   `model_output_labels`: `string` \
+    Path to the file containing labels, that is used to interpret the output of
+    the model. E.g. in case of mobilenets, this is the path to
+    `mobilenet_labels.txt` where each label is in the same order as the output
+    1001 dimension tensor.
+
+and the following optional parameters:
+
+*   `denylist_file_path`: `string` \
+    Path to denylist file. This file contains the indices of images that are
+    denylisted for evaluation. 1762 images are denylisted in ILSVRC dataset.
+    For details please refer to readme.txt of ILSVRC2014 devkit.
+
+*   `num_images`: `int` (default=0) \
+    The number of images to process, if 0, all images in the directory are
+    processed otherwise only num_images will be processed.
+
+*   `num_threads`: `int` (default=4) \
+    The number of threads to use for evaluation. Note: This does not change the
+    number of TFLite Interpreter threads, but shards the dataset to speed up
+    evaluation.
+
+*   `output_file_path`: `string` \
+    The final metrics are dumped into `output_file_path` as a string-serialized
+    instance of `tflite::evaluation::EvaluationStageMetrics`.
+
+The following optional parameters can be used to modify the inference runtime:
+
+*   `num_interpreter_threads`: `int` (default=1) \
+    This modifies the number of threads used by the TFLite Interpreter for
+    inference.
+
+*   `delegate`: `string` \
+    If provided, tries to use the specified delegate for accuracy evaluation.
+    Valid values: "nnapi", "gpu", "hexagon".
+
+    NOTE: Please refer to the
+    [Hexagon delegate documentation](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/performance/hexagon_delegate.md)
+    for instructions on how to set it up for the Hexagon delegate. The tool
+    assumes that `libhexagon_interface.so` and Qualcomm libraries lie in
+    `/data/local/tmp`.
+
+This script also supports runtime/delegate arguments introduced by the
+[delegate registrar](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/delegates).
+If there is any conflict (for example, `num_threads` vs
+`num_interpreter_threads` here), the parameters of this
+script are given precedence.
+
+When **multiple delegates** are specified to be used in the commandline flags
+via the support of delegate registrar, the order of delegates applied to the
+TfLite runtime will be same as their enabling commandline flag is specified. For
+example, "--use_xnnpack=true --use_gpu=true" means applying the XNNPACK delegate
+first, and then the GPU delegate secondly. In comparison,
+"--use_gpu=true --use_xnnpack=true" means applying the GPU delegate first, and
+then the XNNPACK delegate secondly.
+
+Note, one could specify `--help` when launching the binary to see the full list
+of supported arguments.
+
+## Downloading ILSVRC
+
+In order to use this tool to run evaluation on the full 50K ImageNet dataset,
+download the data set from http://image-net.org/request.
+
+## Ground truth label generation
+
+The ILSVRC 2012 devkit `validation_ground_truth.txt` contains IDs that
+correspond to synset of the image. The accuracy binary however expects the
+ground truth labels to contain the actual name of category instead of synset
+ids. A conversion script has been provided to convert the validation ground
+truth to category labels. The `validation_ground_truth.txt` can be converted by
+the following steps:
+
+```
+ILSVRC_2012_DEVKIT_DIR=[set to path to ILSVRC 2012 devkit]
+VALIDATION_LABELS=[set to  path to output]
+
+python tensorflow/lite/tools/evaluation/tasks/imagenet_image_classification/generate_validation_labels.py \
+--ilsvrc_devkit_dir=${ILSVRC_2012_DEVKIT_DIR} \
+--validation_labels_output=${VALIDATION_LABELS}
+```
+
+## Running the binary
+
+### On Android
+
+(0) Refer to
+https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android
+for configuring NDK and SDK.
+
+(1) Build using the following command:
+
+```
+bazel build -c opt \
+  --config=android_arm64 \
+  --cxxopt='--std=c++17' \
+  //tensorflow/lite/tools/evaluation/tasks/imagenet_image_classification:run_eval
+```
+
+(2) Connect your phone. Push the binary to your phone with adb push (make the
+directory if required):
+
+```
+adb push bazel-bin/tensorflow/lite/tools/evaluation/tasks/imagenet_image_classification/run_eval /data/local/tmp
+```
+
+(3) Make the binary executable.
+
+```
+adb shell chmod +x /data/local/tmp/run_eval
+```
+
+(4) Push the TFLite model that you need to test. For example:
+
+```
+adb push mobilenet_quant_v1_224.tflite /data/local/tmp
+```
+
+(5) Push the imagenet images to device, make sure device has sufficient storage
+available before pushing the dataset:
+
+```
+adb shell mkdir /data/local/tmp/ilsvrc_images && \
+adb push ${IMAGENET_IMAGES_DIR} /data/local/tmp/ilsvrc_images
+```
+
+(6) Push the generated validation ground labels to device.
+
+```
+adb push ${VALIDATION_LABELS} /data/local/tmp/ilsvrc_validation_labels.txt
+```
+
+(7) Push the model labels text file to device.
+
+```
+adb push ${MODEL_LABELS_TXT} /data/local/tmp/model_output_labels.txt
+```
+
+(8) Run the binary.
+
+```
+adb shell /data/local/tmp/run_eval \
+  --model_file=/data/local/tmp/mobilenet_quant_v1_224.tflite \
+  --ground_truth_images_path=/data/local/tmp/ilsvrc_images \
+  --ground_truth_labels=/data/local/tmp/ilsvrc_validation_labels.txt \
+  --model_output_labels=/data/local/tmp/model_output_labels.txt \
+  --output_file_path=/data/local/tmp/accuracy_output.txt \
+  --num_images=0 # Run on all images.
+```
+
+### On Desktop
+
+(1) Build and run using the following command:
+
+```
+bazel run -c opt \
+  -- \
+  //tensorflow/lite/tools/evaluation/tasks/imagenet_image_classification:run_eval \
+  --model_file=mobilenet_quant_v1_224.tflite \
+  --ground_truth_images_path=${IMAGENET_IMAGES_DIR} \
+  --ground_truth_labels=${VALIDATION_LABELS} \
+  --model_output_labels=${MODEL_LABELS_TXT} \
+  --output_file_path=/tmp/accuracy_output.txt \
+  --num_images=0 # Run on all images.
+```

tensorflow/lite/tools/evaluation/tasks/imagenet_image_classification/generate_validation_labels.py

@@ -0,0 +1,101 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tool to convert ILSVRC devkit validation ground truth to synset labels."""
+
+import argparse
+from os import path
+import sys
+import scipy.io
+
+_SYNSET_ARRAYS_RELATIVE_PATH = 'data/meta.mat'
+_VALIDATION_FILE_RELATIVE_PATH = 'data/ILSVRC2012_validation_ground_truth.txt'
+
+
+def _synset_to_word(filepath):
+  """Returns synset to word dictionary by reading sysnset arrays."""
+  mat = scipy.io.loadmat(filepath)
+  entries = mat['synsets']
+  # These fields are listed in devkit readme.txt
+  fields = [
+      'synset_id', 'WNID', 'words', 'gloss', 'num_children', 'children',
+      'wordnet_height', 'num_train_images'
+  ]
+  synset_index = fields.index('synset_id')
+  words_index = fields.index('words')
+  synset_to_word = {}
+  for entry in entries:
+    entry = entry[0]
+    synset_id = int(entry[synset_index][0])
+    first_word = entry[words_index][0].split(',')[0]
+    synset_to_word[synset_id] = first_word
+  return synset_to_word
+
+
+def _validation_file_path(ilsvrc_dir):
+  return path.join(ilsvrc_dir, _VALIDATION_FILE_RELATIVE_PATH)
+
+
+def _synset_array_path(ilsvrc_dir):
+  return path.join(ilsvrc_dir, _SYNSET_ARRAYS_RELATIVE_PATH)
+
+
+def _generate_validation_labels(ilsvrc_dir, output_file):
+  synset_to_word = _synset_to_word(_synset_array_path(ilsvrc_dir))
+  with open(_validation_file_path(ilsvrc_dir), 'r') as synset_id_file, open(
+      output_file, 'w') as output:
+    for synset_id in synset_id_file:
+      synset_id = int(synset_id)
+      output.write('%s\n' % synset_to_word[synset_id])
+
+
+def _check_arguments(args):
+  if not args.validation_labels_output:
+    raise ValueError('Invalid path to output file.')
+  ilsvrc_dir = args.ilsvrc_devkit_dir
+  if not ilsvrc_dir or not path.isdir(ilsvrc_dir):
+    raise ValueError('Invalid path to ilsvrc_dir')
+  if not path.exists(_validation_file_path(ilsvrc_dir)):
+    raise ValueError('Invalid path to ilsvrc_dir, cannot find validation file.')
+  if not path.exists(_synset_array_path(ilsvrc_dir)):
+    raise ValueError(
+        'Invalid path to ilsvrc_dir, cannot find synset arrays file.')
+
+
+def main():
+  parser = argparse.ArgumentParser(
+      description='Converts ILSVRC devkit validation_ground_truth.txt to synset'
+      ' labels file that can be used by the accuracy script.')
+  parser.add_argument(
+      '--validation_labels_output',
+      type=str,
+      help='Full path for outputting validation labels.')
+  parser.add_argument(
+      '--ilsvrc_devkit_dir',
+      type=str,
+      help='Full path to ILSVRC 2012 devkit directory.')
+  args = parser.parse_args()
+  try:
+    _check_arguments(args)
+  except ValueError as e:
+    parser.print_usage()
+    file_name = path.basename(sys.argv[0])
+    sys.stderr.write('{0}: error: {1}\n'.format(file_name, str(e)))
+    sys.exit(1)
+  _generate_validation_labels(args.ilsvrc_devkit_dir,
+                              args.validation_labels_output)
+
+
+if __name__ == '__main__':
+  main()

tensorflow/lite/tools/evaluation/tasks/imagenet_image_classification/run_eval.cc

@@ -0,0 +1,211 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <fstream>
+#include <ios>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "tensorflow/lite/core/c/common.h"
+#include "tensorflow/lite/tools/command_line_flags.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_delegate_provider.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+#include "tensorflow/lite/tools/evaluation/stages/image_classification_stage.h"
+#include "tensorflow/lite/tools/evaluation/tasks/task_executor.h"
+#include "tensorflow/lite/tools/evaluation/utils.h"
+#include "tensorflow/lite/tools/logging.h"
+
+namespace tflite {
+namespace evaluation {
+
+constexpr char kModelFileFlag[] = "model_file";
+constexpr char kGroundTruthImagesPathFlag[] = "ground_truth_images_path";
+constexpr char kGroundTruthLabelsFlag[] = "ground_truth_labels";
+constexpr char kOutputFilePathFlag[] = "output_file_path";
+constexpr char kModelOutputLabelsFlag[] = "model_output_labels";
+constexpr char kDenylistFilePathFlag[] = "denylist_file_path";
+constexpr char kNumImagesFlag[] = "num_images";
+constexpr char kInterpreterThreadsFlag[] = "num_interpreter_threads";
+constexpr char kDelegateFlag[] = "delegate";
+
+template <typename T>
+std::vector<T> GetFirstN(const std::vector<T>& v, int n) {
+  if (n >= v.size()) return v;
+  std::vector<T> result(v.begin(), v.begin() + n);
+  return result;
+}
+
+class ImagenetClassification : public TaskExecutor {
+ public:
+  ImagenetClassification() : num_images_(0), num_interpreter_threads_(1) {}
+  ~ImagenetClassification() override {}
+
+ protected:
+  std::vector<Flag> GetFlags() final;
+
+  // If the run is successful, the latest metrics will be returned.
+  std::optional<EvaluationStageMetrics> RunImpl() final;
+
+ private:
+  void OutputResult(const EvaluationStageMetrics& latest_metrics) const;
+  std::string model_file_path_;
+  std::string ground_truth_images_path_;
+  std::string ground_truth_labels_path_;
+  std::string model_output_labels_path_;
+  std::string denylist_file_path_;
+  std::string output_file_path_;
+  std::string delegate_;
+  int num_images_;
+  int num_interpreter_threads_;
+};
+
+std::vector<Flag> ImagenetClassification::GetFlags() {
+  std::vector<tflite::Flag> flag_list = {
+      tflite::Flag::CreateFlag(kModelFileFlag, &model_file_path_,
+                               "Path to test tflite model file."),
+      tflite::Flag::CreateFlag(
+          kModelOutputLabelsFlag, &model_output_labels_path_,
+          "Path to labels that correspond to output of model."
+          " E.g. in case of mobilenet, this is the path to label "
+          "file where each label is in the same order as the output"
+          " of the model."),
+      tflite::Flag::CreateFlag(
+          kGroundTruthImagesPathFlag, &ground_truth_images_path_,
+          "Path to ground truth images. These will be evaluated in "
+          "alphabetical order of filename"),
+      tflite::Flag::CreateFlag(
+          kGroundTruthLabelsFlag, &ground_truth_labels_path_,
+          "Path to ground truth labels, corresponding to alphabetical ordering "
+          "of ground truth images."),
+      tflite::Flag::CreateFlag(
+          kDenylistFilePathFlag, &denylist_file_path_,
+          "Path to denylist file (optional) where each line is a single "
+          "integer that is "
+          "equal to index number of denylisted image."),
+      tflite::Flag::CreateFlag(kOutputFilePathFlag, &output_file_path_,
+                               "File to output metrics proto to."),
+      tflite::Flag::CreateFlag(kNumImagesFlag, &num_images_,
+                               "Number of examples to evaluate, pass 0 for all "
+                               "examples. Default: 0"),
+      tflite::Flag::CreateFlag(
+          kInterpreterThreadsFlag, &num_interpreter_threads_,
+          "Number of interpreter threads to use for inference."),
+      tflite::Flag::CreateFlag(
+          kDelegateFlag, &delegate_,
+          "Delegate to use for inference, if available. "
+          "Must be one of {'nnapi', 'gpu', 'hexagon', 'xnnpack'}"),
+  };
+  return flag_list;
+}
+
+std::optional<EvaluationStageMetrics> ImagenetClassification::RunImpl() {
+  // Process images in filename-sorted order.
+  std::vector<std::string> image_files, ground_truth_image_labels;
+  if (GetSortedFileNames(StripTrailingSlashes(ground_truth_images_path_),
+                         &image_files) != kTfLiteOk) {
+    return std::nullopt;
+  }
+  if (!ReadFileLines(ground_truth_labels_path_, &ground_truth_image_labels)) {
+    TFLITE_LOG(ERROR) << "Could not read ground truth labels file";
+    return std::nullopt;
+  }
+  if (image_files.size() != ground_truth_image_labels.size()) {
+    TFLITE_LOG(ERROR) << "Number of images and ground truth labels is not same";
+    return std::nullopt;
+  }
+  std::vector<ImageLabel> image_labels;
+  image_labels.reserve(image_files.size());
+  for (int i = 0; i < image_files.size(); i++) {
+    image_labels.push_back({image_files[i], ground_truth_image_labels[i]});
+  }
+
+  // Filter out denylisted/unwanted images.
+  if (FilterDenyListedImages(denylist_file_path_, &image_labels) != kTfLiteOk) {
+    return std::nullopt;
+  }
+  if (num_images_ > 0) {
+    image_labels = GetFirstN(image_labels, num_images_);
+  }
+
+  std::vector<std::string> model_labels;
+  if (!ReadFileLines(model_output_labels_path_, &model_labels)) {
+    TFLITE_LOG(ERROR) << "Could not read model output labels file";
+    return std::nullopt;
+  }
+
+  EvaluationStageConfig eval_config;
+  eval_config.set_name("image_classification");
+  auto* classification_params = eval_config.mutable_specification()
+                                    ->mutable_image_classification_params();
+  auto* inference_params = classification_params->mutable_inference_params();
+  inference_params->set_model_file_path(model_file_path_);
+  inference_params->set_num_threads(num_interpreter_threads_);
+  inference_params->set_delegate(ParseStringToDelegateType(delegate_));
+  classification_params->mutable_topk_accuracy_eval_params()->set_k(10);
+
+  ImageClassificationStage eval(eval_config);
+
+  eval.SetAllLabels(model_labels);
+  if (eval.Init(&delegate_providers_) != kTfLiteOk) return std::nullopt;
+
+  const int step = image_labels.size() / 100;
+  for (int i = 0; i < image_labels.size(); ++i) {
+    if (step > 1 && i % step == 0) {
+      TFLITE_LOG(INFO) << "Evaluated: " << i / step << "%";
+    }
+    eval.SetInputs(image_labels[i].image, image_labels[i].label);
+    if (eval.Run() != kTfLiteOk) return std::nullopt;
+  }
+
+  const auto latest_metrics = eval.LatestMetrics();
+  OutputResult(latest_metrics);
+  return std::make_optional(latest_metrics);
+}
+
+void ImagenetClassification::OutputResult(
+    const EvaluationStageMetrics& latest_metrics) const {
+  if (!output_file_path_.empty()) {
+    std::ofstream metrics_ofile;
+    metrics_ofile.open(output_file_path_, std::ios::out);
+    metrics_ofile << latest_metrics.SerializeAsString();
+    metrics_ofile.close();
+  }
+
+  TFLITE_LOG(INFO) << "Num evaluation runs: " << latest_metrics.num_runs();
+  const auto& metrics =
+      latest_metrics.process_metrics().image_classification_metrics();
+  const auto& preprocessing_latency = metrics.pre_processing_latency();
+  TFLITE_LOG(INFO) << "Preprocessing latency: avg="
+                   << preprocessing_latency.avg_us() << "(us), std_dev="
+                   << preprocessing_latency.std_deviation_us() << "(us)";
+  const auto& inference_latency = metrics.inference_latency();
+  TFLITE_LOG(INFO) << "Inference latency: avg=" << inference_latency.avg_us()
+                   << "(us), std_dev=" << inference_latency.std_deviation_us()
+                   << "(us)";
+  const auto& accuracy_metrics = metrics.topk_accuracy_metrics();
+  for (int i = 0; i < accuracy_metrics.topk_accuracies_size(); ++i) {
+    TFLITE_LOG(INFO) << "Top-" << i + 1
+                     << " Accuracy: " << accuracy_metrics.topk_accuracies(i);
+  }
+}
+
+std::unique_ptr<TaskExecutor> CreateTaskExecutor() {
+  return std::unique_ptr<TaskExecutor>(new ImagenetClassification());
+}
+
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/tasks/inference_diff/BUILD

@@ -0,0 +1,42 @@
+load("@rules_cc//cc:cc_binary.bzl", "cc_binary")
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
+load("//tensorflow/lite:build_def.bzl", "tflite_copts")
+load("//tensorflow/lite/tools/evaluation/tasks:build_def.bzl", "task_linkopts")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_visibility = [
+        "//visibility:public",
+    ],
+    licenses = ["notice"],
+)
+
+cc_library(
+    name = "run_eval_lib",
+    srcs = ["run_eval.cc"],
+    copts = tflite_copts(),
+    linkopts = task_linkopts(),
+    deps = [
+        "//tensorflow/lite/core/c:common",
+        "//tensorflow/lite/tools:command_line_flags",
+        "//tensorflow/lite/tools:logging",
+        "//tensorflow/lite/tools/evaluation:evaluation_delegate_provider",
+        "//tensorflow/lite/tools/evaluation:evaluation_stage",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
+        "//tensorflow/lite/tools/evaluation/stages:inference_profiler_stage",
+        "//tensorflow/lite/tools/evaluation/tasks:task_executor",
+        "@com_google_absl//absl/types:optional",
+    ],
+    alwayslink = 1,
+)
+
+cc_binary(
+    name = "run_eval",
+    copts = tflite_copts(),
+    linkopts = task_linkopts(),
+    deps = [
+        ":run_eval_lib",
+        "//tensorflow/lite/tools/evaluation/tasks:task_executor_main",
+    ],
+)

tensorflow/lite/tools/evaluation/tasks/inference_diff/README.md

@@ -0,0 +1,125 @@
+## Inference Diff tool
+
+**NOTE: This is an experimental tool to analyze TensorFlow Lite behavior on
+delegates.**
+
+For a given model, this binary compares TensorFlow Lite execution (in terms of
+latency & output-value deviation) in two settings:
+
+*   Single-threaded CPU Inference
+*   User-defined Inference
+
+To do so, the tool generates random gaussian data and passes it through two
+TFLite Interpreters - one running single-threaded CPU kernels and the other
+parameterized by the user's arguments.
+
+It measures the latency of both, as well as the absolute difference between the
+output tensors from each Interpreter, on a per-element basis.
+
+The final output (logged to stdout) typically looks like this:
+
+```
+Num evaluation runs: 50
+Reference run latency: avg=84364.2(us), std_dev=12525(us)
+Test run latency: avg=7281.64(us), std_dev=2089(us)
+OutputDiff[0]: avg_error=1.96277e-05, std_dev=6.95767e-06
+```
+
+There is one instance of `OutputDiff` for each output tensor in the model, and
+the statistics in `OutputDiff[i]` correspond to the absolute difference in raw
+values across all elements for the `i`th output.
+
+## Parameters
+
+(In this section, 'test Interpreter' refers to the User-defined Inference
+mentioned above. The reference setting is always single-threaded CPU).
+
+The binary takes the following parameters:
+
+*   `model_file` : `string` \
+    Path to the TFlite model file.
+
+and the following optional parameters:
+
+*   `num_runs`: `int` \
+    How many runs to perform to compare execution in reference and test setting.
+    Default: 50. The binary performs runs 3 invocations per 'run', to get more
+    accurate latency numbers.
+
+*   `num_interpreter_threads`: `int` (default=1) \
+    This modifies the number of threads used by the test Interpreter for
+    inference.
+
+*   `delegate`: `string` \
+    If provided, tries to use the specified delegate on the test Interpreter.
+    Valid values: "nnapi", "gpu", "hexagon", "coreml".
+
+    NOTE: Please refer to the
+    [Hexagon delegate documentation](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/performance/hexagon_delegate.md)
+    for instructions on how to set it up for the Hexagon delegate. The tool
+    assumes that `libhexagon_interface.so` and Qualcomm libraries lie in
+    `/data/local/tmp`.
+
+*   `output_file_path`: `string` \
+    The final metrics are dumped into `output_file_path` as a serialized
+    instance of `tflite::evaluation::EvaluationStageMetrics`
+
+This script also supports runtime/delegate arguments introduced by the
+[delegate registrar](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/delegates).
+If there is any conflict (for example, `num_threads` vs
+`num_interpreter_threads` here), the parameters of this
+script are given precedence.
+
+When **multiple delegates** are specified to be used in the commandline flags
+via the support of delegate registrar, the order of delegates applied to the
+TfLite runtime will be same as their enabling commandline flag is specified. For
+example, "--use_xnnpack=true --use_gpu=true" means applying the XNNPACK delegate
+first, and then the GPU delegate secondly. In comparison,
+"--use_gpu=true --use_xnnpack=true" means applying the GPU delegate first, and
+then the XNNPACK delegate secondly.
+
+Note, one could specify `--help` when launching the binary to see the full list
+of supported arguments.
+
+## Running the binary on Android
+
+(1) Build using the following command:
+
+```
+bazel build -c opt \
+  --config=android_arm64 \
+  //tensorflow/lite/tools/evaluation/tasks/inference_diff:run_eval
+```
+
+(2) Connect your phone. Push the binary to your phone with adb push (make the
+directory if required):
+
+```
+adb push bazel-bin/third_party/tensorflow/lite/tools/evaluation/tasks/inference_diff/run_eval /data/local/tmp
+```
+
+(3) Push the TFLite model that you need to test. For example:
+
+```
+adb push mobilenet_v1_1.0_224.tflite /data/local/tmp
+```
+
+(3) Run the binary.
+
+```
+adb shell /data/local/tmp/run_eval \
+  --model_file=/data/local/tmp/mobilenet_v1_1.0_224.tflite \
+  --delegate=gpu
+```
+
+(5) Pull the results.
+
+```
+adb pull /data/local/tmp/inference_diff.txt ~/accuracy_tool
+```
+
+## Running the binary on iOS
+
+Follow the instructions [here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/evaluation/tasks/ios/README.md)
+to run the binary on iOS using the
+[iOS evaluation app](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/evaluation/tasks/ios).

tensorflow/lite/tools/evaluation/tasks/inference_diff/run_eval.cc

@@ -0,0 +1,148 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <fstream>
+#include <ios>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "tensorflow/lite/core/c/common.h"
+#include "tensorflow/lite/tools/command_line_flags.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_delegate_provider.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+#include "tensorflow/lite/tools/evaluation/stages/inference_profiler_stage.h"
+#include "tensorflow/lite/tools/evaluation/tasks/task_executor.h"
+#include "tensorflow/lite/tools/logging.h"
+
+namespace tflite {
+namespace evaluation {
+
+constexpr char kModelFileFlag[] = "model_file";
+constexpr char kOutputFilePathFlag[] = "output_file_path";
+constexpr char kNumRunsFlag[] = "num_runs";
+constexpr char kInterpreterThreadsFlag[] = "num_interpreter_threads";
+constexpr char kDelegateFlag[] = "delegate";
+
+class InferenceDiff : public TaskExecutor {
+ public:
+  InferenceDiff() : num_runs_(50), num_interpreter_threads_(1) {}
+  ~InferenceDiff() override {}
+
+ protected:
+  std::vector<Flag> GetFlags() final;
+
+  // If the run is successful, the latest metrics will be returned.
+  std::optional<EvaluationStageMetrics> RunImpl() final;
+
+ private:
+  void OutputResult(const EvaluationStageMetrics& latest_metrics) const;
+  std::string model_file_path_;
+  std::string output_file_path_;
+  std::string delegate_;
+  int num_runs_;
+  int num_interpreter_threads_;
+};
+
+std::vector<Flag> InferenceDiff::GetFlags() {
+  // Command Line Flags.
+  std::vector<tflite::Flag> flag_list = {
+      tflite::Flag::CreateFlag(kModelFileFlag, &model_file_path_,
+                               "Path to test tflite model file."),
+      tflite::Flag::CreateFlag(kOutputFilePathFlag, &output_file_path_,
+                               "File to output metrics proto to."),
+      tflite::Flag::CreateFlag(kNumRunsFlag, &num_runs_,
+                               "Number of runs of test & reference inference "
+                               "each. Default value: 50"),
+      tflite::Flag::CreateFlag(
+          kInterpreterThreadsFlag, &num_interpreter_threads_,
+          "Number of interpreter threads to use for test inference."),
+      tflite::Flag::CreateFlag(
+          kDelegateFlag, &delegate_,
+          "Delegate to use for test inference, if available. "
+          "Must be one of {'nnapi', 'gpu', 'hexagon', 'xnnpack', 'coreml'}"),
+  };
+
+  return flag_list;
+}
+
+std::optional<EvaluationStageMetrics> InferenceDiff::RunImpl() {
+  // Initialize evaluation stage.
+  EvaluationStageConfig eval_config;
+  eval_config.set_name("inference_profiling");
+  auto* inference_params =
+      eval_config.mutable_specification()->mutable_tflite_inference_params();
+  inference_params->set_model_file_path(model_file_path_);
+  inference_params->set_num_threads(num_interpreter_threads_);
+  // This ensures that latency measurement isn't hampered by the time spent in
+  // generating random data.
+  inference_params->set_invocations_per_run(3);
+  inference_params->set_delegate(ParseStringToDelegateType(delegate_));
+  if (!delegate_.empty() &&
+      inference_params->delegate() == TfliteInferenceParams::NONE) {
+    TFLITE_LOG(WARN) << "Unsupported TFLite delegate: " << delegate_;
+    return std::nullopt;
+  }
+
+  InferenceProfilerStage eval(eval_config);
+  if (eval.Init(&delegate_providers_) != kTfLiteOk) return std::nullopt;
+
+  // Run inference & check diff for specified number of runs.
+  for (int i = 0; i < num_runs_; ++i) {
+    if (eval.Run() != kTfLiteOk) return std::nullopt;
+  }
+
+  const auto latest_metrics = eval.LatestMetrics();
+  OutputResult(latest_metrics);
+  return std::make_optional(latest_metrics);
+}
+
+void InferenceDiff::OutputResult(
+    const EvaluationStageMetrics& latest_metrics) const {
+  // Output latency & diff metrics.
+  if (!output_file_path_.empty()) {
+    std::ofstream metrics_ofile;
+    metrics_ofile.open(output_file_path_, std::ios::out);
+    metrics_ofile << latest_metrics.SerializeAsString();
+    metrics_ofile.close();
+  }
+
+  TFLITE_LOG(INFO) << "Num evaluation runs: " << latest_metrics.num_runs();
+  const auto& metrics =
+      latest_metrics.process_metrics().inference_profiler_metrics();
+  const auto& ref_latency = metrics.reference_latency();
+  TFLITE_LOG(INFO) << "Reference run latency: avg=" << ref_latency.avg_us()
+                   << "(us), std_dev=" << ref_latency.std_deviation_us()
+                   << "(us)";
+  const auto& test_latency = metrics.test_latency();
+  TFLITE_LOG(INFO) << "Test run latency: avg=" << test_latency.avg_us()
+                   << "(us), std_dev=" << test_latency.std_deviation_us()
+                   << "(us)";
+  const auto& output_errors = metrics.output_errors();
+  for (int i = 0; i < output_errors.size(); ++i) {
+    const auto& error = output_errors.at(i);
+    TFLITE_LOG(INFO) << "OutputDiff[" << i
+                     << "]: avg_error=" << error.avg_value()
+                     << ", std_dev=" << error.std_deviation();
+  }
+}
+
+std::unique_ptr<TaskExecutor> CreateTaskExecutor() {
+  return std::unique_ptr<TaskExecutor>(new InferenceDiff());
+}
+
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/tasks/ios/BUILD.apple

@@ -0,0 +1,37 @@
+load("@bazel_skylib//rules:build_test.bzl", "build_test")
+load("@build_bazel_rules_apple//apple:ios.bzl", "ios_static_framework")
+load("//tensorflow/lite/ios:ios.bzl", "TFL_MINIMUM_OS_VERSION")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
+    licenses = ["notice"],
+)
+
+# Main target for the inference diff tool iOS framework.
+# bazel build --config=ios_arm64 -c opt --cxxopt=-std=c++17 //tensorflow/lite/tools/evaluation/tasks/ios:TensorFlowLiteInferenceDiffC_framework
+ios_static_framework(
+    name = "TensorFlowLiteInferenceDiffC_framework",
+    hdrs = [
+        "//tensorflow/lite/tools:logging.h",
+        "//tensorflow/lite/tools/evaluation/tasks:task_executor_c_api.h",
+    ],
+    bundle_name = "TensorFlowLiteInferenceDiffC",
+    minimum_os_version = TFL_MINIMUM_OS_VERSION,
+    deps = [
+        "//tensorflow/lite/tools/evaluation/tasks:task_executor_c_api",
+        "//tensorflow/lite/tools/evaluation/tasks/inference_diff:run_eval_lib",
+    ],
+)
+
+# Used for building TensorFlowLiteInferenceDiffC_framework framework.
+build_test(
+    name = "framework_build_test",
+    # build_test targets are not meant to be run with sanitizers.
+    tags = [
+        "nomsan",
+        "notsan",
+    ],
+    targets = [
+        ":TensorFlowLiteInferenceDiffC_framework",
+    ],
+)

tensorflow/lite/tools/evaluation/tasks/ios/README.md

@@ -0,0 +1,43 @@
+# TFLite iOS evaluation app.
+
+## Description
+
+An iOS app to evaluate TFLite models. This is mainly for running different
+evaluation tasks on iOS. Right now it only supports evaluating the inference
+difference between cpu and delegates.
+
+The app reads evaluation parameters from a JSON file named
+`evaluation_params.json` in its `evaluation_data` directory. Any downloaded
+models for evaluation should also be placed in `evaluation_data` directory.
+
+The JSON file specifies the name of the model file and other evaluation
+parameters like number of iterations, number of threads, delegate name. The
+default values in the JSON file are for the Mobilenet_v2_1.0_224 model
+([tflite&pb][mobilenet-model]).
+
+## Building / running the app
+
+*   Follow the [iOS build instructions][build-ios] to configure the Bazel
+    workspace and `.bazelrc` file correctly.
+
+*   Run `build_evaluation_framework.sh` script to build the evaluation
+    framework. This script will build the evaluation framework targeting iOS
+    arm64 and put it under `TFLiteEvaluation/TFLiteEvaluation/Frameworks`
+    directory.
+
+*   Update evaluation parameters in `evaluation_params.json`.
+
+*   Change `Build Phases -> Copy Bundle Resources` and add the model file to the
+    resources that need to be copied.
+
+*   Ensure that `Build Phases -> Link Binary With Library` contains the
+    `Accelerate framework` and `TensorFlowLiteInferenceDiffC.framework`.
+
+*   Now try running the app. The app has a single button that runs the
+    evaluation on the model and displays results in a text view below. You can
+    also see the console output section in your Xcode to see more detailed
+    information.
+
+[build-ios]: https://tensorflow.org/lite/guide/build_ios
+[mobilenet-model]: https://github.com/tensorflow/tflite-support/raw/master/tensorflow_lite_support/metadata/python/tests/testdata/image_classifier/mobilenet_v2_1.0_224.tflite
+[mobilenet-paper]: https://arxiv.org/pdf/1704.04861.pdf

tensorflow/lite/tools/evaluation/tasks/ios/TFLiteEvaluation/TFLiteEvaluation/AppDelegate.h

@@ -0,0 +1,22 @@
+// Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <UIKit/UIKit.h>
+
+@interface AppDelegate : UIResponder <UIApplicationDelegate>
+
+@property(strong, nonatomic) UIWindow *window;
+
+@end

tensorflow/lite/tools/evaluation/tasks/ios/TFLiteEvaluation/TFLiteEvaluation/AppDelegate.m

@@ -0,0 +1,27 @@
+// Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "AppDelegate.h"
+
+@interface AppDelegate ()
+
+@end
+
+@implementation AppDelegate
+- (BOOL)application:(UIApplication *)application
+    didFinishLaunchingWithOptions:(NSDictionary *)launchOptions {
+  return YES;
+}
+@end

tensorflow/lite/tools/evaluation/tasks/ios/TFLiteEvaluation/TFLiteEvaluation/Assets.xcassets/AccentColor.colorset/Contents.json

@@ -0,0 +1,11 @@
+{
+  "colors" : [
+    {
+      "idiom" : "universal"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

tensorflow/lite/tools/evaluation/tasks/ios/TFLiteEvaluation/TFLiteEvaluation/Assets.xcassets/AppIcon.appiconset/Contents.json

@@ -0,0 +1,98 @@
+{
+  "images" : [
+    {
+      "idiom" : "iphone",
+      "scale" : "2x",
+      "size" : "20x20"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "3x",
+      "size" : "20x20"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "2x",
+      "size" : "29x29"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "3x",
+      "size" : "29x29"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "2x",
+      "size" : "40x40"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "3x",
+      "size" : "40x40"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "2x",
+      "size" : "60x60"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "3x",
+      "size" : "60x60"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "1x",
+      "size" : "20x20"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "2x",
+      "size" : "20x20"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "1x",
+      "size" : "29x29"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "2x",
+      "size" : "29x29"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "1x",
+      "size" : "40x40"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "2x",
+      "size" : "40x40"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "1x",
+      "size" : "76x76"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "2x",
+      "size" : "76x76"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "2x",
+      "size" : "83.5x83.5"
+    },
+    {
+      "idiom" : "ios-marketing",
+      "scale" : "1x",
+      "size" : "1024x1024"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

tensorflow/lite/tools/evaluation/tasks/ios/TFLiteEvaluation/TFLiteEvaluation/Assets.xcassets/Contents.json

@@ -0,0 +1,6 @@
+{
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

tensorflow/lite/tools/evaluation/tasks/ios/TFLiteEvaluation/TFLiteEvaluation/Base.lproj/Main.storyboard

@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="23727" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
+    <device id="retina6_1" orientation="portrait" appearance="light"/>
+    <dependencies>
+        <deployment identifier="iOS"/>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="23721"/>
+        <capability name="Safe area layout guides" minToolsVersion="9.0"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--Evaluation View Controller-->
+        <scene sceneID="tne-QT-ifu">
+            <objects>
+                <viewController id="BYZ-38-t0r" customClass="EvaluationViewController" sceneMemberID="viewController">
+                    <view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
+                        <rect key="frame" x="0.0" y="0.0" width="414" height="896"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <subviews>
+                            <button opaque="NO" contentMode="scaleToFill" misplaced="YES" contentHorizontalAlignment="center" contentVerticalAlignment="center" buttonType="system" lineBreakMode="middleTruncation" translatesAutoresizingMaskIntoConstraints="NO" id="j0O-Lq-1tJ">
+                                <rect key="frame" x="64" y="97" width="286" height="63"/>
+                                <constraints>
+                                    <constraint firstAttribute="height" constant="63" id="8VO-Ln-L2h"/>
+                                </constraints>
+                                <fontDescription key="fontDescription" type="system" pointSize="24"/>
+                                <state key="normal" title="Evaluate model"/>
+                                <connections>
+                                    <action selector="onEvaluateModel:" destination="BYZ-38-t0r" eventType="touchUpInside" id="Rb1-hs-Mub"/>
+                                </connections>
+                            </button>
+                            <textView clipsSubviews="YES" multipleTouchEnabled="YES" contentMode="scaleToFill" editable="NO" textAlignment="natural" selectable="NO" translatesAutoresizingMaskIntoConstraints="NO" id="Vd4-Gf-qKO">
+                                <rect key="frame" x="26" y="177" width="368" height="641"/>
+                                <color key="backgroundColor" white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
+                                <fontDescription key="fontDescription" type="system" pointSize="14"/>
+                                <textInputTraits key="textInputTraits" autocapitalizationType="sentences"/>
+                            </textView>
+                        </subviews>
+                        <viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                        <constraints>
+                            <constraint firstItem="Vd4-Gf-qKO" firstAttribute="top" secondItem="j0O-Lq-1tJ" secondAttribute="bottom" constant="18" id="Kd3-pP-C1k"/>
+                            <constraint firstItem="j0O-Lq-1tJ" firstAttribute="centerX" secondItem="8bC-Xf-vdC" secondAttribute="centerX" id="QJU-cq-L87"/>
+                            <constraint firstItem="Vd4-Gf-qKO" firstAttribute="trailing" secondItem="8bC-Xf-vdC" secondAttribute="trailingMargin" id="Tew-W4-Vq5"/>
+                            <constraint firstItem="j0O-Lq-1tJ" firstAttribute="top" secondItem="6Tk-OE-BBY" secondAttribute="top" id="Uce-n7-kZI"/>
+                            <constraint firstItem="j0O-Lq-1tJ" firstAttribute="leading" secondItem="6Tk-OE-BBY" secondAttribute="leading" constant="64" id="Uhq-Rw-NKT"/>
+                            <constraint firstItem="Vd4-Gf-qKO" firstAttribute="leading" secondItem="6Tk-OE-BBY" secondAttribute="leading" constant="26" id="aXc-6M-kyL"/>
+                            <constraint firstItem="6Tk-OE-BBY" firstAttribute="bottom" secondItem="Vd4-Gf-qKO" secondAttribute="bottom" constant="10" id="tz5-wP-LZs"/>
+                        </constraints>
+                    </view>
+                    <connections>
+                        <outlet property="resultsView" destination="Vd4-Gf-qKO" id="dBT-f6-SYw"/>
+                    </connections>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="140" y="122.78860569715144"/>
+        </scene>
+    </scenes>
+</document>

tensorflow/lite/tools/evaluation/tasks/ios/TFLiteEvaluation/TFLiteEvaluation/EvaluationViewController.h

@@ -0,0 +1,22 @@
+// Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <Foundation/Foundation.h>
+#import <UIKit/UIKit.h>
+
+@interface EvaluationViewController : UIViewController
+@property(weak, nonatomic) IBOutlet UITextView *resultsView;
+
+@end

tensorflow/lite/tools/evaluation/tasks/ios/TFLiteEvaluation/TFLiteEvaluation/EvaluationViewController.mm

@@ -0,0 +1,141 @@
+// Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "EvaluationViewController.h"
+#import <algorithm>
+#include <fstream>
+#import <sstream>
+#import <string>
+#import <vector>
+
+#import <TensorFlowLiteInferenceDiffC/TensorFlowLiteInferenceDiffC.h>
+
+namespace {
+
+NSString* const kDocumentsPrefix = @"/Documents/";
+NSString* const kModelFileKey = @"model_file";
+NSString* const kOutputFilePathKey = @"output_file_path";
+
+NSString* FilePathForResourceName(NSString* filename) {
+  NSString* name = [filename stringByDeletingPathExtension];
+  NSString* extension = [filename pathExtension];
+  NSString* file_path = [[NSBundle mainBundle] pathForResource:name ofType:extension];
+  if (file_path == NULL) {
+    TFLITE_LOG(FATAL) << "Couldn't find '" << [name UTF8String] << "." << [extension UTF8String]
+                      << "' in bundle.";
+  }
+  return file_path;
+}
+
+NSDictionary* ParseEvaluationParamsJson() {
+  NSString* params_json_path = FilePathForResourceName(@"evaluation_params.json");
+  NSData* data = [NSData dataWithContentsOfFile:params_json_path];
+  NSDictionary* dict = [NSJSONSerialization JSONObjectWithData:data options:kNilOptions error:nil];
+  return dict;
+}
+
+std::string FormatCommandLineParam(NSString* key, NSString* value) {
+  std::ostringstream stream;
+  stream << "--" << [key UTF8String] << "=" << [value UTF8String];
+  return stream.str();
+}
+
+// Reads the |evaluation_params.json| to read command line parameters and returns them as a vector
+// of strings.
+// Returns the evaluation parameters as key-value pairs.
+void ReadCommandLineParameters(std::vector<std::string>* params) {
+  NSDictionary* param_dict = ParseEvaluationParamsJson();
+  for (NSString* key in param_dict) {
+    NSString* value = param_dict[key];
+    if ([key isEqualToString:kModelFileKey]) {
+      value = FilePathForResourceName(value);
+    }
+    if ([key isEqualToString:kOutputFilePathKey]) {
+      if (![value hasPrefix:kDocumentsPrefix]) {
+        TFLITE_LOG(FATAL) << "Output file must be under the Document directory";
+      }
+      // Replace the prefix "/Documents/" with the actual documents path on the device.
+      NSString* documents = [NSSearchPathForDirectoriesInDomains(
+          NSDocumentDirectory, NSUserDomainMask, YES) objectAtIndex:0];
+      NSString* relpath = [value substringFromIndex:[kDocumentsPrefix length]];
+      value = [documents stringByAppendingPathComponent:relpath];
+
+      // Create the output directory if necessary.
+      NSString* path = value.stringByDeletingLastPathComponent;
+      if (![[NSFileManager defaultManager] createDirectoryAtPath:path
+                                     withIntermediateDirectories:YES
+                                                      attributes:nil
+                                                           error:nil]) {
+        TFLITE_LOG(FATAL) << "Cannot create output directory: " << [path UTF8String];
+      }
+
+      // Create the output file.
+      std::string output_file_path_ = std::string([value UTF8String]);
+      std::unique_ptr<std::ofstream> output_file_ =
+          std::make_unique<std::ofstream>(output_file_path_, std::ios::out | std::ios::binary);
+      if (!output_file_->is_open()) {
+        TFLITE_LOG(ERROR) << "Cannot open output file: " << output_file_path_;
+      } else {
+        TFLITE_LOG(INFO) << "Create output file: " << output_file_path_;
+      }
+    }
+    params->push_back(FormatCommandLineParam(key, value));
+  }
+}
+
+std::vector<char*> StringVecToCharPtrVec(const std::vector<std::string>& str_vec) {
+  std::vector<char*> charptr_vec;
+  std::transform(str_vec.begin(), str_vec.end(), std::back_inserter(charptr_vec),
+                 [](const std::string& s) -> char* { return const_cast<char*>(s.c_str()); });
+  return charptr_vec;
+}
+
+std::string EvaluationMetricsToString(TfLiteEvaluationMetrics* metrics) {
+  std::ostringstream stream;
+  stream << "Num evaluation runs: " << TfLiteEvaluationMetricsGetNumRuns(metrics);
+  TfLiteEvaluationMetricsLatency ref_latency = TfLiteEvaluationMetricsGetReferenceLatency(metrics);
+  stream << "\nReference run latency: avg=" << ref_latency.avg_us
+         << "(us), std_dev=" << ref_latency.std_deviation_us << "(us)";
+  TfLiteEvaluationMetricsLatency test_latency = TfLiteEvaluationMetricsGetTestLatency(metrics);
+  stream << "\nTest run latency: avg=" << test_latency.avg_us
+         << "(us), std_dev=" << test_latency.std_deviation_us << "(us)";
+  for (int i = 0; i < TfLiteEvaluationMetricsGetOutputErrorCount(metrics); ++i) {
+    TfLiteEvaluationMetricsAccuracy error = TfLiteEvaluationMetricsGetOutputError(metrics, i);
+    stream << "\nOutputDiff[" << i << "]: avg_error=" << error.avg_value
+           << ", std_dev=" << error.std_deviation;
+  }
+
+  return stream.str();
+}
+
+TfLiteEvaluationMetrics* RunEvaluation() {
+  std::vector<std::string> command_line_params;
+  ReadCommandLineParameters(&command_line_params);
+  std::vector<char*> argv = StringVecToCharPtrVec(command_line_params);
+  int argc = static_cast<int>(argv.size());
+  TfLiteEvaluationTask* task = TfLiteEvaluationTaskCreate();
+  return TfLiteEvaluationTaskRunWithArgs(task, argc, argv.data());
+}
+}  // namespace
+
+@interface EvaluationViewController ()
+@end
+
+@implementation EvaluationViewController
+- (IBAction)onEvaluateModel:(UIButton*)sender {
+  TfLiteEvaluationMetrics* metrics = RunEvaluation();
+  [_resultsView setText:[NSString stringWithUTF8String:EvaluationMetricsToString(metrics).c_str()]];
+}
+@end

tensorflow/lite/tools/evaluation/tasks/ios/TFLiteEvaluation/TFLiteEvaluation/Info.plist

@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>UILaunchStoryboardName</key>
+	<string>Main</string>
+	<key>CFBundleDevelopmentRegion</key>
+	<string>$(DEVELOPMENT_LANGUAGE)</string>
+	<key>CFBundleExecutable</key>
+	<string>$(EXECUTABLE_NAME)</string>
+	<key>CFBundleIdentifier</key>
+	<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
+	<key>CFBundleInfoDictionaryVersion</key>
+	<string>6.0</string>
+	<key>CFBundleName</key>
+	<string>$(PRODUCT_NAME)</string>
+	<key>CFBundlePackageType</key>
+	<string>APPL</string>
+	<key>CFBundleShortVersionString</key>
+	<string>1.0</string>
+	<key>CFBundleVersion</key>
+	<string>1</string>
+	<key>LSRequiresIPhoneOS</key>
+	<true/>
+	<key>UIMainStoryboardFile</key>
+	<string>Main</string>
+	<key>UIRequiredDeviceCapabilities</key>
+	<array>
+		<string>arm64</string>
+	</array>
+	<key>UISupportedInterfaceOrientations</key>
+	<array>
+		<string>UIInterfaceOrientationPortrait</string>
+	</array>
+	<key>UISupportedInterfaceOrientations~ipad</key>
+	<array>
+		<string>UIInterfaceOrientationPortrait</string>
+		<string>UIInterfaceOrientationPortraitUpsideDown</string>
+		<string>UIInterfaceOrientationLandscapeLeft</string>
+		<string>UIInterfaceOrientationLandscapeRight</string>
+	</array>
+</dict>
+</plist>

tensorflow/lite/tools/evaluation/tasks/ios/TFLiteEvaluation/TFLiteEvaluation/evaluation_data/evaluation_params.json

@@ -0,0 +1,8 @@
+{
+    "model_file" : "mobilenet_v2_1.0_224.tflite",
+    "output_file_path" : "/Documents/outputs/inference_diff.txt",
+    "num_runs" : "50",
+    "num_threads" : "1",
+    "num_interpreter_threads" : "1",
+    "delegate" : "coreml"
+}

tensorflow/lite/tools/evaluation/tasks/ios/TFLiteEvaluation/TFLiteEvaluation/main.m

@@ -0,0 +1,26 @@
+// Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <UIKit/UIKit.h>
+#import "AppDelegate.h"
+
+int main(int argc, char* argv[]) {
+  NSString* appDelegateClassName;
+  @autoreleasepool {
+    // Setup code that might create autoreleased objects goes here.
+    appDelegateClassName = NSStringFromClass([AppDelegate class]);
+  }
+  return UIApplicationMain(argc, argv, nil, appDelegateClassName);
+}

tensorflow/lite/tools/evaluation/tasks/ios/build_evaluation_framework.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+set -e
+
+WORKSPACE_ROOT=$(bazel info workspace 2> /dev/null)
+EVALUATION_DIR=tensorflow/lite/tools/evaluation/tasks
+DEST_DIR="${EVALUATION_DIR}/ios/TFLiteEvaluation/TFLiteEvaluation/Frameworks"
+FRAMEWORK_TARGET=TensorFlowLiteInferenceDiffC_framework
+
+function usage() {
+  echo "Usage: $(basename "$0")"
+  exit 1
+}
+
+function check_ios_configured() {
+  if [ ! -f "${WORKSPACE_ROOT}/${EVALUATION_DIR}/ios/BUILD" ]; then
+    echo "ERROR: Inference Diff framework BUILD file not found."
+    echo "Please enable iOS support by running the \"./configure\" script" \
+         "from the workspace root."
+    exit 1
+  fi
+}
+
+function build_framework() {
+  set -x
+  pushd "${WORKSPACE_ROOT}"
+
+# Build the framework.
+  bazel build --config=ios_arm64 -c opt --cxxopt=-std=c++17 \
+      "//${EVALUATION_DIR}/ios:${FRAMEWORK_TARGET}"
+
+# Get the generated framework path.
+BAZEL_OUTPUT_FILE_PATH=$(bazel cquery "//${EVALUATION_DIR}/ios:${FRAMEWORK_TARGET}" --config=ios_arm64 --output=starlark --starlark:expr="' '.join([f.path for f in target.files.to_list()])")
+
+# Copy the framework into the destination and unzip.
+  mkdir -p "${DEST_DIR}"
+  cp -f "${BAZEL_OUTPUT_FILE_PATH}" \
+      "${DEST_DIR}"
+  pushd "${DEST_DIR}"
+  unzip -o "${FRAMEWORK_TARGET}.zip"
+  rm -f "${FRAMEWORK_TARGET}.zip"
+
+  popd
+  popd
+}
+
+check_ios_configured
+build_framework

tensorflow/lite/tools/evaluation/tasks/task_executor.cc

@@ -0,0 +1,50 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/evaluation/tasks/task_executor.h"
+
+#include <optional>
+#include <string>
+
+#include "tensorflow/lite/tools/command_line_flags.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/logging.h"
+
+namespace tflite {
+namespace evaluation {
+std::optional<EvaluationStageMetrics> TaskExecutor::Run(int* argc,
+                                                        char* argv[]) {
+  auto flag_list = GetFlags();
+  auto delegate_flags = delegate_providers_.GetFlags();
+
+  flag_list.insert(flag_list.end(), delegate_flags.begin(),
+                   delegate_flags.end());
+  bool parse_result =
+      tflite::Flags::Parse(argc, const_cast<const char**>(argv), flag_list);
+  if (!parse_result) {
+    std::string usage = Flags::Usage(argv[0], flag_list);
+    TFLITE_LOG(ERROR) << usage;
+    return std::nullopt;
+  }
+
+  std::string unconsumed_args =
+      Flags::ArgsToString(*argc, const_cast<const char**>(argv));
+  if (!unconsumed_args.empty()) {
+    TFLITE_LOG(WARN) << "Unconsumed cmdline flags: " << unconsumed_args;
+  }
+
+  return RunImpl();
+}
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/tasks/task_executor.h

@@ -0,0 +1,51 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_EVALUATION_TASKS_TASK_EXECUTOR_H_
+#define TENSORFLOW_LITE_TOOLS_EVALUATION_TASKS_TASK_EXECUTOR_H_
+
+#include <optional>
+
+#include "absl/types/optional.h"
+#include "tensorflow/lite/tools/command_line_flags.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_delegate_provider.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+
+namespace tflite {
+namespace evaluation {
+// A common task execution API to avoid boilerpolate code in defining the main
+// function.
+class TaskExecutor {
+ public:
+  virtual ~TaskExecutor() {}
+
+  // If the run is successful, the latest metrics will be returned.
+  std::optional<EvaluationStageMetrics> Run(int* argc, char* argv[]);
+
+ protected:
+  // Returns a list of commandline flags that this task defines.
+  virtual std::vector<Flag> GetFlags() = 0;
+
+  virtual std::optional<EvaluationStageMetrics> RunImpl() = 0;
+
+  DelegateProviders delegate_providers_;
+};
+
+// Just a declaration. In order to avoid the boilerpolate main-function code,
+// every evaluation task should define this function.
+std::unique_ptr<TaskExecutor> CreateTaskExecutor();
+}  // namespace evaluation
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_EVALUATION_TASKS_TASK_EXECUTOR_H_

tensorflow/lite/tools/evaluation/tasks/task_executor_c_api.cc

@@ -0,0 +1,100 @@
+/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/evaluation/tasks/task_executor_c_api.h"
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <utility>
+
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+#include "tensorflow/lite/tools/evaluation/tasks/task_executor.h"
+#include "tensorflow/lite/tools/logging.h"
+
+extern "C" {
+
+struct TfLiteEvaluationMetrics {
+  tflite::evaluation::EvaluationStageMetrics metrics;
+};
+
+struct TfLiteEvaluationTask {
+  std::unique_ptr<tflite::evaluation::TaskExecutor> task_executor;
+};
+
+int32_t TfLiteEvaluationMetricsGetNumRuns(
+    const TfLiteEvaluationMetrics* metrics) {
+  return metrics->metrics.num_runs();
+}
+
+extern TfLiteEvaluationMetricsLatency TfLiteEvaluationMetricsGetTestLatency(
+    const TfLiteEvaluationMetrics* metrics) {
+  const auto& latency = metrics->metrics.process_metrics()
+                            .inference_profiler_metrics()
+                            .test_latency();
+  return {latency.last_us(), latency.max_us(), latency.min_us(),
+          latency.sum_us(),  latency.avg_us(), latency.std_deviation_us()};
+}
+
+extern TfLiteEvaluationMetricsLatency
+TfLiteEvaluationMetricsGetReferenceLatency(
+    const TfLiteEvaluationMetrics* metrics) {
+  const auto& latency = metrics->metrics.process_metrics()
+                            .inference_profiler_metrics()
+                            .reference_latency();
+  return {latency.last_us(), latency.max_us(), latency.min_us(),
+          latency.sum_us(),  latency.avg_us(), latency.std_deviation_us()};
+}
+
+extern size_t TfLiteEvaluationMetricsGetOutputErrorCount(
+    const TfLiteEvaluationMetrics* metrics) {
+  return metrics->metrics.process_metrics()
+      .inference_profiler_metrics()
+      .output_errors()
+      .size();
+}
+
+extern TfLiteEvaluationMetricsAccuracy TfLiteEvaluationMetricsGetOutputError(
+    const TfLiteEvaluationMetrics* metrics, int32_t output_error_index) {
+  int32_t output_count = TfLiteEvaluationMetricsGetOutputErrorCount(metrics);
+  if (output_error_index < 0 || output_error_index >= output_count) {
+    return {};
+  }
+  const auto& accuracy = metrics->metrics.process_metrics()
+                             .inference_profiler_metrics()
+                             .output_errors()
+                             .at(output_error_index);
+  return {accuracy.max_value(), accuracy.min_value(), accuracy.avg_value(),
+          accuracy.std_deviation()};
+}
+
+TfLiteEvaluationTask* TfLiteEvaluationTaskCreate() {
+  auto task_executor = tflite::evaluation::CreateTaskExecutor();
+  return new TfLiteEvaluationTask{std::move(task_executor)};
+}
+
+TfLiteEvaluationMetrics* TfLiteEvaluationTaskRunWithArgs(
+    TfLiteEvaluationTask* evaluation_task, int argc, char** argv) {
+  const auto metrics_optional =
+      evaluation_task->task_executor->Run(&argc, argv);
+  if (!metrics_optional.has_value()) {
+    TFLITE_LOG(ERROR) << "Failed to run the task evaluation!";
+    return new TfLiteEvaluationMetrics{};
+  }
+  return new TfLiteEvaluationMetrics{std::move(metrics_optional.value())};
+}
+
+}  // extern "C"

tensorflow/lite/tools/evaluation/tasks/task_executor_c_api.h

@@ -0,0 +1,93 @@
+/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_EVALUATION_TASKS_TASK_EXECUTOR_C_API_H_
+#define TENSORFLOW_LITE_TOOLS_EVALUATION_TASKS_TASK_EXECUTOR_C_API_H_
+
+#include <cstddef>
+#include <cstdint>
+#ifdef __cplusplus
+extern "C" {
+#endif  // __cplusplus
+
+// -----------------------------------------------------------------------------
+// C APIs corresponding to tflite::evaluation::LatencyMetrics proto.
+// -----------------------------------------------------------------------------
+
+typedef struct TfLiteEvaluationMetricsLatency {
+  // Latency for the last Run.
+  int64_t last_us;
+  // Maximum latency observed for any Run.
+  int64_t max_us;
+  // Minimum latency observed for any Run.
+  int64_t min_us;
+  // Sum of all Run latencies.
+  int64_t sum_us;
+  // Average latency across all Runs.
+  double avg_us;
+  // Standard deviation for latency across all Runs.
+  int64_t std_deviation_us;
+} TfLiteEvaluationMetricsLatency;
+
+// -----------------------------------------------------------------------------
+// C APIs corresponding to tflite::evaluation::AccuracyMetrics proto.
+// -----------------------------------------------------------------------------
+
+typedef struct TfLiteEvaluationMetricsAccuracy {
+  // Maximum value observed for any Run.
+  float max_value;
+  // Minimum value observed for any Run.
+  float min_value;
+  // Average value across all Runs.
+  double avg_value;
+  // Standard deviation across all Runs.
+  float std_deviation;
+} TfLiteEvaluationMetricsAccuracy;
+
+// -----------------------------------------------------------------------------
+// C APIs corresponding to tflite::evaluation::EvaluationStageMetrics type.
+// -----------------------------------------------------------------------------
+typedef struct TfLiteEvaluationMetrics TfLiteEvaluationMetrics;
+
+extern int32_t TfLiteEvaluationMetricsGetNumRuns(
+    const TfLiteEvaluationMetrics* metrics);
+
+extern TfLiteEvaluationMetricsLatency TfLiteEvaluationMetricsGetTestLatency(
+    const TfLiteEvaluationMetrics* metrics);
+
+extern TfLiteEvaluationMetricsLatency
+TfLiteEvaluationMetricsGetReferenceLatency(
+    const TfLiteEvaluationMetrics* metrics);
+
+extern size_t TfLiteEvaluationMetricsGetOutputErrorCount(
+    const TfLiteEvaluationMetrics* metrics);
+
+extern TfLiteEvaluationMetricsAccuracy TfLiteEvaluationMetricsGetOutputError(
+    const TfLiteEvaluationMetrics* metrics, int32_t output_error_index);
+
+// -----------------------------------------------------------------------------
+// C APIs corresponding to tflite::evaluation::TaskExecutor type.
+// -----------------------------------------------------------------------------
+typedef struct TfLiteEvaluationTask TfLiteEvaluationTask;
+
+extern TfLiteEvaluationTask* TfLiteEvaluationTaskCreate();
+
+extern TfLiteEvaluationMetrics* TfLiteEvaluationTaskRunWithArgs(
+    TfLiteEvaluationTask* evaluation_task, int argc, char** argv);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif  // __cplusplus
+
+#endif  // TENSORFLOW_LITE_TOOLS_EVALUATION_TASKS_TASK_EXECUTOR_C_API_H_

tensorflow/lite/tools/evaluation/tasks/task_executor_main.cc

@@ -0,0 +1,33 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <cstdlib>
+
+#include "tensorflow/lite/tools/evaluation/tasks/task_executor.h"
+#include "tensorflow/lite/tools/logging.h"
+
+// This could serve as the main function for all eval tools.
+int main(int argc, char* argv[]) {
+  auto task_executor = tflite::evaluation::CreateTaskExecutor();
+  if (task_executor == nullptr) {
+    TFLITE_LOG(ERROR) << "Could not create the task evaluation!";
+    return EXIT_FAILURE;
+  }
+  const auto metrics = task_executor->Run(&argc, argv);
+  if (!metrics.has_value()) {
+    TFLITE_LOG(ERROR) << "Could not run the task evaluation!";
+    return EXIT_FAILURE;
+  }
+  return EXIT_SUCCESS;
+}

tensorflow/lite/tools/evaluation/testdata/labels.txt

@@ -0,0 +1,2 @@
+label1
+label2

tensorflow/lite/tools/evaluation/utils.cc

@@ -0,0 +1,297 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/tools/evaluation/utils.h"
+
+#include <algorithm>
+#include <cctype>
+#include <cstddef>
+#include <cstdint>
+#include <fstream>
+#include <string>
+#include <unordered_set>
+#include <vector>
+
+#include "flatbuffers/buffer.h"  // from @flatbuffers
+#include "flatbuffers/string.h"  // from @flatbuffers
+#include "tensorflow/lite/tools/delegates/delegate_provider.h"
+#include "tensorflow/lite/tools/logging.h"
+
+#if defined(__APPLE__)
+#include "TargetConditionals.h"
+#if (TARGET_OS_IPHONE && !TARGET_IPHONE_SIMULATOR) || \
+    (TARGET_OS_OSX && TARGET_CPU_ARM64)
+// Only enable coreml delegate when using a real iPhone device or Apple Silicon.
+#define REAL_IPHONE_DEVICE
+#include "tensorflow/lite/delegates/coreml/coreml_delegate.h"
+#endif
+#endif
+
+#ifndef TFLITE_WITHOUT_XNNPACK
+#include "tensorflow/lite/acceleration/configuration/c/delegate_plugin.h"
+#include "tensorflow/lite/acceleration/configuration/c/xnnpack_plugin.h"
+#include "tensorflow/lite/acceleration/configuration/configuration_generated.h"
+#include "tensorflow/lite/c/common.h"
+#include "tensorflow/lite/delegates/xnnpack/xnnpack_delegate.h"
+#endif  // !defined(TFLITE_WITHOUT_XNNPACK)
+
+#if !defined(_WIN32)
+#include <dirent.h>
+#endif
+#include <sys/stat.h>
+
+namespace tflite {
+namespace evaluation {
+
+std::string StripTrailingSlashes(const std::string& path) {
+  int end = path.size();
+  while (end > 0 && path[end - 1] == '/') {
+    end--;
+  }
+  return path.substr(0, end);
+}
+
+bool ReadFileLines(const std::string& file_path,
+                   std::vector<std::string>* lines_output) {
+  if (!lines_output) {
+    return false;
+  }
+  std::ifstream stream(file_path.c_str());
+  if (!stream) {
+    return false;
+  }
+  std::string line;
+  while (std::getline(stream, line)) {
+    lines_output->push_back(line);
+  }
+  return true;
+}
+
+#if !defined(_WIN32)
+TfLiteStatus GetSortedFileNames(
+    const std::string& directory, std::vector<std::string>* result,
+    const std::unordered_set<std::string>& extensions) {
+  DIR* dir;
+  struct dirent* ent;
+  if (result == nullptr) {
+    return kTfLiteError;
+  }
+  result->clear();
+  std::string dir_path = StripTrailingSlashes(directory);
+  if ((dir = opendir(dir_path.c_str())) != nullptr) {
+    while ((ent = readdir(dir)) != nullptr) {
+      if (ent->d_type == DT_DIR) continue;
+      std::string filename(std::string(ent->d_name));
+      size_t lastdot = filename.find_last_of('.');
+      std::string ext = lastdot != std::string::npos ? filename.substr(lastdot)
+                                                     : std::string();
+      std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);
+      if (!extensions.empty() && extensions.find(ext) == extensions.end()) {
+        continue;
+      }
+      result->emplace_back(dir_path + "/" + filename);
+    }
+    closedir(dir);
+  } else {
+    return kTfLiteError;
+  }
+  std::sort(result->begin(), result->end());
+  return kTfLiteOk;
+}
+#endif
+
+TfLiteDelegatePtr CreateNNAPIDelegate() {
+#if TFLITE_SUPPORTS_NNAPI_DELEGATE
+  return TfLiteDelegatePtr(
+      NnApiDelegate(),
+      // NnApiDelegate() returns a singleton, so provide a no-op deleter.
+      [](TfLiteDelegate*) {});
+#else   // TFLITE_SUPPORTS_NNAPI_DELEGATE
+  return tools::CreateNullDelegate();
+#endif  // TFLITE_SUPPORTS_NNAPI_DELEGATE
+}
+
+#if TFLITE_SUPPORTS_NNAPI_DELEGATE
+TfLiteDelegatePtr CreateNNAPIDelegate(StatefulNnApiDelegate::Options options) {
+  return TfLiteDelegatePtr(
+      new StatefulNnApiDelegate(options), [](TfLiteDelegate* delegate) {
+        delete reinterpret_cast<StatefulNnApiDelegate*>(delegate);
+      });
+}
+#endif  // TFLITE_SUPPORTS_NNAPI_DELEGATE
+
+#if TFLITE_SUPPORTS_GPU_DELEGATE
+TfLiteDelegatePtr CreateGPUDelegate(TfLiteGpuDelegateOptionsV2* options) {
+  return TfLiteDelegatePtr(TfLiteGpuDelegateV2Create(options),
+                           &TfLiteGpuDelegateV2Delete);
+}
+#endif  // TFLITE_SUPPORTS_GPU_DELEGATE
+
+TfLiteDelegatePtr CreateGPUDelegate() {
+#if TFLITE_SUPPORTS_GPU_DELEGATE
+  TfLiteGpuDelegateOptionsV2 options = TfLiteGpuDelegateOptionsV2Default();
+  options.inference_priority1 = TFLITE_GPU_INFERENCE_PRIORITY_MIN_LATENCY;
+  options.inference_preference =
+      TFLITE_GPU_INFERENCE_PREFERENCE_SUSTAINED_SPEED;
+
+  return CreateGPUDelegate(&options);
+#else
+  return tools::CreateNullDelegate();
+#endif  // TFLITE_SUPPORTS_GPU_DELEGATE
+}
+
+TfLiteDelegatePtr CreateHexagonDelegate(
+    const std::string& library_directory_path, bool profiling) {
+#if TFLITE_ENABLE_HEXAGON
+  TfLiteHexagonDelegateOptions options = {0};
+  options.print_graph_profile = profiling;
+  return CreateHexagonDelegate(&options, library_directory_path);
+#else
+  return tools::CreateNullDelegate();
+#endif  // TFLITE_ENABLE_HEXAGON
+}
+
+#if TFLITE_ENABLE_HEXAGON
+TfLiteDelegatePtr CreateHexagonDelegate(
+    const TfLiteHexagonDelegateOptions* options,
+    const std::string& library_directory_path) {
+  if (library_directory_path.empty()) {
+    TfLiteHexagonInit();
+  } else {
+    TfLiteHexagonInitWithPath(library_directory_path.c_str());
+  }
+
+  TfLiteDelegate* delegate = TfLiteHexagonDelegateCreate(options);
+  if (!delegate) {
+    TfLiteHexagonTearDown();
+    return tools::CreateNullDelegate();
+  }
+  return TfLiteDelegatePtr(delegate, [](TfLiteDelegate* delegate) {
+    TfLiteHexagonDelegateDelete(delegate);
+    TfLiteHexagonTearDown();
+  });
+}
+#endif  // TFLITE_ENABLE_HEXAGON
+
+#ifdef TFLITE_WITHOUT_XNNPACK
+TfLiteDelegatePtr CreateXNNPACKDelegate(int num_threads, bool force_fp16,
+                                        const char* weight_cache_file_path) {
+  return tools::CreateNullDelegate();
+}
+#else  // !defined(TFLITE_WITHOUT_XNNPACK)
+// This method replicates the implementation from
+// https://github.com/tensorflow/tensorflow/blob/55e3b5643a791c4cc320746649d455cacfadf6ed/tensorflow/lite/delegates/xnnpack/xnnpack_delegate.cc#L5235
+// to avoid having an entire copy of XNNPack.
+TfLiteXNNPackDelegateOptions XNNPackDelegateOptionsDefault() {
+  TfLiteXNNPackDelegateOptions options = {0};
+
+  // Quantized inference is enabled by default on Web platform
+#ifdef XNNPACK_DELEGATE_ENABLE_QS8
+  options.flags |= TFLITE_XNNPACK_DELEGATE_FLAG_QS8;
+#endif  // XNNPACK_DELEGATE_ENABLE_QS8
+#ifdef XNNPACK_DELEGATE_ENABLE_QU8
+  options.flags |= TFLITE_XNNPACK_DELEGATE_FLAG_QU8;
+#endif  // XNNPACK_DELEGATE_ENABLE_QU8
+
+  // Enable quantized inference for the delegate build used in unit tests.
+#ifdef XNNPACK_DELEGATE_TEST_MODE
+  options.flags |= TFLITE_XNNPACK_DELEGATE_FLAG_QS8;
+  options.flags |= TFLITE_XNNPACK_DELEGATE_FLAG_QU8;
+#endif  // XNNPACK_DELEGATE_TEST_MODE
+  return options;
+}
+
+TfLiteDelegatePtr CreateXNNPACKDelegate() {
+  TfLiteXNNPackDelegateOptions xnnpack_options =
+      XNNPackDelegateOptionsDefault();
+  return CreateXNNPACKDelegate(&xnnpack_options);
+}
+
+TfLiteDelegatePtr CreateXNNPACKDelegate(
+    const TfLiteXNNPackDelegateOptions* xnnpack_options) {
+  flatbuffers::FlatBufferBuilder flatbuffer_builder;
+  flatbuffers::Offset<flatbuffers::String> weight_cache_file_path;
+  if (xnnpack_options->weight_cache_file_path) {
+    TFLITE_LOG(INFO) << "XNNPack file-backed weight cache enabled.";
+    weight_cache_file_path = flatbuffer_builder.CreateString(
+        xnnpack_options->weight_cache_file_path);
+  }
+
+  tflite::XNNPackSettingsBuilder xnnpack_settings_builder(flatbuffer_builder);
+  int num_threads = xnnpack_options->num_threads;
+  if (num_threads >= 0) {
+    xnnpack_settings_builder.add_num_threads(num_threads);
+  }
+  if (xnnpack_options->flags & TFLITE_XNNPACK_DELEGATE_FLAG_FORCE_FP16) {
+    TFLITE_LOG(INFO) << "XNNPack FP16 inference enabled.";
+  }
+  xnnpack_settings_builder.fbb_.AddElement<int32_t>(
+      XNNPackSettings::VT_FLAGS, static_cast<int32_t>(xnnpack_options->flags),
+      0);
+  xnnpack_settings_builder.fbb_.AddElement<int32_t>(
+      XNNPackSettings::VT_RUNTIME_FLAGS,
+      static_cast<int32_t>(xnnpack_options->runtime_flags), 0);
+  xnnpack_settings_builder.add_weight_cache_file_path(weight_cache_file_path);
+  flatbuffers::Offset<tflite::XNNPackSettings> xnnpack_settings =
+      xnnpack_settings_builder.Finish();
+  tflite::TFLiteSettingsBuilder tflite_settings_builder(flatbuffer_builder);
+  tflite_settings_builder.add_xnnpack_settings(xnnpack_settings);
+  tflite_settings_builder.add_delegate(tflite::Delegate_XNNPACK);
+  flatbuffers::Offset<tflite::TFLiteSettings> tflite_settings =
+      tflite_settings_builder.Finish();
+  flatbuffer_builder.Finish(tflite_settings);
+  const tflite::TFLiteSettings* tflite_settings_flatbuffer =
+      flatbuffers::GetRoot<tflite::TFLiteSettings>(
+          flatbuffer_builder.GetBufferPointer());
+  // Create an XNNPack delegate plugin using the settings from the flatbuffer.
+  const TfLiteOpaqueDelegatePlugin* delegate_plugin =
+      TfLiteXnnpackDelegatePluginCApi();
+  TfLiteOpaqueDelegate* delegate =
+      delegate_plugin->create(tflite_settings_flatbuffer);
+  void (*delegate_deleter)(TfLiteOpaqueDelegate*) = delegate_plugin->destroy;
+  return TfLiteDelegatePtr(delegate, delegate_deleter);
+}
+
+TfLiteDelegatePtr CreateXNNPACKDelegate(int num_threads, bool force_fp16,
+                                        const char* weight_cache_file_path) {
+  auto opts = XNNPackDelegateOptionsDefault();
+  // Note that we don't want to use the thread pool for num_threads == 1.
+  opts.num_threads = num_threads > 1 ? num_threads : 0;
+  if (force_fp16) {
+    opts.flags |= TFLITE_XNNPACK_DELEGATE_FLAG_FORCE_FP16;
+  }
+  if (weight_cache_file_path && weight_cache_file_path[0] != '\0') {
+    opts.weight_cache_file_path = weight_cache_file_path;
+  }
+  return CreateXNNPACKDelegate(&opts);
+}
+#endif
+
+TfLiteDelegatePtr CreateCoreMlDelegate() {
+#ifdef REAL_IPHONE_DEVICE
+  TfLiteCoreMlDelegateOptions coreml_options = {
+      .enabled_devices = TfLiteCoreMlDelegateAllDevices};
+  TfLiteDelegate* delegate = TfLiteCoreMlDelegateCreate(&coreml_options);
+  if (!delegate) {
+    return tools::CreateNullDelegate();
+  }
+  return TfLiteDelegatePtr(delegate, &TfLiteCoreMlDelegateDelete);
+#else
+  return tools::CreateNullDelegate();
+#endif  // REAL_IPHONE_DEVICE
+}
+
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/utils.h

@@ -0,0 +1,116 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_TOOLS_EVALUATION_UTILS_H_
+#define TENSORFLOW_LITE_TOOLS_EVALUATION_UTILS_H_
+
+#include <memory>
+#include <string>
+#include <unordered_set>
+#include <vector>
+
+#if !TFLITE_WITH_STABLE_ABI
+// TODO(b/240438534): enable nnapi.
+#if defined(__ANDROID__)
+#define TFLITE_SUPPORTS_NNAPI_DELEGATE 1
+#define TFLITE_SUPPORTS_GPU_DELEGATE 1
+#elif defined(CL_DELEGATE_NO_GL)
+#define TFLITE_SUPPORTS_GPU_DELEGATE 1
+#endif  // defined(__ANDROID__)
+#endif  // TFLITE_WITH_STABLE_ABI
+
+// XNNPACK does not support s390x
+// (see <https://github.com/tensorflow/tensorflow/pull/51655>).
+#ifdef __s390x__
+#define TFLITE_WITHOUT_XNNPACK 1
+#endif
+
+#if TFLITE_SUPPORTS_GPU_DELEGATE
+#include "tensorflow/lite/delegates/gpu/delegate.h"
+#endif
+
+#if TFLITE_ENABLE_HEXAGON
+#include "tensorflow/lite/delegates/hexagon/hexagon_delegate.h"
+#endif
+
+#if TFLITE_SUPPORTS_NNAPI_DELEGATE
+#include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
+#endif  // TFLITE_SUPPORTS_NNAPI_DELEGATE
+
+#ifndef TFLITE_WITHOUT_XNNPACK
+#include "tensorflow/lite/delegates/xnnpack/xnnpack_delegate.h"
+#endif  // !defined(TFLITE_WITHOUT_XNNPACK)
+
+#include "tensorflow/lite/c/common.h"
+
+namespace tflite {
+namespace evaluation {
+
+// Same as Interpreter::TfLiteDelegatePtr, defined here to avoid pulling
+// in tensorflow/lite/interpreter.h dependency.
+using TfLiteDelegatePtr =
+    std::unique_ptr<TfLiteOpaqueDelegate, void (*)(TfLiteOpaqueDelegate*)>;
+
+std::string StripTrailingSlashes(const std::string& path);
+
+bool ReadFileLines(const std::string& file_path,
+                   std::vector<std::string>* lines_output);
+
+// If extension set is empty, all files will be listed. The strings in
+// extension set are expected to be in lowercase and include the dot.
+TfLiteStatus GetSortedFileNames(
+    const std::string& directory, std::vector<std::string>* result,
+    const std::unordered_set<std::string>& extensions);
+
+inline TfLiteStatus GetSortedFileNames(const std::string& directory,
+                                       std::vector<std::string>* result) {
+  return GetSortedFileNames(directory, result,
+                            std::unordered_set<std::string>());
+}
+
+// Returns nullptr on error, e.g. if NNAPI isn't supported on this platform.
+TfLiteDelegatePtr CreateNNAPIDelegate();
+#if TFLITE_SUPPORTS_NNAPI_DELEGATE
+TfLiteDelegatePtr CreateNNAPIDelegate(StatefulNnApiDelegate::Options options);
+#endif  // TFLITE_SUPPORTS_NNAPI_DELEGATE
+
+TfLiteDelegatePtr CreateGPUDelegate();
+#if TFLITE_SUPPORTS_GPU_DELEGATE
+TfLiteDelegatePtr CreateGPUDelegate(TfLiteGpuDelegateOptionsV2* options);
+#endif  // TFLITE_SUPPORTS_GPU_DELEGATE
+
+TfLiteDelegatePtr CreateHexagonDelegate(
+    const std::string& library_directory_path, bool profiling);
+#if TFLITE_ENABLE_HEXAGON
+TfLiteDelegatePtr CreateHexagonDelegate(
+    const TfLiteHexagonDelegateOptions* options,
+    const std::string& library_directory_path);
+#endif
+
+#ifndef TFLITE_WITHOUT_XNNPACK
+TfLiteXNNPackDelegateOptions XNNPackDelegateOptionsDefault();
+TfLiteDelegatePtr CreateXNNPACKDelegate();
+TfLiteDelegatePtr CreateXNNPACKDelegate(
+    const TfLiteXNNPackDelegateOptions* options);
+#endif  // !defined(TFLITE_WITHOUT_XNNPACK)
+TfLiteDelegatePtr CreateXNNPACKDelegate(
+    int num_threads, bool force_fp16,
+    const char* weight_cache_file_path = nullptr);
+
+TfLiteDelegatePtr CreateCoreMlDelegate();
+}  // namespace evaluation
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_EVALUATION_UTILS_H_

tensorflow/lite/tools/evaluation/utils_test.cc

@@ -0,0 +1,83 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/evaluation/utils.h"
+
+#include <string>
+#include <vector>
+
+#include <gtest/gtest.h>
+#include "tensorflow/lite/context.h"
+
+namespace tflite {
+namespace evaluation {
+namespace {
+
+constexpr char kLabelsPath[] =
+    "tensorflow/lite/tools/evaluation/testdata/labels.txt";
+constexpr char kDirPath[] =
+    "tensorflow/lite/tools/evaluation/testdata";
+constexpr char kEmptyFilePath[] =
+    "tensorflow/lite/tools/evaluation/testdata/empty.txt";
+
+TEST(UtilsTest, StripTrailingSlashesTest) {
+  std::string path = "/usr/local/folder/";
+  EXPECT_EQ(StripTrailingSlashes(path), "/usr/local/folder");
+
+  path = "/usr/local/folder";
+  EXPECT_EQ(StripTrailingSlashes(path), path);
+
+  path = "folder";
+  EXPECT_EQ(StripTrailingSlashes(path), path);
+}
+
+TEST(UtilsTest, ReadFileErrors) {
+  std::string correct_path(kLabelsPath);
+  std::string empty_path(kEmptyFilePath);
+  std::string wrong_path("xyz.txt");
+  std::vector<std::string> lines;
+  EXPECT_FALSE(ReadFileLines(correct_path, nullptr));
+  EXPECT_FALSE(ReadFileLines(empty_path, nullptr));
+  EXPECT_FALSE(ReadFileLines(wrong_path, &lines));
+}
+
+TEST(UtilsTest, ReadFileCorrectly) {
+  std::string file_path(kLabelsPath);
+  std::string empty_path(kEmptyFilePath);
+  std::vector<std::string> lines;
+  std::vector<std::string> empty_lines;
+
+  EXPECT_TRUE(ReadFileLines(file_path, &lines));
+  EXPECT_EQ(lines.size(), 2);
+  EXPECT_EQ(lines[0], "label1");
+  EXPECT_EQ(lines[1], "label2");
+
+  EXPECT_TRUE(ReadFileLines(empty_path, &empty_lines));
+  EXPECT_EQ(empty_lines.size(), 0);
+}
+
+TEST(UtilsTest, SortedFilenamesTest) {
+  std::vector<std::string> files;
+  EXPECT_EQ(GetSortedFileNames(kDirPath, &files), kTfLiteOk);
+
+  EXPECT_EQ(files.size(), 2);
+  EXPECT_EQ(files[0], kEmptyFilePath);
+  EXPECT_EQ(files[1], kLabelsPath);
+
+  EXPECT_EQ(GetSortedFileNames("wrong_path", &files), kTfLiteError);
+}
+
+}  // namespace
+}  // namespace evaluation
+}  // namespace tflite