chore: import upstream snapshot with attribution

tensorflow/lite/tools/benchmark/experimental/c/BUILD

@@ -0,0 +1,40 @@
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
+load("//tensorflow/lite:build_def.bzl", "tflite_copts")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_visibility = ["benchmark"],
+    licenses = ["notice"],
+)
+
+package_group(
+    name = "benchmark",
+    packages = [
+        "//tensorflow/lite/tools/benchmark/...",
+    ],
+)
+
+exports_files(
+    ["benchmark_c_api.h"],
+    visibility = ["//tensorflow/lite/tools/benchmark/experimental/c:benchmark"],
+)
+
+cc_library(
+    name = "benchmark_c_api",
+    srcs = ["benchmark_c_api.cc"],
+    hdrs = [
+        "benchmark_c_api.h",
+    ],
+    copts = tflite_copts(),
+    visibility = [
+        "benchmark",
+    ],
+    deps = [
+        "//tensorflow/lite/c:c_api_types",
+        "//tensorflow/lite/core/c:c_api_types",
+        "//tensorflow/lite/tools/benchmark:benchmark_model_lib",
+        "//tensorflow/lite/tools/benchmark:benchmark_params",
+        "//tensorflow/lite/tools/benchmark:benchmark_tflite_model_lib",
+        "@xla//xla/tsl/util:stats_calculator_portable",
+    ],
+)

tensorflow/lite/tools/benchmark/experimental/c/benchmark_c_api.cc

@@ -0,0 +1,187 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/tools/benchmark/experimental/c/benchmark_c_api.h"
+
+#include <cstdint>
+#include <memory>
+#include <utility>
+
+#include "xla/tsl/util/stats_calculator.h"
+#include "tensorflow/lite/c/c_api_types.h"
+#include "tensorflow/lite/tools/benchmark/benchmark_model.h"
+#include "tensorflow/lite/tools/benchmark/benchmark_params.h"
+#include "tensorflow/lite/tools/benchmark/benchmark_tflite_model.h"
+
+extern "C" {
+
+// -----------------------------------------------------------------------------
+// C APIs corresponding to tflite::benchmark::BenchmarkResults type.
+// -----------------------------------------------------------------------------
+struct TfLiteBenchmarkResults {
+  const tflite::benchmark::BenchmarkResults* results;
+};
+
+// Converts the given int64_t stat into a TfLiteBenchmarkInt64Stat struct.
+TfLiteBenchmarkInt64Stat ConvertStat(const tsl::Stat<int64_t>& stat) {
+  return {
+      stat.empty(),    stat.first(), stat.newest(),        stat.max(),
+      stat.min(),      stat.count(), stat.sum(),           stat.squared_sum(),
+      stat.all_same(), stat.avg(),   stat.std_deviation(),
+  };
+}
+
+TfLiteBenchmarkInt64Stat TfLiteBenchmarkResultsGetInferenceTimeMicroseconds(
+    const TfLiteBenchmarkResults* results) {
+  return ConvertStat(results->results->inference_time_us());
+}
+
+TfLiteBenchmarkInt64Stat TfLiteBenchmarkResultsGetWarmupTimeMicroseconds(
+    const TfLiteBenchmarkResults* results) {
+  return ConvertStat(results->results->warmup_time_us());
+}
+
+int64_t TfLiteBenchmarkResultsGetStartupLatencyMicroseconds(
+    const TfLiteBenchmarkResults* results) {
+  return results->results->startup_latency_us();
+}
+
+uint64_t TfLiteBenchmarkResultsGetInputBytes(
+    const TfLiteBenchmarkResults* results) {
+  return results->results->input_bytes();
+}
+
+double TfLiteBenchmarkResultsGetThroughputMbPerSecond(
+    const TfLiteBenchmarkResults* results) {
+  return results->results->throughput_MB_per_second();
+}
+
+// -----------------------------------------------------------------------------
+// C APIs corresponding to tflite::benchmark::BenchmarkListener type.
+// -----------------------------------------------------------------------------
+class BenchmarkListenerAdapter : public tflite::benchmark::BenchmarkListener {
+ public:
+  void OnBenchmarkStart(
+      const tflite::benchmark::BenchmarkParams& params) override {
+    if (on_benchmark_start_fn_ != nullptr) {
+      on_benchmark_start_fn_(user_data_);
+    }
+  }
+
+  void OnSingleRunStart(tflite::benchmark::RunType runType) override {
+    if (on_single_run_start_fn_ != nullptr) {
+      on_single_run_start_fn_(user_data_, runType == tflite::benchmark::WARMUP
+                                              ? TfLiteBenchmarkWarmup
+                                              : TfLiteBenchmarkRegular);
+    }
+  }
+
+  void OnSingleRunEnd() override {
+    if (on_single_run_end_fn_ != nullptr) {
+      on_single_run_end_fn_(user_data_);
+    }
+  }
+
+  void OnBenchmarkEnd(
+      const tflite::benchmark::BenchmarkResults& results) override {
+    if (on_benchmark_end_fn_ != nullptr) {
+      TfLiteBenchmarkResults* wrapper = new TfLiteBenchmarkResults{&results};
+      on_benchmark_end_fn_(user_data_, wrapper);
+      delete wrapper;
+    }
+  }
+
+  // Keep the user_data pointer provided when setting the callbacks.
+  void* user_data_;
+
+  // Function pointers set by the TfLiteBenchmarkListenerSetCallbacks call.
+  // Only non-null callbacks will be actually called.
+  void (*on_benchmark_start_fn_)(void* user_data);
+  void (*on_single_run_start_fn_)(void* user_data,
+                                  TfLiteBenchmarkRunType runType);
+  void (*on_single_run_end_fn_)(void* user_data);
+  void (*on_benchmark_end_fn_)(void* user_data,
+                               TfLiteBenchmarkResults* results);
+};
+
+struct TfLiteBenchmarkListener {
+  std::unique_ptr<BenchmarkListenerAdapter> adapter;
+};
+
+TfLiteBenchmarkListener* TfLiteBenchmarkListenerCreate() {
+  std::unique_ptr<BenchmarkListenerAdapter> adapter =
+      std::make_unique<BenchmarkListenerAdapter>();
+  return new TfLiteBenchmarkListener{std::move(adapter)};
+}
+
+void TfLiteBenchmarkListenerDelete(TfLiteBenchmarkListener* listener) {
+  delete listener;
+}
+
+void TfLiteBenchmarkListenerSetCallbacks(
+    TfLiteBenchmarkListener* listener, void* user_data,
+    void (*on_benchmark_start_fn)(void* user_data),
+    void (*on_single_run_start_fn)(void* user_data,
+                                   TfLiteBenchmarkRunType runType),
+    void (*on_single_run_end_fn)(void* user_data),
+    void (*on_benchmark_end_fn)(void* user_data,
+                                TfLiteBenchmarkResults* results)) {
+  listener->adapter->user_data_ = user_data;
+  listener->adapter->on_benchmark_start_fn_ = on_benchmark_start_fn;
+  listener->adapter->on_single_run_start_fn_ = on_single_run_start_fn;
+  listener->adapter->on_single_run_end_fn_ = on_single_run_end_fn;
+  listener->adapter->on_benchmark_end_fn_ = on_benchmark_end_fn;
+}
+
+// -----------------------------------------------------------------------------
+// C APIs corresponding to tflite::benchmark::BenchmarkTfLiteModel type.
+// -----------------------------------------------------------------------------
+struct TfLiteBenchmarkTfLiteModel {
+  std::unique_ptr<tflite::benchmark::BenchmarkTfLiteModel> benchmark_model;
+};
+
+TfLiteBenchmarkTfLiteModel* TfLiteBenchmarkTfLiteModelCreate() {
+  std::unique_ptr<tflite::benchmark::BenchmarkTfLiteModel> benchmark_model(
+      new tflite::benchmark::BenchmarkTfLiteModel());
+  return new TfLiteBenchmarkTfLiteModel{std::move(benchmark_model)};
+}
+
+void TfLiteBenchmarkTfLiteModelDelete(
+    TfLiteBenchmarkTfLiteModel* benchmark_model) {
+  delete benchmark_model;
+}
+
+TfLiteStatus TfLiteBenchmarkTfLiteModelInit(
+    TfLiteBenchmarkTfLiteModel* benchmark_model) {
+  return benchmark_model->benchmark_model->Init();
+}
+
+TfLiteStatus TfLiteBenchmarkTfLiteModelRun(
+    TfLiteBenchmarkTfLiteModel* benchmark_model) {
+  return benchmark_model->benchmark_model->Run();
+}
+
+TfLiteStatus TfLiteBenchmarkTfLiteModelRunWithArgs(
+    TfLiteBenchmarkTfLiteModel* benchmark_model, int argc, char** argv) {
+  return benchmark_model->benchmark_model->Run(argc, argv);
+}
+
+void TfLiteBenchmarkTfLiteModelAddListener(
+    TfLiteBenchmarkTfLiteModel* benchmark_model,
+    const TfLiteBenchmarkListener* listener) {
+  return benchmark_model->benchmark_model->AddListener(listener->adapter.get());
+}
+
+}  // extern "C"

tensorflow/lite/tools/benchmark/experimental/c/benchmark_c_api.h

@@ -0,0 +1,137 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_C_BENCHMARK_C_API_H_
+#define TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_C_BENCHMARK_C_API_H_
+
+#include <cstdint>
+
+#include "tensorflow/lite/core/c/c_api_types.h"
+
+// -----------------------------------------------------------------------------
+// Experimental C APIs for the benchmark tool, mainly intended to be used for
+// building a standalone TensorFlow Lite benchmark framework for iOS. This
+// header only has a minimal dependency to the C API types, which can be
+// included in the framework itself.
+// -----------------------------------------------------------------------------
+
+#ifdef __cplusplus
+extern "C" {
+#endif  // __cplusplus
+
+typedef enum {
+  TfLiteBenchmarkWarmup,
+  TfLiteBenchmarkRegular,
+} TfLiteBenchmarkRunType;
+
+// -----------------------------------------------------------------------------
+// C APIs corresponding to tsl::Stat<int64_t> type.
+// -----------------------------------------------------------------------------
+typedef struct TfLiteBenchmarkInt64Stat {
+  bool empty;
+  int64_t first;
+  int64_t newest;
+  int64_t max;
+  int64_t min;
+  int64_t count;
+  int64_t sum;
+  double squared_sum;
+  bool all_same;
+  double avg;
+  int64_t std_deviation;
+} TfLiteBenchmarkInt64Stat;
+
+// -----------------------------------------------------------------------------
+// C APIs corresponding to tflite::benchmark::BenchmarkResults type.
+// -----------------------------------------------------------------------------
+typedef struct TfLiteBenchmarkResults TfLiteBenchmarkResults;
+
+extern TfLiteBenchmarkInt64Stat
+TfLiteBenchmarkResultsGetInferenceTimeMicroseconds(
+    const TfLiteBenchmarkResults* results);
+
+extern TfLiteBenchmarkInt64Stat TfLiteBenchmarkResultsGetWarmupTimeMicroseconds(
+    const TfLiteBenchmarkResults* results);
+
+extern int64_t TfLiteBenchmarkResultsGetStartupLatencyMicroseconds(
+    const TfLiteBenchmarkResults* results);
+
+extern uint64_t TfLiteBenchmarkResultsGetInputBytes(
+    const TfLiteBenchmarkResults* results);
+
+extern double TfLiteBenchmarkResultsGetThroughputMbPerSecond(
+    const TfLiteBenchmarkResults* results);
+
+// -----------------------------------------------------------------------------
+// C APIs corresponding to tflite::benchmark::BenchmarkListener type.
+// -----------------------------------------------------------------------------
+typedef struct TfLiteBenchmarkListener TfLiteBenchmarkListener;
+
+extern TfLiteBenchmarkListener* TfLiteBenchmarkListenerCreate();
+
+extern void TfLiteBenchmarkListenerDelete(TfLiteBenchmarkListener* listener);
+
+// Sets the listener callbacks. Only non-null callback functions will be called
+// when the following events occur. The user_data pointer provided by the caller
+// will also be forwarded as a parameter of each callback function.
+//
+// - on_benchmark_start: Called before the (outer) inference loop begins. Note
+//     that this is called *after* the interpreter has been initialized, but
+//     *before* any warmup runs have been executed.
+// - on_single_run_start: Called before a single (inner) inference call starts.
+// - on_single_run_end: Called before a single (inner) inference call ends.
+// - on_benchmark_end: Called after the (outer) inference loop ends.
+//
+// In case of `on_benchmark_end` callback, the passed in `results` pointer is
+// only valid during the callback function execution, and will be destroyed
+// afterwards.
+extern void TfLiteBenchmarkListenerSetCallbacks(
+    TfLiteBenchmarkListener* listener, void* user_data,
+    void (*on_benchmark_start_fn)(void* user_data),
+    void (*on_single_run_start_fn)(void* user_data,
+                                   TfLiteBenchmarkRunType runType),
+    void (*on_single_run_end_fn)(void* user_data),
+    void (*on_benchmark_end_fn)(void* user_data,
+                                TfLiteBenchmarkResults* results));
+
+// -----------------------------------------------------------------------------
+// C APIs corresponding to tflite::benchmark::BenchmarkTfLiteModel type.
+// -----------------------------------------------------------------------------
+typedef struct TfLiteBenchmarkTfLiteModel TfLiteBenchmarkTfLiteModel;
+
+// TODO(b/144321502): Support BenchmarkParams.
+extern TfLiteBenchmarkTfLiteModel* TfLiteBenchmarkTfLiteModelCreate();
+
+extern void TfLiteBenchmarkTfLiteModelDelete(
+    TfLiteBenchmarkTfLiteModel* benchmark_model);
+
+extern TfLiteStatus TfLiteBenchmarkTfLiteModelInit(
+    TfLiteBenchmarkTfLiteModel* benchmark_model);
+
+extern TfLiteStatus TfLiteBenchmarkTfLiteModelRun(
+    TfLiteBenchmarkTfLiteModel* benchmark_model);
+
+extern TfLiteStatus TfLiteBenchmarkTfLiteModelRunWithArgs(
+    TfLiteBenchmarkTfLiteModel* benchmark_model, int argc, char** argv);
+
+extern void TfLiteBenchmarkTfLiteModelAddListener(
+    TfLiteBenchmarkTfLiteModel* benchmark_model,
+    const TfLiteBenchmarkListener* listener);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif  // __cplusplus
+
+#endif  // TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_C_BENCHMARK_C_API_H_

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/AndroidManifest.xml

@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools"
+    package="org.tensorflow.lite.benchmark.delegateperformance">
+
+    <!-- Necessary for loading custom models from disk and writing result to disk. -->
+    <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE"/>
+    <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE"/>
+
+    <!--
+        Align the minSdkVersion value with that for the TFLite Model Benchmark Tool Android Apk.
+        -->
+    <uses-sdk
+        android:minSdkVersion="23"
+        android:targetSdkVersion="33" />
+
+    <application android:debuggable="true">
+        <!-- DPB needs to enable debugging to allow developers to copy their stable delegate
+        libraries into the application's data directory, to enable testing of such stable delegate
+        libraries. -->
+        <!-- This Activity runs Delegate Performance Benchmark for latency. -->
+        <activity
+            android:name=".BenchmarkLatencyActivity"
+            android:screenOrientation="portrait"
+            android:label="TFLite Delegate Performance Benchmark - Latency"
+            android:exported="true"
+            android:noHistory="true">
+        </activity>
+        <!-- This Activity runs Delegate Performance Benchmark for accuracy. -->
+        <activity
+            android:name=".BenchmarkAccuracyActivity"
+            android:screenOrientation="portrait"
+            android:label="TFLite Delegate Performance Benchmark - Accuracy"
+            android:exported="true"
+            android:noHistory="true">
+        </activity>
+    </application>
+
+</manifest>

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/BUILD

@@ -0,0 +1,236 @@
+# Description:
+#   Delegate Performance Benchmark (DPB) Android app.
+#   This provides model-level latency & accuracy testings for delegates, on Android.
+
+load("@build_bazel_rules_android//android:rules.bzl", "android_library")
+load("//tensorflow/lite/core/shims:cc_library_with_tflite.bzl", "android_binary_with_tflite", "android_library_with_tflite")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_visibility = ["//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android:__subpackages__"],
+    licenses = ["notice"],
+)
+
+android_library(
+    name = "benchmark_accuracy_activity",
+    srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkAccuracyActivity.java"],
+    deps = [":benchmark_accuracy_impl"],
+)
+
+android_library(
+    name = "benchmark_accuracy_impl",
+    srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkAccuracyImpl.java"],
+    deps = [
+        ":benchmark_accuracy",
+        ":benchmark_report",
+        ":csv_writer",
+        ":delegate_performance_benchmark_utils",
+        ":html_writer",
+        ":json_writer",
+        ":model_benchmark_report",
+        ":raw_delegate_metrics_entry",
+        ":tflite_settings_list_entry",
+        "//tensorflow/lite/acceleration/configuration:configuration_fbs_android",
+    ],
+)
+
+android_library(
+    name = "benchmark_accuracy",
+    srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkAccuracy.java"],
+)
+
+android_library(
+    name = "benchmark_latency_activity",
+    srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkLatencyActivity.java"],
+    deps = [":benchmark_latency_impl"],
+)
+
+android_library(
+    name = "benchmark_latency_impl",
+    srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkLatencyImpl.java"],
+    deps = [
+        ":benchmark_report",
+        ":csv_writer",
+        ":delegate_performance_benchmark_utils",
+        ":html_writer",
+        ":json_writer",
+        ":model_benchmark_report",
+        ":model_benchmark_report_interface",
+        ":preconditions",
+        ":raw_delegate_metrics_entry",
+        ":tflite_settings_list_entry",
+        "//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto:delegate_performance_java_proto_lite",
+    ],
+)
+
+android_library_with_tflite(
+    name = "benchmark_report",
+    srcs = [
+        "src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkReport.java",
+        "src/main/java/org/tensorflow/lite/benchmark/delegateperformance/ReportWriter.java",
+    ],
+    deps = [
+        ":benchmark_result_type",
+        ":delegate_performance_benchmark_utils",
+        ":model_benchmark_report_interface",
+    ],
+)
+
+android_library(
+    name = "benchmark_result_type",
+    srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkResultType.java"],
+)
+
+android_library_with_tflite(
+    name = "csv_writer",
+    srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/CsvWriter.java"],
+    tflite_deps = [":benchmark_report"],
+    deps = [
+        ":delegate_metrics_entry",
+        ":metrics_entry",
+        ":model_benchmark_report_interface",
+        ":preconditions",
+    ],
+)
+
+android_library(
+    name = "delegate_metrics_entry",
+    srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/DelegateMetricsEntry.java"],
+    deps = [
+        ":benchmark_result_type",
+        ":metrics_entry",
+    ],
+)
+
+android_library_with_tflite(
+    name = "delegate_performance_benchmark_lib",
+    srcs = [
+        "src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkAccuracyActivity.java",
+        "src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkLatencyActivity.java",
+    ],
+    tflite_deps = [
+        "//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native:benchmark_native",
+    ],
+    deps = [
+        ":benchmark_accuracy_impl",
+        ":benchmark_latency_impl",
+    ],
+)
+
+android_library(
+    name = "delegate_performance_benchmark_utils",
+    srcs = [
+        "src/main/java/org/tensorflow/lite/benchmark/delegateperformance/DelegatePerformanceBenchmark.java",
+    ],
+    deps = [
+        ":benchmark_result_type",
+        ":preconditions",
+        ":tflite_settings_list_entry",
+        "//tensorflow/lite/acceleration/configuration:configuration_fbs_android",
+        "//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto:delegate_performance_java_proto_lite",
+        "@flatbuffers//:runtime_android",
+    ],
+)
+
+android_library_with_tflite(
+    name = "html_writer",
+    srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/HtmlWriter.java"],
+    tflite_deps = [":benchmark_report"],
+    deps = [
+        ":benchmark_result_type",
+        ":delegate_metrics_entry",
+        ":metrics_entry",
+        ":model_benchmark_report_interface",
+        ":preconditions",
+    ],
+)
+
+android_library_with_tflite(
+    name = "json_writer",
+    srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/JsonWriter.java"],
+    tflite_deps = [":benchmark_report"],
+)
+
+android_library(
+    name = "metrics_entry",
+    srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/MetricsEntry.java"],
+    deps = [":benchmark_result_type"],
+)
+
+android_library_with_tflite(
+    name = "model_benchmark_report",
+    srcs = [
+        "src/main/java/org/tensorflow/lite/benchmark/delegateperformance/AccuracyBenchmarkReport.java",
+        "src/main/java/org/tensorflow/lite/benchmark/delegateperformance/LatencyBenchmarkReport.java",
+        "src/main/java/org/tensorflow/lite/benchmark/delegateperformance/ModelBenchmarkReport.java",
+    ],
+    deps = [
+        ":benchmark_result_type",
+        ":delegate_metrics_entry",
+        ":delegate_performance_benchmark_utils",
+        ":metrics_entry",
+        ":model_benchmark_report_interface",
+        ":preconditions",
+        ":raw_delegate_metrics_entry",
+        ":tflite_settings_list_entry",
+        "//tensorflow/lite/acceleration/configuration:configuration_fbs_android",
+        "//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto:delegate_performance_java_proto_lite",
+    ],
+)
+
+android_library(
+    name = "model_benchmark_report_interface",
+    srcs = [
+        "src/main/java/org/tensorflow/lite/benchmark/delegateperformance/ModelBenchmarkReportInterface.java",
+    ],
+    deps = [
+        ":benchmark_result_type",
+        ":delegate_metrics_entry",
+    ],
+)
+
+android_library(
+    name = "preconditions",
+    srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/Preconditions.java"],
+)
+
+android_library(
+    name = "raw_delegate_metrics_entry",
+    srcs = ["src/main/java/org/tensorflow/lite/benchmark/delegateperformance/RawDelegateMetricsEntry.java"],
+    deps = ["//tensorflow/lite/acceleration/configuration:configuration_fbs_android"],
+)
+
+android_library(
+    name = "tflite_settings_list_entry",
+    srcs = [
+        "src/main/java/org/tensorflow/lite/benchmark/delegateperformance/TfLiteSettingsListEntry.java",
+    ],
+    deps = [
+        ":preconditions",
+        "//tensorflow/lite/acceleration/configuration:configuration_fbs_android",
+        "//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto:delegate_performance_java_proto_lite",
+    ],
+)
+
+# The main test app.
+android_binary_with_tflite(
+    name = "delegate_performance_benchmark",
+    assets = [
+        "//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/models:accuracy_models",
+        "//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/models:latency_criteria_files",
+        "//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/models:latency_models",
+        "//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto:default_latency_criteria",
+    ],
+    assets_dir = "assets",
+    custom_package = "org.tensorflow.lite.benchmark.delegateperformance",
+    manifest = "AndroidManifest.xml",
+    nocompress_extensions = [".tflite"],
+    # In some platforms we don't have an Android SDK/NDK and this target
+    # can't be built. We need to prevent the build system from trying to
+    # use the target in that case.
+    tags = ["manual"],
+    tflite_deps = [
+        ":delegate_performance_benchmark_lib",
+    ],
+    visibility = ["//visibility:public"],
+)

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/README.md

@@ -0,0 +1,352 @@
+# TensorFlow Lite Delegate Performance Benchmark (Android APK)
+
+## Description
+
+This Android Delegate Performance Benchmark (DPB) app is a simple wrapper around
+the TensorFlow Lite
+[benchmark tool](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark)
+and
+[MiniBenchmark](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/acceleration/mini_benchmark)
+with the focus on testing Tensorflow Lite delegates that implement stable
+delegate ABI.
+
+Development of TensorFlow Lite delegates needs both accuracy and latency
+evaluations for catching potential performance regressions. Pushing and
+executing both latency and accuracy testing binaries directly on an Android
+device is a valid approach to benchmarking, but it can result in subtle (but
+observable) differences in performance relative to execution within an actual
+Android app. In particular, Android's scheduler tailors behavior based on thread
+and process priorities, which differ between a foreground Activity/Application
+and a regular background binary executed via `adb shell ...`.
+
+In addition to that, having multiple benchmarking apps for different performance
+metric evaluations could potentially cost development effort unnecessarily.
+
+To those ends, this app offers a more faithful view of runtime performance
+(accuracy and latency) that developers can expect when using TensorFlow Lite
+delegates with Android apps, and the app provides a single entrypoint to various
+performance metrics to avoid the need to switch between different benchmarking
+apps.
+
+## To build/install/run
+
+### Build
+
+1.  Clone the TensorFlow repo with
+
+    ```
+    git clone --recurse-submodules https://github.com/tensorflow/tensorflow.git
+    ```
+
+    Note: --recurse-submodules is necessary to prevent some issues with protobuf
+    compilation.
+
+1.  Refer to
+    [this page](https://www.tensorflow.org/lite/android/lite_build#set_up_build_environment_without_docker)
+    for setting up a development environment. Although there are several
+    practical tips:
+
+    -   When installing Bazel, for Ubuntu Linux, `sudo apt update && sudo apt
+        install bazel` may be the easiest way. However sometimes you may need
+        `sudo apt update && sudo apt install bazel-5.3.0` if prompted.
+    -   When installing Android NDK and SDK, using Android Studio's SDK Manager
+        may be the easiest way.
+    -   Run the `./configure` script in the root TensorFlow checkout directory,
+        and answer "Yes" when the script asks to interactively configure the
+        `./WORKSPACE` for Android builds.
+    -   The versions which we have verified are working:
+        -   Android NDK version: 21.4.7075529
+            -   Provide the value as part of a path when the `./configure`
+                script asks to specify the home path of the Android NDK.
+        -   Android NDK API level: 26
+            -   Provide the value when the `./configure` script asks to specify
+                the Android NDK API level.
+        -   Android SDK API level: 33
+            -   Provide the value when the `./configure` script asks to specify
+                the Android SDK API level.
+        -   Android build tools version: 30.0.0
+            -   Provide the value when the `./configure` script asks to specify
+                the Android build tools version.
+
+1.  Build for your specific platform, e.g.:
+
+    ```
+    bazel build -c opt \
+      --config=android_arm64 \
+      tensorflow/lite/tools/benchmark/experimental/delegate_performance/android:delegate_performance_benchmark
+    ```
+
+### Install
+
+1.  Connect to a physical device. Install the benchmark APK with adb:
+
+    ```
+    adb install -r -d -g bazel-bin/tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/delegate_performance_benchmark.apk
+    ```
+
+    Note: Make sure to install with "-g" option to grant the permission for
+    reading external storage.
+
+### Run
+
+#### Benchmarking with stable delegates
+
+The delegate-under-test must implement the
+[stable_delegate_interface](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/delegates/utils/experimental/stable_delegate/stable_delegate_interface.h)
+API. The stable delegate provider dynamically loads stable delegate symbols from
+the provided binary (shared object) file. In order to use Delegate Performance
+Benchmark with a stable delegate, you would need to push the shared object file
+to the file directory of Delegate Performance Benchmark:
+`/data/data/org.tensorflow.lite.benchmark.delegateperformance/files/`.
+
+1.  Build and push the stable delegate binary that you want to test. Here we use
+    the
+    [sample stable delegate](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/delegates/utils/experimental/sample_stable_delegate)
+    as an example.
+
+    ```
+    bazel build -c opt \
+      --config=android_arm64 \
+      tensorflow/lite/delegates/utils/experimental/sample_stable_delegate:tensorflowlite_sample_stable_delegate
+
+    # Set the permissions so that we can overwrite a previously installed delegate.
+    chmod 755 bazel-bin/tensorflow/lite/delegates/utils/experimental/sample_stable_delegate/libtensorflowlite_sample_stable_delegate.so
+
+    # Ensure the delegateperformance files path exists.
+    adb shell run-as org.tensorflow.lite.benchmark.delegateperformance mkdir -p /data/data/org.tensorflow.lite.benchmark.delegateperformance/files
+
+    # Install the sample delegate.
+    adb push \
+      bazel-bin/tensorflow/lite/delegates/utils/experimental/sample_stable_delegate/libtensorflowlite_sample_stable_delegate.so \
+      /data/local/tmp/
+    adb shell run-as org.tensorflow.lite.benchmark.delegateperformance \
+      cp /data/local/tmp/libtensorflowlite_sample_stable_delegate.so \
+        /data/data/org.tensorflow.lite.benchmark.delegateperformance/files/
+    ```
+
+1.  Dump the test sample delegate settings file on device. Example command:
+
+    ```
+    adb shell 'echo "{
+      \"delegate\": \"NONE\",  // Replace NONE with the test target delegate type.
+      \"stable_delegate_loader_settings\": {
+        \"delegate_path\": \"/data/data/org.tensorflow.lite.benchmark.delegateperformance/files/libtensorflowlite_sample_stable_delegate.so\"
+      }
+      // Add concrete delegate settings for the test target delegate.
+    }
+    "> /data/local/tmp/stable_delegate_settings.json'
+    ```
+
+#### Supported models
+
+Currently DPB uses a `mobilenet_v1_1.0_224.tflite` and
+`mobilenet_quant_v1_224.tflite` model for latency and accuracy benchmarking. The
+TF Lite model files are bundled into the app during the build process. We plan
+to expand the supported models based on future use cases.
+
+Note: The sample stable delegate provided here only supports ADD and SUB
+operations thus aforementioned mobilenet models would not actually be delegated.
+To test your own delegate against the models, please update
+`stable_delegate_loader_settings` with your delegate path. To get feedback early
+in the development process, e.g. while working towards supporting more OPs, you
+can run the `benchmark_model` tool, which supports stable delegates and can be
+supplied with arbitrary models via the `--graph` CLI parameter. See
+[this document](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/delegates/utils/experimental/sample_stable_delegate/README.md#tf-lite-benchmark-tool)
+which shows how to run a model with ADD operations through the sample stable
+delegate.
+
+#### Latency benchmarking
+
+##### Options
+
+-   `tflite_settings_files`: `str` (required) the comma-delimited paths to the
+    JSON-encoded delegate `TFLiteSettings` file(s), which is defined in
+    [configuration.proto](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/acceleration/configuration/configuration.proto).
+-   Additional optional command-line flags are documented
+    [here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark/README.md)
+    and can be appended to the `args` string (note that all args must be nested
+    in the single quoted string that follows the args key).
+
+##### Recommendation Criteria
+
+The latency benchmark generates a `PASS`, `PASS_WITH_WARNING`, or `FAIL`
+recommendation by checking if the regressions of the below metrics for each pair
+of the test target delegate and a reference delegate breach the thresholds:
+
+1.  Startup overhead latency: the combined overhead start from initialization to
+    inferences with stable latency. It is calculated as `initialization time +
+    average warmup time - average inference time`.
+1.  Average inference latency: average time for the inferences after warmup in
+    the benchmark run.
+
+When the test target delegate type is the same as the reference delegate, the
+checks are more strict. Otherwise, the checks are relaxed. Please see
+[BenchmarkResultType.java](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkResultType.java)
+for the meanings of `PASS`, `PASS_WITH_WARNING` and `FAIL`.
+
+##### Steps
+
+1.  Run the latency benchmark by supplying the settings file via the required
+    `--tflite_settings_files` flag.
+
+    ```
+    adb shell "am start -S \
+    -n org.tensorflow.lite.benchmark.delegateperformance/org.tensorflow.lite.benchmark.delegateperformance.BenchmarkLatencyActivity \
+    --esa --tflite_settings_files '/data/local/tmp/stable_delegate_settings.json'"
+    ```
+
+1.  The results will be available in Android logcat as well as the app's file
+    directory, e.g.:
+
+    ```
+    adb logcat -c && adb logcat -v color  | grep "Inference timings in us"
+
+    ... tflite  : Inference timings in us: Init: 5811, First inference: 67743, Warmup (avg): 65539, Inference (avg): 65155.5
+    ```
+
+    The tool also shows overall results.
+
+    ```
+    adb logcat -c && adb logcat -v color  | grep 'Latency benchmark result'
+    ```
+
+    which might show output like the following.
+
+    ```
+    ... TfLiteLatencyImpl: Latency benchmark result for /data/local/tmp/stable_delegate_settings.json: PASS
+    ```
+
+    For a summarized view, run
+
+    ```
+    adb shell run-as org.tensorflow.lite.benchmark.delegateperformance "cat /data/user/0/org.tensorflow.lite.benchmark.delegateperformance/files/delegate_performance_result/latency/report.html" > /tmp/dpb-latency.html && xdg-open /tmp/dpb-latency.html
+    ```
+
+    It would open a page in the browser like the following:
+
+    Summary | FAIL
+    ------- | ----
+
+    Model                      | Metric                                                | Delegate: NONE (/data/local/tmp/stable_delegate_settings.json) | Delegate: NONE (default_delegate) | Change    | Status
+    -------------------------- | ----------------------------------------------------- | -------------------------------------------------------------- | --------------------------------- | --------- | ------
+    mobilenet_v1_1.0_224       | model_size_megabyte                                   | -1.0E-6                                                        | -1.0E-6                           | 0.0%      | N/A
+    mobilenet_v1_1.0_224       | initialization_latency_us                             | 515667.0                                                       | 844938.0                          | -39.0%    | N/A
+    mobilenet_v1_1.0_224       | warmup_latency_average_us                             | 334263.5                                                       | 1666704.0                         | -79.9%    | N/A
+    mobilenet_v1_1.0_224       | warmup_latency_min_us                                 | 318494.0                                                       | 1666704.0                         | -80.9%    | N/A
+    mobilenet_v1_1.0_224       | warmup_latency_max_us                                 | 350033.0                                                       | 1666704.0                         | -79.0%    | N/A
+    mobilenet_v1_1.0_224       | warmup_latency_standard_deviation_us                  | 15769.0                                                        | 0.0                               | Infinity% | N/A
+    mobilenet_v1_1.0_224       | inference_latency_average_us                          | 316702.6                                                       | 630715.5                          | -49.8%    | PASS
+    mobilenet_v1_1.0_224       | inference_latency_min_us                              | 308218.0                                                       | 314117.0                          | -1.9%     | N/A
+    mobilenet_v1_1.0_224       | inference_latency_max_us                              | 338494.0                                                       | 1601144.0                         | -78.9%    | N/A
+    mobilenet_v1_1.0_224       | inference_latency_standard_deviation_us               | 4896.0                                                         | 347805.0                          | -98.6%    | N/A
+    mobilenet_v1_1.0_224       | initialization_memory_max_rss_mebibyte                | 0.0                                                            | 34.48828                          | -100.0%   | N/A
+    mobilenet_v1_1.0_224       | initialization_memory_total_non_mmapped_heap_mebibyte | 0.0                                                            | 0.0                               | 0.0%      | N/A
+    mobilenet_v1_1.0_224       | initialization_memory_in_use_heap_mebibyte            | 26.140594                                                      | 21.560455                         | 21.2%     | N/A
+    mobilenet_v1_1.0_224       | overall_memory_max_rss_mebibyte                       | 0.0                                                            | 50.371094                         | -100.0%   | N/A
+    mobilenet_v1_1.0_224       | overall_memory_total_non_mmapped_heap_mebibyte        | 0.0                                                            | 0.0                               | 0.0%      | N/A
+    mobilenet_v1_1.0_224       | overall_memory_in_use_heap_mebibyte                   | 28.22168                                                       | 23.295578                         | 21.1%     | N/A
+    mobilenet_v1_1.0_224       | startup_overhead_latency_us                           | 533227.9                                                       | 1880926.5                         | -71.7%    | PASS
+    mobilenet_v1_1.0_224       | delegate_summary                                      |                                                                |                                   |           | PASS (strict)
+    mobilenet_v1_1.0_224       | model_summary                                         | PASS                                                           |                                   |           |
+    mobilenet_v1_1.0_224_quant | model_size_megabyte                                   | -1.0E-6                                                        | -1.0E-6                           | 0.0%      | N/A
+    mobilenet_v1_1.0_224_quant | initialization_latency_us                             | 25318.0                                                        | 8271.0                            | 206.1%    | N/A
+    mobilenet_v1_1.0_224_quant | warmup_latency_average_us                             | 189565.0                                                       | 188034.0                          | 0.8%      | N/A
+    mobilenet_v1_1.0_224_quant | warmup_latency_min_us                                 | 181333.0                                                       | 175592.0                          | 3.3%      | N/A
+    mobilenet_v1_1.0_224_quant | warmup_latency_max_us                                 | 199285.0                                                       | 199388.0                          | -0.1%     | N/A
+    mobilenet_v1_1.0_224_quant | warmup_latency_standard_deviation_us                  | 7404.0                                                         | 9745.0                            | -24.0%    | N/A
+    mobilenet_v1_1.0_224_quant | inference_latency_average_us                          | 178905.2                                                       | 178897.69                         | 0.0%      | PASS_WITH_WARNING
+    mobilenet_v1_1.0_224_quant | inference_latency_min_us                              | 170126.0                                                       | 170102.0                          | 0.0%      | N/A
+    mobilenet_v1_1.0_224_quant | inference_latency_max_us                              | 200089.0                                                       | 193949.0                          | 3.2%      | N/A
+    mobilenet_v1_1.0_224_quant | inference_latency_standard_deviation_us               | 6355.0                                                         | 6387.0                            | -0.5%     | N/A
+    mobilenet_v1_1.0_224_quant | initialization_memory_max_rss_mebibyte                | 0.0                                                            | 0.0                               | 0.0%      | N/A
+    mobilenet_v1_1.0_224_quant | initialization_memory_total_non_mmapped_heap_mebibyte | 0.0                                                            | 0.0                               | 0.0%      | N/A
+    mobilenet_v1_1.0_224_quant | initialization_memory_in_use_heap_mebibyte            | 1.4762268                                                      | 1.4715118                         | 0.3%      | N/A
+    mobilenet_v1_1.0_224_quant | overall_memory_max_rss_mebibyte                       | 0.0                                                            | 0.0                               | 0.0%      | N/A
+    mobilenet_v1_1.0_224_quant | overall_memory_total_non_mmapped_heap_mebibyte        | 0.0                                                            | 0.0                               | 0.0%      | N/A
+    mobilenet_v1_1.0_224_quant | overall_memory_in_use_heap_mebibyte                   | 3.3774261                                                      | 3.38266                           | -0.2%     | N/A
+    mobilenet_v1_1.0_224_quant | startup_overhead_latency_us                           | 35977.797                                                      | 17407.312                         | 106.7%    | FAIL
+    mobilenet_v1_1.0_224_quant | delegate_summary                                      |                                                                |                                   |           | FAIL (strict)
+    mobilenet_v1_1.0_224_quant | model_summary                                         | FAIL                                                           |                                   |           |
+
+#### Accuracy benchmarking
+
+##### Options
+
+-   `tflite_settings_files`: `str` (required) the comma-delimited paths to the
+    JSON-encoded delegate `TFLiteSettings` file(s), which is defined in
+    [configuration.proto](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/acceleration/configuration/configuration.proto).
+    The first path is the test target delegate and all other paths are treated
+    as reference delegates. The test target delegate will be compared against
+    each reference delegate.
+
+##### Recommendation Criteria
+
+The accuracy benchmark delegates the accuracy metric threshold checks to the
+metric scripts, which are embedded together with the test input inside the
+models. The metric scripts generate an "ok" result by aggregating the outcomes
+for every model and every delegate. The accuracy benchmark generates a `PASS`,
+`PASS_WITH_WARNING`, or `FAIL` recommendation by aggregating the "ok" results.
+
+When the test target delegate type is the same as the reference delegate, the
+checks are more strict. Otherwise, the checks are relaxed. Please see
+[BenchmarkResultType.java](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkResultType.java)
+for the meanings of `PASS`, `PASS_WITH_WARNING` and `FAIL`.
+
+##### Steps
+
+1.  Run the accuracy benchmark by supplying the settings file via the required
+    `--tflite_settings_files` flag.
+
+    ```
+    adb shell "am start -S \
+    -n org.tensorflow.lite.benchmark.delegateperformance/org.tensorflow.lite.benchmark.delegateperformance.BenchmarkAccuracyActivity \
+    --esa --tflite_settings_files '/data/local/tmp/stable_delegate_settings.json'"
+    ```
+
+1.  The results will be available in Android logcat, e.g.:
+
+    ```
+    adb logcat -c && adb logcat -v color | grep "tflite"
+
+    ... tflite  : tflite  :   accuracy: ok
+    ```
+
+    For a summarized view, run
+
+    ```
+    adb shell run-as org.tensorflow.lite.benchmark.delegateperformance "cat /data/user/0/org.tensorflow.lite.benchmark.delegateperformance/files/delegate_performance_result/accuracy/report.html" > /tmp/dpb-accuracy.html && xdg-open /tmp/dpb-accuracy.html
+    ```
+
+    It would open a page in the browser like the following:
+
+    Summary | PASS
+    ------- | ----
+
+    Model                                      | Metric                           | Delegate: NONE (/data/local/tmp/stable_delegate_settings.json) | Delegate: NONE (default_delegate) | Change | Status
+    ------------------------------------------ | -------------------------------- | -------------------------------------------------------------- | --------------------------------- | ------ | ------
+    mobilenet_v1_1.0_224_quant_with_validation | mse(average)                     | 1.917638E-6                                                    | 1.917638E-6                       | 0.0%   | N/A
+    mobilenet_v1_1.0_224_quant_with_validation | symmetric_kl_divergence(average) | 0.049423933                                                    | 0.049423933                       | 0.0%   | N/A
+    mobilenet_v1_1.0_224_quant_with_validation | ok                               | 0.0                                                            | 0.0                               | N/A    | PASS
+    mobilenet_v1_1.0_224_quant_with_validation | max_memory_kb                    | 0.0                                                            | 0.0                               | 0.0%   | N/A
+    mobilenet_v1_1.0_224_quant_with_validation | delegate_summary                 |                                                                |                                   |        | PASS (strict)
+    mobilenet_v1_1.0_224_quant_with_validation | model_summary                    | PASS                                                           |                                   |        |
+    mobilenet_v1_1.0_224_with_validation       | mse(average)                     | 1.0577066E-16                                                  | 1.0577066E-16                     | 0.0%   | N/A
+    mobilenet_v1_1.0_224_with_validation       | symmetric_kl_divergence(average) | 7.2540787E-9                                                   | 7.2540787E-9                      | 0.0%   | N/A
+    mobilenet_v1_1.0_224_with_validation       | ok                               | 0.0                                                            | 0.0                               | N/A    | PASS
+    mobilenet_v1_1.0_224_with_validation       | max_memory_kb                    | 0.0                                                            | 0.0                               | 0.0%   | N/A
+    mobilenet_v1_1.0_224_with_validation       | delegate_summary                 |                                                                |                                   |        | PASS (strict)
+    mobilenet_v1_1.0_224_with_validation       | model_summary                    | PASS                                                           |                                   |        |
+
+## FAQ
+
+### 1. What does a delegate summary result with a `(strict)` suffix mean?
+
+The `(strict)` suffix is added to reference delegates that have the same
+delegate type as the test target delegate. The purpose of the suffix is to let a
+user know that the performance metrics are being checked to a higher standard.
+The expectation is that the test target delegate is better, or at least not
+substantially worse, than the reference delegate in all metrics.
+
+Please see
+[BenchmarkResultType.java](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkResultType.java)
+for more details.

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/build_defs.bzl

@@ -0,0 +1,31 @@
+"""Definitions for targets in DPB (Delegate Performance Benchmark)."""
+
+def latency_benchmark_extra_deps():
+    """Defines extra dependencies for latency benchmark. Currently empty."""
+    return []
+
+def accuracy_benchmark_extra_deps():
+    """Defines extra dependencies for accuracy benchmark. Currently empty."""
+    return []
+
+def latency_benchmark_extra_models():
+    """Defines extra models for latency benchmark. Currently empty.
+
+    Returns a list of tuples where each tuple has two fields: 1) the model name and 2) the model target label. Example:
+    [
+        ("model1.tflite", "@repo//package:model1.tflite"),
+        ("model2.tflite", "@repo//package:model2.tflite"),
+    ]
+    """
+    return []
+
+def accuracy_benchmark_extra_models():
+    """Defines extra models for accuracy benchmark. Currently empty.
+
+    Returns a list of tuples where each tuple has two fields: 1) the model name and 2) the model target label. Example:
+    [
+        ("model1_with_validation.tflite", "@repo//package:model1_with_validation.tflite"),
+        ("model2_with_validation.tflite", "@repo//package:model2_with_validation.tflite"),
+    ]
+    """
+    return []

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/models/BUILD

@@ -0,0 +1,109 @@
+# Description:
+#  Holds model-specific files. The app will bundle the files into assets.
+
+load("//tensorflow/lite/experimental/acceleration/mini_benchmark:build_defs.bzl", "validation_model")
+load("//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android:build_defs.bzl", "accuracy_benchmark_extra_models", "latency_benchmark_extra_models")
+load("//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android:proto.bzl", "proto_data")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_visibility = ["//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android:__subpackages__"],
+    licenses = ["notice"],
+)
+
+# Embedded models for accuracy benchmarking.
+validation_model(
+    name = "mobilenet_v1_1.0_224_with_validation.tflite",
+    jpegs = "//tensorflow/lite/experimental/acceleration/mini_benchmark:odt_classifier_testfiles",
+    main_model = "//tensorflow/lite/experimental/acceleration/mini_benchmark/models:mobilenet_v1_1.0_224.tflite",
+    metrics_model = "//tensorflow/lite/experimental/acceleration/mini_benchmark/metrics:mobilenet_metrics_tflite",
+)
+
+validation_model(
+    name = "mobilenet_v1_1.0_224_quant_with_validation.tflite",
+    jpegs = "//tensorflow/lite/experimental/acceleration/mini_benchmark:odt_classifier_testfiles",
+    main_model = "//tensorflow/lite/experimental/acceleration/mini_benchmark/models:mobilenet_v1_1.0_224_quant.tflite",
+    metrics_model = "//tensorflow/lite/experimental/acceleration/mini_benchmark/metrics:mobilenet_metrics_tflite",
+)
+
+# Migrate the models into assets folder.
+ACCURACY_MODELS = [
+    (
+        "mobilenet_v1_1.0_224_with_validation.tflite",
+        ":mobilenet_v1_1.0_224_with_validation.tflite",
+    ),
+    (
+        "mobilenet_v1_1.0_224_quant_with_validation.tflite",
+        ":mobilenet_v1_1.0_224_quant_with_validation.tflite",
+    ),
+] + accuracy_benchmark_extra_models()
+
+BASIC_LATENCY_MODELS = [
+    (
+        "mobilenet_v1_1.0_224.tflite",
+        "@tflite_mobilenet_float//:mobilenet_v1_1.0_224.tflite",
+    ),
+    (
+        "mobilenet_v1_1.0_224_quant.tflite",
+        "@tflite_mobilenet_quant//:mobilenet_v1_1.0_224_quant.tflite",
+    ),
+]
+
+LATENCY_MODELS = BASIC_LATENCY_MODELS + latency_benchmark_extra_models()
+
+COPY_CMD = """
+  srcs=($(SRCS))
+  outs=($(OUTS))
+  for ((i = 0; i < $${#srcs[@]}; ++i)); do
+    echo $${srcs[$$i]};
+    cp $${srcs[$$i]} $${outs[$$i]};
+  done
+"""
+
+genrule(
+    name = "accuracy_models",
+    srcs = [target for _, target in ACCURACY_MODELS],
+    outs = ["assets/accuracy/%s" % name for name, _ in ACCURACY_MODELS],
+    cmd = COPY_CMD,
+)
+
+genrule(
+    name = "latency_models",
+    srcs = [target for _, target in LATENCY_MODELS],
+    outs = ["assets/latency/%s" % name for name, _ in LATENCY_MODELS],
+    cmd = COPY_CMD,
+)
+
+filegroup(
+    name = "latency_models_test_only",
+    testonly = True,
+    srcs = [
+        "assets/latency/mobilenet_v1_1.0_224.tflite",
+        "assets/latency/mobilenet_v1_1.0_224_quant.tflite",
+    ],
+)
+
+# Latency criteria for latency benchmarking.
+filegroup(
+    name = "latency_criteria_files",
+    srcs = [
+        ":mobilenet_v1_1_0_224_latency_criteria",
+        ":mobilenet_v1_1_0_224_quant_latency_criteria",
+    ],
+)
+
+proto_data(
+    name = "mobilenet_v1_1_0_224_latency_criteria",
+    src = "mobilenet_v1_1.0_224.textproto",
+    out = "assets/proto/mobilenet_v1_1.0_224.binarypb",
+    proto_deps = ["//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto:delegate_performance_proto"],
+    proto_name = "tflite.proto.benchmark.LatencyCriteria",
+)
+
+proto_data(
+    name = "mobilenet_v1_1_0_224_quant_latency_criteria",
+    src = "mobilenet_v1_1.0_224_quant.textproto",
+    out = "assets/proto/mobilenet_v1_1.0_224_quant.binarypb",
+    proto_deps = ["//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto:delegate_performance_proto"],
+    proto_name = "tflite.proto.benchmark.LatencyCriteria",
+)

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/models/mobilenet_v1_1.0_224.textproto

@@ -0,0 +1,19 @@
+# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+#
+# proto-file: third_party/tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto/delegate_performance.proto
+# proto-message: LatencyCriteria
+
+startup_overhead_max_regression_percentage_allowed: 5  # 5%
+average_inference_max_regression_percentage_allowed: 5  # 5%

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/models/mobilenet_v1_1.0_224_quant.textproto

@@ -0,0 +1,19 @@
+# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+#
+# proto-file: third_party/tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto/delegate_performance.proto
+# proto-message: LatencyCriteria
+
+startup_overhead_max_regression_percentage_allowed: 5  # 5%
+average_inference_max_regression_percentage_allowed: 5  # 5%

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto.bzl

@@ -0,0 +1,86 @@
+"""BUILD rules for converting proto text files into binary format."""
+
+load("@bazel_skylib//lib:new_sets.bzl", "sets")
+# copybara:uncomment load("@com_google_protobuf//bazel/common:proto_info.bzl", "ProtoInfo")
+
+def _descriptor_set_list(deps):
+    """Makes a list of distinct FileDescriptorSet files of deps's transitive dependencies"""
+    descriptor_set_set = sets.make()
+    for dep in deps:
+        if ProtoInfo in dep:
+            for descriptor_set in dep[ProtoInfo].transitive_descriptor_sets.to_list():
+                sets.insert(descriptor_set_set, descriptor_set)
+    return sets.to_list(descriptor_set_set)
+
+def _proto_data_impl(ctx):
+    output = ctx.actions.declare_file(
+        ctx.attr.out if ctx.attr.out else "%s.pb" % (ctx.attr.name,),
+    )
+
+    args = []
+    args.append("--deterministic_output")
+    args.append("--encode=%s" % (ctx.attr.proto_name,))
+
+    # Determine all proto descriptor set dependencies, as well as transitive dependencies. Pass
+    # them via --descriptor_set_in flag to the protoc tool.
+    #
+    # If descriptor_set_in exceeds 20000 characters, this implementation will need to be ported to
+    # support passing descriptor_set_in as a flagfile argument.
+    descriptor_set_in = []
+    descriptor_set_list = _descriptor_set_list(ctx.attr.proto_deps)
+    for file in ctx.files.proto_deps:
+        descriptor_set_in.append(file.path)
+    if descriptor_set_list:
+        args.append("--descriptor_set_in=%s" % ":".join([d.path for d in descriptor_set_list]))
+
+    redirect = [
+        # textproto input is passed via stdin.
+        "< '%s\'" % ctx.file.src.path,
+        # binaryproto output is emitted via stdout.
+        "> '%s'" % output.path,
+    ]
+
+    ctx.actions.run_shell(
+        outputs = [output],
+        inputs = [ctx.file.src] + descriptor_set_list,
+        mnemonic = "ProtoDataEncode",
+        tools = [ctx.executable._tool],
+        command = " ".join([ctx.executable._tool.path] + args + redirect),
+        use_default_shell_env = False,
+    )
+    return DefaultInfo(
+        files = depset([output]),
+        runfiles = ctx.runfiles(files = [output]),
+    )
+
+_TOOL = "@com_google_protobuf//:protoc"
+
+# BUILD rule to convert a protocol buffer in text format into the standard binary format.
+#
+# Args:
+#   name: The name of the build target.
+#   src: A text formatted protocol buffer.
+#   proto_name: The name of the message type in the .proto files that "src" file represents.
+#   proto_deps: The list of proto_library targets where "proto" is defined.
+#               Transitive dependencies are pulled in automatically.
+#   out: (optional) The name of output file. If out is not set then name of
+#        output file is name + ".binarypb" extension.
+proto_data = rule(
+    implementation = _proto_data_impl,
+    attrs = {
+        "src": attr.label(
+            allow_single_file = True,
+        ),
+        "proto_name": attr.string(),
+        "proto_deps": attr.label_list(
+            providers = [ProtoInfo],
+        ),
+        "out": attr.string(),
+        "_tool": attr.label(
+            executable = True,
+            cfg = "exec",
+            allow_files = True,
+            default = Label(_TOOL),
+        ),
+    },
+)

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto/BUILD

@@ -0,0 +1,36 @@
+# Description:
+#  Holds model-agnostic files and proto definitions. The app will bundle the files into assets.
+
+load("@com_google_protobuf//bazel:java_lite_proto_library.bzl", "java_lite_proto_library")
+load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
+load("//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android:proto.bzl", "proto_data")
+# copybara:uncomment load("//tools/build_defs/proto/cpp:cc_proto_library.bzl", "cc_proto_library")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_visibility = ["//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android:__subpackages__"],
+    licenses = ["notice"],
+)
+
+proto_library(
+    name = "delegate_performance_proto",
+    srcs = ["delegate_performance.proto"],
+)
+
+cc_proto_library(
+    name = "delegate_performance_cc_proto",
+    deps = [":delegate_performance_proto"],
+)
+
+java_lite_proto_library(
+    name = "delegate_performance_java_proto_lite",
+    deps = [":delegate_performance_proto"],
+)
+
+proto_data(
+    name = "default_latency_criteria",
+    src = "default_latency_criteria.textproto",
+    out = "assets/proto/default_latency_criteria.binarypb",
+    proto_deps = [":delegate_performance_proto"],
+    proto_name = "tflite.proto.benchmark.LatencyCriteria",
+)

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto/default_latency_criteria.textproto

@@ -0,0 +1,19 @@
+# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+#
+# proto-file: third_party/tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto/delegate_performance.proto
+# proto-message: LatencyCriteria
+
+startup_overhead_max_regression_percentage_allowed: 5  # 5%
+average_inference_max_regression_percentage_allowed: 5  # 5%

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto/delegate_performance.proto

@@ -0,0 +1,113 @@
+// Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// =============================================================================
+
+syntax = "proto3";
+
+package tflite.proto.benchmark;
+
+option java_package = "tflite.proto.benchmark";
+
+// Parameters for latency thresholds of the delegate latency benchmarking
+// results. The delegate performance benchmark app generates a
+// "PASS"/"PASS_WITH_WARNING"/"FAIL" result for each pair of test target
+// delegate with a reference delegate and an overall result by aggregating all
+// the results. The result computation involves computing the test target
+// delegate performance regressions and checking if the regression thresholds,
+// specified by LatencyCriteria, are breached.
+//
+// Please see
+// tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkResultType.java
+// for the meanings of "PASS", "PASS_WITH_WARNING" and "FAIL".
+//
+// The latency criteria is designed to be model specific.
+//
+// Next ID: 3
+message LatencyCriteria {
+  // The maximum regression (%) of startup overhead time that is allowed.
+  // If initialization_max_regression_percentage_allow is 5 and the
+  // initialization time of the test delegate is slower than the reference
+  // delegate by more than 5%, the threshold is breached.
+  //
+  // Startup overhead time is calculated as below:
+  // initialization time + average warmup time - average inference time
+  optional float startup_overhead_max_regression_percentage_allowed = 1;
+  // The maximum regression (%) of inference time that is allowed.
+  // If average_inference_max_regression_percentage_allowed is 5 and the
+  // inference time of the test delegate is slower than the reference delegate
+  // by more than 5%, the threshold is breached.
+  optional float average_inference_max_regression_percentage_allowed = 2;
+}
+
+// Which stage of benchmarking the event is for.
+enum BenchmarkEventType {
+  BENCHMARK_EVENT_TYPE_UNDEFINED = 0;
+  // Benchmark start. A start without an end can be interpreted as a test that
+  // has crashed or hung.
+  BENCHMARK_EVENT_TYPE_START = 1;
+  // Benchmarking completion. A model was successfully loaded, acceleration
+  // configured and inference run without errors. There may still be an issue
+  // with correctness of results, or with performance.
+  BENCHMARK_EVENT_TYPE_END = 2;
+  // Benchmark was not completed due to an error. The error may be a handled
+  // error (e.g., failure in a delegate), or a crash.
+  BENCHMARK_EVENT_TYPE_ERROR = 3;
+}
+
+// A handled error.
+//
+// Next ID: 2
+message ErrorCode {
+  // What the TF Lite level error is.
+  // See TfLiteStatus in tensorflow/lite/core/c/c_api_types.h for the meaning of
+  // the values.
+  optional int32 tflite_error = 1;
+}
+
+// An error that occurred during benchmarking.
+//
+// Used with event type ERROR.
+//
+// Next ID: 3
+message BenchmarkError {
+  // Handled tflite error.
+  optional ErrorCode error_code = 1;
+  optional string error_message = 2;
+}
+
+// A metric from a benchmark, for example an average inference time in
+// microseconds.
+message BenchmarkMetric {
+  // Metric name, for example inference_latency_average_us.
+  optional string name = 1;
+  // Metric value, for example 180000.
+  optional float value = 2;
+}
+
+// Outcome of a latency benchmark run.  If the benchmark run was successfully
+// completed, the message contains the latency metrics. The information is
+// intended to be compared against with other candidate acceleration
+// configurations. If the benchmark run was failed, the message contains the
+// handled errors for the callers to investigate further.
+//
+// Next ID: 4
+message LatencyResults {
+  // Type of the benchmark event.
+  optional BenchmarkEventType event_type = 1;
+  // Metrics that are intended to be compared against other acceleration
+  // configurations, used when type is END.
+  repeated BenchmarkMetric metrics = 2;
+  // Error during benchmark, used when type is ERROR.
+  optional BenchmarkError error = 3;
+}

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/AccuracyBenchmarkReport.java

@@ -0,0 +1,93 @@
+/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+package org.tensorflow.lite.benchmark.delegateperformance;
+
+import static org.tensorflow.lite.benchmark.delegateperformance.Preconditions.checkNotNull;
+
+import android.util.Log;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import tflite.BenchmarkEvent;
+import tflite.BenchmarkEventType;
+import tflite.BenchmarkMetric;
+import tflite.BenchmarkResult;
+
+/** Model-level accuracy benchmark report class. */
+final class AccuracyBenchmarkReport extends ModelBenchmarkReport {
+  public static final double PASS = 0;
+  public static final double FAIL = 1;
+  private static final String TAG = "AccuracyBenchmarkReport";
+
+  private AccuracyBenchmarkReport(
+      String modelName, List<RawDelegateMetricsEntry> rawDelegateMetricsEntries) {
+    super(modelName);
+    Log.i(TAG, "Creating an accuracy benchmark report for " + modelName);
+    // Adds a regression threshold for "ok" here to make sure that "ok" will be consumed during
+    // metric computation.
+    // TODO(b/267313326): replace the mitigation with a proper accuracy benchmark criteria.
+    maxRegressionPercentageAllowed.put("ok", 0.0);
+    computeModelReport(rawDelegateMetricsEntries);
+  }
+
+  /**
+   * Parses {@link BenchmarkEvent} into the unified {@link RawDelegateMetricsEntry} format for
+   * further processing.
+   */
+  public static RawDelegateMetricsEntry parseResults(
+      BenchmarkEvent event, TfLiteSettingsListEntry entry) {
+    Map<String, Double> metrics = new LinkedHashMap<>();
+    if (event == null || event.eventType() != BenchmarkEventType.END || event.result() == null) {
+      Log.w(TAG, "The accuracy benchmarking is not completed successfully for " + entry.filePath());
+      return RawDelegateMetricsEntry.create(
+          entry.tfliteSettings().delegate(), entry.filePath(), entry.isTestTarget(), metrics);
+    }
+    BenchmarkResult accuracyResults = event.result();
+    for (int i = 0; i < accuracyResults.metricsLength(); i++) {
+      BenchmarkMetric metric = accuracyResults.metrics(i);
+      checkNotNull(metric);
+      if (metric.valuesLength() == 0) {
+        Log.i(TAG, "The metric " + metric.name() + " is empty. Skipping to the next metric.");
+        continue;
+      }
+      String metricName = metric.name();
+      double metricValue = metric.values(0);
+      if (metric.valuesLength() > 1) {
+        // TODO(b/267765648): consider updating the metric aggregation logic.
+        metricName += "(average)";
+        double sum = 0f;
+        for (int j = 0; j < metric.valuesLength(); j++) {
+          sum += metric.values(j);
+        }
+        metricValue = sum / metric.valuesLength();
+      }
+      metrics.put(metricName, metricValue);
+    }
+    // The value of {@code ok} is set to {@code 0} when the delegate-under-test has passed the
+    // accuracy checks performed by MiniBenchmark. Otherwise, the value of {@code ok} is set to
+    // {@code 1}.
+    // TODO(b/267313326): replace the mitigation with a proper accuracy benchmark criteria.
+    metrics.put("ok", accuracyResults.ok() ? PASS : FAIL);
+    metrics.put("max_memory_kb", (double) accuracyResults.maxMemoryKb());
+
+    return RawDelegateMetricsEntry.create(
+        entry.tfliteSettings().delegate(), entry.filePath(), entry.isTestTarget(), metrics);
+  }
+
+  static AccuracyBenchmarkReport create(
+      String modelName, List<RawDelegateMetricsEntry> rawDelegateMetricsEntries) {
+    return new AccuracyBenchmarkReport(modelName, rawDelegateMetricsEntries);
+  }
+}

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkAccuracy.java

@@ -0,0 +1,28 @@
+/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+package org.tensorflow.lite.benchmark.delegateperformance;
+
+import android.content.Context;
+
+/** Interface for Delegate Performance Accuracy Benchmark. */
+public interface BenchmarkAccuracy {
+  /**
+   * Initializes and runs the accuracy benchmark.
+   *
+   * @param context the context to use for finding the test models and exporting reports
+   * @param tfliteSettingsJsonFiles the list of paths to delegate JSON configurations
+   */
+  void benchmark(Context context, String[] tfliteSettingsJsonFiles);
+}

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkAccuracyActivity.java

@@ -0,0 +1,46 @@
+/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.lite.benchmark.delegateperformance;
+
+import android.app.Activity;
+import android.content.Intent;
+import android.os.Bundle;
+import android.util.Log;
+
+/**
+ * {@link Activity} class for Delegate Performance Accuracy Benchmark.
+ *
+ * <p>This Activity receives test arguments via a command line specified in an intent extra. It
+ * performs accuracy benchmark tests via TFLite MiniBenchmark based on the input arguments. Please
+ * check the test example in
+ * tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/README.md.
+ */
+public class BenchmarkAccuracyActivity extends Activity {
+
+  private static final String TAG = "TfLiteBenchmarkAccuracy";
+  private static final String TFLITE_SETTINGS_FILES_INTENT_KEY_0 = "--tflite_settings_files";
+
+  @Override
+  public void onCreate(Bundle savedInstanceState) {
+    Log.i(TAG, "Create benchmark accuracy activity.");
+    super.onCreate(savedInstanceState);
+
+    Intent intent = getIntent();
+    Bundle bundle = intent.getExtras();
+    String[] tfliteSettingsJsonFiles = bundle.getStringArray(TFLITE_SETTINGS_FILES_INTENT_KEY_0);
+    new BenchmarkAccuracyImpl().benchmark(this, tfliteSettingsJsonFiles);
+  }
+}

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkAccuracyImpl.java

@@ -0,0 +1,178 @@
+/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.lite.benchmark.delegateperformance;
+
+import android.app.Activity;
+import android.content.Context;
+import android.content.res.AssetFileDescriptor;
+import android.util.Log;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import tflite.BenchmarkEvent;
+
+/**
+ * Impl class for Delegate Performance Accuracy Benchmark.
+ *
+ * <p>It performs accuracy benchmark tests via TFLite MiniBenchmark based on the input arguments.
+ * Please check the test example in
+ * tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/README.md.
+ *
+ * <p>Generates a PASS/PASS_WITH_WARNING/FAIL result.
+ *
+ * <ul>
+ *   <li>PASS: The test target delegate passed the embedded metric thresholds in all models.
+ *   <li>PASS_WITH_WARNING: Both the test target delegate and the reference delegates breached the
+ *       embedded metric thresholds.
+ *   <li>FAIL: The test target delegate failed at least 1 embedded metric threshold in the models,
+ *       and at least 1 reference delegate passed the embedded metric thresholds in all models.
+ * </ul>
+ *
+ * <p>Generates below list of files to describe the benchmark results under
+ * delegate_performance_result/accuracy folder in the app files directory.
+ *
+ * <ul>
+ *   <li>1. delegate_performance_result/accuracy/report.csv: the performance of each acceleration
+ *       configuration and relative performance differences as percentages in CSV.
+ *   <li>2. delegate_performance_result/accuracy/report.json: detailed performance results. The file
+ *       contains the metric-level, delegate-level and model-level results and the raw metric
+ *       outputs from the native layer in JSON.
+ *   <li>3. delegate_performance_result/accuracy/report.html: the performance of each acceleration
+ *       configuration and relative performance differences as percentages in HTML.
+ * </ul>
+ */
+public class BenchmarkAccuracyImpl implements BenchmarkAccuracy {
+
+  private static final String TAG = "TfLiteAccuracyImpl";
+  private static final String ACCURACY_FOLDER_NAME = "accuracy";
+
+  private Context context;
+  private String[] tfliteSettingsJsonFiles;
+  private BenchmarkReport report;
+
+  @Override
+  public void benchmark(Context context, String[] tfliteSettingsJsonFiles) {
+    Activity activity = (Activity) context;
+    if (!benchmarkInternal(context.getApplicationContext(), tfliteSettingsJsonFiles)) {
+      Log.e(TAG, "Failed to complete accuracy benchmark.");
+    }
+    activity.finish();
+  }
+
+  public boolean benchmarkInternal(Context context, String[] tfliteSettingsJsonFiles) {
+    if (!initialize(context.getApplicationContext(), tfliteSettingsJsonFiles)) {
+      Log.e(TAG, "Failed to initialize accuracy benchmark.");
+      return false;
+    }
+    return benchmarkDelegatesAndExportReport();
+  }
+
+  private boolean benchmarkDelegatesAndExportReport() {
+    Log.i(
+        TAG,
+        "Running accuracy benchmark with TFLiteSettings JSON files: "
+            + Arrays.toString(tfliteSettingsJsonFiles));
+    List<TfLiteSettingsListEntry> tfliteSettingsList =
+        DelegatePerformanceBenchmark.loadTfLiteSettingsList(tfliteSettingsJsonFiles);
+    if (tfliteSettingsList.size() < 2) {
+      Log.e(TAG, "Failed to load the TFLiteSettings JSON file.");
+      return false;
+    }
+    String[] assets;
+    try {
+      assets = context.getAssets().list(ACCURACY_FOLDER_NAME);
+    } catch (IOException e) {
+      Log.e(TAG, "Failed to list files from assets folder.", e);
+      return false;
+    }
+    for (String asset : assets) {
+      if (!asset.endsWith(".tflite")) {
+        Log.i(TAG, asset + " is not a model file. Skipping.");
+        continue;
+      }
+      String modelResultPath;
+      String modelName = DelegatePerformanceBenchmark.getModelName(asset);
+      try {
+        modelResultPath =
+            DelegatePerformanceBenchmark.createResultFolder(
+                context.getFilesDir(), ACCURACY_FOLDER_NAME + "/" + modelName);
+      } catch (IOException e) {
+        Log.e(TAG, "Failed to create result folder for " + modelName + ". Exiting application.", e);
+        return false;
+      }
+      try (AssetFileDescriptor modelFileDescriptor =
+          context.getAssets().openFd(ACCURACY_FOLDER_NAME + "/" + asset)) {
+        List<RawDelegateMetricsEntry> rawDelegateMetricsEntries = new ArrayList<>();
+        for (TfLiteSettingsListEntry tfliteSettingsListEntry : tfliteSettingsList) {
+          BenchmarkEvent benchmarkEvent =
+              DelegatePerformanceBenchmark.runAccuracyBenchmark(
+                  tfliteSettingsListEntry,
+                  modelFileDescriptor.getParcelFileDescriptor().getFd(),
+                  modelFileDescriptor.getStartOffset(),
+                  modelFileDescriptor.getLength(),
+                  modelResultPath);
+
+          rawDelegateMetricsEntries.add(
+              AccuracyBenchmarkReport.parseResults(benchmarkEvent, tfliteSettingsListEntry));
+        }
+        report.addModelBenchmarkReport(
+            AccuracyBenchmarkReport.create(modelName, rawDelegateMetricsEntries));
+      } catch (IOException e) {
+        Log.e(TAG, "Failed to open assets file " + asset, e);
+        return false;
+      }
+    }
+    // Computes the aggregated results and export the report to local files.
+    report.export();
+    TfLiteSettingsListEntry testTarget = tfliteSettingsList.get(tfliteSettingsList.size() - 1);
+    Log.i(
+        TAG,
+        String.format(
+            "Accuracy benchmark result for %s: %s.", testTarget.filePath(), report.result()));
+    return true;
+  }
+
+  /**
+   * Initializes the test environment. Checks the validity of input arguments and creates the result
+   * folder.
+   *
+   * @return {@code true} if the initialization was successful. Otherwise, returns {@code false}.
+   */
+  private boolean initialize(Context context, String[] tfliteSettingsJsonFiles) {
+    if (tfliteSettingsJsonFiles == null || tfliteSettingsJsonFiles.length == 0) {
+      Log.e(TAG, "No TFLiteSettings file provided.");
+      return false;
+    }
+    this.context = context;
+    this.tfliteSettingsJsonFiles = tfliteSettingsJsonFiles;
+    report = BenchmarkReport.create();
+
+    try {
+      // Creates root result folder.
+      String resultFolderPath =
+          DelegatePerformanceBenchmark.createResultFolder(
+              context.getFilesDir(), ACCURACY_FOLDER_NAME);
+      report.addWriter(JsonWriter.create(resultFolderPath));
+      report.addWriter(CsvWriter.create(resultFolderPath));
+      report.addWriter(HtmlWriter.create(resultFolderPath));
+    } catch (IOException e) {
+      Log.e(TAG, "Failed to create result folder", e);
+      return false;
+    }
+    return true;
+  }
+}

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkLatencyActivity.java

@@ -0,0 +1,56 @@
+/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.lite.benchmark.delegateperformance;
+
+import android.app.Activity;
+import android.content.Intent;
+import android.os.Bundle;
+import android.util.Log;
+
+/**
+ * {@link Activity} class for Delegate Performance Latency Benchmark.
+ *
+ * <p>This Activity receives test arguments via a command line specified in an intent extra. It
+ * passes the arguments to the {@link BenchmarkLatencyImpl} class to perform latency benchmark tests
+ * via TFLite Benchmark Tool. Please check the test example in
+ * tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/README.md.
+ */
+public class BenchmarkLatencyActivity extends Activity {
+
+  private static final String TAG = "TfLiteBenchmarkLatency";
+  private static final String TFLITE_SETTINGS_FILES_INTENT_KEY_0 = "--tflite_settings_files";
+  private static final String ARGS_INTENT_KEY_0 = "--args";
+
+  @Override
+  public void onCreate(Bundle savedInstanceState) {
+    Log.i(TAG, "Create benchmark latency activity.");
+    super.onCreate(savedInstanceState);
+
+    Intent intent = getIntent();
+    Bundle bundle = intent.getExtras();
+    String[] tfliteSettingsJsonFiles = bundle.getStringArray(TFLITE_SETTINGS_FILES_INTENT_KEY_0);
+    String[] args = bundle.getStringArray(ARGS_INTENT_KEY_0);
+
+    BenchmarkLatencyImpl impl =
+        new BenchmarkLatencyImpl(getApplicationContext(), tfliteSettingsJsonFiles, args);
+    if (impl.initialize()) {
+      impl.benchmark();
+    } else {
+      Log.e(TAG, "Failed to initialize the latency benchmarking.");
+    }
+    finish();
+  }
+}

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkLatencyImpl.java

@@ -0,0 +1,215 @@
+/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.lite.benchmark.delegateperformance;
+
+import static org.tensorflow.lite.benchmark.delegateperformance.Preconditions.checkState;
+
+import android.content.Context;
+import android.content.res.AssetFileDescriptor;
+import android.util.Log;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import tflite.proto.benchmark.DelegatePerformance.LatencyCriteria;
+import tflite.proto.benchmark.DelegatePerformance.LatencyResults;
+
+/**
+ * Impl class for Delegate Performance Latency Benchmark.
+ *
+ * <p>It performs latency benchmark tests via TFLite Benchmark Tool based on the input arguments.
+ * Please check the test example in
+ * tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/README.md.
+ *
+ * <p>Generates below list of files under delegate_performance_result/latency folder to describe the
+ * benchmark results.
+ *
+ * <ul>
+ *   <li>1. delegate_performance_result/latency/report.csv: the performance of each acceleration
+ *       configuration and relative performance differences as percentages in CSV.
+ *   <li>2. delegate_performance_result/latency/report.json: detailed performance results. The file
+ *       contains the metric-level, delegate-level and model-level results, the latency criteria and
+ *       the raw metric outputs from the native layer in JSON.
+ *   <li>3. delegate_performance_result/latency/report.html: the performance of each acceleration
+ *       configuration and relative performance differences as percentages in HTML.
+ * </ul>
+ *
+ * The metrics for generating the Pass/Pass with Warning/Fail decision:
+ *
+ * <ul>
+ *   <li>1. startup overhead latency: it is equal to (initialization time + average warmup latency -
+ *       average inference time).
+ *   <li>2. average inference latency: average time for the inferences in the benchmark run.
+ * </ul>
+ */
+public final class BenchmarkLatencyImpl {
+
+  private static final String LATENCY_FOLDER_NAME = "latency";
+  private static final String PROTO_FOLDER_NAME = "proto";
+  private static final String TAG = "TfLiteLatencyImpl";
+  private static final String DEFAULT_LATENCY_CRITERIA_FILENAME = "default_latency_criteria";
+  private static final String LATENCY_CRITERIA_FILE_EXT = ".binarypb";
+
+  private final Context context;
+  private final String[] tfliteSettingsJsonFiles;
+  private final String[] args;
+  private final BenchmarkReport report;
+  private LatencyCriteria defaultLatencyCriteria;
+
+  public BenchmarkLatencyImpl(Context context, String[] tfliteSettingsJsonFiles, String[] args) {
+    this.context = context;
+    this.tfliteSettingsJsonFiles = tfliteSettingsJsonFiles;
+    if (args == null) {
+      // The "--args" extra key was not provided.
+      this.args = new String[0];
+    } else {
+      this.args = args;
+    }
+    this.report = BenchmarkReport.create();
+  }
+
+  /**
+   * Initializes the test environment. Creates the result folder and loads the default latency
+   * criteria file.
+   *
+   * <p>Returns {@code true} if the initialization was successful. Otherwise, returns {@code false}.
+   */
+  public boolean initialize() {
+    if (tfliteSettingsJsonFiles == null || tfliteSettingsJsonFiles.length == 0) {
+      Log.e(TAG, "No TFLiteSettings file provided.");
+      return false;
+    }
+    try {
+      // Creates root result folder.
+      String resultFolderPath =
+          DelegatePerformanceBenchmark.createResultFolder(
+              context.getFilesDir(), LATENCY_FOLDER_NAME);
+      report.addWriter(JsonWriter.create(resultFolderPath));
+      report.addWriter(CsvWriter.create(resultFolderPath));
+      report.addWriter(HtmlWriter.create(resultFolderPath));
+    } catch (IOException e) {
+      Log.e(
+          TAG, "Failed to create result folder " + LATENCY_FOLDER_NAME + " in files directory.", e);
+      return false;
+    }
+
+    try {
+      // Loads default latency criteria.
+      defaultLatencyCriteria = loadLatencyCriteria(DEFAULT_LATENCY_CRITERIA_FILENAME);
+    } catch (IOException e) {
+      Log.e(TAG, "Failed to load default latency criteria " + DEFAULT_LATENCY_CRITERIA_FILENAME, e);
+      return false;
+    }
+    return true;
+  }
+
+  /** Benchmarks the embedded model files with the input TFLiteSettings JSON files. */
+  public void benchmark() {
+    List<TfLiteSettingsListEntry> tfliteSettingsList =
+        DelegatePerformanceBenchmark.loadTfLiteSettingsList(tfliteSettingsJsonFiles);
+    if (tfliteSettingsList.size() < 2) {
+      Log.e(TAG, "Failed to load the TFLiteSettings JSON file.");
+      return;
+    }
+
+    String[] assets;
+    try {
+      assets = context.getAssets().list(LATENCY_FOLDER_NAME);
+    } catch (IOException e) {
+      Log.e(TAG, "Failed to list files from assets folder.", e);
+      return;
+    }
+    for (String asset : assets) {
+      if (asset.endsWith(".tflite")) {
+        report.addModelBenchmarkReport(benchmarkModel(asset, tfliteSettingsList, args));
+      }
+    }
+    // Computes the aggregated results and export the report to local files.
+    report.export();
+    TfLiteSettingsListEntry testTarget = tfliteSettingsList.get(tfliteSettingsList.size() - 1);
+    checkState(testTarget.isTestTarget());
+    Log.i(
+        TAG,
+        String.format(
+            "Latency benchmark result for %s: %s", testTarget.filePath(), report.result()));
+  }
+
+  /**
+   * Benchmarks a model file with the TfLiteSettingsListEntry list.
+   *
+   * <p>Returns {@link ModelBenchmarkReportInterface}, which is the model level benchmark report
+   * that has the delegate information and the raw metric results from the native layer.
+   */
+  private ModelBenchmarkReportInterface benchmarkModel(
+      String modelFilename, List<TfLiteSettingsListEntry> tfliteSettingsList, String[] args) {
+    String modelName = DelegatePerformanceBenchmark.getModelName(modelFilename);
+    LatencyCriteria latencyCriteria = tryLoadLatencyCriteria(modelName);
+    List<RawDelegateMetricsEntry> rawDelegateMetricsEntries = new ArrayList<>();
+    try (AssetFileDescriptor modelFileDescriptor =
+        context.getAssets().openFd(LATENCY_FOLDER_NAME + "/" + modelFilename)) {
+      for (TfLiteSettingsListEntry tfliteSettingsListEntry : tfliteSettingsList) {
+        Log.i(
+            TAG,
+            "Running latency benchmark with model: "
+                + modelName
+                + ", settings: "
+                + tfliteSettingsListEntry.filePath()
+                + ", args: "
+                + Arrays.toString(args));
+        LatencyResults results =
+            DelegatePerformanceBenchmark.runLatencyBenchmark(
+                args,
+                tfliteSettingsListEntry,
+                modelFileDescriptor.getParcelFileDescriptor().getFd(),
+                modelFileDescriptor.getStartOffset(),
+                modelFileDescriptor.getLength());
+        rawDelegateMetricsEntries.add(
+            LatencyBenchmarkReport.parseResults(results, tfliteSettingsListEntry));
+      }
+    } catch (IOException e) {
+      Log.e(TAG, "Failed to open asset file " + LATENCY_FOLDER_NAME + "/" + modelFilename);
+    }
+    return LatencyBenchmarkReport.create(modelName, rawDelegateMetricsEntries, latencyCriteria);
+  }
+
+  /**
+   * Tries to load the model-specific latency criteria file by the model name.
+   *
+   * <p>Returns the latency criteria for the specific model if the loading was successful.
+   * Otherwise, returns the default latency criteria.
+   */
+  private LatencyCriteria tryLoadLatencyCriteria(String fileBasename) {
+    try {
+      return loadLatencyCriteria(fileBasename);
+    } catch (IOException e) {
+      Log.w(
+          TAG,
+          "Failed to load the latency criteria of "
+              + fileBasename
+              + ". Fallback to the default latency criteria.");
+    }
+    return defaultLatencyCriteria;
+  }
+
+  /** Loads the latency criteria file from Assets. */
+  private LatencyCriteria loadLatencyCriteria(String fileBasename) throws IOException {
+    String latencyCriteriaFileAssetPath =
+        PROTO_FOLDER_NAME + "/" + fileBasename + LATENCY_CRITERIA_FILE_EXT;
+    InputStream latencyCriteriaFile = context.getAssets().open(latencyCriteriaFileAssetPath);
+    return LatencyCriteria.parseFrom(latencyCriteriaFile);
+  }
+}

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkReport.java

@@ -0,0 +1,98 @@
+/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+package org.tensorflow.lite.benchmark.delegateperformance;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+/**
+ * Benchmark session-level report class to store the model-level {@link
+ * ModelBenchmarkReportInterface} reports. It allows {@link ReportWriter} subscriptions for
+ * exporting this report.
+ */
+final class BenchmarkReport {
+  private static final String TAG = "TfLiteBenchmarkReport";
+  private static final String NAME = "report";
+
+  private final List<ReportWriter> writers = new ArrayList<>();
+  private final List<ModelBenchmarkReportInterface> modelBenchmarkReports = new ArrayList<>();
+  // The benchmark session-level pass status.
+  // Possible values:
+  // - UNKNOWN: The metric computation has not started or failed to complete.
+  // - PASS: All model-level results are "PASS" or "NOT_APPLICABLE".
+  // - PASS_WITH_WARNING: At least 1 "PASS_WITH_WARNING" model-level result and 0 "FAIL" model-level
+  // result.
+  // - FAIL: At least 1 "FAIL" model-level result.
+  private BenchmarkResultType result = BenchmarkResultType.UNKNOWN;
+
+  void addModelBenchmarkReport(ModelBenchmarkReportInterface modelBenchmarkReport) {
+    modelBenchmarkReports.add(modelBenchmarkReport);
+  }
+
+  void addWriter(ReportWriter writer) {
+    writers.add(writer);
+  }
+
+  void export() {
+    if (result == BenchmarkResultType.UNKNOWN) {
+      // The result is not computed.
+      computeBenchmarkReport();
+    }
+    for (ReportWriter writer : writers) {
+      writer.writeReport(this);
+    }
+  }
+
+  // TODO(b/268338967): Use a more informative name for the report.
+  String name() {
+    return NAME;
+  }
+
+  List<ModelBenchmarkReportInterface> modelBenchmarkReports() {
+    return Collections.unmodifiableList(modelBenchmarkReports);
+  }
+
+  BenchmarkResultType result() {
+    return result;
+  }
+
+  JSONObject toJsonObject() throws JSONException {
+    JSONObject jsonObject = new JSONObject();
+    jsonObject.put("name", NAME);
+    JSONArray jsonArray = new JSONArray();
+    for (ModelBenchmarkReportInterface modelBenchmarkReport : modelBenchmarkReports) {
+      jsonArray.put(modelBenchmarkReport.toJsonObject());
+    }
+    jsonObject.put("reports", jsonArray);
+    jsonObject.put("result", result.toString());
+    return jsonObject;
+  }
+
+  private void computeBenchmarkReport() {
+    List<BenchmarkResultType> results = new ArrayList<>();
+    for (ModelBenchmarkReportInterface modelBenchmarkReport : modelBenchmarkReports) {
+      results.add(modelBenchmarkReport.result());
+    }
+    result = DelegatePerformanceBenchmark.aggregateResults(/* strict= */ true, results);
+  }
+
+  static BenchmarkReport create() {
+    return new BenchmarkReport();
+  }
+}

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkResultType.java

@@ -0,0 +1,82 @@
+/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.lite.benchmark.delegateperformance;
+
+/** Enumerates the possible benchmark result values. */
+public enum BenchmarkResultType {
+  // Unknown benchmark result, possibly due to internal failures.
+  UNKNOWN("UNKNOWN"),
+  // The benchmark results are not involved in the Pass/Pass with Warning/Fail result generation.
+  NOT_APPLICABLE("N/A"),
+  // The benchmark activity skips the Pass/Pass with Warning/Fail result generation.
+  SKIP("SKIP"),
+
+  // The meanings of "PASS", "PASS_WITH_WARNING" and "FAIL":
+  //
+  // Latency benchmark:
+  // 1. When the test target delegate type is the same as the reference delegate:
+  // the test target delegate is expected to be better, or at least not
+  // substantially worse, than the reference delegate in ALL metrics.
+  //    - PASS: All performance metrics of the test target delegate are better
+  //    than or equal to the reference delegate.
+  //    - PASS_WITH_WARNING: At least 1 metric is worse than the reference
+  //    delegate, but not substantially worse. No regression thresholds are
+  //    breached.
+  //    - FAIL: At least 1 regression threshold is breached.
+  // 2. When the test target delegate type is different from the reference
+  // delegate: the test delegate is only expected to be better, or at least not
+  // substantially worse, than the reference delegate in AT LEAST 1 metric.
+  //    - PASS: All performance metrics of the test target delegate are better than
+  //    or equal to the reference delegate.
+  //    - PASS_WITH_WARNING: At least 1 metric is worse than the reference
+  //    delegate, but, at least 1 regression threshold is not breached.
+  //    - FAIL: All regression thresholds are breached.
+  // 3. The overall result is generated by aggregating the results from the pairs.
+  //    - PASS: All results are "PASS".
+  //    - PASS_WITH_WARNING: At least 1 "PASS_WITH_WARNING" result and 0 "FAIL"
+  //    result.
+  //    - FAIL: At least 1 "FAIL" result.
+  //
+  // Accuracy benchmark:
+  // The accuracy benchmark delegates the accuracy metric threshold checks to the metric scripts,
+  // which are embedded together with the test input and the ground truth inside the models. The
+  // metric scripts are responsible for computing the accuracy metrics (e.g. Mean square error) from
+  // the model predictions and the ground truth data, checking the accuracy metrics against the
+  // pre-defined thresholds and aggregating the results into a binarized pass status.
+  // 1. When the metric scripts check the outcomes for every model and every delegate:
+  //    - PASS: No accuracy threshold in the model is breached by the delegate.
+  //    - FAIL: At least 1 accuracy threshold threshold is breached.
+  // 2. The overall result is generated by aggregating the results from the delegate results for
+  // each model.
+  //    - PASS: The test target delegate result is "PASS".
+  //    - PASS_WITH_WARNING: The test delegate and all reference delegates results are "FAIL".
+  //    - FAIL: The test target delegate result is "FAIL" and at least 1 reference delegate result
+  //    is "PASS".
+  PASS("PASS"),
+  PASS_WITH_WARNING("PASS_WITH_WARNING"),
+  FAIL("FAIL");
+
+  private final String name;
+
+  BenchmarkResultType(String name) {
+    this.name = name;
+  }
+
+  @Override
+  public String toString() {
+    return name;
+  }
+}

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/CsvWriter.java

@@ -0,0 +1,113 @@
+/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.lite.benchmark.delegateperformance;
+
+import static org.tensorflow.lite.benchmark.delegateperformance.Preconditions.checkState;
+
+import android.util.Log;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.List;
+
+/** Helper class for writing the final report in CSV format. */
+final class CsvWriter implements ReportWriter {
+  private static final String TAG = "TfLiteCsvWriter";
+
+  private final String destinationFolderPath;
+
+  private CsvWriter(String destinationFolderPath) {
+    this.destinationFolderPath = destinationFolderPath;
+  }
+
+  /** Writes the benchmark results into a CSV file. */
+  @Override
+  public void writeReport(BenchmarkReport report) {
+    // Example output file:
+    // Model, Metric, DELEGATE_TYPE (PATH), DELEGATE_TYPE (PATH), Change, Status,...
+    // model_1, metric_1, 900 , 1000, -10%, PASS, ...
+    // model_1, metric_2, ...
+    // model_1, delegate_summary,,,, PASS, ...
+    // model_1, model_summary, PASS,
+    // model_2, ...
+    // ...
+    // Summary, Summary, PASS
+    StringBuilder sb = new StringBuilder();
+    sb.append(destinationFolderPath).append("/").append(report.name()).append(".csv");
+    String filePath = sb.toString();
+    Log.i(TAG, "Generating CSV report to " + filePath);
+
+    List<ModelBenchmarkReportInterface> modelReports = report.modelBenchmarkReports();
+    checkState(!modelReports.isEmpty());
+    sb = new StringBuilder();
+    // Heading row. It is structured as below:
+    // Model, Metric, DELEGATE_TYPE (PATH), DELEGATE_TYPE (PATH), Change, Status, ...
+    sb.append("Model, Metric");
+    for (DelegateMetricsEntry entry : modelReports.get(0).processedDelegateMetrics()) {
+      sb.append(", Delegate: ").append(entry.delegateIdentifier());
+      if (!entry.isTestTarget()) {
+        sb.append(", Change, Status");
+      }
+    }
+    sb.append('\n');
+    for (ModelBenchmarkReportInterface modelReport : modelReports) {
+      writerModelReport(modelReport, sb);
+    }
+    sb.append("Summary").append(", Summary,").append(report.result());
+    sb.append('\n');
+    try (PrintWriter writer = new PrintWriter(filePath)) {
+      writer.write(sb.toString());
+    } catch (IOException e) {
+      Log.e(TAG, "Failed to open report file " + filePath);
+    }
+  }
+
+  private void writerModelReport(ModelBenchmarkReportInterface modelReport, StringBuilder sb) {
+    String modelName = modelReport.modelName();
+    if (modelReport.processedDelegateMetrics().isEmpty()) {
+      Log.w(TAG, "The computed metric is empty.");
+    } else {
+      for (String metricName : modelReport.processedDelegateMetrics().get(0).metrics().keySet()) {
+        sb.append(modelName).append(",").append(metricName);
+        for (DelegateMetricsEntry delegateMetricsEntry : modelReport.processedDelegateMetrics()) {
+          MetricsEntry metricEntry = delegateMetricsEntry.metrics().get(metricName);
+          sb.append(",").append(metricEntry.value());
+          if (!delegateMetricsEntry.isTestTarget()) {
+            sb.append(",")
+                .append(metricEntry.regression())
+                .append(",")
+                .append(metricEntry.result());
+          }
+        }
+        sb.append('\n');
+      }
+    }
+    sb.append(modelName).append(",delegate_summary");
+    for (DelegateMetricsEntry delegateMetricsEntry : modelReport.processedDelegateMetrics()) {
+      sb.append(",");
+      if (!delegateMetricsEntry.isTestTarget()) {
+        // Position the delegate-level result correctly.
+        sb.append(",,").append(delegateMetricsEntry.result());
+      }
+    }
+    sb.append('\n');
+    sb.append(modelName).append(",model_summary,").append(modelReport.result());
+    sb.append('\n');
+  }
+
+  static ReportWriter create(String destinationFolderPath) {
+    return new CsvWriter(destinationFolderPath);
+  }
+}

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/DelegateMetricsEntry.java

@@ -0,0 +1,126 @@
+/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+package org.tensorflow.lite.benchmark.delegateperformance;
+
+import java.util.Collections;
+import java.util.Map;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+/**
+ * Helper class to store the performance results that were computed by {@link ModelBenchmarkReport}
+ * from the raw performance metrics.
+ *
+ * <p>The computation compares the performance metrics of a reference delegate and the test target
+ * delegate. Then it checks the performance regressions (if any) with the criteria to generate
+ * metric-level and delegate-level results.
+ *
+ * <p>An instance of {@link DelegateMetricsEntry} corresponds with each tested model and each pair
+ * reference delegate and the test target delegate.
+ */
+final class DelegateMetricsEntry {
+  // Identifies the delegate involved in the computation.
+  // Format: "DELEGATE_TYPE (PATH_TO_DELEGATE_SETTINGS_FILE)"
+  // TODO(b/267488243): use the delegate name instead of delegate type.
+  private final String delegateIdentifier;
+  /** Map from performance metric names to {@link MetricsEntry}. */
+  private final Map<String, MetricsEntry> metrics;
+  // The delegate-level pass status. The value computation involves computing the test target
+  // delegate performance regressions and checking if the regression thresholds, specified in the
+  // criteria, are breached.
+  // Possible values:
+  // - NOT_APPLICABLE: The reference delegate is the test target delegate.
+  // 1. When the test target delegate type is the same as the reference delegate.
+  //    - PASS: All performance metrics of the test target delegate are better than
+  //    or equal to the reference delegate.
+  //    - PASS_WITH_WARNING: No regression thresholds are breached.
+  //    - FAIL: At least 1 regression threshold is breached.
+  // 2. When the test target delegate type is different from the reference delegate.
+  //    - PASS: All performance metrics of the test target delegate are better or
+  //    equal to the reference delegate.
+  //    - PASS_WITH_WARNING: At least 1 regression threshold is not breached.
+  //    - FAIL: All regression thresholds are breached.
+  private final BenchmarkResultType result;
+  private final boolean isTestTarget;
+  /**
+   * The value is {@code true} when the results are generated from comparing two delegates of the
+   * same type. Otherwise, the value is {@code false}.
+   */
+  private final boolean isStrictCriteria;
+
+  private DelegateMetricsEntry(
+      String delegateIdentifier,
+      Map<String, MetricsEntry> metrics,
+      BenchmarkResultType result,
+      boolean isTestTarget,
+      boolean isStrictCriteria) {
+    this.delegateIdentifier = delegateIdentifier;
+    this.metrics = metrics;
+    this.result = result;
+    this.isTestTarget = isTestTarget;
+    this.isStrictCriteria = isStrictCriteria;
+  }
+
+  /** Returns an identifer to the delegate involved in the computation. */
+  String delegateIdentifier() {
+    return delegateIdentifier;
+  }
+
+  /** Returns an unmodifiable map from performance metric names to {@link MetricsEntry}. */
+  Map<String, MetricsEntry> metrics() {
+    return Collections.unmodifiableMap(metrics);
+  }
+
+  /** Returns the delegate-level pass status. */
+  BenchmarkResultType result() {
+    return result;
+  }
+
+  boolean isTestTarget() {
+    return isTestTarget;
+  }
+
+  /**
+   * Returns {@code true} when the results are generated from comparing two delegates of the same
+   * type. Otherwise, returns {@code false}.
+   */
+  boolean isStrictCriteria() {
+    return isStrictCriteria;
+  }
+
+  JSONObject toJsonObject() throws JSONException {
+    JSONObject jsonObject = new JSONObject();
+    jsonObject.put("delegate_identifier", delegateIdentifier);
+    jsonObject.put("result", result.toString());
+    JSONObject metricsObject = new JSONObject();
+    for (Map.Entry<String, MetricsEntry> entry : metrics.entrySet()) {
+      metricsObject.put(entry.getKey(), entry.getValue().toJsonObject());
+    }
+    jsonObject.put("metrics", metricsObject);
+    jsonObject.put("is_test_target", isTestTarget);
+    jsonObject.put("is_strict_criteria", isStrictCriteria);
+    return jsonObject;
+  }
+
+  static DelegateMetricsEntry create(
+      String delegateIdentifier,
+      Map<String, MetricsEntry> metrics,
+      BenchmarkResultType result,
+      boolean isTestTarget,
+      boolean isStrictCriteria) {
+    return new DelegateMetricsEntry(
+        delegateIdentifier, metrics, result, isTestTarget, isStrictCriteria);
+  }
+}

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/DelegatePerformanceBenchmark.java

@@ -0,0 +1,258 @@
+/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.lite.benchmark.delegateperformance;
+
+import static org.tensorflow.lite.benchmark.delegateperformance.Preconditions.checkArgument;
+import static org.tensorflow.lite.benchmark.delegateperformance.Preconditions.checkNotNull;
+import static org.tensorflow.lite.benchmark.delegateperformance.Preconditions.checkState;
+
+import android.util.Log;
+import com.google.flatbuffers.FlatBufferBuilder;
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Set;
+import tflite.BenchmarkEvent;
+import tflite.TFLiteSettings;
+import tflite.proto.benchmark.DelegatePerformance.BenchmarkEventType;
+import tflite.proto.benchmark.DelegatePerformance.LatencyResults;
+
+/** Helper class for running delegate performance benchmark. */
+class DelegatePerformanceBenchmark {
+  private static final String DELEGATE_PERFORMANCE_RESULT_FOLDER = "delegate_performance_result";
+  private static final String TAG = "TfLiteBenchmarkHelper";
+  private static final String MODEL_EXT = ".tflite";
+
+  static {
+    System.loadLibrary("delegate_performance_benchmark");
+  }
+
+  public static String createResultFolder(File filesDir, String resultFolder) throws IOException {
+    File resultDir = new File(filesDir, DELEGATE_PERFORMANCE_RESULT_FOLDER + "/" + resultFolder);
+    String resultPath = resultDir.getAbsolutePath();
+    if (resultDir.exists() || resultDir.mkdirs()) {
+      Log.i(TAG, "Logging the result to " + resultPath);
+      return resultPath;
+    }
+    throw new IOException("Failed to create directory for " + resultPath);
+  }
+
+  /**
+   * Extracts the model name from the model file name.
+   *
+   * <p>Strips out the ".tflite" extension from the input. Returns "model" if the input filename is
+   * "model.tflite".
+   */
+  public static String getModelName(String filename) {
+    checkNotNull(filename);
+    checkArgument(filename.endsWith(MODEL_EXT));
+    return filename.substring(0, filename.length() - MODEL_EXT.length());
+  }
+
+  /**
+   * Returns a {@code LatencyResults} by parsing the outcome from a TFLite Benchmark Tool execution.
+   * If it fails to parse the outcome, this method returns a {@code LatencyResults} with an error
+   * event type.
+   */
+  public static LatencyResults runLatencyBenchmark(
+      String[] args,
+      TfLiteSettingsListEntry tfliteSettingslistEntry,
+      int modelFd,
+      long modelOffset,
+      long modelSize) {
+    byte[] tfliteSettingsByteArray =
+        new byte[tfliteSettingslistEntry.tfliteSettings().getByteBuffer().remaining()];
+    tfliteSettingslistEntry.tfliteSettings().getByteBuffer().get(tfliteSettingsByteArray);
+    tfliteSettingslistEntry.tfliteSettings().getByteBuffer().rewind();
+
+    byte[] latencyResultsByteArray =
+        latencyBenchmarkNativeRun(
+            args,
+            tfliteSettingsByteArray,
+            tfliteSettingslistEntry.filePath(),
+            modelFd,
+            modelOffset,
+            modelSize);
+    if (latencyResultsByteArray == null || latencyResultsByteArray.length == 0) {
+      Log.w(
+          TAG,
+          String.format(
+              "Received null response from native for %s. Treating this as error.",
+              tfliteSettingslistEntry.filePath()));
+      return LatencyResults.newBuilder()
+          .setEventType(BenchmarkEventType.BENCHMARK_EVENT_TYPE_ERROR)
+          .build();
+    }
+    try {
+      return LatencyResults.parseFrom(latencyResultsByteArray);
+    } catch (IOException e) {
+      Log.w(
+          TAG,
+          String.format(
+              "Failed to parse the results running %s with exception %s.",
+              tfliteSettingslistEntry.filePath(), e));
+      return LatencyResults.newBuilder()
+          .setEventType(BenchmarkEventType.BENCHMARK_EVENT_TYPE_ERROR)
+          .build();
+    }
+  }
+
+  /** Returns a {@code BenchmarkEvent} by parsing the outcome from a MiniBenchmark execution. */
+  public static BenchmarkEvent runAccuracyBenchmark(
+      TfLiteSettingsListEntry tfliteSettingslistEntry,
+      int modelFd,
+      long modelOffset,
+      long modelSize,
+      String resultPath) {
+    byte[] tfliteSettingsByteArray =
+        new byte[tfliteSettingslistEntry.tfliteSettings().getByteBuffer().remaining()];
+    tfliteSettingslistEntry.tfliteSettings().getByteBuffer().get(tfliteSettingsByteArray);
+    tfliteSettingslistEntry.tfliteSettings().getByteBuffer().rewind();
+
+    byte[] accuracyResultsByteArray =
+        accuracyBenchmarkNativeRun(
+            tfliteSettingsByteArray, modelFd, modelOffset, modelSize, resultPath);
+    ByteBuffer byteBuffer = ByteBuffer.wrap(accuracyResultsByteArray);
+    return BenchmarkEvent.getRootAsBenchmarkEvent(byteBuffer);
+  }
+
+  /**
+   * Loads the input TFLiteSettings JSON files into TfLiteSettingsListEntry instances.
+   *
+   * <p>Adds default entry at the beginning as a reference delegate. The default entry contains a
+   * dummy TFLiteSettings structure, which lets the interpreter to apply the default acceleration.
+   * Positions the test target delegate at the end of the entry list.
+   */
+  public static List<TfLiteSettingsListEntry> loadTfLiteSettingsList(String[] jsonFilePaths) {
+    List<TfLiteSettingsListEntry> tfliteSettingsList = new ArrayList<>();
+    // Always add the default delegate to compare.
+    FlatBufferBuilder tfliteSettingsBuilder = new FlatBufferBuilder();
+      TFLiteSettings.startTFLiteSettings(tfliteSettingsBuilder);
+      int tfliteSettingsOffset = TFLiteSettings.endTFLiteSettings(tfliteSettingsBuilder);
+      tfliteSettingsBuilder.finish(tfliteSettingsOffset);
+    tfliteSettingsList.add(
+        TfLiteSettingsListEntry.create(
+            TFLiteSettings.getRootAsTFLiteSettings(tfliteSettingsBuilder.dataBuffer()),
+            "default_delegate",
+            /* isTestTarget= */ false));
+    // To avoid system-level initialization negatively impacting benchmark results, the test target
+    // delegate is positioned at the end of the list in reverse order. This ensures initialization
+    // latency is added to the reference delegate that is executed first among the delegates that
+    // share the same delegate type. Otherwise, it could introduce false positive results when
+    // comparing  delegates that share the same initialization. The order of the input settings
+    // files will be reinstated when computing the model-level reports.
+    for (int i = jsonFilePaths.length - 1; i >= 0; i--) {
+      String jsonFilePath = jsonFilePaths[i];
+      byte[] tfliteSettingsByteArray = loadTfLiteSettingsJsonNative(jsonFilePath);
+      if (tfliteSettingsByteArray == null || tfliteSettingsByteArray.length == 0) {
+        Log.e(TAG, "Failed to load TFLiteSetting from JSON file " + jsonFilePath);
+        return new ArrayList<>();
+      }
+
+      ByteBuffer byteBuffer = ByteBuffer.wrap(tfliteSettingsByteArray);
+      tfliteSettingsList.add(
+          TfLiteSettingsListEntry.create(
+              TFLiteSettings.getRootAsTFLiteSettings(byteBuffer),
+              jsonFilePath,
+              /* isTestTarget= */ i == 0));
+    }
+    return tfliteSettingsList;
+  }
+
+  /**
+   * Aggregates a list of {@link BenchmarkResultType} into a {@link BenchmarkResultType} based on
+   * the requirements. {@code strict} is set to {@code true} when comparing two delegates with the
+   * same types or aggregating the results into model-level and session-level. {@code strict} is set
+   * to {@code false} when comparing two delegates with different types.
+   *
+   * <p>If {@code strict} is set to {@code true}, returns:
+   *
+   * <ul>
+   *   <li>PASS if all elements in {@code results} are PASS.
+   *   <li>PASS_WITH_WARNING if at least one element in {@code results} is PASS_WITH_WARNING and
+   *       {@code results} doesn't contain FAIL.
+   *   <li>FAIL if at least one element in {@code results} is FAIL.
+   * </ul>
+   *
+   * Otherwise, returns:
+   *
+   * <ul>
+   *   <li>PASS if all elements in {@code results} are PASS.
+   *   <li>PASS_WITH_WARNING if at least one element in {@code results} is PASS_WITH_WARNING or
+   *       PASS.
+   *   <li>FAIL if all elements in {@code results} are FAIL.
+   * </ul>
+   */
+  public static BenchmarkResultType aggregateResults(
+      boolean strict, List<BenchmarkResultType> results) {
+    checkState(!results.isEmpty());
+    checkState(
+        allMatch(
+            results,
+            EnumSet.of(
+                BenchmarkResultType.PASS,
+                BenchmarkResultType.PASS_WITH_WARNING,
+                BenchmarkResultType.FAIL)));
+
+    if (strict) {
+      if (results.contains(BenchmarkResultType.FAIL)) {
+        return BenchmarkResultType.FAIL;
+      }
+      if (results.contains(BenchmarkResultType.PASS_WITH_WARNING)) {
+        return BenchmarkResultType.PASS_WITH_WARNING;
+      }
+      return BenchmarkResultType.PASS;
+    } else {
+      if (allMatch(results, EnumSet.of(BenchmarkResultType.PASS))) {
+        return BenchmarkResultType.PASS;
+      }
+      if (allMatch(results, EnumSet.of(BenchmarkResultType.FAIL))) {
+        return BenchmarkResultType.FAIL;
+      }
+      return BenchmarkResultType.PASS_WITH_WARNING;
+    }
+  }
+
+  /**
+   * Returns {@code true} when all elements in {@code results} are in the value range specified by
+   * {@code targets}.
+   */
+  private static boolean allMatch(
+      List<BenchmarkResultType> results, Set<BenchmarkResultType> targets) {
+    for (BenchmarkResultType result : results) {
+      if (!targets.contains(result)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  private static native byte[] latencyBenchmarkNativeRun(
+      String[] args,
+      byte[] tfliteSettings,
+      String tfliteSettingsPath,
+      int modelFd,
+      long modelOffset,
+      long modelSize);
+
+  private static native byte[] accuracyBenchmarkNativeRun(
+      byte[] tfliteSettings, int modelFd, long modelOffset, long modelSize, String resultPath);
+
+  private static native byte[] loadTfLiteSettingsJsonNative(String jsonFilePath);
+}

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/HtmlWriter.java

@@ -0,0 +1,205 @@
+/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.lite.benchmark.delegateperformance;
+
+import static org.tensorflow.lite.benchmark.delegateperformance.Preconditions.checkState;
+
+import android.util.Log;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.List;
+
+/** Helper class for writing the final report in HTML format. */
+final class HtmlWriter implements ReportWriter {
+  private static final String TAG = "TfLiteHtmlWriter";
+
+  private final String destinationFolderPath;
+
+  private HtmlWriter(String destinationFolderPath) {
+    this.destinationFolderPath = destinationFolderPath;
+  }
+
+  /** Writes the benchmark results into an HTML file. */
+  @Override
+  public void writeReport(BenchmarkReport report) {
+    // Example output file contains an overall summary table like:
+    //
+    // Summary PASS
+    //
+    // And a detailed methods table like:
+    //
+    // Model   Metric           DELEGATE_TYPE (PATH) DELEGATE_TYPE (PATH) Change Status ...
+    // model_1 metric_1         900                  1000                 -10%   PASS   ...
+    // model_1 metric_2         ...
+    // model_1 delegate_summary                                                  PASS   ...
+    // model_1 model_summary    PASS
+    // model_2 ...
+    // ...
+    StringBuilder sb = new StringBuilder();
+    sb.append(destinationFolderPath).append("/").append(report.name()).append(".html");
+    String filePath = sb.toString();
+    Log.i(TAG, "Generating an HTML report to " + filePath);
+
+    List<ModelBenchmarkReportInterface> modelReports = report.modelBenchmarkReports();
+    checkState(!modelReports.isEmpty());
+
+    // File header
+    sb =
+        new StringBuilder()
+            .append("<html>\n")
+            .append("<head>\n")
+            .append("<title>Delegate Performance Benchmark Report</title>\n")
+            .append("<style>\n")
+            .append("body {\n")
+            .append("  font-family: -apple-system, system-ui, BlinkMacSystemFont, \"Segoe UI\",\n")
+            .append("               \"Roboto\", \"Helvetica Neue\", Arial, sans-serif;\n")
+            .append("}\n")
+            .append("table, th, td {\n")
+            .append("  border: 1px solid black;\n")
+            .append("  border-collapse: collapse;\n")
+            .append("  padding: 15px;\n")
+            .append("  text-align: center;\n")
+            .append("}\n")
+            .append("th {\n")
+            .append("  background-color:LightSkyBlue\n")
+            .append("}\n")
+            .append("tr:nth-child(even) {\n")
+            .append("  background-color: whitesmoke;\n")
+            .append("}\n")
+            .append(".status-PASS {\n")
+            .append("  background-color: chartreuse;\n")
+            .append("}\n")
+            .append(".status-FAIL {\n")
+            .append("  background-color: red;\n")
+            .append("}\n")
+            .append(".status-PASS_WITH_WARNING {\n")
+            .append("  background-color: yellow;\n")
+            .append("}\n")
+            .append(".status-NOT_APPLICABLE {\n")
+            .append("  background-color: grey;\n")
+            .append("}\n")
+            .append(".ls-info-box {\n")
+            .append("  padding: 5px 10px;\n")
+            .append("  margin: 10px;\n")
+            .append("  border-style: solid;\n")
+            .append("  border-left-width: 5px;\n")
+            .append("  border-radius: 4px;\n")
+            .append("  color: #666;\n")
+            .append("  font-size: 13px;\n")
+            .append("  line-height: 1.3\n")
+            .append("}\n")
+            .append(".ls-info-box {\n")
+            .append("  background-color: #f0f0f0;\n")
+            .append("  border-left-color: silver\n")
+            .append("}\n")
+            .append("</style>\n")
+            .append("</head>\n")
+            .append("<body>\n")
+            // TODO(b/268338967): Use a more informative name for the report.
+            .append("<h1>Delegate Performance Benchmark Report</h1>\n");
+    // Summary table
+    sb.append("<table>\n").append("<tr>\n").append("<td>Summary</td>\n");
+    addResultCell(report.result(), /* isStrictCriteria= */ null, sb);
+    sb.append("</tr>\n").append("</table>\n");
+
+    // Heading row for the detailed metric table. It is structured as below:
+    // Model, Metric, DELEGATE_TYPE (PATH), DELEGATE_TYPE (PATH), Change, Status, ...
+    sb.append("<table>\n")
+        .append("<thead>\n")
+        .append("<tr>\n")
+        .append("<th>Model</th>\n")
+        .append("<th>Metric</th>\n");
+    for (DelegateMetricsEntry entry : modelReports.get(0).processedDelegateMetrics()) {
+      sb.append("<th>Delegate: ").append(entry.delegateIdentifier()).append("</th>\n");
+      if (!entry.isTestTarget()) {
+        sb.append("<th>Change (%)</th>\n").append("<th>Status</th>\n");
+      }
+    }
+    sb.append("</thead>\n").append("<tbody>\n");
+    for (ModelBenchmarkReportInterface modelReport : modelReports) {
+      writerModelReport(modelReport, sb);
+    }
+    sb.append("</tbody>\n")
+        .append("</table>\n")
+        .append("<div class=\"ls-info-box\">\n")
+        .append("<p>\n")
+        .append(
+            "When the test target delegate type is the same as the reference delegate, the checks"
+                + " are more strict. Otherwise, the checks are relaxed. Please see \n")
+        .append(
+            "<a href=\"https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/BenchmarkResultType.java\">BenchmarkResultType.java</a>\n")
+        .append(" for the meanings of PASS, PASS_WITH_WARNING and FAIL.\n")
+        .append("</p>\n")
+        .append("</div>\n")
+        .append("</body>\n")
+        .append("</html>\n");
+    try (PrintWriter writer = new PrintWriter(filePath)) {
+      writer.write(sb.toString());
+    } catch (IOException e) {
+      Log.e(TAG, "Failed to open report file " + filePath);
+    }
+  }
+
+  private void writerModelReport(ModelBenchmarkReportInterface modelReport, StringBuilder sb) {
+    String modelName = modelReport.modelName();
+    if (modelReport.processedDelegateMetrics().isEmpty()) {
+      Log.w(TAG, "The computed metric is empty.");
+    } else {
+      for (String metricName : modelReport.processedDelegateMetrics().get(0).metrics().keySet()) {
+        sb.append("<tr>\n<td>")
+            .append(modelName)
+            .append("</td>\n<td>")
+            .append(metricName)
+            .append("</td>\n");
+        for (DelegateMetricsEntry delegateMetricsEntry : modelReport.processedDelegateMetrics()) {
+          MetricsEntry metricEntry = delegateMetricsEntry.metrics().get(metricName);
+          sb.append("<td>").append(metricEntry.value()).append("</td>\n");
+          if (!delegateMetricsEntry.isTestTarget()) {
+            sb.append("<td>").append(metricEntry.regression()).append("</td>\n");
+            addResultCell(metricEntry.result(), /* isStrictCriteria= */ null, sb);
+          }
+        }
+        sb.append("</tr>\n");
+      }
+      sb.append("<tr>\n<td>").append(modelName).append("</td>\n<td>delegate_summary</td>\n");
+      for (DelegateMetricsEntry delegateMetricsEntry : modelReport.processedDelegateMetrics()) {
+        // Position the delegate-level result correctly.
+        sb.append("<td/>\n");
+        if (!delegateMetricsEntry.isTestTarget()) {
+          sb.append("<td/>\n");
+          addResultCell(delegateMetricsEntry.result(), delegateMetricsEntry.isStrictCriteria(), sb);
+        }
+      }
+      sb.append("</tr>\n<tr>\n<td>").append(modelName).append("</td>\n<td>model_summary</td>\n");
+      addResultCell(modelReport.result(), /* isStrictCriteria= */ null, sb);
+      sb.append("</tr>\n");
+    }
+  }
+
+  /** Adds a colored result cell to the table. */
+  private void addResultCell(
+      BenchmarkResultType result, Boolean isStrictCriteria, StringBuilder sb) {
+    sb.append("<td class=\"status-").append(result.name()).append("\">").append(result);
+    if (isStrictCriteria != null && isStrictCriteria) {
+      sb.append(" (strict)");
+    }
+    sb.append("</td>\n");
+  }
+
+  static ReportWriter create(String destinationFolderPath) {
+    return new HtmlWriter(destinationFolderPath);
+  }
+}

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/JsonWriter.java

@@ -0,0 +1,67 @@
+/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.lite.benchmark.delegateperformance;
+
+import android.util.Log;
+import java.io.IOException;
+import java.io.PrintWriter;
+import org.json.JSONException;
+
+/** Helper class for writing the final report in JSON format. */
+final class JsonWriter implements ReportWriter {
+  private static final String TAG = "TfLiteJsonWriter";
+
+  private final String destinationFolderPath;
+
+  private JsonWriter(String destinationFolderPath) {
+    this.destinationFolderPath = destinationFolderPath;
+  }
+
+  // Writes the benchmark results into a JSON file.
+  // Example output file:
+  // {
+  //    "reports": [
+  //       {
+  //          "model": "model_name",
+  //          "metrics": [...],
+  //          "raw_metrics": [... ],
+  //          "max_regression_percentage_allowed": { ... },
+  //          "result": "PASS"
+  //       }
+  //       , ...
+  //    ],
+  //    "name": "report",
+  //    "result": "PASS"
+  // }
+  @Override
+  public void writeReport(BenchmarkReport report) {
+    StringBuilder sb = new StringBuilder();
+    sb.append(destinationFolderPath).append("/").append(report.name()).append(".json");
+    String filePath = sb.toString();
+    try (PrintWriter writer = new PrintWriter(filePath, "UTF-8")) {
+      writer.println(report.toJsonObject().toString(/* indentFactor= */ 4));
+      Log.i(TAG, "Generated report " + filePath + ".");
+    } catch (IOException e) {
+      Log.e(TAG, "Failed to open report file " + filePath + "." + e);
+    } catch (JSONException e) {
+      Log.e(TAG, "Failed to convert the benchmark report to JSON." + e);
+    }
+  }
+
+  static ReportWriter create(String destinationFolderPath) {
+    return new JsonWriter(destinationFolderPath);
+  }
+}

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/LatencyBenchmarkReport.java

@@ -0,0 +1,93 @@
+/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+package org.tensorflow.lite.benchmark.delegateperformance;
+
+import static java.lang.Math.max;
+import static org.tensorflow.lite.benchmark.delegateperformance.Preconditions.checkNotNull;
+import static org.tensorflow.lite.benchmark.delegateperformance.Preconditions.checkState;
+
+import android.util.Log;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import tflite.proto.benchmark.DelegatePerformance.BenchmarkEventType;
+import tflite.proto.benchmark.DelegatePerformance.BenchmarkMetric;
+import tflite.proto.benchmark.DelegatePerformance.LatencyCriteria;
+import tflite.proto.benchmark.DelegatePerformance.LatencyResults;
+
+/** Model-level latency benchmark report class. */
+final class LatencyBenchmarkReport extends ModelBenchmarkReport {
+  private static final String TAG = "LatencyBenchmarkReport";
+
+  private LatencyBenchmarkReport(
+      String modelName,
+      List<RawDelegateMetricsEntry> rawDelegateMetricsEntries,
+      LatencyCriteria criteria) {
+    super(modelName);
+    checkNotNull(criteria);
+    this.maxRegressionPercentageAllowed.put(
+        "startup_overhead_latency_us",
+        (double) criteria.getStartupOverheadMaxRegressionPercentageAllowed());
+    this.maxRegressionPercentageAllowed.put(
+        "inference_latency_average_us",
+        (double) criteria.getAverageInferenceMaxRegressionPercentageAllowed());
+    Log.i(TAG, "Creating a latency benchmark report for " + modelName);
+    computeModelReport(rawDelegateMetricsEntries);
+  }
+
+  /**
+   * Parses {@link LatencyResults} into the unified {@link RawDelegateMetricsEntry} format for
+   * further processing.
+   */
+  public static RawDelegateMetricsEntry parseResults(
+      LatencyResults results, TfLiteSettingsListEntry entry) {
+    // Use {@code LinkedHashMap} to keep the metrics insertion order.
+    Map<String, Double> metrics = new LinkedHashMap<>();
+    if (results.getEventType() != BenchmarkEventType.BENCHMARK_EVENT_TYPE_END) {
+      Log.w(TAG, "The latency benchmarking is not completed successfully for " + entry.filePath());
+      return RawDelegateMetricsEntry.create(
+          entry.tfliteSettings().delegate(), entry.filePath(), entry.isTestTarget(), metrics);
+    }
+    for (BenchmarkMetric metric : results.getMetricsList()) {
+      if (metric != null) {
+        metrics.put(metric.getName(), (double) metric.getValue());
+      }
+    }
+    checkState(metrics.containsKey("initialization_latency_us"));
+    checkState(metrics.containsKey("warmup_latency_average_us"));
+    checkState(metrics.containsKey("inference_latency_average_us"));
+    metrics.put(
+        "startup_overhead_latency_us",
+        max(
+            metrics.get("initialization_latency_us")
+                + metrics.get("warmup_latency_average_us")
+                - metrics.get("inference_latency_average_us"),
+            // The average warmup latency is generally assumed to be greater than the average
+            // inference latency. Therefore, it is highly unusual for the sum of the initialization
+            // latency and the average warmup latency to be less than the average inference latency.
+            // In order to ensure that the regression is computed successfully, we keep the startup
+            // overhead latency at minimum 0.
+            0d));
+    return RawDelegateMetricsEntry.create(
+        entry.tfliteSettings().delegate(), entry.filePath(), entry.isTestTarget(), metrics);
+  }
+
+  static LatencyBenchmarkReport create(
+      String modelName,
+      List<RawDelegateMetricsEntry> rawDelegateMetricsEntries,
+      LatencyCriteria criteria) {
+    return new LatencyBenchmarkReport(modelName, rawDelegateMetricsEntries, criteria);
+  }
+}

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/MetricsEntry.java

@@ -0,0 +1,72 @@
+/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+package org.tensorflow.lite.benchmark.delegateperformance;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+
+/**
+ * Helper class to store the metric value, the test target delegate regression compared with the
+ * reference delegate and the metric-level results based on the criteria.
+ *
+ * <p>An instance of {@link MetricsEntry} corresponds with each tested model metric and each pair of
+ * test target delegate and reference delegate.
+ *
+ * <p>TODO(b/267429312): use AutoValue here to simplify the source code.
+ */
+final class MetricsEntry {
+  private final double value;
+  // The test target delegate's performance regression compared against the reference delegate.
+  // Example value: "5%".
+  private final String regression;
+  // The metric pass status after checking the regression with the criteria.
+  // Possible values:
+  // - NOT_APPLICABLE: This performance metric is not involved in the criteria.
+  // - PASS: The performance metric of the test target delegate are better or equal to the reference
+  // delegate.
+  // - PASS_WITH_WARNING: The regression doesn't breach the threshold specified in the criteria.
+  // - FAIL: The regression breaches the threshold specified in the criteria.
+  private final BenchmarkResultType result;
+
+  private MetricsEntry(double value, String regression, BenchmarkResultType result) {
+    this.value = value;
+    this.regression = regression;
+    this.result = result;
+  }
+
+  double value() {
+    return value;
+  }
+
+  String regression() {
+    return regression;
+  }
+
+  BenchmarkResultType result() {
+    return result;
+  }
+
+  JSONObject toJsonObject() throws JSONException {
+    JSONObject jsonObject = new JSONObject();
+    jsonObject.put("value", value);
+    jsonObject.put("regression", regression);
+    jsonObject.put("result", result.toString());
+    return jsonObject;
+  }
+
+  static MetricsEntry create(double value, String regression, BenchmarkResultType result) {
+    return new MetricsEntry(value, regression, result);
+  }
+}

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/ModelBenchmarkReport.java

@@ -0,0 +1,203 @@
+/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+package org.tensorflow.lite.benchmark.delegateperformance;
+
+import static org.tensorflow.lite.benchmark.delegateperformance.Preconditions.checkState;
+
+import android.util.Log;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+/**
+ * Model-level benchmark report class to be extended by {@link AccuracyBenchmarkReport} and {@link
+ * LatencyBenchmarkReport}.
+ *
+ * <p>This class contains helper functions to convert the raw performance results from the native
+ * layer into metric-level, delegate-level and model-level pass status values.
+ */
+public abstract class ModelBenchmarkReport implements ModelBenchmarkReportInterface {
+  private static final String TAG = "ModelBenchmarkReport";
+
+  protected final String modelName;
+  /* Map from performance metric names to the maximum regression percentage thresholds allowed. */
+  protected final Map<String, Double> maxRegressionPercentageAllowed = new HashMap<>();
+  /** List of {@link DelegateMetricsEntry}, which stores delegate-level performance results. */
+  protected final List<DelegateMetricsEntry> processedDelegateMetrics = new ArrayList<>();
+  /** Model-level pass status. */
+  protected BenchmarkResultType result = BenchmarkResultType.UNKNOWN;
+
+  protected ModelBenchmarkReport(String modelName) {
+    this.modelName = modelName;
+  }
+
+  @Override
+  public String modelName() {
+    return modelName;
+  }
+
+  @Override
+  public List<DelegateMetricsEntry> processedDelegateMetrics() {
+    return Collections.unmodifiableList(processedDelegateMetrics);
+  }
+
+  @Override
+  public BenchmarkResultType result() {
+    return result;
+  }
+
+  @Override
+  public JSONObject toJsonObject() throws JSONException {
+    JSONObject jsonObject = new JSONObject();
+    jsonObject.put("model", modelName);
+    jsonObject.put("result", result.toString());
+    JSONArray processedDelegateMetricsArray = new JSONArray();
+    for (DelegateMetricsEntry entry : processedDelegateMetrics) {
+      processedDelegateMetricsArray.put(entry.toJsonObject());
+    }
+    jsonObject.put("metrics", processedDelegateMetricsArray);
+    JSONObject maxRegressionPercentageAllowedObject = new JSONObject();
+    for (Map.Entry<String, Double> entry : maxRegressionPercentageAllowed.entrySet()) {
+      maxRegressionPercentageAllowedObject.put(entry.getKey(), entry.getValue());
+    }
+    jsonObject.put("max_regression_percentage_allowed", maxRegressionPercentageAllowedObject);
+    return jsonObject;
+  }
+
+  /**
+   * Converts the prepopulated list of {@link RawDelegateMetricsEntry}, the raw performance results
+   * collected from the native layer, into a list of {@link DelegateMetricsEntry}.
+   *
+   * <p>The computation compares the test target delegate with every reference delegate by computing
+   * the regression of the raw performance results. The list of {@link DelegateMetricsEntry} stores
+   * the calculated results for all pairs. {@link ModelBenchmarkReport} stores both the raw
+   * performance results and the processed performance results.
+   */
+  protected void computeModelReport(List<RawDelegateMetricsEntry> rawDelegateMetrics) {
+    if (!processedDelegateMetrics.isEmpty()) {
+      // Metrics are computed.
+      Log.i(TAG, "Delegate metrics are already computed. Skips the computation.");
+      return;
+    }
+    // At least 2 delegates (the default delegate and the test target delegate) are tested.
+    checkState(rawDelegateMetrics.size() >= 2);
+    // The test target delegate is the last delegate to benchmark. The order will be guaranteed by
+    // DelegatePerformanceBenchmark#loadTfLiteSettingsList().
+    RawDelegateMetricsEntry testTarget = rawDelegateMetrics.get(rawDelegateMetrics.size() - 1);
+    checkState(testTarget.isTestTarget());
+    // Use {@code LinkedHashMap} to keep the insertion order.
+    Map<String, MetricsEntry> metrics = new LinkedHashMap<>();
+    for (String metricName : testTarget.metrics().keySet()) {
+      metrics.put(
+          metricName,
+          MetricsEntry.create(
+              testTarget.metrics().get(metricName), "N/A", BenchmarkResultType.NOT_APPLICABLE));
+    }
+    processedDelegateMetrics.add(
+        DelegateMetricsEntry.create(
+            testTarget.delegateIdentifier(),
+            metrics,
+            BenchmarkResultType.NOT_APPLICABLE,
+            testTarget.isTestTarget(),
+            /* isStrictCriteria= */ false));
+
+    // Processes the reference delegate results. Compute the performance regressions by comparing
+    // them with the results from the test target delegate.
+    List<BenchmarkResultType> referenceResults = new ArrayList<>();
+    // Traverses the list in reverse order, so that the exported order of items matches with the
+    // input delegate setting files order.
+    for (int reference = rawDelegateMetrics.size() - 2; reference >= 0; reference--) {
+      RawDelegateMetricsEntry entry = rawDelegateMetrics.get(reference);
+      Map<String, MetricsEntry> referenceMetrics = new LinkedHashMap<>();
+      List<BenchmarkResultType> metricResults = new ArrayList<>();
+      for (String metricName : testTarget.metrics().keySet()) {
+        MetricsEntry metricEntry =
+            computeReferenceMetricEntry(
+                entry.metrics().get(metricName), testTarget.metrics().get(metricName), metricName);
+        referenceMetrics.put(metricName, metricEntry);
+        // Filters for the metrics that are involved in the Pass/Pass with Warning/Fail result
+        // generation.
+        if (metricEntry.result() != BenchmarkResultType.NOT_APPLICABLE) {
+          metricResults.add(metricEntry.result());
+        }
+      }
+      // TODO(b/267488243): Load the delegate name from the native layer.
+      boolean sameDelegateType = entry.delegateName().equals(testTarget.delegateName());
+      BenchmarkResultType referenceDelegateResult =
+          DelegatePerformanceBenchmark.aggregateResults(sameDelegateType, metricResults);
+      referenceResults.add(referenceDelegateResult);
+      processedDelegateMetrics.add(
+          DelegateMetricsEntry.create(
+              entry.delegateIdentifier(),
+              referenceMetrics,
+              referenceDelegateResult,
+              entry.isTestTarget(),
+              sameDelegateType));
+    }
+    result = DelegatePerformanceBenchmark.aggregateResults(/* strict= */ true, referenceResults);
+  }
+
+  private MetricsEntry computeReferenceMetricEntry(
+      Double referenceValue, Double testTargetValue, String metricName) {
+    boolean checkRegression = maxRegressionPercentageAllowed.containsKey(metricName);
+    String regression = "N/A";
+    BenchmarkResultType result =
+        checkRegression ? BenchmarkResultType.FAIL : BenchmarkResultType.NOT_APPLICABLE;
+    if (referenceValue == null || testTargetValue == null) {
+      return MetricsEntry.create(referenceValue, regression, result);
+    }
+    // Here is a mitigation to the lack of support for criteria operators. "ok" metric is handled
+    // specifically for the accuracy benchmarking results.
+    // TODO(b/267313326): remove the mitigation after the criteria operators are added.
+    if (metricName.equals("ok")) {
+      if (testTargetValue == AccuracyBenchmarkReport.PASS) {
+        // The test target delegate passed the accuracy checks.
+        result = BenchmarkResultType.PASS;
+      } else if (referenceValue == AccuracyBenchmarkReport.FAIL) {
+        // Both the test target and the reference delegates failed the accuracy checks. Therefore,
+        // it is not considered as a regression.
+        result = BenchmarkResultType.PASS_WITH_WARNING;
+      }
+      return MetricsEntry.create(referenceValue, regression, result);
+    }
+
+    // Here we assume that lower values of the metric are better, for all of our metrics.
+    // TODO(b/267313326): Remove the assumption with the criteria operator support.
+    double regressionValue = 0.0;
+    if (!testTargetValue.equals(referenceValue)) {
+      regressionValue = (testTargetValue - referenceValue) / referenceValue;
+    }
+    if (checkRegression) {
+      if (regressionValue <= 0) {
+        result = BenchmarkResultType.PASS;
+      } else if (regressionValue <= maxRegressionPercentageAllowed.get(metricName) / 100f) {
+        result = BenchmarkResultType.PASS_WITH_WARNING;
+      }
+    }
+    regression = toPercentage(regressionValue);
+    return MetricsEntry.create(referenceValue, regression, result);
+  }
+
+  private String toPercentage(double n) {
+    return String.format(Locale.ENGLISH, "%.1f", n * 100) + "%";
+  }
+}

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/ModelBenchmarkReportInterface.java

@@ -0,0 +1,48 @@
+/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+package org.tensorflow.lite.benchmark.delegateperformance;
+
+import java.util.List;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+/**
+ * An interface to handle metric computations and exporting results for a benchmark run on a single
+ * model. It is used in {@link ModelBenchmarkReport}.
+ */
+public interface ModelBenchmarkReportInterface {
+  /** Returns the model name. */
+  String modelName();
+
+  /** Returns the list of processed performance results for all delegates. */
+  List<DelegateMetricsEntry> processedDelegateMetrics();
+
+  /**
+   * Returns the model-level pass status.
+   *
+   * <p>Possible return values:
+   *
+   * <ul>
+   *   <li>1. PASS: All delegate-level results are "PASS" or "NOT_APPLICABLE".
+   *   <li>2. PASS_WITH_WARNING: At least 1 "PASS_WITH_WARNING" delegate-level result and 0 "FAIL"
+   *       delegate-level result.
+   *   <li>3. FAIL: At least 1 "FAIL" delegate-level result.
+   * </ul>
+   */
+  BenchmarkResultType result();
+
+  /** Serializes the report and returns a {@code JSONObject}. */
+  JSONObject toJsonObject() throws JSONException;
+}

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/Preconditions.java

@@ -0,0 +1,62 @@
+/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+package org.tensorflow.lite.benchmark.delegateperformance;
+
+/** Helper class for checking preconditions. */
+final class Preconditions {
+
+  /**
+   * Ensures that an object reference passed as a parameter to the calling method is not null.
+   *
+   * <p>TODO(b/250876587): Consider adding proper annotation support.
+   *
+   * @param reference an object reference
+   * @return the non-null reference that was validated
+   * @throws NullPointerException if {@code reference} is null
+   */
+  public static <T> T checkNotNull(/* @Nullable */ T reference) {
+    if (reference == null) {
+      throw new NullPointerException();
+    }
+    return reference;
+  }
+
+  /**
+   * Ensures the truth of an expression involving one or more parameters to the calling method.
+   *
+   * @param expression a boolean expression
+   * @throws IllegalArgumentException if {@code expression} is false
+   */
+  public static void checkArgument(boolean expression) {
+    if (!expression) {
+      throw new IllegalArgumentException();
+    }
+  }
+
+  /**
+   * Ensures the truth of an expression involving the state of the calling instance, but not
+   * involving any parameters to the calling method.
+   *
+   * @param expression a boolean expression
+   * @throws IllegalStateException if {@code expression} is false
+   */
+  public static void checkState(boolean expression) {
+    if (!expression) {
+      throw new IllegalStateException();
+    }
+  }
+
+  private Preconditions() {}
+}

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/RawDelegateMetricsEntry.java

@@ -0,0 +1,78 @@
+/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+package org.tensorflow.lite.benchmark.delegateperformance;
+
+import java.util.Collections;
+import java.util.Map;
+import tflite.Delegate;
+
+/**
+ * Helper class to store the raw performance results from the native layer.
+ *
+ * <p>An instance of {@link RawDelegateMetricsEntry} corresponds with each tested model and each
+ * delegate.
+ */
+final class RawDelegateMetricsEntry {
+  // The name of the delegate. The available names are listed in
+  // tensorflow/tensorflow/lite/acceleration/configuration/configuration.proto
+  // TODO(b/267431570): consider replacing the field with an Enum value.
+  private final String delegateName;
+  // Specifies the path to the delegate settings file.
+  private final String path;
+  // If {@link isTestTarget} is set to {@code true}, the metrics are gathered from the benchmark run
+  // with the test target delegate.
+  private final boolean isTestTarget;
+  private final Map<String, Double> metrics;
+
+  private RawDelegateMetricsEntry(
+      int delegate, String path, boolean isTestTarget, Map<String, Double> metrics) {
+    this.delegateName = Delegate.name(delegate);
+    this.path = path;
+    this.isTestTarget = isTestTarget;
+    this.metrics = metrics;
+  }
+
+  /* Returns the name of the delegate. */
+  String delegateName() {
+    return delegateName;
+  }
+
+  /* Returns the path to the delegate settings file. */
+  String path() {
+    return path;
+  }
+
+  /* Returns whether the delegate is the test target. */
+  boolean isTestTarget() {
+    return isTestTarget;
+  }
+
+  Map<String, Double> metrics() {
+    return Collections.unmodifiableMap(metrics);
+  }
+
+  /**
+   * Returns an identifier for the delegate. The idenfier consists of the delegate type and the path
+   * to the delegate settings file.
+   */
+  String delegateIdentifier() {
+    return delegateName + " (" + path + ")";
+  }
+
+  static RawDelegateMetricsEntry create(
+      int delegate, String path, boolean isTestTarget, Map<String, Double> metrics) {
+    return new RawDelegateMetricsEntry(delegate, path, isTestTarget, metrics);
+  }
+}

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/ReportWriter.java

@@ -0,0 +1,24 @@
+/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+package org.tensorflow.lite.benchmark.delegateperformance;
+
+/**
+ * An interface to handle exporting {@link BenchmarkReport}. It will be used in {@link CsvWriter},
+ * {@link JsonWriter} and {@link HtmlWriter}.
+ */
+public interface ReportWriter {
+  /** Exports {@link BenchmarkReport} to the target format. */
+  void writeReport(BenchmarkReport report);
+}

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/java/org/tensorflow/lite/benchmark/delegateperformance/TfLiteSettingsListEntry.java

@@ -0,0 +1,70 @@
+/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.lite.benchmark.delegateperformance;
+
+import static org.tensorflow.lite.benchmark.delegateperformance.Preconditions.checkNotNull;
+
+import tflite.TFLiteSettings;
+
+/** Helper class for store the data before and after benchmark runs. */
+final class TfLiteSettingsListEntry {
+  private static final String TAG = "TfLiteSettingsListEntry";
+
+  private final TFLiteSettings tfliteSettings;
+  private final String filePath;
+  private final boolean isTestTarget;
+
+  private TfLiteSettingsListEntry(
+      TFLiteSettings tfliteSettings, String filePath, boolean isTestTarget) {
+    checkNotNull(tfliteSettings);
+    checkNotNull(filePath);
+    this.tfliteSettings = tfliteSettings;
+    this.filePath = filePath;
+    this.isTestTarget = isTestTarget;
+  }
+
+  TFLiteSettings tfliteSettings() {
+    return tfliteSettings;
+  }
+
+  String filePath() {
+    return filePath;
+  }
+
+  boolean isTestTarget() {
+    return isTestTarget;
+  }
+
+  @Override
+  public String toString() {
+    return "TfLiteSettingsListEntry{"
+        // TODO(b/265268620): Dump the entire TFLiteSettings buffer.
+        + "delegate="
+        + tfliteSettings.delegate()
+        + ", "
+        + "filePath="
+        + filePath
+        + ", "
+        + "isTestTarget="
+        + isTestTarget
+        + "}";
+  }
+
+  static TfLiteSettingsListEntry create(
+      TFLiteSettings tfliteSettings, String filePath, boolean isTestTarget) {
+    return new TfLiteSettingsListEntry(tfliteSettings, filePath, isTestTarget);
+  }
+}

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/BUILD

@@ -0,0 +1,78 @@
+# Description:
+#  Holds the native layer of the app.
+
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
+load("//tensorflow/lite/core/shims:cc_library_with_tflite.bzl", "cc_library_with_tflite", "jni_binary_with_tflite")
+load("//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android:build_defs.bzl", "accuracy_benchmark_extra_deps", "latency_benchmark_extra_deps")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_visibility = ["//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android:__subpackages__"],
+    licenses = ["notice"],
+)
+
+jni_binary_with_tflite(
+    name = "libdelegate_performance_benchmark.so",
+    srcs = ["delegate_performance_benchmark_jni.cc"],
+    tflite_deps = [":accuracy_benchmark"],
+    deps = [
+        ":latency_benchmark",
+        "//tensorflow/lite/acceleration/configuration:configuration_fbs",
+        "//tensorflow/lite/delegates/utils/experimental/stable_delegate:tflite_settings_json_parser",
+        "//tensorflow/lite/java/jni",
+        "//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto:delegate_performance_cc_proto",
+        "@flatbuffers",
+    ],
+)
+
+cc_library(
+    name = "status_codes",
+    hdrs = ["status_codes.h"],
+)
+
+cc_library(
+    name = "latency_benchmark",
+    srcs = ["latency_benchmark.cc"],
+    hdrs = ["latency_benchmark.h"],
+    deps = [
+        "//tensorflow/core/util:stats_calculator_portable",
+        "//tensorflow/lite:minimal_logging",
+        "//tensorflow/lite/acceleration/configuration:configuration_fbs",
+        "//tensorflow/lite/c:c_api_types",
+        "//tensorflow/lite/profiling:memory_info",
+        "//tensorflow/lite/tools/benchmark:benchmark_model_lib",
+        "//tensorflow/lite/tools/benchmark:benchmark_params",
+        "//tensorflow/lite/tools/benchmark:benchmark_tflite_model_lib",
+        "//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto:delegate_performance_cc_proto",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/strings:str_format",
+    ] + latency_benchmark_extra_deps(),
+)
+
+cc_library_with_tflite(
+    name = "accuracy_benchmark",
+    srcs = ["accuracy_benchmark.cc"],
+    hdrs = ["accuracy_benchmark.h"],
+    deps = [
+        ":status_codes",
+        "//tensorflow/lite:minimal_logging",
+        "//tensorflow/lite/acceleration/configuration:configuration_fbs",
+        "//tensorflow/lite/acceleration/configuration:gpu_plugin",
+        "//tensorflow/lite/acceleration/configuration:stable_delegate_plugin",
+        "//tensorflow/lite/acceleration/configuration:xnnpack_plugin",
+        "//tensorflow/lite/core/acceleration/configuration:nnapi_plugin",
+        "//tensorflow/lite/delegates/utils/experimental/stable_delegate:tflite_settings_json_parser",
+        "//tensorflow/lite/experimental/acceleration/mini_benchmark:libjpeg_handle_static_link",
+        "//tensorflow/lite/experimental/acceleration/mini_benchmark:status_codes",
+        "//tensorflow/lite/experimental/acceleration/mini_benchmark/c:c_api_in_process",
+        "//tensorflow/lite/kernels/internal:compatibility",
+        "//tensorflow/lite/tools:command_line_flags",
+        "//tensorflow/lite/tools:tool_params",
+        "@flatbuffers",
+    ] + accuracy_benchmark_extra_deps(),
+)
+
+cc_library_with_tflite(
+    name = "benchmark_native",
+    tflite_jni_binaries = [":libdelegate_performance_benchmark.so"],
+)

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/accuracy_benchmark.cc

@@ -0,0 +1,137 @@
+/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/accuracy_benchmark.h"
+
+#include <errno.h>
+#include <stdio.h>
+#include <sys/stat.h>
+
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <string>
+#include <vector>
+
+#include "flatbuffers/base.h"  // from @flatbuffers
+#include "flatbuffers/buffer.h"  // from @flatbuffers
+#include "flatbuffers/flatbuffer_builder.h"  // from @flatbuffers
+#include "flatbuffers/flatbuffers.h"  // from @flatbuffers
+#include "flatbuffers/vector.h"  // from @flatbuffers
+#include "tensorflow/lite/acceleration/configuration/configuration_generated.h"
+#include "tensorflow/lite/experimental/acceleration/mini_benchmark/c/c_api.h"
+#include "tensorflow/lite/kernels/internal/compatibility.h"
+#include "tensorflow/lite/logger.h"
+#include "tensorflow/lite/minimal_logging.h"
+#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/status_codes.h"
+
+namespace tflite {
+namespace benchmark {
+namespace accuracy {
+
+namespace {
+std::vector<const tflite::BenchmarkEvent*> ToBenchmarkEvents(uint8_t* data,
+                                                             size_t size) {
+  std::vector<const tflite::BenchmarkEvent*> results;
+  uint8_t* current_root = data;
+  while (current_root < data + size) {
+    flatbuffers::uoffset_t current_size =
+        flatbuffers::GetPrefixedSize(current_root);
+    results.push_back(
+        flatbuffers::GetSizePrefixedRoot<tflite::BenchmarkEvent>(current_root));
+    current_root += current_size + sizeof(flatbuffers::uoffset_t);
+  }
+  // Checks the data read is not over the bounds of the buffer.
+  TFLITE_CHECK_EQ(current_root, data + size);
+  return results;
+}
+}  // namespace
+
+flatbuffers::Offset<BenchmarkEvent> Benchmark(
+    flatbuffers::FlatBufferBuilder& fbb, const TFLiteSettings& tflite_settings,
+    int model_fd, size_t model_offset, size_t model_size,
+    const char* result_path_chars) {
+  std::string result_path(result_path_chars);
+  std::string storage_path = result_path + "/storage_path.fb";
+  int return_code = std::remove(storage_path.c_str());
+  if (return_code) {
+    TFLITE_LOG_PROD(TFLITE_LOG_WARNING,
+                    "Failed to remove storage file (%s): %s.",
+                    storage_path.c_str(), strerror(errno));
+  }
+
+  flatbuffers::FlatBufferBuilder mini_benchmark_fbb;
+  TFLiteSettingsT tflite_settings_t;
+  tflite_settings.UnPackTo(&tflite_settings_t);
+  flatbuffers::Offset<TFLiteSettings> tflite_settings_offset =
+      CreateTFLiteSettings(mini_benchmark_fbb, &tflite_settings_t);
+  flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<TFLiteSettings>>>
+      tflite_settings_vector_offset =
+          mini_benchmark_fbb.CreateVector({tflite_settings_offset});
+  ModelFileBuilder model_file_builder(mini_benchmark_fbb);
+  model_file_builder.add_fd(model_fd);
+  model_file_builder.add_offset(model_offset);
+  model_file_builder.add_length(model_size);
+  flatbuffers::Offset<ModelFile> model_file_offset =
+      model_file_builder.Finish();
+  flatbuffers::Offset<BenchmarkStoragePaths> storage_paths_offset =
+      CreateBenchmarkStoragePaths(mini_benchmark_fbb,
+                                  mini_benchmark_fbb.CreateString(storage_path),
+                                  mini_benchmark_fbb.CreateString(result_path));
+  flatbuffers::Offset<ValidationSettings> validation_settings_offset =
+      CreateValidationSettings(mini_benchmark_fbb,
+                               /*per_test_timeout_ms=*/5000);
+  mini_benchmark_fbb.Finish(CreateMinibenchmarkSettings(
+      mini_benchmark_fbb, tflite_settings_vector_offset, model_file_offset,
+      storage_paths_offset, validation_settings_offset));
+
+  TfLiteMiniBenchmarkSettings* settings = TfLiteMiniBenchmarkSettingsCreate();
+  TfLiteMiniBenchmarkSettingsSetFlatBufferData(
+      settings, mini_benchmark_fbb.GetBufferPointer(),
+      mini_benchmark_fbb.GetSize());
+
+  TfLiteMiniBenchmarkResult* result =
+      TfLiteBlockingValidatorRunnerTriggerValidation(settings);
+  std::vector<const BenchmarkEvent*> events =
+      ToBenchmarkEvents(TfLiteMiniBenchmarkResultFlatBufferData(result),
+                        TfLiteMiniBenchmarkResultFlatBufferDataSize(result));
+  TfLiteMiniBenchmarkSettingsFree(settings);
+
+  // The settings contains one test only. Therefore, the benchmark checks for
+  // the first result only.
+  if (events.size() != 1) {
+    TfLiteMiniBenchmarkResultFree(result);
+    TFLITE_LOG_PROD(
+        TFLITE_LOG_ERROR,
+        "Number of result events (%zu) doesn't match the expectation (%zu).",
+        events.size(), 1);
+    flatbuffers::Offset<BenchmarkError> error =
+        CreateBenchmarkError(fbb, BenchmarkStage_INFERENCE,
+                             /*exit_code=*/kBenchmarkResultCountMismatch);
+    BenchmarkEventBuilder builder(fbb);
+    builder.add_event_type(BenchmarkEventType_ERROR);
+    builder.add_error(error);
+    return builder.Finish();
+  }
+  BenchmarkEventT benchmark_event;
+  events[0]->UnPackTo(&benchmark_event);
+  TfLiteMiniBenchmarkResultFree(result);
+  return CreateBenchmarkEvent(fbb, &benchmark_event);
+}
+
+}  // namespace accuracy
+}  // namespace benchmark
+}  // namespace tflite

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/accuracy_benchmark.h

@@ -0,0 +1,51 @@
+/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_DELEGATE_PERFORMANCE_ANDROID_SRC_MAIN_NATIVE_ACCURACY_BENCHMARK_H_
+#define TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_DELEGATE_PERFORMANCE_ANDROID_SRC_MAIN_NATIVE_ACCURACY_BENCHMARK_H_
+
+#include <cstddef>
+#include <string>
+#include <vector>
+
+#include "flatbuffers/buffer.h"  // from @flatbuffers
+#include "flatbuffers/flatbuffer_builder.h"  // from @flatbuffers
+#include "tensorflow/lite/acceleration/configuration/configuration_generated.h"
+
+namespace tflite {
+namespace benchmark {
+namespace accuracy {
+
+// Triggers MiniBenchmark testings. Uses the arguments passed from the testing
+// app to configure MiniBenchmark ValidatorRunner. The tests will access and
+// execute the pre-embedded models in the app via the model file descriptor. The
+// contents of a model are initialized using model_size bytes starting at
+// model_offset position in the file described by model_fd. Any intermediate
+// data and results will be dumped to the result path given.
+//
+// Returns a BenchmarkEvent flatbuffer offset. If the benchmark tests finish
+// successfully with a pass from MiniBenchmark, the returned offset contains the
+// concrete accuracy metrics and the overall result from MiniBenchmark.
+// Otherwise, the returned value contains an error code.
+flatbuffers::Offset<BenchmarkEvent> Benchmark(
+    flatbuffers::FlatBufferBuilder& fbb, const TFLiteSettings& tflite_settings,
+    int model_fd, size_t model_offset, size_t model_size,
+    const char* result_path_chars);
+
+}  // namespace accuracy
+}  // namespace benchmark
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_DELEGATE_PERFORMANCE_ANDROID_SRC_MAIN_NATIVE_ACCURACY_BENCHMARK_H_

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/delegate_performance_benchmark_jni.cc

@@ -0,0 +1,155 @@
+/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <jni.h>
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+#include "flatbuffers/flatbuffer_builder.h"  // from @flatbuffers
+#include "tensorflow/lite/acceleration/configuration/configuration_generated.h"
+#include "tensorflow/lite/delegates/utils/experimental/stable_delegate/tflite_settings_json_parser.h"
+#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto/delegate_performance.pb.h"
+#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/accuracy_benchmark.h"
+
+#ifndef TFLITE_WITH_STABLE_ABI
+#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/latency_benchmark.h"
+#endif  // !TFLITE_WITH_STABLE_ABI
+
+namespace {
+
+// A helper method that converts an array of strings passed from Java to a
+// vector of strings in C++.
+std::vector<std::string> toStringVector(JNIEnv* env,
+                                        jobjectArray string_array) {
+  int len = env->GetArrayLength(string_array);
+  std::vector<std::string> vec(len);
+  for (int i = 0; i < len; ++i) {
+    jstring str =
+        static_cast<jstring>(env->GetObjectArrayElement(string_array, i));
+    const char* chars = env->GetStringUTFChars(str, nullptr);
+    vec[i] = std::string(chars);
+    env->ReleaseStringUTFChars(str, chars);
+    env->DeleteLocalRef(str);
+  }
+  return vec;
+}
+
+// Serializes the proto message into jbyteArray.
+jbyteArray CppProtoToBytes(JNIEnv* env, const google::protobuf::MessageLite& proto) {
+  jbyteArray array = nullptr;
+  const int byte_size = proto.ByteSizeLong();
+  if (byte_size) {
+    array = env->NewByteArray(byte_size);
+    void* ptr = env->GetPrimitiveArrayCritical(array, nullptr);
+    proto.SerializeWithCachedSizesToArray(static_cast<uint8_t*>(ptr));
+    env->ReleasePrimitiveArrayCritical(array, ptr, 0);
+  }
+  return array;
+}
+
+}  // namespace
+
+extern "C" {
+
+JNIEXPORT jbyteArray JNICALL
+Java_org_tensorflow_lite_benchmark_delegateperformance_DelegatePerformanceBenchmark_latencyBenchmarkNativeRun(
+    JNIEnv* env, jclass clazz, jobjectArray args_obj,
+    jbyteArray tflite_settings_byte_array, jstring tflite_settings_path_obj,
+    jint model_fd, jlong model_offset, jlong model_size) {
+  tflite::proto::benchmark::LatencyResults results;
+
+// The latency benchmark doesn't support TF Lite with the stable ABI path.
+#ifndef TFLITE_WITH_STABLE_ABI
+  std::vector<std::string> args = toStringVector(env, args_obj);
+  const char* tflite_settings_path_chars =
+      env->GetStringUTFChars(tflite_settings_path_obj, nullptr);
+  jbyte* tflite_settings_bytes =
+      env->GetByteArrayElements(tflite_settings_byte_array, nullptr);
+  const tflite::TFLiteSettings* tflite_settings =
+      flatbuffers::GetRoot<tflite::TFLiteSettings>(
+          reinterpret_cast<const char*>(tflite_settings_bytes));
+
+  results = tflite::benchmark::latency::Benchmark(
+      *tflite_settings, tflite_settings_path_chars, static_cast<int>(model_fd),
+      static_cast<size_t>(model_offset), static_cast<size_t>(model_size), args);
+
+  env->ReleaseByteArrayElements(tflite_settings_byte_array,
+                                tflite_settings_bytes, JNI_ABORT);
+  env->ReleaseStringUTFChars(tflite_settings_path_obj,
+                             tflite_settings_path_chars);
+#endif  // !TFLITE_WITH_STABLE_ABI
+  return CppProtoToBytes(env, results);
+}
+
+// TODO(b/262411020): Consider returning jobject directly.
+JNIEXPORT jbyteArray JNICALL
+Java_org_tensorflow_lite_benchmark_delegateperformance_DelegatePerformanceBenchmark_accuracyBenchmarkNativeRun(
+    JNIEnv* env, jclass clazz, jbyteArray tflite_settings_byte_array,
+    jint model_fd, jlong model_offset, jlong model_size,
+    jstring result_path_obj) {
+  const char* result_path_chars =
+      env->GetStringUTFChars(result_path_obj, nullptr);
+  jbyte* tflite_settings_bytes =
+      env->GetByteArrayElements(tflite_settings_byte_array, nullptr);
+  const tflite::TFLiteSettings* tflite_settings =
+      flatbuffers::GetRoot<tflite::TFLiteSettings>(
+          reinterpret_cast<const char*>(tflite_settings_bytes));
+  flatbuffers::FlatBufferBuilder fbb;
+
+  flatbuffers::Offset<tflite::BenchmarkEvent> benchmark_event =
+      tflite::benchmark::accuracy::Benchmark(
+          fbb, *tflite_settings, static_cast<int>(model_fd),
+          static_cast<size_t>(model_offset), static_cast<size_t>(model_size),
+          result_path_chars);
+  fbb.Finish(benchmark_event);
+
+  env->ReleaseByteArrayElements(tflite_settings_byte_array,
+                                tflite_settings_bytes, JNI_ABORT);
+  env->ReleaseStringUTFChars(result_path_obj, result_path_chars);
+
+  jbyteArray byte_array = nullptr;
+  if (fbb.GetSize() > 0) {
+    byte_array = env->NewByteArray(fbb.GetSize());
+    env->SetByteArrayRegion(
+        byte_array, 0, fbb.GetSize(),
+        reinterpret_cast<const jbyte*>(fbb.GetBufferPointer()));
+  }
+  return byte_array;
+}
+
+JNIEXPORT jbyteArray JNICALL
+Java_org_tensorflow_lite_benchmark_delegateperformance_DelegatePerformanceBenchmark_loadTfLiteSettingsJsonNative(
+    JNIEnv* env, jclass clazz, jstring json_file_path_obj) {
+  const char* json_file_path_chars =
+      env->GetStringUTFChars(json_file_path_obj, nullptr);
+
+  tflite::delegates::utils::TfLiteSettingsJsonParser parser;
+  parser.Parse(json_file_path_chars);
+
+  jbyteArray tflite_settings_byte_array = nullptr;
+  flatbuffers::uoffset_t tflite_settings_size = parser.GetBufferSize();
+  if (tflite_settings_size > 0) {
+    tflite_settings_byte_array = env->NewByteArray(tflite_settings_size);
+    env->SetByteArrayRegion(
+        tflite_settings_byte_array, 0, tflite_settings_size,
+        reinterpret_cast<const jbyte*>(parser.GetBufferPointer()));
+  }
+  env->ReleaseStringUTFChars(json_file_path_obj, json_file_path_chars);
+  return tflite_settings_byte_array;
+}
+
+}  // extern "C"

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/latency_benchmark.cc

@@ -0,0 +1,254 @@
+/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/latency_benchmark.h"
+
+#include <errno.h>
+#include <sys/stat.h>
+
+#include <cstddef>
+#include <cstdint>
+#include <string>
+#include <vector>
+
+#include "absl/strings/str_cat.h"
+#include "absl/strings/str_format.h"
+#include "tensorflow/core/util/stats_calculator.h"
+#include "tensorflow/lite/acceleration/configuration/configuration_generated.h"
+#include "tensorflow/lite/c/c_api_types.h"
+#include "tensorflow/lite/logger.h"
+#include "tensorflow/lite/minimal_logging.h"
+#include "tensorflow/lite/profiling/memory_info.h"
+#include "tensorflow/lite/tools/benchmark/benchmark_model.h"
+#include "tensorflow/lite/tools/benchmark/benchmark_params.h"
+#include "tensorflow/lite/tools/benchmark/benchmark_tflite_model.h"
+#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto/delegate_performance.pb.h"
+
+namespace tflite {
+namespace benchmark {
+namespace latency {
+namespace {
+
+static constexpr char kBenchmarkToolName[] = "(BenchmarkModelAndroid)";
+
+// The listener subscribes to the benchmark lifecycle and parses the
+// benchmarking results into a LatencyResults proto message. The message will be
+// reported later to the app.
+class DelegatePerformanceReportingListener : public BenchmarkListener {
+ public:
+  void OnBenchmarkStart(const BenchmarkParams& unused) override {
+    results_proto_.set_event_type(proto::benchmark::BENCHMARK_EVENT_TYPE_START);
+  }
+
+  // TFLite Benchmark Tool triggers this method at the end of a benchmark for
+  // logging the results.
+  void OnBenchmarkEnd(const BenchmarkResults& results) override {
+    ReportResult(results);
+  }
+
+  void ReportFailure(TfLiteStatus status) {
+    std::string status_msg =
+        status == kTfLiteError
+            ? "TFLite error"
+            : (status == kTfLiteDelegateError ? "TFLite delegate error"
+                                              : "unexpected TFLite status");
+    TFLITE_LOG_PROD(TFLITE_LOG_ERROR,
+                    "Benchmark failed due to %s with status code %d.",
+                    status_msg.c_str(), status);
+    results_proto_.set_event_type(proto::benchmark::BENCHMARK_EVENT_TYPE_ERROR);
+    results_proto_.mutable_error()->mutable_error_code()->set_tflite_error(
+        status);
+    results_proto_.mutable_error()->set_error_message(status_msg);
+  }
+
+  const proto::benchmark::LatencyResults& GetResults() {
+    return results_proto_;
+  }
+
+ private:
+  // TODO(b/262399611): Consider putting metric related logic into a separate
+  // file.
+  void ReportResult(const BenchmarkResults& results) {
+    tensorflow::Stat<int64_t> warmup_us = results.warmup_time_us();
+    tensorflow::Stat<int64_t> inference_us = results.inference_time_us();
+    profiling::memory::MemoryUsage init_mem_usage = results.init_mem_usage();
+    profiling::memory::MemoryUsage overall_mem_usage =
+        results.overall_mem_usage();
+
+    if (results.model_size_mb() > 0) {
+      // Ignores invalid model sizes to avoid confusions.
+      AddMetric(/*name=*/"model_size_megabyte",
+                /*value=*/results.model_size_mb());
+    }
+    AddMetric(/*name=*/"initialization_latency_us",
+              /*value=*/results.startup_latency_us());
+    AddMetric(/*name=*/"warmup_latency_average_us", /*value=*/warmup_us.avg());
+    AddMetric(/*name=*/"warmup_latency_min_us", /*value=*/warmup_us.min());
+    AddMetric(/*name=*/"warmup_latency_max_us", /*value=*/warmup_us.max());
+    AddMetric(/*name=*/"warmup_latency_standard_deviation_us",
+              /*value=*/warmup_us.std_deviation());
+    AddMetric(/*name=*/"inference_latency_average_us",
+              /*value=*/inference_us.avg());
+    AddMetric(/*name=*/"inference_latency_min_us",
+              /*value=*/inference_us.min());
+    AddMetric(/*name=*/"inference_latency_max_us",
+              /*value=*/inference_us.max());
+    AddMetric(/*name=*/"inference_latency_standard_deviation_us",
+              /*value=*/inference_us.std_deviation());
+    AddMetric(/*name=*/"initialization_memory_max_rss_mebibyte",
+              /*value=*/init_mem_usage.mem_footprint_kb / 1024.0);
+    AddMetric(/*name=*/"initialization_memory_total_non_mmapped_heap_mebibyte",
+              /*value=*/init_mem_usage.total_allocated_bytes / 1024.0 / 1024.0);
+    AddMetric(
+        /*name=*/"initialization_memory_in_use_heap_mebibyte",
+        /*value=*/init_mem_usage.in_use_allocated_bytes / 1024.0 / 1024.0);
+    AddMetric(/*name=*/"overall_memory_max_rss_mebibyte",
+              /*value=*/overall_mem_usage.mem_footprint_kb / 1024.0);
+    AddMetric(
+        /*name=*/"overall_memory_total_non_mmapped_heap_mebibyte",
+        /*value=*/overall_mem_usage.total_allocated_bytes / 1024.0 / 1024.0);
+    AddMetric(
+        /*name=*/"overall_memory_in_use_heap_mebibyte",
+        /*value=*/overall_mem_usage.in_use_allocated_bytes / 1024.0 / 1024.0);
+    results_proto_.set_event_type(proto::benchmark::BENCHMARK_EVENT_TYPE_END);
+    TFLITE_LOG_PROD(TFLITE_LOG_INFO, "Benchmark finished.");
+  }
+
+  void AddMetric(std::string name, float value) {
+    proto::benchmark::BenchmarkMetric* metric = results_proto_.add_metrics();
+    metric->set_name(name);
+    metric->set_value(value);
+  }
+
+  proto::benchmark::LatencyResults results_proto_;
+};
+
+// Converts the input TFLiteSettings into TFLite Benchmark Tool arguments.
+// Please see tensorflow/lite/tools/benchmark.
+std::vector<std::string> ParseArgumentsFromTfLiteSettings(
+    const TFLiteSettings& tflite_settings,
+    const std::string& tflite_settings_path) {
+  std::vector<std::string> args;
+  if (tflite_settings_path.empty()) {
+    return args;
+  }
+  if (tflite_settings.stable_delegate_loader_settings()) {
+    args.push_back(absl::StrFormat("--stable_delegate_settings_file=%s",
+                                   tflite_settings_path));
+    return args;
+  }
+  switch (tflite_settings.delegate()) {
+    case Delegate_XNNPACK: {
+      args.push_back("--use_xnnpack=true");
+      if (tflite_settings.xnnpack_settings()) {
+        if (tflite_settings.xnnpack_settings()->num_threads()) {
+          args.push_back(absl::StrFormat(
+              "--num_threads=%d",
+              tflite_settings.xnnpack_settings()->num_threads()));
+        }
+        if (tflite_settings.xnnpack_settings()->flags() ==
+            XNNPackFlags_TFLITE_XNNPACK_DELEGATE_FLAG_FORCE_FP16) {
+          args.push_back("--xnnpack_force_fp16=true");
+        }
+      }
+      return args;
+    }
+    case Delegate_GPU: {
+      args.push_back("--use_gpu=true");
+      const tflite::GPUSettings* gpu_settings = tflite_settings.gpu_settings();
+      if (gpu_settings) {
+        if (gpu_settings->is_precision_loss_allowed()) {
+          args.push_back("--gpu_precision_loss_allowed=true");
+        }
+        if (gpu_settings->enable_quantized_inference()) {
+          args.push_back("--gpu_experimental_enable_quant=true");
+        }
+        if (gpu_settings->inference_preference() ==
+            GPUInferenceUsage_GPU_INFERENCE_PREFERENCE_SUSTAINED_SPEED) {
+          args.push_back("--gpu_inference_for_sustained_speed=true");
+        }
+        if (gpu_settings->force_backend() == GPUBackend_OPENCL) {
+          args.push_back("--gpu_backend=cl");
+        } else if (gpu_settings->force_backend() == GPUBackend_OPENGL) {
+          args.push_back("--gpu_backend=gl");
+        }
+        if (gpu_settings->cache_directory()) {
+          args.push_back(
+              absl::StrFormat("--delegate_serialize_dir=%s",
+                              gpu_settings->cache_directory()->c_str()));
+        }
+        if (gpu_settings->model_token()) {
+          args.push_back(absl::StrFormat("--delegate_serialize_token=%s",
+                                         gpu_settings->model_token()->c_str()));
+        }
+      }
+      break;
+    }
+    case Delegate_EDGETPU: {
+      args.push_back("--use_edgetpu=true");
+      break;
+    }
+    default:
+      TFLITE_LOG_PROD(TFLITE_LOG_WARNING,
+                      "Delegate type %s is not enabled by the latency module.",
+                      EnumNameDelegate(tflite_settings.delegate()));
+      break;
+  }
+  if (tflite_settings.disable_default_delegates()) {
+    // Currently TFLite Benchmark Tool doesn't support handling the case with
+    // applying XNNPack delegate explicitly and disabling the XNNPack delegate
+    // as the default delegate at the same time. When the
+    // "disable_default_delegates" configuration is set to true, it only takes
+    // effect if the delegate is not set to XNNPack. Otherwise, the default
+    // delegates will still be enabled.
+    args.push_back("--use_xnnpack=false");
+  }
+  return args;
+}
+}  // namespace
+
+proto::benchmark::LatencyResults Benchmark(
+    const TFLiteSettings& tflite_settings,
+    const std::string& tflite_settings_path, int model_fd, size_t model_offset,
+    size_t model_size, const std::vector<std::string>& args) {
+  // Constructs a fake argv command-line object for the benchmark.
+  std::vector<char*> argv;
+  argv.push_back(const_cast<char*>(kBenchmarkToolName));
+  std::string arg_graph =
+      absl::StrCat("--graph=fd:", model_fd, ":", model_offset, ":", model_size);
+  argv.push_back(const_cast<char*>(arg_graph.data()));
+  std::vector<std::string> args_from_tflite_settings =
+      ParseArgumentsFromTfLiteSettings(tflite_settings, tflite_settings_path);
+  for (const std::string& arg : args_from_tflite_settings) {
+    argv.push_back(const_cast<char*>(arg.data()));
+  }
+  // Keep the args here for any additional TFLite Benchmark Tool configurations.
+  for (const std::string& arg : args) {
+    argv.push_back(const_cast<char*>(arg.data()));
+  }
+
+  BenchmarkTfLiteModel benchmark;
+  DelegatePerformanceReportingListener delegatePerformanceReporting;
+  benchmark.AddListener(&delegatePerformanceReporting);
+  TfLiteStatus status = benchmark.Run(argv.size(), argv.data());
+  if (status != kTfLiteOk) {
+    delegatePerformanceReporting.ReportFailure(status);
+  }
+  return delegatePerformanceReporting.GetResults();
+}
+
+}  // namespace latency
+}  // namespace benchmark
+}  // namespace tflite

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/latency_benchmark.h

@@ -0,0 +1,49 @@
+/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_DELEGATE_PERFORMANCE_ANDROID_SRC_MAIN_NATIVE_LATENCY_BENCHMARK_H_
+#define TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_DELEGATE_PERFORMANCE_ANDROID_SRC_MAIN_NATIVE_LATENCY_BENCHMARK_H_
+
+#include <string>
+#include <vector>
+
+#include "tensorflow/lite/acceleration/configuration/configuration_generated.h"
+#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto/delegate_performance.pb.h"
+
+namespace tflite {
+namespace benchmark {
+namespace latency {
+
+// Triggers TFLite Benchmark Tool. Passes the "args" from the testing app to
+// directly to TFLite Benchmark Tool. Converts the "tflite_settings" to
+// command-line options to configure TFLite Benchmark Tool. If the latency
+// benchmarking uses a stable delegate, the "tflite_settings_path" is passed to
+// enable the stable delegate provider. The contents of the tested model are
+// initialized using model_size bytes starting at model_offset position in the
+// file referenced by the file descriptor model_fd.
+//
+// Returns a LatencyResults proto message. If the benchmark tests finish
+// successfully from TFLite Benchmark Tool, the message contains the latency
+// metrics. Otherwise, the message contains the corresponding error.
+proto::benchmark::LatencyResults Benchmark(
+    const TFLiteSettings& tflite_settings,
+    const std::string& tflite_settings_path, int model_fd, size_t model_offset,
+    size_t model_size, const std::vector<std::string>& args);
+
+}  // namespace latency
+}  // namespace benchmark
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_DELEGATE_PERFORMANCE_ANDROID_SRC_MAIN_NATIVE_LATENCY_BENCHMARK_H_

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/status_codes.h

@@ -0,0 +1,35 @@
+/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_DELEGATE_PERFORMANCE_ANDROID_SRC_MAIN_NATIVE_STATUS_CODES_H_
+#define TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_DELEGATE_PERFORMANCE_ANDROID_SRC_MAIN_NATIVE_STATUS_CODES_H_
+
+namespace tflite {
+namespace benchmark {
+
+enum DelegatePerformanceBenchmarkStatus {
+  kBenchmarkUnknownStatus = 0,
+
+  // Set of error codes that are used as the return codes to communicate between
+  // the native layer and the caller app.
+  kBenchmarkRunnerInitializationFailed = 1000,
+  kBenchmarkResultCountMismatch = 1001,
+  kBenchmarkInvalidTfLiteSettings = 1002,
+};
+
+}  // namespace benchmark
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_BENCHMARK_EXPERIMENTAL_DELEGATE_PERFORMANCE_ANDROID_SRC_MAIN_NATIVE_STATUS_CODES_H_

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/test/native/BUILD

@@ -0,0 +1,61 @@
+# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# Description:
+#  Holds the tests for the native layer of the app.
+
+load("@rules_cc//cc:cc_test.bzl", "cc_test")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    licenses = ["notice"],
+)
+
+cc_test(
+    name = "latency_benchmark_test",
+    srcs = ["latency_benchmark_test.cc"],
+    data = [
+        "//tensorflow/lite/delegates/utils/experimental/sample_stable_delegate:tensorflowlite_sample_stable_delegate",
+        "//tensorflow/lite/tools/delegates/experimental/stable_delegate:test_sample_stable_delegate_settings.json",
+        "@tflite_mobilenet_float//:mobilenet_v1_1.0_224.tflite",
+    ],
+    deps = [
+        "//tensorflow/lite/acceleration/configuration:configuration_fbs",
+        "//tensorflow/lite/delegates/utils/experimental/stable_delegate:tflite_settings_json_parser",
+        "//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto:delegate_performance_cc_proto",
+        "//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native:latency_benchmark",
+        "@com_google_googletest//:gtest_main",
+        "@flatbuffers",
+    ],
+)
+
+cc_test(
+    name = "accuracy_benchmark_test",
+    srcs = ["accuracy_benchmark_test.cc"],
+    data = [
+        "//tensorflow/lite/delegates/utils/experimental/sample_stable_delegate:tensorflowlite_sample_stable_delegate",
+        "//tensorflow/lite/delegates/utils/experimental/stable_delegate:test_xnnpack_settings.json",
+        "//tensorflow/lite/tools/delegates/experimental/stable_delegate:test_sample_stable_delegate_settings.json",
+    ],
+    deps = [
+        "//tensorflow/lite/acceleration/configuration:configuration_fbs",
+        "//tensorflow/lite/delegates/utils/experimental/stable_delegate:tflite_settings_json_parser",
+        "//tensorflow/lite/experimental/acceleration/mini_benchmark:embedded_mobilenet_validation_model",
+        "//tensorflow/lite/experimental/acceleration/mini_benchmark:mini_benchmark_test_helper",
+        "//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native:accuracy_benchmark",
+        "//tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native:status_codes",
+        "@com_google_googletest//:gtest_main",
+        "@flatbuffers",
+    ],
+)

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/test/native/accuracy_benchmark_test.cc

@@ -0,0 +1,149 @@
+/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/accuracy_benchmark.h"
+
+#include <fcntl.h>
+#include <stdio.h>
+
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include <gtest/gtest.h>
+#include "flatbuffers/buffer.h"  // from @flatbuffers
+#include "flatbuffers/flatbuffer_builder.h"  // from @flatbuffers
+#include "tensorflow/lite/acceleration/configuration/configuration_generated.h"
+#include "tensorflow/lite/delegates/utils/experimental/stable_delegate/tflite_settings_json_parser.h"
+#include "tensorflow/lite/experimental/acceleration/mini_benchmark/embedded_mobilenet_validation_model.h"
+#include "tensorflow/lite/experimental/acceleration/mini_benchmark/mini_benchmark_test_helper.h"
+#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/status_codes.h"
+
+namespace tflite {
+namespace benchmark {
+namespace accuracy {
+namespace {
+
+class AccuracyBenchmarkTest : public ::testing::Test {
+ protected:
+  void SetUp() override {
+    acceleration::MiniBenchmarkTestHelper helper;
+    should_perform_test_ = helper.should_perform_test();
+
+    if (!should_perform_test_) {
+      return;
+    }
+    std::string embedded_model_path = helper.DumpToTempFile(
+        "mobilenet_quant_with_validation.tflite",
+        g_tflite_acceleration_embedded_mobilenet_validation_model,
+        g_tflite_acceleration_embedded_mobilenet_validation_model_len);
+    ASSERT_FALSE(embedded_model_path.empty());
+
+    model_fp_ = fopen(embedded_model_path.c_str(), "rb");
+    ASSERT_NE(model_fp_, nullptr);
+    ASSERT_EQ(fseek(model_fp_, 0, SEEK_END), 0);
+    model_size_ = ftell(model_fp_);
+    ASSERT_NE(model_size_, -1);
+    ASSERT_EQ(fseek(model_fp_, 0, SEEK_SET), 0);
+
+    result_path_ = ::testing::TempDir();
+  }
+
+  void TearDown() override { fclose(model_fp_); }
+
+  std::string result_path_;
+  size_t model_size_;
+  FILE* model_fp_;
+  bool should_perform_test_ = true;
+};
+
+TEST_F(AccuracyBenchmarkTest, FailedWithInvalidModelFileDescriptor) {
+  if (!should_perform_test_) {
+    std::cerr << "Skipping test";
+    return;
+  }
+  delegates::utils::TfLiteSettingsJsonParser parser;
+  flatbuffers::FlatBufferBuilder builder;
+  std::vector<std::string> args;
+  const TFLiteSettings* tflite_settings = parser.Parse(
+      "tensorflow/lite/tools/delegates/experimental/"
+      "stable_delegate/test_sample_stable_delegate_settings.json");
+
+  flatbuffers::Offset<BenchmarkEvent> offset =
+      Benchmark(builder, *tflite_settings, /*model_fd=*/0,
+                /*model_offset=*/0, /*model_size=*/0, result_path_.c_str());
+  builder.Finish(offset);
+  const BenchmarkEvent* event =
+      flatbuffers::GetRoot<BenchmarkEvent>(builder.GetBufferPointer());
+
+  ASSERT_NE(event, nullptr);
+  EXPECT_EQ(event->event_type(), BenchmarkEventType_ERROR);
+  ASSERT_NE(event->error(), nullptr);
+  EXPECT_EQ(event->error()->stage(), BenchmarkStage_INFERENCE);
+  EXPECT_EQ(event->error()->exit_code(),
+            DelegatePerformanceBenchmarkStatus::kBenchmarkResultCountMismatch);
+}
+
+TEST_F(AccuracyBenchmarkTest, SucceedWithSampleStableDelegate) {
+  if (!should_perform_test_) {
+    std::cerr << "Skipping test";
+    return;
+  }
+  delegates::utils::TfLiteSettingsJsonParser parser;
+  flatbuffers::FlatBufferBuilder builder;
+  const TFLiteSettings* tflite_settings = parser.Parse(
+      "tensorflow/lite/tools/delegates/experimental/"
+      "stable_delegate/test_sample_stable_delegate_settings.json");
+
+  flatbuffers::Offset<BenchmarkEvent> offset = Benchmark(
+      builder, *tflite_settings, /*model_fd=*/fileno(model_fp_),
+      /*model_offset=*/0, /*model_size=*/model_size_, result_path_.c_str());
+  builder.Finish(offset);
+  const BenchmarkEvent* event =
+      flatbuffers::GetRoot<BenchmarkEvent>(builder.GetBufferPointer());
+
+  // TODO(b/253442685): verify that the stable delegate was used.
+  ASSERT_NE(event, nullptr);
+  EXPECT_EQ(event->event_type(), BenchmarkEventType_END);
+  EXPECT_EQ(event->error(), nullptr);
+}
+
+TEST_F(AccuracyBenchmarkTest, SucceedWithEmbeddedValidationAndXNNPack) {
+  if (!should_perform_test_) {
+    std::cerr << "Skipping test";
+    return;
+  }
+  delegates::utils::TfLiteSettingsJsonParser parser;
+  flatbuffers::FlatBufferBuilder builder;
+  const TFLiteSettings* tflite_settings = parser.Parse(
+      "tensorflow/lite/delegates/utils/experimental/"
+      "stable_delegate/test_xnnpack_settings.json");
+
+  flatbuffers::Offset<BenchmarkEvent> offset = Benchmark(
+      builder, *tflite_settings, /*model_fd=*/fileno(model_fp_),
+      /*model_offset=*/0, /*model_size=*/model_size_, result_path_.c_str());
+  builder.Finish(offset);
+  const BenchmarkEvent* event =
+      flatbuffers::GetRoot<BenchmarkEvent>(builder.GetBufferPointer());
+
+  // TODO(b/253442685): verify that the XNNPack delegate was used.
+  ASSERT_NE(event, nullptr);
+  EXPECT_EQ(event->event_type(), BenchmarkEventType_END);
+  EXPECT_EQ(event->error(), nullptr);
+}
+
+}  // namespace
+}  // namespace accuracy
+}  // namespace benchmark
+}  // namespace tflite

tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/test/native/latency_benchmark_test.cc

@@ -0,0 +1,153 @@
+/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/src/main/native/latency_benchmark.h"
+
+#include <fcntl.h>
+
+#include <string>
+#include <vector>
+
+#include <gtest/gtest.h>
+#include "flatbuffers/buffer.h"  // from @flatbuffers
+#include "flatbuffers/flatbuffer_builder.h"  // from @flatbuffers
+#include "tensorflow/lite/acceleration/configuration/configuration_generated.h"
+#include "tensorflow/lite/delegates/utils/experimental/stable_delegate/tflite_settings_json_parser.h"
+#include "tensorflow/lite/tools/benchmark/experimental/delegate_performance/android/proto/delegate_performance.pb.h"
+
+namespace tflite {
+namespace benchmark {
+namespace latency {
+namespace {
+
+static constexpr char kModelPath[] =
+    "../tflite_mobilenet_float/"
+    "mobilenet_v1_1.0_224.tflite";
+static constexpr char kSettingsFilePath[] =
+    "tensorflow/lite/tools/delegates/experimental/stable_delegate/"
+    "test_sample_stable_delegate_settings.json";
+
+class LatencyBenchmarkTest : public ::testing::Test {
+ protected:
+  void SetUp() override {
+    model_fp_ = fopen(kModelPath, "rb");
+    ASSERT_TRUE(model_fp_ != nullptr);
+    ASSERT_EQ(fseek(model_fp_, 0, SEEK_END), 0);
+    model_size_ = ftell(model_fp_);
+    ASSERT_NE(model_size_, -1);
+    ASSERT_EQ(fseek(model_fp_, 0, SEEK_SET), 0);
+    settings_ = parser_.Parse(kSettingsFilePath);
+  }
+
+  delegates::utils::TfLiteSettingsJsonParser parser_;
+  const TFLiteSettings* settings_;
+  size_t model_size_;
+  FILE* model_fp_;
+  std::vector<std::string> args_;
+};
+
+TEST_F(LatencyBenchmarkTest, FailedWithNullFileDescriptor) {
+  EXPECT_TRUE(Benchmark(*settings_, kSettingsFilePath,
+                        /*model_fd=*/0, /*model_offset=*/0,
+                        /*model_size=*/0, args_)
+                  .has_error());
+}
+
+TEST_F(LatencyBenchmarkTest, FailedWithInvalidNumThreadsSettings) {
+  flatbuffers::FlatBufferBuilder fbb;
+  flatbuffers::Offset<tflite::XNNPackSettings> xnnpack_settings =
+      CreateXNNPackSettings(fbb, /*num_threads=*/-3);
+  TFLiteSettingsBuilder tflite_settings_builder(fbb);
+  tflite_settings_builder.add_delegate(Delegate_XNNPACK);
+  tflite_settings_builder.add_xnnpack_settings(xnnpack_settings);
+  fbb.Finish(tflite_settings_builder.Finish());
+  const TFLiteSettings* settings =
+      flatbuffers::GetRoot<TFLiteSettings>(fbb.GetBufferPointer());
+
+  EXPECT_TRUE(Benchmark(*settings,
+                        /*tflite_settings_path=*/"example_path",
+                        fileno(model_fp_),
+                        /*model_offset=*/0, model_size_, args_)
+                  .has_error());
+}
+
+TEST_F(LatencyBenchmarkTest, SucceedWithEmptyTfLiteSettings) {
+  flatbuffers::FlatBufferBuilder fbb;
+  TFLiteSettingsBuilder tflite_settings_builder(fbb);
+  fbb.Finish(tflite_settings_builder.Finish());
+  const TFLiteSettings* settings =
+      flatbuffers::GetRoot<TFLiteSettings>(fbb.GetBufferPointer());
+
+  // TODO(b/253442685): verify that the default delegate was used.
+  EXPECT_EQ(Benchmark(*settings, /*tflite_settings_path=*/"example_path",
+                      fileno(model_fp_), /*model_offset=*/0, model_size_, args_)
+                .event_type(),
+            proto::benchmark::BENCHMARK_EVENT_TYPE_END);
+}
+
+TEST_F(LatencyBenchmarkTest, SucceedWithCpuTfLiteSettings) {
+  flatbuffers::FlatBufferBuilder fbb;
+  TFLiteSettingsBuilder tflite_settings_builder(fbb);
+  tflite_settings_builder.add_disable_default_delegates(true);
+  fbb.Finish(tflite_settings_builder.Finish());
+  const TFLiteSettings* settings =
+      flatbuffers::GetRoot<TFLiteSettings>(fbb.GetBufferPointer());
+
+  // TODO(b/253442685): verify that no model delegation has occurred.
+  EXPECT_EQ(Benchmark(*settings, /*tflite_settings_path=*/"example_path",
+                      fileno(model_fp_), /*model_offset=*/0, model_size_, args_)
+                .event_type(),
+            proto::benchmark::BENCHMARK_EVENT_TYPE_END);
+}
+
+#ifdef __ANDROID__
+TEST_F(LatencyBenchmarkTest, SucceedWithGpuTfLiteSettings) {
+  flatbuffers::FlatBufferBuilder fbb;
+  TFLiteSettingsBuilder tflite_settings_builder(fbb);
+  tflite_settings_builder.add_delegate(Delegate_GPU);
+  fbb.Finish(tflite_settings_builder.Finish());
+  const TFLiteSettings* settings =
+      flatbuffers::GetRoot<TFLiteSettings>(fbb.GetBufferPointer());
+
+  // TODO(b/253442685): verify that the GPU delegate was used.
+  EXPECT_EQ(Benchmark(*settings, /*tflite_settings_path=*/"example_path",
+                      fileno(model_fp_), /*model_offset=*/0, model_size_, args_)
+                .event_type(),
+            proto::benchmark::BENCHMARK_EVENT_TYPE_END);
+}
+#endif  // __ANDROID__
+
+TEST_F(LatencyBenchmarkTest, SucceedWithSampleStableDelegate) {
+  // TODO(b/253442685): verify that the stable delegate was used.
+  EXPECT_EQ(Benchmark(*settings_, kSettingsFilePath, fileno(model_fp_),
+                      /*model_offset=*/0, model_size_, args_)
+                .event_type(),
+            proto::benchmark::BENCHMARK_EVENT_TYPE_END);
+}
+
+TEST_F(LatencyBenchmarkTest,
+       SucceedWithSampleStableDelegateAndBenchmarkToolArguments) {
+  std::vector<std::string> args = {"--warmup_runs=10"};
+
+  // TODO(b/253442685): verify that the stable delegate was used.
+  EXPECT_EQ(Benchmark(*settings_, kSettingsFilePath, fileno(model_fp_),
+                      /*model_offset=*/0, model_size_, args)
+                .event_type(),
+            proto::benchmark::BENCHMARK_EVENT_TYPE_END);
+}
+
+}  // namespace
+}  // namespace latency
+}  // namespace benchmark
+}  // namespace tflite

tensorflow/lite/tools/benchmark/experimental/firebase/android/AndroidManifest.xml

@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="org.tensorflow.lite.benchmark.firebase">
+
+    <!-- Necessary for loading custom models from disk and writing result to disk. -->
+    <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE"/>
+    <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE"/>
+
+    <uses-sdk
+        android:minSdkVersion="23"
+        android:targetSdkVersion="23" />
+
+    <application>
+        <!-- Number of Firebase Game Loop test scenarios defined in this application. -->
+        <meta-data
+            android:name="com.google.test.loops"
+            android:value="10" />
+        <!-- This Activity runs on the Firebase Test Lab. -->
+        <activity
+            android:name=".BenchmarkModelActivity"
+            android:screenOrientation="portrait"
+            android:label="TFLite Benchmark on Firebase Test Lab"
+            android:theme="@android:style/Theme.NoDisplay"
+            android:exported="true"
+            android:noHistory="true">
+            <!-- Intent filter for the Firebase Game Loop test.  -->
+            <intent-filter>
+                <action android:name="com.google.intent.action.TEST_LOOP" />
+                <category android:name="android.intent.category.DEFAULT" />
+                <data android:mimeType="application/javascript" />
+            </intent-filter>
+        </activity>
+    </application>
+
+</manifest>

tensorflow/lite/tools/benchmark/experimental/firebase/android/BUILD

@@ -0,0 +1,62 @@
+# Description:
+#   BenchmarkModel Android harness for Firebase Test Lab.
+
+load("@build_bazel_rules_android//android:rules.bzl", "android_binary")
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
+load("//tensorflow/lite:build_def.bzl", "tflite_jni_binary")
+load("//tensorflow/lite:special_rules.bzl", "tflite_hexagon_nn_skel_libraries")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_visibility = ["//visibility:private"],
+    licenses = ["notice"],
+)
+
+android_binary(
+    name = "benchmark_model_firebase",
+    srcs = glob([
+        "src/**/*.java",
+    ]),
+    custom_package = "org.tensorflow.lite.benchmark.firebase",
+    manifest = "AndroidManifest.xml",
+    # In some platforms we don't have an Android SDK/NDK and this target
+    # can't be built. We need to prevent the build system from trying to
+    # use the target in that case.
+    tags = ["manual"],
+    deps = [
+        ":hexagon_libs",
+        ":tensorflowlite_benchmark_firebase_native",
+    ],
+)
+
+tflite_jni_binary(
+    name = "libtensorflowlite_benchmark_firebase.so",
+    srcs = glob([
+        "jni/**/*.cc",
+        "jni/**/*.h",
+    ]),
+    deps = [
+        "//tensorflow/lite/java/jni",
+        "//tensorflow/lite/tools/benchmark:benchmark_tflite_model_lib",
+    ],
+)
+
+cc_library(
+    name = "tensorflowlite_benchmark_firebase_native",
+    srcs = ["libtensorflowlite_benchmark_firebase.so"],
+    visibility = ["//visibility:private"],
+)
+
+cc_library(
+    name = "hexagon_libs",
+    srcs = select({
+        "//tensorflow:android_arm64": [
+            "//tensorflow/lite/delegates/hexagon/hexagon_nn:libhexagon_interface.so",
+        ] + tflite_hexagon_nn_skel_libraries(),
+        "//tensorflow:android_arm": [
+            "//tensorflow/lite/delegates/hexagon/hexagon_nn:libhexagon_interface.so",
+        ] + tflite_hexagon_nn_skel_libraries(),
+        "//conditions:default": [],
+    }),
+    visibility = ["//visibility:private"],
+)

tensorflow/lite/tools/benchmark/experimental/firebase/android/jni/benchmark_model_jni.cc

@@ -0,0 +1,290 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <errno.h>
+#include <jni.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include <fstream>
+#include <sstream>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "tensorflow/lite/tools/benchmark/benchmark_tflite_model.h"
+
+#ifdef __ANDROID__
+#include <android/log.h>
+#endif
+
+namespace tflite {
+namespace benchmark {
+namespace {
+
+const char kOutputDir[] = "/sdcard/benchmark_output";
+const char kSerializeDir[] = "/sdcard/serialize";
+
+bool CreateDir(const char* path) {
+  struct stat st;
+  if (stat(path, &st) != 0) {
+    if (mkdir(path, 0777) != 0 && errno != EEXIST) {
+      return false;
+    }
+  } else if (!S_ISDIR(st.st_mode)) {
+    errno = ENOTDIR;
+    return false;
+  }
+  return true;
+}
+
+class FirebaseReportingListener : public BenchmarkListener {
+ public:
+  explicit FirebaseReportingListener(std::string tag, int report_fd)
+      : tag_(tag), report_fd_(report_fd) {
+    if (report_fd < 0) {
+#ifdef __ANDROID__
+      __android_log_print(
+          ANDROID_LOG_ERROR, "tflite",
+          "Report would be streamed only to local log not to Firebase "
+          "since the Firebase log file is not opened.");
+#else
+      fprintf(stderr,
+              "Report would be streamed only to local log not to Firebase "
+              "since the Firebase log file is not opened.");
+#endif
+    }
+  }
+
+  void OnBenchmarkEnd(const BenchmarkResults& results) override {
+    ReportResult(results);
+  }
+
+  void ReportFailure(TfLiteStatus status) {
+    std::string status_msg =
+        status == kTfLiteError
+            ? "TFLite error"
+            : (status == kTfLiteDelegateError ? "TFLite delegate error"
+                                              : "Unknown error code");
+    Report(status_msg, std::vector<std::pair<std::string, std::string>>());
+  }
+
+ private:
+  void Report(
+      const std::string& status,
+      const std::vector<std::pair<std::string, std::string>>& contents) {
+    // The output format of Firebase Game Loop test is json.
+    // https://firebase.google.com/docs/test-lab/android/game-loop#output-example
+    std::stringstream report;
+    report << "{\n"
+           << "  \"name\": \"TFLite benchmark\",\n"
+           << "  \"benchmark config\": \"" << tag_ << "\",\n"
+           << "  \"status\": \"" << status << "\"";
+    for (const auto& content : contents) {
+      report << ",\n"
+             << "  \"" << content.first << "\": \"" << content.second << "\"";
+    }
+    report << "\n}\n";
+
+    auto report_str = report.str();
+    if (report_fd_ >= 0) {
+      write(report_fd_, report_str.c_str(), report_str.size());
+    }
+
+#ifdef __ANDROID__
+    __android_log_print(ANDROID_LOG_ERROR, "tflite", "%s", report_str.c_str());
+#else
+    fprintf(stderr, "%s", report_str.c_str());
+#endif
+  }
+
+  void ReportResult(const BenchmarkResults& results) {
+    std::vector<std::pair<std::string, std::string>> contents;
+    std::stringstream avg_time;
+    avg_time << "init: " << results.startup_latency_us() << ", "
+             << "warmup: " << results.warmup_time_us().avg() << ", "
+             << "inference: " << results.inference_time_us().avg();
+    contents.emplace_back("average time in us", avg_time.str());
+    std::stringstream overall_mem_usage;
+    overall_mem_usage << results.overall_mem_usage();
+    contents.emplace_back("overall memory usage", overall_mem_usage.str());
+
+    Report("OK", contents);
+  }
+
+  std::string tag_;
+  int report_fd_;
+};
+
+class CsvExportingListener : public BenchmarkListener {
+ public:
+  explicit CsvExportingListener(std::string tag) : tag_(tag) {}
+
+  void OnBenchmarkEnd(const BenchmarkResults& results) override {
+    if (!CreateDir(kOutputDir)) {
+#ifdef __ANDROID__
+      __android_log_print(ANDROID_LOG_ERROR, "tflite",
+                          "Failed to create output directory %s.", kOutputDir);
+#else
+      fprintf(stderr, "Failed to create output directory %s.", kOutputDir);
+#endif
+      return;
+    }
+    WriteBenchmarkResultCsv(results);
+  }
+
+ private:
+  void WriteBenchmarkResultCsv(const BenchmarkResults& results) {
+    auto init_us = results.startup_latency_us();
+    auto warmup_us = results.warmup_time_us();
+    auto inference_us = results.inference_time_us();
+    auto init_mem_usage = results.init_mem_usage();
+    auto overall_mem_usage = results.overall_mem_usage();
+
+    std::stringstream file_name;
+    file_name << kOutputDir << "/benchmark_result_" << tag_;
+
+    std::ofstream file;
+    file.open(file_name.str().c_str());
+    file << "config_key,model_size,init_time,"
+         << "warmup_avg,warmup_min,warmup_max,warmup_stddev,"
+         << "inference_avg,inference_min,inference_max,inference_stddev,"
+         << "init_max_rss,init_total_alloc,init_in_use_alloc,"
+         << "overall_max_rss,overall_total_alloc,overall_in_use_alloc\n";
+    file << tag_ << "," << results.model_size_mb() << "," << init_us << ","
+         << warmup_us.avg() << "," << warmup_us.min() << "," << warmup_us.max()
+         << "," << warmup_us.std_deviation() << "," << inference_us.avg() << ","
+         << inference_us.min() << "," << inference_us.max() << ","
+         << inference_us.std_deviation() << ","
+         << (init_mem_usage.mem_footprint_kb / 1024.0) << ","
+         << (init_mem_usage.total_allocated_bytes / 1024.0 / 1024.0) << ","
+         << (init_mem_usage.in_use_allocated_bytes / 1024.0 / 1024.0) << ","
+         << (overall_mem_usage.mem_footprint_kb / 1024.0) << ","
+         << (overall_mem_usage.total_allocated_bytes / 1024.0 / 1024.0) << ","
+         << (overall_mem_usage.in_use_allocated_bytes / 1024.0 / 1024.0)
+         << "\n";
+    file.close();
+  }
+
+  std::string tag_;
+};
+
+std::string GetScenarioConfig(const std::string& library_dir, int scenario,
+                              std::vector<std::string>& args) {
+  // The number of scenarios should equal to the value specified in
+  // AndroidManifest.xml file.
+  std::unordered_map<int, std::pair<std::string, std::vector<std::string>>>
+      all_scenarios = {
+          {1, {"cpu_1thread", {"--num_threads=1"}}},
+          {2, {"cpu_2threads", {"--num_threads=2"}}},
+          {3, {"cpu_4threads", {"--num_threads=4"}}},
+          {4, {"xnnpack_1thread", {"--use_xnnpack=true", "--num_threads=1"}}},
+          {5, {"xnnpack_2threads", {"--use_xnnpack=true", "--num_threads=2"}}},
+          {6, {"xnnpack_4threads", {"--use_xnnpack=true", "--num_threads=4"}}},
+          {7,
+           {"gpu_default",
+            {"--use_gpu=true", "--gpu_precision_loss_allowed=false"}}},
+          {8,
+           {"gpu_fp16",
+            {"--use_gpu=true", "--gpu_precision_loss_allowed=true"}}},
+          {9, {"dsp_hexagon", {"--use_hexagon=true"}}},
+          {10, {"nnapi", {"--use_nnapi=true"}}},
+          {11,
+           {"gpu_default_with_serialization",
+            {"--use_gpu=true", "--gpu_precision_loss_allowed=false",
+             "--delegate_serialize_token=dummy_token"}}},
+          {12,
+           {"gpu_fp16_with_serialization",
+            {"--use_gpu=true", "--gpu_precision_loss_allowed=true",
+             "--delegate_serialize_token=dummy_token"}}},
+      };
+
+  std::string tag;
+  args.emplace_back("(BenchmarkModelAndroid)");
+  args.emplace_back("--graph=/data/local/tmp/graph");
+
+  auto it = all_scenarios.find(scenario);
+  if (it != all_scenarios.end()) {
+    const auto& scenario_info = it->second;
+    tag = scenario_info.first;
+    for (const auto& arg : scenario_info.second) {
+      args.push_back(arg);
+    }
+  }
+  if (scenario == 9) {
+    std::stringstream hexagon_lib_path;
+    hexagon_lib_path << "--hexagon_lib_path=" << library_dir;
+    args.push_back(hexagon_lib_path.str());
+  }
+
+  if (scenario == 11 || scenario == 12) {
+    if (CreateDir(kSerializeDir)) {
+      std::stringstream serialize_dir;
+      serialize_dir << "--delegate_serialize_dir=" << kSerializeDir;
+      args.push_back(serialize_dir.str());
+    } else {
+#ifdef __ANDROID__
+      __android_log_print(ANDROID_LOG_ERROR, "tflite",
+                          "Failed to create serialize directory %s.",
+                          kSerializeDir);
+#else
+      fprintf(stderr, "Failed to create serialize directory %s.",
+              kSerializeDir);
+#endif
+    }
+  }
+  return tag;
+}
+
+void RunScenario(const std::string& library_dir, int scenario, int report_fd) {
+  std::vector<std::string> args;
+  std::string tag = GetScenarioConfig(library_dir, scenario, args);
+  std::vector<char*> argv;
+  argv.reserve(args.size());
+  for (auto& arg : args) {
+    argv.push_back(const_cast<char*>(arg.data()));
+  }
+
+  BenchmarkTfLiteModel benchmark;
+  FirebaseReportingListener firebaseReporting(tag, report_fd);
+  benchmark.AddListener(&firebaseReporting);
+  CsvExportingListener csvExporting(tag);
+  benchmark.AddListener(&csvExporting);
+  auto status = benchmark.Run(static_cast<int>(argv.size()), argv.data());
+  if (status != kTfLiteOk) {
+    firebaseReporting.ReportFailure(status);
+  }
+}
+
+}  // namespace
+}  // namespace benchmark
+}  // namespace tflite
+
+extern "C" {
+
+JNIEXPORT void JNICALL
+Java_org_tensorflow_lite_benchmark_firebase_BenchmarkModel_nativeRun(
+    JNIEnv* env, jclass clazz, jstring library_dir, jint scenario,
+    jint report_fd) {
+  const char* lib_dir = env->GetStringUTFChars(library_dir, nullptr);
+
+  tflite::benchmark::RunScenario(lib_dir, static_cast<int>(scenario),
+                                 static_cast<int>(report_fd));
+
+  env->ReleaseStringUTFChars(library_dir, lib_dir);
+}
+
+}  // extern "C"

tensorflow/lite/tools/benchmark/experimental/firebase/android/src/org/tensorflow/lite/benchmark/firebase/BenchmarkModel.java

@@ -0,0 +1,34 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.lite.benchmark.firebase;
+
+import android.content.Context;
+
+/** Helper class for running a native TensorFlow Lite benchmark. */
+class BenchmarkModel {
+  static {
+    System.loadLibrary("tensorflowlite_benchmark_firebase");
+  }
+
+  // Executes a standard TensorFlow Lite benchmark with predefined args for each scenario.
+  // Result and status will be reported to the file with reportFd file descriptor.
+  public static void run(Context context, int scenario, int reportFd) {
+    String libraryDir = context.getApplicationInfo().nativeLibraryDir;
+    nativeRun(libraryDir, scenario, reportFd);
+  }
+
+  private static native void nativeRun(String libraryDir, int scenario, int reportFd);
+}

tensorflow/lite/tools/benchmark/experimental/firebase/android/src/org/tensorflow/lite/benchmark/firebase/BenchmarkModelActivity.java

@@ -0,0 +1,74 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.lite.benchmark.firebase;
+
+import android.app.Activity;
+import android.content.Intent;
+import android.net.Uri;
+import android.os.Bundle;
+import android.os.ParcelFileDescriptor;
+import android.util.Log;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+/**
+ * {@code Activity} class for Firebase Game Loop test.
+ *
+ * <p>This Activity receives and handles an {@code Intent} for Firebase Game Loop test. Refer to
+ * https://firebase.google.com/docs/test-lab/android/game-loop.
+ */
+public class BenchmarkModelActivity extends Activity {
+
+  private static final String TAG = "tflite_BenchmarkModelActivity";
+
+  @Override
+  public void onCreate(Bundle savedInstanceState) {
+    super.onCreate(savedInstanceState);
+
+    Intent intent = getIntent();
+    if (!intent.getAction().equals("com.google.intent.action.TEST_LOOP")) {
+      Log.e(TAG, "Received non Firebase Game Loop test intent " + intent.getAction());
+      finish();
+    }
+    int scenario = intent.getIntExtra("scenario", 0);
+    Log.i(TAG, "Running TensorFlow Lite benchmark with scenario: " + scenario);
+
+    ParcelFileDescriptor parcelFileDescriptor = null;
+    Uri reportFile = intent.getData();
+    if (reportFile != null) {
+      Log.i(TAG, "Logging the result to " + reportFile.getEncodedPath());
+      try {
+        parcelFileDescriptor =
+            getContentResolver().openAssetFileDescriptor(reportFile, "w").getParcelFileDescriptor();
+      } catch (FileNotFoundException | NullPointerException e) {
+        Log.e(TAG, "Error while opening Firebase Test Lab report file", e);
+      }
+    }
+
+    int reportFd = parcelFileDescriptor != null ? parcelFileDescriptor.getFd() : -1;
+    BenchmarkModel.run(this, scenario, reportFd);
+
+    if (parcelFileDescriptor != null) {
+      try {
+        parcelFileDescriptor.close();
+      } catch (IOException e) {
+        Log.e(TAG, "Failed to close Firebase Test Lab result file", e);
+      }
+    }
+
+    finish();
+  }
+}

tensorflow/lite/tools/benchmark/experimental/ios/BUILD.apple

@@ -0,0 +1,45 @@
+load("@bazel_skylib//rules:build_test.bzl", "build_test")
+load("@build_bazel_rules_apple//apple:ios.bzl", "ios_static_framework")
+load("//tensorflow/lite/ios:ios.bzl", "TFL_MINIMUM_OS_VERSION", "strip_common_include_path_prefix")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
+    licenses = ["notice"],
+)
+
+strip_common_include_path_prefix(
+    name = "strip_common_include_path_benchmark",
+    hdr_labels = [
+        "//tensorflow/lite/core/c:c_api_types.h",
+        "//tensorflow/lite/tools:logging.h",
+        "//tensorflow/lite/tools/benchmark/experimental/c:benchmark_c_api.h",
+    ],
+)
+
+# Main target for the benchmark tool iOS framework.
+# bazel build --config=ios_fat -c opt //tensorflow/lite/tools/benchmark/experimental/ios:TensorFlowLiteBenchmarkC_framework
+ios_static_framework(
+    name = "TensorFlowLiteBenchmarkC_framework",
+    hdrs = [
+        ":benchmark_c_api.h",
+        ":c_api_types.h",
+        ":logging.h",
+    ],
+    bundle_name = "TensorFlowLiteBenchmarkC",
+    minimum_os_version = TFL_MINIMUM_OS_VERSION,
+    deps = [
+        "//tensorflow/lite/tools/benchmark/experimental/c:benchmark_c_api",
+    ],
+)
+
+# Used for building TensorFlowLiteBenchmarkC_framework framework.
+build_test(
+    name = "framework_build_test",
+    tags = [
+        "nomsan",  # b/145205324
+        "notsan",  # b/145205324
+    ],
+    targets = [
+        ":TensorFlowLiteBenchmarkC_framework",
+    ],
+)