chore: import upstream snapshot with attribution

tensorflow/lite/tools/evaluation/tasks/BUILD

@@ -0,0 +1,61 @@
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
+load("//tensorflow/lite:build_def.bzl", "tflite_copts")
+load("//tensorflow/lite/tools/evaluation/tasks:build_def.bzl", "task_linkopts")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_visibility = [
+        "//visibility:public",
+    ],
+    licenses = ["notice"],
+)
+
+exports_files(
+    ["task_executor_c_api.h"],
+    visibility = ["//tensorflow/lite/tools/evaluation/tasks:__subpackages__"],
+)
+
+cc_library(
+    name = "task_executor",
+    srcs = ["task_executor.cc"],
+    hdrs = ["task_executor.h"],
+    copts = tflite_copts(),
+    linkopts = task_linkopts(),
+    deps = [
+        "//tensorflow/lite/tools:command_line_flags",
+        "//tensorflow/lite/tools:logging",
+        "//tensorflow/lite/tools/evaluation:evaluation_delegate_provider",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
+        "@com_google_absl//absl/types:optional",
+    ],
+)
+
+cc_library(
+    name = "task_executor_main",
+    srcs = ["task_executor_main.cc"],
+    copts = tflite_copts(),
+    linkopts = task_linkopts(),
+    deps = [
+        ":task_executor",
+        "//tensorflow/lite/tools:logging",
+        "@com_google_absl//absl/types:optional",
+    ],
+)
+
+cc_library(
+    name = "task_executor_c_api",
+    srcs = ["task_executor_c_api.cc"],
+    hdrs = [
+        "task_executor_c_api.h",
+    ],
+    copts = tflite_copts(),
+    visibility = [
+        "//tensorflow/lite/tools/evaluation/tasks:__subpackages__",
+    ],
+    deps = [
+        ":task_executor",
+        "//tensorflow/lite/tools:logging",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
+    ],
+)

tensorflow/lite/tools/evaluation/tasks/README.md

@@ -0,0 +1,45 @@
+# TFLite Model Task Evaluation
+
+This page describes how you can check the accuracy of quantized models to verify
+that any degradation in accuracy is within acceptable limits.
+
+## Accuracy & correctness
+
+TensorFlow Lite has two types of tooling to measure how accurately a delegate
+behaves for a given model: Task-Based and Task-Agnostic.
+
+**Task-Based Evaluation** TFLite has two tools to evaluate correctness on two
+image-based tasks: - [ILSVRC 2012](http://image-net.org/challenges/LSVRC/2012/)
+(Image Classification) with top-K accuracy -
+[COCO Object Detection](https://cocodataset.org/#detection-2020) (w/ bounding
+boxes) with mean Average Precision (mAP)
+
+**Task-Agnostic Evaluation** For tasks where there isn't an established
+on-device evaluation tool, or if you are experimenting with custom models,
+TensorFlow Lite has the Inference Diff tool.
+
+## Tools
+
+There are three different binaries which are supported. A brief description of
+each is provided below.
+
+### [Inference Diff Tool](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/evaluation/tasks/inference_diff#inference-diff-tool)
+
+This binary compares TensorFlow Lite execution in single-threaded CPU inference
+and user-defined inference.
+
+### [Image Classification Evaluation](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/evaluation/tasks/imagenet_image_classification#image-classification-evaluation-based-on-ilsvrc-2012-task)
+
+This binary evaluates TensorFlow Lite models trained for the
+[ILSVRC 2012 image classification task.](http://www.image-net.org/challenges/LSVRC/2012/)
+
+### [Object Detection Evaluation](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/evaluation/tasks/coco_object_detection#object-detection-evaluation-using-the-2014-coco-minival-dataset)
+
+This binary evaluates TensorFlow Lite models trained for the bounding box-based
+[COCO Object Detection](https://cocodataset.org/#detection-eval) task.
+
+********************************************************************************
+
+For more information visit the TensorFlow Lite guide on
+[Accuracy & correctness](https://www.tensorflow.org/lite/performance/delegates#accuracy_correctness)
+page.

tensorflow/lite/tools/evaluation/tasks/build_def.bzl

@@ -0,0 +1,14 @@
+"""Common BUILD-related definitions across different tasks"""
+
+load("//tensorflow/lite:build_def.bzl", "tflite_linkopts")
+
+def task_linkopts():
+    return tflite_linkopts() + select({
+        "//tensorflow:android": [
+            "-pie",  # Android 5.0 and later supports only PIE
+            "-lm",  # some builtin ops, e.g., tanh, need -lm
+            # Hexagon delegate libraries should be in /data/local/tmp
+            "-Wl,--rpath=/data/local/tmp/",
+        ],
+        "//conditions:default": [],
+    })

tensorflow/lite/tools/evaluation/tasks/coco_object_detection/BUILD

@@ -0,0 +1,56 @@
+load("@rules_cc//cc:cc_binary.bzl", "cc_binary")
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
+load("@xla//third_party/rules_python/python:py_binary.bzl", "py_binary")
+load("//tensorflow/lite:build_def.bzl", "tflite_copts")
+load("//tensorflow/lite/tools/evaluation/tasks:build_def.bzl", "task_linkopts")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_visibility = [
+        "//visibility:public",
+    ],
+    licenses = ["notice"],
+)
+
+py_binary(
+    name = "preprocess_coco_minival",
+    srcs = ["preprocess_coco_minival.py"],
+    strict_deps = True,
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_py",
+        "@absl_py//absl/logging",
+    ],
+)
+
+cc_library(
+    name = "run_eval_lib",
+    srcs = ["run_eval.cc"],
+    copts = tflite_copts(),
+    linkopts = task_linkopts(),
+    deps = [
+        "//tensorflow/lite/core/c:common",
+        "//tensorflow/lite/tools:command_line_flags",
+        "//tensorflow/lite/tools:logging",
+        "//tensorflow/lite/tools/evaluation:evaluation_delegate_provider",
+        "//tensorflow/lite/tools/evaluation:evaluation_stage",
+        "//tensorflow/lite/tools/evaluation:utils",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
+        "//tensorflow/lite/tools/evaluation/stages:object_detection_stage",
+        "//tensorflow/lite/tools/evaluation/tasks:task_executor",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/types:optional",
+    ],
+    alwayslink = 1,
+)
+
+cc_binary(
+    name = "run_eval",
+    copts = tflite_copts(),
+    linkopts = task_linkopts(),
+    deps = [
+        ":run_eval_lib",
+        "//tensorflow/lite/tools/evaluation/tasks:task_executor_main",
+    ],
+)

tensorflow/lite/tools/evaluation/tasks/coco_object_detection/README.md

@@ -0,0 +1,264 @@
+# Object Detection evaluation using the 2014 COCO minival dataset.
+
+This binary evaluates the following parameters of TFLite models trained for the
+**bounding box-based**
+[COCO Object Detection](http://cocodataset.org/#detection-eval) task:
+
+*   Native pre-processing latency
+*   Inference latency
+*   mean Average Precision (mAP) averaged across IoU thresholds from 0.5 to 0.95
+    (in increments of 0.05) and all object categories.
+
+The binary takes the path to validation images and a ground truth proto file as
+inputs, along with the model and inference-specific parameters such as delegate
+and number of threads. It outputs the metrics to std-out as follows:
+
+```
+Num evaluation runs: 8059
+Preprocessing latency: avg=16589.9(us), std_dev=0(us)
+Inference latency: avg=85169.7(us), std_dev=505(us)
+Average Precision [IOU Threshold=0.5]: 0.349581
+Average Precision [IOU Threshold=0.55]: 0.330213
+Average Precision [IOU Threshold=0.6]: 0.307694
+Average Precision [IOU Threshold=0.65]: 0.281025
+Average Precision [IOU Threshold=0.7]: 0.248507
+Average Precision [IOU Threshold=0.75]: 0.210295
+Average Precision [IOU Threshold=0.8]: 0.165011
+Average Precision [IOU Threshold=0.85]: 0.116215
+Average Precision [IOU Threshold=0.9]: 0.0507883
+Average Precision [IOU Threshold=0.95]: 0.0064338
+Overall mAP: 0.206576
+```
+
+To run the binary, please follow the
+[Preprocessing section](#preprocessing-the-minival-dataset) to prepare the data,
+and then execute the commands in the
+[Running the binary section](#running-the-binary).
+
+## Parameters
+
+The binary takes the following parameters:
+
+*   `model_file` : `string` \
+    Path to the TFlite model file. It should accept images preprocessed in the
+    Inception format, and the output signature should be similar to the
+    [SSD MobileNet model](https://www.tensorflow.org/lite/examples/object_detection/overview#output_signature.):
+
+*   `model_output_labels`: `string` \
+    Path to labels that correspond to output of model. E.g. in case of
+    COCO-trained SSD model, this is the path to a file where each line contains
+    a class detected by the model in correct order, starting from 'background'.
+
+A sample model & label-list combination for COCO can be downloaded from the
+TFLite
+[Hosted models page](https://www.tensorflow.org/lite/guide/hosted_models#object_detection).
+
+*   `ground_truth_images_path`: `string` \
+    The path to the directory containing ground truth images.
+
+*   `ground_truth_proto`: `string` \
+    Path to file containing tflite::evaluation::ObjectDetectionGroundTruth proto
+    in text format. If left empty, mAP numbers are not provided.
+
+The above two parameters can be prepared using the `preprocess_coco_minival`
+script included in this folder.
+
+*   `output_file_path`: `string` \
+    The final metrics are dumped into `output_file_path` as a string-serialized
+    instance of `tflite::evaluation::EvaluationStageMetrics`.
+
+The following optional parameters can be used to modify the inference runtime:
+
+*   `num_interpreter_threads`: `int` (default=1) \
+    This modifies the number of threads used by the TFLite Interpreter for
+    inference.
+
+*   `delegate`: `string` \
+    If provided, tries to use the specified delegate for accuracy evaluation.
+    Valid values: "nnapi", "gpu", "hexagon".
+
+    NOTE: Please refer to the
+    [Hexagon delegate documentation](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/performance/hexagon_delegate.md)
+    for instructions on how to set it up for the Hexagon delegate. The tool
+    assumes that `libhexagon_interface.so` and Qualcomm libraries lie in
+    `/data/local/tmp`.
+
+This script also supports runtime/delegate arguments introduced by the
+[delegate registrar](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/delegates).
+If there is any conflict (for example, `num_threads` vs
+`num_interpreter_threads` here), the parameters of this
+script are given precedence.
+
+When **multiple delegates** are specified to be used in the commandline flags
+via the support of delegate registrar, the order of delegates applied to the
+TfLite runtime will be same as their enabling commandline flag is specified. For
+example, "--use_xnnpack=true --use_gpu=true" means applying the XNNPACK delegate
+first, and then the GPU delegate secondly. In comparison,
+"--use_gpu=true --use_xnnpack=true" means applying the GPU delegate first, and
+then the XNNPACK delegate secondly.
+
+Note, one could specify `--help` when launching the binary to see the full list
+of supported arguments.
+
+### Debug Mode
+
+The script also supports a debug mode with the following parameter:
+
+*   `debug_mode`: `boolean` \
+    Whether to enable debug mode. Per-image predictions are written to std-out
+    along with metrics.
+
+Image-wise predictions are output as follows:
+
+```
+======================================================
+
+Image: image_1.jpg
+
+Object [0]
+  Score: 0.585938
+  Class-ID: 5
+  Bounding Box:
+    Normalized Top: 0.23103
+    Normalized Bottom: 0.388524
+    Normalized Left: 0.559144
+    Normalized Right: 0.763928
+Object [1]
+  Score: 0.574219
+  Class-ID: 5
+  Bounding Box:
+    Normalized Top: 0.269571
+    Normalized Bottom: 0.373971
+    Normalized Left: 0.613175
+    Normalized Right: 0.760507
+======================================================
+
+Image: image_2.jpg
+...
+```
+
+This mode lets you debug the output of an object detection model that isn't
+necessarily trained on the COCO dataset (by leaving `ground_truth_proto` empty).
+The model output signature would still need to follow the convention mentioned
+above, and you we still need an output labels file.
+
+## Preprocessing the minival dataset
+
+To compute mAP in a consistent and interpretable way, we utilize the same 2014
+COCO 'minival' dataset that is mentioned in the
+[Tensorflow detection model zoo](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf1_detection_zoo.md).
+
+The links to download the components of the validation set are:
+
+*   [2014 COCO Validation Images](http://images.cocodataset.org/zips/val2014.zip)
+*   [2014 COCO Train/Val annotations](http://images.cocodataset.org/annotations/annotations_trainval2014.zip):
+    Out of the files from this zip, we only require `instances_val2014.json`.
+*   [minival Image IDs](https://github.com/tensorflow/models/blob/master/research/object_detection/data/mscoco_minival_ids.txt) :
+    Only applies to the 2014 validation set. You would need to copy the contents
+    into a text file.
+
+Since evaluation has to be performed on-device, we first filter the above data
+and extract a subset that only contains the images & ground-truth bounding boxes
+we need.
+
+To do so, we utilize the `preprocess_coco_minival` Python binary as follows:
+
+```
+bazel run //tensorflow/lite/tools/evaluation/tasks/coco_object_detection:preprocess_coco_minival -- \
+  --images_folder=/path/to/val2014 \
+  --instances_file=/path/to/instances_val2014.json \
+  --allowlist_file=/path/to/minival_allowlist.txt \
+  --output_folder=/path/to/output/folder
+
+```
+
+Optionally, you can specify a `--num_images=N` argument, to preprocess the first
+`N` image files (based on sorted list of filenames).
+
+The script generates the following within the output folder:
+
+*   `images/`: the resulting subset of the 2014 COCO Validation images.
+
+*   `ground_truth.pb`: a `.pb` (binary-format proto) file holding
+    `tflite::evaluation::ObjectDetectionGroundTruth` corresponding to image
+    subset.
+
+## Running the binary
+
+### On Android
+
+(0) Refer to
+https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android
+for configuring NDK and SDK.
+
+(1) Build using the following command:
+
+```
+bazel build -c opt \
+  --config=android_arm64 \
+  --cxxopt='--std=c++17' \
+  //tensorflow/lite/tools/evaluation/tasks/coco_object_detection:run_eval
+```
+
+(2) Connect your phone. Push the binary to your phone with adb push (make the
+directory if required):
+
+```
+adb push bazel-bin/third_party/tensorflow/lite/tools/evaluation/tasks/coco_object_detection/run_eval /data/local/tmp
+```
+
+(3) Make the binary executable.
+
+```
+adb shell chmod +x /data/local/tmp/run_eval
+```
+
+(4) Push the TFLite model that you need to test:
+
+```
+adb push ssd_mobilenet_v1_float.tflite /data/local/tmp
+```
+
+(5) Push the model labels text file to device.
+
+```
+adb push /path/to/labelmap.txt /data/local/tmp/labelmap.txt
+```
+
+(6) Preprocess the dataset using the instructions given in the
+[Preprocessing section](#preprocessing-the-minival-dataset) and push the data
+(folder containing images & ground truth proto) to the device:
+
+```
+adb shell mkdir /data/local/tmp/coco_validation && \
+adb push /path/to/output/folder /data/local/tmp/coco_validation
+```
+
+(7) Run the binary.
+
+```
+adb shell /data/local/tmp/run_eval \
+  --model_file=/data/local/tmp/ssd_mobilenet_v1_float.tflite \
+  --ground_truth_images_path=/data/local/tmp/coco_validation/images \
+  --ground_truth_proto=/data/local/tmp/coco_validation/ground_truth.pb \
+  --model_output_labels=/data/local/tmp/labelmap.txt \
+  --output_file_path=/data/local/tmp/coco_output.txt
+```
+
+Optionally, you could also pass in the `--num_interpreter_threads` &
+`--delegate` arguments to run with different configurations.
+
+### On Desktop
+
+(1) Build and run using the following command:
+
+```
+bazel run -c opt \
+  -- \
+  //tensorflow/lite/tools/evaluation/tasks/coco_object_detection:run_eval \
+  --model_file=/path/to/ssd_mobilenet_v1_float.tflite \
+  --ground_truth_images_path=/path/to/images \
+  --ground_truth_proto=/path/to/ground_truth.pb \
+  --model_output_labels=/path/to/labelmap.txt \
+  --output_file_path=/path/to/coco_output.txt
+```

tensorflow/lite/tools/evaluation/tasks/coco_object_detection/preprocess_coco_minival.py

@@ -0,0 +1,227 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Preprocesses COCO minival data for Object Detection evaluation using mean Average Precision.
+
+The 2014 validation images & annotations can be downloaded from:
+http://cocodataset.org/#download
+The minival image ID allowlist, a subset of the 2014 validation set, can be
+found here:
+https://github.com/tensorflow/models/blob/master/research/object_detection/data/mscoco_minival_ids.txt.
+
+This script takes in the original images folder, instances JSON file and
+image ID allowlist and produces the following in the specified output folder:
+A subfolder for allowlisted images (images/), and a file (ground_truth.pbtxt)
+containing an instance of tflite::evaluation::ObjectDetectionGroundTruth.
+"""
+
+import argparse
+import ast
+import collections
+import os
+import shutil
+import sys
+
+from absl import logging
+from tensorflow.lite.tools.evaluation.proto import evaluation_stages_pb2
+
+
+def _get_ground_truth_detections(instances_file,
+                                 allowlist_file=None,
+                                 num_images=None):
+  """Processes the annotations JSON file and returns ground truth data corresponding to allowlisted image IDs.
+
+  Args:
+    instances_file: COCO instances JSON file, usually named as
+      instances_val20xx.json.
+    allowlist_file: File containing COCO minival image IDs to allowlist for
+      evaluation, one per line.
+    num_images: Number of allowlisted images to pre-process. First num_images
+      are chosen based on sorted list of filenames. If None, all allowlisted
+      files are preprocessed.
+
+  Returns:
+    A dict mapping image id (int) to a per-image dict that contains:
+      'filename', 'image' & 'height' mapped to filename & image dimensions
+      respectively
+      AND
+      'detections' to a list of detection dicts, with each mapping:
+        'category_id' to COCO category id (starting with 1) &
+        'bbox' to a list of dimension-normalized [top, left, bottom, right]
+        bounding-box values.
+  """
+  # Read JSON data into a dict.
+  with open(instances_file, 'r') as annotation_dump:
+    data_dict = ast.literal_eval(annotation_dump.readline())
+
+  image_data = collections.OrderedDict()
+
+  # Read allowlist.
+  if allowlist_file is not None:
+    with open(allowlist_file, 'r') as allowlist:
+      image_id_allowlist = set([int(x) for x in allowlist.readlines()])
+  else:
+    image_id_allowlist = [image['id'] for image in data_dict['images']]
+
+  # Get image names and dimensions.
+  for image_dict in data_dict['images']:
+    image_id = image_dict['id']
+    if image_id not in image_id_allowlist:
+      continue
+    image_data_dict = {}
+    image_data_dict['id'] = image_dict['id']
+    image_data_dict['file_name'] = image_dict['file_name']
+    image_data_dict['height'] = image_dict['height']
+    image_data_dict['width'] = image_dict['width']
+    image_data_dict['detections'] = []
+    image_data[image_id] = image_data_dict
+
+  shared_image_ids = set()
+  for annotation_dict in data_dict['annotations']:
+    image_id = annotation_dict['image_id']
+    if image_id in image_data:
+      shared_image_ids.add(image_id)
+
+  output_image_ids = sorted(shared_image_ids)
+  if num_images:
+    if num_images <= 0:
+      logging.warning(
+          '--num_images is %d, hence outputing all annotated images.',
+          num_images)
+    elif num_images > len(shared_image_ids):
+      logging.warning(
+          '--num_images (%d) is larger than the number of annotated images.',
+          num_images)
+    else:
+      output_image_ids = output_image_ids[:num_images]
+
+  for image_id in list(image_data):
+    if image_id not in output_image_ids:
+      del image_data[image_id]
+
+  # Get detected object annotations per image.
+  for annotation_dict in data_dict['annotations']:
+    image_id = annotation_dict['image_id']
+    if image_id not in output_image_ids:
+      continue
+
+    image_data_dict = image_data[image_id]
+    bbox = annotation_dict['bbox']
+    # bbox format is [x, y, width, height]
+    # Refer: http://cocodataset.org/#format-data
+    top = bbox[1]
+    left = bbox[0]
+    bottom = top + bbox[3]
+    right = left + bbox[2]
+    if (top > image_data_dict['height'] or left > image_data_dict['width'] or
+        bottom > image_data_dict['height'] or right > image_data_dict['width']):
+      continue
+    object_d = {}
+    object_d['bbox'] = [
+        top / image_data_dict['height'], left / image_data_dict['width'],
+        bottom / image_data_dict['height'], right / image_data_dict['width']
+    ]
+    object_d['category_id'] = annotation_dict['category_id']
+    image_data_dict['detections'].append(object_d)
+
+  return image_data
+
+
+def _dump_data(ground_truth_detections, images_folder_path, output_folder_path):
+  """Dumps images & data from ground-truth objects into output_folder_path.
+
+  The following are created in output_folder_path:
+    images/: sub-folder for allowlisted validation images.
+    ground_truth.pb: A binary proto file containing all ground-truth
+    object-sets.
+
+  Args:
+    ground_truth_detections: A dict mapping image id to ground truth data.
+      Output of _get_ground_truth_detections.
+    images_folder_path: Validation images folder
+    output_folder_path: folder to output files to.
+  """
+  # Ensure output folders exist.
+  if not os.path.exists(output_folder_path):
+    os.makedirs(output_folder_path)
+  output_images_folder = os.path.join(output_folder_path, 'images')
+  if not os.path.exists(output_images_folder):
+    os.makedirs(output_images_folder)
+  output_proto_file = os.path.join(output_folder_path, 'ground_truth.pb')
+
+  ground_truth_data = evaluation_stages_pb2.ObjectDetectionGroundTruth()
+  for image_dict in ground_truth_detections.values():
+    # Create an ObjectsSet proto for this file's ground truth.
+    detection_result = ground_truth_data.detection_results.add()
+    detection_result.image_id = image_dict['id']
+    detection_result.image_name = image_dict['file_name']
+    for detection_dict in image_dict['detections']:
+      object_instance = detection_result.objects.add()
+      object_instance.bounding_box.normalized_top = detection_dict['bbox'][0]
+      object_instance.bounding_box.normalized_left = detection_dict['bbox'][1]
+      object_instance.bounding_box.normalized_bottom = detection_dict['bbox'][2]
+      object_instance.bounding_box.normalized_right = detection_dict['bbox'][3]
+      object_instance.class_id = detection_dict['category_id']
+    # Copy image.
+    shutil.copy2(
+        os.path.join(images_folder_path, image_dict['file_name']),
+        output_images_folder)
+
+  # Dump proto.
+  with open(output_proto_file, 'wb') as proto_file:
+    proto_file.write(ground_truth_data.SerializeToString())
+
+
+def _parse_args():
+  """Creates a parser that parse the command line arguments.
+
+  Returns:
+    A namespace parsed from command line arguments.
+  """
+  parser = argparse.ArgumentParser(
+      description='preprocess_coco_minival: Preprocess COCO minival dataset')
+  parser.add_argument(
+      '--images_folder',
+      type=str,
+      help='Full path of the validation images folder.',
+      required=True)
+  parser.add_argument(
+      '--instances_file',
+      type=str,
+      help='Full path of the input JSON file, like instances_val20xx.json.',
+      required=True)
+  parser.add_argument(
+      '--allowlist_file',
+      type=str,
+      help='File with COCO image ids to preprocess, one on each line.',
+      required=False)
+  parser.add_argument(
+      '--num_images',
+      type=int,
+      help='Number of allowlisted images to preprocess into the output folder.',
+      required=False)
+  parser.add_argument(
+      '--output_folder',
+      type=str,
+      help='Full path to output images & text proto files into.',
+      required=True)
+  return parser.parse_known_args(args=sys.argv[1:])[0]
+
+
+if __name__ == '__main__':
+  args = _parse_args()
+  ground_truths = _get_ground_truth_detections(args.instances_file,
+                                               args.allowlist_file,
+                                               args.num_images)
+  _dump_data(ground_truths, args.images_folder, args.output_folder)

tensorflow/lite/tools/evaluation/tasks/coco_object_detection/run_eval.cc

@@ -0,0 +1,232 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <fstream>
+#include <ios>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "absl/container/flat_hash_map.h"
+#include "tensorflow/lite/core/c/common.h"
+#include "tensorflow/lite/tools/command_line_flags.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_delegate_provider.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+#include "tensorflow/lite/tools/evaluation/stages/object_detection_stage.h"
+#include "tensorflow/lite/tools/evaluation/tasks/task_executor.h"
+#include "tensorflow/lite/tools/evaluation/utils.h"
+#include "tensorflow/lite/tools/logging.h"
+
+namespace tflite {
+namespace evaluation {
+
+constexpr char kModelFileFlag[] = "model_file";
+constexpr char kGroundTruthImagesPathFlag[] = "ground_truth_images_path";
+constexpr char kModelOutputLabelsFlag[] = "model_output_labels";
+constexpr char kOutputFilePathFlag[] = "output_file_path";
+constexpr char kGroundTruthProtoFileFlag[] = "ground_truth_proto";
+constexpr char kInterpreterThreadsFlag[] = "num_interpreter_threads";
+constexpr char kDebugModeFlag[] = "debug_mode";
+constexpr char kDelegateFlag[] = "delegate";
+
+std::string GetNameFromPath(const std::string& str) {
+  int pos = str.find_last_of("/\\");
+  if (pos == std::string::npos) return "";
+  return str.substr(pos + 1);
+}
+
+class CocoObjectDetection : public TaskExecutor {
+ public:
+  CocoObjectDetection() : debug_mode_(false), num_interpreter_threads_(1) {}
+  ~CocoObjectDetection() override {}
+
+ protected:
+  std::vector<Flag> GetFlags() final;
+
+  // If the run is successful, the latest metrics will be returned.
+  std::optional<EvaluationStageMetrics> RunImpl() final;
+
+ private:
+  void OutputResult(const EvaluationStageMetrics& latest_metrics) const;
+  std::string model_file_path_;
+  std::string model_output_labels_path_;
+  std::string ground_truth_images_path_;
+  std::string ground_truth_proto_file_;
+  std::string output_file_path_;
+  bool debug_mode_;
+  std::string delegate_;
+  int num_interpreter_threads_;
+};
+
+std::vector<Flag> CocoObjectDetection::GetFlags() {
+  std::vector<tflite::Flag> flag_list = {
+      tflite::Flag::CreateFlag(kModelFileFlag, &model_file_path_,
+                               "Path to test tflite model file."),
+      tflite::Flag::CreateFlag(
+          kModelOutputLabelsFlag, &model_output_labels_path_,
+          "Path to labels that correspond to output of model."
+          " E.g. in case of COCO-trained SSD model, this is the path to file "
+          "where each line contains a class detected by the model in correct "
+          "order, starting from background."),
+      tflite::Flag::CreateFlag(
+          kGroundTruthImagesPathFlag, &ground_truth_images_path_,
+          "Path to ground truth images. These will be evaluated in "
+          "alphabetical order of filenames"),
+      tflite::Flag::CreateFlag(kGroundTruthProtoFileFlag,
+                               &ground_truth_proto_file_,
+                               "Path to file containing "
+                               "tflite::evaluation::ObjectDetectionGroundTruth "
+                               "proto in binary serialized format. If left "
+                               "empty, mAP numbers are not output."),
+      tflite::Flag::CreateFlag(
+          kOutputFilePathFlag, &output_file_path_,
+          "File to output to. Contains only metrics proto if debug_mode is "
+          "off, and per-image predictions also otherwise."),
+      tflite::Flag::CreateFlag(kDebugModeFlag, &debug_mode_,
+                               "Whether to enable debug mode. Per-image "
+                               "predictions are written to the output file "
+                               "along with metrics."),
+      tflite::Flag::CreateFlag(
+          kInterpreterThreadsFlag, &num_interpreter_threads_,
+          "Number of interpreter threads to use for inference."),
+      tflite::Flag::CreateFlag(
+          kDelegateFlag, &delegate_,
+          "Delegate to use for inference, if available. "
+          "Must be one of {'nnapi', 'gpu', 'xnnpack', 'hexagon'}"),
+  };
+  return flag_list;
+}
+
+std::optional<EvaluationStageMetrics> CocoObjectDetection::RunImpl() {
+  // Process images in filename-sorted order.
+  std::vector<std::string> image_paths;
+  if (GetSortedFileNames(StripTrailingSlashes(ground_truth_images_path_),
+                         &image_paths) != kTfLiteOk) {
+    return std::nullopt;
+  }
+
+  std::vector<std::string> model_labels;
+  if (!ReadFileLines(model_output_labels_path_, &model_labels)) {
+    TFLITE_LOG(ERROR) << "Could not read model output labels file";
+    return std::nullopt;
+  }
+
+  EvaluationStageConfig eval_config;
+  eval_config.set_name("object_detection");
+  auto* detection_params =
+      eval_config.mutable_specification()->mutable_object_detection_params();
+  auto* inference_params = detection_params->mutable_inference_params();
+  inference_params->set_model_file_path(model_file_path_);
+  inference_params->set_num_threads(num_interpreter_threads_);
+  inference_params->set_delegate(ParseStringToDelegateType(delegate_));
+
+  // Get ground truth data.
+  absl::flat_hash_map<std::string, ObjectDetectionResult> ground_truth_map;
+  if (!ground_truth_proto_file_.empty()) {
+    PopulateGroundTruth(ground_truth_proto_file_, &ground_truth_map);
+  }
+
+  ObjectDetectionStage eval(eval_config);
+
+  eval.SetAllLabels(model_labels);
+  if (eval.Init(&delegate_providers_) != kTfLiteOk) return std::nullopt;
+
+  const int step = image_paths.size() / 100;
+  for (int i = 0; i < image_paths.size(); ++i) {
+    if (step > 1 && i % step == 0) {
+      TFLITE_LOG(INFO) << "Finished: " << i / step << "%";
+    }
+
+    const std::string image_name = GetNameFromPath(image_paths[i]);
+    eval.SetInputs(image_paths[i], ground_truth_map[image_name]);
+    if (eval.Run() != kTfLiteOk) return std::nullopt;
+
+    if (debug_mode_) {
+      ObjectDetectionResult prediction = *eval.GetLatestPrediction();
+      TFLITE_LOG(INFO) << "Image: " << image_name << "\n";
+      for (int i = 0; i < prediction.objects_size(); ++i) {
+        const auto& object = prediction.objects(i);
+        TFLITE_LOG(INFO) << "Object [" << i << "]";
+        TFLITE_LOG(INFO) << "  Score: " << object.score();
+        TFLITE_LOG(INFO) << "  Class-ID: " << object.class_id();
+        TFLITE_LOG(INFO) << "  Bounding Box:";
+        const auto& bounding_box = object.bounding_box();
+        TFLITE_LOG(INFO) << "    Normalized Top: "
+                         << bounding_box.normalized_top();
+        TFLITE_LOG(INFO) << "    Normalized Bottom: "
+                         << bounding_box.normalized_bottom();
+        TFLITE_LOG(INFO) << "    Normalized Left: "
+                         << bounding_box.normalized_left();
+        TFLITE_LOG(INFO) << "    Normalized Right: "
+                         << bounding_box.normalized_right();
+      }
+      TFLITE_LOG(INFO)
+          << "======================================================\n";
+    }
+  }
+
+  // Write metrics to file.
+  EvaluationStageMetrics latest_metrics = eval.LatestMetrics();
+  if (ground_truth_proto_file_.empty()) {
+    TFLITE_LOG(WARN) << "mAP metrics are meaningless w/o ground truth.";
+    latest_metrics.mutable_process_metrics()
+        ->mutable_object_detection_metrics()
+        ->clear_average_precision_metrics();
+  }
+
+  OutputResult(latest_metrics);
+  return std::make_optional(latest_metrics);
+}
+
+void CocoObjectDetection::OutputResult(
+    const EvaluationStageMetrics& latest_metrics) const {
+  if (!output_file_path_.empty()) {
+    std::ofstream metrics_ofile;
+    metrics_ofile.open(output_file_path_, std::ios::out);
+    metrics_ofile << latest_metrics.SerializeAsString();
+    metrics_ofile.close();
+  }
+  TFLITE_LOG(INFO) << "Num evaluation runs: " << latest_metrics.num_runs();
+  const auto object_detection_metrics =
+      latest_metrics.process_metrics().object_detection_metrics();
+  const auto& preprocessing_latency =
+      object_detection_metrics.pre_processing_latency();
+  TFLITE_LOG(INFO) << "Preprocessing latency: avg="
+                   << preprocessing_latency.avg_us() << "(us), std_dev="
+                   << preprocessing_latency.std_deviation_us() << "(us)";
+  const auto& inference_latency = object_detection_metrics.inference_latency();
+  TFLITE_LOG(INFO) << "Inference latency: avg=" << inference_latency.avg_us()
+                   << "(us), std_dev=" << inference_latency.std_deviation_us()
+                   << "(us)";
+  const auto& precision_metrics =
+      object_detection_metrics.average_precision_metrics();
+  for (int i = 0; i < precision_metrics.individual_average_precisions_size();
+       ++i) {
+    const auto ap_metric = precision_metrics.individual_average_precisions(i);
+    TFLITE_LOG(INFO) << "Average Precision [IOU Threshold="
+                     << ap_metric.iou_threshold()
+                     << "]: " << ap_metric.average_precision();
+  }
+  TFLITE_LOG(INFO) << "Overall mAP: "
+                   << precision_metrics.overall_mean_average_precision();
+}
+
+std::unique_ptr<TaskExecutor> CreateTaskExecutor() {
+  return std::unique_ptr<TaskExecutor>(new CocoObjectDetection());
+}
+
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/tasks/imagenet_image_classification/BUILD

@@ -0,0 +1,43 @@
+load("@rules_cc//cc:cc_binary.bzl", "cc_binary")
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
+load("//tensorflow/lite:build_def.bzl", "tflite_copts")
+load("//tensorflow/lite/tools/evaluation/tasks:build_def.bzl", "task_linkopts")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_visibility = [
+        "//visibility:public",
+    ],
+    licenses = ["notice"],
+)
+
+cc_library(
+    name = "run_eval_lib",
+    srcs = ["run_eval.cc"],
+    copts = tflite_copts(),
+    linkopts = task_linkopts(),
+    deps = [
+        "//tensorflow/lite/core/c:common",
+        "//tensorflow/lite/tools:command_line_flags",
+        "//tensorflow/lite/tools:logging",
+        "//tensorflow/lite/tools/evaluation:evaluation_delegate_provider",
+        "//tensorflow/lite/tools/evaluation:evaluation_stage",
+        "//tensorflow/lite/tools/evaluation:utils",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
+        "//tensorflow/lite/tools/evaluation/stages:image_classification_stage",
+        "//tensorflow/lite/tools/evaluation/tasks:task_executor",
+        "@com_google_absl//absl/types:optional",
+    ],
+    alwayslink = 1,
+)
+
+cc_binary(
+    name = "run_eval",
+    copts = tflite_copts(),
+    linkopts = task_linkopts(),
+    deps = [
+        ":run_eval_lib",
+        "//tensorflow/lite/tools/evaluation/tasks:task_executor_main",
+    ],
+)

tensorflow/lite/tools/evaluation/tasks/imagenet_image_classification/README.md

@@ -0,0 +1,216 @@
+## Image Classification evaluation based on ILSVRC 2012 task
+
+This binary evaluates the following parameters of TFLite models trained for the
+[ILSVRC 2012 image classification task](http://www.image-net.org/challenges/LSVRC/2012/):
+
+*   Native pre-processing latency
+*   Inference latency
+*   Top-K (1 to 10) accuracy values
+
+The binary takes the path to validation images and labels as inputs, along with
+the model and inference-specific parameters such as delegate and number of
+threads. It outputs the metrics to std-out as follows:
+
+```
+Num evaluation runs: 300 # Total images evaluated
+Preprocessing latency: avg=13772.5(us), std_dev=0(us)
+Inference latency: avg=76578.4(us), std_dev=600(us)
+Top-1 Accuracy: 0.733333
+Top-2 Accuracy: 0.826667
+Top-3 Accuracy: 0.856667
+Top-4 Accuracy: 0.87
+Top-5 Accuracy: 0.89
+Top-6 Accuracy: 0.903333
+Top-7 Accuracy: 0.906667
+Top-8 Accuracy: 0.913333
+Top-9 Accuracy: 0.92
+Top-10 Accuracy: 0.923333
+```
+
+To run the binary download the ILSVRC 2012 devkit
+[see instructions](#downloading-ilsvrc) and run the
+[`generate_validation_ground_truth` script](#ground-truth-label-generation) to
+generate the ground truth labels.
+
+## Parameters
+
+The binary takes the following parameters:
+
+*   `model_file` : `string` \
+    Path to the TFlite model file.
+
+*   `ground_truth_images_path`: `string` \
+    The path to the directory containing ground truth images.
+
+*   `ground_truth_labels`: `string` \
+    Path to ground truth labels file. This file should contain the same number
+    of labels as the number images in the ground truth directory. The labels are
+    assumed to be in the same order as the sorted filename of images. See
+    [ground truth label generation](#ground-truth-label-generation) section for
+    more information about how to generate labels for images.
+
+*   `model_output_labels`: `string` \
+    Path to the file containing labels, that is used to interpret the output of
+    the model. E.g. in case of mobilenets, this is the path to
+    `mobilenet_labels.txt` where each label is in the same order as the output
+    1001 dimension tensor.
+
+and the following optional parameters:
+
+*   `denylist_file_path`: `string` \
+    Path to denylist file. This file contains the indices of images that are
+    denylisted for evaluation. 1762 images are denylisted in ILSVRC dataset.
+    For details please refer to readme.txt of ILSVRC2014 devkit.
+
+*   `num_images`: `int` (default=0) \
+    The number of images to process, if 0, all images in the directory are
+    processed otherwise only num_images will be processed.
+
+*   `num_threads`: `int` (default=4) \
+    The number of threads to use for evaluation. Note: This does not change the
+    number of TFLite Interpreter threads, but shards the dataset to speed up
+    evaluation.
+
+*   `output_file_path`: `string` \
+    The final metrics are dumped into `output_file_path` as a string-serialized
+    instance of `tflite::evaluation::EvaluationStageMetrics`.
+
+The following optional parameters can be used to modify the inference runtime:
+
+*   `num_interpreter_threads`: `int` (default=1) \
+    This modifies the number of threads used by the TFLite Interpreter for
+    inference.
+
+*   `delegate`: `string` \
+    If provided, tries to use the specified delegate for accuracy evaluation.
+    Valid values: "nnapi", "gpu", "hexagon".
+
+    NOTE: Please refer to the
+    [Hexagon delegate documentation](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/performance/hexagon_delegate.md)
+    for instructions on how to set it up for the Hexagon delegate. The tool
+    assumes that `libhexagon_interface.so` and Qualcomm libraries lie in
+    `/data/local/tmp`.
+
+This script also supports runtime/delegate arguments introduced by the
+[delegate registrar](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/delegates).
+If there is any conflict (for example, `num_threads` vs
+`num_interpreter_threads` here), the parameters of this
+script are given precedence.
+
+When **multiple delegates** are specified to be used in the commandline flags
+via the support of delegate registrar, the order of delegates applied to the
+TfLite runtime will be same as their enabling commandline flag is specified. For
+example, "--use_xnnpack=true --use_gpu=true" means applying the XNNPACK delegate
+first, and then the GPU delegate secondly. In comparison,
+"--use_gpu=true --use_xnnpack=true" means applying the GPU delegate first, and
+then the XNNPACK delegate secondly.
+
+Note, one could specify `--help` when launching the binary to see the full list
+of supported arguments.
+
+## Downloading ILSVRC
+
+In order to use this tool to run evaluation on the full 50K ImageNet dataset,
+download the data set from http://image-net.org/request.
+
+## Ground truth label generation
+
+The ILSVRC 2012 devkit `validation_ground_truth.txt` contains IDs that
+correspond to synset of the image. The accuracy binary however expects the
+ground truth labels to contain the actual name of category instead of synset
+ids. A conversion script has been provided to convert the validation ground
+truth to category labels. The `validation_ground_truth.txt` can be converted by
+the following steps:
+
+```
+ILSVRC_2012_DEVKIT_DIR=[set to path to ILSVRC 2012 devkit]
+VALIDATION_LABELS=[set to  path to output]
+
+python tensorflow/lite/tools/evaluation/tasks/imagenet_image_classification/generate_validation_labels.py \
+--ilsvrc_devkit_dir=${ILSVRC_2012_DEVKIT_DIR} \
+--validation_labels_output=${VALIDATION_LABELS}
+```
+
+## Running the binary
+
+### On Android
+
+(0) Refer to
+https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android
+for configuring NDK and SDK.
+
+(1) Build using the following command:
+
+```
+bazel build -c opt \
+  --config=android_arm64 \
+  --cxxopt='--std=c++17' \
+  //tensorflow/lite/tools/evaluation/tasks/imagenet_image_classification:run_eval
+```
+
+(2) Connect your phone. Push the binary to your phone with adb push (make the
+directory if required):
+
+```
+adb push bazel-bin/tensorflow/lite/tools/evaluation/tasks/imagenet_image_classification/run_eval /data/local/tmp
+```
+
+(3) Make the binary executable.
+
+```
+adb shell chmod +x /data/local/tmp/run_eval
+```
+
+(4) Push the TFLite model that you need to test. For example:
+
+```
+adb push mobilenet_quant_v1_224.tflite /data/local/tmp
+```
+
+(5) Push the imagenet images to device, make sure device has sufficient storage
+available before pushing the dataset:
+
+```
+adb shell mkdir /data/local/tmp/ilsvrc_images && \
+adb push ${IMAGENET_IMAGES_DIR} /data/local/tmp/ilsvrc_images
+```
+
+(6) Push the generated validation ground labels to device.
+
+```
+adb push ${VALIDATION_LABELS} /data/local/tmp/ilsvrc_validation_labels.txt
+```
+
+(7) Push the model labels text file to device.
+
+```
+adb push ${MODEL_LABELS_TXT} /data/local/tmp/model_output_labels.txt
+```
+
+(8) Run the binary.
+
+```
+adb shell /data/local/tmp/run_eval \
+  --model_file=/data/local/tmp/mobilenet_quant_v1_224.tflite \
+  --ground_truth_images_path=/data/local/tmp/ilsvrc_images \
+  --ground_truth_labels=/data/local/tmp/ilsvrc_validation_labels.txt \
+  --model_output_labels=/data/local/tmp/model_output_labels.txt \
+  --output_file_path=/data/local/tmp/accuracy_output.txt \
+  --num_images=0 # Run on all images.
+```
+
+### On Desktop
+
+(1) Build and run using the following command:
+
+```
+bazel run -c opt \
+  -- \
+  //tensorflow/lite/tools/evaluation/tasks/imagenet_image_classification:run_eval \
+  --model_file=mobilenet_quant_v1_224.tflite \
+  --ground_truth_images_path=${IMAGENET_IMAGES_DIR} \
+  --ground_truth_labels=${VALIDATION_LABELS} \
+  --model_output_labels=${MODEL_LABELS_TXT} \
+  --output_file_path=/tmp/accuracy_output.txt \
+  --num_images=0 # Run on all images.
+```

tensorflow/lite/tools/evaluation/tasks/imagenet_image_classification/generate_validation_labels.py

@@ -0,0 +1,101 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tool to convert ILSVRC devkit validation ground truth to synset labels."""
+
+import argparse
+from os import path
+import sys
+import scipy.io
+
+_SYNSET_ARRAYS_RELATIVE_PATH = 'data/meta.mat'
+_VALIDATION_FILE_RELATIVE_PATH = 'data/ILSVRC2012_validation_ground_truth.txt'
+
+
+def _synset_to_word(filepath):
+  """Returns synset to word dictionary by reading sysnset arrays."""
+  mat = scipy.io.loadmat(filepath)
+  entries = mat['synsets']
+  # These fields are listed in devkit readme.txt
+  fields = [
+      'synset_id', 'WNID', 'words', 'gloss', 'num_children', 'children',
+      'wordnet_height', 'num_train_images'
+  ]
+  synset_index = fields.index('synset_id')
+  words_index = fields.index('words')
+  synset_to_word = {}
+  for entry in entries:
+    entry = entry[0]
+    synset_id = int(entry[synset_index][0])
+    first_word = entry[words_index][0].split(',')[0]
+    synset_to_word[synset_id] = first_word
+  return synset_to_word
+
+
+def _validation_file_path(ilsvrc_dir):
+  return path.join(ilsvrc_dir, _VALIDATION_FILE_RELATIVE_PATH)
+
+
+def _synset_array_path(ilsvrc_dir):
+  return path.join(ilsvrc_dir, _SYNSET_ARRAYS_RELATIVE_PATH)
+
+
+def _generate_validation_labels(ilsvrc_dir, output_file):
+  synset_to_word = _synset_to_word(_synset_array_path(ilsvrc_dir))
+  with open(_validation_file_path(ilsvrc_dir), 'r') as synset_id_file, open(
+      output_file, 'w') as output:
+    for synset_id in synset_id_file:
+      synset_id = int(synset_id)
+      output.write('%s\n' % synset_to_word[synset_id])
+
+
+def _check_arguments(args):
+  if not args.validation_labels_output:
+    raise ValueError('Invalid path to output file.')
+  ilsvrc_dir = args.ilsvrc_devkit_dir
+  if not ilsvrc_dir or not path.isdir(ilsvrc_dir):
+    raise ValueError('Invalid path to ilsvrc_dir')
+  if not path.exists(_validation_file_path(ilsvrc_dir)):
+    raise ValueError('Invalid path to ilsvrc_dir, cannot find validation file.')
+  if not path.exists(_synset_array_path(ilsvrc_dir)):
+    raise ValueError(
+        'Invalid path to ilsvrc_dir, cannot find synset arrays file.')
+
+
+def main():
+  parser = argparse.ArgumentParser(
+      description='Converts ILSVRC devkit validation_ground_truth.txt to synset'
+      ' labels file that can be used by the accuracy script.')
+  parser.add_argument(
+      '--validation_labels_output',
+      type=str,
+      help='Full path for outputting validation labels.')
+  parser.add_argument(
+      '--ilsvrc_devkit_dir',
+      type=str,
+      help='Full path to ILSVRC 2012 devkit directory.')
+  args = parser.parse_args()
+  try:
+    _check_arguments(args)
+  except ValueError as e:
+    parser.print_usage()
+    file_name = path.basename(sys.argv[0])
+    sys.stderr.write('{0}: error: {1}\n'.format(file_name, str(e)))
+    sys.exit(1)
+  _generate_validation_labels(args.ilsvrc_devkit_dir,
+                              args.validation_labels_output)
+
+
+if __name__ == '__main__':
+  main()

tensorflow/lite/tools/evaluation/tasks/imagenet_image_classification/run_eval.cc

@@ -0,0 +1,211 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <fstream>
+#include <ios>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "tensorflow/lite/core/c/common.h"
+#include "tensorflow/lite/tools/command_line_flags.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_delegate_provider.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+#include "tensorflow/lite/tools/evaluation/stages/image_classification_stage.h"
+#include "tensorflow/lite/tools/evaluation/tasks/task_executor.h"
+#include "tensorflow/lite/tools/evaluation/utils.h"
+#include "tensorflow/lite/tools/logging.h"
+
+namespace tflite {
+namespace evaluation {
+
+constexpr char kModelFileFlag[] = "model_file";
+constexpr char kGroundTruthImagesPathFlag[] = "ground_truth_images_path";
+constexpr char kGroundTruthLabelsFlag[] = "ground_truth_labels";
+constexpr char kOutputFilePathFlag[] = "output_file_path";
+constexpr char kModelOutputLabelsFlag[] = "model_output_labels";
+constexpr char kDenylistFilePathFlag[] = "denylist_file_path";
+constexpr char kNumImagesFlag[] = "num_images";
+constexpr char kInterpreterThreadsFlag[] = "num_interpreter_threads";
+constexpr char kDelegateFlag[] = "delegate";
+
+template <typename T>
+std::vector<T> GetFirstN(const std::vector<T>& v, int n) {
+  if (n >= v.size()) return v;
+  std::vector<T> result(v.begin(), v.begin() + n);
+  return result;
+}
+
+class ImagenetClassification : public TaskExecutor {
+ public:
+  ImagenetClassification() : num_images_(0), num_interpreter_threads_(1) {}
+  ~ImagenetClassification() override {}
+
+ protected:
+  std::vector<Flag> GetFlags() final;
+
+  // If the run is successful, the latest metrics will be returned.
+  std::optional<EvaluationStageMetrics> RunImpl() final;
+
+ private:
+  void OutputResult(const EvaluationStageMetrics& latest_metrics) const;
+  std::string model_file_path_;
+  std::string ground_truth_images_path_;
+  std::string ground_truth_labels_path_;
+  std::string model_output_labels_path_;
+  std::string denylist_file_path_;
+  std::string output_file_path_;
+  std::string delegate_;
+  int num_images_;
+  int num_interpreter_threads_;
+};
+
+std::vector<Flag> ImagenetClassification::GetFlags() {
+  std::vector<tflite::Flag> flag_list = {
+      tflite::Flag::CreateFlag(kModelFileFlag, &model_file_path_,
+                               "Path to test tflite model file."),
+      tflite::Flag::CreateFlag(
+          kModelOutputLabelsFlag, &model_output_labels_path_,
+          "Path to labels that correspond to output of model."
+          " E.g. in case of mobilenet, this is the path to label "
+          "file where each label is in the same order as the output"
+          " of the model."),
+      tflite::Flag::CreateFlag(
+          kGroundTruthImagesPathFlag, &ground_truth_images_path_,
+          "Path to ground truth images. These will be evaluated in "
+          "alphabetical order of filename"),
+      tflite::Flag::CreateFlag(
+          kGroundTruthLabelsFlag, &ground_truth_labels_path_,
+          "Path to ground truth labels, corresponding to alphabetical ordering "
+          "of ground truth images."),
+      tflite::Flag::CreateFlag(
+          kDenylistFilePathFlag, &denylist_file_path_,
+          "Path to denylist file (optional) where each line is a single "
+          "integer that is "
+          "equal to index number of denylisted image."),
+      tflite::Flag::CreateFlag(kOutputFilePathFlag, &output_file_path_,
+                               "File to output metrics proto to."),
+      tflite::Flag::CreateFlag(kNumImagesFlag, &num_images_,
+                               "Number of examples to evaluate, pass 0 for all "
+                               "examples. Default: 0"),
+      tflite::Flag::CreateFlag(
+          kInterpreterThreadsFlag, &num_interpreter_threads_,
+          "Number of interpreter threads to use for inference."),
+      tflite::Flag::CreateFlag(
+          kDelegateFlag, &delegate_,
+          "Delegate to use for inference, if available. "
+          "Must be one of {'nnapi', 'gpu', 'hexagon', 'xnnpack'}"),
+  };
+  return flag_list;
+}
+
+std::optional<EvaluationStageMetrics> ImagenetClassification::RunImpl() {
+  // Process images in filename-sorted order.
+  std::vector<std::string> image_files, ground_truth_image_labels;
+  if (GetSortedFileNames(StripTrailingSlashes(ground_truth_images_path_),
+                         &image_files) != kTfLiteOk) {
+    return std::nullopt;
+  }
+  if (!ReadFileLines(ground_truth_labels_path_, &ground_truth_image_labels)) {
+    TFLITE_LOG(ERROR) << "Could not read ground truth labels file";
+    return std::nullopt;
+  }
+  if (image_files.size() != ground_truth_image_labels.size()) {
+    TFLITE_LOG(ERROR) << "Number of images and ground truth labels is not same";
+    return std::nullopt;
+  }
+  std::vector<ImageLabel> image_labels;
+  image_labels.reserve(image_files.size());
+  for (int i = 0; i < image_files.size(); i++) {
+    image_labels.push_back({image_files[i], ground_truth_image_labels[i]});
+  }
+
+  // Filter out denylisted/unwanted images.
+  if (FilterDenyListedImages(denylist_file_path_, &image_labels) != kTfLiteOk) {
+    return std::nullopt;
+  }
+  if (num_images_ > 0) {
+    image_labels = GetFirstN(image_labels, num_images_);
+  }
+
+  std::vector<std::string> model_labels;
+  if (!ReadFileLines(model_output_labels_path_, &model_labels)) {
+    TFLITE_LOG(ERROR) << "Could not read model output labels file";
+    return std::nullopt;
+  }
+
+  EvaluationStageConfig eval_config;
+  eval_config.set_name("image_classification");
+  auto* classification_params = eval_config.mutable_specification()
+                                    ->mutable_image_classification_params();
+  auto* inference_params = classification_params->mutable_inference_params();
+  inference_params->set_model_file_path(model_file_path_);
+  inference_params->set_num_threads(num_interpreter_threads_);
+  inference_params->set_delegate(ParseStringToDelegateType(delegate_));
+  classification_params->mutable_topk_accuracy_eval_params()->set_k(10);
+
+  ImageClassificationStage eval(eval_config);
+
+  eval.SetAllLabels(model_labels);
+  if (eval.Init(&delegate_providers_) != kTfLiteOk) return std::nullopt;
+
+  const int step = image_labels.size() / 100;
+  for (int i = 0; i < image_labels.size(); ++i) {
+    if (step > 1 && i % step == 0) {
+      TFLITE_LOG(INFO) << "Evaluated: " << i / step << "%";
+    }
+    eval.SetInputs(image_labels[i].image, image_labels[i].label);
+    if (eval.Run() != kTfLiteOk) return std::nullopt;
+  }
+
+  const auto latest_metrics = eval.LatestMetrics();
+  OutputResult(latest_metrics);
+  return std::make_optional(latest_metrics);
+}
+
+void ImagenetClassification::OutputResult(
+    const EvaluationStageMetrics& latest_metrics) const {
+  if (!output_file_path_.empty()) {
+    std::ofstream metrics_ofile;
+    metrics_ofile.open(output_file_path_, std::ios::out);
+    metrics_ofile << latest_metrics.SerializeAsString();
+    metrics_ofile.close();
+  }
+
+  TFLITE_LOG(INFO) << "Num evaluation runs: " << latest_metrics.num_runs();
+  const auto& metrics =
+      latest_metrics.process_metrics().image_classification_metrics();
+  const auto& preprocessing_latency = metrics.pre_processing_latency();
+  TFLITE_LOG(INFO) << "Preprocessing latency: avg="
+                   << preprocessing_latency.avg_us() << "(us), std_dev="
+                   << preprocessing_latency.std_deviation_us() << "(us)";
+  const auto& inference_latency = metrics.inference_latency();
+  TFLITE_LOG(INFO) << "Inference latency: avg=" << inference_latency.avg_us()
+                   << "(us), std_dev=" << inference_latency.std_deviation_us()
+                   << "(us)";
+  const auto& accuracy_metrics = metrics.topk_accuracy_metrics();
+  for (int i = 0; i < accuracy_metrics.topk_accuracies_size(); ++i) {
+    TFLITE_LOG(INFO) << "Top-" << i + 1
+                     << " Accuracy: " << accuracy_metrics.topk_accuracies(i);
+  }
+}
+
+std::unique_ptr<TaskExecutor> CreateTaskExecutor() {
+  return std::unique_ptr<TaskExecutor>(new ImagenetClassification());
+}
+
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/tasks/inference_diff/BUILD

@@ -0,0 +1,42 @@
+load("@rules_cc//cc:cc_binary.bzl", "cc_binary")
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
+load("//tensorflow/lite:build_def.bzl", "tflite_copts")
+load("//tensorflow/lite/tools/evaluation/tasks:build_def.bzl", "task_linkopts")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_visibility = [
+        "//visibility:public",
+    ],
+    licenses = ["notice"],
+)
+
+cc_library(
+    name = "run_eval_lib",
+    srcs = ["run_eval.cc"],
+    copts = tflite_copts(),
+    linkopts = task_linkopts(),
+    deps = [
+        "//tensorflow/lite/core/c:common",
+        "//tensorflow/lite/tools:command_line_flags",
+        "//tensorflow/lite/tools:logging",
+        "//tensorflow/lite/tools/evaluation:evaluation_delegate_provider",
+        "//tensorflow/lite/tools/evaluation:evaluation_stage",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto",
+        "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto",
+        "//tensorflow/lite/tools/evaluation/stages:inference_profiler_stage",
+        "//tensorflow/lite/tools/evaluation/tasks:task_executor",
+        "@com_google_absl//absl/types:optional",
+    ],
+    alwayslink = 1,
+)
+
+cc_binary(
+    name = "run_eval",
+    copts = tflite_copts(),
+    linkopts = task_linkopts(),
+    deps = [
+        ":run_eval_lib",
+        "//tensorflow/lite/tools/evaluation/tasks:task_executor_main",
+    ],
+)

tensorflow/lite/tools/evaluation/tasks/inference_diff/README.md

@@ -0,0 +1,125 @@
+## Inference Diff tool
+
+**NOTE: This is an experimental tool to analyze TensorFlow Lite behavior on
+delegates.**
+
+For a given model, this binary compares TensorFlow Lite execution (in terms of
+latency & output-value deviation) in two settings:
+
+*   Single-threaded CPU Inference
+*   User-defined Inference
+
+To do so, the tool generates random gaussian data and passes it through two
+TFLite Interpreters - one running single-threaded CPU kernels and the other
+parameterized by the user's arguments.
+
+It measures the latency of both, as well as the absolute difference between the
+output tensors from each Interpreter, on a per-element basis.
+
+The final output (logged to stdout) typically looks like this:
+
+```
+Num evaluation runs: 50
+Reference run latency: avg=84364.2(us), std_dev=12525(us)
+Test run latency: avg=7281.64(us), std_dev=2089(us)
+OutputDiff[0]: avg_error=1.96277e-05, std_dev=6.95767e-06
+```
+
+There is one instance of `OutputDiff` for each output tensor in the model, and
+the statistics in `OutputDiff[i]` correspond to the absolute difference in raw
+values across all elements for the `i`th output.
+
+## Parameters
+
+(In this section, 'test Interpreter' refers to the User-defined Inference
+mentioned above. The reference setting is always single-threaded CPU).
+
+The binary takes the following parameters:
+
+*   `model_file` : `string` \
+    Path to the TFlite model file.
+
+and the following optional parameters:
+
+*   `num_runs`: `int` \
+    How many runs to perform to compare execution in reference and test setting.
+    Default: 50. The binary performs runs 3 invocations per 'run', to get more
+    accurate latency numbers.
+
+*   `num_interpreter_threads`: `int` (default=1) \
+    This modifies the number of threads used by the test Interpreter for
+    inference.
+
+*   `delegate`: `string` \
+    If provided, tries to use the specified delegate on the test Interpreter.
+    Valid values: "nnapi", "gpu", "hexagon", "coreml".
+
+    NOTE: Please refer to the
+    [Hexagon delegate documentation](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/performance/hexagon_delegate.md)
+    for instructions on how to set it up for the Hexagon delegate. The tool
+    assumes that `libhexagon_interface.so` and Qualcomm libraries lie in
+    `/data/local/tmp`.
+
+*   `output_file_path`: `string` \
+    The final metrics are dumped into `output_file_path` as a serialized
+    instance of `tflite::evaluation::EvaluationStageMetrics`
+
+This script also supports runtime/delegate arguments introduced by the
+[delegate registrar](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/delegates).
+If there is any conflict (for example, `num_threads` vs
+`num_interpreter_threads` here), the parameters of this
+script are given precedence.
+
+When **multiple delegates** are specified to be used in the commandline flags
+via the support of delegate registrar, the order of delegates applied to the
+TfLite runtime will be same as their enabling commandline flag is specified. For
+example, "--use_xnnpack=true --use_gpu=true" means applying the XNNPACK delegate
+first, and then the GPU delegate secondly. In comparison,
+"--use_gpu=true --use_xnnpack=true" means applying the GPU delegate first, and
+then the XNNPACK delegate secondly.
+
+Note, one could specify `--help` when launching the binary to see the full list
+of supported arguments.
+
+## Running the binary on Android
+
+(1) Build using the following command:
+
+```
+bazel build -c opt \
+  --config=android_arm64 \
+  //tensorflow/lite/tools/evaluation/tasks/inference_diff:run_eval
+```
+
+(2) Connect your phone. Push the binary to your phone with adb push (make the
+directory if required):
+
+```
+adb push bazel-bin/third_party/tensorflow/lite/tools/evaluation/tasks/inference_diff/run_eval /data/local/tmp
+```
+
+(3) Push the TFLite model that you need to test. For example:
+
+```
+adb push mobilenet_v1_1.0_224.tflite /data/local/tmp
+```
+
+(3) Run the binary.
+
+```
+adb shell /data/local/tmp/run_eval \
+  --model_file=/data/local/tmp/mobilenet_v1_1.0_224.tflite \
+  --delegate=gpu
+```
+
+(5) Pull the results.
+
+```
+adb pull /data/local/tmp/inference_diff.txt ~/accuracy_tool
+```
+
+## Running the binary on iOS
+
+Follow the instructions [here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/evaluation/tasks/ios/README.md)
+to run the binary on iOS using the
+[iOS evaluation app](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/evaluation/tasks/ios).

tensorflow/lite/tools/evaluation/tasks/inference_diff/run_eval.cc

@@ -0,0 +1,148 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <fstream>
+#include <ios>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "tensorflow/lite/core/c/common.h"
+#include "tensorflow/lite/tools/command_line_flags.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_delegate_provider.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+#include "tensorflow/lite/tools/evaluation/stages/inference_profiler_stage.h"
+#include "tensorflow/lite/tools/evaluation/tasks/task_executor.h"
+#include "tensorflow/lite/tools/logging.h"
+
+namespace tflite {
+namespace evaluation {
+
+constexpr char kModelFileFlag[] = "model_file";
+constexpr char kOutputFilePathFlag[] = "output_file_path";
+constexpr char kNumRunsFlag[] = "num_runs";
+constexpr char kInterpreterThreadsFlag[] = "num_interpreter_threads";
+constexpr char kDelegateFlag[] = "delegate";
+
+class InferenceDiff : public TaskExecutor {
+ public:
+  InferenceDiff() : num_runs_(50), num_interpreter_threads_(1) {}
+  ~InferenceDiff() override {}
+
+ protected:
+  std::vector<Flag> GetFlags() final;
+
+  // If the run is successful, the latest metrics will be returned.
+  std::optional<EvaluationStageMetrics> RunImpl() final;
+
+ private:
+  void OutputResult(const EvaluationStageMetrics& latest_metrics) const;
+  std::string model_file_path_;
+  std::string output_file_path_;
+  std::string delegate_;
+  int num_runs_;
+  int num_interpreter_threads_;
+};
+
+std::vector<Flag> InferenceDiff::GetFlags() {
+  // Command Line Flags.
+  std::vector<tflite::Flag> flag_list = {
+      tflite::Flag::CreateFlag(kModelFileFlag, &model_file_path_,
+                               "Path to test tflite model file."),
+      tflite::Flag::CreateFlag(kOutputFilePathFlag, &output_file_path_,
+                               "File to output metrics proto to."),
+      tflite::Flag::CreateFlag(kNumRunsFlag, &num_runs_,
+                               "Number of runs of test & reference inference "
+                               "each. Default value: 50"),
+      tflite::Flag::CreateFlag(
+          kInterpreterThreadsFlag, &num_interpreter_threads_,
+          "Number of interpreter threads to use for test inference."),
+      tflite::Flag::CreateFlag(
+          kDelegateFlag, &delegate_,
+          "Delegate to use for test inference, if available. "
+          "Must be one of {'nnapi', 'gpu', 'hexagon', 'xnnpack', 'coreml'}"),
+  };
+
+  return flag_list;
+}
+
+std::optional<EvaluationStageMetrics> InferenceDiff::RunImpl() {
+  // Initialize evaluation stage.
+  EvaluationStageConfig eval_config;
+  eval_config.set_name("inference_profiling");
+  auto* inference_params =
+      eval_config.mutable_specification()->mutable_tflite_inference_params();
+  inference_params->set_model_file_path(model_file_path_);
+  inference_params->set_num_threads(num_interpreter_threads_);
+  // This ensures that latency measurement isn't hampered by the time spent in
+  // generating random data.
+  inference_params->set_invocations_per_run(3);
+  inference_params->set_delegate(ParseStringToDelegateType(delegate_));
+  if (!delegate_.empty() &&
+      inference_params->delegate() == TfliteInferenceParams::NONE) {
+    TFLITE_LOG(WARN) << "Unsupported TFLite delegate: " << delegate_;
+    return std::nullopt;
+  }
+
+  InferenceProfilerStage eval(eval_config);
+  if (eval.Init(&delegate_providers_) != kTfLiteOk) return std::nullopt;
+
+  // Run inference & check diff for specified number of runs.
+  for (int i = 0; i < num_runs_; ++i) {
+    if (eval.Run() != kTfLiteOk) return std::nullopt;
+  }
+
+  const auto latest_metrics = eval.LatestMetrics();
+  OutputResult(latest_metrics);
+  return std::make_optional(latest_metrics);
+}
+
+void InferenceDiff::OutputResult(
+    const EvaluationStageMetrics& latest_metrics) const {
+  // Output latency & diff metrics.
+  if (!output_file_path_.empty()) {
+    std::ofstream metrics_ofile;
+    metrics_ofile.open(output_file_path_, std::ios::out);
+    metrics_ofile << latest_metrics.SerializeAsString();
+    metrics_ofile.close();
+  }
+
+  TFLITE_LOG(INFO) << "Num evaluation runs: " << latest_metrics.num_runs();
+  const auto& metrics =
+      latest_metrics.process_metrics().inference_profiler_metrics();
+  const auto& ref_latency = metrics.reference_latency();
+  TFLITE_LOG(INFO) << "Reference run latency: avg=" << ref_latency.avg_us()
+                   << "(us), std_dev=" << ref_latency.std_deviation_us()
+                   << "(us)";
+  const auto& test_latency = metrics.test_latency();
+  TFLITE_LOG(INFO) << "Test run latency: avg=" << test_latency.avg_us()
+                   << "(us), std_dev=" << test_latency.std_deviation_us()
+                   << "(us)";
+  const auto& output_errors = metrics.output_errors();
+  for (int i = 0; i < output_errors.size(); ++i) {
+    const auto& error = output_errors.at(i);
+    TFLITE_LOG(INFO) << "OutputDiff[" << i
+                     << "]: avg_error=" << error.avg_value()
+                     << ", std_dev=" << error.std_deviation();
+  }
+}
+
+std::unique_ptr<TaskExecutor> CreateTaskExecutor() {
+  return std::unique_ptr<TaskExecutor>(new InferenceDiff());
+}
+
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/tasks/ios/BUILD.apple

@@ -0,0 +1,37 @@
+load("@bazel_skylib//rules:build_test.bzl", "build_test")
+load("@build_bazel_rules_apple//apple:ios.bzl", "ios_static_framework")
+load("//tensorflow/lite/ios:ios.bzl", "TFL_MINIMUM_OS_VERSION")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
+    licenses = ["notice"],
+)
+
+# Main target for the inference diff tool iOS framework.
+# bazel build --config=ios_arm64 -c opt --cxxopt=-std=c++17 //tensorflow/lite/tools/evaluation/tasks/ios:TensorFlowLiteInferenceDiffC_framework
+ios_static_framework(
+    name = "TensorFlowLiteInferenceDiffC_framework",
+    hdrs = [
+        "//tensorflow/lite/tools:logging.h",
+        "//tensorflow/lite/tools/evaluation/tasks:task_executor_c_api.h",
+    ],
+    bundle_name = "TensorFlowLiteInferenceDiffC",
+    minimum_os_version = TFL_MINIMUM_OS_VERSION,
+    deps = [
+        "//tensorflow/lite/tools/evaluation/tasks:task_executor_c_api",
+        "//tensorflow/lite/tools/evaluation/tasks/inference_diff:run_eval_lib",
+    ],
+)
+
+# Used for building TensorFlowLiteInferenceDiffC_framework framework.
+build_test(
+    name = "framework_build_test",
+    # build_test targets are not meant to be run with sanitizers.
+    tags = [
+        "nomsan",
+        "notsan",
+    ],
+    targets = [
+        ":TensorFlowLiteInferenceDiffC_framework",
+    ],
+)

tensorflow/lite/tools/evaluation/tasks/ios/README.md

@@ -0,0 +1,43 @@
+# TFLite iOS evaluation app.
+
+## Description
+
+An iOS app to evaluate TFLite models. This is mainly for running different
+evaluation tasks on iOS. Right now it only supports evaluating the inference
+difference between cpu and delegates.
+
+The app reads evaluation parameters from a JSON file named
+`evaluation_params.json` in its `evaluation_data` directory. Any downloaded
+models for evaluation should also be placed in `evaluation_data` directory.
+
+The JSON file specifies the name of the model file and other evaluation
+parameters like number of iterations, number of threads, delegate name. The
+default values in the JSON file are for the Mobilenet_v2_1.0_224 model
+([tflite&pb][mobilenet-model]).
+
+## Building / running the app
+
+*   Follow the [iOS build instructions][build-ios] to configure the Bazel
+    workspace and `.bazelrc` file correctly.
+
+*   Run `build_evaluation_framework.sh` script to build the evaluation
+    framework. This script will build the evaluation framework targeting iOS
+    arm64 and put it under `TFLiteEvaluation/TFLiteEvaluation/Frameworks`
+    directory.
+
+*   Update evaluation parameters in `evaluation_params.json`.
+
+*   Change `Build Phases -> Copy Bundle Resources` and add the model file to the
+    resources that need to be copied.
+
+*   Ensure that `Build Phases -> Link Binary With Library` contains the
+    `Accelerate framework` and `TensorFlowLiteInferenceDiffC.framework`.
+
+*   Now try running the app. The app has a single button that runs the
+    evaluation on the model and displays results in a text view below. You can
+    also see the console output section in your Xcode to see more detailed
+    information.
+
+[build-ios]: https://tensorflow.org/lite/guide/build_ios
+[mobilenet-model]: https://github.com/tensorflow/tflite-support/raw/master/tensorflow_lite_support/metadata/python/tests/testdata/image_classifier/mobilenet_v2_1.0_224.tflite
+[mobilenet-paper]: https://arxiv.org/pdf/1704.04861.pdf

tensorflow/lite/tools/evaluation/tasks/ios/TFLiteEvaluation/TFLiteEvaluation/AppDelegate.h

@@ -0,0 +1,22 @@
+// Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <UIKit/UIKit.h>
+
+@interface AppDelegate : UIResponder <UIApplicationDelegate>
+
+@property(strong, nonatomic) UIWindow *window;
+
+@end

tensorflow/lite/tools/evaluation/tasks/ios/TFLiteEvaluation/TFLiteEvaluation/AppDelegate.m

@@ -0,0 +1,27 @@
+// Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "AppDelegate.h"
+
+@interface AppDelegate ()
+
+@end
+
+@implementation AppDelegate
+- (BOOL)application:(UIApplication *)application
+    didFinishLaunchingWithOptions:(NSDictionary *)launchOptions {
+  return YES;
+}
+@end

tensorflow/lite/tools/evaluation/tasks/ios/TFLiteEvaluation/TFLiteEvaluation/Assets.xcassets/AccentColor.colorset/Contents.json

@@ -0,0 +1,11 @@
+{
+  "colors" : [
+    {
+      "idiom" : "universal"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

tensorflow/lite/tools/evaluation/tasks/ios/TFLiteEvaluation/TFLiteEvaluation/Assets.xcassets/AppIcon.appiconset/Contents.json

@@ -0,0 +1,98 @@
+{
+  "images" : [
+    {
+      "idiom" : "iphone",
+      "scale" : "2x",
+      "size" : "20x20"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "3x",
+      "size" : "20x20"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "2x",
+      "size" : "29x29"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "3x",
+      "size" : "29x29"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "2x",
+      "size" : "40x40"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "3x",
+      "size" : "40x40"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "2x",
+      "size" : "60x60"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "3x",
+      "size" : "60x60"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "1x",
+      "size" : "20x20"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "2x",
+      "size" : "20x20"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "1x",
+      "size" : "29x29"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "2x",
+      "size" : "29x29"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "1x",
+      "size" : "40x40"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "2x",
+      "size" : "40x40"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "1x",
+      "size" : "76x76"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "2x",
+      "size" : "76x76"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "2x",
+      "size" : "83.5x83.5"
+    },
+    {
+      "idiom" : "ios-marketing",
+      "scale" : "1x",
+      "size" : "1024x1024"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

tensorflow/lite/tools/evaluation/tasks/ios/TFLiteEvaluation/TFLiteEvaluation/Assets.xcassets/Contents.json

@@ -0,0 +1,6 @@
+{
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

tensorflow/lite/tools/evaluation/tasks/ios/TFLiteEvaluation/TFLiteEvaluation/Base.lproj/Main.storyboard

@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="23727" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
+    <device id="retina6_1" orientation="portrait" appearance="light"/>
+    <dependencies>
+        <deployment identifier="iOS"/>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="23721"/>
+        <capability name="Safe area layout guides" minToolsVersion="9.0"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--Evaluation View Controller-->
+        <scene sceneID="tne-QT-ifu">
+            <objects>
+                <viewController id="BYZ-38-t0r" customClass="EvaluationViewController" sceneMemberID="viewController">
+                    <view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
+                        <rect key="frame" x="0.0" y="0.0" width="414" height="896"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <subviews>
+                            <button opaque="NO" contentMode="scaleToFill" misplaced="YES" contentHorizontalAlignment="center" contentVerticalAlignment="center" buttonType="system" lineBreakMode="middleTruncation" translatesAutoresizingMaskIntoConstraints="NO" id="j0O-Lq-1tJ">
+                                <rect key="frame" x="64" y="97" width="286" height="63"/>
+                                <constraints>
+                                    <constraint firstAttribute="height" constant="63" id="8VO-Ln-L2h"/>
+                                </constraints>
+                                <fontDescription key="fontDescription" type="system" pointSize="24"/>
+                                <state key="normal" title="Evaluate model"/>
+                                <connections>
+                                    <action selector="onEvaluateModel:" destination="BYZ-38-t0r" eventType="touchUpInside" id="Rb1-hs-Mub"/>
+                                </connections>
+                            </button>
+                            <textView clipsSubviews="YES" multipleTouchEnabled="YES" contentMode="scaleToFill" editable="NO" textAlignment="natural" selectable="NO" translatesAutoresizingMaskIntoConstraints="NO" id="Vd4-Gf-qKO">
+                                <rect key="frame" x="26" y="177" width="368" height="641"/>
+                                <color key="backgroundColor" white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
+                                <fontDescription key="fontDescription" type="system" pointSize="14"/>
+                                <textInputTraits key="textInputTraits" autocapitalizationType="sentences"/>
+                            </textView>
+                        </subviews>
+                        <viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                        <constraints>
+                            <constraint firstItem="Vd4-Gf-qKO" firstAttribute="top" secondItem="j0O-Lq-1tJ" secondAttribute="bottom" constant="18" id="Kd3-pP-C1k"/>
+                            <constraint firstItem="j0O-Lq-1tJ" firstAttribute="centerX" secondItem="8bC-Xf-vdC" secondAttribute="centerX" id="QJU-cq-L87"/>
+                            <constraint firstItem="Vd4-Gf-qKO" firstAttribute="trailing" secondItem="8bC-Xf-vdC" secondAttribute="trailingMargin" id="Tew-W4-Vq5"/>
+                            <constraint firstItem="j0O-Lq-1tJ" firstAttribute="top" secondItem="6Tk-OE-BBY" secondAttribute="top" id="Uce-n7-kZI"/>
+                            <constraint firstItem="j0O-Lq-1tJ" firstAttribute="leading" secondItem="6Tk-OE-BBY" secondAttribute="leading" constant="64" id="Uhq-Rw-NKT"/>
+                            <constraint firstItem="Vd4-Gf-qKO" firstAttribute="leading" secondItem="6Tk-OE-BBY" secondAttribute="leading" constant="26" id="aXc-6M-kyL"/>
+                            <constraint firstItem="6Tk-OE-BBY" firstAttribute="bottom" secondItem="Vd4-Gf-qKO" secondAttribute="bottom" constant="10" id="tz5-wP-LZs"/>
+                        </constraints>
+                    </view>
+                    <connections>
+                        <outlet property="resultsView" destination="Vd4-Gf-qKO" id="dBT-f6-SYw"/>
+                    </connections>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="140" y="122.78860569715144"/>
+        </scene>
+    </scenes>
+</document>

tensorflow/lite/tools/evaluation/tasks/ios/TFLiteEvaluation/TFLiteEvaluation/EvaluationViewController.h

@@ -0,0 +1,22 @@
+// Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <Foundation/Foundation.h>
+#import <UIKit/UIKit.h>
+
+@interface EvaluationViewController : UIViewController
+@property(weak, nonatomic) IBOutlet UITextView *resultsView;
+
+@end

tensorflow/lite/tools/evaluation/tasks/ios/TFLiteEvaluation/TFLiteEvaluation/EvaluationViewController.mm

@@ -0,0 +1,141 @@
+// Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "EvaluationViewController.h"
+#import <algorithm>
+#include <fstream>
+#import <sstream>
+#import <string>
+#import <vector>
+
+#import <TensorFlowLiteInferenceDiffC/TensorFlowLiteInferenceDiffC.h>
+
+namespace {
+
+NSString* const kDocumentsPrefix = @"/Documents/";
+NSString* const kModelFileKey = @"model_file";
+NSString* const kOutputFilePathKey = @"output_file_path";
+
+NSString* FilePathForResourceName(NSString* filename) {
+  NSString* name = [filename stringByDeletingPathExtension];
+  NSString* extension = [filename pathExtension];
+  NSString* file_path = [[NSBundle mainBundle] pathForResource:name ofType:extension];
+  if (file_path == NULL) {
+    TFLITE_LOG(FATAL) << "Couldn't find '" << [name UTF8String] << "." << [extension UTF8String]
+                      << "' in bundle.";
+  }
+  return file_path;
+}
+
+NSDictionary* ParseEvaluationParamsJson() {
+  NSString* params_json_path = FilePathForResourceName(@"evaluation_params.json");
+  NSData* data = [NSData dataWithContentsOfFile:params_json_path];
+  NSDictionary* dict = [NSJSONSerialization JSONObjectWithData:data options:kNilOptions error:nil];
+  return dict;
+}
+
+std::string FormatCommandLineParam(NSString* key, NSString* value) {
+  std::ostringstream stream;
+  stream << "--" << [key UTF8String] << "=" << [value UTF8String];
+  return stream.str();
+}
+
+// Reads the |evaluation_params.json| to read command line parameters and returns them as a vector
+// of strings.
+// Returns the evaluation parameters as key-value pairs.
+void ReadCommandLineParameters(std::vector<std::string>* params) {
+  NSDictionary* param_dict = ParseEvaluationParamsJson();
+  for (NSString* key in param_dict) {
+    NSString* value = param_dict[key];
+    if ([key isEqualToString:kModelFileKey]) {
+      value = FilePathForResourceName(value);
+    }
+    if ([key isEqualToString:kOutputFilePathKey]) {
+      if (![value hasPrefix:kDocumentsPrefix]) {
+        TFLITE_LOG(FATAL) << "Output file must be under the Document directory";
+      }
+      // Replace the prefix "/Documents/" with the actual documents path on the device.
+      NSString* documents = [NSSearchPathForDirectoriesInDomains(
+          NSDocumentDirectory, NSUserDomainMask, YES) objectAtIndex:0];
+      NSString* relpath = [value substringFromIndex:[kDocumentsPrefix length]];
+      value = [documents stringByAppendingPathComponent:relpath];
+
+      // Create the output directory if necessary.
+      NSString* path = value.stringByDeletingLastPathComponent;
+      if (![[NSFileManager defaultManager] createDirectoryAtPath:path
+                                     withIntermediateDirectories:YES
+                                                      attributes:nil
+                                                           error:nil]) {
+        TFLITE_LOG(FATAL) << "Cannot create output directory: " << [path UTF8String];
+      }
+
+      // Create the output file.
+      std::string output_file_path_ = std::string([value UTF8String]);
+      std::unique_ptr<std::ofstream> output_file_ =
+          std::make_unique<std::ofstream>(output_file_path_, std::ios::out | std::ios::binary);
+      if (!output_file_->is_open()) {
+        TFLITE_LOG(ERROR) << "Cannot open output file: " << output_file_path_;
+      } else {
+        TFLITE_LOG(INFO) << "Create output file: " << output_file_path_;
+      }
+    }
+    params->push_back(FormatCommandLineParam(key, value));
+  }
+}
+
+std::vector<char*> StringVecToCharPtrVec(const std::vector<std::string>& str_vec) {
+  std::vector<char*> charptr_vec;
+  std::transform(str_vec.begin(), str_vec.end(), std::back_inserter(charptr_vec),
+                 [](const std::string& s) -> char* { return const_cast<char*>(s.c_str()); });
+  return charptr_vec;
+}
+
+std::string EvaluationMetricsToString(TfLiteEvaluationMetrics* metrics) {
+  std::ostringstream stream;
+  stream << "Num evaluation runs: " << TfLiteEvaluationMetricsGetNumRuns(metrics);
+  TfLiteEvaluationMetricsLatency ref_latency = TfLiteEvaluationMetricsGetReferenceLatency(metrics);
+  stream << "\nReference run latency: avg=" << ref_latency.avg_us
+         << "(us), std_dev=" << ref_latency.std_deviation_us << "(us)";
+  TfLiteEvaluationMetricsLatency test_latency = TfLiteEvaluationMetricsGetTestLatency(metrics);
+  stream << "\nTest run latency: avg=" << test_latency.avg_us
+         << "(us), std_dev=" << test_latency.std_deviation_us << "(us)";
+  for (int i = 0; i < TfLiteEvaluationMetricsGetOutputErrorCount(metrics); ++i) {
+    TfLiteEvaluationMetricsAccuracy error = TfLiteEvaluationMetricsGetOutputError(metrics, i);
+    stream << "\nOutputDiff[" << i << "]: avg_error=" << error.avg_value
+           << ", std_dev=" << error.std_deviation;
+  }
+
+  return stream.str();
+}
+
+TfLiteEvaluationMetrics* RunEvaluation() {
+  std::vector<std::string> command_line_params;
+  ReadCommandLineParameters(&command_line_params);
+  std::vector<char*> argv = StringVecToCharPtrVec(command_line_params);
+  int argc = static_cast<int>(argv.size());
+  TfLiteEvaluationTask* task = TfLiteEvaluationTaskCreate();
+  return TfLiteEvaluationTaskRunWithArgs(task, argc, argv.data());
+}
+}  // namespace
+
+@interface EvaluationViewController ()
+@end
+
+@implementation EvaluationViewController
+- (IBAction)onEvaluateModel:(UIButton*)sender {
+  TfLiteEvaluationMetrics* metrics = RunEvaluation();
+  [_resultsView setText:[NSString stringWithUTF8String:EvaluationMetricsToString(metrics).c_str()]];
+}
+@end

tensorflow/lite/tools/evaluation/tasks/ios/TFLiteEvaluation/TFLiteEvaluation/Info.plist

@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>UILaunchStoryboardName</key>
+	<string>Main</string>
+	<key>CFBundleDevelopmentRegion</key>
+	<string>$(DEVELOPMENT_LANGUAGE)</string>
+	<key>CFBundleExecutable</key>
+	<string>$(EXECUTABLE_NAME)</string>
+	<key>CFBundleIdentifier</key>
+	<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
+	<key>CFBundleInfoDictionaryVersion</key>
+	<string>6.0</string>
+	<key>CFBundleName</key>
+	<string>$(PRODUCT_NAME)</string>
+	<key>CFBundlePackageType</key>
+	<string>APPL</string>
+	<key>CFBundleShortVersionString</key>
+	<string>1.0</string>
+	<key>CFBundleVersion</key>
+	<string>1</string>
+	<key>LSRequiresIPhoneOS</key>
+	<true/>
+	<key>UIMainStoryboardFile</key>
+	<string>Main</string>
+	<key>UIRequiredDeviceCapabilities</key>
+	<array>
+		<string>arm64</string>
+	</array>
+	<key>UISupportedInterfaceOrientations</key>
+	<array>
+		<string>UIInterfaceOrientationPortrait</string>
+	</array>
+	<key>UISupportedInterfaceOrientations~ipad</key>
+	<array>
+		<string>UIInterfaceOrientationPortrait</string>
+		<string>UIInterfaceOrientationPortraitUpsideDown</string>
+		<string>UIInterfaceOrientationLandscapeLeft</string>
+		<string>UIInterfaceOrientationLandscapeRight</string>
+	</array>
+</dict>
+</plist>

tensorflow/lite/tools/evaluation/tasks/ios/TFLiteEvaluation/TFLiteEvaluation/evaluation_data/evaluation_params.json

@@ -0,0 +1,8 @@
+{
+    "model_file" : "mobilenet_v2_1.0_224.tflite",
+    "output_file_path" : "/Documents/outputs/inference_diff.txt",
+    "num_runs" : "50",
+    "num_threads" : "1",
+    "num_interpreter_threads" : "1",
+    "delegate" : "coreml"
+}

tensorflow/lite/tools/evaluation/tasks/ios/TFLiteEvaluation/TFLiteEvaluation/main.m

@@ -0,0 +1,26 @@
+// Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <UIKit/UIKit.h>
+#import "AppDelegate.h"
+
+int main(int argc, char* argv[]) {
+  NSString* appDelegateClassName;
+  @autoreleasepool {
+    // Setup code that might create autoreleased objects goes here.
+    appDelegateClassName = NSStringFromClass([AppDelegate class]);
+  }
+  return UIApplicationMain(argc, argv, nil, appDelegateClassName);
+}

tensorflow/lite/tools/evaluation/tasks/ios/build_evaluation_framework.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+set -e
+
+WORKSPACE_ROOT=$(bazel info workspace 2> /dev/null)
+EVALUATION_DIR=tensorflow/lite/tools/evaluation/tasks
+DEST_DIR="${EVALUATION_DIR}/ios/TFLiteEvaluation/TFLiteEvaluation/Frameworks"
+FRAMEWORK_TARGET=TensorFlowLiteInferenceDiffC_framework
+
+function usage() {
+  echo "Usage: $(basename "$0")"
+  exit 1
+}
+
+function check_ios_configured() {
+  if [ ! -f "${WORKSPACE_ROOT}/${EVALUATION_DIR}/ios/BUILD" ]; then
+    echo "ERROR: Inference Diff framework BUILD file not found."
+    echo "Please enable iOS support by running the \"./configure\" script" \
+         "from the workspace root."
+    exit 1
+  fi
+}
+
+function build_framework() {
+  set -x
+  pushd "${WORKSPACE_ROOT}"
+
+# Build the framework.
+  bazel build --config=ios_arm64 -c opt --cxxopt=-std=c++17 \
+      "//${EVALUATION_DIR}/ios:${FRAMEWORK_TARGET}"
+
+# Get the generated framework path.
+BAZEL_OUTPUT_FILE_PATH=$(bazel cquery "//${EVALUATION_DIR}/ios:${FRAMEWORK_TARGET}" --config=ios_arm64 --output=starlark --starlark:expr="' '.join([f.path for f in target.files.to_list()])")
+
+# Copy the framework into the destination and unzip.
+  mkdir -p "${DEST_DIR}"
+  cp -f "${BAZEL_OUTPUT_FILE_PATH}" \
+      "${DEST_DIR}"
+  pushd "${DEST_DIR}"
+  unzip -o "${FRAMEWORK_TARGET}.zip"
+  rm -f "${FRAMEWORK_TARGET}.zip"
+
+  popd
+  popd
+}
+
+check_ios_configured
+build_framework

tensorflow/lite/tools/evaluation/tasks/task_executor.cc

@@ -0,0 +1,50 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/evaluation/tasks/task_executor.h"
+
+#include <optional>
+#include <string>
+
+#include "tensorflow/lite/tools/command_line_flags.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/logging.h"
+
+namespace tflite {
+namespace evaluation {
+std::optional<EvaluationStageMetrics> TaskExecutor::Run(int* argc,
+                                                        char* argv[]) {
+  auto flag_list = GetFlags();
+  auto delegate_flags = delegate_providers_.GetFlags();
+
+  flag_list.insert(flag_list.end(), delegate_flags.begin(),
+                   delegate_flags.end());
+  bool parse_result =
+      tflite::Flags::Parse(argc, const_cast<const char**>(argv), flag_list);
+  if (!parse_result) {
+    std::string usage = Flags::Usage(argv[0], flag_list);
+    TFLITE_LOG(ERROR) << usage;
+    return std::nullopt;
+  }
+
+  std::string unconsumed_args =
+      Flags::ArgsToString(*argc, const_cast<const char**>(argv));
+  if (!unconsumed_args.empty()) {
+    TFLITE_LOG(WARN) << "Unconsumed cmdline flags: " << unconsumed_args;
+  }
+
+  return RunImpl();
+}
+}  // namespace evaluation
+}  // namespace tflite

tensorflow/lite/tools/evaluation/tasks/task_executor.h

@@ -0,0 +1,51 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_EVALUATION_TASKS_TASK_EXECUTOR_H_
+#define TENSORFLOW_LITE_TOOLS_EVALUATION_TASKS_TASK_EXECUTOR_H_
+
+#include <optional>
+
+#include "absl/types/optional.h"
+#include "tensorflow/lite/tools/command_line_flags.h"
+#include "tensorflow/lite/tools/evaluation/evaluation_delegate_provider.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+
+namespace tflite {
+namespace evaluation {
+// A common task execution API to avoid boilerpolate code in defining the main
+// function.
+class TaskExecutor {
+ public:
+  virtual ~TaskExecutor() {}
+
+  // If the run is successful, the latest metrics will be returned.
+  std::optional<EvaluationStageMetrics> Run(int* argc, char* argv[]);
+
+ protected:
+  // Returns a list of commandline flags that this task defines.
+  virtual std::vector<Flag> GetFlags() = 0;
+
+  virtual std::optional<EvaluationStageMetrics> RunImpl() = 0;
+
+  DelegateProviders delegate_providers_;
+};
+
+// Just a declaration. In order to avoid the boilerpolate main-function code,
+// every evaluation task should define this function.
+std::unique_ptr<TaskExecutor> CreateTaskExecutor();
+}  // namespace evaluation
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_EVALUATION_TASKS_TASK_EXECUTOR_H_

tensorflow/lite/tools/evaluation/tasks/task_executor_c_api.cc

@@ -0,0 +1,100 @@
+/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/evaluation/tasks/task_executor_c_api.h"
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <utility>
+
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h"
+#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h"
+#include "tensorflow/lite/tools/evaluation/tasks/task_executor.h"
+#include "tensorflow/lite/tools/logging.h"
+
+extern "C" {
+
+struct TfLiteEvaluationMetrics {
+  tflite::evaluation::EvaluationStageMetrics metrics;
+};
+
+struct TfLiteEvaluationTask {
+  std::unique_ptr<tflite::evaluation::TaskExecutor> task_executor;
+};
+
+int32_t TfLiteEvaluationMetricsGetNumRuns(
+    const TfLiteEvaluationMetrics* metrics) {
+  return metrics->metrics.num_runs();
+}
+
+extern TfLiteEvaluationMetricsLatency TfLiteEvaluationMetricsGetTestLatency(
+    const TfLiteEvaluationMetrics* metrics) {
+  const auto& latency = metrics->metrics.process_metrics()
+                            .inference_profiler_metrics()
+                            .test_latency();
+  return {latency.last_us(), latency.max_us(), latency.min_us(),
+          latency.sum_us(),  latency.avg_us(), latency.std_deviation_us()};
+}
+
+extern TfLiteEvaluationMetricsLatency
+TfLiteEvaluationMetricsGetReferenceLatency(
+    const TfLiteEvaluationMetrics* metrics) {
+  const auto& latency = metrics->metrics.process_metrics()
+                            .inference_profiler_metrics()
+                            .reference_latency();
+  return {latency.last_us(), latency.max_us(), latency.min_us(),
+          latency.sum_us(),  latency.avg_us(), latency.std_deviation_us()};
+}
+
+extern size_t TfLiteEvaluationMetricsGetOutputErrorCount(
+    const TfLiteEvaluationMetrics* metrics) {
+  return metrics->metrics.process_metrics()
+      .inference_profiler_metrics()
+      .output_errors()
+      .size();
+}
+
+extern TfLiteEvaluationMetricsAccuracy TfLiteEvaluationMetricsGetOutputError(
+    const TfLiteEvaluationMetrics* metrics, int32_t output_error_index) {
+  int32_t output_count = TfLiteEvaluationMetricsGetOutputErrorCount(metrics);
+  if (output_error_index < 0 || output_error_index >= output_count) {
+    return {};
+  }
+  const auto& accuracy = metrics->metrics.process_metrics()
+                             .inference_profiler_metrics()
+                             .output_errors()
+                             .at(output_error_index);
+  return {accuracy.max_value(), accuracy.min_value(), accuracy.avg_value(),
+          accuracy.std_deviation()};
+}
+
+TfLiteEvaluationTask* TfLiteEvaluationTaskCreate() {
+  auto task_executor = tflite::evaluation::CreateTaskExecutor();
+  return new TfLiteEvaluationTask{std::move(task_executor)};
+}
+
+TfLiteEvaluationMetrics* TfLiteEvaluationTaskRunWithArgs(
+    TfLiteEvaluationTask* evaluation_task, int argc, char** argv) {
+  const auto metrics_optional =
+      evaluation_task->task_executor->Run(&argc, argv);
+  if (!metrics_optional.has_value()) {
+    TFLITE_LOG(ERROR) << "Failed to run the task evaluation!";
+    return new TfLiteEvaluationMetrics{};
+  }
+  return new TfLiteEvaluationMetrics{std::move(metrics_optional.value())};
+}
+
+}  // extern "C"

tensorflow/lite/tools/evaluation/tasks/task_executor_c_api.h

@@ -0,0 +1,93 @@
+/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_EVALUATION_TASKS_TASK_EXECUTOR_C_API_H_
+#define TENSORFLOW_LITE_TOOLS_EVALUATION_TASKS_TASK_EXECUTOR_C_API_H_
+
+#include <cstddef>
+#include <cstdint>
+#ifdef __cplusplus
+extern "C" {
+#endif  // __cplusplus
+
+// -----------------------------------------------------------------------------
+// C APIs corresponding to tflite::evaluation::LatencyMetrics proto.
+// -----------------------------------------------------------------------------
+
+typedef struct TfLiteEvaluationMetricsLatency {
+  // Latency for the last Run.
+  int64_t last_us;
+  // Maximum latency observed for any Run.
+  int64_t max_us;
+  // Minimum latency observed for any Run.
+  int64_t min_us;
+  // Sum of all Run latencies.
+  int64_t sum_us;
+  // Average latency across all Runs.
+  double avg_us;
+  // Standard deviation for latency across all Runs.
+  int64_t std_deviation_us;
+} TfLiteEvaluationMetricsLatency;
+
+// -----------------------------------------------------------------------------
+// C APIs corresponding to tflite::evaluation::AccuracyMetrics proto.
+// -----------------------------------------------------------------------------
+
+typedef struct TfLiteEvaluationMetricsAccuracy {
+  // Maximum value observed for any Run.
+  float max_value;
+  // Minimum value observed for any Run.
+  float min_value;
+  // Average value across all Runs.
+  double avg_value;
+  // Standard deviation across all Runs.
+  float std_deviation;
+} TfLiteEvaluationMetricsAccuracy;
+
+// -----------------------------------------------------------------------------
+// C APIs corresponding to tflite::evaluation::EvaluationStageMetrics type.
+// -----------------------------------------------------------------------------
+typedef struct TfLiteEvaluationMetrics TfLiteEvaluationMetrics;
+
+extern int32_t TfLiteEvaluationMetricsGetNumRuns(
+    const TfLiteEvaluationMetrics* metrics);
+
+extern TfLiteEvaluationMetricsLatency TfLiteEvaluationMetricsGetTestLatency(
+    const TfLiteEvaluationMetrics* metrics);
+
+extern TfLiteEvaluationMetricsLatency
+TfLiteEvaluationMetricsGetReferenceLatency(
+    const TfLiteEvaluationMetrics* metrics);
+
+extern size_t TfLiteEvaluationMetricsGetOutputErrorCount(
+    const TfLiteEvaluationMetrics* metrics);
+
+extern TfLiteEvaluationMetricsAccuracy TfLiteEvaluationMetricsGetOutputError(
+    const TfLiteEvaluationMetrics* metrics, int32_t output_error_index);
+
+// -----------------------------------------------------------------------------
+// C APIs corresponding to tflite::evaluation::TaskExecutor type.
+// -----------------------------------------------------------------------------
+typedef struct TfLiteEvaluationTask TfLiteEvaluationTask;
+
+extern TfLiteEvaluationTask* TfLiteEvaluationTaskCreate();
+
+extern TfLiteEvaluationMetrics* TfLiteEvaluationTaskRunWithArgs(
+    TfLiteEvaluationTask* evaluation_task, int argc, char** argv);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif  // __cplusplus
+
+#endif  // TENSORFLOW_LITE_TOOLS_EVALUATION_TASKS_TASK_EXECUTOR_C_API_H_

tensorflow/lite/tools/evaluation/tasks/task_executor_main.cc

@@ -0,0 +1,33 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <cstdlib>
+
+#include "tensorflow/lite/tools/evaluation/tasks/task_executor.h"
+#include "tensorflow/lite/tools/logging.h"
+
+// This could serve as the main function for all eval tools.
+int main(int argc, char* argv[]) {
+  auto task_executor = tflite::evaluation::CreateTaskExecutor();
+  if (task_executor == nullptr) {
+    TFLITE_LOG(ERROR) << "Could not create the task evaluation!";
+    return EXIT_FAILURE;
+  }
+  const auto metrics = task_executor->Run(&argc, argv);
+  if (!metrics.has_value()) {
+    TFLITE_LOG(ERROR) << "Could not run the task evaluation!";
+    return EXIT_FAILURE;
+  }
+  return EXIT_SUCCESS;
+}