chore: import upstream snapshot with attribution

tensorflow/lite/delegates/gpu/common/testing/BUILD

@@ -0,0 +1,24 @@
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_visibility = ["//visibility:public"],
+    licenses = ["notice"],
+)
+
+cc_library(
+    name = "interpreter_utils",
+    testonly = 1,
+    srcs = ["interpreter_utils.cc"],
+    hdrs = ["interpreter_utils.h"],
+    deps = [
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/core:framework",
+        "//tensorflow/lite/core/api",
+        "//tensorflow/lite/core/c:common",
+        "//tensorflow/lite/core/kernels:builtin_ops",
+        "//tensorflow/lite/delegates/gpu/common:status",
+        "//tensorflow/lite/delegates/gpu/common:tensor",
+        "//tensorflow/lite/schema:schema_fbs",
+    ],
+)

tensorflow/lite/delegates/gpu/common/testing/feature_parity/BUILD

@@ -0,0 +1,102 @@
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
+load("@rules_cc//cc:cc_test.bzl", "cc_test")
+load(
+    "//tensorflow/core/platform:build_config_root.bzl",
+    "tf_gpu_tests_tags",
+)
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_testonly = 1,
+    default_visibility = ["//visibility:public"],
+    licenses = ["notice"],
+)
+
+cc_library(
+    name = "feature_parity",
+    hdrs = ["feature_parity.h"],
+    deps = [
+        ":utils",
+        "//tensorflow/lite/delegates/gpu/common/testing/feature_parity/generators:all_generators",
+    ],
+)
+
+cc_library(
+    name = "utils",
+    srcs = ["utils.cc"],
+    hdrs = ["utils.h"],
+    deps = [
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite:string",
+        "//tensorflow/lite/core:framework",
+        "//tensorflow/lite/core/c:common",
+        "//tensorflow/lite/core/kernels:builtin_ops",
+        "//tensorflow/lite/schema:schema_fbs",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:span",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+cc_test(
+    name = "opengl_test",
+    srcs = ["opengl_test.cc"],
+    linkopts = [
+        "-lEGL",
+        "-lGLESv3",
+    ],
+    tags = tf_gpu_tests_tags() + [
+        "local",
+        "nobuilder",
+        "notap",
+        "tflite_not_portable_ios",
+    ],
+    deps = [
+        ":feature_parity",
+        ":utils",
+        "//tensorflow/lite:framework_lib",
+        "//tensorflow/lite/delegates/gpu:gl_delegate",
+        "//tensorflow/lite/schema:schema_fbs",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
+cc_test(
+    name = "opencl_test",
+    srcs = ["opencl_test.cc"],
+    linkstatic = True,
+    tags = tf_gpu_tests_tags() + [
+        "linux",
+        "local",
+        "tflite_not_portable_ios",
+    ],
+    deps = [
+        ":feature_parity",
+        ":utils",
+        "//tensorflow/lite:framework_lib",
+        "//tensorflow/lite/delegates/gpu:delegate",
+        "//tensorflow/lite/delegates/gpu:delegate_options",
+        "//tensorflow/lite/schema:schema_fbs",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
+cc_test(
+    name = "xnnpack_test",
+    srcs = ["xnnpack_test.cc"],
+    tags = tf_gpu_tests_tags() + [
+        "local",
+        "nobuilder",
+        "notap",
+        "tflite_not_portable_ios",
+    ],
+    deps = [
+        ":feature_parity",
+        ":utils",
+        "//tensorflow/lite:framework_lib",
+        "//tensorflow/lite/delegates/xnnpack:xnnpack_delegate",
+        "//tensorflow/lite/schema:schema_fbs",
+        "@com_google_googletest//:gtest_main",
+    ],
+)

tensorflow/lite/delegates/gpu/common/testing/feature_parity/README.md

@@ -0,0 +1,7 @@
+### TFLite Delegate Feature Parity Testing
+
+These tests ensure feature parity across TFLite GPU delegates. Every test
+receives a simple automatically generated tflite model as an input. Model runs
+with default tflite cpu interpreter and an delegated interpreter. Test succeeds
+when output results for both interpretes match with the given accuracy.
+

tensorflow/lite/delegates/gpu/common/testing/feature_parity/feature_parity.h

@@ -0,0 +1,34 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_DELEGATES_GPU_COMMON_TESTING_FEATURE_PARITY_FEATURE_PARITY_H_
+#define TENSORFLOW_LITE_DELEGATES_GPU_COMMON_TESTING_FEATURE_PARITY_FEATURE_PARITY_H_
+
+#include <vector>
+
+#include "tensorflow/lite/delegates/gpu/common/testing/feature_parity/generators/add.h"
+#include "tensorflow/lite/delegates/gpu/common/testing/feature_parity/utils.h"
+
+namespace tflite {
+
+// Specifies the common ops feature parity for all gpu delegates. NameModel
+// structure stores the model name and model itself.
+inline std::vector<TestParams> GetFeatureParity() {
+  return {Add2SameShapeTensors(), AddBroadcast()};
+}
+
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_DELEGATES_GPU_COMMON_TESTING_FEATURE_PARITY_FEATURE_PARITY_H_

tensorflow/lite/delegates/gpu/common/testing/feature_parity/generators/BUILD

@@ -0,0 +1,31 @@
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_testonly = 1,
+    default_visibility = ["//visibility:public"],
+    licenses = ["notice"],
+)
+
+cc_library(
+    name = "all_generators",
+    hdrs = [
+        "add.h",
+    ],
+    deps = [
+        ":add",
+        "//tensorflow/lite/delegates/gpu/common/testing/feature_parity:utils",
+    ],
+)
+
+cc_library(
+    name = "add",
+    srcs = ["add.cc"],
+    hdrs = ["add.h"],
+    deps = [
+        "//tensorflow/lite:schema_fbs_version",
+        "//tensorflow/lite/delegates/gpu/common/testing/feature_parity:utils",
+        "//tensorflow/lite/schema:schema_fbs",
+        "@flatbuffers",
+    ],
+)

tensorflow/lite/delegates/gpu/common/testing/feature_parity/generators/add.cc

@@ -0,0 +1,124 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/delegates/gpu/common/testing/feature_parity/generators/add.h"
+
+#include <stdint.h>
+
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "flatbuffers/buffer.h"  // from @flatbuffers
+#include "flatbuffers/flatbuffer_builder.h"  // from @flatbuffers
+#include "flatbuffers/string.h"  // from @flatbuffers
+#include "tensorflow/lite/delegates/gpu/common/testing/feature_parity/utils.h"
+#include "tensorflow/lite/schema/schema_generated.h"
+#include "tensorflow/lite/version.h"
+
+namespace tflite {
+namespace {
+
+class AddModelBuilder {
+ public:
+  std::vector<uint8_t> Build() {
+    flatbuffers::FlatBufferBuilder builder;
+    flatbuffers::Offset<OperatorCode> operator_code =
+        CreateOperatorCode(builder, BuiltinOperator_ADD, 0);
+
+    flatbuffers::Offset<AddOptions> add_options =
+        CreateAddOptions(builder, ActivationFunctionType_NONE);
+
+    flatbuffers::Offset<Buffer> buffers[1] = {
+        CreateBuffer(builder, builder.CreateVector({})),
+    };
+    std::vector<flatbuffers::Offset<Tensor>> tensors;
+    tensors.reserve(input_shapes_.size());
+    for (int8_t i = 0; i < input_shapes_.size(); i++) {
+      tensors.push_back(CreateTensor(
+          builder, builder.CreateVector<int32_t>(input_shapes_[i].data(), 4),
+          TensorType_FLOAT32,
+          /*buffer=*/0, builder.CreateString(std::to_string(i))));
+    }
+    tensors.push_back(CreateTensor(
+        builder, builder.CreateVector<int32_t>(output_shape_.data(), 4),
+        TensorType_FLOAT32,
+        /*buffer=*/0,
+        builder.CreateString(std::to_string(input_shapes_.size()))));
+
+    const int32_t op_inputs[2] = {0, 1};
+    const int32_t op_outputs[1] = {2};
+
+    flatbuffers::Offset<Operator> op =
+        CreateOperator(builder, /*opcode_index=*/0,
+                       builder.CreateVector<int32_t>(op_inputs, 2),
+                       builder.CreateVector<int32_t>(op_outputs, 1),
+                       BuiltinOptions_AddOptions, add_options.Union());
+
+    int32_t subgraph_inputs[2] = {0, 1};
+    int32_t subgraph_outputs[1] = {2};
+    flatbuffers::Offset<SubGraph> subgraph =
+        CreateSubGraph(builder, builder.CreateVector(&tensors[0], 3),
+                       builder.CreateVector<int32_t>(subgraph_inputs, 2),
+                       builder.CreateVector<int32_t>(subgraph_outputs, 1),
+                       builder.CreateVector(&op, 1));
+
+    flatbuffers::Offset<flatbuffers::String> description =
+        builder.CreateString("Add model");
+    flatbuffers::Offset<Model> model_buffer = CreateModel(
+        builder, TFLITE_SCHEMA_VERSION, builder.CreateVector(&operator_code, 1),
+        builder.CreateVector(&subgraph, 1), description,
+        builder.CreateVector(buffers, 1));
+
+    builder.Finish(model_buffer);
+    return std::vector<uint8_t>(builder.GetBufferPointer(),
+                                builder.GetBufferPointer() + builder.GetSize());
+  }
+
+  void SetInputShape(uint32_t input, std::vector<int32_t>&& shape) {
+    if (input_shapes_.size() <= input) {
+      input_shapes_.resize(input + 1);
+    }
+
+    input_shapes_[input] = std::move(shape);
+  }
+
+  void SetOutputShape(std::vector<int32_t>&& shape) {
+    output_shape_ = std::move(shape);
+  }
+
+ private:
+  std::vector<std::vector<int32_t>> input_shapes_;
+  std::vector<int32_t> output_shape_;
+};
+}  // namespace
+
+TestParams Add2SameShapeTensors() {
+  AddModelBuilder builder;
+  builder.SetInputShape(0, {1, 2, 2, 2});
+  builder.SetInputShape(1, {1, 2, 2, 2});
+  builder.SetOutputShape({1, 2, 2, 2});
+  return {"Add2SameShapeTensors", builder.Build()};
+}
+
+TestParams AddBroadcast() {
+  AddModelBuilder builder;
+  builder.SetInputShape(0, {1, 2, 2, 2});
+  builder.SetInputShape(1, {1, 1, 1, 2});
+  builder.SetOutputShape({1, 2, 2, 2});
+  return {"AddBroadcast", builder.Build()};
+}
+
+}  // namespace tflite

tensorflow/lite/delegates/gpu/common/testing/feature_parity/generators/add.h

@@ -0,0 +1,28 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_DELEGATES_GPU_COMMON_TESTING_FEATURE_PARITY_GENERATORS_ADD_H_
+#define TENSORFLOW_LITE_DELEGATES_GPU_COMMON_TESTING_FEATURE_PARITY_GENERATORS_ADD_H_
+
+#include "tensorflow/lite/delegates/gpu/common/testing/feature_parity/utils.h"
+
+namespace tflite {
+
+TestParams Add2SameShapeTensors();
+TestParams AddBroadcast();
+
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_DELEGATES_GPU_COMMON_TESTING_FEATURE_PARITY_GENERATORS_ADD_H_

tensorflow/lite/delegates/gpu/common/testing/feature_parity/opencl_test.cc

@@ -0,0 +1,70 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <stdint.h>
+
+#include <memory>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+#include "tensorflow/lite/delegates/gpu/common/testing/feature_parity/feature_parity.h"
+#include "tensorflow/lite/delegates/gpu/common/testing/feature_parity/utils.h"
+#include "tensorflow/lite/delegates/gpu/delegate.h"
+#include "tensorflow/lite/delegates/gpu/delegate_options.h"
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/schema/schema_generated.h"
+
+namespace tflite {
+
+class OpenCLBackend : public testing::TestWithParam<TestParams> {};
+
+TEST_P(OpenCLBackend, DelegateTest) {
+  TfLiteGpuDelegateOptionsV2 options = TfLiteGpuDelegateOptionsV2();
+  std::unique_ptr<TfLiteDelegate, decltype(&TfLiteGpuDelegateV2Delete)>
+      cl_delegate(TfLiteGpuDelegateV2Create(&options),
+                  TfLiteGpuDelegateV2Delete);
+  const TestParams& param = GetParam();
+  const std::vector<uint8_t>& model_binary = param.model;
+  ASSERT_NE(model_binary.empty(), true);
+  std::unique_ptr<Interpreter> tflite_cpu_interpreter;
+  ASSERT_OK(
+      BuildInterpreter(GetModel(model_binary.data()), &tflite_cpu_interpreter));
+  ASSERT_OK(AllocateTensors(&tflite_cpu_interpreter));
+  std::unique_ptr<Interpreter> opencl_interpreter;
+  ASSERT_OK(
+      BuildInterpreter(GetModel(model_binary.data()), &opencl_interpreter));
+  // Ensures that tensors are allocated.
+  ASSERT_OK(ModifyGraphWithDelegate(&opencl_interpreter, cl_delegate.get()));
+  // Inputs are initialized with consequent values of the fixed range.
+  InitializeInputs(/*left=*/0, /*right=*/100, &tflite_cpu_interpreter);
+  InitializeInputs(/*left=*/0, /*right=*/100, &opencl_interpreter);
+  ASSERT_OK(Invoke(&tflite_cpu_interpreter));
+  ASSERT_OK(Invoke(&opencl_interpreter));
+  for (int i = 0; i < tflite_cpu_interpreter->outputs().size(); ++i) {
+    int id = tflite_cpu_interpreter->outputs()[i];
+    const TfLiteTensor* cpu = tflite_cpu_interpreter->tensor(id);
+    const TfLiteTensor* gpu = opencl_interpreter->tensor(id);
+    EXPECT_THAT(*gpu, TensorEq(testing::FloatNear(1e-6), *cpu))
+        << " for output tensor #" << i << " with id " << id;
+  }
+}
+
+INSTANTIATE_TEST_SUITE_P(FeatureParityTests, OpenCLBackend,
+                         testing::ValuesIn(GetFeatureParity()),
+                         [](const testing::TestParamInfo<TestParams>& info) {
+                           return info.param.name;
+                         });
+}  // namespace tflite

tensorflow/lite/delegates/gpu/common/testing/feature_parity/opengl_test.cc

@@ -0,0 +1,68 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+#include "tensorflow/lite/delegates/gpu/common/testing/feature_parity/feature_parity.h"
+#include "tensorflow/lite/delegates/gpu/common/testing/feature_parity/utils.h"
+#include "tensorflow/lite/delegates/gpu/gl_delegate.h"
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/schema/schema_generated.h"
+
+namespace tflite {
+
+class OpenGLBackend : public testing::TestWithParam<TestParams> {};
+
+TEST_P(OpenGLBackend, DelegateTest) {
+  TfLiteGpuDelegateOptions options = TfLiteGpuDelegateOptions();
+  std::unique_ptr<TfLiteDelegate, decltype(&TfLiteGpuDelegateDelete)>
+      gl_delegate(TfLiteGpuDelegateCreate(&options), TfLiteGpuDelegateDelete);
+  const TestParams& param = GetParam();
+  const std::vector<uint8_t>& model_binary = param.model;
+  ASSERT_NE(model_binary.empty(), true);
+  std::unique_ptr<Interpreter> tflite_cpu_interpreter;
+  ASSERT_OK(
+      BuildInterpreter(GetModel(model_binary.data()), &tflite_cpu_interpreter));
+  ASSERT_OK(AllocateTensors(&tflite_cpu_interpreter));
+  std::unique_ptr<Interpreter> opengl_interpreter;
+  ASSERT_OK(
+      BuildInterpreter(GetModel(model_binary.data()), &opengl_interpreter));
+  // Ensures that tensors are allocated.
+  ASSERT_OK(ModifyGraphWithDelegate(&opengl_interpreter, gl_delegate.get()));
+  // Inputs are initialized with consequent values of the fixed range.
+  InitializeInputs(/*left=*/0, /*right=*/100, &tflite_cpu_interpreter);
+  InitializeInputs(/*left=*/0, /*right=*/100, &opengl_interpreter);
+  ASSERT_OK(Invoke(&tflite_cpu_interpreter));
+  ASSERT_OK(Invoke(&opengl_interpreter));
+  for (int i = 0; i < tflite_cpu_interpreter->outputs().size(); ++i) {
+    int id = tflite_cpu_interpreter->outputs()[i];
+    const TfLiteTensor* cpu = tflite_cpu_interpreter->tensor(id);
+    const TfLiteTensor* gpu = opengl_interpreter->tensor(id);
+    EXPECT_THAT(*gpu, TensorEq(testing::FloatNear(1e-6), *cpu))
+        << " for output tensor #" << i << " with id " << id;
+  }
+}
+
+INSTANTIATE_TEST_SUITE_P(FeatureParityTests, OpenGLBackend,
+                         testing::ValuesIn(GetFeatureParity()),
+                         [](const testing::TestParamInfo<TestParams>& info) {
+                           return info.param.name;
+                         });
+
+}  // namespace tflite

tensorflow/lite/delegates/gpu/common/testing/feature_parity/utils.cc

@@ -0,0 +1,171 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/delegates/gpu/common/testing/feature_parity/utils.h"
+
+#include <memory>
+#include <optional>
+#include <ostream>
+#include <string>
+#include <utility>
+
+#include "absl/status/status.h"
+#include "absl/strings/substitute.h"
+#include "tensorflow/lite/core/interpreter_builder.h"
+#include "tensorflow/lite/core/kernels/register.h"
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/schema/schema_generated.h"
+
+std::ostream& operator<<(std::ostream& os, const TfLiteTensor& tensor) {
+  std::string shape;
+  std::optional<std::string> result = tflite::ShapeToString(tensor.dims);
+  if (result.has_value()) {
+    shape = std::move(result.value());
+  } else {
+    shape = "[error: unsupported number of dimensions]";
+  }
+  return os << "tensor of shape " << shape;
+}
+
+namespace tflite {
+
+std::optional<std::string> ShapeToString(TfLiteIntArray* shape) {
+  std::string result;
+  int* data = shape->data;
+  switch (shape->size) {
+    case 1:
+      result = absl::Substitute("Linear=[$0]", data[0]);
+      break;
+    case 2:
+      result = absl::Substitute("HW=[$0, $1]", data[0], data[1]);
+      break;
+    case 3:
+      result = absl::Substitute("HWC=[$0, $1, $2]", data[0], data[1], data[2]);
+      break;
+    case 4:
+      result = absl::Substitute("BHWC=[$0, $1, $2, $3]", data[0], data[1],
+                                data[2], data[3]);
+      break;
+    default:
+      // This printer doesn't expect shapes of more than 4 dimensions.
+      return std::nullopt;
+  }
+  return result;
+}
+
+std::optional<std::string> CoordinateToString(TfLiteIntArray* shape,
+                                              int linear) {
+  std::string result;
+  switch (shape->size) {
+    case 1: {
+      result = absl::Substitute("[$0]", linear);
+      break;
+    } break;
+    case 2: {
+      const int tensor_width = shape->data[1];
+      const int h_coord = linear / tensor_width;
+      const int w_coord = linear % tensor_width;
+      result = absl::Substitute("[$0, $1]", h_coord, w_coord);
+      break;
+    } break;
+    case 3: {
+      const int tensor_width = shape->data[1];
+      const int tensor_channels = shape->data[2];
+      const int h_coord = linear / (tensor_width * tensor_channels);
+      const int w_coord =
+          (linear % (tensor_width * tensor_channels)) / tensor_channels;
+      const int c_coord =
+          (linear % (tensor_width * tensor_channels)) % tensor_channels;
+      result = absl::Substitute("[$0, $1, $2]", h_coord, w_coord, c_coord);
+      break;
+    } break;
+    case 4: {
+      const int tensor_height = shape->data[1];
+      const int tensor_width = shape->data[2];
+      const int tensor_channels = shape->data[3];
+      const int b_coord =
+          linear / (tensor_height * tensor_width * tensor_channels);
+      const int h_coord =
+          (linear % (tensor_height * tensor_width * tensor_channels)) /
+          (tensor_width * tensor_channels);
+      const int w_coord =
+          ((linear % (tensor_height * tensor_width * tensor_channels)) %
+           (tensor_width * tensor_channels)) /
+          tensor_channels;
+      const int c_coord =
+          ((linear % (tensor_height * tensor_width * tensor_channels)) %
+           (tensor_width * tensor_channels)) %
+          tensor_channels;
+      result = absl::Substitute("[$0, $1, $2, $3]", b_coord, h_coord, w_coord,
+                                c_coord);
+      break;
+    }
+    default:
+      // This printer doesn't expect shapes of more than 4 dimensions.
+      return std::nullopt;
+  }
+  return result;
+}
+
+// Builds interpreter for a model, allocates tensors.
+absl::Status BuildInterpreter(const Model* model,
+                              std::unique_ptr<Interpreter>* interpreter) {
+  TfLiteStatus status =
+      InterpreterBuilder(model, ops::builtin::BuiltinOpResolver())(interpreter);
+  if (status != kTfLiteOk || !*interpreter) {
+    return absl::InternalError(
+        "Failed to initialize interpreter with model binary.");
+  }
+  return absl::OkStatus();
+}
+
+absl::Status AllocateTensors(std::unique_ptr<Interpreter>* interpreter) {
+  if ((*interpreter)->AllocateTensors() != kTfLiteOk) {
+    return absl::InternalError("Failed to allocate tensors.");
+  }
+  return absl::OkStatus();
+}
+
+absl::Status ModifyGraphWithDelegate(std::unique_ptr<Interpreter>* interpreter,
+                                     TfLiteDelegate* delegate) {
+  if ((*interpreter)->ModifyGraphWithDelegate(delegate) != kTfLiteOk) {
+    return absl::InternalError("Failed to modify graph with delegate.");
+  }
+  return absl::OkStatus();
+}
+
+void InitializeInputs(int left, int right,
+                      std::unique_ptr<Interpreter>* interpreter) {
+  for (int id : (*interpreter)->inputs()) {
+    float* input_data = (*interpreter)->typed_tensor<float>(id);
+    int input_size = (*interpreter)->input_tensor(id)->bytes;
+    for (int i = 0; i < input_size; i++) {
+      input_data[i] = left + i % right;
+    }
+  }
+}
+
+absl::Status Invoke(std::unique_ptr<Interpreter>* interpreter) {
+  if ((*interpreter)->Invoke() != kTfLiteOk) {
+    return absl::InternalError("Failed during inference.");
+  }
+  return absl::OkStatus();
+}
+
+std::ostream& operator<<(std::ostream& os, const TestParams& param) {
+  return os << param.name;
+}
+
+}  // namespace tflite

tensorflow/lite/delegates/gpu/common/testing/feature_parity/utils.h

@@ -0,0 +1,223 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_DELEGATES_GPU_COMMON_TESTING_FEATURE_PARITY_UTILS_H_
+#define TENSORFLOW_LITE_DELEGATES_GPU_COMMON_TESTING_FEATURE_PARITY_UTILS_H_
+
+#include <stddef.h>
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <ostream>
+#include <string>
+#include <tuple>
+#include <utility>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+#include "absl/status/status.h"
+#include "absl/types/span.h"
+#include "tensorflow/lite/core/model.h"
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/schema/schema_generated.h"
+#include "tensorflow/lite/string_type.h"
+
+namespace tflite {
+
+// These two functions implement usability printing for TfLiteTensor dimensions
+// and coordinates. By default dimensions are interpreted depending on the size:
+// 1:Linear, 2:HW, 3: HWC, 4:BHWC. If there are more than 4 dimensions,
+// absl::nullopt will be returned.
+std::optional<std::string> ShapeToString(TfLiteIntArray* shape);
+std::optional<std::string> CoordinateToString(TfLiteIntArray* shape,
+                                              int linear);
+
+template <typename TupleMatcher>
+class TensorEqMatcher {
+ public:
+  TensorEqMatcher(const TupleMatcher& tuple_matcher, const TfLiteTensor& rhs)
+      : tuple_matcher_(tuple_matcher), rhs_(rhs) {}
+
+  // Make TensorEqMatcher movable only (The copy operations are implicitly
+  // deleted).
+  TensorEqMatcher(TensorEqMatcher&& other) = default;
+  TensorEqMatcher& operator=(TensorEqMatcher&& other) = default;
+
+  template <typename T>
+  operator testing::Matcher<T>() const {  // NOLINT
+    return testing::Matcher<T>(new Impl(tuple_matcher_, rhs_));
+  }
+
+  class Impl : public testing::MatcherInterface<TfLiteTensor> {
+   public:
+    typedef ::std::tuple<float, float> InnerMatcherArg;
+
+    Impl(const TupleMatcher& tuple_matcher, const TfLiteTensor& rhs)
+        : mono_tuple_matcher_(
+              testing::SafeMatcherCast<InnerMatcherArg>(tuple_matcher)),
+          rhs_(rhs) {}
+
+    // Make Impl movable only (The copy operations are implicitly deleted).
+    Impl(Impl&& other) = default;
+    Impl& operator=(Impl&& other) = default;
+
+    // Define what gtest framework will print for the Expected field.
+    void DescribeTo(std::ostream* os) const override {
+      std::string shape;
+      std::optional<std::string> result = ShapeToString(rhs_.dims);
+      if (result.has_value()) {
+        shape = std::move(result.value());
+      } else {
+        shape = "[error: unsupported number of dimensions]";
+      }
+      *os << "tensor which has the shape of " << shape
+          << ", where each value and its corresponding expected value ";
+      mono_tuple_matcher_.DescribeTo(os);
+    }
+
+    bool MatchAndExplain(
+        TfLiteTensor lhs,
+        testing::MatchResultListener* listener) const override {
+      // 1. Check that TfLiteTensor data type is supported.
+      // Support for other data types will be added on demand.
+      if (lhs.type != kTfLiteFloat32 || rhs_.type != kTfLiteFloat32) {
+        *listener << "which data type is not float32, which is not currently "
+                     "supported.";
+        return false;
+      }
+
+      // 2. Check that dimensions' sizes match. Otherwise, we are not able to
+      // compare tensors.
+      if (lhs.dims->size != rhs_.dims->size) {
+        *listener << "which is different from the expected shape of size "
+                  << rhs_.dims->size;
+        return false;
+      }
+      // 3. Check that dimensions' values are equal as well. We are not able to
+      // compare tensors of different shapes, even if the total elements count
+      // matches.
+      bool dims_are_equal = true;
+      for (int i = 0; i < lhs.dims->size; i++) {
+        dims_are_equal &= lhs.dims->data[i] == rhs_.dims->data[i];
+      }
+      if (!dims_are_equal) {
+        std::string shape;
+        std::optional<std::string> result = ShapeToString(rhs_.dims);
+        if (result.has_value()) {
+          shape = std::move(result.value());
+        } else {
+          shape = "[error: unsupported number of dimensions]";
+        }
+        *listener << "which is different from the expected shape " << shape;
+        return false;
+      }
+
+      // 4. Proceed to data comparison. Iterate through elements as they lay
+      // flat. If some pair of elements don't match, deduct the coordinate
+      // basing on the dimensions, then return.
+      absl::Span<float> lhs_span(lhs.data.f, lhs.bytes / sizeof(float));
+      absl::Span<float> rhs_span(rhs_.data.f, rhs_.bytes / sizeof(float));
+
+      auto left = lhs_span.begin();
+      auto right = rhs_span.begin();
+      for (size_t i = 0; i != lhs_span.size(); ++i, ++left, ++right) {
+        if (listener->IsInterested()) {
+          testing::StringMatchResultListener inner_listener;
+          if (!mono_tuple_matcher_.MatchAndExplain({*left, *right},
+                                                   &inner_listener)) {
+            *listener << "where the value pair (";
+            testing::internal::UniversalPrint(*left, listener->stream());
+            *listener << ", ";
+            testing::internal::UniversalPrint(*right, listener->stream());
+            std::string coordinate;
+            std::optional<std::string> result = CoordinateToString(lhs.dims, i);
+            if (result.has_value()) {
+              coordinate = std::move(result.value());
+            } else {
+              coordinate = "[error: unsupported number of dimensions]";
+            }
+            *listener << ") with coordinate " << coordinate << " don't match";
+            testing::internal::PrintIfNotEmpty(inner_listener.str(),
+                                               listener->stream());
+            return false;
+          }
+        } else {
+          if (!mono_tuple_matcher_.Matches({*left, *right})) return false;
+        }
+      }
+
+      return true;
+    }
+
+   private:
+    const testing::Matcher<InnerMatcherArg> mono_tuple_matcher_;
+    const TfLiteTensor rhs_;
+  };
+
+ private:
+  const TupleMatcher tuple_matcher_;
+  const TfLiteTensor rhs_;
+};
+
+// Builds interpreter for a model, allocates tensors.
+absl::Status BuildInterpreter(const Model* model,
+                              std::unique_ptr<Interpreter>* interpreter);
+
+// Allocates tensors for a given interpreter.
+absl::Status AllocateTensors(std::unique_ptr<Interpreter>* interpreter);
+
+// Modifies graph with given delegate.
+absl::Status ModifyGraphWithDelegate(std::unique_ptr<Interpreter>* interpreter,
+                                     TfLiteDelegate* delegate);
+
+// Initializes inputs with consequent values of some fixed range.
+void InitializeInputs(int left, int right,
+                      std::unique_ptr<Interpreter>* interpreter);
+
+// Invokes a prebuilt interpreter.
+absl::Status Invoke(std::unique_ptr<Interpreter>* interpreter);
+
+// Usability structure, which is used to pass parameters data to parameterized
+// tests.
+struct TestParams {
+  // A gtest name, which will be used for a generated tests.
+  std::string name;
+
+  // Function, which returns a TFLite model, associated with this test name.
+  std::vector<uint8_t> model;
+};
+
+// Defines how the TestParams should be printed into the command line if
+// something fails during testing.
+std::ostream& operator<<(std::ostream& os, const TestParams& param);
+
+}  // namespace tflite
+
+// Gtest framework uses this function to describe TfLiteTensor if something
+// fails. TfLiteTensor is defined in global namespace, same should be done for
+// streaming operator.
+std::ostream& operator<<(std::ostream& os, const TfLiteTensor& tensor);
+
+// Defines a matcher to compare two TfLiteTensors pointwise using the given
+// tuple matcher for comparing their values.
+template <typename TupleMatcherT>
+inline tflite::TensorEqMatcher<TupleMatcherT> TensorEq(
+    const TupleMatcherT& matcher, const TfLiteTensor& rhs) {
+  return tflite::TensorEqMatcher<TupleMatcherT>(matcher, rhs);
+}
+
+#endif  // TENSORFLOW_LITE_DELEGATES_GPU_COMMON_TESTING_FEATURE_PARITY_UTILS_H_

tensorflow/lite/delegates/gpu/common/testing/feature_parity/xnnpack_test.cc

@@ -0,0 +1,69 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <stdint.h>
+
+#include <memory>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+#include "tensorflow/lite/delegates/gpu/common/testing/feature_parity/feature_parity.h"
+#include "tensorflow/lite/delegates/gpu/common/testing/feature_parity/utils.h"
+#include "tensorflow/lite/delegates/xnnpack/xnnpack_delegate.h"
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/schema/schema_generated.h"
+
+namespace tflite {
+
+class XnnPackBackend : public testing::TestWithParam<TestParams> {};
+
+TEST_P(XnnPackBackend, DelegateTest) {
+  std::unique_ptr<TfLiteDelegate, decltype(&TfLiteXNNPackDelegateDelete)>
+      xnnpack_delegate(TfLiteXNNPackDelegateCreate(nullptr),
+                       TfLiteXNNPackDelegateDelete);
+  const TestParams& param = GetParam();
+  const std::vector<uint8_t>& model_binary = param.model;
+  ASSERT_NE(model_binary.empty(), true);
+  std::unique_ptr<Interpreter> tflite_cpu_interpreter;
+  ASSERT_OK(
+      BuildInterpreter(GetModel(model_binary.data()), &tflite_cpu_interpreter));
+  ASSERT_OK(AllocateTensors(&tflite_cpu_interpreter));
+  std::unique_ptr<Interpreter> xnnpack_interpreter;
+  ASSERT_OK(
+      BuildInterpreter(GetModel(model_binary.data()), &xnnpack_interpreter));
+  // Ensures that tensors are allocated.
+  ASSERT_OK(
+      ModifyGraphWithDelegate(&xnnpack_interpreter, xnnpack_delegate.get()));
+  // Inputs are initialized with consequent values of the fixed range.
+  InitializeInputs(/*left=*/0, /*right=*/100, &tflite_cpu_interpreter);
+  InitializeInputs(/*left=*/0, /*right=*/100, &xnnpack_interpreter);
+  ASSERT_OK(Invoke(&tflite_cpu_interpreter));
+  ASSERT_OK(Invoke(&xnnpack_interpreter));
+  for (int i = 0; i < tflite_cpu_interpreter->outputs().size(); ++i) {
+    int id = tflite_cpu_interpreter->outputs()[i];
+    const TfLiteTensor* cpu = tflite_cpu_interpreter->tensor(id);
+    const TfLiteTensor* xnnpack = xnnpack_interpreter->tensor(id);
+    EXPECT_THAT(*xnnpack, TensorEq(testing::FloatNear(1e-6), *cpu))
+        << " for output tensor #" << i << " with id " << id;
+  }
+}
+
+INSTANTIATE_TEST_SUITE_P(FeatureParityTests, XnnPackBackend,
+                         testing::ValuesIn(GetFeatureParity()),
+                         [](const testing::TestParamInfo<TestParams>& info) {
+                           return info.param.name;
+                         });
+}  // namespace tflite

tensorflow/lite/delegates/gpu/common/testing/interpreter_utils.cc

@@ -0,0 +1,119 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/delegates/gpu/common/testing/interpreter_utils.h"
+
+#include <cstring>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "tensorflow/lite/core/api/op_resolver.h"
+#include "tensorflow/lite/core/interpreter_builder.h"
+#include "tensorflow/lite/core/kernels/register.h"
+#include "tensorflow/lite/delegates/gpu/common/status.h"
+#include "tensorflow/lite/delegates/gpu/common/tensor.h"
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/schema/schema_generated.h"
+
+namespace tflite {
+namespace gpu {
+namespace testing {
+
+absl::Status InterpreterInvokeWithOpResolver(
+    const ::tflite::Model* model, TfLiteDelegate* delegate,
+    const OpResolver& op_resolver, const std::vector<TensorFloat32>& inputs,
+    std::vector<TensorFloat32>* outputs) {
+  auto interpreter = std::make_unique<Interpreter>();
+  if (InterpreterBuilder(model, op_resolver)(&interpreter) != kTfLiteOk) {
+    return absl::InternalError("Unable to create TfLite InterpreterBuilder");
+  }
+  if (delegate && interpreter->ModifyGraphWithDelegate(delegate) != kTfLiteOk) {
+    return absl::InternalError(
+        "Unable to modify TfLite graph with the delegate");
+  }
+  interpreter->SetNumThreads(1);
+  if (interpreter->AllocateTensors() != kTfLiteOk) {
+    return absl::InternalError("Unable to allocate TfLite tensors");
+  }
+  for (int i = 0; i < inputs.size(); ++i) {
+    if (interpreter->tensor(interpreter->inputs()[i])->type != kTfLiteFloat32) {
+      return absl::InternalError("input data_type is not float32");
+    }
+    float* tflite_data =
+        interpreter->typed_tensor<float>(interpreter->inputs()[i]);
+    if (inputs[i].data.size() * sizeof(float) >
+        interpreter->tensor(interpreter->inputs()[i])->bytes) {
+      return absl::InternalError("too big input data");
+    }
+    std::memcpy(tflite_data, inputs[i].data.data(),
+                inputs[i].data.size() * sizeof(float));
+  }
+  if (interpreter->Invoke() != kTfLiteOk) {
+    return absl::InternalError("Unable to invoke TfLite interpreter");
+  }
+  if (!outputs || !outputs->empty()) {
+    return absl::InternalError("Invalid outputs pointer");
+  }
+  outputs->reserve(interpreter->outputs().size());
+  for (auto t : interpreter->outputs()) {
+    const TfLiteTensor* out_tensor = interpreter->tensor(t);
+    TensorFloat32 bhwc;
+    bhwc.id = t;
+    // TODO(impjdi) Relax this condition to arbitrary batch size.
+    if (out_tensor->dims->data[0] != 1) {
+      return absl::InternalError("Batch dimension is expected to be 1");
+    }
+    bhwc.shape.b = out_tensor->dims->data[0];
+    switch (out_tensor->dims->size) {
+      case 2:
+        bhwc.shape.h = 1;
+        bhwc.shape.w = 1;
+        bhwc.shape.c = out_tensor->dims->data[1];
+        break;
+      case 3:
+        bhwc.shape.h = 1;
+        bhwc.shape.w = out_tensor->dims->data[1];
+        bhwc.shape.c = out_tensor->dims->data[2];
+        break;
+      case 4:
+        bhwc.shape.h = out_tensor->dims->data[1];
+        bhwc.shape.w = out_tensor->dims->data[2];
+        bhwc.shape.c = out_tensor->dims->data[3];
+        break;
+      default:
+        return absl::InternalError("Unsupported dimensions size " +
+                                   std::to_string(out_tensor->dims->size));
+    }
+    bhwc.data = std::vector<float>(
+        out_tensor->data.f,
+        out_tensor->data.f + out_tensor->bytes / sizeof(float));
+    outputs->push_back(bhwc);
+  }
+  return absl::OkStatus();
+}
+
+absl::Status InterpreterInvoke(const ::tflite::Model* model,
+                               TfLiteDelegate* delegate,
+                               const std::vector<TensorFloat32>& inputs,
+                               std::vector<TensorFloat32>* outputs) {
+  ops::builtin::BuiltinOpResolver builtin_op_resolver;
+  return InterpreterInvokeWithOpResolver(model, delegate, builtin_op_resolver,
+                                         inputs, outputs);
+}
+
+}  // namespace testing
+}  // namespace gpu
+}  // namespace tflite

tensorflow/lite/delegates/gpu/common/testing/interpreter_utils.h

@@ -0,0 +1,51 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_DELEGATES_GPU_COMMON_TESTING_INTERPRETER_UTILS_H_
+#define TENSORFLOW_LITE_DELEGATES_GPU_COMMON_TESTING_INTERPRETER_UTILS_H_
+
+#include <vector>
+
+#include "tensorflow/lite/core/api/op_resolver.h"
+#include "tensorflow/lite/core/c/common.h"
+#include "tensorflow/lite/delegates/gpu/common/status.h"
+#include "tensorflow/lite/delegates/gpu/common/tensor.h"
+#include "tensorflow/lite/schema/schema_generated.h"
+
+namespace tflite {
+namespace gpu {
+namespace testing {
+
+// Runs Tensorflow Lite model using Tensorflow Lite with a delegate and
+// an appropriate operations resolver. If delegate is nullptr, inference will
+// be done only on CPU.
+absl::Status InterpreterInvokeWithOpResolver(
+    const ::tflite::Model* model, TfLiteDelegate* delegate,
+    const OpResolver& op_resolver, const std::vector<TensorFloat32>& inputs,
+    std::vector<TensorFloat32>* outputs);
+
+// Runs Tensorflow Lite model using Tensorflow Lite with a delegate and
+// builtin operations resolver. If delegate is nullptr, inference will
+// be done only on CPU.
+absl::Status InterpreterInvoke(const ::tflite::Model* model,
+                               TfLiteDelegate* delegate,
+                               const std::vector<TensorFloat32>& inputs,
+                               std::vector<TensorFloat32>* outputs);
+
+}  // namespace testing
+}  // namespace gpu
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_DELEGATES_GPU_COMMON_TESTING_INTERPRETER_UTILS_H_