chore: import upstream snapshot with attribution

tensorflow/lite/testing/op_tests/abs.py

@@ -0,0 +1,68 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for abs."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_abs_tests(options):
+  """Make a set of tests to do abs."""
+
+  # Chose a set of parameters
+  test_parameters = [{
+      "input_shape": [[], [1], [2, 3], [1, 1, 1, 1], [1, 3, 4, 3],
+                      [3, 15, 14, 3], [3, 1, 2, 4, 6], [2, 2, 3, 4, 5, 6]],
+      "dtype": [tf.float32],
+      "dynamic_range_quantize": [False, True],
+      "fully_quantize": [False],
+      "input_range": [(-10, 10), (-10, 0)],
+  }, {
+      "input_shape": [[], [1], [2, 3], [1, 1, 1, 1], [1, 3, 4, 3],
+                      [3, 15, 14, 3], [3, 1, 2, 4, 6], [2, 2, 3, 4, 5, 6]],
+      "dtype": [tf.float32],
+      "dynamic_range_quantize": [False],
+      "fully_quantize": [True],
+      "input_range": [(-10, 10)],
+  }, {
+      "input_shape": [[], [1], [2, 3], [1, 1, 1, 1],
+                      [1, 3, 4, 3], [3, 15, 14, 3],
+                      [3, 1, 2, 4, 6], [2, 2, 3, 4, 5, 6]],
+      "dtype": [tf.int16],
+  }]
+
+  def build_graph(parameters):
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+    out = tf.abs(input_tensor)
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    min_value, max_value = (-10, 10)
+    if "input_range" in parameters:
+      min_value, max_value = parameters["input_range"]
+    input_values = create_tensor_data(
+        parameters["dtype"],
+        parameters["input_shape"],
+        min_value=min_value,
+        max_value=max_value)
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/add_n.py

@@ -0,0 +1,74 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for add_n."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_add_n_tests(options):
+  """Make a set of tests for AddN op."""
+
+  test_parameters = [
+      {
+          "dtype": [tf.float32, tf.int32],
+          "input_shape": [[2, 5, 3, 1]],
+          "num_inputs": [2, 3, 4, 5],
+          "dynamic_range_quantize": [False],
+      },
+      {
+          "dtype": [tf.float32, tf.int32],
+          "input_shape": [[5]],
+          "num_inputs": [2, 3, 4, 5],
+          "dynamic_range_quantize": [False],
+      },
+      {
+          "dtype": [tf.float32, tf.int32],
+          "input_shape": [[]],
+          "num_inputs": [2, 3, 4, 5],
+          "dynamic_range_quantize": [False],
+      },
+      {
+          "dtype": [tf.float32],
+          "input_shape": [[]],
+          "num_inputs": [2, 3, 4, 5],
+          "dynamic_range_quantize": [True],
+      },
+  ]
+
+  def build_graph(parameters):
+    """Builds the graph given the current parameters."""
+    input_tensors = []
+    for i in range(parameters["num_inputs"]):
+      input_tensors.append(
+          tf.compat.v1.placeholder(
+              dtype=parameters["dtype"],
+              name="input_{}".format(i),
+              shape=parameters["input_shape"]))
+    out = tf.add_n(input_tensors)
+    return input_tensors, [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    """Builds operand inputs for op."""
+    input_data = []
+    for _ in range(parameters["num_inputs"]):
+      input_data.append(
+          create_tensor_data(parameters["dtype"], parameters["input_shape"]))
+    return input_data, sess.run(
+        outputs, feed_dict={i: d for i, d in zip(inputs, input_data)})
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/arg_min_max.py

@@ -0,0 +1,85 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for arg_min_max."""
+import random
+
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_arg_min_max_tests(options):
+  """Make a set of tests to do arg_max."""
+
+  test_parameters = [
+      {
+          "input_dtype": [tf.float32, tf.int32],
+          "input_shape": [[], [1, 1, 1, 3], [2, 3, 4, 5], [2, 3, 3], [5, 5],
+                          [10]],
+          "output_type": [tf.int32, tf.int64],
+          "is_arg_max": [True],
+          "is_last_axis": [False],
+          "dynamic_range_quantize": [False, True],
+      },
+      {
+          "input_dtype": [tf.float32, tf.int32],
+          "input_shape": [[1], [2], [3], [4], [5], [6], [7], [8], [9], [10],
+                          [2, 10], [3, 4, 50], [2, 3, 5, 100]],
+          "output_type": [tf.int32, tf.int64],
+          "is_arg_max": [False, True],
+          "is_last_axis": [True],
+          "dynamic_range_quantize": [False, True],
+      },
+      {
+          "input_dtype": [tf.bool],
+          "input_shape": [[1, 1, 1, 3], [2, 3, 4, 5], [2, 3, 3], [5, 5], [10]],
+          "output_type": [tf.int32, tf.int64],
+          "is_arg_max": [True],
+          "is_last_axis": [False],
+      },
+  ]
+
+  def build_graph(parameters):
+    """Build the topk op testing graph."""
+    input_value = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+    if not parameters["is_last_axis"]:
+      axis = random.randint(0, max(len(parameters["input_shape"]) - 1, 0))
+    else:
+      axis = -1
+    if parameters["is_arg_max"]:
+      out = tf.math.argmax(
+          input=input_value, axis=axis, output_type=parameters["output_type"])
+    else:
+      out = tf.math.argmin(
+          input=input_value, axis=axis, output_type=parameters["output_type"])
+    return [input_value], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value = create_tensor_data(parameters["input_dtype"],
+                                     parameters["input_shape"])
+    return [input_value], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_value])))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=8)

tensorflow/lite/testing/op_tests/atan2.py

@@ -0,0 +1,52 @@
+# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for atan2."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_atan2_tests(options):
+  """Make a set of tests to do atan2."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32, tf.float64],
+      "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]]
+  }]
+
+  def build_graph(parameters):
+    """Build the atan2 op testing graph."""
+    input_value1 = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="y",
+        shape=parameters["input_shape"])
+    input_value2 = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="x",
+        shape=parameters["input_shape"])
+    out = tf.math.atan2(input_value1, input_value2)
+    return [input_value1, input_value2], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value1 = create_tensor_data(parameters["input_dtype"],
+                                      parameters["input_shape"])
+    input_value2 = create_tensor_data(parameters["input_dtype"],
+                                      parameters["input_shape"])
+    return [input_value1, input_value2], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_value1, input_value2])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/batch_to_space_nd.py

@@ -0,0 +1,135 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for batch_to_space_nd."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_batch_to_space_nd_tests(options):
+  """Make a set of tests to do batch_to_space_nd."""
+
+  test_parameters = [
+      {
+          "dtype": [tf.float32, tf.int64, tf.int32],
+          "input_shape": [[12, 3, 3, 1]],
+          "block_shape": [[1, 4], [2, 2], [3, 4]],
+          "crops": [[[0, 0], [0, 0]], [[1, 1], [1, 1]]],
+          "constant_block_shape": [True, False],
+          "constant_crops": [True, False],
+          "dynamic_range_quantize": [False],
+      },
+      # Single batch (no-op)
+      {
+          "dtype": [tf.float32],
+          "input_shape": [[1, 3, 3, 1]],
+          "block_shape": [[1, 1]],
+          "crops": [[[0, 0], [0, 0]], [[1, 1], [1, 1]]],
+          "constant_block_shape": [True],
+          "constant_crops": [True],
+          "dynamic_range_quantize": [True, False],
+      },
+      {
+          "dtype": [tf.float32],
+          "input_shape": [[1, 3, 3, 1]],
+          "block_shape": [[1, 1]],
+          "crops": [[[0, 0], [0, 0]], [[1, 1], [1, 1]]],
+          "constant_block_shape": [True],
+          "constant_crops": [True],
+          "fully_quantize": [True],
+          "quant_16x8": [False, True],
+      },
+      # 3D use case.
+      {
+          "dtype": [tf.float32],
+          "input_shape": [[1, 3, 3]],
+          "block_shape": [[1]],
+          "crops": [[[0, 0]], [[1, 1]]],
+          "constant_block_shape": [True],
+          "constant_crops": [True],
+          "dynamic_range_quantize": [True, False],
+      },
+      {
+          "dtype": [tf.float32],
+          "input_shape": [[1, 3, 3]],
+          "block_shape": [[1]],
+          "crops": [[[0, 0]], [[1, 1]]],
+          "constant_block_shape": [True],
+          "constant_crops": [True],
+          "fully_quantize": [True],
+          "quant_16x8": [False, True],
+      },
+  ]
+
+  if options.run_with_flex:
+    # Non-4D use case: 1 batch dimension, 3 spatial dimensions, 2 others.
+    test_parameters = test_parameters + [{
+        "dtype": [tf.float32],
+        "input_shape": [[8, 2, 2, 2, 1, 1]],
+        "block_shape": [[2, 2, 2]],
+        "crops": [[[0, 0], [0, 0], [0, 0]]],
+        "constant_block_shape": [True, False],
+        "constant_crops": [True, False],
+        "dynamic_range_quantize": [False],
+    }]
+
+  def build_graph(parameters):
+    """Build a batch_to_space graph given `parameters`."""
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+    input_tensors = [input_tensor]
+
+    # Get block_shape either as a const or as a placeholder (tensor).
+    if parameters["constant_block_shape"]:
+      block_shape = parameters["block_shape"]
+    else:
+      shape = [len(parameters["block_shape"])]
+      block_shape = tf.compat.v1.placeholder(
+          dtype=tf.int32, name="shape", shape=shape)
+      input_tensors.append(block_shape)
+
+    # Get crops either as a const or as a placeholder (tensor).
+    if parameters["constant_crops"]:
+      crops = parameters["crops"]
+    else:
+      shape = [len(parameters["crops"]), 2]
+      crops = tf.compat.v1.placeholder(
+          dtype=tf.int32, name="crops", shape=shape)
+      input_tensors.append(crops)
+
+    out = tf.batch_to_space(input_tensor, block_shape, crops)
+    return input_tensors, [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    values = [
+        create_tensor_data(
+            parameters["dtype"],
+            parameters["input_shape"],
+            min_value=-1.0,
+            max_value=1.0,
+        )
+    ]
+    if not parameters["constant_block_shape"]:
+      values.append(np.array(parameters["block_shape"]))
+    if not parameters["constant_crops"]:
+      values.append(np.array(parameters["crops"]))
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/batchmatmul.py

@@ -0,0 +1,107 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for batchmatmul."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function("make_batchmatmul_tests")
+def make_batchmatmul_tests(options):
+  """Make a set of tests to do basic batch matrix multiply."""
+
+  test_parameters = [
+      {
+          "dtype": [tf.float32],
+          "shapes": [((3, 4, 7), (7, 9), (3, 4, 7), (7, 9)),
+                     ((None, 4, 5), (None, 5, 6), (3, 4, 5), (3, 5, 6)),
+                     ((None, 1, 3, 4), (None, 4, 2), (2, 1, 3, 4), (5, 4, 2)),
+                     ((None, None, None, 3, 4), (None, None, None, 4, 3),
+                      (2, 2, 2, 3, 4), (2, 2, 2, 4, 3))],
+          "adjoint_b": [False, True],
+          "adjoint_a": [False, True],
+          "rhs_constant": [False],
+          "fully_quantize": [False, True],
+      },
+  ]
+
+  def swap_last_two_dims(*args):
+    """Return a tuple with the last two dimensions swapped."""
+    return args[:-2] + (args[-1],) + (args[-2],)
+
+  def build_graph(parameters):
+    """Build a simple graph with BatchMatMul."""
+    placeholder0_shape = parameters["shapes"][0]
+    adj_a = parameters["adjoint_a"]
+    adj_b = parameters["adjoint_b"]
+    rhs_constant = parameters["rhs_constant"]
+    if adj_a:
+      placeholder0_shape = swap_last_two_dims(*placeholder0_shape)
+    input0_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"], shape=placeholder0_shape)
+    if rhs_constant:
+      if adj_b:
+        constant1_shape = swap_last_two_dims(*parameters["shapes"][3])
+      else:
+        constant1_shape = parameters["shapes"][3]
+      data = create_tensor_data(
+          parameters["dtype"], constant1_shape, min_value=-1.0, max_value=1.0)
+      input1_constant = tf.constant(
+          data, shape=constant1_shape, dtype=parameters["dtype"])
+      out = tf.matmul(
+          input0_tensor, input1_constant, adjoint_a=adj_a, adjoint_b=adj_b)
+      return [input0_tensor], [out]
+    else:
+      if adj_b:
+        placeholder1_shape = swap_last_two_dims(*parameters["shapes"][1])
+      else:
+        placeholder1_shape = parameters["shapes"][1]
+      input1_tensor = tf.compat.v1.placeholder(
+          dtype=parameters["dtype"], shape=placeholder1_shape)
+      out = tf.matmul(
+          input0_tensor, input1_tensor, adjoint_a=adj_a, adjoint_b=adj_b)
+      return [input0_tensor, input1_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    """Feed inputs, assign variables, and freeze graph."""
+    input0_shape = parameters["shapes"][2]
+    adj_a = parameters["adjoint_a"]
+    adj_b = parameters["adjoint_b"]
+    rhs_constant = parameters["rhs_constant"]
+    if adj_a:
+      input0_shape = swap_last_two_dims(*input0_shape)
+    input0_value = create_tensor_data(
+        parameters["dtype"], input0_shape, min_value=-1.0, max_value=1.0)
+    if rhs_constant:
+      output_values = sess.run(
+          outputs, feed_dict=dict(zip(inputs, [input0_value])))
+      return [input0_value], output_values
+    else:
+      input1_shape = parameters["shapes"][
+          3] if not adj_b else swap_last_two_dims(*parameters["shapes"][3])
+      input1_value = create_tensor_data(
+          parameters["dtype"], input1_shape, min_value=-1.0, max_value=1.0)
+      output_values = sess.run(
+          outputs, feed_dict=dict(zip(inputs, [input0_value, input1_value])))
+      return [input0_value, input1_value], output_values
+
+  options.disable_batchmatmul_unfold = True
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=0)

tensorflow/lite/testing/op_tests/binary_op.py

@@ -0,0 +1,393 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for binary_op."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+def make_binary_op_tests(options,
+                         binary_operator,
+                         allow_fully_quantize=False,
+                         expected_tf_failures=0,
+                         test_parameters=None):
+  """Make a set of tests to do binary ops with and without broadcast."""
+
+  if test_parameters is None:
+    test_parameters = []
+
+  test_parameters = test_parameters + [
+      # Avoid creating all combinations to keep the test size small.
+      {
+          "dtype": [tf.float32, tf.int32],
+          "input_shape_1": [[1, 3, 4, 3]],
+          "input_shape_2": [[1, 3, 4, 3]],
+          "activation": [True],
+          "fully_quantize": [False],
+          "dynamic_range_quantize": [False],
+      },
+      {
+          "dtype": [tf.float32],
+          "input_shape_1": [[5]],
+          "input_shape_2": [[5]],
+          "activation": [False, True],
+          "fully_quantize": [False],
+          "dynamic_range_quantize": [False],
+      },
+      {
+          "dtype": [tf.float32, tf.int32, tf.int64],
+          "input_shape_1": [[1, 3, 4, 3]],
+          "input_shape_2": [[3]],
+          "activation": [True, False],
+          "fully_quantize": [False],
+          "dynamic_range_quantize": [False],
+      },
+      {
+          "dtype": [tf.float32, tf.int32],
+          "input_shape_1": [[3]],
+          "input_shape_2": [[1, 3, 4, 3]],
+          "activation": [True, False],
+          "fully_quantize": [False],
+          "dynamic_range_quantize": [False],
+      },
+      {
+          "dtype": [tf.float32],
+          "input_shape_1": [[]],
+          "input_shape_2": [[]],
+          "activation": [False],
+          "fully_quantize": [False],
+          "dynamic_range_quantize": [False],
+      },
+      {
+          "dtype": [tf.float32],
+          "input_shape_1": [[1, 3, 4, 3]],
+          "input_shape_2": [[1, 3, 4, 3]],
+          "activation": [False],
+          "fully_quantize": [True],
+          "dynamic_range_quantize": [False],
+      },
+      {
+          "dtype": [tf.float32],
+          "input_shape_1": [[5]],
+          "input_shape_2": [[5]],
+          "activation": [False],
+          "fully_quantize": [True],
+          "dynamic_range_quantize": [False],
+      },
+      {
+          "dtype": [tf.float32],
+          "input_shape_1": [[1, 3, 4, 3]],
+          "input_shape_2": [[3]],
+          "activation": [False],
+          "fully_quantize": [True],
+          "dynamic_range_quantize": [False],
+      },
+      {
+          "dtype": [tf.float32],
+          "input_shape_1": [[3]],
+          "input_shape_2": [[1, 3, 4, 3]],
+          "activation": [False],
+          "fully_quantize": [True],
+          "dynamic_range_quantize": [False],
+      },
+      {
+          "dtype": [tf.float32],
+          "input_shape_1": [[]],
+          "input_shape_2": [[]],
+          "activation": [False],
+          "fully_quantize": [True],
+          "dynamic_range_quantize": [False],
+      },
+      {
+          "dtype": [tf.float32],
+          "input_shape_1": [[1, 3, 4, 3]],
+          "input_shape_2": [[1, 3, 4, 3]],
+          "activation": [False],
+          "fully_quantize": [False],
+          "dynamic_range_quantize": [True],
+      },
+      {
+          "dtype": [tf.float32],
+          "input_shape_1": [[5]],
+          "input_shape_2": [[5]],
+          "activation": [False],
+          "fully_quantize": [False],
+          "dynamic_range_quantize": [True],
+      },
+      {
+          "dtype": [tf.float32],
+          "input_shape_1": [[1, 3, 4, 3]],
+          "input_shape_2": [[3]],
+          "activation": [False],
+          "fully_quantize": [False],
+          "dynamic_range_quantize": [True],
+      },
+      {
+          "dtype": [tf.float32],
+          "input_shape_1": [[3]],
+          "input_shape_2": [[1, 3, 4, 3]],
+          "activation": [False],
+          "fully_quantize": [False],
+          "dynamic_range_quantize": [True],
+      },
+      {
+          "dtype": [tf.float32],
+          "input_shape_1": [[]],
+          "input_shape_2": [[]],
+          "activation": [False],
+          "fully_quantize": [False],
+          "dynamic_range_quantize": [True],
+      },
+  ]
+
+  # float64 types are supported via flex only.
+  if options.run_with_flex:
+    test_parameters = test_parameters + [
+        {
+            "dtype": [tf.float64],
+            "input_shape_1": [[7]],
+            "input_shape_2": [[7]],
+            "activation": [False],
+            "fully_quantize": [False],
+            "dynamic_range_quantize": [False],
+        },
+    ]
+
+  if not options.skip_high_dimension_inputs:
+    test_parameters = test_parameters + [
+        # High dimension broadcasting support in MLIR converter.
+        # Note(b/204360746): XNNPack delegate don't support high dimension.
+        {
+            "dtype": [tf.float32],
+            "input_shape_1": [[8, 7, 6, 5, 4, 3, 2, 1],
+                              [8, 7, 6, 5, None, 3, 2, 1], [2, None]],
+            "input_shape_2": [[4, 3, 2, 1], [None, 3, 2, 1]],
+            "activation": [False],
+            "fully_quantize": [False],
+            "dynamic_range_quantize": [False],
+            "dynamic_size_value": [4, 1],
+        }
+    ]
+
+  # test_parameters include fully_quantize option only when
+  # allow_fully_quantize is True.
+  if not allow_fully_quantize:
+    test_parameters = [
+        test_parameter for test_parameter in test_parameters
+        if True not in test_parameter["fully_quantize"]
+    ]
+
+  def populate_dynamic_shape(parameters, input_shape):
+    return [
+        parameters["dynamic_size_value"] if x is None else x
+        for x in input_shape
+    ]
+
+  def build_graph(parameters):
+    """Builds the graph given the current parameters."""
+    input1 = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"],
+        name="input1",
+        shape=parameters["input_shape_1"])
+    input2 = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"],
+        name="input2",
+        shape=parameters["input_shape_2"])
+    out = binary_operator(input1, input2)
+    if parameters["activation"] and (parameters["dtype"] != tf.int32 and
+                                     parameters["dtype"] != tf.int64):
+      out = tf.nn.relu(out)
+    return [input1, input2], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    """Builds operand inputs for op."""
+    input_shape_1 = populate_dynamic_shape(parameters,
+                                           parameters["input_shape_1"])
+    input_shape_2 = populate_dynamic_shape(parameters,
+                                           parameters["input_shape_2"])
+    if allow_fully_quantize:
+      input1 = create_tensor_data(
+          parameters["dtype"], input_shape_1, min_value=-1, max_value=1)
+      input2 = create_tensor_data(
+          parameters["dtype"], input_shape_2, min_value=-1, max_value=1)
+    else:
+      input1 = create_tensor_data(parameters["dtype"], input_shape_1)
+      input2 = create_tensor_data(parameters["dtype"], input_shape_2)
+    return [input1, input2], sess.run(
+        outputs, feed_dict={
+            inputs[0]: input1,
+            inputs[1]: input2
+        })
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=expected_tf_failures)
+
+
+def make_binary_op_tests_func(binary_operator):
+  """Return a function that does a test on a binary operator."""
+  return lambda options: make_binary_op_tests(options, binary_operator)
+
+
+@register_make_test_function()
+def make_add_tests(options):
+  """Make zip tests for add op with uint32 and int16 case."""
+  test_parameters = [
+      {
+          "dtype": [tf.uint32],
+          "input_shape_1": [[1, 3, 3, 3], [1], [3, 3]],
+          "input_shape_2": [[3], [1]],
+          "activation": [False],
+          "fully_quantize": [False],
+          "dynamic_range_quantize": [False],
+      },
+      {
+          "dtype": [tf.int16],
+          "input_shape_1": [[1, 3, 3, 3], [1], [3, 3]],
+          "input_shape_2": [[3], [1]],
+          "activation": [False],
+          "fully_quantize": [False],
+          "dynamic_range_quantize": [False],
+      },
+  ]
+  make_binary_op_tests(
+      options,
+      tf.add,
+      allow_fully_quantize=True,
+      test_parameters=test_parameters)
+
+
+@register_make_test_function()
+def make_div_tests(options):
+  """Make zip tests for div op with 5D case."""
+  test_parameters = [
+      {
+          "dtype": [tf.float32],
+          "input_shape_1": [[1, 3, 3, 3, 3]],
+          "input_shape_2": [[3]],
+          "activation": [False],
+          "fully_quantize": [False],
+          "dynamic_range_quantize": [False, True],
+      },
+  ]
+  make_binary_op_tests(
+      options, tf.compat.v1.div, test_parameters=test_parameters)
+
+
+@register_make_test_function()
+def make_sub_tests(options):
+  """Make zip tests for sub op with additional cases."""
+  test_parameters = [
+      {
+          "dtype": [tf.float32],
+          "input_shape_1": [[1, 3, 3, 3, 3]],
+          "input_shape_2": [[3]],
+          "activation": [False],
+          "fully_quantize": [False],
+          "dynamic_range_quantize": [False, True],
+      },
+  ]
+  make_binary_op_tests(
+      options,
+      tf.subtract,
+      allow_fully_quantize=True,
+      test_parameters=test_parameters)
+
+
+@register_make_test_function()
+def make_mul_tests(options):
+  """Make zip tests for mul op with additional complex cases."""
+  test_parameters = [
+      {
+          "dtype": [tf.complex64],
+          "input_shape_1": [[1, 3, 3, 3, 3]],
+          "input_shape_2": [[3]],
+          "activation": [False],
+          "fully_quantize": [False],
+          "dynamic_range_quantize": [False],
+      },
+      {
+          "dtype": [tf.int16],
+          "input_shape_1": [[1, 3, 3, 3]],
+          "input_shape_2": [[3], [1, 3, 3, 3]],
+          "activation": [False],
+          "fully_quantize": [False],
+          "dynamic_range_quantize": [False],
+      },
+      {
+          "dtype": [tf.uint32],
+          "input_shape_1": [[1, 3, 3, 3]],
+          "input_shape_2": [[3], [1, 3, 3, 3]],
+          "activation": [False],
+          "fully_quantize": [False],
+          "dynamic_range_quantize": [False],
+      },
+  ]
+  make_binary_op_tests(
+      options,
+      tf.multiply,
+      allow_fully_quantize=True,
+      test_parameters=test_parameters)
+
+
+@register_make_test_function()
+def make_pow_tests(options):
+  make_binary_op_tests(options, tf.pow, expected_tf_failures=7)
+
+
+@register_make_test_function()
+def make_floor_div_tests(options):
+  """Make zip tests for floor_div op with int16 case."""
+  test_parameters = [
+      {
+          "dtype": [tf.int8, tf.int16],
+          "input_shape_1": [[1, 3, 3, 3, 3]],
+          "input_shape_2": [[3]],
+          "activation": [False],
+          "fully_quantize": [False],
+          "dynamic_range_quantize": [False],
+      },
+  ]
+  make_binary_op_tests(
+      options, tf.math.floordiv, test_parameters=test_parameters
+  )
+
+
+@register_make_test_function()
+def make_floor_mod_tests(options):
+  """Make zip tests for floor_mod op with int16 case."""
+  test_parameters = [
+      {
+          "dtype": [tf.int8, tf.int16],
+          "input_shape_1": [[1, 3, 3, 3, 3]],
+          "input_shape_2": [[3]],
+          "activation": [False],
+          "fully_quantize": [False],
+          "dynamic_range_quantize": [False],
+      },
+  ]
+  make_binary_op_tests(
+      options, tf.math.floormod, test_parameters=test_parameters
+  )
+
+
+@register_make_test_function()
+def make_squared_difference_tests(options):
+  make_binary_op_tests(
+      options, tf.math.squared_difference, allow_fully_quantize=True)

tensorflow/lite/testing/op_tests/bitcast.py

@@ -0,0 +1,86 @@
+# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for bitcast."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_bitcast_tests(options):
+  """Generate examples for bitcast."""
+  test_parameters = [
+      {
+          "input_dtype": [tf.int32],
+          "output_dtype": [tf.uint32],
+          "input_shape": [[], [1], [1, 2], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.uint32],
+          "output_dtype": [tf.int32],
+          "input_shape": [[], [1], [1, 2], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.uint32],
+          "output_dtype": [tf.int16],
+          "input_shape": [[], [1], [1, 2], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.int16],
+          "output_dtype": [tf.uint32],
+          "input_shape": [[2], [1, 2], [1, 2, 2], [3, 4, 5, 6, 2]],
+      },
+      {
+          "input_dtype": [tf.int32],
+          "output_dtype": [tf.int16],
+          "input_shape": [[], [1], [1, 2], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.uint16],
+          "output_dtype": [tf.uint32],
+          "input_shape": [[2], [1, 2], [1, 2, 2], [3, 4, 5, 6, 2]],
+      },
+      {
+          "input_dtype": [tf.float32],
+          "output_dtype": [tf.int16],
+          "input_shape": [[], [1], [1, 2], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.int16],
+          "output_dtype": [tf.float32],
+          "input_shape": [[2], [1, 2], [1, 2, 2], [3, 4, 5, 6, 2]],
+      }
+  ]
+
+  def build_graph(parameters):
+    """Build the bitcast testing graph."""
+    input_value = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input",
+        shape=parameters["input_shape"],
+    )
+    out = tf.bitcast(input_value, parameters["output_dtype"])
+    return [input_value], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value = create_tensor_data(
+        parameters["input_dtype"], parameters["input_shape"]
+    )
+    return [input_value], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_value]))
+    )
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/bitwise_xor.py

@@ -0,0 +1,78 @@
+# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for bitwise_xor operator."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_bitwise_xor_tests(options):
+  """Generate examples for bitwise_xor."""
+  test_parameters = [
+      {
+          "input_dtype": [
+              tf.uint8,
+              tf.int8,
+              tf.uint16,
+              tf.int16,
+              tf.uint32,
+              tf.int32,
+          ],
+          "input_shape_pair": [
+              ([], []),
+              ([2, 3, 4], [2, 3, 4]),
+              ([1, 1, 1, 3], [1, 1, 1, 3]),
+              ([5, 5], [1]),
+              ([10], [2, 4, 10]),
+              ([2, 3, 3], [2, 3]),  # this test case is intended to fail
+          ],
+      },
+  ]
+
+  def build_graph(parameters):
+    """Build the bitwise_xor testing graph."""
+    input_value1 = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input1",
+        shape=parameters["input_shape_pair"][0],
+    )
+    input_value2 = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input2",
+        shape=parameters["input_shape_pair"][1],
+    )
+    out = tf.bitwise.bitwise_xor(input_value1, input_value2)
+    return [input_value1, input_value2], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value1 = create_tensor_data(
+        parameters["input_dtype"], parameters["input_shape_pair"][0]
+    )
+    input_value2 = create_tensor_data(
+        parameters["input_dtype"], parameters["input_shape_pair"][1]
+    )
+    return [input_value1, input_value2], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_value1, input_value2]))
+    )
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=6,
+  )

tensorflow/lite/testing/op_tests/broadcast_args.py

@@ -0,0 +1,62 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for broadcast_args."""
+import numpy as np
+import tensorflow as tf
+
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function("make_broadcast_args_tests")
+def make_broadcast_args_tests(options):
+  """Make a set of tests to do broadcast_args."""
+
+  # Chose a set of parameters
+  test_parameters = [{
+      "dtype": [tf.int64, tf.int32],
+      "input1_shape": [[1], [4], [3, 4], [1, 3, 4]],
+      "input2_shape": [[6, 4, 3, 4]],
+  }, {
+      "dtype": [tf.int64, tf.int32],
+      "input1_shape": [[1, 4, 0]],
+      "input2_shape": [[3, 1, 0], [3, 4, 1]],
+  }]
+
+  def build_graph(parameters):
+    """Build the graph for broadcast_args tests."""
+    shape1_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"],
+        name="input1",
+        shape=[len(parameters["input1_shape"])])
+    shape2_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"],
+        name="input2",
+        shape=[len(parameters["input2_shape"])])
+
+    out = tf.raw_ops.BroadcastArgs(s0=shape1_tensor, s1=shape2_tensor)
+    return [shape1_tensor, shape2_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_values = [
+        np.array(parameters["input1_shape"]).astype(
+            parameters["dtype"].as_numpy_dtype),
+        np.array(parameters["input2_shape"]).astype(
+            parameters["dtype"].as_numpy_dtype),
+    ]
+    return input_values, sess.run(
+        outputs, feed_dict=dict(zip(inputs, input_values)))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/broadcast_gradient_args.py

@@ -0,0 +1,68 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for broadcast_gradient_args."""
+import numpy as np
+import tensorflow as tf
+
+from tensorflow.lite.testing.zip_test_utils import ExtraConvertOptions
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_broadcast_gradient_args_tests(options):
+  """Make a set of tests to do broadcast_gradient_args."""
+
+  test_parameters = [{
+      'input_case': ['ALL_EQUAL', 'ONE_DIM', 'NON_BROADCASTABLE'],
+      'dtype': [tf.dtypes.int32, tf.dtypes.int64],
+  }]
+
+  def build_graph(parameters):
+    """Build the op testing graph."""
+    input1 = tf.compat.v1.placeholder(dtype=parameters['dtype'], name='input1')
+    input2 = tf.compat.v1.placeholder(dtype=parameters['dtype'], name='input2')
+    output1, output2 = tf.raw_ops.BroadcastGradientArgs(s0=input1, s1=input2)
+    return [input1, input2], [output1, output2]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    dtype = parameters['dtype'].as_numpy_dtype()
+
+    if parameters['input_case'] == 'ALL_EQUAL':
+      values = [
+          np.array([2, 4, 1, 3], dtype=dtype),
+          np.array([2, 4, 1, 3], dtype=dtype)
+      ]
+    elif parameters['input_case'] == 'ONE_DIM':
+      values = [
+          np.array([2, 4, 1, 3], dtype=dtype),
+          np.array([2, 1, 1, 3], dtype=dtype)
+      ]
+    elif parameters['input_case'] == 'NON_BROADCASTABLE':
+      values = [
+          np.array([2, 4, 1, 3], dtype=dtype),
+          np.array([2, 5, 1, 3], dtype=dtype)
+      ]
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  extra_convert_options = ExtraConvertOptions()
+  extra_convert_options.allow_custom_ops = True
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      extra_convert_options,
+      expected_tf_failures=2)

tensorflow/lite/testing/op_tests/broadcast_to.py

@@ -0,0 +1,72 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for broadcast_to."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function("make_broadcast_to_tests")
+def make_broadcast_to_tests(options):
+  """Make a set of tests to do broadcast_to."""
+
+  # Chose a set of parameters
+  test_parameters = [{
+      "input_dtype": [tf.float32, tf.int32],
+      "input_shape": [[1, 2], [2, 3, 4], [1], [2, 5, 2, 3, 4]],
+      "output_shape": [[3, 1, 2], [5, 2, 3, 4], [10, 10],
+                       [1, 2, 1, 2, 5, 2, 3, 4]],
+  }, {
+      "input_dtype": [tf.float32, tf.int32],
+      "input_shape": [[3, 2, 3, 4, 5, 6, 7, 8]],
+      "output_shape": [[3, 2, 3, 4, 5, 6, 7, 8]],
+  }, {
+      "input_dtype": [tf.float32, tf.int32],
+      "input_shape": [[1, 3, 1, 2, 1, 4, 1, 1]],
+      "output_shape": [[2, 3, 1, 2, 2, 4, 1, 1]],
+  }, {
+      "input_dtype": [tf.float32, tf.int32],
+      "input_shape": [[2, 1, 1, 2, 1, 4, 1, 1]],
+      "output_shape": [[2, 3, 2, 2, 2, 4, 1, 1]],
+  }, {
+      "input_dtype": [tf.float32, tf.int32],
+      "input_shape": [[3, 4, 1]],
+      "output_shape": [[3, 4, 0]],
+  }]
+
+  def build_graph(parameters):
+    """Build the graph for cond tests."""
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+
+    out = tf.broadcast_to(input_tensor, shape=parameters["output_shape"])
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_values = [
+        create_tensor_data(parameters["input_dtype"], parameters["input_shape"])
+    ]
+    return input_values, sess.run(
+        outputs, feed_dict=dict(zip(inputs, input_values)))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=16)

tensorflow/lite/testing/op_tests/cast.py

@@ -0,0 +1,148 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for cast."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_cast_tests(options):
+  """Generate examples for cast."""
+  test_parameters = [
+      {
+          "input_dtype": [tf.float32],
+          "output_dtype": [tf.int16],
+          "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.int16],
+          "output_dtype": [tf.float32],
+          "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.int32],
+          "output_dtype": [tf.float32],
+          "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.int8],
+          "output_dtype": [tf.float32],
+          "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.float32],
+          "output_dtype": [tf.int8],
+          "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.uint16],
+          "output_dtype": [tf.float32],
+          "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.uint32],
+          "output_dtype": [tf.int32],
+          "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.uint8],
+          "output_dtype": [tf.int8],
+          "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.int8],
+          "output_dtype": [tf.uint8],
+          "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.uint16],
+          "output_dtype": [tf.int16],
+          "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.int16],
+          "output_dtype": [tf.uint16],
+          "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.int32],
+          "output_dtype": [tf.float64],
+          "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.float64],
+          "output_dtype": [tf.int32],
+          "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.float32],
+          "output_dtype": [tf.float64],
+          "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.float64],
+          "output_dtype": [tf.float32],
+          "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.int64],
+          "output_dtype": [tf.float64],
+          "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.float64],
+          "output_dtype": [tf.int64],
+          "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.float32],
+          "output_dtype": [tf.float16],
+          "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.float16],
+          "output_dtype": [tf.float32],
+          "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.bfloat16],
+          "output_dtype": [tf.float32],
+          "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+      },
+      {
+          "input_dtype": [tf.float32],
+          "output_dtype": [tf.bfloat16],
+          "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+      },
+  ]
+
+  def build_graph(parameters):
+    """Build the cast testing graph."""
+    input_value = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+    out = tf.cast(input_value, parameters["output_dtype"])
+    return [input_value], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value = create_tensor_data(parameters["input_dtype"],
+                                     parameters["input_shape"])
+    return [input_value], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_value])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/ceil.py

@@ -0,0 +1,45 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for ceil."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_ceil_tests(options):
+  """Make a set of tests to do ceil."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32],
+      "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+  }]
+
+  def build_graph(parameters):
+    """Build the ceil op testing graph."""
+    input_value = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input1",
+        shape=parameters["input_shape"])
+    out = tf.math.ceil(input_value)
+    return [input_value], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value = create_tensor_data(parameters["input_dtype"],
+                                     parameters["input_shape"])
+    return [input_value], sess.run(outputs, feed_dict={inputs[0]: input_value})
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/complex_abs.py

@@ -0,0 +1,54 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for complex abs."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_complex_abs_tests(options):
+  """Make a set of tests to do complex abs."""
+
+  # Chose a set of parameters
+  test_parameters = [{
+      "dtype": [tf.complex64],
+      "input_shape": [[], [1], [2, 3], [1, 3, 4, 3], [2, 2, 3, 4, 5, 6]],
+      "Tout": [tf.float32]
+  }, {
+      "dtype": [tf.complex128],
+      "input_shape": [[], [1], [2, 3], [1, 3, 4, 3], [2, 2, 3, 4, 5, 6]],
+      "Tout": [tf.float64]
+  }]
+
+  def build_graph(parameters):
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+    out = tf.raw_ops.ComplexAbs(x=input_tensor, Tout=parameters["Tout"])
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_values = create_tensor_data(
+        parameters["dtype"].as_numpy_dtype,
+        parameters["input_shape"],
+        min_value=-10,
+        max_value=10)
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/concat.py

@@ -0,0 +1,106 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for concat."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_concat_tests(options):
+  """Make a set of tests to do concatenation."""
+
+  test_parameters = [{
+      "base_shape": [[1, 3, 4, 3], [3, 4]],
+      "num_tensors": [1, 2, 3, 4, 5, 6],
+      "axis": [0, 1, 2, 3, -3, -2, -1],
+      "type": [tf.float32, tf.uint8, tf.int32, tf.int64],
+      "fully_quantize": [False],
+      "quant_16x8": [False],
+      "dynamic_range_quantize": [False],
+  }, {
+      "base_shape": [[1, 3, 4, 3], [3, 4], [2, 3, 4, 3]],
+      "num_tensors": [1, 2, 3, 4, 5, 6],
+      "axis": [1, 2, 3, -3, -2, -1],
+      "type": [tf.float32],
+      "fully_quantize": [True],
+      "quant_16x8": [False],
+      "dynamic_range_quantize": [False],
+  }, {
+      "base_shape": [[1, 3, 4, 3]],
+      "num_tensors": [6],
+      "axis": [-1],
+      "type": [tf.float32],
+      "fully_quantize": [True],
+      "quant_16x8": [True],
+      "dynamic_range_quantize": [False],
+  }, {
+      "base_shape": [[1, 3, 4, 3]],
+      "num_tensors": [6],
+      "axis": [1],
+      "type": [tf.float32],
+      "fully_quantize": [False],
+      "quant_16x8": [False],
+      "dynamic_range_quantize": [True],
+  }, {
+      "base_shape": [[1, 3, 4, 3]],
+      "num_tensors": [6],
+      "axis": [1],
+      "type": [tf.bool],
+      "fully_quantize": [False],
+      "quant_16x8": [False],
+      "dynamic_range_quantize": [True],
+  }]
+
+  def get_shape(parameters, delta):
+    """Return a tweaked version of 'base_shape'."""
+    axis = parameters["axis"]
+    shape = parameters["base_shape"][:]
+    if axis < 0:
+      axis += len(shape)
+    if axis < len(shape):
+      shape[axis] += delta
+    return shape
+
+  def build_graph(parameters):
+    all_tensors = []
+    for n in range(0, parameters["num_tensors"]):
+      input_tensor = tf.compat.v1.placeholder(
+          dtype=parameters["type"],
+          name=("input%d" % n),
+          shape=get_shape(parameters, n))
+      all_tensors.append(input_tensor)
+    out = tf.concat(all_tensors, parameters["axis"])
+    return all_tensors, [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    all_values = []
+    for n in range(0, parameters["num_tensors"]):
+      input_values = create_tensor_data(
+          parameters["type"],
+          get_shape(parameters, n),
+          min_value=-1,
+          max_value=1)
+      all_values.append(input_values)
+    return all_values, sess.run(
+        outputs, feed_dict=dict(zip(inputs, all_values)))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=75)

tensorflow/lite/testing/op_tests/cond.py

@@ -0,0 +1,62 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for cond."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+from tensorflow.python.framework import test_util
+
+
+@register_make_test_function("make_cond_tests")
+@test_util.enable_control_flow_v2
+def make_cond_tests(options):
+  """Make a set of tests to do relu1."""
+
+  # Chose a set of parameters
+  test_parameters = [{
+      # Note: The `tf.string` test case also serves as a regression test to
+      # ensure that branch subgraph with dynamically allocated inputs/outputs
+      # are handled correctly.
+      "dtype": [tf.float32, tf.string],
+      "pred": [False, True],
+  }]
+
+  def build_graph(parameters):
+    """Build the graph for cond tests."""
+    input1 = tf.compat.v1.placeholder(dtype=parameters["dtype"], shape=(1,))
+    input2 = tf.compat.v1.placeholder(dtype=parameters["dtype"], shape=(1,))
+    # MLIR TFLite converter can't handle scalar inputs. This is a workaround
+    # to input (1,) tensors and then reshape to scalar.
+    # TODO(b/129003347): Remove the workaround after scalar inputs are
+    # supported.
+    pred = tf.compat.v1.placeholder(dtype=tf.bool, shape=(1,))
+    pred_scalar = tf.reshape(pred, ())
+
+    out = tf.cond(
+        pred=pred_scalar, true_fn=lambda: input1, false_fn=lambda: input2)
+    return [input1, input2, pred], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_values = [
+        create_tensor_data(parameters["dtype"], (1,)),
+        create_tensor_data(parameters["dtype"], (1,)),
+        np.array([parameters["pred"]], dtype=np.bool_),
+    ]
+    return input_values, sess.run(
+        outputs, feed_dict=dict(zip(inputs, input_values)))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/constant.py

@@ -0,0 +1,66 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for constant ops."""
+import numpy as np
+import tensorflow as tf
+
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import MAP_TF_TO_NUMPY_TYPE
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+# This function tests various TensorFLow functions that generates Const op,
+# including `tf.ones`, `tf.zeros` and random functions.
+@register_make_test_function()
+def make_constant_tests(options):
+  """Make a set of tests to do constant ops."""
+
+  test_parameters = [{
+      "dtype": [tf.float32, tf.int32],
+      "input_shape": [[], [1], [2], [1, 1, 1, 1], [2, 2, 2, 2]],
+      "constant_is_also_output": [True, False],
+      # Models should not be rejected regardless whether it has unread inputs.
+      "has_unread_input": [True, False],
+  }]
+
+  def build_graph(parameters):
+    """Build a constant graph given `parameters`."""
+    dummy_input = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"],
+        name="input1",
+        shape=parameters["input_shape"])
+    constant = tf.constant(
+        create_tensor_data(parameters["dtype"], parameters["input_shape"]))
+    outputs = [tf.maximum(dummy_input, constant)]
+    if parameters["constant_is_also_output"]:
+      outputs.append(constant)
+    inputs = [dummy_input]
+    if parameters["has_unread_input"]:
+      unread_input = tf.compat.v1.placeholder(
+          dtype=parameters["dtype"],
+          name="unread_input",
+          shape=parameters["input_shape"])
+      inputs.append(unread_input)
+
+    return inputs, outputs
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    dummy_input = np.zeros(
+        parameters["input_shape"],
+        dtype=MAP_TF_TO_NUMPY_TYPE[parameters["dtype"]])
+    return [dummy_input], sess.run(outputs, feed_dict={inputs[0]: dummy_input})
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/control_dep.py

@@ -0,0 +1,56 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for control_dep."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+TEST_INPUT_DEPTH = 3
+
+
+@register_make_test_function()
+def make_control_dep_tests(options):
+  """Make a set of tests that use control dependencies."""
+
+  test_parameters = [{
+      "input_shape": [[], [1, 1, 1, 1], [1, 15, 14, 1], [3, 15, 14, 3]],
+  }]
+
+  def build_graph(parameters):
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="input", shape=parameters["input_shape"])
+    filter_value = tf.zeros((3, 3, TEST_INPUT_DEPTH, 8), tf.float32)
+    assert_op = tf.compat.v1.assert_greater_equal(input_tensor,
+                                                  input_tensor - 1)
+    with tf.control_dependencies([assert_op]):
+      out = tf.nn.conv2d(
+          input=input_tensor,
+          filters=filter_value,
+          strides=(1, 1, 1, 1),
+          padding="SAME")
+      return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_values = create_tensor_data(tf.float32, parameters["input_shape"])
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=3)

tensorflow/lite/testing/op_tests/conv.py

@@ -0,0 +1,148 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for conv."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_conv_tests(options):
+  """Make a set of tests to do convolution."""
+
+  test_parameters = [
+      {
+          "input_shape": [[1, 3, 4, 3], [4, 6, 6, 1]],
+          "filter_shape": [[1, 1], [2, 3], [3, 3]],
+          "strides": [[1, 1, 1, 1], [1, 2, 3, 1]],
+          "dilations": [[1, 1, 1, 1], [1, 3, 2, 1], [1, 2, 2, 1]],
+          "padding": ["SAME", "VALID"],
+          "data_format": ["NHWC"],  # TODO(aselle): NCHW  would be good
+          "constant_filter": [True, False],
+          "channel_multiplier": [1, 2],
+          "fully_quantize": [False],
+          "quant_16x8": [False],
+          "dynamic_range_quantize": [False]
+      },
+      {
+          "input_shape": [[1, 3, 4, 3]],
+          "filter_shape": [[1, 1], [2, 3]],
+          "strides": [[1, 1, 1, 1]],
+          "dilations": [[1, 1, 1, 1]],
+          "padding": ["SAME"],
+          "data_format": ["NHWC"],
+          "constant_filter": [True],
+          "channel_multiplier": [1, 2],
+          "fully_quantize": [True],
+          "quant_16x8": [True],
+          "dynamic_range_quantize": [False],
+      },
+      # TODO(b/134702301): The fully_quantize param is just ignored by the MLIR
+      # testing path now, resulting in duplicate tests. Either ignore these
+      # tests or handle it properly in the mlir_convert() function.
+      {
+          "input_shape": [[1, 3, 4, 3], [4, 6, 6, 1]],
+          "filter_shape": [[1, 1], [2, 3], [3, 3]],
+          "strides": [[1, 1, 1, 1], [1, 2, 3, 1]],
+          "dilations": [[1, 1, 1, 1], [1, 3, 2, 1], [1, 2, 2, 1]],
+          "padding": ["SAME", "VALID"],
+          "data_format": ["NHWC"],  # TODO(aselle): NCHW  would be good
+          "constant_filter": [True],
+          "channel_multiplier": [1, 2],
+          "fully_quantize": [True],
+          "quant_16x8": [False],
+          "dynamic_range_quantize": [False]
+      },
+      {
+          "input_shape": [[1, 3, 4, 3]],
+          "filter_shape": [[1, 1]],
+          "strides": [[1, 1, 1, 1], [1, 2, 3, 1]],
+          "dilations": [[1, 1, 1, 1]],
+          "padding": ["SAME", "VALID"],
+          "data_format": ["NHWC"],
+          "constant_filter": [True],
+          "channel_multiplier": [2],
+          "fully_quantize": [False],
+          "quant_16x8": [False],
+          "dynamic_range_quantize": [True]
+      },
+      {
+          "input_shape": [[1, 3, 4, 3]],
+          "filter_shape": [[1, 1]],
+          "strides": [[1, 1, 1, 1], [1, 2, 3, 1]],
+          "dilations": [[1, 1, 1, 1]],
+          "padding": [[[0, 0], [1, 1], [1, 1], [0, 0]]],
+          "data_format": ["NHWC"],
+          "constant_filter": [True],
+          "channel_multiplier": [2],
+          "fully_quantize": [False],
+          "quant_16x8": [False],
+          "dynamic_range_quantize": [True]
+      },
+  ]
+
+  def get_tensor_shapes(parameters):
+    input_shape = parameters["input_shape"]
+    filter_size = parameters["filter_shape"]
+    filter_shape = filter_size + [
+        input_shape[3], parameters["channel_multiplier"]
+    ]
+    return [input_shape, filter_shape]
+
+  def build_graph(parameters):
+    """Build a conv graph given `parameters`."""
+    input_shape, filter_shape = get_tensor_shapes(parameters)
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="input", shape=input_shape)
+
+    # Get filter input either as a placeholder or constants. Also get a list of
+    # the input tensors that are represented as placeholders.
+    if parameters["constant_filter"]:
+      filter_input = create_tensor_data(
+          np.float32, filter_shape, min_value=-10, max_value=10)
+      input_tensors = [input_tensor]
+    else:
+      filter_input = tf.compat.v1.placeholder(
+          dtype=tf.float32, name="filter", shape=filter_shape)
+      input_tensors = [input_tensor, filter_input]
+
+    out = tf.nn.conv2d(
+        input=input_tensor,
+        filters=filter_input,
+        strides=parameters["strides"],
+        dilations=parameters["dilations"],
+        padding=parameters["padding"],
+        data_format=parameters["data_format"])
+    return input_tensors, [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    # Build list of input values either containing 1 tensor (input) or 2 tensors
+    # (input, filter) based on whether filter is constant or variable input.
+    input_shape, filter_shape = get_tensor_shapes(parameters)
+    values = [
+        create_tensor_data(np.float32, input_shape, min_value=-1, max_value=1)
+    ]
+    if not parameters["constant_filter"]:
+      values.append(create_tensor_data(np.float32, filter_shape))
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=60)

tensorflow/lite/testing/op_tests/conv2d_transpose.py

@@ -0,0 +1,79 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for conv2d_transpose."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_conv2d_transpose_tests(options):
+  """Make a set of tests to do transpose_conv."""
+
+  test_parameters = [{
+      "input_shape": [[1, 50, 54, 3]],
+      "filter_shape": [[1, 1, 8, 3], [1, 2, 8, 3], [1, 3, 8, 3], [1, 4, 8, 3]],
+      "output_shape": [[1, 100, 108, 8]],
+      "dynamic_output_shape": [True, False],
+  }, {
+      "input_shape": [[1, 16, 1, 512]],
+      "filter_shape": [[4, 1, 512, 512]],
+      "output_shape": [[1, 32, 1, 512]],
+      "dynamic_output_shape": [True, False],
+  }, {
+      "input_shape": [[1, 128, 128, 1]],
+      "filter_shape": [[4, 4, 1, 1]],
+      "output_shape": [[1, 256, 256, 1]],
+      "dynamic_output_shape": [True, False],
+  }]
+
+  def build_graph(parameters):
+    """Build a transpose_conv graph given `parameters`."""
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="input", shape=parameters["input_shape"])
+
+    filter_tensor = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="filter", shape=parameters["filter_shape"])
+
+    input_tensors = [input_tensor, filter_tensor]
+
+    if parameters["dynamic_output_shape"]:
+      output_shape = tf.compat.v1.placeholder(dtype=tf.int32, shape=[4])
+      input_tensors.append(output_shape)
+    else:
+      output_shape = parameters["output_shape"]
+
+    out = tf.nn.conv2d_transpose(
+        input_tensor,
+        filter_tensor,
+        output_shape=output_shape,
+        padding="SAME",
+        strides=(1, 2, 2, 1))
+
+    return input_tensors, [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    values = [
+        create_tensor_data(np.float32, parameters["input_shape"]),
+        create_tensor_data(np.float32, parameters["filter_shape"])
+    ]
+    if parameters["dynamic_output_shape"]:
+      values.append(np.array(parameters["output_shape"]))
+
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/conv3d.py

@@ -0,0 +1,70 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for exp."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_conv3d_tests(options):
+  """Make a set of tests to do conv3d."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32],
+      "input_shape": [[2, 3, 4, 5, 3], [2, 5, 6, 8, 3]],
+      "filter_shape": [[2, 2, 2, 3, 2], [1, 2, 2, 3, 2]],
+      "strides": [(1, 1, 1, 1, 1), (1, 1, 1, 2, 1), (1, 1, 2, 2, 1),
+                  (1, 2, 1, 2, 1), (1, 2, 2, 2, 1)],
+      "dilations": [(1, 1, 1, 1, 1)],
+      "padding": ["SAME", "VALID"],
+  }]
+
+  def build_graph(parameters):
+    """Build the exp op testing graph."""
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+    filter_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="filter",
+        shape=parameters["filter_shape"])
+
+    out = tf.nn.conv3d(
+        input_tensor,
+        filter_tensor,
+        strides=parameters["strides"],
+        dilations=parameters["dilations"],
+        padding=parameters["padding"])
+    return [input_tensor, filter_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    values = [
+        create_tensor_data(
+            parameters["input_dtype"],
+            parameters["input_shape"],
+            min_value=-100,
+            max_value=9),
+        create_tensor_data(
+            parameters["input_dtype"],
+            parameters["filter_shape"],
+            min_value=-3,
+            max_value=3)
+    ]
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/conv3d_transpose.py

@@ -0,0 +1,96 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for conv3d_transpose."""
+import numpy as np
+import tensorflow as tf
+
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_conv3d_transpose_tests(options):
+  """Make a set of tests to do conv3d_transpose."""
+
+  test_parameters = [{
+      "shape_dtype": [tf.int32, tf.int64],
+      "input_dtype": [tf.float32],
+      "input_shape": [[2, 3, 4, 5, 2], [2, 5, 6, 8, 2]],
+      "filter_shape": [[2, 2, 2, 3, 2], [1, 2, 2, 3, 2]],
+      "strides": [(1, 1, 1, 1, 1), (1, 1, 1, 2, 1), (1, 1, 2, 2, 1),
+                  (1, 2, 1, 2, 1), (1, 2, 2, 2, 1)],
+      "dilations": [(1, 1, 1, 1, 1)],
+      "padding": ["SAME", "VALID"],
+  }]
+
+  def build_graph(parameters):
+    """Build the exp op testing graph."""
+    output_shape = tf.compat.v1.placeholder(
+        dtype=parameters["shape_dtype"], name="input", shape=[5])
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+    filter_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="filter",
+        shape=parameters["filter_shape"])
+
+    out = tf.nn.conv3d_transpose(
+        input_tensor,
+        filter_tensor,
+        output_shape,
+        strides=parameters["strides"],
+        dilations=parameters["dilations"],
+        padding=parameters["padding"])
+    return [input_tensor, filter_tensor, output_shape], [out]
+
+  def calculate_output_shape(parameters):
+
+    def calculate_shape(idx):
+      input_size = parameters["input_shape"][idx]
+      filter_size = parameters["filter_shape"][idx - 1]
+      stride = parameters["strides"][idx]
+      if parameters["padding"] == "SAME":
+        return (input_size - 1) * stride + 1
+      else:
+        return (input_size - 1) * stride + filter_size
+
+    output_shape_values = [parameters["input_shape"][0]]
+    output_shape_values.append(calculate_shape(1))
+    output_shape_values.append(calculate_shape(2))
+    output_shape_values.append(calculate_shape(3))
+    output_shape_values.append(parameters["filter_shape"][3])
+    return np.dtype(
+        parameters["shape_dtype"].as_numpy_dtype()).type(output_shape_values)
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    values = [
+        create_tensor_data(
+            parameters["input_dtype"],
+            parameters["input_shape"],
+            min_value=-100,
+            max_value=9),
+        create_tensor_data(
+            parameters["input_dtype"],
+            parameters["filter_shape"],
+            min_value=-3,
+            max_value=3),
+        calculate_output_shape(parameters)
+    ]
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/conv_activation.py

@@ -0,0 +1,152 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for conv with activations."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+def make_conv_activation_tests(activation_op):
+  """Make a set of tests to do convolution with activation."""
+
+  def f(options):
+    """Actual function that generates examples."""
+    test_parameters = [
+        {
+            "input_shape": [[1, 3, 4, 3], [4, 6, 6, 1]],
+            "filter_shape": [[1, 1], [2, 3], [3, 3]],
+            "strides": [[1, 1, 1, 1], [1, 2, 3, 1]],
+            "dilations": [[1, 1, 1, 1], [1, 3, 2, 1], [1, 2, 2, 1]],
+            "padding": ["SAME", "VALID"],
+            "data_format": ["NHWC"],  # TODO(aselle): NCHW  would be good
+            "constant_filter": [True, False],
+            "channel_multiplier": [1, 2],
+            "fully_quantize": [False],
+            "quant_16x8": [False],
+            "dynamic_range_quantize": [False],
+        },
+        # TODO(b/134702301): The fully_quantize param is just ignored by the
+        # MLIR testing path now, resulting in duplicate tests. Either ignore
+        # these tests or handle it properly in the mlir_convert() function.
+        {
+            "input_shape": [[1, 3, 4, 3], [4, 6, 6, 1]],
+            "filter_shape": [[1, 1], [2, 3]],
+            "strides": [[1, 1, 1, 1], [1, 2, 3, 1]],
+            "dilations": [[1, 1, 1, 1], [1, 3, 2, 1]],
+            "padding": ["SAME", "VALID"],
+            "data_format": ["NHWC"],  # TODO(aselle): NCHW  would be good
+            "constant_filter": [True],
+            "channel_multiplier": [1, 2],
+            "fully_quantize": [True],
+            "quant_16x8": [False, True],
+            "dynamic_range_quantize": [False],
+        },
+        {
+            "input_shape": [[1, 3, 4, 3]],
+            "filter_shape": [[1, 1], [2, 3], [3, 3]],
+            "strides": [[1, 1, 1, 1], [1, 2, 3, 1]],
+            "dilations": [[1, 1, 1, 1]],
+            "padding": ["SAME", "VALID"],
+            "data_format": ["NHWC"],
+            "constant_filter": [True],
+            "channel_multiplier": [1, 2],
+            "fully_quantize": [False],
+            "quant_16x8": [False],
+            "dynamic_range_quantize": [True],
+        },
+    ]
+
+    def get_tensor_shapes(parameters):
+      input_shape = parameters["input_shape"]
+      filter_size = parameters["filter_shape"]
+      filter_shape = filter_size + [
+          input_shape[3], parameters["channel_multiplier"]
+      ]
+      return [input_shape, filter_shape]
+
+    def build_graph(parameters):
+      """Build a conv graph given `parameters`."""
+      input_shape, filter_shape = get_tensor_shapes(parameters)
+      input_tensor = tf.compat.v1.placeholder(
+          dtype=tf.float32, name="input", shape=input_shape)
+
+      # Get filter input either as a placeholder or constants. Also get a list
+      # of the input tensors that are represented as placeholders.
+      if parameters["constant_filter"]:
+        filter_input = create_tensor_data(
+            np.float32, filter_shape, min_value=-10, max_value=10)
+        input_tensors = [input_tensor]
+      else:
+        filter_input = tf.compat.v1.placeholder(
+            dtype=tf.float32, name="filter", shape=filter_shape)
+        input_tensors = [input_tensor, filter_input]
+
+      out = tf.nn.conv2d(
+          input=input_tensor,
+          filters=filter_input,
+          strides=parameters["strides"],
+          dilations=parameters["dilations"],
+          padding=parameters["padding"],
+          data_format=parameters["data_format"])
+      out = activation_op(out)
+      return input_tensors, [out]
+
+    def build_inputs(parameters, sess, inputs, outputs):
+      """Build inputs for conv with activation."""
+
+      input_shape, filter_shape = get_tensor_shapes(parameters)
+      values = [
+          create_tensor_data(
+              np.float32, input_shape, min_value=-1, max_value=1)
+      ]
+      if not parameters["constant_filter"]:
+        values.append(create_tensor_data(np.float32, filter_shape))
+      return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+    make_zip_of_tests(
+        options,
+        test_parameters,
+        build_graph,
+        build_inputs,
+        expected_tf_failures=48)
+
+  return f
+
+
+@register_make_test_function()
+def make_conv_relu6_tests(options):
+  """Make a set of tests to do conv_relu6."""
+  return make_conv_activation_tests(tf.nn.relu6)(options)
+
+
+@register_make_test_function()
+def make_conv_relu_tests(options):
+  """Make a set of tests to do conv_relu."""
+  return make_conv_activation_tests(tf.nn.relu)(options)
+
+
+def relu1(input_tensor):
+  # Note that the following is not supported:
+  #   out = tf.maximum(-1.0, tf.minimum(input_tensor, 1.0))
+  out = tf.minimum(1.0, tf.maximum(input_tensor, -1.0))
+  return out
+
+
+@register_make_test_function()
+def make_conv_relu1_tests(options):
+  """Make a set of tests to do conv_relu1."""
+  return make_conv_activation_tests(relu1)(options)

tensorflow/lite/testing/op_tests/conv_bias_activation.py

@@ -0,0 +1,145 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for conv followed with bias Add and activations."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+def make_conv_bias_activation_tests(activation_op):
+  """Make a set of tests to do convolution with activation and bias.
+
+  This test will create multiple consecutive convolutions with NCHW layout to
+  make sure that the tranformations to NHWC works as expected. Note this
+  doesn't check any performance so manual checking of the generated model is
+  advised.
+
+  Args:
+    activation_op: The activation op to be used in the test.
+
+  Returns:
+    The function that creates the test.
+  """
+
+  def create_test(options):
+    """Actual function that generates examples."""
+    test_parameters = [
+        {
+            "input_shape": [[1, 3, 4, 3]],
+            "filter_shape": [[2, 3], [3, 3]],
+            "filter_2_shape": [[2, 1, 1, 3]],
+            "strides": [[1, 1, 1, 1]],
+            "dilations": [[1, 1, 1, 1]],
+            "data_format": ["NCHW"],
+            "channel_multiplier": [1, 2],
+            "fully_quantize": [False],
+            "dynamic_range_quantize": [False],
+        },
+    ]
+
+    def get_tensor_shapes(parameters):
+      input_shape = parameters["input_shape"]
+      filter_size = parameters["filter_shape"]
+      filter_shape = filter_size + [
+          input_shape[3], parameters["channel_multiplier"]
+      ]
+      return [input_shape, filter_shape]
+
+    # TF CPU doesn't support cases with NCHW. Instead
+    # use XLA which doesn't have the same restrictions.
+    @tf.function(jit_compile=True)
+    def add_conv(input_tensor, filter_input, parameters):
+      out = tf.nn.conv2d(
+          input=input_tensor,
+          filters=filter_input,
+          strides=parameters["strides"],
+          dilations=parameters["dilations"],
+          padding="VALID",
+          data_format=parameters["data_format"])
+      return out
+
+    def add_bias_add(data_input, filter_shape):
+      bias_input = create_tensor_data(np.float32, (filter_shape[-1],))
+      out = tf.nn.bias_add(data_input, bias_input, data_format="NHWC")
+      return out
+
+    def build_graph(parameters):
+      """Build a conv graph given `parameters`."""
+      input_shape, filter_shape = get_tensor_shapes(parameters)
+      input_tensor = tf.compat.v1.placeholder(
+          dtype=tf.float32, name="input", shape=input_shape)
+
+      filter_input = create_tensor_data(
+          np.float32, filter_shape, min_value=-10, max_value=10)
+      input_tensors = [input_tensor]
+
+      if parameters["data_format"] == "NCHW":
+        out = add_conv(input_tensor, filter_input, parameters)
+      else:
+        out = tf.nn.conv2d(
+            input=input_tensor,
+            filters=filter_input,
+            strides=parameters["strides"],
+            dilations=parameters["dilations"],
+            padding="VALID",
+            data_format=parameters["data_format"])
+      out = add_bias_add(out, filter_shape)
+      out = activation_op(out)
+
+      # Add another conv + bias_add + activation.
+
+      # Create constant filter for the second conv2d.
+      filter_input_2 = create_tensor_data(
+          np.float32, parameters["filter_2_shape"], min_value=-10, max_value=10)
+      if parameters["data_format"] == "NCHW":
+        out = add_conv(out, filter_input_2, parameters)
+      else:
+        out = tf.nn.conv2d(
+            input=out,
+            filters=filter_input_2,
+            strides=parameters["strides"],
+            dilations=parameters["dilations"],
+            padding="VALID",
+            data_format=parameters["data_format"])
+      out = add_bias_add(out, filter_shape)
+      out = activation_op(out)
+      return input_tensors, [out]
+
+    def build_inputs(parameters, sess, inputs, outputs):
+      """Build inputs for conv with activation."""
+
+      input_shape, _ = get_tensor_shapes(parameters)
+      values = [
+          create_tensor_data(
+              np.float32, input_shape, min_value=-1, max_value=1)
+      ]
+      return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+    make_zip_of_tests(
+        options,
+        test_parameters,
+        build_graph,
+        build_inputs,
+        expected_tf_failures=2)
+
+  return create_test
+
+
+@register_make_test_function()
+def make_conv_bias_relu6_tests(options):
+  """Make a set of tests to do conv_bias_relu6."""
+  return make_conv_bias_activation_tests(tf.nn.relu6)(options)

tensorflow/lite/testing/op_tests/conv_to_depthwiseconv_with_shared_weights.py

@@ -0,0 +1,83 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for conv."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_conv_to_depthwiseconv_with_shared_weights_tests(options):
+  """Make a test where 2 Conv ops shared the same constant weight tensor."""
+
+  test_parameters = [{
+      "input_shape": [[1, 10, 10, 1]],
+      "filter_shape": [[3, 3]],
+      "strides": [[1, 1, 1, 1]],
+      "dilations": [[1, 1, 1, 1]],
+      "padding": ["SAME"],
+      "data_format": ["NHWC"],
+      "channel_multiplier": [3],
+  }]
+
+  def get_tensor_shapes(parameters):
+    input_shape = parameters["input_shape"]
+    filter_size = parameters["filter_shape"]
+    filter_shape = filter_size + [
+        input_shape[3], parameters["channel_multiplier"]
+    ]
+    return [input_shape, filter_shape]
+
+  def build_graph(parameters):
+    """Build a conv graph given `parameters`."""
+    input_shape, filter_shape = get_tensor_shapes(parameters)
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="input", shape=input_shape)
+
+    # Construct a constant weights tensor which will be used by both Conv2D.
+    filter_tensor = tf.constant(
+        create_tensor_data(np.float32, filter_shape), dtype=tf.float32)
+    input_tensors = [input_tensor]
+
+    # Construct 2 Conv2D operations which use exactly the same input and
+    # weights.
+    result1 = tf.nn.conv2d(
+        input=input_tensor,
+        filters=filter_tensor,
+        strides=parameters["strides"],
+        dilations=parameters["dilations"],
+        padding=parameters["padding"],
+        data_format=parameters["data_format"])
+    result2 = tf.nn.conv2d(
+        input=input_tensor,
+        filters=filter_tensor,
+        strides=parameters["strides"],
+        dilations=parameters["dilations"],
+        padding=parameters["padding"],
+        data_format=parameters["data_format"])
+    # Add the 2 results up.
+    out = result1 + result2
+    return input_tensors, [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    # Build list of input values either containing 1 tensor (input) or 2 tensors
+    # (input, filter) based on whether filter is constant or variable input.
+    input_shape, unused_filter_shape = get_tensor_shapes(parameters)
+    values = [create_tensor_data(np.float32, input_shape)]
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/conv_with_shared_weights.py

@@ -0,0 +1,92 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for conv_with_shared_weights."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_conv_with_shared_weights_tests(options):
+  """Make a test where 2 Conv ops shared the same constant weight tensor."""
+
+  test_parameters = [{
+      "input_shape": [[1, 10, 10, 3]],
+      "filter_shape": [[3, 3]],
+      "strides": [[1, 1, 1, 1]],
+      "dilations": [[1, 1, 1, 1]],
+      "padding": ["SAME"],
+      "data_format": ["NHWC"],
+      "channel_multiplier": [1],
+      "dynamic_range_quantize": [False, True],
+  }]
+
+  def get_tensor_shapes(parameters):
+    input_shape = parameters["input_shape"]
+    filter_size = parameters["filter_shape"]
+    filter_shape = filter_size + [
+        input_shape[3], parameters["channel_multiplier"]
+    ]
+    return [input_shape, filter_shape]
+
+  def build_graph(parameters):
+    """Build a conv graph given `parameters`."""
+    input_shape, filter_shape = get_tensor_shapes(parameters)
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="input", shape=input_shape)
+    input_tensors = [input_tensor]
+
+    # Construct a constant weights tensor which will be used by both Conv2D.
+    filter_tensor = tf.constant(
+        create_tensor_data(np.float32, filter_shape), dtype=tf.float32)
+
+    # Ensure that FuseBinaryIntoFollowingAffine works with an input which
+    # is shared by multiple affine ops.
+    conv_input = input_tensor + 0.1
+
+    # Construct 2 Conv2D operations which use exactly the same input and
+    # weights.
+    result1 = tf.nn.conv2d(
+        input=conv_input,
+        filters=filter_tensor,
+        strides=parameters["strides"],
+        dilations=parameters["dilations"],
+        padding=parameters["padding"],
+        data_format=parameters["data_format"])
+    result2 = tf.nn.conv2d(
+        input=conv_input,
+        filters=filter_tensor,
+        strides=parameters["strides"],
+        dilations=parameters["dilations"],
+        padding=parameters["padding"],
+        data_format=parameters["data_format"])
+    # Add MUL ops after Conv2D ops. These MUL ops should be fused into the
+    # weights of Conv2D.
+    result1 = result1 * 2
+    result2 = result2 * 3
+    # Add the 2 results up.
+    out = result1 + result2
+    return input_tensors, [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    # Build list of input values either containing 1 tensor (input) or 2 tensors
+    # (input, filter) based on whether filter is constant or variable input.
+    input_shape, unused_filter_shape = get_tensor_shapes(parameters)
+    values = [create_tensor_data(np.float32, input_shape)]
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/cos.py

@@ -0,0 +1,52 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for cos."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_cos_tests(options):
+  """Make a set of tests to do cos."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32],
+      "input_shape": [[], [3], [1, 100], [4, 2, 3], [5, 224, 224, 3]],
+  }]
+
+  def build_graph(parameters):
+    """Build the cos op testing graph."""
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+
+    out = tf.cos(input_tensor)
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    values = [
+        create_tensor_data(
+            parameters["input_dtype"],
+            parameters["input_shape"],
+            min_value=-np.pi,
+            max_value=np.pi)
+    ]
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/cumsum.py

@@ -0,0 +1,49 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for cumsum."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_cumsum_tests(options):
+  """Make a set of tests to do cumsum."""
+
+  test_parameters = [{
+      "shape": [(3, 6), (8, 9, 7), (2, 4, 3, 5)],
+      "dtype": [tf.int32, tf.int64, tf.float32],
+      "axis": [0, 1],
+      "exclusive": [True, False],
+      "reverse": [True, False],
+  }]
+
+  def build_graph(parameters):
+    """Build the cumsum op testing graph."""
+    input1 = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"], shape=parameters["shape"])
+    out = tf.math.cumsum(
+        input1,
+        parameters["axis"],
+        exclusive=parameters["exclusive"],
+        reverse=parameters["reverse"])
+    return [input1], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input1 = create_tensor_data(parameters["dtype"], parameters["shape"])
+    return [input1], sess.run(outputs, feed_dict=dict(zip(inputs, [input1])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/depth_to_space.py

@@ -0,0 +1,60 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for depth_to_space."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_depth_to_space_tests(options):
+  """Make a set of tests to do depth_to_space."""
+
+  test_parameters = [{
+      "dtype": [tf.int32, tf.uint8, tf.int64],
+      "input_shape": [[2, 3, 4, 16]],
+      "block_size": [2, 4],
+      "fully_quantize": [False],
+  }, {
+      "dtype": [tf.float32],
+      "input_shape": [[2, 3, 4, 16]],
+      "block_size": [2, 4],
+      "fully_quantize": [True, False],
+  }]
+
+  def build_graph(parameters):
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+    out = tf.compat.v1.depth_to_space(
+        input_tensor, block_size=parameters["block_size"])
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    if not parameters["fully_quantize"]:
+      input_values = create_tensor_data(parameters["dtype"],
+                                        parameters["input_shape"])
+    else:
+      input_values = create_tensor_data(
+          parameters["dtype"],
+          parameters["input_shape"],
+          min_value=-1,
+          max_value=1)
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/depthwiseconv.py

@@ -0,0 +1,152 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for depthwiseconv."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_depthwiseconv_tests(options):
+  """Make a set of tests to do convolution."""
+
+  # Tensorflow only supports equal strides
+  test_parameters = [
+      {
+          "input_shape": [[1, 3, 4, 3], [1, 10, 10, 3]],
+          "filter_size": [[1, 1], [1, 2], [3, 3]],
+          "strides": [[1, 1, 1, 1], [1, 3, 3, 1]],
+          "dilations": [[1, 1, 1, 1], [1, 3, 2, 1], [1, 2, 2, 1]],
+          "channel_multiplier": [1, 2],
+          "rate": [[1, 1]],
+          "padding": ["SAME", "VALID"],
+          "data_format": ["NHWC"],
+          "constant_filter": [True, False],
+          "fully_quantize": [False],
+          "quant_16x8": [False]
+      },
+      {
+          "input_shape": [[1, 3, 4, 3]],
+          "filter_size": [[1, 1]],
+          "strides": [[1, 1, 2, 1]],  # TF needs [1, x, x, 1]
+          "dilations": [[1, 1, 1, 1], [1, 2, 2, 1]],
+          "channel_multiplier": [2],
+          "rate": [[2, 2]],  #  Only [1, 1] is supported
+          "padding": ["SAME"],
+          "data_format": ["NHWC"],
+          "constant_filter": [True, False],
+          "fully_quantize": [False],
+          "quant_16x8": [False]
+      },
+      {
+          "input_shape": [[1, 3, 4, 3], [1, 10, 10, 3]],
+          "filter_size": [[1, 1], [1, 2], [3, 3]],
+          "strides": [[1, 1, 1, 1], [1, 3, 3, 1]],
+          "dilations": [[1, 1, 1, 1], [1, 3, 2, 1], [1, 2, 2, 1]],
+          "channel_multiplier": [1, 2],
+          "rate": [[1, 1]],
+          "padding": ["SAME", "VALID"],
+          "data_format": ["NHWC"],
+          "constant_filter": [True],
+          "fully_quantize": [True],
+          "quant_16x8": [False]
+      },
+      {
+          "input_shape": [[1, 3, 3, 3000]],
+          "filter_size": [[3, 3]],
+          "strides": [[1, 1, 1, 1]],
+          "dilations": [[1, 1, 1, 1]],
+          "channel_multiplier": [1],
+          "rate": [[1, 1]],
+          "padding": ["VALID"],
+          "data_format": ["NHWC"],
+          "constant_filter": [True],
+          "fully_quantize": [True],
+          "quant_16x8": [False]
+      },
+      {
+          "input_shape": [[1, 3, 4, 3]],
+          "filter_size": [[1, 2]],
+          "strides": [[1, 3, 3, 1]],
+          "dilations": [[1, 3, 2, 1]],
+          "channel_multiplier": [1],
+          "rate": [[1, 1]],
+          "padding": ["SAME", "VALID"],
+          "data_format": ["NHWC"],
+          "constant_filter": [True],
+          "fully_quantize": [True],
+          "quant_16x8": [True]
+      },
+  ]
+
+  def get_tensor_shapes(parameters):
+    input_shape = parameters["input_shape"]
+    filter_size = parameters["filter_size"]
+    filter_shape = filter_size + [
+        input_shape[3], parameters["channel_multiplier"]
+    ]
+    return [input_shape, filter_shape]
+
+  def build_graph(parameters):
+    """Build a depthwise conv graph given `parameters`."""
+    input_shape, filter_shape = get_tensor_shapes(parameters)
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="input", shape=input_shape)
+
+    # Get filter input either as a placeholder or constants. Also get a list of
+    # the input tensors that are represented as placeholders.
+    if parameters["constant_filter"]:
+      filter_input = create_tensor_data(np.float32, filter_shape)
+      input_tensors = [input_tensor]
+    else:
+      filter_input = tf.compat.v1.placeholder(
+          dtype=tf.float32, name="filter", shape=filter_shape)
+      input_tensors = [input_tensor, filter_input]
+
+    out = tf.nn.depthwise_conv2d(
+        input=input_tensor,
+        filter=filter_input,
+        strides=parameters["strides"],
+        dilations=parameters["rate"],
+        padding=parameters["padding"],
+        data_format=parameters["data_format"])
+    return input_tensors, [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    # pylint: disable=g-doc-return-or-yield, g-doc-args
+    """Build list of input values.
+
+    It either contains 1 tensor (input) or 2 tensors (input, filter) based on
+    whether filter is constant or variable input.
+    """
+
+    input_shape, filter_shape = get_tensor_shapes(parameters)
+    values = [
+        create_tensor_data(np.float32, input_shape, min_value=-1, max_value=1)
+    ]
+    if not parameters["constant_filter"]:
+      values.append(
+          create_tensor_data(
+              np.float32, filter_shape, min_value=-1, max_value=1))
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=4)

tensorflow/lite/testing/op_tests/dynamic_rnn.py

@@ -0,0 +1,75 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for dynamic_rnn."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import rnn
+
+
+@register_make_test_function("make_dynamic_rnn_tests")
+@test_util.enable_control_flow_v2
+def make_dynamic_rnn_tests(options):
+  """Make a set of tests to do basic Lstm cell."""
+
+  test_parameters = [
+      {
+          "dtype": [tf.float32],
+          "num_batches": [4, 2],
+          "time_step_size": [4, 3],
+          "input_vec_size": [3, 2],
+          "num_cells": [4, 2],
+      },
+  ]
+
+  def build_graph(parameters):
+    """Build a simple graph with BasicLSTMCell."""
+    num_batches = parameters["num_batches"]
+    time_step_size = parameters["time_step_size"]
+    input_vec_size = parameters["input_vec_size"]
+    num_cells = parameters["num_cells"]
+    input_shape = (num_batches, time_step_size, input_vec_size)
+
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"], shape=input_shape)
+    lstm_cell = tf.compat.v1.nn.rnn_cell.BasicLSTMCell(
+        num_cells, activation=tf.nn.relu)
+
+    output, _ = rnn.dynamic_rnn(
+        lstm_cell, input_tensor, dtype=parameters["dtype"])
+    return [input_tensor], [output]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    """Feed inputs, assign variables, and freeze graph."""
+    sess.run(tf.compat.v1.global_variables_initializer())
+
+    num_batches = parameters["num_batches"]
+    time_step_size = parameters["time_step_size"]
+    input_vec_size = parameters["input_vec_size"]
+    input_shape = (num_batches, time_step_size, input_vec_size)
+    input_value = create_tensor_data(parameters["dtype"], input_shape)
+
+    output_values = sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_value])))
+    return [input_value], output_values
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      use_frozen_graph=True)

tensorflow/lite/testing/op_tests/dynamic_update_slice.py

@@ -0,0 +1,84 @@
+# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for tensor_list_set_item."""
+import functools
+
+import tensorflow as tf
+from tensorflow.lite.python import lite
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+from tensorflow.python.ops import list_ops
+
+
+def _tflite_convert_verify_op(tflite_convert_function, *args, **kwargs):
+  """Verifies that the result of the conversion contains DynamicUpdateSlice op."""
+  result = tflite_convert_function(*args, **kwargs)
+  tflite_model_binary = result[0]
+  if not result[0]:
+    tf.compat.v1.logging.error(result[1])  # stderr from running tflite_convert.
+    raise RuntimeError("Failed to build model: \n\n" + result[1])
+  interpreter = lite.Interpreter(model_content=tflite_model_binary)
+  interpreter.allocate_tensors()
+  for op in interpreter._get_ops_details():  # pylint: disable=protected-access
+    if op["op_name"] == "DYNAMIC_UPDATE_SLICE":
+      return result
+  raise RuntimeError(
+      "Expected to generate DYNAMIC_UPDATE_SLICE op node in graph.")
+
+
+@register_make_test_function()
+def make_dynamic_update_slice_tests(options):
+  """Make a set of tests to do TensorListSetItem."""
+
+  test_parameters = [
+      {
+          "element_dtype": [tf.float32, tf.int32, tf.bool],
+          "num_elements": [4, 5, 6],
+          "element_shape": [[], [5], [3, 3]],
+          "index": [0, 1, 2, 3],
+      },
+  ]
+
+  def build_graph(parameters):
+    """Build the TensorListSetItem op testing graph."""
+    data = tf.compat.v1.placeholder(
+        dtype=parameters["element_dtype"],
+        shape=[parameters["num_elements"]] + parameters["element_shape"])
+    item = tf.compat.v1.placeholder(
+        dtype=parameters["element_dtype"], shape=parameters["element_shape"])
+    tensor_list = list_ops.tensor_list_from_tensor(data,
+                                                   parameters["element_shape"])
+    tensor_list = list_ops.tensor_list_set_item(tensor_list,
+                                                parameters["index"], item)
+    out = list_ops.tensor_list_stack(
+        tensor_list,
+        num_elements=parameters["num_elements"],
+        element_dtype=parameters["element_dtype"])
+    return [data, item], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    data = create_tensor_data(parameters["element_dtype"],
+                              [parameters["num_elements"]] +
+                              parameters["element_shape"])
+    item = create_tensor_data(parameters["element_dtype"],
+                              parameters["element_shape"])
+    return [data, item], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [data, item])))
+
+  options.enable_dynamic_update_slice = True
+  options.tflite_convert_function = functools.partial(
+      _tflite_convert_verify_op, options.tflite_convert_function)
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/einsum.py

@@ -0,0 +1,89 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for einsum."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+from tensorflow.python.framework import test_util
+
+
+@register_make_test_function("make_einsum_tests")
+@test_util.enable_control_flow_v2
+def make_einsum_tests(options):
+  """Make a set of tests to do basic einsum ops."""
+
+  test_parameters = [
+      {
+          "dtype": [tf.float32],
+          "shapes": [
+              ((None, None, 8, 64), (4, None, 8, 64), "BQNH,BTNH->BQNT"),
+              ((1, None, 8, None), (1, None, 8, 64), "BQNT,BTNH->BQNH"),
+              ((None, None, 8, 64), (8, 8, 64), "ABNH,NDH->ABD"),
+              ((None, None, 128), (128, 8, 64), "ABD,DNH->ABNH"),
+              ((3, 4, 5), (3, 5, 6), "ijk,ikm->ijm"),
+              ((3, 4, 5), (5, 6), "ijk,km->ijm"),
+              ((2, 5, 7), (5, 2), "LBH,BL->BH"),
+              ((2, 5, 7), (5, 3, 2), "LBH,BKL->BKH"),
+              ((2, 5, 7, 3), (2, 4, 7, 3), "BFNH,BTNH->BNFT"),
+              ((2, 5, 7, 3), (7, 3, 4), "BFND,NDH->BFH"),
+              ((3, 4, 5), (5, 6, 2), "BFD,DNH->BFNH"),
+              ((7, 11, 13), (7, 11, 13, 5), "BIN,BINJ->BIJ"),
+              ((7, 11, 19), (7, 11, 13, 19), "BIJ,BINJ->BIN"),
+              ((5, 13, 3, 11), (5, 11, 13, 8), "ACBE,AECD->ABCD"),
+              ((5, 11, 7, 3), (5, 8, 7, 3), "AECD,ABCD->ACBE"),
+              ((5, 4, 3), (3, 2, 1), "...ij,j...->i..."),
+              ((5, 4, 3), (3, 2, 1), "...ij,j...->...i"),
+              ((1, 11, 19), (7, 11, 13, 19), "...IJ,...INJ->...IN"),
+              ((1, 11, 19), (7, 11, 13, 19), "...IJ,...INJ->IN..."),
+              ((4, 3, 2, 5), (3, 6, 1), "ij...,jk...->ik..."),
+              ((4, 3, 2, 5), (3, 6, 1), "ij...,jk...->...ik"),
+          ],
+      },
+  ]
+
+  def set_dynamic_shape(shape):
+    """Convert dynamic shapes to static shapes."""
+    return [4 if x is None else x for x in shape]
+
+  def build_graph(parameters):
+    """Build a simple graph with einsum Op."""
+    input0_shape = parameters["shapes"][0]
+    input1_shape = parameters["shapes"][1]
+    equation = parameters["shapes"][2]
+
+    input0_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"], shape=input0_shape)
+    input1_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"], shape=input1_shape)
+    out = tf.einsum(equation, input0_tensor, input1_tensor)
+    return [input0_tensor, input1_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    """Feed inputs, assign variables, and freeze graph."""
+    input0_shape = set_dynamic_shape(parameters["shapes"][0])
+    input1_shape = set_dynamic_shape(parameters["shapes"][1])
+    input0_value = create_tensor_data(parameters["dtype"], input0_shape)
+    input1_value = create_tensor_data(parameters["dtype"], input1_shape)
+    output_values = sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input0_value, input1_value])))
+    return [input0_value, input1_value], output_values
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      use_frozen_graph=True)

tensorflow/lite/testing/op_tests/elementwise.py

@@ -0,0 +1,114 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for elementwise ops."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+def _make_elementwise_tests(
+    op,
+    allow_fully_quantize=False,
+    min_value=-100,
+    max_value=100,
+):
+  """Make a set of tests to do element-wise operations."""
+
+  def f(options):
+    """Actual function that generates examples."""
+    test_parameters = [
+        {
+            "input_dtype": [tf.float32],
+            "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+            "fully_quantize": [False],
+            "quant_16x8": [False],
+            "input_range": [[min_value, max_value]],
+        },
+        {
+            "input_dtype": [tf.float32],
+            "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+            "fully_quantize": [True],
+            "quant_16x8": [False, True],
+            "input_range": [[min_value, max_value]],
+        },
+    ]
+
+    if not allow_fully_quantize:
+      test_parameters = [
+          test_parameter for test_parameter in test_parameters
+          if True not in test_parameter["fully_quantize"]
+      ]
+
+    def build_graph(parameters):
+      """Build the unary op testing graph."""
+      input_value = tf.compat.v1.placeholder(
+          dtype=parameters["input_dtype"],
+          name="input1",
+          shape=parameters["input_shape"])
+      out = op(input_value)
+      return [input_value], [out]
+
+    def build_inputs(parameters, sess, inputs, outputs):
+      input_value = create_tensor_data(parameters["input_dtype"],
+                                       parameters["input_shape"],
+                                       min_value=min_value,
+                                       max_value=max_value)
+      return [input_value], sess.run(
+          outputs, feed_dict={inputs[0]: input_value})
+
+    make_zip_of_tests(options, test_parameters, build_graph, build_inputs)
+
+  return f
+
+
+@register_make_test_function()
+def make_sin_tests(options):
+  """Make a set of tests to do sin."""
+  return _make_elementwise_tests(tf.sin)(options)
+
+
+@register_make_test_function()
+def make_log_tests(options):
+  """Make a set of tests to do log."""
+  return _make_elementwise_tests(
+      tf.math.log,
+      allow_fully_quantize=True,
+      min_value=0.1,
+      max_value=10,
+  )(options)
+
+
+@register_make_test_function()
+def make_sqrt_tests(options):
+  """Make a set of tests to do sqrt."""
+  return _make_elementwise_tests(tf.sqrt)(options)
+
+
+@register_make_test_function()
+def make_rsqrt_tests(options):
+  """Make a set of tests to do 1/sqrt."""
+  return _make_elementwise_tests(
+      tf.math.rsqrt,
+      allow_fully_quantize=True,
+      min_value=0.1,
+      max_value=1,
+  )(options)
+
+
+@register_make_test_function()
+def make_square_tests(options):
+  """Make a set of tests to do square."""
+  return _make_elementwise_tests(tf.square)(options)

tensorflow/lite/testing/op_tests/elu.py

@@ -0,0 +1,49 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for elu."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_elu_tests(options):
+  """Make a set of tests to do (float) tf.nn.elu."""
+
+  test_parameters = [
+      {
+          "input_shape": [[], [1], [2, 3], [1, 1, 1, 1], [1, 3, 4, 3],
+                          [3, 15, 14, 3], [3, 1, 2, 4, 6], [2, 2, 3, 4, 5, 6]],
+      },
+  ]
+
+  def build_graph(parameters):
+    """Build the graph for the test case."""
+
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="input", shape=parameters["input_shape"])
+    out = tf.nn.elu(input_tensor)
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    """Build the inputs for the test case."""
+    input_values = create_tensor_data(
+        np.float32, parameters["input_shape"], min_value=-4, max_value=10)
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/embedding_lookup.py

@@ -0,0 +1,56 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for embedding_lookup."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_embedding_lookup_tests(options):
+  """Make a set of tests to do gather."""
+
+  test_parameters = [
+      {
+          "params_dtype": [tf.float32],
+          "params_shape": [[10], [10, 10]],
+          "ids_dtype": [tf.int32],
+          "ids_shape": [[3], [5]],
+      },
+  ]
+
+  def build_graph(parameters):
+    """Build the gather op testing graph."""
+    params = tf.compat.v1.placeholder(
+        dtype=parameters["params_dtype"],
+        name="params",
+        shape=parameters["params_shape"])
+    ids = tf.compat.v1.placeholder(
+        dtype=parameters["ids_dtype"],
+        name="ids",
+        shape=parameters["ids_shape"])
+    out = tf.nn.embedding_lookup(params=params, ids=ids)
+    return [params, ids], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    params = create_tensor_data(parameters["params_dtype"],
+                                parameters["params_shape"])
+    ids = create_tensor_data(parameters["ids_dtype"], parameters["ids_shape"],
+                             0, parameters["params_shape"][0] - 1)
+    return [params, ids], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [params, ids])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/equal.py

@@ -0,0 +1,64 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for equal."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_equal_tests(options):
+  """Make a set of tests to do equal."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32, tf.int16, tf.int32, tf.int64, tf.string],
+      "input_shape_pair": [([], []), ([1, 1, 1, 3], [1, 1, 1, 3]),
+                           ([2, 3, 4, 5], [2, 3, 4, 5]), ([2, 3, 3], [2, 3]),
+                           ([5, 5], [1]), ([10], [2, 4, 10])],
+      "fully_quantize": [False],
+  }, {
+      "input_dtype": [tf.float32],
+      "input_shape_pair": [([1, 1, 1, 3], [1, 1, 1, 3]), ([2, 3, 3], [2, 3])],
+      "fully_quantize": [True],
+  }]
+
+  def build_graph(parameters):
+    """Build the equal op testing graph."""
+    input_value1 = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input1",
+        shape=parameters["input_shape_pair"][0])
+    input_value2 = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input2",
+        shape=parameters["input_shape_pair"][1])
+    out = tf.equal(input_value1, input_value2)
+    return [input_value1, input_value2], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value1 = create_tensor_data(parameters["input_dtype"],
+                                      parameters["input_shape_pair"][0])
+    input_value2 = create_tensor_data(parameters["input_dtype"],
+                                      parameters["input_shape_pair"][1])
+    return [input_value1, input_value2], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_value1, input_value2])))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=6)

tensorflow/lite/testing/op_tests/exp.py

@@ -0,0 +1,63 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for exp."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_exp_tests(options):
+  """Make a set of tests to do exp."""
+
+  test_parameters = [
+      {
+          "input_dtype": [tf.float32],
+          "input_shape": [[], [3], [1, 100], [4, 2, 3], [5, 224, 224, 3]],
+          "input_range": [(-100, 9)],
+      },
+      {
+          "input_dtype": [tf.float32],
+          "input_shape": [[], [3], [1, 100], [4, 2, 3], [5, 224, 224, 3]],
+          "input_range": [(-2, 2)],
+          "fully_quantize": [True],
+          "quant_16x8": [False, True],
+      },
+  ]
+
+  def build_graph(parameters):
+    """Build the exp op testing graph."""
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+
+    out = tf.exp(input_tensor)
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    min_value, max_value = parameters["input_range"]
+    values = [
+        create_tensor_data(
+            parameters["input_dtype"],
+            parameters["input_shape"],
+            min_value=min_value,
+            max_value=max_value,
+        )
+    ]
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/expand_dims.py

@@ -0,0 +1,75 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for expand_dims."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_expand_dims_tests(options):
+  """Make a set of tests to do expand_dims."""
+
+  test_parameters = [{
+      "input_type": [tf.float32, tf.int32],
+      "input_shape": [[5, 4], [1, 5, 4]],
+      "axis_value": [0, 1, 2, -1, -2, -3],
+      "constant_axis": [True, False],
+      "fully_quantize": [False],
+  }, {
+      "input_type": [tf.float32],
+      "input_shape": [[5, 4], [1, 5, 4]],
+      "axis_value": [0, 1, 2, -1, -2, -3],
+      "constant_axis": [True],
+      "fully_quantize": [True],
+  }]
+
+  def build_graph(parameters):
+    """Build the where op testing graph."""
+    inputs = []
+    input_value = tf.compat.v1.placeholder(
+        dtype=parameters["input_type"],
+        name="input",
+        shape=parameters["input_shape"])
+    inputs.append(input_value)
+
+    if parameters["constant_axis"]:
+      axis_value = tf.constant(
+          parameters["axis_value"], dtype=tf.int32, shape=[1])
+    else:
+      axis_value = tf.compat.v1.placeholder(
+          dtype=tf.int32, name="axis", shape=[1])
+      inputs.append(axis_value)
+
+    out = tf.expand_dims(input_value, axis=axis_value)
+    return inputs, [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    """Builds the inputs for expand_dims."""
+    input_values = []
+    input_values.append(
+        create_tensor_data(
+            parameters["input_type"],
+            parameters["input_shape"],
+            min_value=-1,
+            max_value=1))
+    if not parameters["constant_axis"]:
+      input_values.append(np.array([parameters["axis_value"]], dtype=np.int32))
+    return input_values, sess.run(
+        outputs, feed_dict=dict(zip(inputs, input_values)))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/expm1.py

@@ -0,0 +1,49 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for expm1."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_expm1_tests(options):
+  """Make a set of tests to do (float) tf.math.expm1."""
+
+  test_parameters = [
+      {
+          "input_shape": [[], [1], [2, 3], [1, 1, 1, 1], [1, 3, 4, 3],
+                          [3, 15, 14, 3], [3, 1, 2, 4, 6], [2, 2, 3, 4, 5, 6]],
+      },
+  ]
+
+  def build_graph(parameters):
+    """Build the graph for the test case."""
+
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="input", shape=parameters["input_shape"])
+    out = tf.math.expm1(input_tensor)
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    """Build the inputs for the test case."""
+    input_values = create_tensor_data(
+        np.float32, parameters["input_shape"], min_value=-4, max_value=10)
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/eye.py

@@ -0,0 +1,63 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for eye."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_scalar_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_eye_tests(options):
+  """Make a set of tests for tf.eye op."""
+
+  test_parameters = [{
+      "num_rows_shape": [[]],
+      "num_cols_shape": [[]],
+      "batch_shape": [[3], [2, 4], [4, 5, 6], None],
+      "use_num_cols": [True, False],
+      "dtype": [tf.float32, tf.int32],
+  }]
+
+  def build_graph(parameters):
+    """Make a set of tests to do eye."""
+
+    input_tensor0 = tf.compat.v1.placeholder(
+        dtype=tf.int32, name="num_rows", shape=parameters["num_rows_shape"])
+    input_tensor1 = tf.compat.v1.placeholder(
+        dtype=tf.int32, name="num_columns", shape=parameters["num_cols_shape"])
+    if parameters["use_num_cols"]:
+      outs = tf.eye(
+          num_rows=input_tensor0,
+          num_columns=input_tensor1,
+          batch_shape=parameters["batch_shape"],
+          dtype=parameters["dtype"])
+      return [input_tensor0, input_tensor1], [outs]
+    else:
+      outs = tf.eye(num_rows=input_tensor0, dtype=parameters["dtype"])
+      return [input_tensor0], [outs]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value0 = create_scalar_data(dtype=np.int32, min_value=1)
+    input_value1 = create_scalar_data(dtype=np.int32, min_value=1)
+    if parameters["use_num_cols"]:
+      return [input_value0, input_value1], sess.run(
+          outputs, feed_dict=dict(zip(inputs, [input_value0, input_value1])))
+    else:
+      return [input_value0], sess.run(
+          outputs, feed_dict=dict(zip(inputs, [input_value0])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/fill.py

@@ -0,0 +1,88 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for fill."""
+import tensorflow.compat.v2 as tf
+from tensorflow.lite.testing.zip_test_utils import create_scalar_data
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_fill_tests(options):
+  """Make a set of tests to do fill."""
+
+  test_parameters = [{
+      "dims_dtype": [tf.int32, tf.int64],
+      "dims_shape": [[], [1], [3], [3, 3]],
+      "value_dtype": [tf.int32, tf.int64, tf.float32, tf.bool, tf.string],
+  }]
+
+  def build_graph(parameters):
+    """Build the fill op testing graph."""
+    input1 = tf.compat.v1.placeholder(
+        dtype=parameters["dims_dtype"],
+        name="dims",
+        shape=parameters["dims_shape"])
+    input2 = tf.compat.v1.placeholder(
+        dtype=parameters["value_dtype"], name="value", shape=[])
+    out = tf.fill(input1, input2)
+    return [input1, input2], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input1 = create_tensor_data(parameters["dims_dtype"],
+                                parameters["dims_shape"], 1)
+    input2 = create_scalar_data(parameters["value_dtype"])
+    return [input1, input2], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input1, input2])))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=20)
+
+
+@register_make_test_function()
+def make_fill_16_tests(options):
+  """Make a set of tests to do fill with fp16."""
+
+  test_parameters = [{
+      "dims_dtype": [tf.int32, tf.int64],
+      "dims_shape": [[], [1], [3], [3, 3]],
+  }]
+
+  def build_graph(parameters):
+    """Build the fill op testing graph."""
+    input1 = tf.compat.v1.placeholder(
+        dtype=parameters["dims_dtype"],
+        name="dims",
+        shape=parameters["dims_shape"])
+    const_fp16 = tf.constant(1.0, dtype=tf.float16)
+    out = tf.fill(input1, const_fp16)
+    return [input1], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input1 = create_tensor_data(parameters["dims_dtype"],
+                                parameters["dims_shape"], 1)
+    return [input1], sess.run(outputs, feed_dict=dict(zip(inputs, [input1])))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=0)

tensorflow/lite/testing/op_tests/floor.py

@@ -0,0 +1,45 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for floor."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_floor_tests(options):
+  """Make a set of tests to do floor."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32],
+      "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+  }]
+
+  def build_graph(parameters):
+    """Build the floor op testing graph."""
+    input_value = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input1",
+        shape=parameters["input_shape"])
+    out = tf.floor(input_value)
+    return [input_value], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value = create_tensor_data(parameters["input_dtype"],
+                                     parameters["input_shape"])
+    return [input_value], sess.run(outputs, feed_dict={inputs[0]: input_value})
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/fully_connected.py

@@ -0,0 +1,172 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for fully_connected."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_fully_connected_tests(options):
+  """Make a set of tests to do fully_connected."""
+
+  test_parameters = [{
+      "shape1": [[3, 3]],
+      "shape2": [[3, 3]],
+      "transpose_a": [True, False],
+      "transpose_b": [True, False],
+      "constant_filter": [True, False],
+      "fully_quantize": [False],
+      "quant_16x8": [False]
+  }, {
+      "shape1": [[4, 4], [1, 4], [4]],
+      "shape2": [[4, 4], [4, 1], [4]],
+      "transpose_a": [False],
+      "transpose_b": [False],
+      "constant_filter": [True, False],
+      "fully_quantize": [False],
+      "quant_16x8": [False]
+  }, {
+      "shape1": [[40, 37]],
+      "shape2": [[37, 40]],
+      "transpose_a": [False],
+      "transpose_b": [False],
+      "constant_filter": [True, False],
+      "fully_quantize": [False],
+      "quant_16x8": [False]
+  }, {
+      "shape1": [[40, 37]],
+      "shape2": [[40, 37]],
+      "transpose_a": [False],
+      "transpose_b": [True],
+      "constant_filter": [True, False],
+      "fully_quantize": [False],
+      "quant_16x8": [False]
+  }, {
+      "shape1": [[5, 3]],
+      "shape2": [[5, 3]],
+      "transpose_a": [True],
+      "transpose_b": [False],
+      "constant_filter": [True, False],
+      "fully_quantize": [False],
+      "quant_16x8": [False]
+  }, {
+      "shape1": [[1, 3]],
+      "shape2": [[3, 3]],
+      "transpose_a": [False],
+      "transpose_b": [False],
+      "constant_filter": [True],
+      "fully_quantize": [True],
+      "quant_16x8": [False]
+  }, {
+      "shape1": [[1, 4], [4]],
+      "shape2": [[4, 4], [4, 1], [4]],
+      "transpose_a": [False],
+      "transpose_b": [False],
+      "constant_filter": [True],
+      "fully_quantize": [True],
+      "quant_16x8": [False]
+  }, {
+      "shape1": [[1, 37], [2, 37]],
+      "shape2": [[37, 40]],
+      "transpose_a": [False],
+      "transpose_b": [False],
+      "constant_filter": [True],
+      "fully_quantize": [True],
+      "quant_16x8": [False]
+  }, {
+      "shape1": [[1, 3], [2, 3]],
+      "shape2": [[3, 5], [3, 1]],
+      "transpose_a": [False],
+      "transpose_b": [False],
+      "constant_filter": [True],
+      "fully_quantize": [True],
+      "quant_16x8": [False]
+  }, {
+      "shape1": [[2, 3]],
+      "shape2": [[3, 5]],
+      "transpose_a": [False],
+      "transpose_b": [False],
+      "constant_filter": [True],
+      "fully_quantize": [True],
+      "quant_16x8": [True]
+  }, {
+      "shape1": [[0, 3]],
+      "shape2": [[3, 3]],
+      "transpose_a": [False],
+      "transpose_b": [False],
+      "constant_filter": [True, False],
+      "fully_quantize": [False],
+      "quant_16x8": [False]
+  }, {
+      "shape1": [[3, 0]],
+      "shape2": [[0, 3]],
+      "transpose_a": [False],
+      "transpose_b": [False],
+      "constant_filter": [True, False],
+      "fully_quantize": [False],
+      "quant_16x8": [False]
+  }]
+
+  def build_graph(parameters):
+    """Build a matmul graph given `parameters`."""
+    input_tensor1 = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="input1", shape=parameters["shape1"])
+
+    # Get input_tensor2 either as a placeholder or constants. Also get a list of
+    # the input tensors that are represented as placeholders.
+    if parameters["constant_filter"]:
+      input_tensor2 = create_tensor_data(
+          np.float32, parameters["shape2"], min_value=-1, max_value=1)
+      input_tensors = [input_tensor1]
+    else:
+      input_tensor2 = tf.compat.v1.placeholder(
+          dtype=tf.float32, name="input2", shape=parameters["shape2"])
+      input_tensors = [input_tensor1, input_tensor2]
+
+    out = tf.matmul(
+        input_tensor1,
+        input_tensor2,
+        transpose_a=parameters["transpose_a"],
+        transpose_b=parameters["transpose_b"])
+    return input_tensors, [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    # pylint: disable=g-doc-return-or-yield, g-doc-args
+    """Build list of input values.
+
+    It either contains 1 tensor (input_values1) or
+    2 tensors (input_values1, input_values2) based on whether the second input
+    is a constant or variable input.
+    """
+
+    values = [
+        create_tensor_data(
+            np.float32, shape=parameters["shape1"], min_value=-1, max_value=1)
+    ]
+    if not parameters["constant_filter"]:
+      values.append(
+          create_tensor_data(
+              np.float32, parameters["shape2"], min_value=-1, max_value=1))
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=14)

tensorflow/lite/testing/op_tests/fully_connected_4bit_hybrid.py

@@ -0,0 +1,90 @@
+# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for fully_connected_4bit."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_fully_connected_4bit_hybrid_tests(options):
+  """Make a set of tests to do fully_connected."""
+
+  test_parameters = [
+      # Simple 3x3 test.
+      {
+          "shape1": [[3, 3]],
+          "shape2": [[3, 3]],
+          "dynamic_range_quantize": [True],
+      },
+      # Use optimized kernel.
+      {
+          "shape1": [[40, 42]],
+          "shape2": [[42, 40]],
+          "dynamic_range_quantize": [True],
+      },
+      # No optimization.
+      {
+          "shape1": [[1, 40]],
+          "shape2": [[40, 3]],
+          "dynamic_range_quantize": [True],
+      },
+  ]
+
+  def build_graph(parameters):
+    """Build a matmul graph given `parameters`."""
+    input_tensor1 = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="input1", shape=parameters["shape1"]
+    )
+    # Create a float filter with no quantization loss.
+    float_data = np.random.uniform(-1, 1, parameters["shape2"])
+    scale = np.abs(float_data).max() / 7.0
+    int_data = np.round(float_data / scale)
+    input_tensor2 = tf.constant(int_data, dtype=tf.float32)
+    quantized = tf.quantization.fake_quant_with_min_max_vars(
+        input_tensor2, min=-7, max=7, num_bits=4, narrow_range=True
+    )
+    out = tf.matmul(input_tensor1, quantized)
+    return [input_tensor1], [out]
+
+  def create_input_data(parameters):
+    """Create a float input with no quantization loss."""
+    float_data = np.random.random(parameters["shape1"]).astype(np.float32)
+    # Note that since the default ops dynamically quantize the inputs to four
+    # bits, but e.g. XNNPACK dynamically quantizes the inputs to 8 bits, we
+    # generate inputs in {-1, 0, 1} which will be quantized exactly by both
+    # schemes.
+    scale = np.abs(float_data).max(axis=1, keepdims=True) / 1.0
+    return np.round(float_data / scale)
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    # pylint: disable=g-doc-return-or-yield, g-doc-args
+    """Build list of input values.
+
+    Use the specialized method, as dynamic range quantization will cause
+    differing outputs from TF, which does not quantize inputs.
+    """
+    values = [create_input_data(parameters)]
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  options.experimental_low_bit_qat = True
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=0,
+  )

tensorflow/lite/testing/op_tests/fused_batch_norm.py

@@ -0,0 +1,89 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for fused_batch_norm."""
+import numpy as np
+
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_fused_batch_norm_tests(options):
+  """Make a set of tests to do fused_batch_norm."""
+
+  test_parameters = [{
+      "dtype": [tf.float32],
+      "input_shape": [[1, 1, 6, 2]],
+      "epsilon": [0.001, 0.1],
+      "is_training": [False],
+  }, {
+      "dtype": [tf.float32],
+      "input_shape": [[1, 1, 6, 2]],
+      "epsilon": [0.001, 0.1],
+      "is_training": [True],
+  }, {
+      "dtype": [tf.float32],
+      "input_shape": [[1, None, 6, 2]],
+      "epsilon": [0.001, 0.1],
+      "is_training": [True, False],
+  }]
+
+  def build_graph(parameters):
+    """Build the testing graph for fused batch normalization."""
+    input_shape = parameters["input_shape"]
+    scale_shape = input_shape[3]
+
+    scale = create_tensor_data(parameters["dtype"], scale_shape)
+    offset = create_tensor_data(parameters["dtype"], scale_shape)
+    mean = create_tensor_data(parameters["dtype"], scale_shape)
+    variance = create_tensor_data(parameters["dtype"], scale_shape)
+
+    x = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"], name="x", shape=parameters["input_shape"])
+    [x_norm, _, _] = tf.compat.v1.nn.fused_batch_norm(
+        x,
+        scale,
+        offset,
+        mean,
+        variance,
+        parameters["epsilon"],
+        data_format="NHWC",
+        is_training=parameters["is_training"])
+
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+    out = tf.add(input_tensor, x_norm)
+    return [x, input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    # Fill dynamic shape with a random number.
+    input_shape = parameters["input_shape"]
+    input_shape = [
+        np.random.randint(1, 10) if v is None else v for v in input_shape
+    ]
+
+    input_values = [
+        create_tensor_data(parameters["dtype"], input_shape),
+        create_tensor_data(parameters["dtype"], input_shape)
+    ]
+
+    return input_values, sess.run(
+        outputs, feed_dict=dict(zip(inputs, input_values)))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/gather.py

@@ -0,0 +1,128 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for gather."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_gather_tests(options):
+  """Make a set of tests to do gather."""
+
+  test_parameters = [
+      {
+          "params_dtype": [tf.float32, tf.int32, tf.int64, tf.int16],
+          "params_shape": [[1, 2, 20]],
+          "indices_dtype": [tf.int32, tf.int64],
+          "indices_shape": [[3], [5]],
+          "axis": [-1, 0, 1],
+          "batch_dims": [0],
+          "constant_params": [False, True],
+      },
+      {
+          "params_dtype": [tf.string],
+          "params_shape": [[8]],
+          "indices_dtype": [tf.int32],
+          "indices_shape": [[3], [3, 2]],
+          "axis": [0],
+          "batch_dims": [0],
+          "constant_params": [False, True],
+      },
+      {
+          "params_dtype": [tf.float32],
+          "params_shape": [[1, 2, 20]],
+          "indices_dtype": [tf.int32, tf.int64],
+          "indices_shape": [[3], [5]],
+          "axis": [-1, 0, 1],
+          "batch_dims": [0],
+          "constant_params": [False],
+          # Fix the indice values to prevent representative dataset generator
+          # from generating invalid values.
+          "constant_indices": [True],
+          "fully_quantize": [True],
+          "input_range": [(-10, 10)],
+      },
+      {
+          # Test with batch_dims.
+          "params_dtype": [tf.float32, tf.int32],
+          "params_shape": [[2, 2, 3, 5]],
+          "indices_dtype": [tf.int32],
+          "indices_shape": [[2, 2, 2]],
+          "axis": [0, 2],
+          "batch_dims": [1, 2],
+          "constant_params": [False, True],
+      }
+  ]
+
+  def build_graph(parameters):
+    """Build the gather op testing graph."""
+    inputs = []
+
+    if parameters["constant_params"]:
+      params = create_tensor_data(parameters["params_dtype"],
+                                  parameters["params_shape"])
+    else:
+      params = tf.compat.v1.placeholder(
+          dtype=parameters["params_dtype"],
+          name="params",
+          shape=parameters["params_shape"])
+      inputs.append(params)
+
+    if parameters.get("constant_indices", False):
+      indices = create_tensor_data(
+          parameters["indices_dtype"],
+          parameters["indices_shape"],
+          min_value=0,
+          max_value=parameters["params_shape"][0] - 1)
+    else:
+      indices = tf.compat.v1.placeholder(
+          dtype=parameters["indices_dtype"],
+          name="indices",
+          shape=parameters["indices_shape"])
+      inputs.append(indices)
+
+    axis = min(len(parameters["params_shape"]), parameters["axis"])
+    out = tf.gather(
+        params, indices, axis=axis, batch_dims=parameters["batch_dims"])
+    return inputs, [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_values = []
+    min_value, max_value = parameters.get("input_range", (-10, 10))
+    if not parameters["constant_params"]:
+      params = create_tensor_data(
+          parameters["params_dtype"],
+          parameters["params_shape"],
+          min_value=min_value,
+          max_value=max_value)
+      input_values.append(params)
+    if not parameters.get("constant_indices", False):
+      indices = create_tensor_data(
+          parameters["indices_dtype"],
+          parameters["indices_shape"],
+          min_value=0,
+          max_value=parameters["params_shape"][0] - 1)
+      input_values.append(indices)
+    return input_values, sess.run(
+        outputs, feed_dict=dict(zip(inputs, input_values)))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=0)

tensorflow/lite/testing/op_tests/gather_nd.py

@@ -0,0 +1,96 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for gather_nd."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_gather_nd_tests(options):
+  """Make a set of tests to do gather_nd."""
+
+  test_parameters = [
+      {
+          "params_dtype": [
+              tf.float32,
+              tf.int16,
+              tf.int32,
+              tf.int64,
+              tf.string,
+              tf.bool,
+          ],
+          "params_shape": [[5, 1]],
+          "indices_dtype": [tf.int16, tf.int32, tf.int64],
+          "indices_shape": [[1, 1]],
+      },
+      {
+          "params_dtype": [
+              tf.float32,
+              tf.int16,
+              tf.int32,
+              tf.int64,
+              tf.string,
+              tf.bool,
+          ],
+          "params_shape": [[5, 5]],
+          "indices_dtype": [tf.int16, tf.int32, tf.int64],
+          "indices_shape": [[2, 1], [2, 2]],
+      },
+      {
+          "params_dtype": [
+              tf.float32,
+              tf.int16,
+              tf.int32,
+              tf.int64,
+              tf.string,
+              tf.bool,
+          ],
+          "params_shape": [[5, 5, 10]],
+          "indices_dtype": [tf.int16, tf.int32, tf.int64],
+          "indices_shape": [[3, 1], [2, 2], [2, 3], [2, 1, 3]],
+      },
+      {
+          "params_dtype": [tf.float32, tf.string],
+          "params_shape": [[1, 0]],
+          "indices_dtype": [tf.int64],
+          "indices_shape": [[0, 2]],
+      },
+  ]
+
+  def build_graph(parameters):
+    """Build the gather_nd op testing graph."""
+    params = tf.compat.v1.placeholder(
+        dtype=parameters["params_dtype"],
+        name="params",
+        shape=parameters["params_shape"])
+    indices = tf.compat.v1.placeholder(
+        dtype=parameters["indices_dtype"],
+        name="indices",
+        shape=parameters["indices_shape"])
+    out = tf.gather_nd(params, indices)
+    return [params, indices], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    params = create_tensor_data(parameters["params_dtype"],
+                                parameters["params_shape"])
+    indices = create_tensor_data(parameters["indices_dtype"],
+                                 parameters["indices_shape"], 0,
+                                 parameters["params_shape"][0] - 1)
+    return [params, indices], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [params, indices])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/gather_with_constant.py

@@ -0,0 +1,50 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for gather_with_constant."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_gather_with_constant_tests(options):
+  """Make a set of test which feed a constant to gather."""
+
+  test_parameters = [{
+      "input_shape": [[3]],
+      "reference_shape": [[2]],
+  }, {
+      "input_shape": [[2, 3]],
+      "reference_shape": [[2, 3]],
+  }]
+
+  def build_graph(parameters):
+    """Build a graph where the inputs to Gather are constants."""
+    reference = tf.compat.v1.placeholder(
+        dtype=tf.int32, shape=parameters["reference_shape"])
+    gather_input = tf.constant(
+        create_tensor_data(tf.int32, parameters["input_shape"]))
+    gather_indices = tf.constant([0, 1], tf.int32)
+    out = tf.equal(reference, tf.gather(gather_input, gather_indices))
+    return [reference], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    reference_values = np.zeros(parameters["reference_shape"], dtype=np.int32)
+    return [reference_values], sess.run(
+        outputs, feed_dict={inputs[0]: reference_values})
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/gelu.py

@@ -0,0 +1,77 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for gelu."""
+import functools
+
+import tensorflow as tf
+from tensorflow.lite.python import lite
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+def _tflite_convert_verify_op(tflite_convert_function, *args, **kwargs):
+  """Verifies that the result of the conversion contains Gelu op."""
+  result = tflite_convert_function(*args, **kwargs)
+  tflite_model_binary = result[0]
+  if not result[0]:
+    tf.compat.v1.logging.error(result[1])  # stderr from running tflite_convert.
+    raise RuntimeError("Failed to build model: \n\n" + result[1])
+  interpreter = lite.Interpreter(model_content=tflite_model_binary)
+  interpreter.allocate_tensors()
+  for op in interpreter._get_ops_details():  # pylint: disable=protected-access
+    if op["op_name"] == "GELU":
+      return result
+  raise RuntimeError("Expected to generate GELU op node in graph.")
+
+
+@register_make_test_function()
+def make_gelu_tests(options):
+  """Makes a set of tests for gelu."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32],
+      "input_shape": [[], [1], [2, 3], [1, 1, 1, 1], [1, 3, 4, 3],
+                      [3, 15, 14, 3], [3, 1, 2, 4, 6], [2, 2, 3, 4, 5, 6]],
+      "fully_quantize": [False, True],
+      "input_range": [(-10, 10)],
+      "approximate": [True, False],
+  }]
+
+  def build_graph(parameters):
+    """Builds the gelu op testing graph."""
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+
+    out = tf.nn.gelu(input_tensor, approximate=parameters["approximate"])
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    values = [
+        create_tensor_data(
+            parameters["input_dtype"],
+            parameters["input_shape"],
+            min_value=-8,
+            max_value=8)
+    ]
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  if not options.run_with_flex:
+    options.tflite_convert_function = functools.partial(
+        _tflite_convert_verify_op,
+        options.tflite_convert_function)
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/global_batch_norm.py

@@ -0,0 +1,61 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for global_batch_norm."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_global_batch_norm_tests(options):
+  """Make a set of tests to do batch_norm_with_global_normalization."""
+
+  test_parameters = [{
+      "dtype": [tf.float32],
+      "input_shape": [[1, 1, 6, 2], [3, 4, 5, 4]],
+      "epsilon": [0.1, 0.0001],
+      "scale_after": [True, False],
+  }]
+
+  def build_graph(parameters):
+    """Build the global batch norm testing graph."""
+    input_shape = parameters["input_shape"]
+    scale_shape = input_shape[3]
+
+    scale = create_tensor_data(parameters["dtype"], scale_shape)
+    offset = create_tensor_data(parameters["dtype"], scale_shape)
+    mean = create_tensor_data(parameters["dtype"], scale_shape)
+    variance = create_tensor_data(parameters["dtype"], scale_shape)
+
+    x = create_tensor_data(parameters["dtype"], parameters["input_shape"])
+    x_norm = tf.nn.batch_norm_with_global_normalization(
+        x, mean, variance, scale, offset, parameters["epsilon"],
+        parameters["scale_after"])
+
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+    out = tf.add(input_tensor, x_norm)
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value = create_tensor_data(parameters["dtype"],
+                                     parameters["input_shape"])
+    return [input_value], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_value])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/greater.py

@@ -0,0 +1,64 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for conv."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_greater_tests(options):
+  """Make a set of tests to do greater."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32, tf.int32, tf.int64],
+      "input_shape_pair": [([1, 1, 1, 3], [1, 1, 1, 3]),
+                           ([2, 3, 4, 5], [2, 3, 4, 5]), ([2, 3, 3], [2, 3]),
+                           ([5, 5], [1]), ([10], [2, 4, 10])],
+      "fully_quantize": [False],
+  }, {
+      "input_dtype": [tf.float32],
+      "input_shape_pair": [([1, 1, 1, 3], [1, 1, 1, 3]), ([2, 3, 3], [2, 3])],
+      "fully_quantize": [True],
+  }]
+
+  def build_graph(parameters):
+    """Build the greater op testing graph."""
+    input_value1 = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input1",
+        shape=parameters["input_shape_pair"][0])
+    input_value2 = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input2",
+        shape=parameters["input_shape_pair"][1])
+    out = tf.greater(input_value1, input_value2)
+    return [input_value1, input_value2], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value1 = create_tensor_data(parameters["input_dtype"],
+                                      parameters["input_shape_pair"][0])
+    input_value2 = create_tensor_data(parameters["input_dtype"],
+                                      parameters["input_shape_pair"][1])
+    return [input_value1, input_value2], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_value1, input_value2])))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=4)

tensorflow/lite/testing/op_tests/greater_equal.py

@@ -0,0 +1,64 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for greater_equal."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_greater_equal_tests(options):
+  """Make a set of tests to do greater_equal."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32, tf.int16, tf.int32, tf.int64],
+      "input_shape_pair": [([1, 1, 1, 3], [1, 1, 1, 3]),
+                           ([2, 3, 4, 5], [2, 3, 4, 5]), ([2, 3, 3], [2, 3]),
+                           ([5, 5], [1]), ([10], [2, 4, 10])],
+      "fully_quantize": [False],
+  }, {
+      "input_dtype": [tf.float32],
+      "input_shape_pair": [([1, 1, 1, 3], [1, 1, 1, 3]), ([2, 3, 3], [2, 3])],
+      "fully_quantize": [True],
+  }]
+
+  def build_graph(parameters):
+    """Build the greater_equal op testing graph."""
+    input_value1 = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input1",
+        shape=parameters["input_shape_pair"][0])
+    input_value2 = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input2",
+        shape=parameters["input_shape_pair"][1])
+    out = tf.greater_equal(input_value1, input_value2)
+    return [input_value1, input_value2], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value1 = create_tensor_data(parameters["input_dtype"],
+                                      parameters["input_shape_pair"][0])
+    input_value2 = create_tensor_data(parameters["input_dtype"],
+                                      parameters["input_shape_pair"][1])
+    return [input_value1, input_value2], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_value1, input_value2])))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=5)

tensorflow/lite/testing/op_tests/hardswish.py

@@ -0,0 +1,80 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for hardswish."""
+import functools
+
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.python import lite
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+def _tflite_convert_verify_num_ops(tflite_convert_function, *args, **kwargs):
+  """Verifies that the result of the conversion is a single op."""
+  num_ops = kwargs.pop("num_ops", 2)
+  result = tflite_convert_function(*args, **kwargs)
+  tflite_model_binary = result[0]
+  if not result[0]:
+    tf.compat.v1.logging.error(result[1])  # stderr from running tflite_convert.
+    raise RuntimeError("Failed to build model: \n\n" + result[1])
+  interpreter = lite.Interpreter(model_content=tflite_model_binary)
+  interpreter.allocate_tensors()
+  if len(interpreter.get_tensor_details()) != num_ops:
+    raise RuntimeError(
+        "Expected to generate two node graph got %s " %
+        "\n".join(str(x) for x in interpreter.get_tensor_details()))
+  return result
+
+
+@register_make_test_function()
+def make_hardswish_tests(options):
+  """Make a set of tests to do hardswish."""
+
+  # Chose a set of parameters
+  if options.run_with_flex:
+    # Only Flex is able to execute on the data bigger than four dimension.
+    test_parameters = [{
+        "input_shape": [[], [1], [2, 3], [1, 1, 1, 1], [1, 3, 4, 3],
+                        [3, 15, 14, 3], [3, 1, 2, 4, 6], [2, 2, 3, 4, 5, 6]],
+    }]
+  else:
+    test_parameters = [{
+        "input_shape": [[], [1], [2, 3], [1, 1, 1, 1], [1, 3, 4, 3],
+                        [3, 15, 14, 3]],
+    }]
+
+  def build_graph(parameters):
+    inp = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="input", shape=parameters["input_shape"])
+    out = inp * tf.nn.relu6(inp + np.float32(3)) * np.float32(1. / 6.)
+
+    return [inp], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_values = create_tensor_data(
+        np.float32, parameters["input_shape"], min_value=-10, max_value=10)
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  # Add additional validation if we are using converter.
+  # Flex doesn't yet support this.
+  if not options.run_with_flex:
+    options.tflite_convert_function = functools.partial(
+        _tflite_convert_verify_num_ops,
+        options.tflite_convert_function,
+        num_ops=2)
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/identify_dilated_conv.py

@@ -0,0 +1,92 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for identifying dilated conv."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_identify_dilated_conv_tests(options):
+  """Make a set of tests to do dilated convolution."""
+
+  test_parameters = [
+      {
+          "input_shape": [[1, 3, 4, 3], [4, 6, 6, 1]],
+          "filter_shape": [[1, 1], [2, 3], [3, 3]],
+          "strides": [[1, 1, 1, 1], [1, 2, 3, 1]],
+          "dilations": [[1, 3, 2, 1], [1, 2, 2, 1], [1, 2, 1, 1]],
+          "padding": ["VALID", "SAME"],
+          "data_format": ["NHWC"],
+          "constant_filter": [True, False],
+          "channel_multiplier": [1, 2],
+      },
+  ]
+
+  def get_tensor_shapes(parameters):
+    input_shape = parameters["input_shape"]
+    filter_size = parameters["filter_shape"]
+    filter_shape = filter_size + [
+        input_shape[3], parameters["channel_multiplier"]
+    ]
+    return [input_shape, filter_shape]
+
+  def build_graph(parameters):
+    """Build a conv graph given `parameters`."""
+    input_shape, filter_shape = get_tensor_shapes(parameters)
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="input", shape=input_shape)
+
+    # Get filter input either as a placeholder or constants. Also get a list of
+    # the input tensors that are represented as placeholders.
+    if parameters["constant_filter"]:
+      filter_input = create_tensor_data(
+          np.float32, filter_shape, min_value=-10, max_value=10)
+      input_tensors = [input_tensor]
+    else:
+      filter_input = tf.compat.v1.placeholder(
+          dtype=tf.float32, name="filter", shape=filter_shape)
+      input_tensors = [input_tensor, filter_input]
+
+    # Use `tf.nn.convolution` here since it will create the `batch_to_space` and
+    # the `space_to_batch` ops respectively.
+    out = tf.nn.convolution(
+        input=input_tensor,
+        filters=filter_input,
+        strides=parameters["strides"],
+        dilations=parameters["dilations"],
+        padding=parameters["padding"],
+        data_format=parameters["data_format"])
+    return input_tensors, [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    # Build list of input values either containing 1 tensor (input) or 2 tensors
+    # (input, filter) based on whether filter is constant or variable input.
+    input_shape, filter_shape = get_tensor_shapes(parameters)
+    values = [
+        create_tensor_data(np.float32, input_shape, min_value=-1, max_value=1)
+    ]
+    if not parameters["constant_filter"]:
+      values.append(create_tensor_data(np.float32, filter_shape))
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=168)

tensorflow/lite/testing/op_tests/identify_dilated_conv1d.py

@@ -0,0 +1,75 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for identifying dilated Conv1D."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_identify_dilated_conv1d_tests(options):
+  """Make a set of tests to do 1D dilated convolution."""
+
+  test_parameters = [
+      {
+          "input_shape": [[1, 3, 3], [4, 6, 1]],
+          "filter_size": [1, 2, 3],
+          "stride": [1, 2],
+          "dilations": [1, 2, 3],
+          "padding": ["VALID", "SAME"],
+          "num_filters": [1, 2],
+      },
+  ]
+
+  def get_tensor_shapes(parameters):
+    input_shape = parameters["input_shape"]
+    filter_size = parameters["filter_size"]
+    filter_shape = [filter_size, input_shape[2], parameters["num_filters"]]
+    return [input_shape, filter_shape]
+
+  def build_graph(parameters):
+    """Build a conv graph given `parameters`."""
+    input_shape, filter_shape = get_tensor_shapes(parameters)
+    filter_input = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="filter", shape=filter_shape)
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="input", shape=input_shape)
+    input_tensors = [input_tensor, filter_input]
+
+    out = tf.nn.conv1d(
+        input=input_tensor,
+        filters=filter_input,
+        stride=parameters["stride"],
+        dilations=parameters["dilations"],
+        padding=parameters["padding"])
+    return input_tensors, [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_shape, filter_shape = get_tensor_shapes(parameters)
+    values = [
+        create_tensor_data(np.float32, input_shape, min_value=-1, max_value=1),
+        create_tensor_data(np.float32, filter_shape)
+    ]
+
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=16)

tensorflow/lite/testing/op_tests/identity.py

@@ -0,0 +1,73 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for identity."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+from tensorflow.python.ops import array_ops
+
+
+@register_make_test_function()
+def make_identity_tests(options):
+  """Make a set of tests to do identity."""
+
+  # Chose a set of parameters
+  test_parameters = [{
+      "input_shape": [[], [1], [3, 3]],
+      "op_to_use": [
+          "identity", "identity_n", "snapshot", "identity_n_with_2_inputs"
+      ],
+  }]
+
+  def build_graph(parameters):
+    """Make a set of tests to do identity."""
+
+    input_tensors = []
+    input_count = (2 if parameters["op_to_use"] == "identity_n_with_2_inputs"
+                   else 1)
+    input_tensors = [
+        tf.compat.v1.placeholder(
+            dtype=tf.float32, name="input", shape=parameters["input_shape"])
+        for _ in range(input_count)
+    ]
+
+    # We add the Multiply before Identity just as a walk-around to make the test
+    # pass when input_shape is scalar.
+    # During graph transformation, converter will replace the Identity op with
+    # Reshape when input has shape. However, currently converter can't
+    # distinguish between missing shape and scalar shape. As a result, when
+    # input has scalar shape, this conversion still fails.
+    inputs_doubled = [input_tensor * 2.0 for input_tensor in input_tensors]
+    if parameters["op_to_use"] == "identity":
+      identity_outputs = [tf.identity(inputs_doubled[0])]
+    elif parameters["op_to_use"] == "snapshot":
+      identity_outputs = [array_ops.snapshot(inputs_doubled[0])]
+    elif parameters["op_to_use"] in ("identity_n", "identity_n_with_2_inputs"):
+      identity_outputs = tf.identity_n(inputs_doubled)
+    return input_tensors, identity_outputs
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_values = [
+        create_tensor_data(
+            np.float32, parameters["input_shape"], min_value=-4, max_value=10)
+        for _ in range(len(inputs))
+    ]
+
+    return input_values, sess.run(
+        outputs, feed_dict=dict(zip(inputs, input_values)))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/imag.py

@@ -0,0 +1,49 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for imag op."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_imag_tests(options):
+  """Make a set of tests to do imag op."""
+
+  # Chose a set of parameters
+  test_parameters = [{
+      "dtype": [tf.complex64, tf.complex128],
+      "input_shape": [[], [1], [2, 3], [1, 3, 4, 3], [2, 2, 3, 4, 5, 6]],
+  }]
+
+  def build_graph(parameters):
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+    out = tf.math.imag(input_tensor)
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_values = create_tensor_data(
+        parameters["dtype"].as_numpy_dtype,
+        parameters["input_shape"],
+        min_value=-10,
+        max_value=10)
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/irfft2d.py

@@ -0,0 +1,64 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for irfft2d."""
+import numpy as np
+import tensorflow as tf
+
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import ExtraConvertOptions
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_irfft2d_tests(options):
+  """Make a set of tests to do irfft2d."""
+
+  test_parameters = [{
+      "input_dtype": [tf.complex64],
+      "input_shape": [[4, 3]],
+      "fft_length": [[4, 4], [2, 2], [2, 4]]
+  }, {
+      "input_dtype": [tf.complex64],
+      "input_shape": [[3, 8, 5]],
+      "fft_length": [[2, 4], [2, 8], [8, 8]]
+  }, {
+      "input_dtype": [tf.complex64],
+      "input_shape": [[3, 1, 9]],
+      "fft_length": [[1, 8], [1, 16]]
+  }]
+
+  def build_graph(parameters):
+    input_value = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+    outs = tf.signal.irfft2d(input_value, fft_length=parameters["fft_length"])
+    return [input_value], [outs]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    rfft_length = []
+    rfft_length.append(parameters["input_shape"][-2])
+    rfft_length.append((parameters["input_shape"][-1] - 1) * 2)
+    rfft_input = create_tensor_data(np.float32, parameters["input_shape"])
+    rfft_result = np.fft.rfft2(rfft_input, rfft_length)
+
+    return [rfft_result], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [rfft_result])))
+
+  extra_convert_options = ExtraConvertOptions()
+  extra_convert_options.allow_custom_ops = True
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs,
+                    extra_convert_options)

tensorflow/lite/testing/op_tests/is_finite.py

@@ -0,0 +1,62 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for is_finite."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_is_finite_tests(options):
+  """Make a set of tests to do is_finite."""
+
+  test_parameters = [
+      {
+          "input_shape": [[100], [3, 15, 14, 3]],
+      },
+  ]
+
+  def build_graph(parameters):
+    """Build the graph for the test case."""
+
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="input", shape=parameters["input_shape"])
+    out = tf.math.is_finite(input_tensor)
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    """Build the inputs for the test case."""
+    input_values = create_tensor_data(
+        np.float32, parameters["input_shape"], min_value=-10, max_value=10)
+
+    # Inject NaN and Inf value.
+    def random_index(shape):
+      result = []
+      for dim in shape:
+        result.append(np.random.randint(low=0, high=dim))
+      return tuple(result)
+
+    input_values[random_index(input_values.shape)] = np.inf
+    input_values[random_index(input_values.shape)] = -np.inf
+    input_values[random_index(input_values.shape)] = np.nan
+    input_values[random_index(input_values.shape)] = tf.float32.max
+    input_values[random_index(input_values.shape)] = tf.float32.min
+
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/l2norm.py

@@ -0,0 +1,66 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for l2norm."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_l2norm_tests(options):
+  """Make a set of tests to do l2norm."""
+
+  # Chose a set of parameters
+  test_parameters = [{
+      "input_shape": [[5, 7], [1, 1, 1, 1], [1, 3, 4, 3], [3, 15, 14, 3]],
+      "dim": [0, 1, 2, 3, [2, 3], -2],
+      "epsilon": [None, 1e-12, 1e-3],
+      "fully_quantize": [False],
+  }, {
+      "input_shape": [[1, 1, 1, 1], [1, 3, 4, 3], [3, 15, 14, 3]],
+      "dim": [3],
+      "epsilon": [None, 1e-12],
+      "fully_quantize": [True],
+  }, {  # use another group of test so the dim is set to fuse to tfl.l2norm
+      "input_shape": [[5, 7]],
+      "dim": [1],
+      "epsilon": [None, 1e-12],
+      "fully_quantize": [True],
+  }]
+
+  def build_graph(parameters):
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="input", shape=parameters["input_shape"])
+    if parameters["epsilon"]:
+      out = tf.nn.l2_normalize(
+          input_tensor, parameters["dim"], epsilon=parameters["epsilon"])
+    else:
+      out = tf.nn.l2_normalize(input_tensor, parameters["dim"])
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_values = create_tensor_data(
+        np.float32, parameters["input_shape"], min_value=-1, max_value=1)
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=9)

tensorflow/lite/testing/op_tests/l2norm_shared_epsilon.py

@@ -0,0 +1,49 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for l2norm_shared_epsilon."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_l2norm_shared_epsilon_tests(options):
+  """Regression test for a bug (b/122651451)."""
+
+  # Chose a set of parameters
+  test_parameters = [{
+      "input_shape": [[5, 7]],
+      "dim": [1],
+      "epsilon": [1e-8],
+  }]
+
+  def build_graph(parameters):
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="input", shape=parameters["input_shape"])
+    epsilon = tf.constant(parameters["epsilon"])
+    out1 = tf.nn.l2_normalize(input_tensor, parameters["dim"], epsilon=epsilon)
+    out2 = tf.nn.l2_normalize(input_tensor, parameters["dim"], epsilon=epsilon)
+    out = out1 + out2
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_values = create_tensor_data(
+        np.float32, parameters["input_shape"], min_value=-4, max_value=10)
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/leaky_relu.py

@@ -0,0 +1,50 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for leaky_relu."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_leaky_relu_tests(options):
+  """Make a set of tests to do LeakyRelu."""
+
+  test_parameters = [{
+      "input_shape": [[], [1], [5], [1, 10, 10, 3], [3, 3, 3, 3]],
+      "alpha": [0.1, 1.0, 2.0, -0.1, -1.0, -2.0],
+      "fully_quantize": [False, True],
+      "input_range": [(-3, 10)],
+      "quant_16x8": [False, True],
+  }]
+
+  def build_graph(parameters):
+    """Build the graph for the test case."""
+
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="input", shape=parameters["input_shape"])
+    out = tf.nn.leaky_relu(input_tensor, alpha=parameters["alpha"])
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    """Build the inputs for the test case."""
+    input_values = create_tensor_data(
+        np.float32, parameters["input_shape"], min_value=-3, max_value=10)
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/less.py

@@ -0,0 +1,64 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for less."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_less_tests(options):
+  """Make a set of tests to do less."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32, tf.int16, tf.int32, tf.int64],
+      "input_shape_pair": [([1, 1, 1, 3], [1, 1, 1, 3]),
+                           ([2, 3, 4, 5], [2, 3, 4, 5]), ([2, 3, 3], [2, 3]),
+                           ([5, 5], [1]), ([10], [2, 4, 10])],
+      "fully_quantize": [False],
+  }, {
+      "input_dtype": [tf.float32],
+      "input_shape_pair": [([1, 1, 1, 3], [1, 1, 1, 3]), ([2, 3, 3], [2, 3])],
+      "fully_quantize": [True],
+  }]
+
+  def build_graph(parameters):
+    """Build the less op testing graph."""
+    input_value1 = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input1",
+        shape=parameters["input_shape_pair"][0])
+    input_value2 = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input2",
+        shape=parameters["input_shape_pair"][1])
+    out = tf.less(input_value1, input_value2)
+    return [input_value1, input_value2], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value1 = create_tensor_data(parameters["input_dtype"],
+                                      parameters["input_shape_pair"][0])
+    input_value2 = create_tensor_data(parameters["input_dtype"],
+                                      parameters["input_shape_pair"][1])
+    return [input_value1, input_value2], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_value1, input_value2])))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=5)

tensorflow/lite/testing/op_tests/less_equal.py

@@ -0,0 +1,87 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for less_equal."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_less_equal_tests(options):
+  """Make a set of tests to do less_equal."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32, tf.int32, tf.int64],
+      "input_shape_pair": [([1, 1, 1, 3], [1, 1, 1, 3]),
+                           ([2, 3, 4, 5], [2, 3, 4, 5]), ([2, 3, 3], [2, 3]),
+                           ([5, 5], [1]), ([10], [2, 4, 10])],
+      "fully_quantize": [False],
+  }, {
+      "input_dtype": [tf.float32],
+      "input_shape_pair": [([1, 1, 1, 3], [1, 1, 1, 3]), ([2, 3, 3], [2, 3])],
+      "fully_quantize": [True],
+  }]
+
+  # High dimension broadcasting support in MLIR converter.
+  # Note(b/204360746): XNNPack delegate don't support high dimension.
+  if not options.skip_high_dimension_inputs:
+    test_parameters = test_parameters + [
+        {
+            "input_dtype": [tf.float32, tf.int32],
+            "input_shape_pair": [([6, 5, 4, 3, 2, 1], [4, 3, 2, 1]),
+                                 ([6, 5, 4, 3, 2, 1], [None, 3, 2, 1]),
+                                 ([6, 5, None, 3, 2, 1], [None, 3, 2, 1])],
+            "fully_quantize": [False],
+            "dynamic_size_value": [4, 1],
+        },
+    ]
+
+  def populate_dynamic_shape(parameters, input_shape):
+    return [
+        parameters["dynamic_size_value"] if x is None else x
+        for x in input_shape
+    ]
+
+  def build_graph(parameters):
+    """Build the less_equal op testing graph."""
+    input_value1 = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input1",
+        shape=parameters["input_shape_pair"][0])
+    input_value2 = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input2",
+        shape=parameters["input_shape_pair"][1])
+    out = tf.less_equal(input_value1, input_value2)
+    return [input_value1, input_value2], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_shape_1 = populate_dynamic_shape(parameters,
+                                           parameters["input_shape_pair"][0])
+    input_shape_2 = populate_dynamic_shape(parameters,
+                                           parameters["input_shape_pair"][1])
+
+    input_value1 = create_tensor_data(parameters["input_dtype"], input_shape_1)
+    input_value2 = create_tensor_data(parameters["input_dtype"], input_shape_2)
+    return [input_value1, input_value2], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_value1, input_value2])))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=4)

tensorflow/lite/testing/op_tests/local_response_norm.py

@@ -0,0 +1,53 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for local_response_norm."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_local_response_norm_tests(options):
+  """Make a set of tests to do local_response_norm."""
+
+  # Chose a set of parameters
+  test_parameters = [{
+      "input_shape": [[1, 1, 1, 1], [1, 3, 4, 3], [3, 15, 14, 3]],
+      "depth_radius": [None, 0, 1, 3, 5],
+      "bias": [None, 0.3, -0.1],
+      "alpha": [None, 2, -3],
+      "beta": [None, 0.25, 2],
+  }]
+
+  def build_graph(parameters):
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="input", shape=parameters["input_shape"])
+    out = tf.nn.local_response_normalization(
+        input_tensor,
+        depth_radius=parameters["depth_radius"],
+        bias=parameters["bias"],
+        alpha=parameters["alpha"],
+        beta=parameters["beta"])
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_values = create_tensor_data(
+        np.float32, parameters["input_shape"], min_value=-4, max_value=10)
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/log_softmax.py

@@ -0,0 +1,51 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for log_softmax."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_log_softmax_tests(options):
+  """Make a set of tests to do log_softmax."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32],
+      "input_shape": [[1, 100], [4, 2], [5, 224]],
+  }]
+
+  def build_graph(parameters):
+    """Build the log_softmax op testing graph."""
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+
+    out = tf.nn.log_softmax(input_tensor)
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    values = [
+        create_tensor_data(
+            parameters["input_dtype"],
+            parameters["input_shape"],
+            min_value=-100,
+            max_value=9)
+    ]
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/logic.py

@@ -0,0 +1,76 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for logic operators."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+def _make_logical_tests(op):
+  """Make a set of tests to do logical operations."""
+
+  def logical(options, expected_tf_failures=0):
+    """Generate examples."""
+    test_parameters = [{
+        "input_shape_pair": [([], []), ([1, 1, 1, 3], [1, 1, 1, 3]),
+                             ([2, 3, 4, 5], [2, 3, 4, 5]), ([2, 3, 3], [2, 3]),
+                             ([5, 5], [1]), ([10], [2, 4, 10])],
+    }]
+
+    def build_graph(parameters):
+      """Build the logical testing graph."""
+      input_value1 = tf.compat.v1.placeholder(
+          dtype=tf.bool, name="input1", shape=parameters["input_shape_pair"][0])
+      input_value2 = tf.compat.v1.placeholder(
+          dtype=tf.bool, name="input2", shape=parameters["input_shape_pair"][1])
+      out = op(input_value1, input_value2)
+      return [input_value1, input_value2], [out]
+
+    def build_inputs(parameters, sess, inputs, outputs):
+      input_value1 = create_tensor_data(tf.bool,
+                                        parameters["input_shape_pair"][0])
+      input_value2 = create_tensor_data(tf.bool,
+                                        parameters["input_shape_pair"][1])
+      return [input_value1, input_value2], sess.run(
+          outputs, feed_dict=dict(zip(inputs, [input_value1, input_value2])))
+
+    make_zip_of_tests(
+        options,
+        test_parameters,
+        build_graph,
+        build_inputs,
+        expected_tf_failures=expected_tf_failures)
+
+  return logical
+
+
+@register_make_test_function()
+def make_logical_or_tests(options):
+  """Make a set of tests to do logical_or."""
+  return _make_logical_tests(tf.logical_or)(options, expected_tf_failures=1)
+
+
+@register_make_test_function()
+def make_logical_and_tests(options):
+  """Make a set of tests to do logical_and."""
+  return _make_logical_tests(tf.logical_and)(options, expected_tf_failures=1)
+
+
+@register_make_test_function()
+def make_logical_xor_tests(options):
+  """Make a set of tests to do logical_xor, test logical_not as well."""
+  return _make_logical_tests(tf.math.logical_xor)(
+      options, expected_tf_failures=1)

tensorflow/lite/testing/op_tests/lstm.py

@@ -0,0 +1,102 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for lstm."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import ExtraConvertOptions
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+from tensorflow.python.ops import rnn
+
+
+@register_make_test_function()
+def make_lstm_tests(options):
+  """Make a set of tests to do basic Lstm cell."""
+
+  test_parameters = [
+      {
+          "dtype": [tf.float32],
+          "num_batchs": [1],
+          "time_step_size": [1],
+          "input_vec_size": [3],
+          "num_cells": [4],
+          "split_tflite_lstm_inputs": [False],
+      },
+  ]
+
+  def build_graph(parameters):
+    """Build a simple graph with BasicLSTMCell."""
+
+    num_batchs = parameters["num_batchs"]
+    time_step_size = parameters["time_step_size"]
+    input_vec_size = parameters["input_vec_size"]
+    num_cells = parameters["num_cells"]
+    inputs_after_split = []
+    for i in range(time_step_size):
+      one_timestamp_input = tf.compat.v1.placeholder(
+          dtype=parameters["dtype"],
+          name="split_{}".format(i),
+          shape=[num_batchs, input_vec_size])
+      inputs_after_split.append(one_timestamp_input)
+    # Currently lstm identifier has a few limitations: only supports
+    # forget_bias == 0, inner state activation == tanh.
+    # TODO(zhixianyan): Add another test with forget_bias == 1.
+    # TODO(zhixianyan): Add another test with relu as activation.
+    lstm_cell = tf.compat.v1.nn.rnn_cell.BasicLSTMCell(
+        num_cells, forget_bias=0.0, state_is_tuple=True)
+    cell_outputs, _ = rnn.static_rnn(
+        lstm_cell, inputs_after_split, dtype=tf.float32)
+    out = cell_outputs[-1]
+    return inputs_after_split, [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    """Feed inputs, assign variables, and freeze graph."""
+
+    with tf.compat.v1.variable_scope("", reuse=True):
+      kernel = tf.compat.v1.get_variable("rnn/basic_lstm_cell/kernel")
+      bias = tf.compat.v1.get_variable("rnn/basic_lstm_cell/bias")
+      kernel_values = create_tensor_data(parameters["dtype"],
+                                         [kernel.shape[0], kernel.shape[1]], -1,
+                                         1)
+      bias_values = create_tensor_data(parameters["dtype"], [bias.shape[0]], 0,
+                                       1)
+      sess.run(tf.group(kernel.assign(kernel_values), bias.assign(bias_values)))
+
+    num_batchs = parameters["num_batchs"]
+    time_step_size = parameters["time_step_size"]
+    input_vec_size = parameters["input_vec_size"]
+    input_values = []
+    for _ in range(time_step_size):
+      tensor_data = create_tensor_data(parameters["dtype"],
+                                       [num_batchs, input_vec_size], 0, 1)
+      input_values.append(tensor_data)
+    out = sess.run(outputs, feed_dict=dict(zip(inputs, input_values)))
+    return input_values, out
+
+  # TODO(zhixianyan): Automatically generate rnn_states for lstm cell.
+  extra_convert_options = ExtraConvertOptions()
+  extra_convert_options.rnn_states = (
+      "{state_array:rnn/BasicLSTMCellZeroState/zeros,"
+      "back_edge_source_array:rnn/basic_lstm_cell/Add_1,size:4},"
+      "{state_array:rnn/BasicLSTMCellZeroState/zeros_1,"
+      "back_edge_source_array:rnn/basic_lstm_cell/Mul_2,size:4}")
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      extra_convert_options,
+      use_frozen_graph=True)

tensorflow/lite/testing/op_tests/matrix_band_part.py

@@ -0,0 +1,74 @@
+# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for matrix_band_part."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_matrix_band_part_tests(options):
+  """Make a set of tests for tf.linalg.band_part op."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32],
+      "input_shape": [[1, 2], [3, 4, 5], [6, 7, 8, 9], [None, None], [10, None],
+                      [None, 10], [3, None, 10], [None, None, None], None],
+      # TFL range doesn't accept I64.
+      "index_dtype": [tf.int32],
+  }]
+
+  def get_static_shape(shape):
+    """Randomly assign static number for dynamic dimension."""
+    if not shape:
+      return np.random.randint(
+          low=1, high=10, size=np.random.randint(low=2, high=10,
+                                                 size=())).tolist()
+    return [x or np.random.randint(low=5, high=10, size=()) for x in shape]
+
+  def build_graph(parameters):
+    """Build the sign op testing graph."""
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+    num_lower = tf.compat.v1.placeholder(
+        dtype=parameters["index_dtype"], name="num_lower", shape=())
+    num_upper = tf.compat.v1.placeholder(
+        dtype=parameters["index_dtype"], name="num_upper", shape=())
+    out = tf.linalg.band_part(input_tensor, num_lower, num_upper)
+    return [input_tensor, num_lower, num_upper], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    static_input_shape = get_static_shape(parameters["input_shape"])
+    input_value = create_tensor_data(parameters["input_dtype"],
+                                     static_input_shape)
+    num_lower = create_tensor_data(
+        parameters["index_dtype"],
+        shape=(),
+        min_value=-1,
+        max_value=static_input_shape[-2])
+    num_upper = create_tensor_data(
+        parameters["index_dtype"],
+        shape=(),
+        min_value=-1,
+        max_value=static_input_shape[-1])
+    return [input_value, num_lower, num_upper
+           ], sess.run(outputs,
+                       dict(zip(inputs, [input_value, num_lower, num_upper])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/matrix_diag.py

@@ -0,0 +1,47 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for matrix_diag."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_matrix_diag_tests(options):
+  """Make a set of tests for tf.linalg.diag op."""
+
+  test_parameters = [
+      {
+          "input_shape": [[3], [2, 3], [3, 4, 5], [2, 4, 6, 8]],
+          "input_dtype": [tf.int32, tf.float32],
+      },
+  ]
+
+  def build_graph(parameters):
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+    outs = tf.linalg.diag(input_tensor)
+    return [input_tensor], [outs]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_values = create_tensor_data(parameters["input_dtype"],
+                                      parameters["input_shape"])
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/matrix_set_diag.py

@@ -0,0 +1,53 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for matrix_set_diag."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_matrix_set_diag_tests(options):
+  """Make a set of tests for tf.linalg.set_diag op."""
+
+  test_parameters = [
+      {
+          "input_diag_shapes": [([3, 3], [3]), ([2, 3], [2]), ([2, 4,
+                                                                4], [2, 4]),
+                                ([3, 4, 5, 6], [3, 4, 5])],
+          "input_dtype": [tf.int32, tf.float32, tf.uint8],
+      },
+  ]
+
+  def build_graph(parameters):
+    input_shape = parameters["input_diag_shapes"][0]
+    diag_shape = parameters["input_diag_shapes"][1]
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"], name="input", shape=input_shape)
+    diag_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"], name="diagonal", shape=diag_shape)
+    outs = tf.linalg.set_diag(input_tensor, diag_tensor)
+    return [input_tensor, diag_tensor], [outs]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_shape = parameters["input_diag_shapes"][0]
+    diag_shape = parameters["input_diag_shapes"][1]
+    input_values = create_tensor_data(parameters["input_dtype"], input_shape)
+    diag_values = create_tensor_data(parameters["input_dtype"], diag_shape)
+    return [input_values, diag_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values, diag_values])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/max_pool_with_argmax.py

@@ -0,0 +1,62 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for max_pool_with_argmax."""
+import tensorflow as tf
+
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import ExtraConvertOptions
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_max_pool_with_argmax_tests(options):
+  """Make a set of tests to do max_pool_with_argmax."""
+
+  test_parameters = [{
+      'input_size': [[2, 4, 2, 2], [2, 4, 3, 2]],
+      'pool_size': [(2, 2), (2, 1)],
+      'strides': [(2, 2)],
+      'padding': ['SAME', 'VALID'],
+  }, {
+      'input_size': [[2, 4, 10, 2], [2, 4, 11, 2], [2, 4, 12, 2]],
+      'pool_size': [(2, 2)],
+      'strides': [(2, 3)],
+      'padding': ['SAME', 'VALID'],
+  }]
+
+  def build_graph(parameters):
+    """Build the exp op testing graph."""
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=tf.float32, name='input', shape=parameters['input_size'])
+    updates, indices = tf.nn.max_pool_with_argmax(
+        input_tensor,
+        ksize=parameters['pool_size'],
+        strides=parameters['strides'],
+        padding=parameters['padding'],
+        output_dtype=tf.dtypes.int32)
+    return [input_tensor], [updates, indices]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    values = [
+        create_tensor_data(
+            tf.float32, parameters['input_size'], min_value=-10, max_value=10)
+    ]
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  extra_convert_options = ExtraConvertOptions()
+  extra_convert_options.allow_custom_ops = True
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs,
+                    extra_convert_options)

tensorflow/lite/testing/op_tests/maximum.py

@@ -0,0 +1,70 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for maximum."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_maximum_tests(options):
+  """Make a set of tests to do maximum."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32],
+      "input_shape_1": [[], [3], [1, 100], [4, 2, 3], [5, 224, 224, 3],
+                        [5, 32, 32, 3, 1], [5, 32, 32, 3, 1]],
+      "input_shape_2": [[], [3], [1, 100], [4, 2, 3], [5, 224, 224, 3],
+                        [5, 32, 32, 3, 3], [5, 32, 32, 3, 1]],
+      "fully_quantize": [False, True],
+  }]
+
+  def build_graph(parameters):
+    """Build the maximum op testing graph."""
+    input_tensor_1 = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input_1",
+        shape=parameters["input_shape_1"])
+    input_tensor_2 = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input_2",
+        shape=parameters["input_shape_2"])
+
+    out = tf.maximum(input_tensor_1, input_tensor_2)
+    return [input_tensor_1, input_tensor_2], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    """Builds the inputs for the model above."""
+    values = [
+        create_tensor_data(
+            parameters["input_dtype"],
+            parameters["input_shape_1"],
+            min_value=-1,
+            max_value=1),
+        create_tensor_data(
+            parameters["input_dtype"],
+            parameters["input_shape_2"],
+            min_value=-1,
+            max_value=1)
+    ]
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=44)

tensorflow/lite/testing/op_tests/minimum.py

@@ -0,0 +1,70 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for minimum."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_minimum_tests(options):
+  """Make a set of tests to do minimum."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32],
+      "input_shape_1": [[], [3], [1, 100], [4, 2, 3], [5, 224, 224, 3],
+                        [5, 32, 32, 1, 1], [5, 32, 32, 1, 1]],
+      "input_shape_2": [[], [3], [1, 100], [4, 2, 3], [5, 224, 224, 3],
+                        [5, 32, 32, 1, 1], [5, 32, 32, 1, 3]],
+      "fully_quantize": [False, True],
+  }]
+
+  def build_graph(parameters):
+    """Build the minimum op testing graph."""
+    input_tensor_1 = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input_1",
+        shape=parameters["input_shape_1"])
+    input_tensor_2 = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input_2",
+        shape=parameters["input_shape_2"])
+
+    out = tf.minimum(input_tensor_1, input_tensor_2)
+    return [input_tensor_1, input_tensor_2], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    """Builds the inputs for the model above."""
+    values = [
+        create_tensor_data(
+            parameters["input_dtype"],
+            parameters["input_shape_1"],
+            min_value=-1,
+            max_value=1),
+        create_tensor_data(
+            parameters["input_dtype"],
+            parameters["input_shape_2"],
+            min_value=-1,
+            max_value=1)
+    ]
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=44)

tensorflow/lite/testing/op_tests/mirror_pad.py

@@ -0,0 +1,119 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for mirror_pad."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_mirror_pad_tests(options):
+  """Make a set of tests to do mirror_pad."""
+
+  test_parameters = [
+      {
+          "input_shape": [[2, 3]],
+          "padding_matrix": [[[1, 1], [2, 1]]],
+          "mode": ["REFLECT"],
+          "type": ["const"],
+          "fully_quantize": [True, False],
+      },
+      {
+          "input_shape": [[2, 3]],
+          "padding_matrix": [[[1, 1], [1, 1]]],
+          "mode": ["REFLECT"],
+          "type": ["const"],
+          "fully_quantize": [False],
+      },
+      {
+          "input_shape": [[2, 3]],
+          "padding_matrix": [[[1, 1], [2, 1]]],
+          "mode": ["SYMMETRIC"],
+          "type": ["placeholder"],
+          "fully_quantize": [False],
+      },
+      {
+          "input_shape": [[2, 3]],
+          "padding_matrix": [[[1, 1], [2, 1]]],
+          "mode": ["REFLECT"],
+          "type": ["placeholder"],
+          "fully_quantize": [False],
+      },
+      {
+          "input_shape": [[3]],
+          "padding_matrix": [[[0, 2]]],
+          "mode": ["SYMMETRIC"],
+          "type": ["placeholder"],
+          "fully_quantize": [False],
+      },
+      {
+          "input_shape": [[3]],
+          "padding_matrix": [[[0, 2]]],
+          "mode": ["SYMMETRIC"],
+          "type": ["const"],
+          "fully_quantize": [False],
+      },
+      {
+          "input_shape": [[3]],
+          "padding_matrix": [[[0, 2]]],
+          "mode": ["REFLECT"],
+          "type": ["const"],
+          "fully_quantize": [False, True],
+          "quant_16x8": [False, True],
+      },
+      {
+          "input_shape": [[3, 2, 4, 5]],
+          "padding_matrix": [[[1, 1], [2, 2], [1, 1], [1, 1]]],
+          "mode": ["SYMMETRIC"],
+          "type": ["placeholder"],
+          "fully_quantize": [False],
+      },
+  ]
+
+  def build_graph(parameters):
+    """Build the graph for the test case."""
+
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="input", shape=parameters["input_shape"])
+    if parameters["type"] != "const" and not parameters["fully_quantize"]:
+      padding_matrix = tf.compat.v1.placeholder(
+          dtype=tf.int32,
+          name="padding",
+          shape=[len(parameters["input_shape"]), 2])
+      input_tensors = [input_tensor, padding_matrix]
+    else:
+      padding_matrix = tf.constant(np.array(parameters["padding_matrix"]))
+      input_tensors = [input_tensor]
+    output = tf.pad(
+        tensor=input_tensor, paddings=padding_matrix, mode=parameters["mode"])
+
+    return input_tensors, [output]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    if not parameters["fully_quantize"]:
+      input_values = [create_tensor_data(tf.float32, parameters["input_shape"])]
+    else:
+      input_values = [
+          create_tensor_data(
+              tf.float32, parameters["input_shape"], min_value=-1, max_value=1)
+      ]
+    if parameters["type"] != "const":
+      input_values.append(np.array(parameters["padding_matrix"]))
+    return input_values, sess.run(
+        outputs, feed_dict=dict(zip(inputs, input_values)))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/multinomial.py

@@ -0,0 +1,66 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for multinomial."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_multinomial_tests(options):
+  """Make a set of tests to do multinomial."""
+  test_parameters = [{
+      "logits_shape": [[1, 2], [2, 5]],
+      "dtype": [tf.int64, tf.int32],
+      "seed": [None, 1234],
+      "seed2": [5678],
+  }, {
+      "logits_shape": [[1, 2]],
+      "dtype": [tf.int64, tf.int32],
+      "seed": [1234],
+      "seed2": [None]
+  }]
+
+  def build_graph(parameters):
+    """Build the op testing graph."""
+    tf.compat.v1.set_random_seed(seed=parameters["seed"])
+    logits_tf = tf.compat.v1.placeholder(
+        name="logits", dtype=tf.float32, shape=parameters["logits_shape"])
+    num_samples_tf = tf.compat.v1.placeholder(
+        name="num_samples", dtype=tf.int32, shape=None)
+    out = tf.random.categorical(
+        logits=logits_tf,
+        num_samples=num_samples_tf,
+        dtype=parameters["dtype"],
+        seed=parameters["seed2"])
+    return [logits_tf, num_samples_tf], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_values = [
+        create_tensor_data(
+            dtype=tf.float32, shape=parameters["logits_shape"], min_value=-2,
+            max_value=-1),
+        create_tensor_data(
+            dtype=tf.int32, shape=None, min_value=10, max_value=100)
+    ]
+    return input_values, sess.run(
+        outputs, feed_dict=dict(zip(inputs, input_values)))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/nearest_upsample.py

@@ -0,0 +1,75 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for nearest upsample."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_nearest_upsample_tests(options):
+  """Make a set of tests to do nearest_upsample."""
+
+  # Chose a set of parameters
+  test_parameters = [{
+      "input_shape": [[1, 10, 10, 64], [3, 8, 32]],
+      "scale_n_axis": [([2, 2], [1, 2]), ([3, 4], [1, 2]), ([3], [1])],
+      "dtype": [tf.float32, tf.int32],
+  }]
+
+  def new_shape_for_upsample(original_shape, scales, axis):
+    """Calculate the input shape & ones shape, also the upsample shape."""
+    input_new_shape = []
+    ones_new_shape = []
+    upsample_new_shape = []
+    j = 0
+    for i in range(len(original_shape)):
+      input_new_shape.append(original_shape[i])
+      ones_new_shape.append(1)
+      if j < len(scales) and axis[j] == i:
+        input_new_shape.append(1)
+        ones_new_shape.append(scales[j])
+        upsample_new_shape.append(original_shape[i] * scales[j])
+        j += 1
+      else:
+        upsample_new_shape.append(original_shape[i])
+    return input_new_shape, ones_new_shape, upsample_new_shape
+
+  def build_graph(parameters):
+    """Build the nearest upsample testing graph."""
+    input_shape = parameters["input_shape"]
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"], name="input", shape=input_shape)
+    scales, axis = parameters["scale_n_axis"]
+    input_new_shape, ones_new_shape, new_shape = new_shape_for_upsample(
+        input_shape, scales, axis)
+
+    out = tf.compat.v1.reshape(input_tensor,
+                               input_new_shape) * tf.compat.v1.ones(
+                                   ones_new_shape, dtype=parameters["dtype"])
+    out = tf.compat.v1.reshape(out, new_shape)
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_values = create_tensor_data(
+        parameters["dtype"],
+        parameters["input_shape"],
+        min_value=-10,
+        max_value=10)
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/neg.py

@@ -0,0 +1,45 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for neg."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_neg_tests(options):
+  """Make a set of tests to do neg."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32, tf.int32],
+      "input_shape": [[1, 3, 4, 3], [5], []],
+  }]
+
+  def build_graph(parameters):
+    """Build the neg op testing graph."""
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+    out = tf.negative(input_tensor)
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    values = create_tensor_data(parameters["input_dtype"],
+                                parameters["input_shape"])
+    return [values], sess.run(outputs, feed_dict=dict(zip(inputs, [values])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/not_equal.py

@@ -0,0 +1,59 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for not_equal."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_not_equal_tests(options):
+  """Make a set of tests to do not equal."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32, tf.int32, tf.int64, tf.string],
+      "input_shape_pair": [([1, 1, 1, 3], [1, 1, 1, 3]),
+                           ([2, 3, 4, 5], [2, 3, 4, 5]), ([2, 3, 3], [2, 3]),
+                           ([5, 5], [1]), ([10], [2, 4, 10])],
+  }]
+
+  def build_graph(parameters):
+    """Build the not equal op testing graph."""
+    input_value1 = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input1",
+        shape=parameters["input_shape_pair"][0])
+    input_value2 = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input2",
+        shape=parameters["input_shape_pair"][1])
+    out = tf.not_equal(input_value1, input_value2)
+    return [input_value1, input_value2], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value1 = create_tensor_data(parameters["input_dtype"],
+                                      parameters["input_shape_pair"][0])
+    input_value2 = create_tensor_data(parameters["input_dtype"],
+                                      parameters["input_shape_pair"][1])
+    return [input_value1, input_value2], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_value1, input_value2])))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=4)

tensorflow/lite/testing/op_tests/one_hot.py

@@ -0,0 +1,81 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for one_hot."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_one_hot_tests(options):
+  """Make a set of tests to do one_hot."""
+
+  test_parameters = [{
+      "indices_type": [tf.int32, tf.int64],
+      "indices_shape": [[3], [4, 4], [1, 5], [5, 1]],
+      "axis": [0, 1],
+      "dtype": [tf.int32, tf.int64, tf.float32],
+      "provide_optional_inputs": [True, False],
+  }]
+
+  def build_graph(parameters):
+    """Build the one_hot op testing graph."""
+    indices = tf.compat.v1.placeholder(
+        dtype=parameters["indices_type"],
+        name="indices",
+        shape=parameters["indices_shape"])
+    depth = tf.compat.v1.placeholder(dtype=tf.int32, name="depth", shape=())
+
+    if not parameters["provide_optional_inputs"]:
+      out = tf.one_hot(indices=indices, depth=depth)
+      return [indices, depth], [out]
+
+    on_value = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"], name="on_value", shape=())
+    off_value = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"], name="off_value", shape=())
+    out = tf.one_hot(
+        indices=indices,
+        depth=depth,
+        on_value=on_value,
+        off_value=off_value,
+        axis=parameters["axis"],
+        dtype=parameters["dtype"])
+    return [indices, depth, on_value, off_value], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    """Build the input for one_hot op."""
+    input_values = [
+        create_tensor_data(
+            parameters["indices_type"],
+            shape=parameters["indices_shape"],
+            min_value=-1,
+            max_value=10),
+        create_tensor_data(tf.int32, shape=None, min_value=1, max_value=10),
+    ]
+
+    if parameters["provide_optional_inputs"]:
+      input_values.append(
+          create_tensor_data(
+              parameters["dtype"], shape=None, min_value=1, max_value=10))
+      input_values.append(
+          create_tensor_data(
+              parameters["dtype"], shape=None, min_value=-1, max_value=0))
+
+    return input_values, sess.run(
+        outputs, feed_dict=dict(zip(inputs, input_values)))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/pack.py

@@ -0,0 +1,96 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for pack."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_pack_tests(options):
+  """Make a set of tests to do stack."""
+
+  test_parameters = [
+      # Avoid creating all combinations to keep the test size small.
+      {
+          "dtype": [tf.float32],
+          "base_shape": [[3, 4, 3], [3, 4], [5]],
+          "num_tensors": [1, 2, 3, 4, 5, 6],
+          "axis": [0, 1, 2, 3],
+          "additional_shape": [1, 2, 3],
+          "fully_quantize": [False],
+      },
+      {
+          "dtype": [tf.int32],
+          "base_shape": [[3, 4, 3], [3, 4], [5]],
+          "num_tensors": [6],
+          "axis": [0, 1, 2, 3],
+          "additional_shape": [1, 2, 3],
+          "fully_quantize": [False],
+      },
+      {
+          "dtype": [tf.int64],
+          "base_shape": [[3, 4, 3], [3, 4], [5]],
+          "num_tensors": [5],
+          "axis": [0, 1, 2, 3],
+          "additional_shape": [1, 2, 3],
+          "fully_quantize": [False],
+      },
+      {
+          "dtype": [tf.float32],
+          "base_shape": [[1, 4, 3], [3, 4], [5]],
+          "num_tensors": [2, 3, 4, 5, 6],  # 1 tensor would go to Reshape.
+          "axis": [0, 1, 2, 3],
+          "additional_shape": [1, 2, 3],
+          "fully_quantize": [True],
+      },
+  ]
+
+  def get_shape(parameters):
+    """Return a tweaked version of 'base_shape'."""
+    axis = parameters["axis"]
+    shape = parameters["base_shape"][:]
+    if axis < len(shape):
+      shape[axis] += parameters["additional_shape"]
+    return shape
+
+  def build_graph(parameters):
+    all_tensors = []
+    for n in range(0, parameters["num_tensors"]):
+      input_tensor = tf.compat.v1.placeholder(
+          dtype=parameters["dtype"],
+          name=("input%d" % n),
+          shape=get_shape(parameters))
+      all_tensors.append(input_tensor)
+    out = tf.stack(all_tensors, parameters["axis"])
+    return all_tensors, [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    all_values = []
+    for _ in range(0, parameters["num_tensors"]):
+      input_values = create_tensor_data(
+          np.float32, get_shape(parameters), min_value=-1, max_value=1)
+      all_values.append(input_values)
+    return all_values, sess.run(
+        outputs, feed_dict=dict(zip(inputs, all_values)))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=117)

tensorflow/lite/testing/op_tests/pad.py

@@ -0,0 +1,146 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for pad."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_pad_tests(options):
+  """Make a set of tests to do pad."""
+
+  # TODO(nupurgarg): Add test for tf.uint8.
+  test_parameters = [
+      # 5D:
+      {
+          "dtype": [tf.int32, tf.int64, tf.float32, tf.bool],
+          "input_shape": [[1, 1, 2, 1, 1], [2, 1, 1, 1, 1]],
+          "padding_dtype": [tf.int32, tf.int64],
+          "paddings": [
+              [[0, 0], [0, 1], [2, 3], [0, 0], [1, 0]],
+              [[0, 1], [0, 0], [0, 0], [2, 3], [1, 0]],
+          ],
+          "constant_paddings": [True, False],
+          "fully_quantize": [False],
+          "quant_16x8": [False],
+      },
+      # 4D:
+      {
+          "dtype": [tf.int32, tf.int64, tf.float32, tf.bool],
+          "input_shape": [[1, 1, 2, 1], [2, 1, 1, 1]],
+          "padding_dtype": [tf.int32, tf.int64],
+          "paddings": [
+              [[0, 0], [0, 1], [2, 3], [0, 0]],
+              [[0, 1], [0, 0], [0, 0], [2, 3]],
+          ],
+          "constant_paddings": [True, False],
+          "fully_quantize": [False],
+          "quant_16x8": [False],
+      },
+      # 2D:
+      {
+          "dtype": [tf.int32, tf.int64, tf.float32, tf.bool],
+          "input_shape": [[1, 2]],
+          "padding_dtype": [tf.int32, tf.int64],
+          "paddings": [[[0, 1], [2, 3]]],
+          "constant_paddings": [True, False],
+          "fully_quantize": [False],
+          "quant_16x8": [False],
+      },
+      # 1D:
+      {
+          "dtype": [tf.int32, tf.bool],
+          "input_shape": [[1]],
+          "padding_dtype": [tf.int32, tf.int64],
+          "paddings": [[[1, 2]]],
+          "constant_paddings": [False],
+          "fully_quantize": [False],
+          "quant_16x8": [False],
+      },
+      # 4D:
+      {
+          "dtype": [tf.float32, tf.bool],
+          "input_shape": [[1, 1, 2, 1], [2, 1, 1, 1]],
+          "padding_dtype": [tf.int32, tf.int64],
+          "paddings": [
+              [[0, 0], [0, 1], [2, 3], [0, 0]],
+              [[0, 1], [0, 0], [0, 0], [2, 3]],
+              [[0, 0], [0, 0], [0, 0], [0, 0]],
+          ],
+          "constant_paddings": [True],
+          "fully_quantize": [True],
+          "quant_16x8": [False, True],
+      },
+      # 2D:
+      {
+          "dtype": [tf.float32, tf.bool],
+          "input_shape": [[1, 2]],
+          "padding_dtype": [tf.int32, tf.int64],
+          "paddings": [[[0, 1], [2, 3]]],
+          "constant_paddings": [True],
+          "fully_quantize": [True],
+          "quant_16x8": [False, True],
+      },
+      # 1D:
+      {
+          "dtype": [tf.float32, tf.bool],
+          "input_shape": [[1]],
+          "padding_dtype": [tf.int32, tf.int64],
+          "paddings": [[[1, 2]]],
+          "constant_paddings": [True],
+          "fully_quantize": [True],
+          "quant_16x8": [False, True],
+      },
+  ]
+
+  def build_graph(parameters):
+    """Build a pad graph given `parameters`."""
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+
+    # Get paddings as either a placeholder or constants.
+    if parameters["constant_paddings"]:
+      paddings = parameters["paddings"]
+      input_tensors = [input_tensor]
+    else:
+      shape = [len(parameters["paddings"]), 2]
+      paddings = tf.compat.v1.placeholder(
+          dtype=parameters["padding_dtype"], name="padding", shape=shape
+      )
+      input_tensors = [input_tensor, paddings]
+
+    out = tf.pad(tensor=input_tensor, paddings=paddings)
+    return input_tensors, [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    """Build inputs for pad op."""
+
+    values = [
+        create_tensor_data(
+            parameters["dtype"],
+            parameters["input_shape"],
+            min_value=-1,
+            max_value=1)
+    ]
+    if not parameters["constant_paddings"]:
+      values.append(np.array(parameters["paddings"]))
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/padv2.py

@@ -0,0 +1,96 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for padv2."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_padv2_tests(options):
+  """Make a set of tests to do padv2."""
+
+  # TODO(nupurgarg): Add test for tf.uint8.
+  test_parameters = [
+      # 5D:
+      {
+          "dtype": [tf.int32, tf.int64, tf.float32],
+          "input_shape": [[1, 1, 2, 1, 1], [2, 1, 1, 1, 1]],
+          "paddings": [[[0, 0], [0, 1], [2, 3], [0, 0], [1, 0]],
+                       [[0, 1], [0, 0], [0, 0], [2, 3], [1, 0]]],
+          "constant_paddings": [True, False],
+          "constant_values": [0, 2],
+      },
+      # 4D:
+      {
+          "dtype": [tf.int32, tf.int64, tf.float32],
+          "input_shape": [[1, 1, 2, 1], [2, 1, 1, 1]],
+          "paddings": [[[0, 0], [0, 1], [2, 3], [0, 0]],
+                       [[0, 1], [0, 0], [0, 0], [2, 3]]],
+          "constant_paddings": [True, False],
+          "constant_values": [0, 2],
+      },
+      # 2D:
+      {
+          "dtype": [tf.int32, tf.int64, tf.float32],
+          "input_shape": [[1, 2]],
+          "paddings": [[[0, 1], [2, 3]]],
+          "constant_paddings": [True, False],
+          "constant_values": [0, 2],
+      },
+      # 1D:
+      {
+          "dtype": [tf.int32],
+          "input_shape": [[1]],
+          "paddings": [[[0, 1]]],
+          "constant_paddings": [False],
+          "constant_values": [0, 2],
+      },
+  ]
+
+  def build_graph(parameters):
+    """Build a pad graph given `parameters`."""
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+
+    # Get paddings as either a placeholder or constants.
+    if parameters["constant_paddings"]:
+      paddings = parameters["paddings"]
+      input_tensors = [input_tensor]
+    else:
+      shape = [len(parameters["paddings"]), 2]
+      paddings = tf.compat.v1.placeholder(
+          dtype=tf.int32, name="padding", shape=shape)
+      input_tensors = [input_tensor, paddings]
+
+    out = tf.pad(
+        tensor=input_tensor,
+        paddings=paddings,
+        constant_values=parameters["constant_values"])
+    return input_tensors, [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    values = [
+        create_tensor_data(parameters["dtype"], parameters["input_shape"])
+    ]
+    if not parameters["constant_paddings"]:
+      values.append(np.array(parameters["paddings"]))
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/parse_example.py

@@ -0,0 +1,93 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for parse example."""
+import string
+
+import numpy as np
+import tensorflow as tf
+
+from tensorflow.lite.testing.zip_test_utils import ExtraConvertOptions
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+def create_example_data(feature_dtype, feature_shape):
+  """Create structured example data."""
+  features = {}
+  if feature_dtype in (tf.float32, tf.float16, tf.float64):
+    data = np.random.rand(*feature_shape)
+    features["x"] = tf.train.Feature(
+        float_list=tf.train.FloatList(value=list(data)))
+  elif feature_dtype in (tf.int32, tf.uint8, tf.int64, tf.int16):
+    data = np.random.randint(-100, 100, size=feature_shape)
+    features["x"] = tf.train.Feature(
+        int64_list=tf.train.Int64List(value=list(data)))
+  elif feature_dtype == tf.string:
+    letters = list(string.ascii_uppercase)
+    data = "".join(np.random.choice(letters, size=10)).encode("utf-8")
+    features["x"] = tf.train.Feature(
+        bytes_list=tf.train.BytesList(value=[data]*feature_shape[0]))
+  example = tf.train.Example(features=tf.train.Features(feature=features))
+  return np.array([example.SerializeToString()])
+
+
+@register_make_test_function("make_parse_example_tests")
+def make_parse_example_tests(options):
+  """Make a set of tests to use parse_example."""
+
+  # Chose a set of parameters
+  test_parameters = [{
+      "feature_dtype": [tf.string, tf.float32, tf.int64],
+      "is_dense": [True, False],
+      "feature_shape": [[1], [2], [16]],
+  }]
+
+  def build_graph(parameters):
+    """Build the graph for parse_example tests."""
+    feature_dtype = parameters["feature_dtype"]
+    feature_shape = parameters["feature_shape"]
+    is_dense = parameters["is_dense"]
+    input_value = tf.compat.v1.placeholder(
+        dtype=tf.string, name="input", shape=[1])
+    if is_dense:
+      feature_default_value = np.zeros(shape=feature_shape)
+      if feature_dtype == tf.string:
+        feature_default_value = np.array(["missing"] * feature_shape[0])
+      features = {
+          "x":
+              tf.io.FixedLenFeature(
+                  shape=feature_shape,
+                  dtype=feature_dtype,
+                  default_value=feature_default_value)
+      }
+    else:  # Sparse
+      features = {"x": tf.io.VarLenFeature(dtype=feature_dtype)}
+    out = tf.io.parse_example(serialized=input_value, features=features)
+    output_tensor = out["x"]
+    if not is_dense:
+      output_tensor = out["x"].values
+    return [input_value], [output_tensor]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    feature_dtype = parameters["feature_dtype"]
+    feature_shape = parameters["feature_shape"]
+    input_values = [create_example_data(feature_dtype, feature_shape)]
+    return input_values, sess.run(
+        outputs, feed_dict=dict(zip(inputs, input_values)))
+
+  extra_convert_options = ExtraConvertOptions()
+  extra_convert_options.allow_custom_ops = True
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs,
+                    extra_convert_options)

tensorflow/lite/testing/op_tests/placeholder_with_default.py

@@ -0,0 +1,49 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for placeholder_with_default."""
+import numpy as np
+import tensorflow as tf
+
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import MAP_TF_TO_NUMPY_TYPE
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_placeholder_with_default_tests(options):
+  """Make a set of tests to test placeholder_with_default."""
+
+  test_parameters = [{
+      "dtype": [tf.float32, tf.int32, tf.int64],
+  }]
+
+  def build_graph(parameters):
+    """Build the placeholder_with_default testing graph."""
+    const_node = tf.constant([1, 2, 2, 0],
+                             shape=[2, 2],
+                             dtype=parameters["dtype"])
+    input_tensor = tf.compat.v1.placeholder_with_default(
+        const_node, shape=[2, 2], name="input")
+    out = tf.equal(input_tensor, const_node, name="output")
+
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    numpy_type = MAP_TF_TO_NUMPY_TYPE[parameters["dtype"]]
+    input_value = np.array([[1, 0], [2, 1]], numpy_type)
+    return [input_value], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_value])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/pool.py

@@ -0,0 +1,142 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for pool operators."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+def make_pool_tests(pool_op_in, allow_fully_quantize=False):
+  """Make a set of tests to do average pooling.
+
+  Args:
+    pool_op_in: TensorFlow pooling operation to test  i.e. `tf.nn.avg_pool2d`.
+    allow_fully_quantize: bool, whether fully_quantize is allowed.
+
+  Returns:
+    A function representing the true generator (after curried pool_op_in).
+  """
+
+  pool_op = pool_op_in
+
+  def f(options, expected_tf_failures=0):
+    """Actual function that generates examples.
+
+    Args:
+      options: An Options instance.
+      expected_tf_failures: number of expected tensorflow failures.
+    """
+
+    # Chose a set of parameters
+    test_parameters = [
+        {
+            "ksize": [[2, 1, 1, 2], [1, 1, 1, 1], [1, 1, 2, 1], [1, 10, 11, 1]],
+            "strides": [[2, 1, 1, 2], [1, 1, 1, 1], [1, 1, 2, 1],
+                        [1, 10, 11, 1]],
+            # TODO(aselle): should add a degenerate shape (e.g. [1, 0, 1, 1]).
+            "input_shape": [[], [1, 1, 1, 1], [1, 15, 14, 1], [3, 15, 14, 3]],
+            "padding": ["SAME", "VALID"],
+            "data_format": ["NHWC"],  # TODO(aselle): NCHW  would be good
+            "fully_quantize": [False],
+            "quant_16x8": [False]
+        },
+        {
+            "ksize": [[2, 1, 1, 2], [1, 1, 1, 1], [1, 1, 2, 1], [1, 10, 11, 1]],
+            "strides": [[2, 1, 1, 2], [1, 1, 1, 1], [1, 1, 2, 1],
+                        [1, 10, 11, 1]],
+            # TODO(aselle): should add a degenerate shape (e.g. [1, 0, 1, 1]).
+            "input_shape": [[], [1, 1, 1, 1], [1, 15, 14, 1], [3, 15, 14, 3]],
+            "padding": ["SAME", "VALID"],
+            "data_format": ["NHWC"],  # TODO(aselle): NCHW  would be good
+            "fully_quantize": [True],
+            "quant_16x8": [False]
+        },
+        {
+            "ksize": [[1, 1, 1, 1]],
+            "strides": [[1, 1, 1, 1]],
+            "input_shape": [[1, 1, 1, 1]],
+            "padding": ["SAME", "VALID"],
+            "data_format": ["NHWC"],
+            "fully_quantize": [True],
+            "quant_16x8": [True]
+        }
+    ]
+    # test_parameters include fully_quantize option only when
+    # allow_fully_quantize is True.
+    if not allow_fully_quantize:
+      test_parameters = [
+          test_parameter for test_parameter in test_parameters
+          if True not in test_parameter["fully_quantize"]
+      ]
+
+    def build_graph(parameters):
+      input_tensor = tf.compat.v1.placeholder(
+          dtype=tf.float32, name="input", shape=parameters["input_shape"])
+      out = pool_op(
+          input_tensor,
+          ksize=parameters["ksize"],
+          strides=parameters["strides"],
+          data_format=parameters["data_format"],
+          padding=parameters["padding"])
+      return [input_tensor], [out]
+
+    def build_inputs(parameters, sess, inputs, outputs):
+      if allow_fully_quantize:
+        input_values = create_tensor_data(
+            tf.float32, parameters["input_shape"], min_value=-1, max_value=1)
+      else:
+        input_values = create_tensor_data(tf.float32, parameters["input_shape"])
+      return [input_values], sess.run(
+          outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+    make_zip_of_tests(
+        options,
+        test_parameters,
+        build_graph,
+        build_inputs,
+        expected_tf_failures=expected_tf_failures)
+
+  return f
+
+
+def make_l2_pool(input_tensor, ksize, strides, padding, data_format):
+  """Given an input perform a sequence of TensorFlow ops to produce l2pool."""
+  return tf.sqrt(
+      tf.nn.avg_pool2d(
+          input=tf.square(input_tensor),
+          ksize=ksize,
+          strides=strides,
+          padding=padding,
+          data_format=data_format))
+
+
+@register_make_test_function()
+def make_l2_pool_tests(options):
+  make_pool_tests(make_l2_pool)(options, expected_tf_failures=80)
+
+
+@register_make_test_function()
+def make_avg_pool_tests(options):
+  make_pool_tests(
+      tf.nn.avg_pool2d, allow_fully_quantize=True)(
+          options, expected_tf_failures=160)
+
+
+@register_make_test_function()
+def make_max_pool_tests(options):
+  make_pool_tests(
+      tf.nn.max_pool2d, allow_fully_quantize=True)(
+          options, expected_tf_failures=160)

tensorflow/lite/testing/op_tests/pool3d.py

@@ -0,0 +1,89 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for pool operators."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import ExtraConvertOptions
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+def make_pool3d_tests(pool_op):
+  """Make a set of tests to do pooling.
+
+  Args:
+    pool_op: TensorFlow pooling operation to test  i.e. `tf.nn.max_pool3d`.
+
+  Returns:
+    A function representing the true generator (after curried pool_op).
+  """
+
+  def f(options, expected_tf_failures=0):
+    """Actual function that generates examples.
+
+    Args:
+      options: An Options instance.
+      expected_tf_failures: number of expected tensorflow failures.
+    """
+
+    # Chose a set of parameters
+    test_parameters = [
+        {
+            "ksize": [[1, 1, 1, 1, 1], [1, 2, 2, 2, 1], [1, 2, 3, 4, 1]],
+            "strides": [[1, 1, 1, 1, 1], [1, 2, 1, 2, 1], [1, 2, 2, 4, 1]],
+            "input_shape": [[1, 1, 1, 1, 1], [1, 16, 15, 14, 1],
+                            [3, 16, 15, 14, 3]],
+            "padding": ["SAME", "VALID"],
+            "data_format": ["NDHWC"],
+        },
+    ]
+
+    def build_graph(parameters):
+      input_tensor = tf.compat.v1.placeholder(
+          dtype=tf.float32, name="input", shape=parameters["input_shape"])
+      out = pool_op(
+          input_tensor,
+          ksize=parameters["ksize"],
+          strides=parameters["strides"],
+          data_format=parameters["data_format"],
+          padding=parameters["padding"])
+      return [input_tensor], [out]
+
+    def build_inputs(parameters, sess, inputs, outputs):
+      input_values = create_tensor_data(tf.float32, parameters["input_shape"])
+      return [input_values], sess.run(
+          outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+    extra_convert_options = ExtraConvertOptions()
+    extra_convert_options.allow_custom_ops = True
+    make_zip_of_tests(
+        options,
+        test_parameters,
+        build_graph,
+        build_inputs,
+        extra_convert_options,
+        expected_tf_failures=expected_tf_failures)
+
+  return f
+
+
+@register_make_test_function()
+def make_avg_pool3d_tests(options):
+  make_pool3d_tests(tf.nn.avg_pool3d)(options, expected_tf_failures=6)
+
+
+@register_make_test_function()
+def make_max_pool3d_tests(options):
+  make_pool3d_tests(tf.nn.max_pool3d)(options, expected_tf_failures=6)

tensorflow/lite/testing/op_tests/prelu.py

@@ -0,0 +1,100 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for prelu."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_prelu_tests(options):
+  """Make a set of tests to do PReLU."""
+
+  test_parameters = [
+      {
+          # The canonical case for image processing is having a 4D `input`
+          # (NHWC)and `shared_axes`=[1, 2], so the alpha parameter is per
+          # channel.
+          "input_shape": [[1, 10, 10, 3], [3, 3, 3, 3]],
+          "shared_axes": [[1, 2], [1]],
+          "fully_quantize": [False],
+          "input_range": [(-10, 10)],
+      },
+      {
+          # 2D-3D example. Share the 2nd axis.
+          "input_shape": [[20, 20], [20, 20, 20]],
+          "shared_axes": [[1]],
+          "fully_quantize": [False],
+          "input_range": [(-10, 10)],
+      },
+      # Quantized cases.
+      {
+          # The canonical case for image processing is having a 4D `input`
+          # (NHWC)and `shared_axes`=[1, 2], so the alpha parameter is per
+          # channel.
+          "input_shape": [[1, 10, 10, 3], [3, 3, 3, 3]],
+          "shared_axes": [[1, 2], [1]],
+          "fully_quantize": [True],
+          "input_range": [(-10, 10)],
+      },
+      {
+          # 2D-3D example. Share the 2nd axis.
+          "input_shape": [[20, 20], [20, 20, 20]],
+          "shared_axes": [[1]],
+          "fully_quantize": [True],
+          "input_range": [(-10, 10)],
+      },
+  ]
+
+  def build_graph(parameters):
+    """Build the graph for the test case."""
+
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="input", shape=parameters["input_shape"])
+    prelu = tf.keras.layers.PReLU(shared_axes=parameters["shared_axes"])
+    out = prelu(input_tensor)
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    """Build the inputs for the test case."""
+
+    input_shape = parameters["input_shape"]
+    input_values = create_tensor_data(
+        np.float32, input_shape, min_value=-10, max_value=10)
+    shared_axes = parameters["shared_axes"]
+
+    alpha_shape = []
+    for dim in range(1, len(input_shape)):
+      alpha_shape.append(1 if dim in shared_axes else input_shape[dim])
+
+    alpha_values = create_tensor_data(
+        np.float32, alpha_shape, min_value=-5, max_value=5)
+
+    # There should be only 1 trainable variable tensor.
+    variables = tf.compat.v1.all_variables()
+    assert len(variables) == 1
+    sess.run(variables[0].assign(alpha_values))
+
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      use_frozen_graph=True)

tensorflow/lite/testing/op_tests/random_standard_normal.py

@@ -0,0 +1,66 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for random_standard_normal."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_random_standard_normal_tests(options):
+  """Make a set of tests to do random_standard_normal."""
+
+  test_parameters = [{
+      "input_shape": [[1]],
+      "input_dtype": [tf.int32],
+      "shape": [[10]],
+      "seed": [None, 0, 1234],
+      "seed2": [0, 5678],
+      "dtype": [tf.float32],
+  }, {
+      "input_shape": [[3]],
+      "input_dtype": [tf.int32],
+      "shape": [[2, 3, 4]],
+      "seed": [0, 1234],
+      "seed2": [None, 0, 5678],
+      "dtype": [tf.float32],
+  }]
+
+  def build_graph(parameters):
+    """Build the op testing graph."""
+    tf.compat.v1.set_random_seed(seed=parameters["seed"])
+    input_value = tf.compat.v1.placeholder(
+        name="shape",
+        shape=parameters["input_shape"],
+        dtype=parameters["input_dtype"])
+    out = tf.random.normal(
+        shape=input_value, dtype=parameters["dtype"], seed=parameters["seed2"])
+    return [input_value], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value = create_tensor_data(
+        parameters["input_dtype"],
+        parameters["input_shape"],
+        min_value=1,
+        max_value=10)
+    return [input_value], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_value])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/random_uniform.py

@@ -0,0 +1,66 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for random_uniform."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_random_uniform_tests(options):
+  """Make a set of tests to do random_uniform."""
+
+  test_parameters = [{
+      "input_shape": [[1]],
+      "input_dtype": [tf.int32],
+      "shape": [[10]],
+      "seed": [None, 0, 1234],
+      "seed2": [0, 5678],
+      "dtype": [tf.float32],
+  }, {
+      "input_shape": [[3]],
+      "input_dtype": [tf.int32],
+      "shape": [[2, 3, 4]],
+      "seed": [0, 1234],
+      "seed2": [None, 0, 5678],
+      "dtype": [tf.float32],
+  }]
+
+  def build_graph(parameters):
+    """Build the op testing graph."""
+    tf.compat.v1.set_random_seed(seed=parameters["seed"])
+    input_value = tf.compat.v1.placeholder(
+        name="shape",
+        shape=parameters["input_shape"],
+        dtype=parameters["input_dtype"])
+    out = tf.random.uniform(
+        shape=input_value, dtype=parameters["dtype"], seed=parameters["seed2"])
+    return [input_value], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value = create_tensor_data(
+        parameters["input_dtype"],
+        parameters["input_shape"],
+        min_value=1,
+        max_value=10)
+    return [input_value], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_value])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/range.py

@@ -0,0 +1,51 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for range."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_scalar_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_range_tests(options):
+  """Make a set of tests to do range."""
+
+  test_parameters = [{
+      "dtype": [tf.int32, tf.float32, tf.int64],
+      "offset": [10, 100, 1000, 0],
+      "delta": [1, 2, 3, 4, -1, -2, -3, -4],
+  }]
+
+  def build_graph(parameters):
+    """Build the range op testing graph."""
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"], name=("start"), shape=[])
+    if parameters["delta"] < 0:
+      offset = parameters["offset"] * -1
+    else:
+      offset = parameters["offset"]
+    delta = parameters["delta"]
+    limit_tensor = input_tensor + offset
+    delta_tensor = tf.constant(delta, dtype=parameters["dtype"])
+    out = tf.range(input_tensor, limit_tensor, delta_tensor)
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value = create_scalar_data(parameters["dtype"])
+    return [input_value], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_value])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/rank.py

@@ -0,0 +1,44 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for rank."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_rank_tests(options):
+  """Make a set of tests to do rank."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32, tf.int32],
+      "input_shape": [[], [0], [1, 1, 1, 3], [2, 3, 4, 5], [5, 5], [10]],
+  }]
+
+  def build_graph(parameters):
+    """Build the rank op testing graph."""
+    input_value = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"], name="input")
+    out = tf.rank(input_value)
+    return [input_value], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value = create_tensor_data(parameters["input_dtype"],
+                                     parameters["input_shape"])
+    return [input_value], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_value])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/real.py

@@ -0,0 +1,49 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for real op."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_real_tests(options):
+  """Make a set of tests to do real op."""
+
+  # Chose a set of parameters
+  test_parameters = [{
+      "dtype": [tf.complex64, tf.complex128],
+      "input_shape": [[], [1], [2, 3], [1, 3, 4, 3], [2, 2, 3, 4, 5, 6]],
+  }]
+
+  def build_graph(parameters):
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+    out = tf.math.real(input_tensor)
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_values = create_tensor_data(
+        parameters["dtype"].as_numpy_dtype,
+        parameters["input_shape"],
+        min_value=-10,
+        max_value=10)
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/reciprocal.py

@@ -0,0 +1,54 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for reciprocal."""
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function("make_reciprocal_tests")
+def make_reciprocal_tests(options):
+  """Make a set of tests to do reciprocal."""
+
+  # Chose a set of parameters
+  test_parameters = [{
+      "input_dtype": [tf.float32, tf.int32, tf.int64],
+      "input_shape": [[1, 2], [1, 2, 3, 4], [10]],
+  }]
+
+  def build_graph(parameters):
+    """Build the graph for cond tests."""
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input",
+        shape=parameters["input_shape"])
+
+    out = tf.math.reciprocal(input_tensor)
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_values = [
+        create_tensor_data(parameters["input_dtype"], parameters["input_shape"])
+    ]
+    return input_values, sess.run(
+        outputs, feed_dict=dict(zip(inputs, input_values)))
+
+  make_zip_of_tests(
+      options,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_failures=6)

tensorflow/lite/testing/op_tests/reduce.py

@@ -0,0 +1,278 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for reduce operators."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+def make_reduce_tests(reduce_op,
+                      min_value=-10,
+                      max_value=10,
+                      boolean_tensor_only=False,
+                      allow_fully_quantize=False):
+  """Make a set of tests to do reduce operation.
+
+  Args:
+    reduce_op: TensorFlow reduce operation to test, i.e. `tf.reduce_mean`.
+    min_value: min value for created tensor data.
+    max_value: max value for created tensor data.
+    boolean_tensor_only: If true, will only generate tensor with boolean value.
+    allow_fully_quantize: bool, whether fully_quantize is allowed.
+
+  Returns:
+    a function representing the true generator with `reduce_op_in` curried.
+  """
+
+  def f(options):
+    """Actual function that generates examples."""
+
+    test_parameters = [
+        {
+            "input_dtype": [tf.float32, tf.int32, tf.int64],
+            "input_shape": [[3, 3, 2, 4]],
+            "axis": [
+                0,
+                1,
+                2,
+                [0, 1],
+                [0, 2],
+                [1, 2],
+                [0, 1, 2],
+                [1, 0],
+                [2, 0],
+                [2, 1],
+                [2, 1, 0],
+                [2, 0, 1],
+                -1,
+                -2,
+                -3,
+                [1, -1],
+                [0, -1],
+                [-1, 0],
+                [-1, -2, -3],
+            ],
+            "const_axis": [True, False],
+            "keepdims": [True, False],
+            "fully_quantize": [False],
+        },
+        {
+            "input_dtype": [tf.float32],
+            "input_shape": [[1, 8, 8, 3]],
+            "axis": [
+                0,
+                1,
+                2,
+                3,
+                [1, 2],
+                [0, 3],
+                [1, 2, 3],
+                [0, 1, 2, 3],
+                [3, 2, 1, 0],
+                [3, 1, 0, 2],
+                [2, 0],
+                [3, 0],
+                [3, 1],
+                [1, 0],
+                -1,
+                -2,
+                -3,
+                -4,
+                [0, -2],
+                [2, 3, 1, 0],
+                [3, 1, 2],
+                [3, -4],
+            ],
+            "const_axis": [True, False],
+            "keepdims": [True, False],
+            "fully_quantize": [False],
+        },
+        {
+            "input_dtype": [tf.float32],
+            "input_shape": [[], [1, 8, 8, 3], [3, 2, 4]],
+            "axis": [[]],  # shape is: [0]
+            "const_axis": [False],
+            "keepdims": [True, False],
+            "fully_quantize": [False],
+        },
+        {
+            "input_dtype": [tf.float32],
+            "input_shape": [[], [1, 8, 8, 3], [3, 2, 4]],
+            "axis": [None],  # shape is: []
+            "const_axis": [True],
+            "keepdims": [True, False],
+            "fully_quantize": [False],
+        },
+        {
+            "input_dtype": [tf.float32],
+            "input_shape": [[3, 3, 2, 4]],
+            "axis": [
+                0,
+                1,
+                2,
+                [0, 1],
+                [0, 2],
+                [1, 2],
+                [0, 1, 2],
+                [1, 0],
+                [2, 0],
+                [2, 1],
+                [2, 1, 0],
+                [2, 0, 1],
+                -1,
+                -2,
+                -3,
+                [1, -1],
+                [0, -1],
+                [-1, 0],
+                [-1, -2, -3],
+            ],
+            "const_axis": [True],
+            "keepdims": [True, False],
+            "fully_quantize": [True],
+        },
+        {
+            "input_dtype": [tf.float32],
+            "input_shape": [[1, 8, 8, 4], [1, 8, 8, 3]],
+            "axis": [
+                0, 1, 2, 3, [0], [1], [2], [3], [-1], [-2], [-3], [1, 2],
+                [0, 3], [1, 2, 3], [1, 3], [2, 3]
+            ],
+            "const_axis": [True],
+            "keepdims": [True, False],
+            "fully_quantize": [True],
+        },
+        {
+            "input_dtype": [tf.float32, tf.int32],
+            "input_shape": [[2, 0, 2], [0]],
+            "axis": [0],
+            "const_axis": [True],
+            "keepdims": [True, False],
+            "fully_quantize": [False],
+        },
+    ]
+    # test_parameters include fully_quantize option only when
+    # allow_fully_quantize is True.
+    if not allow_fully_quantize:
+      test_parameters = [
+          test_parameter for test_parameter in test_parameters
+          if True not in test_parameter["fully_quantize"]
+      ]
+
+    def build_graph(parameters):
+      """Build the mean op testing graph."""
+      dtype = parameters["input_dtype"]
+      if boolean_tensor_only:
+        dtype = tf.bool
+      input_tensor = tf.compat.v1.placeholder(
+          dtype=dtype, name="input", shape=parameters["input_shape"])
+
+      # Get axis as either a placeholder or constants.
+      if parameters["const_axis"]:
+        axis = parameters["axis"]
+        input_tensors = [input_tensor]
+      else:
+        if isinstance(parameters["axis"], list):
+          shape = [len(parameters["axis"])]
+        else:
+          shape = []  # shape for None or integers.
+        axis = tf.compat.v1.placeholder(
+            dtype=tf.int32, name="axis", shape=shape)
+        input_tensors = [input_tensor, axis]
+
+      out = reduce_op(input_tensor, axis=axis, keepdims=parameters["keepdims"])
+      return input_tensors, [out]
+
+    def build_inputs(parameters, sess, inputs, outputs):
+      """Build the inputs for reduced operators."""
+
+      dtype = parameters["input_dtype"]
+      if boolean_tensor_only:
+        dtype = tf.bool
+      values = [
+          create_tensor_data(
+              dtype,
+              parameters["input_shape"],
+              min_value=min_value,
+              max_value=max_value)
+      ]
+      if not parameters["const_axis"]:
+        values.append(np.array(parameters["axis"]))
+      return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+    make_zip_of_tests(options, test_parameters, build_graph, build_inputs)
+
+  return f
+
+
+@register_make_test_function()
+def make_mean_tests(options):
+  """Make a set of tests to do mean."""
+  return make_reduce_tests(
+      tf.reduce_mean,
+      min_value=-1,
+      max_value=1,
+      boolean_tensor_only=False,
+      allow_fully_quantize=True)(
+          options)
+
+
+@register_make_test_function()
+def make_sum_tests(options):
+  """Make a set of tests to do sum."""
+  return make_reduce_tests(
+      tf.reduce_sum,
+      min_value=-1,
+      max_value=1,
+      boolean_tensor_only=False,
+      allow_fully_quantize=True)(
+          options)
+
+
+@register_make_test_function()
+def make_reduce_prod_tests(options):
+  """Make a set of tests to do prod."""
+  # set min max value to be -2, 2 to avoid overflow.
+  return make_reduce_tests(tf.reduce_prod, -2, 2)(options)
+
+
+@register_make_test_function()
+def make_reduce_max_tests(options):
+  """Make a set of tests to do max."""
+  return make_reduce_tests(
+      tf.reduce_max, allow_fully_quantize=True, min_value=-1, max_value=1)(
+          options)
+
+
+@register_make_test_function()
+def make_reduce_min_tests(options):
+  """Make a set of tests to do min."""
+  return make_reduce_tests(
+      tf.reduce_min, allow_fully_quantize=True, min_value=-1, max_value=1)(
+          options)
+
+
+@register_make_test_function()
+def make_reduce_any_tests(options):
+  """Make a set of tests to do any."""
+  return make_reduce_tests(tf.reduce_any, boolean_tensor_only=True)(options)
+
+
+@register_make_test_function()
+def make_reduce_all_tests(options):
+  """Make a set of tests to do all."""
+  return make_reduce_tests(tf.reduce_all, boolean_tensor_only=True)(options)

tensorflow/lite/testing/op_tests/relu.py

@@ -0,0 +1,48 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for relu."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_relu_tests(options):
+  """Make a set of tests to do relu."""
+
+  # Chose a set of parameters
+  test_parameters = [{
+      "input_shape": [[], [1], [2, 3], [1, 1, 1, 1], [1, 3, 4, 3],
+                      [3, 15, 14, 3], [3, 1, 2, 4, 6], [2, 2, 3, 4, 5, 6]],
+      "fully_quantize": [True, False],
+      "input_range": [(-8, 8)]
+  }]
+
+  def build_graph(parameters):
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="input", shape=parameters["input_shape"])
+    out = tf.nn.relu(input_tensor)
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    min_value, max_value = parameters["input_range"]
+    input_values = create_tensor_data(
+        np.float32, parameters["input_shape"], min_value, max_value)
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/relu1.py

@@ -0,0 +1,49 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for relu1."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_relu1_tests(options):
+  """Make a set of tests to do relu1."""
+
+  # Chose a set of parameters
+  test_parameters = [{
+      "input_shape": [[], [1, 1, 1, 1], [1, 3, 4, 3], [3, 15, 14, 3]],
+      "fully_quantize": [True, False],
+      "input_range": [(-2, 8)]
+  }]
+
+  def build_graph(parameters):
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="input", shape=parameters["input_shape"])
+    # Note that the following is not supported:
+    #   out = tf.maximum(-1.0, tf.minimum(input_tensor, 1.0))
+    out = tf.minimum(1.0, tf.maximum(input_tensor, -1.0))
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    min_value, max_value = parameters["input_range"]
+    input_values = create_tensor_data(
+        np.float32, parameters["input_shape"], min_value, max_value)
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)

tensorflow/lite/testing/op_tests/relu6.py

@@ -0,0 +1,48 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test configs for relu6."""
+import numpy as np
+import tensorflow as tf
+from tensorflow.lite.testing.zip_test_utils import create_tensor_data
+from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests
+from tensorflow.lite.testing.zip_test_utils import register_make_test_function
+
+
+@register_make_test_function()
+def make_relu6_tests(options):
+  """Make a set of tests to do relu6."""
+
+  # Chose a set of parameters
+  test_parameters = [{
+      "input_shape": [[], [1, 1, 1, 1], [1, 3, 4, 3], [3, 15, 14, 3],
+                      [3, 1, 2, 4, 6], [2, 2, 3, 4, 5, 6]],
+      "fully_quantize": [True, False],
+      "input_range": [(-2, 8)]
+  }]
+
+  def build_graph(parameters):
+    input_tensor = tf.compat.v1.placeholder(
+        dtype=tf.float32, name="input", shape=parameters["input_shape"])
+    out = tf.nn.relu6(input_tensor)
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    min_value, max_value = parameters["input_range"]
+    input_values = create_tensor_data(np.float32, parameters["input_shape"],
+                                      min_value, max_value)
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(options, test_parameters, build_graph, build_inputs)