chore: import upstream snapshot with attribution

tensorflow/lite/delegates/gpu/cl/testing/BUILD

@@ -0,0 +1,108 @@
+load("@rules_cc//cc:cc_binary.bzl", "cc_binary")
+load("@rules_cc//cc:cc_test.bzl", "cc_test")
+load("//tensorflow/lite/delegates/gpu:build_defs.bzl", "gtest_main_no_heapcheck_deps")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_visibility = ["//visibility:public"],
+    licenses = ["notice"],
+)
+
+cc_binary(
+    name = "delegate_testing",
+    srcs = ["delegate_testing.cc"],
+    tags = [
+        "nobuilder",
+        "notap",
+    ],
+    deps = [
+        "//tensorflow/lite/delegates/gpu:delegate",
+        "//tensorflow/lite/delegates/gpu/cl:gpu_api_delegate",
+        "//tensorflow/lite/delegates/gpu/common:model_builder",
+        "//tensorflow/lite/delegates/gpu/common:status",
+        "//tensorflow/lite/kernels:builtin_ops",
+        "//tensorflow/lite/kernels:kernel_util",
+        "@com_google_absl//absl/time",
+    ],
+)
+
+cc_test(
+    name = "gpu_model_test",
+    srcs = ["gpu_model_test.cc"],
+    tags = [
+        "noasan",
+        "nomsan",
+        "notsan",
+        "requires-gpu-nvidia",
+    ],
+    deps = [
+        "//tensorflow/lite/delegates/gpu/cl/kernels:cl_test",
+        "//tensorflow/lite/delegates/gpu/common:gpu_model_test_util",
+        "//tensorflow/lite/delegates/gpu/common:status",
+    ] + gtest_main_no_heapcheck_deps(),  # constant buffers leak on nvidia
+)
+
+cc_binary(
+    name = "internal_api_samples",
+    srcs = ["internal_api_samples.cc"],
+    linkopts = select({
+        "//tensorflow:android": [
+            "-lEGL",
+            "-lGLESv3",
+        ],
+        "//conditions:default": [],
+    }),
+    tags = [
+        "nobuilder",
+        "notap",
+    ],
+    deps = [
+        "//tensorflow/lite/delegates/gpu:api",
+        "//tensorflow/lite/delegates/gpu/cl:api",
+        "//tensorflow/lite/delegates/gpu/cl:environment",
+        "//tensorflow/lite/delegates/gpu/cl:inference_context",
+        "//tensorflow/lite/delegates/gpu/common:model",
+        "//tensorflow/lite/delegates/gpu/common:model_builder",
+        "//tensorflow/lite/delegates/gpu/common:status",
+        "//tensorflow/lite/kernels:builtin_ops",
+        "//tensorflow/lite/kernels:kernel_util",
+        "@com_google_absl//absl/time",
+        "@com_google_absl//absl/types:span",
+    ],
+)
+
+cc_binary(
+    name = "memory_sharing_sample",
+    srcs = ["memory_sharing_sample.cc"],
+    deps = [
+        "//tensorflow/lite/core/kernels:builtin_ops",
+        "//tensorflow/lite/delegates/gpu/cl:environment",
+        "//tensorflow/lite/delegates/gpu/cl:inference_context",
+        "//tensorflow/lite/delegates/gpu/common:model",
+        "//tensorflow/lite/delegates/gpu/common:model_builder",
+        "//tensorflow/lite/delegates/gpu/common:status",
+        "@com_google_absl//absl/time",
+    ],
+)
+
+cc_binary(
+    name = "performance_profiling",
+    srcs = ["performance_profiling.cc"],
+    copts = [
+        "-DGOOGLE_COMMANDLINEFLAGS_FULL_API=1",
+    ],
+    deps = [
+        "//tensorflow/lite/core/kernels:builtin_ops",
+        "//tensorflow/lite/delegates/gpu/cl:cl_command_buffer",
+        "//tensorflow/lite/delegates/gpu/cl:environment",
+        "//tensorflow/lite/delegates/gpu/cl:inference_context",
+        "//tensorflow/lite/delegates/gpu/cl:opencl_wrapper",
+        "//tensorflow/lite/delegates/gpu/common:model",
+        "//tensorflow/lite/delegates/gpu/common:model_builder",
+        "//tensorflow/lite/delegates/gpu/common:status",
+        "@com_google_absl//absl/flags:flag",
+        "@com_google_absl//absl/flags:parse",
+        "@com_google_absl//absl/time",
+        "@opencl_headers",
+    ],
+)

tensorflow/lite/delegates/gpu/cl/testing/delegate_testing.cc

@@ -0,0 +1,202 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <algorithm>
+#include <chrono>  // NOLINT(build/c++11)
+#include <cmath>
+#include <cstdlib>
+#include <iostream>
+#include <memory>
+#include <string>
+
+#include "absl/time/time.h"
+#include "tensorflow/lite/delegates/gpu/cl/gpu_api_delegate.h"
+#include "tensorflow/lite/delegates/gpu/common/model_builder.h"
+#include "tensorflow/lite/delegates/gpu/common/status.h"
+#include "tensorflow/lite/delegates/gpu/delegate.h"
+#include "tensorflow/lite/kernels/kernel_util.h"
+#include "tensorflow/lite/kernels/register.h"
+
+namespace {
+
+void FillInputTensor(tflite::Interpreter* interpreter) {
+  for (int k = 0; k < interpreter->inputs().size(); ++k) {
+    TfLiteTensor* tensor_ptr = interpreter->tensor(interpreter->inputs()[k]);
+    const auto tensor_elements_count = tflite::NumElements(tensor_ptr);
+    if (tensor_ptr->type == kTfLiteFloat32) {
+      float* p = interpreter->typed_input_tensor<float>(k);
+      for (int i = 0; i < tensor_elements_count; ++i) {
+        p[i] = std::sin(i);
+      }
+    }
+    if (tensor_ptr->type == kTfLiteInt32) {
+      int* p = interpreter->typed_input_tensor<int>(k);
+      for (int i = 0; i < tensor_elements_count; ++i) {
+        p[i] = i % 32;
+      }
+    }
+    if (tensor_ptr->type == kTfLiteInt8) {
+      int8_t* p = interpreter->typed_input_tensor<int8_t>(k);
+      for (int i = 0; i < tensor_elements_count; ++i) {
+        p[i] = i % 256 - 128;
+      }
+    }
+    if (tensor_ptr->type == kTfLiteUInt8) {
+      uint8_t* p = interpreter->typed_input_tensor<uint8_t>(k);
+      for (int i = 0; i < tensor_elements_count; ++i) {
+        p[i] = i % 256;
+      }
+    }
+  }
+}
+
+void CompareCPUGPUResults(tflite::Interpreter* cpu, tflite::Interpreter* gpu,
+                          float eps) {
+  for (int i = 0; i < cpu->outputs().size(); ++i) {
+    TfLiteTensor* tensor_ptr = cpu->tensor(cpu->outputs()[i]);
+    const auto tensor_elements_count = tflite::NumElements(tensor_ptr);
+
+    std::cout << "Output " << tensor_ptr->name << ":" << std::endl;
+
+    const int kMaxPrint = 10;
+    int printed = 0;
+    int total_different = 0;
+    for (int k = 0; k < tensor_elements_count; ++k) {
+      float cpu_val = 0.0f;
+      float gpu_val = 0.0f;
+      if (tensor_ptr->type == kTfLiteFloat32) {
+        const float* cpu_out = cpu->typed_output_tensor<float>(i);
+        const float* gpu_out = gpu->typed_output_tensor<float>(i);
+        cpu_val = cpu_out[k];
+        gpu_val = gpu_out[k];
+      }
+      if (tensor_ptr->type == kTfLiteInt8) {
+        int8_t* cpu_out = cpu->typed_output_tensor<int8_t>(i);
+        int8_t* gpu_out = gpu->typed_output_tensor<int8_t>(i);
+        cpu_val = cpu_out[k];
+        gpu_val = gpu_out[k];
+      }
+      if (tensor_ptr->type == kTfLiteUInt8) {
+        uint8_t* cpu_out = cpu->typed_output_tensor<uint8_t>(i);
+        uint8_t* gpu_out = gpu->typed_output_tensor<uint8_t>(i);
+        cpu_val = cpu_out[k];
+        gpu_val = gpu_out[k];
+      }
+      const float abs_diff = fabs(cpu_val - gpu_val);
+      if (abs_diff > eps) {
+        total_different++;
+        if (printed < kMaxPrint) {
+          std::cout << "Element #" << k << ": CPU value - " << cpu_val
+                    << ", GPU value - " << gpu_val << ", abs diff - "
+                    << abs_diff << std::endl;
+          printed++;
+        }
+        if (printed == kMaxPrint) {
+          std::cout << "Printed " << kMaxPrint
+                    << " different elements, threshhold - " << eps
+                    << ", next different elements skipped" << std::endl;
+          printed++;
+        }
+      }
+    }
+    std::cout << "Total " << total_different
+              << " different elements, for output #" << i << ", threshhold - "
+              << eps << std::endl;
+  }
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+  if (argc <= 1) {
+    std::cerr << "Expected model path as second argument." << std::endl;
+    return -1;
+  }
+
+  auto model = tflite::FlatBufferModel::BuildFromFile(argv[1]);
+  if (!model) {
+    std::cerr << "FlatBufferModel::BuildFromFile failed, model path - "
+              << argv[1] << std::endl;
+    return -1;
+  }
+  tflite::ops::builtin::BuiltinOpResolver op_resolver;
+  tflite::InterpreterBuilder builder(*model, op_resolver);
+
+  // CPU.
+  std::unique_ptr<tflite::Interpreter> cpu_inference;
+  builder(&cpu_inference);
+  if (!cpu_inference) {
+    std::cerr << "Failed to build CPU inference." << std::endl;
+    return -1;
+  }
+  auto status = cpu_inference->AllocateTensors();
+  if (status != kTfLiteOk) {
+    std::cerr << "Failed to AllocateTensors for CPU inference." << std::endl;
+    return -1;
+  }
+  FillInputTensor(cpu_inference.get());
+  status = cpu_inference->Invoke();
+  if (status != kTfLiteOk) {
+    std::cerr << "Failed to Invoke CPU inference." << std::endl;
+    return -1;
+  }
+
+  // GPU.
+  std::unique_ptr<tflite::Interpreter> gpu_inference;
+  builder(&gpu_inference);
+  if (!gpu_inference) {
+    std::cerr << "Failed to build GPU inference." << std::endl;
+    return -1;
+  }
+  TfLiteGpuDelegateOptionsV2 options;
+  options.is_precision_loss_allowed = -1;
+  options.inference_preference =
+      TFLITE_GPU_INFERENCE_PREFERENCE_FAST_SINGLE_ANSWER;
+  options.inference_priority1 = TFLITE_GPU_INFERENCE_PRIORITY_MIN_LATENCY;
+  options.inference_priority2 = TFLITE_GPU_INFERENCE_PRIORITY_MIN_MEMORY_USAGE;
+  options.inference_priority3 = TFLITE_GPU_INFERENCE_PRIORITY_MAX_PRECISION;
+  options.experimental_flags = TFLITE_GPU_EXPERIMENTAL_FLAGS_ENABLE_QUANT;
+  options.max_delegated_partitions = 1;
+  auto* gpu_delegate = TfLiteGpuDelegateV2Create(&options);
+  status = gpu_inference->ModifyGraphWithDelegate(gpu_delegate);
+  if (status != kTfLiteOk) {
+    std::cerr << "ModifyGraphWithDelegate failed." << std::endl;
+    return -1;
+  }
+  FillInputTensor(gpu_inference.get());
+  status = gpu_inference->Invoke();
+  if (status != kTfLiteOk) {
+    std::cerr << "Failed to Invoke GPU inference." << std::endl;
+    return -1;
+  }
+
+  CompareCPUGPUResults(cpu_inference.get(), gpu_inference.get(), 1e-4f);
+
+  // CPU inference latency.
+  auto start = std::chrono::high_resolution_clock::now();
+  cpu_inference->Invoke();
+  auto end = std::chrono::high_resolution_clock::now();
+  std::cout << "CPU time - " << (end - start).count() * 1e-6f << "ms"
+            << std::endl;
+
+  // GPU inference latency.
+  start = std::chrono::high_resolution_clock::now();
+  gpu_inference->Invoke();
+  end = std::chrono::high_resolution_clock::now();
+  std::cout << "GPU time(CPU->GPU->CPU) - " << (end - start).count() * 1e-6f
+            << "ms" << std::endl;
+
+  TfLiteGpuDelegateV2Delete(gpu_delegate);
+  return EXIT_SUCCESS;
+}

tensorflow/lite/delegates/gpu/cl/testing/gpu_model_test.cc

@@ -0,0 +1,96 @@
+/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <gtest/gtest.h>
+#include "tensorflow/lite/delegates/gpu/cl/kernels/cl_test.h"
+#include "tensorflow/lite/delegates/gpu/common/gpu_model_test_util.h"
+#include "tensorflow/lite/delegates/gpu/common/status.h"
+
+namespace tflite {
+namespace gpu {
+namespace cl {
+namespace {
+
+TEST_F(OpenCLOperationTest, LinkingConvolutionAndCosOp) {
+  auto status = TestLinkingConvolutionAndCosOp(&exec_env_);
+  ASSERT_TRUE(status.ok()) << status.message();
+}
+
+TEST_F(OpenCLOperationTest, LinkingConvolution2InputMul2InputMul) {
+  auto status = TestLinkingConvolution2InputMul2InputMul(&exec_env_);
+  ASSERT_TRUE(status.ok()) << status.message();
+}
+
+TEST_F(OpenCLOperationTest, LinkingConvolution2InputBroadcastMul2InputMul) {
+  auto status = TestLinkingConvolution2InputBroadcastMul2InputMul(&exec_env_);
+  ASSERT_TRUE(status.ok()) << status.message();
+}
+
+TEST_F(OpenCLOperationTest, LinkingConvolution2InputMul2InputBroadcastMul) {
+  auto status = TestLinkingConvolution2InputMul2InputBroadcastMul(&exec_env_);
+  ASSERT_TRUE(status.ok()) << status.message();
+}
+
+TEST_F(OpenCLOperationTest, LinkingConvolution2InputMul2InputMulCos) {
+  auto status = TestLinkingConvolution2InputMul2InputMulCos(&exec_env_);
+  ASSERT_TRUE(status.ok()) << status.message();
+}
+
+TEST_F(OpenCLOperationTest, LinkingConvolutionFirstTanh2InputDiff) {
+  auto status = TestLinkingConvolutionFirstTanh2InputDiff(&exec_env_);
+  ASSERT_TRUE(status.ok()) << status.message();
+}
+
+TEST_F(OpenCLOperationTest, LinkingConvolutionSecondTanh2InputDiff) {
+  auto status = TestLinkingConvolutionSecondTanh2InputDiff(&exec_env_);
+  ASSERT_TRUE(status.ok()) << status.message();
+}
+
+TEST_F(OpenCLOperationTest, LinkingConvolutionFirstTanhSecondCos2InputDiff) {
+  auto status = TestLinkingConvolutionFirstTanhSecondCos2InputDiff(&exec_env_);
+  ASSERT_TRUE(status.ok()) << status.message();
+}
+
+TEST_F(OpenCLOperationTest, LinkingComplex0) {
+  auto status = TestLinkingComplex0(&exec_env_);
+  ASSERT_TRUE(status.ok()) << status.message();
+}
+
+TEST_F(OpenCLOperationTest, LinkingConvElem2InputAddElemsOp) {
+  auto status = TestLinkingConvElem2InputAddElemsOp(&exec_env_);
+  ASSERT_TRUE(status.ok()) << status.message();
+}
+
+TEST_F(OpenCLOperationTest, LinkingSliceCastOp) {
+  auto status = TestLinkingSliceCastOp(&exec_env_);
+  ASSERT_TRUE(status.ok()) << status.message();
+}
+
+TEST_F(OpenCLOperationTest, LinkingAddAddMulOp) {
+  auto status = TestLinkingAddAddMulOp(&exec_env_,
+                                       /*use_second_input_add=*/true);
+  ASSERT_TRUE(status.ok()) << status.message();
+}
+
+TEST_F(OpenCLOperationTest, LinkingAddMulOp) {
+  auto status =
+      TestLinkingAddAddMulOp(&exec_env_, /*use_second_input_add=*/false);
+  ASSERT_TRUE(status.ok()) << status.message();
+}
+
+}  // namespace
+}  // namespace cl
+}  // namespace gpu
+}  // namespace tflite

tensorflow/lite/delegates/gpu/cl/testing/internal_api_samples.cc

@@ -0,0 +1,468 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <algorithm>
+#include <chrono>  // NOLINT(build/c++11)
+#include <iostream>
+#include <string>
+
+#include "absl/time/time.h"
+#include "absl/types/span.h"
+#include "tensorflow/lite/delegates/gpu/api.h"
+#include "tensorflow/lite/delegates/gpu/cl/api.h"
+#include "tensorflow/lite/delegates/gpu/cl/environment.h"
+#include "tensorflow/lite/delegates/gpu/cl/inference_context.h"
+#include "tensorflow/lite/delegates/gpu/common/model.h"
+#include "tensorflow/lite/delegates/gpu/common/model_builder.h"
+#include "tensorflow/lite/delegates/gpu/common/status.h"
+#include "tensorflow/lite/kernels/kernel_util.h"
+#include "tensorflow/lite/kernels/register.h"
+
+namespace tflite {
+namespace gpu {
+namespace cl {
+namespace {
+void FillInputTensors(tflite::Interpreter* interpreter) {
+  for (int k = 0; k < interpreter->inputs().size(); ++k) {
+    TfLiteTensor* tensor_ptr = interpreter->tensor(interpreter->inputs()[k]);
+    const auto tensor_elements_count = tflite::NumElements(tensor_ptr);
+    if (tensor_ptr->type == kTfLiteFloat32) {
+      float* p = interpreter->typed_input_tensor<float>(k);
+      for (int i = 0; i < tensor_elements_count; ++i) {
+        p[i] = std::sin(i);
+      }
+    } else {
+      std::cout << "No support of non Float32 input/output tensors"
+                << std::endl;
+    }
+  }
+}
+
+void CompareCPUGPUResults(tflite::Interpreter* cpu,
+                          const std::vector<int64_t>& outputs,
+                          const std::vector<std::vector<float>>& gpu,
+                          float eps) {
+  for (int i = 0; i < gpu.size(); ++i) {
+    TfLiteTensor* tensor_ptr = cpu->tensor(outputs[i]);
+    const float* cpu_out = tensor_ptr->data.f;
+    const float* gpu_out = gpu[i].data();
+    const int kMaxPrint = 10;
+    int printed = 0;
+    int total_different = 0;
+    for (int k = 0; k < tensor_ptr->bytes / 4; ++k) {
+      const float abs_diff = fabs(cpu_out[k] - gpu_out[k]);
+      if (abs_diff > eps) {
+        total_different++;
+        if (printed < kMaxPrint) {
+          std::cout << "Output #" << i << ": element #" << k << ": CPU value - "
+                    << cpu_out[k] << ", GPU value - " << gpu_out[k]
+                    << ", abs diff - " << abs_diff << std::endl;
+          printed++;
+        }
+        if (printed == kMaxPrint) {
+          std::cout << "Printed " << kMaxPrint
+                    << " different elements, threshhold - " << eps
+                    << ", next different elements skipped" << std::endl;
+          printed++;
+        }
+      }
+    }
+    std::cout << "Total " << total_different
+              << " different elements, for output #" << i << ", threshhold - "
+              << eps << std::endl;
+  }
+}
+}  // namespace
+
+absl::Status RunModelSampleWithInternalAPISerializedKernels(
+    const std::string& model_name, const std::vector<uint8_t>& kernel_cache);
+
+absl::Status RunModelSampleWithInternalAPISerialized(
+    tflite::Interpreter* cpu, const std::vector<uint8_t>& kernel_cache,
+    const std::vector<uint8_t>& serialized_model);
+
+// Run Jet with OpenCL internal API and compares correctness with TFLite CPU
+absl::Status RunModelSampleWithInternalAPI(const std::string& model_name,
+                                           std::vector<uint8_t>* kernel_cache) {
+  auto flatbuffer = tflite::FlatBufferModel::BuildFromFile(model_name.c_str());
+
+  ops::builtin::BuiltinOpResolver op_resolver;
+  InterpreterBuilder tfl_builder(*flatbuffer, op_resolver);
+
+  // CPU.
+  std::unique_ptr<tflite::Interpreter> cpu_inference;
+  tfl_builder(&cpu_inference);
+  if (!cpu_inference) {
+    return absl::InternalError("Failed to build CPU inference.");
+  }
+  auto status = cpu_inference->AllocateTensors();
+  if (status != kTfLiteOk) {
+    return absl::InternalError("Failed to AllocateTensors for CPU inference.");
+  }
+  for (int k = 0; k < cpu_inference->inputs().size(); ++k) {
+    TfLiteTensor* tensor_ptr =
+        cpu_inference->tensor(cpu_inference->inputs()[k]);
+    if (tensor_ptr->type != kTfLiteFloat32) {
+      return absl::InvalidArgumentError(
+          "Internal api supports only F32 input tensors");
+    }
+  }
+  for (int k = 0; k < cpu_inference->outputs().size(); ++k) {
+    TfLiteTensor* tensor_ptr =
+        cpu_inference->tensor(cpu_inference->outputs()[k]);
+    if (tensor_ptr->type != kTfLiteFloat32) {
+      return absl::InvalidArgumentError(
+          "Internal api supports only F32 output tensors");
+    }
+  }
+  FillInputTensors(cpu_inference.get());
+  status = cpu_inference->Invoke();
+  if (status != kTfLiteOk) {
+    return absl::InternalError("Failed to Invoke CPU inference.");
+  }
+
+  const auto start = std::chrono::high_resolution_clock::now();
+  GraphFloat32 graph_cl;
+  RETURN_IF_ERROR(BuildFromFlatBuffer(*flatbuffer, op_resolver, &graph_cl));
+
+  auto inputs = graph_cl.inputs();
+  auto outputs = graph_cl.outputs();
+  std::vector<int64_t> in_refs(inputs.size());
+  std::vector<int64_t> out_refs(outputs.size());
+  for (int i = 0; i < inputs.size(); ++i) {
+    in_refs[i] = inputs[i]->tensor.ref;
+  }
+  for (int i = 0; i < outputs.size(); ++i) {
+    out_refs[i] = outputs[i]->tensor.ref;
+  }
+
+  Environment env;
+  RETURN_IF_ERROR(CreateEnvironment(&env));
+
+  std::unique_ptr<InferenceEnvironment> inf_env;
+  // Initializes environment.
+  InferenceEnvironmentOptions env_options;
+  env_options.device = env.device().id();
+  env_options.context = env.context().context();
+  env_options.command_queue = env.queue()->queue();
+  RETURN_IF_ERROR(NewInferenceEnvironment(env_options, &inf_env, nullptr));
+
+  std::unique_ptr<InferenceBuilder> builder;
+  // Initializes builder.
+  InferenceOptions options;
+  options.priority1 = InferencePriority::MIN_LATENCY;
+  options.priority2 = InferencePriority::MIN_MEMORY_USAGE;
+  options.priority3 = InferencePriority::MAX_PRECISION;
+  options.usage = InferenceUsage::SUSTAINED_SPEED;
+
+  RETURN_IF_ERROR(
+      inf_env->NewInferenceBuilder(options, std::move(graph_cl), &builder));
+
+  // Sets input/output object def for builder_.
+  ObjectDef obj_def;
+  obj_def.data_type = DataType::FLOAT32;
+  obj_def.data_layout = DataLayout::BHWC;
+  obj_def.object_type = ObjectType::CPU_MEMORY;
+  obj_def.user_provided = true;
+  for (int i = 0; i < in_refs.size(); ++i) {
+    RETURN_IF_ERROR(builder->SetInputObjectDef(i, obj_def));
+  }
+  for (int i = 0; i < out_refs.size(); ++i) {
+    RETURN_IF_ERROR(builder->SetOutputObjectDef(i, obj_def));
+  }
+
+  std::unique_ptr<::tflite::gpu::InferenceRunner> runner;
+  // Builds runner.
+  RETURN_IF_ERROR(builder->Build(&runner));
+
+  const auto end = std::chrono::high_resolution_clock::now();
+  std::cout << "Initialization total time - " << (end - start).count() * 1e-6f
+            << "ms" << std::endl;
+
+  if (kernel_cache) {
+    *kernel_cache = inf_env->GetSerializedBinaryCache();
+    std::cout << "Kernel cache size - " << kernel_cache->size() << std::endl;
+  }
+
+  // Sets the input/output object.
+  for (int i = 0; i < in_refs.size(); ++i) {
+    TfLiteTensor* tensor_ptr = cpu_inference->tensor(in_refs[i]);
+    RETURN_IF_ERROR(runner->SetInputObject(
+        i, CpuMemory{tensor_ptr->data.data, tensor_ptr->bytes}));
+  }
+
+  std::vector<std::vector<float>> output_tensors(out_refs.size());
+  for (int i = 0; i < out_refs.size(); ++i) {
+    TfLiteTensor* tensor_ptr = cpu_inference->tensor(out_refs[i]);
+    output_tensors[i].resize(tensor_ptr->bytes / 4);
+    RETURN_IF_ERROR(runner->SetOutputObject(
+        i, CpuMemory{output_tensors[i].data(), tensor_ptr->bytes}));
+  }
+
+  RETURN_IF_ERROR(runner->Run());
+
+  CompareCPUGPUResults(cpu_inference.get(), out_refs, output_tensors, 1e-4f);
+
+  return absl::OkStatus();
+}
+
+absl::Status RunModelSampleWithInternalAPISerializedKernels(
+    const std::string& model_name, const std::vector<uint8_t>& kernel_cache) {
+  auto flatbuffer = tflite::FlatBufferModel::BuildFromFile(model_name.c_str());
+
+  ops::builtin::BuiltinOpResolver op_resolver;
+  InterpreterBuilder tfl_builder(*flatbuffer, op_resolver);
+
+  // CPU.
+  std::unique_ptr<tflite::Interpreter> cpu_inference;
+  tfl_builder(&cpu_inference);
+  if (!cpu_inference) {
+    return absl::InternalError("Failed to build CPU inference.");
+  }
+  auto status = cpu_inference->AllocateTensors();
+  if (status != kTfLiteOk) {
+    return absl::InternalError("Failed to AllocateTensors for CPU inference.");
+  }
+  for (int k = 0; k < cpu_inference->inputs().size(); ++k) {
+    TfLiteTensor* tensor_ptr =
+        cpu_inference->tensor(cpu_inference->inputs()[k]);
+    if (tensor_ptr->type != kTfLiteFloat32) {
+      return absl::InvalidArgumentError(
+          "Internal api supports only F32 input tensors");
+    }
+  }
+  for (int k = 0; k < cpu_inference->outputs().size(); ++k) {
+    TfLiteTensor* tensor_ptr =
+        cpu_inference->tensor(cpu_inference->outputs()[k]);
+    if (tensor_ptr->type != kTfLiteFloat32) {
+      return absl::InvalidArgumentError(
+          "Internal api supports only F32 output tensors");
+    }
+  }
+  FillInputTensors(cpu_inference.get());
+  status = cpu_inference->Invoke();
+  if (status != kTfLiteOk) {
+    return absl::InternalError("Failed to Invoke CPU inference.");
+  }
+
+  const auto start = std::chrono::high_resolution_clock::now();
+  GraphFloat32 graph_cl;
+  RETURN_IF_ERROR(BuildFromFlatBuffer(*flatbuffer, op_resolver, &graph_cl));
+
+  auto inputs = graph_cl.inputs();
+  auto outputs = graph_cl.outputs();
+  std::vector<int64_t> in_refs(inputs.size());
+  std::vector<int64_t> out_refs(outputs.size());
+  for (int i = 0; i < inputs.size(); ++i) {
+    in_refs[i] = inputs[i]->tensor.ref;
+  }
+  for (int i = 0; i < outputs.size(); ++i) {
+    out_refs[i] = outputs[i]->tensor.ref;
+  }
+
+  Environment env;
+  RETURN_IF_ERROR(CreateEnvironment(&env));
+
+  std::unique_ptr<InferenceEnvironment> inf_env;
+  // Initializes environment.
+  InferenceEnvironmentOptions env_options;
+  env_options.device = env.device().id();
+  env_options.context = env.context().context();
+  env_options.command_queue = env.queue()->queue();
+  env_options.serialized_binary_cache =
+      absl::MakeSpan(kernel_cache.data(), kernel_cache.size());
+  RETURN_IF_ERROR(NewInferenceEnvironment(env_options, &inf_env, nullptr));
+
+  InferenceOptions options;
+  options.priority1 = InferencePriority::MIN_LATENCY;
+  options.priority2 = InferencePriority::MIN_MEMORY_USAGE;
+  options.priority3 = InferencePriority::MAX_PRECISION;
+  options.usage = InferenceUsage::SUSTAINED_SPEED;
+
+  std::vector<uint8_t> serialized_model;
+  RETURN_IF_ERROR(inf_env->BuildSerializedModel(options, std::move(graph_cl),
+                                                &serialized_model));
+  std::unique_ptr<InferenceBuilder> builder;
+  RETURN_IF_ERROR(inf_env->NewInferenceBuilder(serialized_model, &builder));
+
+  // Sets input/output object def for builder_.
+  ObjectDef obj_def;
+  obj_def.data_type = DataType::FLOAT32;
+  obj_def.data_layout = DataLayout::BHWC;
+  obj_def.object_type = ObjectType::CPU_MEMORY;
+  obj_def.user_provided = true;
+  for (int i = 0; i < in_refs.size(); ++i) {
+    RETURN_IF_ERROR(builder->SetInputObjectDef(i, obj_def));
+  }
+  for (int i = 0; i < out_refs.size(); ++i) {
+    RETURN_IF_ERROR(builder->SetOutputObjectDef(i, obj_def));
+  }
+
+  std::unique_ptr<::tflite::gpu::InferenceRunner> runner;
+  // Builds runner.
+  RETURN_IF_ERROR(builder->Build(&runner));
+
+  const auto end = std::chrono::high_resolution_clock::now();
+  std::cout << "Initialization total time";
+  if (!kernel_cache.empty()) {
+    std::cout << "(with kernel cache)";
+  }
+  std::cout << " - " << (end - start).count() * 1e-6f << "ms" << std::endl;
+
+  // Sets the input/output object.
+  for (int i = 0; i < in_refs.size(); ++i) {
+    TfLiteTensor* tensor_ptr = cpu_inference->tensor(in_refs[i]);
+    RETURN_IF_ERROR(runner->SetInputObject(
+        i, CpuMemory{tensor_ptr->data.data, tensor_ptr->bytes}));
+  }
+
+  std::vector<std::vector<float>> output_tensors(out_refs.size());
+  for (int i = 0; i < out_refs.size(); ++i) {
+    TfLiteTensor* tensor_ptr = cpu_inference->tensor(out_refs[i]);
+    output_tensors[i].resize(tensor_ptr->bytes / 4);
+    RETURN_IF_ERROR(runner->SetOutputObject(
+        i, CpuMemory{output_tensors[i].data(), tensor_ptr->bytes}));
+  }
+
+  RETURN_IF_ERROR(runner->Run());
+
+  CompareCPUGPUResults(cpu_inference.get(), out_refs, output_tensors, 1e-4f);
+
+  RETURN_IF_ERROR(RunModelSampleWithInternalAPISerialized(
+      cpu_inference.get(), kernel_cache, serialized_model));
+
+  return absl::OkStatus();
+}
+
+absl::Status RunModelSampleWithInternalAPISerialized(
+    tflite::Interpreter* cpu, const std::vector<uint8_t>& kernel_cache,
+    const std::vector<uint8_t>& serialized_model) {
+  FillInputTensors(cpu);
+  auto status = cpu->Invoke();
+  if (status != kTfLiteOk) {
+    return absl::InternalError("Failed to Invoke CPU inference.");
+  }
+
+  const auto start = std::chrono::high_resolution_clock::now();
+
+  Environment env;
+  RETURN_IF_ERROR(CreateEnvironment(&env));
+
+  std::unique_ptr<InferenceEnvironment> inf_env;
+  // Initializes environment.
+  InferenceEnvironmentOptions env_options;
+  env_options.device = env.device().id();
+  env_options.context = env.context().context();
+  env_options.command_queue = env.queue()->queue();
+  env_options.serialized_binary_cache =
+      absl::MakeSpan(kernel_cache.data(), kernel_cache.size());
+  RETURN_IF_ERROR(NewInferenceEnvironment(env_options, &inf_env, nullptr));
+
+  std::vector<int64_t> in_refs;
+  std::vector<int64_t> out_refs;
+  RETURN_IF_ERROR(GetInOutRefs(serialized_model, &in_refs, &out_refs));
+  std::unique_ptr<InferenceBuilder> builder;
+  RETURN_IF_ERROR(inf_env->NewInferenceBuilder(serialized_model, &builder));
+
+  // Sets input/output object def for builder_.
+  ObjectDef obj_def;
+  obj_def.data_type = DataType::FLOAT32;
+  obj_def.data_layout = DataLayout::BHWC;
+  obj_def.object_type = ObjectType::CPU_MEMORY;
+  obj_def.user_provided = true;
+  for (int i = 0; i < in_refs.size(); ++i) {
+    RETURN_IF_ERROR(builder->SetInputObjectDef(i, obj_def));
+  }
+  for (int i = 0; i < out_refs.size(); ++i) {
+    RETURN_IF_ERROR(builder->SetOutputObjectDef(i, obj_def));
+  }
+
+  std::unique_ptr<::tflite::gpu::InferenceRunner> runner;
+  // Builds runner.
+  RETURN_IF_ERROR(builder->Build(&runner));
+
+  const auto end = std::chrono::high_resolution_clock::now();
+  std::cout << "Serialized initialization total time";
+  if (kernel_cache.empty()) {
+    std::cout << "(without kernel cache)";
+  } else {
+    std::cout << "(with kernel cache)";
+  }
+  std::cout << " - " << (end - start).count() * 1e-6f << "ms" << std::endl;
+
+  // Sets the input/output object.
+  for (int i = 0; i < in_refs.size(); ++i) {
+    TfLiteTensor* tensor_ptr = cpu->tensor(in_refs[i]);
+    RETURN_IF_ERROR(runner->SetInputObject(
+        i, CpuMemory{tensor_ptr->data.data, tensor_ptr->bytes}));
+  }
+
+  std::vector<std::vector<float>> output_tensors(out_refs.size());
+  for (int i = 0; i < out_refs.size(); ++i) {
+    TfLiteTensor* tensor_ptr = cpu->tensor(out_refs[i]);
+    output_tensors[i].resize(tensor_ptr->bytes / 4);
+    RETURN_IF_ERROR(runner->SetOutputObject(
+        i, CpuMemory{output_tensors[i].data(), tensor_ptr->bytes}));
+  }
+
+  RETURN_IF_ERROR(runner->Run());
+
+  std::cout << "Comparing results second time:" << std::endl;
+
+  CompareCPUGPUResults(cpu, out_refs, output_tensors, 1e-4f);
+
+  return absl::OkStatus();
+}
+
+}  // namespace cl
+}  // namespace gpu
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  if (argc <= 1) {
+    std::cerr << "Expected model path as second argument.";
+    return -1;
+  }
+
+  auto load_status = tflite::gpu::cl::LoadOpenCL();
+  if (!load_status.ok()) {
+    std::cerr << load_status.message();
+    return -1;
+  }
+
+  std::vector<uint8_t> kernel_cache;
+  auto run_status =
+      tflite::gpu::cl::RunModelSampleWithInternalAPI(argv[1], &kernel_cache);
+  if (!run_status.ok()) {
+    std::cerr << run_status.message();
+    return -1;
+  }
+  run_status = tflite::gpu::cl::RunModelSampleWithInternalAPISerializedKernels(
+      argv[1], kernel_cache);
+  if (!run_status.ok()) {
+    std::cerr << run_status.message();
+    return -1;
+  }
+
+  // The same with empty kernels cache.
+  run_status = tflite::gpu::cl::RunModelSampleWithInternalAPISerializedKernels(
+      argv[1], {});
+  if (!run_status.ok()) {
+    std::cerr << run_status.message();
+    return -1;
+  }
+
+  return EXIT_SUCCESS;
+}

tensorflow/lite/delegates/gpu/cl/testing/memory_sharing_sample.cc

@@ -0,0 +1,237 @@
+/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <algorithm>
+#include <chrono>  // NOLINT(build/c++11)
+#include <iostream>
+#include <ostream>
+#include <set>
+#include <string>
+
+#include "absl/time/time.h"
+#include "tensorflow/lite/core/kernels/register.h"
+#include "tensorflow/lite/delegates/gpu/cl/environment.h"
+#include "tensorflow/lite/delegates/gpu/cl/inference_context.h"
+#include "tensorflow/lite/delegates/gpu/common/model.h"
+#include "tensorflow/lite/delegates/gpu/common/model_builder.h"
+#include "tensorflow/lite/delegates/gpu/common/status.h"
+
+namespace tflite {
+namespace gpu {
+namespace cl {
+
+// returns storage types that can be used with shared common buffer
+std::set<TensorStorageType> GetSupportedStorages(const GpuInfo& gpu_info) {
+  std::set<TensorStorageType> supported_storages;
+  if (gpu_info.IsCL11OrHigher()) {
+    supported_storages.insert(TensorStorageType::BUFFER);
+  }
+  if (CanUseSubBufferForImage2d(gpu_info)) {
+    supported_storages.insert(TensorStorageType::TEXTURE_2D);
+    supported_storages.insert(TensorStorageType::IMAGE_BUFFER);
+  }
+  return supported_storages;
+}
+
+absl::Status RunSample(const std::string& model_name_mv1,
+                       const std::string& model_name_mv2) {
+  // mv1 postfix here and later is for mobilenet_v1
+  // mv2 postfix here and later is for mobilenet_v2
+  auto flatbuffer_mv1 =
+      tflite::FlatBufferModel::BuildFromFile(model_name_mv1.c_str());
+  GraphFloat32 graph_mv1;
+  ops::builtin::BuiltinOpResolver op_resolver;
+  RETURN_IF_ERROR(BuildFromFlatBuffer(*flatbuffer_mv1, op_resolver, &graph_mv1,
+                                      /*allow_quant_ops*/ true));
+
+  auto flatbuffer_mv2 =
+      tflite::FlatBufferModel::BuildFromFile(model_name_mv2.c_str());
+  GraphFloat32 graph_mv2;
+  RETURN_IF_ERROR(BuildFromFlatBuffer(*flatbuffer_mv2, op_resolver, &graph_mv2,
+                                      /*allow_quant_ops*/ true));
+
+  Environment env;
+  RETURN_IF_ERROR(CreateEnvironment(&env));
+  const auto& gpu_info = env.GetDevicePtr()->GetInfo();
+
+  auto supported_storages = GetSupportedStorages(gpu_info);
+  if (supported_storages.empty()) {
+    return absl::UnimplementedError("No solution for this device");
+  }
+  TensorStorageType storage_type = *supported_storages.begin();
+  if (gpu_info.IsAdreno()) {
+    if (supported_storages.find(TensorStorageType::TEXTURE_2D) !=
+        supported_storages.end()) {
+      storage_type = TensorStorageType::TEXTURE_2D;
+    } else if (supported_storages.find(TensorStorageType::IMAGE_BUFFER) !=
+               supported_storages.end()) {
+      storage_type = TensorStorageType::IMAGE_BUFFER;
+    }
+  }
+  if (gpu_info.IsMali()) {
+    if (supported_storages.find(TensorStorageType::TEXTURE_2D) !=
+        supported_storages.end()) {
+      storage_type = TensorStorageType::TEXTURE_2D;
+    } else if (supported_storages.find(TensorStorageType::BUFFER) !=
+               supported_storages.end()) {
+      storage_type = TensorStorageType::BUFFER;
+    }
+  }
+
+  CreateGpuModelInfo create_info_mv1;
+  create_info_mv1.precision = env.IsSupported(CalculationsPrecision::F16)
+                                  ? CalculationsPrecision::F16
+                                  : CalculationsPrecision::F32;
+  create_info_mv1.storage_type = storage_type;
+  create_info_mv1.hints.Add(ModelHints::kAllowSpecialKernels);
+
+  CreateGpuModelInfo create_info_mv2 = create_info_mv1;
+
+  Tensor input_224_224, output_mv1, output_mv2;
+  auto data_type = DeduceDataTypeFromPrecision(create_info_mv1.precision);
+  RETURN_IF_ERROR(CreateTensor(
+      env.context(),
+      CreateHwcTensorDescriptor(data_type, TensorStorageType::TEXTURE_2D,
+                                HWC(224, 224, 3)),
+      &input_224_224));
+  RETURN_IF_ERROR(
+      CreateTensor(env.context(),
+                   CreateHwcTensorDescriptor(
+                       data_type, TensorStorageType::BUFFER, HWC(1, 1, 1001)),
+                   &output_mv1));
+  RETURN_IF_ERROR(
+      CreateTensor(env.context(),
+                   CreateHwcTensorDescriptor(
+                       data_type, TensorStorageType::BUFFER, HWC(1, 1, 1001)),
+                   &output_mv2));
+
+  create_info_mv1.external_immutable_tensors = {
+      {graph_mv1.inputs()[0]->id, &input_224_224},
+      {graph_mv1.outputs()[0]->id, &output_mv1},
+  };
+  create_info_mv2.external_immutable_tensors = {
+      {graph_mv2.inputs()[0]->id, &input_224_224},
+      {graph_mv2.outputs()[0]->id, &output_mv2},
+  };
+
+  RETURN_IF_ERROR(RunGraphTransformsForGpuModel(&graph_mv1));
+  GpuModel gpu_model_mv1;
+  RETURN_IF_ERROR(
+      GraphToGpuModel(graph_mv1, create_info_mv1, gpu_info, &gpu_model_mv1));
+  uint64_t total_size_mv1 = 0;
+  RETURN_IF_ERROR(GetTotalBufferSizeForTensors(gpu_model_mv1, create_info_mv1,
+                                               gpu_info, &total_size_mv1));
+
+  RETURN_IF_ERROR(RunGraphTransformsForGpuModel(&graph_mv2));
+  GpuModel gpu_model_mv2;
+  RETURN_IF_ERROR(
+      GraphToGpuModel(graph_mv2, create_info_mv2, gpu_info, &gpu_model_mv2));
+  uint64_t total_size_mv2 = 0;
+  RETURN_IF_ERROR(GetTotalBufferSizeForTensors(gpu_model_mv2, create_info_mv2,
+                                               gpu_info, &total_size_mv2));
+
+  uint64_t total_size = std::max(total_size_mv1, total_size_mv2);
+
+  Buffer shared_buffer;
+  RETURN_IF_ERROR(
+      CreateReadWriteBuffer(total_size, &env.context(), &shared_buffer));
+  InferenceContext context_mv1;
+  RETURN_IF_ERROR(context_mv1.InitFromGpuModel(create_info_mv1, &gpu_model_mv1,
+                                               &env, nullptr, &shared_buffer));
+
+  InferenceContext context_mv2;
+  RETURN_IF_ERROR(context_mv2.InitFromGpuModel(create_info_mv2, &gpu_model_mv2,
+                                               &env, nullptr, &shared_buffer));
+
+  {  // profiling mv1
+    auto* queue = env.profiling_queue();
+    ProfilingInfo profiling_info;
+    RETURN_IF_ERROR(context_mv1.Profile(queue, &profiling_info));
+    std::cout << profiling_info.GetDetailedReport() << std::endl;
+  }
+  {  // profiling mv2
+    auto* queue = env.profiling_queue();
+    ProfilingInfo profiling_info;
+    RETURN_IF_ERROR(context_mv2.Profile(queue, &profiling_info));
+    std::cout << profiling_info.GetDetailedReport() << std::endl;
+  }
+
+  {
+    const uint64_t runtime_mem_bytes =
+        context_mv1.GetSizeOfMemoryAllocatedForIntermediateTensors();
+    std::cout << "Memory for intermediate tensors - "
+              << runtime_mem_bytes / 1024.0 / 1024.0 << " MB" << std::endl;
+    const uint64_t const_mem_bytes = context_mv1.GetConstantTensorsSize();
+    std::cout << "Memory for constant tensors - "
+              << const_mem_bytes / 1024.0 / 1024.0 << " MB" << std::endl;
+    std::cout << "Total tensors memory(const + intermediate) - "
+              << (const_mem_bytes + runtime_mem_bytes) / 1024.0 / 1024.0
+              << " MB" << std::endl;
+  }
+  {
+    const uint64_t runtime_mem_bytes =
+        context_mv2.GetSizeOfMemoryAllocatedForIntermediateTensors();
+    std::cout << "Memory for intermediate tensors - "
+              << runtime_mem_bytes / 1024.0 / 1024.0 << " MB" << std::endl;
+    const uint64_t const_mem_bytes = context_mv2.GetConstantTensorsSize();
+    std::cout << "Memory for constant tensors - "
+              << const_mem_bytes / 1024.0 / 1024.0 << " MB" << std::endl;
+    std::cout << "Total tensors memory(const + intermediate) - "
+              << (const_mem_bytes + runtime_mem_bytes) / 1024.0 / 1024.0
+              << " MB" << std::endl;
+  }
+
+  const uint64_t runtime_mem_bytes = shared_buffer.GetMemorySizeInBytes();
+  const uint64_t inout_mem_bytes = input_224_224.GetMemorySizeInBytes() +
+                                   output_mv1.GetMemorySizeInBytes() +
+                                   output_mv2.GetMemorySizeInBytes();
+  std::cout
+      << "Total consumed memory size(2 models) for intermediate tensors - "
+      << (runtime_mem_bytes + inout_mem_bytes) / 1024.0 / 1024.0 << " MB"
+      << std::endl;
+  const uint64_t total_constant_size = context_mv1.GetConstantTensorsSize() +
+                                       context_mv2.GetConstantTensorsSize();
+  std::cout << "Total consumed memory size(2 models, runtime + constant) - "
+            << (runtime_mem_bytes + inout_mem_bytes + total_constant_size) /
+                   1024.0 / 1024.0
+            << " MB" << std::endl;
+
+  return absl::OkStatus();
+}
+
+}  // namespace cl
+}  // namespace gpu
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  if (argc <= 2) {
+    std::cerr << "Expected 2 model path as arguments.";
+    return -1;
+  }
+
+  auto load_status = tflite::gpu::cl::LoadOpenCL();
+  if (!load_status.ok()) {
+    std::cerr << load_status.message();
+    return -1;
+  }
+
+  auto run_status = tflite::gpu::cl::RunSample(argv[1], argv[2]);
+  if (!run_status.ok()) {
+    std::cerr << run_status.message();
+    return -1;
+  }
+
+  return EXIT_SUCCESS;
+}

tensorflow/lite/delegates/gpu/cl/testing/performance_profiling.cc

@@ -0,0 +1,407 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <algorithm>
+#include <chrono>  // NOLINT(build/c++11)
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include "absl/flags/flag.h"
+#include "absl/flags/parse.h"
+#include "absl/time/time.h"
+#include <CL/cl.h>
+#include <CL/cl_ext.h>
+#include <CL/cl_platform.h>
+#include "tensorflow/lite/core/kernels/register.h"
+#include "tensorflow/lite/delegates/gpu/cl/cl_command_buffer.h"
+#include "tensorflow/lite/delegates/gpu/cl/environment.h"
+#include "tensorflow/lite/delegates/gpu/cl/inference_context.h"
+#include "tensorflow/lite/delegates/gpu/cl/opencl_wrapper.h"
+#include "tensorflow/lite/delegates/gpu/common/model.h"
+#include "tensorflow/lite/delegates/gpu/common/model_builder.h"
+#include "tensorflow/lite/delegates/gpu/common/status.h"
+
+ABSL_FLAG(int, num_tests, 10, "Number of benchmark tests");
+ABSL_FLAG(int, num_runs_per_test, 0,
+          "Number of runs per benchmark test. Use 0 for default");
+ABSL_FLAG(bool, benchmark_command_buffer, true, "Run command buffer benchmark");
+
+namespace tflite {
+namespace gpu {
+namespace cl {
+
+absl::Status RunPredefinedLayoutSample(const std::string& model_name) {
+  auto flatbuffer = tflite::FlatBufferModel::BuildFromFile(model_name.c_str());
+  GraphFloat32 graph_cl;
+  ops::builtin::BuiltinOpResolver op_resolver;
+  RETURN_IF_ERROR(BuildFromFlatBuffer(*flatbuffer, op_resolver, &graph_cl,
+                                      /*allow_quant_ops=*/true));
+
+  Environment env;
+  RETURN_IF_ERROR(CreateEnvironment(&env));
+
+  CreateGpuModelInfo create_info;
+  create_info.precision = env.IsSupported(CalculationsPrecision::F16)
+                              ? CalculationsPrecision::F16
+                              : CalculationsPrecision::F32;
+  create_info.storage_type = GetFastestStorageType(env.device().GetInfo());
+  create_info.hints.Add(ModelHints::kAllowSpecialKernels);
+  {
+    // Example of adding predefined descriptor
+    // Assumed that graph has first input with batch = 1.
+    auto data_type = DeduceDataTypeFromPrecision(create_info.precision);
+    create_info.predefined[graph_cl.inputs()[0]->id] =
+        TensorDescriptor{data_type, TensorStorageType::BUFFER, Layout::HWC};
+  }
+  std::cout << "Precision: " << ToString(create_info.precision) << std::endl;
+  std::cout << "Storage type: " << ToString(create_info.storage_type)
+            << std::endl;
+  InferenceContext context;
+  RETURN_IF_ERROR(
+      context.InitFromGraphWithTransforms(create_info, &graph_cl, &env));
+
+  // After initialization we can receive input tensor
+  // in_ten will have TensorStorageType::BUFFER storage type
+  Tensor* in_ten = context.GetTensor(graph_cl.inputs()[0]->id);
+  if (in_ten->GetStorageType() != TensorStorageType::BUFFER) {
+    return absl::InternalError("Failed preconditiion");
+  }
+
+  RETURN_IF_ERROR(context.AddToQueue(env.queue()));
+
+  std::cout << "Finished RunPredefinedLayoutSample." << std::endl;
+
+  return absl::OkStatus();
+}
+
+absl::Status RunExternalImmutableSample(const std::string& model_name) {
+  auto flatbuffer = tflite::FlatBufferModel::BuildFromFile(model_name.c_str());
+  GraphFloat32 graph_cl;
+  ops::builtin::BuiltinOpResolver op_resolver;
+  RETURN_IF_ERROR(BuildFromFlatBuffer(*flatbuffer, op_resolver, &graph_cl,
+                                      /*allow_quant_ops*/ true));
+
+  Environment env;
+  RETURN_IF_ERROR(CreateEnvironment(&env));
+
+  CreateGpuModelInfo create_info;
+  create_info.precision = env.IsSupported(CalculationsPrecision::F16)
+                              ? CalculationsPrecision::F16
+                              : CalculationsPrecision::F32;
+  create_info.storage_type = GetFastestStorageType(env.device().GetInfo());
+  create_info.hints.Add(ModelHints::kAllowSpecialKernels);
+  // Example of external immutable tensors:
+  std::vector<Tensor> outputs(graph_cl.outputs().size());
+  for (int i = 0; i < graph_cl.outputs().size(); ++i) {
+    // Assumed that graph outputs have batch size = 1.
+    auto data_type = DeduceDataTypeFromPrecision(create_info.precision);
+    TensorDescriptor required_tensor_desc = TensorDescriptor{
+        data_type, TensorStorageType::TEXTURE_ARRAY, Layout::HWC};
+    required_tensor_desc.SetBHWCShape(graph_cl.outputs()[i]->tensor.shape);
+    RETURN_IF_ERROR(
+        CreateTensor(env.context(), required_tensor_desc, &outputs[i]));
+    create_info.external_immutable_tensors[graph_cl.outputs()[i]->id] =
+        &outputs[i];
+  }
+  std::cout << "Precision: " << ToString(create_info.precision) << std::endl;
+  std::cout << "Storage type: " << ToString(create_info.storage_type)
+            << std::endl;
+  InferenceContext context;
+  RETURN_IF_ERROR(
+      context.InitFromGraphWithTransforms(create_info, &graph_cl, &env));
+
+  RETURN_IF_ERROR(context.AddToQueue(env.queue()));
+
+  // outputs can be used here. But AddToQueue do not have cpu
+  // syncronization.
+  RETURN_IF_ERROR(env.queue()->WaitForCompletion());
+
+  TensorDescriptor desc;
+  RETURN_IF_ERROR(outputs[0].ToDescriptor(&desc, env.queue()));
+  TensorFloat32 cpu_tensor;
+  desc.DownloadData(&cpu_tensor);
+  std::cout << "First tensor data at index 0 - " << cpu_tensor.data[0]
+            << std::endl;
+
+  return absl::OkStatus();
+}
+
+absl::Status RunSerializedTest(const std::string& model_name) {
+  auto flatbuffer = tflite::FlatBufferModel::BuildFromFile(model_name.c_str());
+  GraphFloat32 graph_cl;
+  ops::builtin::BuiltinOpResolver op_resolver;
+  RETURN_IF_ERROR(BuildFromFlatBuffer(*flatbuffer, op_resolver, &graph_cl,
+                                      /*allow_quant_ops*/ true));
+
+  Environment env;
+  RETURN_IF_ERROR(CreateEnvironment(&env));
+
+  CreateGpuModelInfo create_info;
+  create_info.precision = env.IsSupported(CalculationsPrecision::F16)
+                              ? CalculationsPrecision::F16
+                              : CalculationsPrecision::F32;
+  create_info.storage_type = GetFastestStorageType(env.device().GetInfo());
+  create_info.hints.Add(ModelHints::kAllowSpecialKernels);
+
+  {  // calculating time without building serialized model
+    InferenceContext test_context;
+    const auto start = std::chrono::high_resolution_clock::now();
+    RETURN_IF_ERROR(
+        test_context.InitFromGraphWithTransforms(create_info, &graph_cl, &env));
+    const auto end = std::chrono::high_resolution_clock::now();
+    const double total_time_ms = (end - start).count() * 1e-6f;
+    std::cout << "Inference context initialization total time - "
+              << total_time_ms << "ms" << std::endl;
+  }
+  InferenceContext context;
+  std::vector<uint8_t> serialized_model;
+  RETURN_IF_ERROR(context.InitFromGraphWithTransforms(create_info, &graph_cl,
+                                                      &env, &serialized_model));
+
+  std::vector<TensorFloat32> src_tensors(graph_cl.inputs().size());
+  for (int i = 0; i < graph_cl.inputs().size(); ++i) {
+    src_tensors[i].id = graph_cl.inputs()[i]->id;
+    src_tensors[i].shape = graph_cl.inputs()[i]->tensor.shape;
+    src_tensors[i].data.resize(src_tensors[i].shape.DimensionsProduct());
+    for (int j = 0; j < src_tensors[i].data.size(); ++j) {
+      src_tensors[i].data[j] = std::sin(j);
+    }
+  }
+  for (int i = 0; i < graph_cl.inputs().size(); ++i) {
+    RETURN_IF_ERROR(context.SetInputTensor(graph_cl.inputs()[i]->id,
+                                           src_tensors[i], env.queue()));
+  }
+  RETURN_IF_ERROR(context.AddToQueue(env.queue()));
+  RETURN_IF_ERROR(env.queue()->WaitForCompletion());
+
+  std::vector<TensorFloat32> dst_tensors(graph_cl.outputs().size());
+  for (int i = 0; i < graph_cl.outputs().size(); ++i) {
+    RETURN_IF_ERROR(context.GetOutputTensor(graph_cl.outputs()[i]->id,
+                                            env.queue(), &dst_tensors[i]));
+  }
+
+  Environment env_v2;
+  RETURN_IF_ERROR(CreateEnvironment(&env_v2));
+  InferenceContext serialized_context;
+  {
+    const auto start = std::chrono::high_resolution_clock::now();
+    RETURN_IF_ERROR(
+        serialized_context.RestoreDeserialized(serialized_model, &env_v2));
+    const auto end = std::chrono::high_resolution_clock::now();
+    const double total_time_ms = (end - start).count() * 1e-6f;
+    std::cout << "Serialized inference context initialization total time - "
+              << total_time_ms << "ms" << std::endl;
+  }
+  for (int i = 0; i < graph_cl.inputs().size(); ++i) {
+    RETURN_IF_ERROR(serialized_context.SetInputTensor(
+        graph_cl.inputs()[i]->id, src_tensors[i], env_v2.queue()));
+  }
+
+  RETURN_IF_ERROR(serialized_context.AddToQueue(env_v2.queue()));
+  RETURN_IF_ERROR(env_v2.queue()->WaitForCompletion());
+
+  std::vector<TensorFloat32> dst_tensors_v2(graph_cl.outputs().size());
+  for (int i = 0; i < graph_cl.outputs().size(); ++i) {
+    RETURN_IF_ERROR(serialized_context.GetOutputTensor(
+        graph_cl.outputs()[i]->id, env_v2.queue(), &dst_tensors_v2[i]));
+  }
+
+  for (int i = 0; i < graph_cl.outputs().size(); ++i) {
+    if (dst_tensors[i].data.size() != dst_tensors_v2[i].data.size()) {
+      std::cout << "Different sizes for " << i << " output tensor" << std::endl;
+      break;
+    }
+    for (int j = 0; j < dst_tensors[i].data.size(); ++j) {
+      if (dst_tensors[i].data[j] != dst_tensors_v2[i].data[j]) {
+        std::cout << "Different elements for " << j << " element in " << i
+                  << " tensor: " << dst_tensors[i].data[j] << " - "
+                  << dst_tensors_v2[i].data[j] << std::endl;
+        break;
+      }
+    }
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status RunCommandBufferSample(int num_tests, double model_time_ms,
+                                    Environment* env,
+                                    InferenceContext* context) {
+  if (!env->device().GetInfo().SupportsExtension("cl_khr_command_buffer")) {
+    return absl::OkStatus();
+  }
+
+  int num_cbs = 3;
+  int num_inferences_in_cb = std::max(1.0, 100.0 / model_time_ms);
+  std::vector<CLCommandBuffer> cbs(num_cbs);
+  for (auto& cb : cbs) {
+    RETURN_IF_ERROR(cb.Init(env->queue(), /*simultaneous_use=*/false));
+    for (int i = 0; i < num_inferences_in_cb; ++i) {
+      RETURN_IF_ERROR(context->AddToCommandBuffer(cb.GetCommandBuffer()));
+    }
+    RETURN_IF_ERROR(cb.Finalize());
+  }
+
+  for (int i = 0; i < num_tests; ++i) {
+    const auto start = std::chrono::high_resolution_clock::now();
+    for (auto& cb : cbs) {
+      RETURN_IF_ERROR(cb.Enqueue(env->queue()));
+    }
+    clFinish(env->queue()->queue());
+    const auto end = std::chrono::high_resolution_clock::now();
+    const double total_time_ms = (end - start).count() * 1e-6f;
+    const double average_inference_time =
+        total_time_ms / (num_cbs * num_inferences_in_cb);
+    std::cout << "Total time CB - " << average_inference_time << "ms"
+              << std::endl;
+  }
+  return absl::OkStatus();
+}
+
+absl::Status RunModelSample(const std::string& model_name) {
+  auto flatbuffer = tflite::FlatBufferModel::BuildFromFile(model_name.c_str());
+  GraphFloat32 graph_cl;
+  ops::builtin::BuiltinOpResolver op_resolver;
+  RETURN_IF_ERROR(BuildFromFlatBuffer(*flatbuffer, op_resolver, &graph_cl,
+                                      /*allow_quant_ops*/ true));
+
+  Environment env;
+  RETURN_IF_ERROR(CreateEnvironment(&env));
+
+  CreateGpuModelInfo create_info;
+  create_info.precision = env.IsSupported(CalculationsPrecision::F16)
+                              ? CalculationsPrecision::F16
+                              : CalculationsPrecision::F32;
+  create_info.storage_type = GetFastestStorageType(env.device().GetInfo());
+  create_info.hints.Add(ModelHints::kAllowSpecialKernels);
+  std::cout << "Precision: " << ToString(create_info.precision) << std::endl;
+  std::cout << "Storage type: " << ToString(create_info.storage_type)
+            << std::endl;
+  InferenceContext context;
+  const auto start_init = std::chrono::high_resolution_clock::now();
+  RETURN_IF_ERROR(
+      context.InitFromGraphWithTransforms(create_info, &graph_cl, &env));
+  const auto end_init = std::chrono::high_resolution_clock::now();
+  std::cout << "Graph initialization time: "
+            << (end_init - start_init).count() * 1e-6f << " ms." << std::endl;
+
+  auto* queue = env.profiling_queue();
+  ProfilingInfo profiling_info;
+  RETURN_IF_ERROR(context.Profile(queue, &profiling_info));
+  std::cout << profiling_info.GetDetailedReport() << std::endl;
+  const uint64_t runtime_mem_bytes =
+      context.GetSizeOfMemoryAllocatedForIntermediateTensors();
+  std::cout << "Memory for intermediate tensors - "
+            << runtime_mem_bytes / 1024.0 / 1024.0 << " MB" << std::endl;
+  const uint64_t const_mem_bytes = context.GetConstantTensorsSize();
+  std::cout << "Memory for constant tensors - "
+            << const_mem_bytes / 1024.0 / 1024.0 << " MB" << std::endl;
+  std::cout << "Total tensors memory(const + intermediate) - "
+            << (const_mem_bytes + runtime_mem_bytes) / 1024.0 / 1024.0 << " MB"
+            << std::endl;
+
+  const int num_tests = absl::GetFlag(FLAGS_num_tests);
+  const double model_time_ms =
+      absl::ToDoubleMilliseconds(profiling_info.GetTotalTime());
+  const int num_runs_per_sec =
+      std::max(1, static_cast<int>(1000.0f / model_time_ms));
+  int num_runs_per_test = absl::GetFlag(FLAGS_num_runs_per_test);
+  if (num_runs_per_test == 0) {
+    num_runs_per_test = num_runs_per_sec;
+  }
+
+  std::cout << "Start running model: "
+            << std::chrono::duration_cast<std::chrono::milliseconds>(
+                   std::chrono::system_clock::now().time_since_epoch())
+                   .count()
+            << std::endl;
+  for (int i = 0; i < num_tests; ++i) {
+    const auto start = std::chrono::high_resolution_clock::now();
+    for (int k = 0; k < num_runs_per_test; ++k) {
+      RETURN_IF_ERROR(context.AddToQueue(env.queue()));
+    }
+    RETURN_IF_ERROR(env.queue()->WaitForCompletion());
+    const auto end = std::chrono::high_resolution_clock::now();
+    const double total_time_ms = (end - start).count() * 1e-6f;
+    const double average_inference_time = total_time_ms / num_runs_per_test;
+    std::cout << "Total time - " << average_inference_time << "ms" << std::endl;
+  }
+  if (absl::GetFlag(FLAGS_benchmark_command_buffer)) {
+    RETURN_IF_ERROR(
+        RunCommandBufferSample(num_tests, model_time_ms, &env, &context));
+  }
+
+  std::cout << "Finished running model: "
+            << std::chrono::duration_cast<std::chrono::milliseconds>(
+                   std::chrono::system_clock::now().time_since_epoch())
+                   .count()
+            << std::endl;
+
+  return absl::OkStatus();
+}
+
+}  // namespace cl
+}  // namespace gpu
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  absl::ParseCommandLine(argc, argv);
+  if (argc <= 1) {
+    std::cerr << "Expected model path as second argument.";
+    return -1;
+  }
+
+  auto load_status = tflite::gpu::cl::LoadOpenCL();
+  if (!load_status.ok()) {
+    std::cerr << load_status.message();
+    return -1;
+  }
+
+  auto run_status = tflite::gpu::cl::RunModelSample(argv[1]);
+  if (!run_status.ok()) {
+    std::cerr << run_status.message();
+    return -1;
+  }
+
+  bool run_serialized_test = false;
+  if (run_serialized_test) {
+    run_status = tflite::gpu::cl::RunSerializedTest(argv[1]);
+    if (!run_status.ok()) {
+      std::cerr << run_status.message();
+      return -1;
+    }
+  }
+
+  bool run_with_external_immutable_tensors = false;
+  if (run_with_external_immutable_tensors) {
+    run_status = tflite::gpu::cl::RunExternalImmutableSample(argv[1]);
+    if (!run_status.ok()) {
+      std::cerr << run_status.message();
+      return -1;
+    }
+  }
+
+  bool run_with_predefined_layout = false;
+  if (run_with_predefined_layout) {
+    run_status = tflite::gpu::cl::RunPredefinedLayoutSample(argv[1]);
+    if (!run_status.ok()) {
+      std::cerr << run_status.message();
+      return -1;
+    }
+  }
+
+  return EXIT_SUCCESS;
+}

tensorflow/lite/delegates/gpu/cl/testing/run_delegate_testing.sh

@@ -0,0 +1,110 @@
+#!/bin/bash
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+shopt -s expand_aliases  # to work with commands aliases in .sh
+
+description="Delegate testing sample:
+Compares GPU backend vs TFLite CPU(speed/correctness).
+How to use:
+[-h or --help, print instructions]
+[-m or --model_path, path to the model in .tflite format]
+[-d or --device, select device](optional, if you have few connected devices)"
+
+model_path=""
+alias ADB='adb'
+host=""
+
+while [[ "$1" != "" ]]; do
+  case $1 in
+    -m | --model_path)
+      shift
+      model_path=$1
+      ;;
+    -d | --device)
+      shift
+      if [[ "$1" == "HOST" ]]
+      then
+      host="HOST"
+      fi
+      alias ADB='adb -s '$1''
+      ;;
+    -h | --help)
+      echo "$description"
+      exit
+      ;;
+  esac
+  shift
+done
+
+if [ "$model_path" = "" ]
+then
+echo "No model provided."
+echo "$description"
+exit
+fi
+
+SHELL_DIR=$(dirname "$0")
+BINARY_NAME=delegate_testing
+declare -a BUILD_CONFIG
+
+if [[ "$host" == "HOST" ]]
+then
+
+os_name=$(uname -s)
+if [[ "$os_name" == "Darwin" ]]; then
+BUILD_CONFIG=( --config=darwin_x86_64 -c opt )
+else
+BUILD_CONFIG=( -c opt )
+fi
+
+bazel build "${BUILD_CONFIG[@]}" --copt -DCL_DELEGATE_NO_GL //"$SHELL_DIR":"$BINARY_NAME"
+chmod +x bazel-bin/"$SHELL_DIR"/"$BINARY_NAME"
+./bazel-bin/"$SHELL_DIR"/"$BINARY_NAME" "$model_path"
+exit
+fi
+
+model_name=${model_path##*/}  # finds last token after '/'
+
+OPENCL_DIR=/data/local/tmp/delegate_testing/
+
+ADB shell mkdir -p $OPENCL_DIR
+
+ADB push "$model_path" "$OPENCL_DIR"
+
+abi_version=$(ADB shell getprop ro.product.cpu.abi | tr -d '\r')
+if [[ "$abi_version" == "armeabi-v7a" ]]; then
+#"32 bit ARM"
+BUILD_CONFIG=( --config=android_arm -c opt --copt=-fPIE --linkopt=-pie )
+elif [[ "$abi_version" == "arm64-v8a" ]]; then
+#"64 bit ARM"
+BUILD_CONFIG=( --config=android_arm64 -c opt )
+elif [[ "$abi_version" == "x86_64" ]]; then
+# x86_64
+BUILD_CONFIG=( --config=android_x86_64 -c opt )
+else
+echo "Error: Unknown processor ABI"
+exit 1
+fi
+
+bazel build "${BUILD_CONFIG[@]}" //$SHELL_DIR:$BINARY_NAME
+
+ADB push bazel-bin/$SHELL_DIR/$BINARY_NAME $OPENCL_DIR
+
+ADB shell chmod +x $OPENCL_DIR/$BINARY_NAME
+ADB shell "cd $OPENCL_DIR && ./$BINARY_NAME $model_name"
+
+# clean up files from device
+ADB shell rm -rf $OPENCL_DIR

tensorflow/lite/delegates/gpu/cl/testing/run_internal_api_samples.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+shopt -s expand_aliases  # to work with commands aliases in .sh
+
+description="Example of intetrnal api usage:
+How to use:
+[-h or --help, print instructions]
+[-m or --model_path, path to the model in .tflite format]
+[-d or --device, select device](optional, if you have few connected devices)"
+
+model_path=""
+alias ADB='adb'
+host=""
+
+while [[ "$1" != "" ]]; do
+  case $1 in
+    -m | --model_path)
+      shift
+      model_path=$1
+      ;;
+    -d | --device)
+      shift
+      if [[ "$1" == "HOST" ]]
+      then
+      host="HOST"
+      fi
+      alias ADB='adb -s '$1''
+      ;;
+    -h | --help)
+      echo "$description"
+      exit
+      ;;
+  esac
+  shift
+done
+
+if [ "$model_path" = "" ]
+then
+echo "No model provided."
+echo "$description"
+exit
+fi
+
+SHELL_DIR=$(dirname "$0")
+BINARY_NAME=internal_api_samples
+declare -a BUILD_CONFIG
+
+if [[ "$host" == "HOST" ]]
+then
+
+os_name=$(uname -s)
+if [[ "$os_name" == "Darwin" ]]; then
+BUILD_CONFIG=( --config=darwin_x86_64 -c opt )
+else
+BUILD_CONFIG=( -c opt )
+fi
+
+bazel build "${BUILD_CONFIG[@]}" --copt -DCL_DELEGATE_NO_GL //"$SHELL_DIR":"$BINARY_NAME"
+chmod +x bazel-bin/"$SHELL_DIR"/"$BINARY_NAME"
+./bazel-bin/"$SHELL_DIR"/"$BINARY_NAME" "$model_path"
+exit
+fi
+
+model_name=${model_path##*/}  # finds last token after '/'
+
+OPENCL_DIR=/data/local/tmp/internal_api_samples/
+
+ADB shell mkdir -p $OPENCL_DIR
+
+ADB push "$model_path" "$OPENCL_DIR"
+
+abi_version=$(ADB shell getprop ro.product.cpu.abi | tr -d '\r')
+if [[ "$abi_version" == "armeabi-v7a" ]]; then
+#"32 bit ARM"
+BUILD_CONFIG=( --config=android_arm -c opt --copt=-fPIE --linkopt=-pie )
+elif [[ "$abi_version" == "arm64-v8a" ]]; then
+#"64 bit ARM"
+BUILD_CONFIG=( --config=android_arm64 -c opt )
+elif [[ "$abi_version" == "x86_64" ]]; then
+# x86_64
+BUILD_CONFIG=( --config=android_x86_64 -c opt )
+else
+echo "Error: Unknown processor ABI"
+exit 1
+fi
+
+bazel build "${BUILD_CONFIG[@]}" --copt -DCL_DELEGATE_NO_GL //$SHELL_DIR:$BINARY_NAME
+
+ADB push bazel-bin/$SHELL_DIR/$BINARY_NAME $OPENCL_DIR
+
+ADB shell chmod +x $OPENCL_DIR/$BINARY_NAME
+ADB shell "cd $OPENCL_DIR && ./$BINARY_NAME $model_name"
+
+# clean up files from device
+ADB shell rm -rf $OPENCL_DIR

tensorflow/lite/delegates/gpu/cl/testing/run_memory_sharing_sample.sh

@@ -0,0 +1,123 @@
+#!/bin/bash
+# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+shopt -s expand_aliases  # to work with commands aliases in .sh
+
+description="Memory sharing sample:
+How to use:
+[-h or --help, print instructions]
+[-m1 or --model1_path, path to the mobilenet v1 model in .tflite format]
+[-m2 or --model2_path, path to the mobilenet v2 model in .tflite format]
+[-d or --device, select device](optional, if you have few connected devices)"
+
+model1_path=""
+model2_path=""
+alias ADB='adb'
+host=""
+
+while [[ "$1" != "" ]]; do
+  case $1 in
+    -m1 | --model1_path)
+      shift
+      model1_path=$1
+      ;;
+    -m2 | --model2_path)
+      shift
+      model2_path=$1
+      ;;
+    -d | --device)
+      shift
+      if [[ "$1" == "HOST" ]]
+      then
+      host="HOST"
+      fi
+      alias ADB='adb -s '$1''
+      ;;
+    -h | --help)
+      echo "$description"
+      exit
+      ;;
+  esac
+  shift
+done
+
+if [ "$model1_path" = "" ]
+then
+echo "No mobilenet v1 model provided."
+echo "$description"
+exit
+fi
+if [ "$model2_path" = "" ]
+then
+echo "No mobilenet v2 model provided."
+echo "$description"
+exit
+fi
+
+SHELL_DIR=$(dirname "$0")
+BINARY_NAME=memory_sharing_sample
+declare -a BUILD_CONFIG
+
+if [[ "$host" == "HOST" ]]
+then
+
+os_name=$(uname -s)
+if [[ "$os_name" == "Darwin" ]]; then
+BUILD_CONFIG=( --config=darwin_x86_64 -c opt )
+else
+BUILD_CONFIG=( -c opt )
+fi
+
+bazel build "${BUILD_CONFIG[@]}" //"$SHELL_DIR":"$BINARY_NAME"
+chmod +x bazel-bin/"$SHELL_DIR"/"$BINARY_NAME"
+./bazel-bin/"$SHELL_DIR"/"$BINARY_NAME" "$model1_path" "$model2_path"
+exit
+fi
+
+model1_name=${model1_path##*/}  # finds last token after '/'
+model2_name=${model2_path##*/}  # finds last token after '/'
+
+OPENCL_DIR=/data/local/tmp/memory_sharing_sample/
+
+ADB shell mkdir -p $OPENCL_DIR
+
+ADB push "$model1_path" "$OPENCL_DIR"
+ADB push "$model2_path" "$OPENCL_DIR"
+
+abi_version=$(ADB shell getprop ro.product.cpu.abi | tr -d '\r')
+if [[ "$abi_version" == "armeabi-v7a" ]]; then
+#"32 bit ARM"
+BUILD_CONFIG=( --config=android_arm -c opt --copt=-fPIE --linkopt=-pie )
+elif [[ "$abi_version" == "arm64-v8a" ]]; then
+#"64 bit ARM"
+BUILD_CONFIG=( --config=android_arm64 -c opt )
+elif [[ "$abi_version" == "x86_64" ]]; then
+# x86_64
+BUILD_CONFIG=( --config=android_x86_64 -c opt )
+else
+echo "Error: Unknown processor ABI"
+exit 1
+fi
+
+bazel build "${BUILD_CONFIG[@]}" //$SHELL_DIR:$BINARY_NAME
+
+ADB push bazel-bin/$SHELL_DIR/$BINARY_NAME $OPENCL_DIR
+
+ADB shell chmod +x $OPENCL_DIR/$BINARY_NAME
+ADB shell "cd $OPENCL_DIR && ./$BINARY_NAME $model1_name $model2_name"
+
+# clean up files from device
+ADB shell rm -rf $OPENCL_DIR

tensorflow/lite/delegates/gpu/cl/testing/run_performance_profiling.sh

@@ -0,0 +1,119 @@
+#!/bin/bash
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+shopt -s expand_aliases  # to work with commands aliases in .sh
+
+description="Performance profiling sample:
+Shows detailed per-layer time in ms for .tflite model.
+Uses OpenCL gpu events for measuring.
+Works good for Snapdragon(Adreno), Nvidia, Mali. For PowerVR the measurement results are not as precise.
+After detailed per-layer timing it measures model execution with CPU timer.
+Warning! CPU time can be much slower then time in real application on some devices on Android.
+This is because in Android, Activity-based processes can have higher priorities than binary processes.
+How to use:
+[-h or --help, print instructions]
+[-m or --model_path, path to the model in .tflite format]
+[-d or --device, select device](optional, if you have few connected devices)"
+
+model_path=""
+alias ADB='adb'
+host=""
+
+while [[ "$1" != "" ]]; do
+  case $1 in
+    -m | --model_path)
+      shift
+      model_path=$1
+      ;;
+    -d | --device)
+      shift
+      if [[ "$1" == "HOST" ]]
+      then
+      host="HOST"
+      fi
+      alias ADB='adb -s '$1''
+      ;;
+    -h | --help)
+      echo "$description"
+      exit
+      ;;
+    --)
+      shift
+      break
+  esac
+  shift
+done
+
+if [ "$model_path" = "" ]
+then
+echo "No model provided."
+echo "$description"
+exit
+fi
+
+SHELL_DIR=$(dirname "$0")
+BINARY_NAME=performance_profiling
+declare -a BUILD_CONFIG
+
+if [[ "$host" == "HOST" ]]
+then
+
+os_name=$(uname -s)
+if [[ "$os_name" == "Darwin" ]]; then
+BUILD_CONFIG=( --config=darwin_x86_64 -c opt )
+else
+BUILD_CONFIG=( -c opt )
+fi
+
+bazel build "${BUILD_CONFIG[@]}" //"$SHELL_DIR":"$BINARY_NAME"
+chmod +x bazel-bin/"$SHELL_DIR"/"$BINARY_NAME"
+./bazel-bin/"$SHELL_DIR"/"$BINARY_NAME" "$model_path"
+exit
+
+fi
+
+model_name=${model_path##*/}  # finds last token after '/'
+
+OPENCL_DIR=/data/local/tmp/profiling_inference/
+
+ADB shell mkdir -p $OPENCL_DIR
+
+ADB push "$model_path" "$OPENCL_DIR"
+
+abi_version=$(ADB shell getprop ro.product.cpu.abi | tr -d '\r')
+if [[ "$abi_version" == "armeabi-v7a" ]]; then
+#"32 bit ARM"
+BUILD_CONFIG=( --config=android_arm -c opt --copt=-fPIE --linkopt=-pie )
+elif [[ "$abi_version" == "arm64-v8a" ]]; then
+#"64 bit ARM"
+BUILD_CONFIG=( --config=android_arm64 -c opt )
+elif [[ "$abi_version" == "x86_64" ]]; then
+# x86_64
+BUILD_CONFIG=( --config=android_x86_64 -c opt )
+else
+echo "Error: Unknown processor ABI"
+exit 1
+fi
+
+bazel build "${BUILD_CONFIG[@]}" //$SHELL_DIR:$BINARY_NAME
+
+ADB push bazel-bin/$SHELL_DIR/$BINARY_NAME $OPENCL_DIR
+
+ADB shell chmod +x $OPENCL_DIR/$BINARY_NAME
+ADB shell "cd $OPENCL_DIR && ./$BINARY_NAME $model_name $@"
+
+# clean up files from device
+ADB shell rm -rf $OPENCL_DIR