tensorflow--tensorflow/tensorflow/lite/interpreter_options.h

/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
/// \file
///
/// Provides options to an interpreter.
///
#ifndef TENSORFLOW_LITE_INTERPRETER_OPTIONS_H_
#define TENSORFLOW_LITE_INTERPRETER_OPTIONS_H_

namespace tflite {

/// Options class for `Interpreter`.
/// WARNING: This is an experimental API and subject to change.
class InterpreterOptions {
 public:
  /// Preserving all intermediates tensors for debugging.
  /// WARNING: This is an experimental API and subject to change.
  void SetPreserveAllTensors(bool value = true) {
    experimental_preserve_all_tensors_ = value;
  }

  /// Returns if the `experimental_preserve_all_tensors_` feature is enabled.
  /// WARNING: This is an experimental API and subject to change.
  bool GetPreserveAllTensors() { return experimental_preserve_all_tensors_; }

  /// Force all intermediate dynamic tensors to be released once they are not
  /// used by the model. Please use this configuration with caution, since it
  /// might reduce the peak memory usage of the model at the cost of a slower
  /// inference speed.
  /// WARNING: This is an experimental API and subject to change.
  void SetEnsureDynamicTensorsAreReleased(bool value = true) {
    experimental_ensure_dynamic_tensors_are_released_ = value;
  }

  /// Returns if the `experimental_ensure_dynamic_tensors_are_released_` feature
  /// is enabled.
  /// WARNING: This is an experimental API and subject to change.
  bool GetEnsureDynamicTensorsAreReleased() {
    return experimental_ensure_dynamic_tensors_are_released_;
  }

  /// Use dynamic tensor allocation and deallocation method for large tensors
  /// instead of static memory planner. Dynamic tensors are allocated just
  /// before when they're needed and released when they're not needed anymore.
  /// It improves peak memory usage but there could be some latency impact. The
  /// value (in bytes, and default is 1024 * 1024) is used to determine large
  /// tensors.
  /// WARNING: This is an experimental API and subject to change.
  void OptimizeMemoryForLargeTensors(int value = 1 << 20) {
    if (value > 0) {
      experimental_optimize_memory_for_large_tensors_ = value;
      experimental_ensure_dynamic_tensors_are_released_ = true;
    }
  }

  /// Returns the size (in bytes) threshold for dynamic tensor allocation
  /// method. It returns zero if the feature is not enabled.
  /// WARNING: This is an experimental API and subject to change.
  int GetDynamicAllocationForLargeTensors() {
    return experimental_optimize_memory_for_large_tensors_;
  }

  // Returns true iff delegate clustering (i.e., reordering execution such that
  // the number of switches between non-delegated and delegated execution of
  // nodes is minimized) is disabled.
  // WARNING: This is an experimental API and subject to change.
  bool GetDisableDelegateClustering() {
    return experimental_disable_delegate_clustering_;
  }

  // If value == true, disable delegate clustering (see above), otherwise,
  // enable it.
  // WARNING: This is an experimental API and subject to change.
  void SetDisableDelegateClustering(bool value = true) {
    experimental_disable_delegate_clustering_ = value;
  }

  // If set to `true`, the CAST op will cache its output when its input is a
  // constant tensor.
  //
  // WARNING: This is an experimental API and subject to change.
  void SetCacheConstantCastOp(bool value) {
    experimental_cache_constant_cast_op_ = value;
  }

  // If `true`, the CAST op will cache its output when its input is a constant
  // tensor.
  //
  // WARNING: This is an experimental API and subject to change.
  bool GetCacheConstantCastOp() const {
    return experimental_cache_constant_cast_op_;
  }

  // Sets the StableHLO Composite op automatic inlining.
  //
  // WARNING: This is an experimental API and subject to change.
  void SetShloCompositeInlining(bool value) {
    experimental_shlo_composite_inlining_ = value;
  }

  // If `true`, the interpreter will try to inline StableHLO Composite
  // operations that haven't been picked up by a delegate.
  //
  // WARNING: This is an experimental API and subject to change.
  bool GetShloCompositeInlining() const {
    return experimental_shlo_composite_inlining_;
  }

  // Controls to update Tensor names with Signature input & output names.
  //
  // WARNING: This is an experimental API and subject to change.
  void SetUseSignatureTensorNames(bool value) {
    experimental_use_signature_tensor_names_ = value;
  }

  // If `true`, the interpreter will use the tensor names from the Signature
  // inputs and outputs.
  //
  // WARNING: This is an experimental API and subject to change.
  bool GetUseSignatureTensorNames() const {
    return experimental_use_signature_tensor_names_;
  }

  // If `true`, per-channel quantization zero-points that are all identical
  // will be compressed into a single value to reduce memory usage.
  //
  // WARNING: This is an experimental API and subject to change.
  void SetCompressQuantizationZeroPoints(bool value) {
    experimental_compress_quantization_zero_points_ = value;
  }

  // Returns whether quantization zero-points compression is enabled.
  //
  // WARNING: This is an experimental API and subject to change.
  bool GetCompressQuantizationZeroPoints() const {
    return experimental_compress_quantization_zero_points_;
  }

  // If value == true, disable node fusion (clustering) when partitioning
  // delegated graphs, forcing single-operator delegated subsets.
  // WARNING: This is an experimental API and subject to change.
  void SetDisableDelegateNodeFusion(bool value) {
    experimental_disable_delegate_node_fusion_ = value;
  }

  bool GetDisableDelegateNodeFusion() const {
    return experimental_disable_delegate_node_fusion_;
  }

  // If value == true, force TFLite to profile delegated nodes even if
  // the delegate supports per-operator internal profiling.
  // WARNING: This is an experimental API and subject to change.
  void SetForceDelegateNodeProfiling(bool value) {
    experimental_force_delegate_node_profiling_ = value;
  }

  bool GetForceDelegateNodeProfiling() const {
    return experimental_force_delegate_node_profiling_;
  }

 private:
  bool experimental_preserve_all_tensors_ = false;
  bool experimental_ensure_dynamic_tensors_are_released_ = false;
  int experimental_optimize_memory_for_large_tensors_ = 0;
  bool experimental_disable_delegate_clustering_ = false;
  bool experimental_cache_constant_cast_op_ = false;
  bool experimental_shlo_composite_inlining_ = false;
  bool experimental_use_signature_tensor_names_ = false;
  bool experimental_compress_quantization_zero_points_ = false;
  bool experimental_disable_delegate_node_fusion_ = false;
  bool experimental_force_delegate_node_profiling_ = false;
};

}  // namespace tflite

#endif  // TENSORFLOW_LITE_INTERPRETER_OPTIONS_H_