chore: import upstream snapshot with attribution

tensorflow/lite/experimental/microfrontend/BUILD

@@ -0,0 +1,148 @@
+# TensorFlow ops for audio front-end processing.
+
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
+load("@rules_cc//cc:cc_test.bzl", "cc_test")
+load("@xla//third_party/rules_python/python:py_library.bzl", "py_library")
+load(
+    "//tensorflow:tensorflow.bzl",
+    "tf_copts",
+    "tf_custom_op_library",
+    "tf_gen_op_libs",
+    "tf_gen_op_wrapper_py",
+    "tf_opts_nortti_if_android",
+)
+load("//tensorflow:tensorflow.default.bzl", "tf_custom_op_py_strict_library", "tf_py_strict_test")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_visibility = ["//visibility:public"],
+    licenses = ["notice"],
+)
+
+cc_library(
+    name = "audio_microfrontend",
+    srcs = ["audio_microfrontend.cc"],
+    hdrs = ["audio_microfrontend.h"],
+    deps = [
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/c:c_api_types",
+        "//tensorflow/lite/c:common",
+        "//tensorflow/lite/experimental/microfrontend/lib:frontend",
+        "//tensorflow/lite/kernels:kernel_util",
+        "//tensorflow/lite/kernels/internal:compatibility",
+        "//tensorflow/lite/kernels/internal:reference",
+        "@flatbuffers",
+    ],
+)
+
+cc_library(
+    name = "audio_microfrontend_op_lib",
+    srcs = ["ops/audio_microfrontend_op.cc"],
+    copts = tf_copts(android_optimization_level_override = None) + tf_opts_nortti_if_android() + [
+        "-Wno-narrowing",
+        "-Wno-sign-compare",
+        "-Wno-overloaded-virtual",
+    ] + select({
+        "//tensorflow:android": [
+            # Selective registration uses constexprs with recursive
+            # string comparisons; that can lead to compiler errors, so
+            # we increase the constexpr recursion depth.
+            "-fconstexpr-depth=1024",
+            "-Oz",
+        ],
+        "//conditions:default": [],
+    }),
+    deps = [
+        "//tensorflow/lite/experimental/microfrontend/lib:frontend",
+    ] + select({
+        "//tensorflow:android": [
+            "//tensorflow/core:portable_tensorflow_lib_lite",
+        ],
+        "//conditions:default": [
+            "//tensorflow/core:framework",
+            "//tensorflow/core:lib",
+        ],
+    }),
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "audio_microfrontend_test",
+    size = "small",
+    srcs = ["audio_microfrontend_test.cc"],
+    tags = ["tflite_not_portable_ios"],
+    deps = [
+        ":audio_microfrontend",
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/core:framework",
+        "//tensorflow/lite/kernels:test_util",
+        "//tensorflow/lite/schema:schema_fbs",
+        "@com_google_googletest//:gtest_main",
+        "@flatbuffers",
+    ],
+)
+
+tf_custom_op_library(
+    name = "python/ops/_audio_microfrontend_op.so",
+    srcs = [
+        "ops/audio_microfrontend_op.cc",
+    ],
+    deps = [
+        "//tensorflow/lite/experimental/microfrontend/lib:frontend",
+    ],
+)
+
+tf_gen_op_libs(
+    op_lib_names = ["audio_microfrontend_op"],
+    deps = [
+        "//tensorflow/core:lib",
+        "//tensorflow/lite/experimental/microfrontend/lib:frontend",
+    ],
+)
+
+tf_gen_op_wrapper_py(
+    name = "audio_microfrontend_op",
+    extra_py_deps = [
+        "//tensorflow/python:pywrap_tfe",
+        "//tensorflow/python/util:dispatch",
+        "//tensorflow/python/util:deprecation",
+        "//tensorflow/python/util:tf_export",
+    ],
+    py_lib_rule = py_library,
+    deps = [":audio_microfrontend_op_op_lib"],
+)
+
+tf_custom_op_py_strict_library(
+    name = "audio_microfrontend_py",
+    srcs = [
+        "python/ops/audio_microfrontend_op.py",
+    ],
+    dso = [":python/ops/_audio_microfrontend_op.so"],
+    kernels = [
+        ":audio_microfrontend_op_op_lib",
+    ],
+    deps = [
+        ":audio_microfrontend_op",
+        "//tensorflow/python/framework:dtypes",
+        "//tensorflow/python/framework:load_library",
+        "//tensorflow/python/framework:ops",
+        "//tensorflow/python/ops:array_ops",
+        "//tensorflow/python/ops:control_flow_ops",
+        "//tensorflow/python/ops:linalg_ops",
+        "//tensorflow/python/ops:math_ops",
+        "//tensorflow/python/platform:client_testlib",
+        "//tensorflow/python/platform:resource_loader",
+    ],
+)
+
+tf_py_strict_test(
+    name = "audio_microfrontend_op_test",
+    size = "small",
+    srcs = ["python/kernel_tests/audio_microfrontend_op_test.py"],
+    exec_properties = {"cpp_link.mem": "16g"},
+    deps = [
+        ":audio_microfrontend_py",
+        "//tensorflow:tensorflow_py",
+        "//tensorflow/python/framework:ops",
+    ],
+)

tensorflow/lite/experimental/microfrontend/README.md

@@ -0,0 +1,75 @@
+# Audio "frontend" TensorFlow operations for feature generation
+The most common module used by most audio processing modules is the feature
+generation (also called frontend). It receives raw audio input, and produces
+filter banks (a vector of values).
+
+More specifically the audio signal goes through a pre-emphasis filter
+(optionally); then gets sliced into (overlapping) frames and a window function
+is applied to each frame; afterwards, we do a Fourier transform on each frame
+(or more specifically a Short-Time Fourier Transform) and calculate the power
+spectrum; and subsequently compute the filter banks.
+
+## Operations
+Here we provide implementations for both a TensorFlow and TensorFlow Lite
+operations that encapsulate the functionality of the audio frontend.
+
+Both frontend Ops receives audio data and produces as many unstacked frames
+(filterbanks) as audio is passed in, according to the configuration.
+
+The processing uses a lightweight library to perform:
+
+1. A slicing window function
+2. Short-time FFTs
+3. Filterbank calculations
+4. Noise reduction
+5. Auto Gain Control
+6. Logarithmic scaling
+
+Please refer to the Op's documentation for details on the different
+configuration parameters.
+
+However, it is important to clarify the contract of the Ops:
+
+> *A frontend OP will produce as many unstacked frames as possible with the
+> given audio input.*
+
+This means:
+
+1. The output is a rank-2 Tensor, where each row corresponds to the
+  sequence/time dimension, and each column is the feature dimension).
+2. It is expected that the Op will receive the right input (in terms of
+  positioning in the audio stream, and the amount), as needed to produce the
+  expected output.
+3. Thus, any logic to slice, cache, or otherwise rearrange the input and/or
+  output of the operation must be handled externally in the graph.
+
+For example, a 200ms audio input will produce an output tensor of shape
+`[18, num_channels]`, when configured with a `window_size=25ms`, and
+`window_step=10ms`. The reason being that when reaching the point in the
+audio at 180ms there’s not enough audio to construct a complete window.
+
+Due to both functional and efficiency reasons, we provide the following
+functionality related to input processing:
+
+**Padding.** A boolean flag `zero_padding` that indicates whether to pad the
+audio with zeros such that we generate output frames based on the `window_step`.
+This means that in the example above, we would generate a tensor of shape
+`[20, num_channels]` by adding enough zeros such that we step over all the
+available audio and still be able to create complete windows of audio (some of
+the window will just have zeros; in the example above, frame 19 and 20 will have
+the equivalent of 5 and 15ms full of zeros respectively).
+
+<!-- TODO
+Stacking. An integer that indicates how many contiguous frames to stack in the output tensor’s first dimension, such that the tensor is shaped [-1, stack_size * num_channels]. For example, if the stack_size is 3, the example above would produce an output tensor shaped [18, 120] is padding is false, and [20, 120] is padding is set to true.
+-->
+
+**Striding.** An integer `frame_stride` that indicates the striding step used to
+generate the output tensor, thus determining the second dimension. In the
+example above, with a `frame_stride=3`, the output tensor would have a shape of
+`[6, 120]` when `zero_padding` is set to false, and `[7, 120]` when
+`zero_padding` is set to true.
+
+<!-- TODO
+Note we would not expect the striding step to be larger than the stack_size
+(should we enforce that?).
+-->

tensorflow/lite/experimental/microfrontend/audio_microfrontend.cc

@@ -0,0 +1,218 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+
+#include "flatbuffers/flexbuffers.h"  // from @flatbuffers
+#include "tensorflow/lite/c/c_api_types.h"
+#include "tensorflow/lite/c/common.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/frontend.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/frontend_util.h"
+#include "tensorflow/lite/kernels/internal/compatibility.h"
+#include "tensorflow/lite/kernels/internal/tensor_ctypes.h"
+#include "tensorflow/lite/kernels/kernel_util.h"
+
+namespace tflite {
+namespace ops {
+namespace custom {
+namespace audio_microfrontend {
+
+constexpr int kInputTensor = 0;
+constexpr int kOutputTensor = 0;
+
+typedef struct {
+  int sample_rate;
+  FrontendState* state;
+  int left_context;
+  int right_context;
+  int frame_stride;
+  bool zero_padding;
+  int out_scale;
+  bool out_float;
+} TfLiteAudioMicrofrontendParams;
+
+void* Init(TfLiteContext* context, const char* buffer, size_t length) {
+  auto* data = new TfLiteAudioMicrofrontendParams;
+
+  const uint8_t* buffer_t = reinterpret_cast<const uint8_t*>(buffer);
+  const flexbuffers::Map& m = flexbuffers::GetRoot(buffer_t, length).AsMap();
+
+  data->sample_rate = m["sample_rate"].AsInt32();
+
+  struct FrontendConfig config;
+  config.window.size_ms = m["window_size"].AsInt32();
+  config.window.step_size_ms = m["window_step"].AsInt32();
+  config.filterbank.num_channels = m["num_channels"].AsInt32();
+  config.filterbank.upper_band_limit = m["upper_band_limit"].AsFloat();
+  config.filterbank.lower_band_limit = m["lower_band_limit"].AsFloat();
+  config.noise_reduction.smoothing_bits = m["smoothing_bits"].AsInt32();
+  config.noise_reduction.even_smoothing = m["even_smoothing"].AsFloat();
+  config.noise_reduction.odd_smoothing = m["odd_smoothing"].AsFloat();
+  config.noise_reduction.min_signal_remaining =
+      m["min_signal_remaining"].AsFloat();
+  config.pcan_gain_control.enable_pcan = m["enable_pcan"].AsBool();
+  config.pcan_gain_control.strength = m["pcan_strength"].AsFloat();
+  config.pcan_gain_control.offset = m["pcan_offset"].AsFloat();
+  config.pcan_gain_control.gain_bits = m["gain_bits"].AsInt32();
+  config.log_scale.enable_log = m["enable_log"].AsBool();
+  config.log_scale.scale_shift = m["scale_shift"].AsInt32();
+
+  data->state = new FrontendState;
+  FrontendPopulateState(&config, data->state, data->sample_rate);
+
+  data->left_context = m["left_context"].AsInt32();
+  data->right_context = m["right_context"].AsInt32();
+  data->frame_stride = m["frame_stride"].AsInt32();
+  data->zero_padding = m["zero_padding"].AsBool();
+  data->out_scale = m["out_scale"].AsInt32();
+  data->out_float = m["out_float"].AsBool();
+
+  return data;
+}
+
+void Free(TfLiteContext* context, void* buffer) {
+  auto* data = reinterpret_cast<TfLiteAudioMicrofrontendParams*>(buffer);
+  FrontendFreeStateContents(data->state);
+  delete data->state;
+  delete data;
+}
+
+TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  auto* data =
+      reinterpret_cast<TfLiteAudioMicrofrontendParams*>(node->user_data);
+
+  TF_LITE_ENSURE_EQ(context, NumInputs(node), 1);
+  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
+
+  const TfLiteTensor* input;
+  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kInputTensor, &input));
+  TfLiteTensor* output;
+  TF_LITE_ENSURE_OK(context,
+                    GetOutputSafe(context, node, kOutputTensor, &output));
+
+  TF_LITE_ENSURE_EQ(context, NumDimensions(input), 1);
+
+  TF_LITE_ENSURE_EQ(context, input->type, kTfLiteInt16);
+  output->type = kTfLiteInt32;
+  if (data->out_float) {
+    output->type = kTfLiteFloat32;
+  }
+
+  TfLiteIntArray* output_size = TfLiteIntArrayCreate(2);
+  int num_frames = 0;
+  if (input->dims->data[0] >= data->state->window.size) {
+    num_frames = (input->dims->data[0] - data->state->window.size) /
+                     data->state->window.step / data->frame_stride +
+                 1;
+  }
+  output_size->data[0] = num_frames;
+  output_size->data[1] = data->state->filterbank.num_channels *
+                         (1 + data->left_context + data->right_context);
+
+  return context->ResizeTensor(context, output, output_size);
+}
+
+template <typename T>
+void GenerateFeatures(TfLiteAudioMicrofrontendParams* data,
+                      const TfLiteTensor* input, TfLiteTensor* output) {
+  const int16_t* audio_data = GetTensorData<int16_t>(input);
+  int64_t audio_size = input->dims->data[0];
+
+  T* filterbanks_flat = GetTensorData<T>(output);
+
+  int num_frames = 0;
+  if (audio_size >= data->state->window.size) {
+    num_frames = (input->dims->data[0] - data->state->window.size) /
+                     data->state->window.step +
+                 1;
+  }
+  std::vector<std::vector<T>> frame_buffer(num_frames);
+
+  int frame_index = 0;
+  while (audio_size > 0) {
+    size_t num_samples_read;
+    struct FrontendOutput output = FrontendProcessSamples(
+        data->state, audio_data, audio_size, &num_samples_read);
+    audio_data += num_samples_read;
+    audio_size -= num_samples_read;
+
+    if (output.values != nullptr) {
+      frame_buffer[frame_index].reserve(output.size);
+      int i;
+      for (i = 0; i < output.size; ++i) {
+        frame_buffer[frame_index].push_back(static_cast<T>(output.values[i]) /
+                                            data->out_scale);
+      }
+      ++frame_index;
+    }
+  }
+
+  int index = 0;
+  std::vector<T> pad(data->state->filterbank.num_channels, 0);
+  int anchor;
+  for (anchor = 0; anchor < frame_buffer.size(); anchor += data->frame_stride) {
+    int frame;
+    for (frame = anchor - data->left_context;
+         frame <= anchor + data->right_context; ++frame) {
+      std::vector<T>* feature;
+      if (data->zero_padding && (frame < 0 || frame >= frame_buffer.size())) {
+        feature = &pad;
+      } else if (frame < 0) {
+        feature = &frame_buffer[0];
+      } else if (frame >= frame_buffer.size()) {
+        feature = &frame_buffer[frame_buffer.size() - 1];
+      } else {
+        feature = &frame_buffer[frame];
+      }
+      for (auto f : *feature) {
+        filterbanks_flat[index++] = f;
+      }
+    }
+  }
+}
+
+TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
+  auto* data =
+      reinterpret_cast<TfLiteAudioMicrofrontendParams*>(node->user_data);
+  FrontendReset(data->state);
+
+  const TfLiteTensor* input;
+  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kInputTensor, &input));
+  TfLiteTensor* output;
+  TF_LITE_ENSURE_OK(context,
+                    GetOutputSafe(context, node, kOutputTensor, &output));
+
+  if (data->out_float) {
+    GenerateFeatures<float>(data, input, output);
+  } else {
+    GenerateFeatures<int32>(data, input, output);
+  }
+
+  return kTfLiteOk;
+}
+
+}  // namespace audio_microfrontend
+
+TfLiteRegistration* Register_AUDIO_MICROFRONTEND() {
+  static TfLiteRegistration r = {
+      audio_microfrontend::Init, audio_microfrontend::Free,
+      audio_microfrontend::Prepare, audio_microfrontend::Eval};
+  return &r;
+}
+
+}  // namespace custom
+}  // namespace ops
+}  // namespace tflite

tensorflow/lite/experimental/microfrontend/audio_microfrontend.h

@@ -0,0 +1,29 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_AUDIO_MICROFRONTEND_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_AUDIO_MICROFRONTEND_H_
+
+#include "tensorflow/lite/context.h"
+
+namespace tflite {
+namespace ops {
+namespace custom {
+TfLiteRegistration* Register_AUDIO_MICROFRONTEND();
+}  // namespace custom
+}  // namespace ops
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_AUDIO_MICROFRONTEND_H_

tensorflow/lite/experimental/microfrontend/audio_microfrontend_test.cc

@@ -0,0 +1,199 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+// Unit test for TFLite Micro Frontend op.
+
+#include "tensorflow/lite/experimental/microfrontend/audio_microfrontend.h"
+
+#include <cstdint>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+#include "flatbuffers/flexbuffers.h"  // from @flatbuffers
+#include "tensorflow/lite/core/interpreter.h"
+#include "tensorflow/lite/kernels/test_util.h"
+#include "tensorflow/lite/schema/schema_generated.h"
+
+namespace tflite {
+namespace ops {
+namespace custom {
+namespace {
+
+using ::testing::ElementsAreArray;
+
+class MicroFrontendOpModel : public SingleOpModel {
+ public:
+  MicroFrontendOpModel(int n_input, int n_frame, int n_frequency_per_frame,
+                       int n_left_context, int n_right_context,
+                       int n_frame_stride,
+                       const std::vector<std::vector<int>>& input_shapes)
+      : n_input_(n_input),
+        n_frame_(n_frame),
+        n_frequency_per_frame_(n_frequency_per_frame),
+        n_left_context_(n_left_context),
+        n_right_context_(n_right_context),
+        n_frame_stride_(n_frame_stride) {
+    input_ = AddInput(TensorType_INT16);
+    output_ = AddOutput(TensorType_INT32);
+
+    // Set up and pass in custom options using flexbuffer.
+    flexbuffers::Builder fbb;
+    fbb.Map([&]() {
+      // Parameters to initialize FFT state.
+      fbb.Int("sample_rate", 1000);
+      fbb.Int("window_size", 25);
+      fbb.Int("window_step", 10);
+      fbb.Int("num_channels", 2);
+      fbb.Float("upper_band_limit", 450.0);
+      fbb.Float("lower_band_limit", 8.0);
+      fbb.Int("smoothing_bits", 10);
+      fbb.Float("even_smoothing", 0.025);
+      fbb.Float("odd_smoothing", 0.06);
+      fbb.Float("min_signal_remaining", 0.05);
+      fbb.Bool("enable_pcan", true);
+      fbb.Float("pcan_strength", 0.95);
+      fbb.Float("pcan_offset", 80.0);
+      fbb.Int("gain_bits", 21);
+      fbb.Bool("enable_log", true);
+      fbb.Int("scale_shift", 6);
+
+      // Parameters for micro frontend.
+      fbb.Int("left_context", n_left_context);
+      fbb.Int("right_context", n_right_context);
+      fbb.Int("frame_stride", n_frame_stride);
+      fbb.Bool("zero_padding", true);
+      fbb.Int("out_scale", 1);
+      fbb.Bool("out_float", false);
+    });
+    fbb.Finish();
+    SetCustomOp("MICRO_FRONTEND", fbb.GetBuffer(),
+                Register_AUDIO_MICROFRONTEND);
+    BuildInterpreter(input_shapes);
+  }
+
+  void SetInput(const std::vector<int16_t>& data) {
+    PopulateTensor(input_, data);
+  }
+
+  std::vector<int> GetOutput() { return ExtractVector<int>(output_); }
+
+  int num_inputs() { return n_input_; }
+  int num_frmes() { return n_frame_; }
+  int num_frequency_per_frame() { return n_frequency_per_frame_; }
+  int num_left_context() { return n_left_context_; }
+  int num_right_context() { return n_right_context_; }
+  int num_frame_stride() { return n_frame_stride_; }
+
+ protected:
+  int input_;
+  int output_;
+  int n_input_;
+  int n_frame_;
+  int n_frequency_per_frame_;
+  int n_left_context_;
+  int n_right_context_;
+  int n_frame_stride_;
+};
+
+class BaseMicroFrontendTest : public ::testing::Test {
+ protected:
+  // Micro frontend input.
+  std::vector<int16_t> micro_frontend_input_;
+
+  // Compares output up to tolerance to the result of the micro_frontend given
+  // the input.
+  void VerifyGoldens(const std::vector<int16_t>& input,
+                     const std::vector<std::vector<int>>& output,
+                     MicroFrontendOpModel* micro_frontend,
+                     float tolerance = 1e-5) {
+    // Dimensionality check.
+    const int num_inputs = micro_frontend->num_inputs();
+    EXPECT_GT(num_inputs, 0);
+
+    const int num_frames = micro_frontend->num_frmes();
+    EXPECT_GT(num_frames, 0);
+    EXPECT_EQ(num_frames, output.size());
+
+    const int num_frequency_per_frame =
+        micro_frontend->num_frequency_per_frame();
+    EXPECT_GT(num_frequency_per_frame, 0);
+    EXPECT_EQ(num_frequency_per_frame, output[0].size());
+
+    // Set up input.
+    micro_frontend->SetInput(input);
+
+    // Call Invoke.
+    ASSERT_EQ(micro_frontend->Invoke(), kTfLiteOk);
+
+    // Mimic padding behaviour with zero_padding = true.
+    std::vector<int> output_flattened;
+    int anchor;
+    for (anchor = 0; anchor < output.size();
+         anchor += micro_frontend->num_frame_stride()) {
+      int frame;
+      for (frame = anchor - micro_frontend->num_left_context();
+           frame <= anchor + micro_frontend->num_right_context(); ++frame) {
+        if (frame < 0 || frame >= output.size()) {
+          // Padding with zeros.
+          int j;
+          for (j = 0; j < num_frequency_per_frame; ++j) {
+            output_flattened.push_back(0.0);
+          }
+        } else {
+          // Copy real output.
+          for (auto data_point : output[frame]) {
+            output_flattened.push_back(data_point);
+          }
+        }
+      }
+    }
+
+    // Validate result.
+    EXPECT_THAT(micro_frontend->GetOutput(),
+                ElementsAreArray(output_flattened));
+  }
+};  // namespace
+
+class TwoConsecutive36InputsMicroFrontendTest : public BaseMicroFrontendTest {
+  void SetUp() override {
+    micro_frontend_input_ = {
+        0, 32767, 0, -32768, 0, 32767, 0, -32768, 0, 32767, 0, -32768,
+        0, 32767, 0, -32768, 0, 32767, 0, -32768, 0, 32767, 0, -32768,
+        0, 32767, 0, -32768, 0, 32767, 0, -32768, 0, 32767, 0, -32768};
+  }
+};
+
+TEST_F(TwoConsecutive36InputsMicroFrontendTest, MicroFrontendBlackBoxTest) {
+  const int n_input = 36;
+  const int n_frame = 2;
+  const int n_frequency_per_frame = 2;
+
+  MicroFrontendOpModel micro_frontend(n_input, n_frame, n_frequency_per_frame,
+                                      1, 1, 1,
+                                      {
+                                          {n_input},
+                                      });
+
+  // Verify the final output.
+  const std::vector<std::vector<int>> micro_frontend_golden_output = {
+      {479, 425}, {436, 378}};
+  VerifyGoldens(micro_frontend_input_, micro_frontend_golden_output,
+                &micro_frontend);
+}
+
+}  // namespace
+}  // namespace custom
+}  // namespace ops
+}  // namespace tflite

tensorflow/lite/experimental/microfrontend/lib/BUILD

@@ -0,0 +1,121 @@
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
+
+# Library for generating feature vectors from audio data
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:LICENSE"],
+    default_visibility = ["//visibility:public"],
+    licenses = ["notice"],
+)
+
+cc_library(
+    name = "bits",
+    hdrs = ["bits.h"],
+)
+
+cc_library(
+    name = "fft",
+    srcs = [
+        "fft.cc",
+        "fft_util.cc",
+    ],
+    hdrs = [
+        "fft.h",
+        "fft_util.h",
+    ],
+    deps = [
+        "@kissfft//:kiss_fftr_16",
+    ],
+)
+
+cc_library(
+    name = "filterbank",
+    srcs = [
+        "filterbank.c",
+        "filterbank_util.c",
+    ],
+    hdrs = [
+        "filterbank.h",
+        "filterbank_util.h",
+    ],
+    deps = [
+        ":bits",
+        ":fft",
+    ],
+)
+
+cc_library(
+    name = "frontend",
+    srcs = [
+        "frontend.c",
+        "frontend_util.c",
+    ],
+    hdrs = [
+        "frontend.h",
+        "frontend_util.h",
+    ],
+    deps = [
+        ":bits",
+        ":fft",
+        ":filterbank",
+        ":log_scale",
+        ":noise_reduction",
+        ":pcan_gain_control",
+        ":window",
+    ],
+)
+
+cc_library(
+    name = "log_scale",
+    srcs = [
+        "log_lut.c",
+        "log_scale.c",
+        "log_scale_util.c",
+    ],
+    hdrs = [
+        "log_lut.h",
+        "log_scale.h",
+        "log_scale_util.h",
+    ],
+    deps = [
+        ":bits",
+    ],
+)
+
+cc_library(
+    name = "noise_reduction",
+    srcs = [
+        "noise_reduction.c",
+        "noise_reduction_util.c",
+    ],
+    hdrs = [
+        "noise_reduction.h",
+        "noise_reduction_util.h",
+    ],
+)
+
+cc_library(
+    name = "pcan_gain_control",
+    srcs = [
+        "pcan_gain_control.c",
+        "pcan_gain_control_util.c",
+    ],
+    hdrs = [
+        "pcan_gain_control.h",
+        "pcan_gain_control_util.h",
+    ],
+    deps = [
+        ":bits",
+    ],
+)
+
+cc_library(
+    name = "window",
+    srcs = [
+        "window.c",
+        "window_util.c",
+    ],
+    hdrs = [
+        "window.h",
+        "window_util.h",
+    ],
+)

tensorflow/lite/experimental/microfrontend/lib/README.md

@@ -0,0 +1,65 @@
+# Audio "frontend" library for feature generation
+
+A feature generation library (also called frontend) that receives raw audio
+input, and produces filter banks (a vector of values).
+
+The raw audio input is expected to be 16-bit PCM features, with a configurable
+sample rate. More specifically the audio signal goes through a pre-emphasis
+filter (optionally); then gets sliced into (potentially overlapping) frames and
+a window function is applied to each frame; afterwards, we do a Fourier
+transform on each frame (or more specifically a Short-Time Fourier Transform)
+and calculate the power spectrum; and subsequently compute the filter banks.
+
+By default the library is configured with a set of defaults to perform the
+different processing tasks. This takes place with the frontend_util.c function:
+
+```c++
+void FrontendFillConfigWithDefaults(struct FrontendConfig* config)
+```
+
+A single invocation looks like:
+
+```c++
+struct FrontendConfig frontend_config;
+FrontendFillConfigWithDefaults(&frontend_config);
+int sample_rate = 16000;
+FrontendPopulateState(&frontend_config, &frontend_state, sample_rate);
+int16_t* audio_data = ;  // PCM audio samples at 16KHz.
+size_t audio_size = ;  // Number of audio samples.
+size_t num_samples_read;  // How many samples were processed.
+struct FrontendOutput output =
+    FrontendProcessSamples(
+        &frontend_state, audio_data, audio_size, &num_samples_read);
+for (i = 0; i < output.size; ++i) {
+  printf("%d ", output.values[i]);  // Print the feature vector.
+}
+```
+
+Something to note in the above example is that the frontend consumes as many
+samples needed from the audio data to produce a single feature vector (according
+to the frontend configuration). If not enough samples were available to generate
+a feature vector, the returned size will be 0 and the values pointer will be
+`NULL`.
+
+An example of how to use the frontend is provided in frontend_main.cc and its
+binary frontend_main. This example, expects a path to a file containing `int16`
+PCM features at a sample rate of 16KHz, and upon execution will printing out
+the coefficients according to the frontend default configuration.
+
+## Extra features
+Extra features of this frontend library include a noise reduction module, as
+well as a gain control module.
+
+**Noise cancellation**. Removes stationary noise from each channel of the signal
+using a low pass filter.
+
+**Gain control**. A novel automatic gain control based dynamic compression to
+replace the widely used static (such as log or root) compression. Disabled
+by default.
+
+## Memory map
+The binary frontend_memmap_main shows a sample usage of how to avoid all the
+initialization code in your application, by first running
+"frontend_generate_memmap" to create a header/source file that uses a baked in
+frontend state. This command could be automated as part of your build process,
+or you can just use the output directly.

tensorflow/lite/experimental/microfrontend/lib/bits.h

@@ -0,0 +1,102 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_BITS_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_BITS_H_
+
+#ifdef __cplusplus
+#include <cstdint>
+
+extern "C" {
+#endif
+
+static inline int CountLeadingZeros32Slow(uint64_t n) {
+  int zeroes = 28;
+  if (n >> 16) zeroes -= 16, n >>= 16;
+  if (n >> 8) zeroes -= 8, n >>= 8;
+  if (n >> 4) zeroes -= 4, n >>= 4;
+  return "\4\3\2\2\1\1\1\1\0\0\0\0\0\0\0"[n] + zeroes;
+}
+
+static inline int CountLeadingZeros32(uint32_t n) {
+#if !defined(__clang__) && defined(_MSC_VER)
+  unsigned long result = 0;  // NOLINT(runtime/int)
+  if (_BitScanReverse(&result, n)) {
+    return 31 - result;
+  }
+  return 32;
+#elif defined(__clang__) && defined(__GNUC__)
+
+  // Handle 0 as a special case because __builtin_clz(0) is undefined.
+  if (n == 0) {
+    return 32;
+  }
+  return __builtin_clz(n);
+#else
+  return CountLeadingZeros32Slow(n);
+#endif
+}
+
+static inline int MostSignificantBit32(uint32_t n) {
+  return 32 - CountLeadingZeros32(n);
+}
+
+static inline int CountLeadingZeros64Slow(uint64_t n) {
+  int zeroes = 60;
+  if (n >> 32) zeroes -= 32, n >>= 32;
+  if (n >> 16) zeroes -= 16, n >>= 16;
+  if (n >> 8) zeroes -= 8, n >>= 8;
+  if (n >> 4) zeroes -= 4, n >>= 4;
+  return "\4\3\2\2\1\1\1\1\0\0\0\0\0\0\0"[n] + zeroes;
+}
+
+static inline int CountLeadingZeros64(uint64_t n) {
+#if !defined(__clang__) && defined(_MSC_VER) && defined(_M_X64)
+  // MSVC does not have __builtin_clzll. Use _BitScanReverse64.
+  unsigned long result = 0;  // NOLINT(runtime/int)
+  if (_BitScanReverse64(&result, n)) {
+    return 63 - result;
+  }
+  return 64;
+#elif !defined(__clang__) && defined(_MSC_VER)
+  // MSVC does not have __builtin_clzll. Compose two calls to _BitScanReverse
+  unsigned long result = 0;  // NOLINT(runtime/int)
+  if ((n >> 32) && _BitScanReverse(&result, n >> 32)) {
+    return 31 - result;
+  }
+  if (_BitScanReverse(&result, n)) {
+    return 63 - result;
+  }
+  return 64;
+#elif defined(__clang__) || defined(__GNUC__)
+
+  // Handle 0 as a special case because __builtin_clzll(0) is undefined.
+  if (n == 0) {
+    return 64;
+  }
+  return __builtin_clzll(n);
+#else
+  return CountLeadingZeros64Slow(n);
+#endif
+}
+
+static inline int MostSignificantBit64(uint64_t n) {
+  return 64 - CountLeadingZeros64(n);
+}
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_BITS_H_

tensorflow/lite/experimental/microfrontend/lib/fft.cc

@@ -0,0 +1,55 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/experimental/microfrontend/lib/fft.h"
+
+#include <string.h>
+
+#include <cstdint>
+
+#define FIXED_POINT 16
+#include "kiss_fft.h"
+#include "kiss_fftr.h"
+
+void FftCompute(struct FftState* state, const int16_t* input,
+                int input_scale_shift) {
+  const size_t input_size = state->input_size;
+  const size_t fft_size = state->fft_size;
+
+  int16_t* fft_input = state->input;
+  // First, scale the input by the given shift.
+  size_t i;
+  for (i = 0; i < input_size; ++i) {
+    fft_input[i] = static_cast<int16_t>(static_cast<uint16_t>(input[i])
+                                        << input_scale_shift);
+  }
+  // Zero out whatever else remains in the top part of the input.
+  for (; i < fft_size; ++i) {
+    fft_input[i] = 0;
+  }
+
+  // Apply the FFT.
+  kiss_fftr(reinterpret_cast<kiss_fftr_cfg>(state->scratch),
+            state->input,
+            reinterpret_cast<kiss_fft_cpx*>(state->output));
+}
+
+void FftInit(struct FftState* state) {
+  // All the initialization is done in FftPopulateState()
+}
+
+void FftReset(struct FftState* state) {
+  memset(state->input, 0, state->fft_size * sizeof(*state->input));
+  memset(state->output, 0, (state->fft_size / 2 + 1) * sizeof(*state->output));
+}

tensorflow/lite/experimental/microfrontend/lib/fft.h

@@ -0,0 +1,50 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FFT_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FFT_H_
+
+#include <stdint.h>
+#include <stdlib.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct complex_int16_t {
+  int16_t real;
+  int16_t imag;
+};
+
+struct FftState {
+  int16_t* input;
+  struct complex_int16_t* output;
+  size_t fft_size;
+  size_t input_size;
+  void* scratch;
+  size_t scratch_size;
+};
+
+void FftCompute(struct FftState* state, const int16_t* input,
+                int input_scale_shift);
+
+void FftInit(struct FftState* state);
+
+void FftReset(struct FftState* state);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FFT_H_

tensorflow/lite/experimental/microfrontend/lib/fft_io.c

@@ -0,0 +1,33 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/experimental/microfrontend/lib/fft_io.h"
+
+void FftWriteMemmapPreamble(FILE* fp, const struct FftState* state) {
+  fprintf(fp, "static int16_t fft_input[%zu];\n", state->fft_size);
+  fprintf(fp, "static struct complex_int16_t fft_output[%zu];\n",
+          state->fft_size / 2 + 1);
+  fprintf(fp, "static char fft_scratch[%zu];\n", state->scratch_size);
+  fprintf(fp, "\n");
+}
+
+void FftWriteMemmap(FILE* fp, const struct FftState* state,
+                    const char* variable) {
+  fprintf(fp, "%s->input = fft_input;\n", variable);
+  fprintf(fp, "%s->output = fft_output;\n", variable);
+  fprintf(fp, "%s->fft_size = %zu;\n", variable, state->fft_size);
+  fprintf(fp, "%s->input_size = %zu;\n", variable, state->input_size);
+  fprintf(fp, "%s->scratch = fft_scratch;\n", variable);
+  fprintf(fp, "%s->scratch_size = %zu;\n", variable, state->scratch_size);
+}

tensorflow/lite/experimental/microfrontend/lib/fft_io.h

@@ -0,0 +1,34 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FFT_IO_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FFT_IO_H_
+
+#include <stdio.h>
+
+#include "tensorflow/lite/experimental/microfrontend/lib/fft.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void FftWriteMemmapPreamble(FILE* fp, const struct FftState* state);
+void FftWriteMemmap(FILE* fp, const struct FftState* state,
+                    const char* variable);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FFT_IO_H_

tensorflow/lite/experimental/microfrontend/lib/fft_util.cc

@@ -0,0 +1,75 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/experimental/microfrontend/lib/fft_util.h"
+
+#include <stdio.h>
+
+#include <cstdint>
+#include <cstdlib>
+
+#define FIXED_POINT 16
+#include "kiss_fft.h"
+#include "kiss_fftr.h"
+
+int FftPopulateState(struct FftState* state, size_t input_size) {
+  state->input_size = input_size;
+  state->fft_size = 1;
+  while (state->fft_size < state->input_size) {
+    state->fft_size <<= 1;
+  }
+
+  state->input = reinterpret_cast<int16_t*>(
+      malloc(state->fft_size * sizeof(*state->input)));
+  if (state->input == nullptr) {
+    fprintf(stderr, "Failed to alloc fft input buffer\n");
+    return 0;
+  }
+
+  state->output = reinterpret_cast<complex_int16_t*>(
+      malloc((state->fft_size / 2 + 1) * sizeof(*state->output) * 2));
+  if (state->output == nullptr) {
+    fprintf(stderr, "Failed to alloc fft output buffer\n");
+    return 0;
+  }
+
+  // Ask kissfft how much memory it wants.
+  size_t scratch_size = 0;
+  kiss_fftr_cfg kfft_cfg = kiss_fftr_alloc(
+      state->fft_size, 0, nullptr, &scratch_size);
+  if (kfft_cfg != nullptr) {
+    fprintf(stderr, "Kiss memory sizing failed.\n");
+    return 0;
+  }
+  state->scratch = malloc(scratch_size);
+  if (state->scratch == nullptr) {
+    fprintf(stderr, "Failed to alloc fft scratch buffer\n");
+    return 0;
+  }
+  state->scratch_size = scratch_size;
+  // Let kissfft configure the scratch space we just allocated
+  kfft_cfg = kiss_fftr_alloc(state->fft_size, 0,
+                                              state->scratch, &scratch_size);
+  if (kfft_cfg != state->scratch) {
+    fprintf(stderr, "Kiss memory preallocation strategy failed.\n");
+    return 0;
+  }
+  return 1;
+}
+
+void FftFreeStateContents(struct FftState* state) {
+  free(state->input);
+  free(state->output);
+  free(state->scratch);
+}

tensorflow/lite/experimental/microfrontend/lib/fft_util.h

@@ -0,0 +1,34 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FFT_UTIL_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FFT_UTIL_H_
+
+#include "tensorflow/lite/experimental/microfrontend/lib/fft.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Prepares and FFT for the given input size.
+int FftPopulateState(struct FftState* state, size_t input_size);
+
+// Frees any allocated buffers.
+void FftFreeStateContents(struct FftState* state);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FFT_UTIL_H_

tensorflow/lite/experimental/microfrontend/lib/filterbank.c

@@ -0,0 +1,134 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/experimental/microfrontend/lib/filterbank.h"
+
+#include <string.h>
+
+#include "tensorflow/lite/experimental/microfrontend/lib/bits.h"
+
+void FilterbankConvertFftComplexToEnergy(struct FilterbankState* state,
+                                         struct complex_int16_t* fft_output,
+                                         int32_t* energy) {
+  const int end_index = state->end_index;
+  int i;
+  energy += state->start_index;
+  fft_output += state->start_index;
+  for (i = state->start_index; i < end_index; ++i) {
+    const int32_t real = fft_output->real;
+    const int32_t imag = fft_output->imag;
+    fft_output++;
+    const uint32_t mag_squared = (real * real) + (imag * imag);
+    *energy++ = mag_squared;
+  }
+}
+
+void FilterbankAccumulateChannels(struct FilterbankState* state,
+                                  const int32_t* energy) {
+  uint64_t* work = state->work;
+  uint64_t weight_accumulator = 0;
+  uint64_t unweight_accumulator = 0;
+
+  const int16_t* channel_frequency_starts = state->channel_frequency_starts;
+  const int16_t* channel_weight_starts = state->channel_weight_starts;
+  const int16_t* channel_widths = state->channel_widths;
+
+  int num_channels_plus_1 = state->num_channels + 1;
+  int i;
+  for (i = 0; i < num_channels_plus_1; ++i) {
+    const int32_t* magnitudes = energy + *channel_frequency_starts++;
+    const int16_t* weights = state->weights + *channel_weight_starts;
+    const int16_t* unweights = state->unweights + *channel_weight_starts++;
+    const int width = *channel_widths++;
+    int j;
+    for (j = 0; j < width; ++j) {
+      weight_accumulator += *weights++ * ((uint64_t)*magnitudes);
+      unweight_accumulator += *unweights++ * ((uint64_t)*magnitudes);
+      ++magnitudes;
+    }
+    *work++ = weight_accumulator;
+    weight_accumulator = unweight_accumulator;
+    unweight_accumulator = 0;
+  }
+}
+
+static uint16_t Sqrt32(uint32_t num) {
+  if (num == 0) {
+    return 0;
+  }
+  uint32_t res = 0;
+  int max_bit_number = 32 - MostSignificantBit32(num);
+  max_bit_number |= 1;
+  uint32_t bit = 1U << (31 - max_bit_number);
+  int iterations = (31 - max_bit_number) / 2 + 1;
+  while (iterations--) {
+    if (num >= res + bit) {
+      num -= res + bit;
+      res = (res >> 1U) + bit;
+    } else {
+      res >>= 1U;
+    }
+    bit >>= 2U;
+  }
+  // Do rounding - if we have the bits.
+  if (num > res && res != 0xFFFF) {
+    ++res;
+  }
+  return res;
+}
+
+static uint32_t Sqrt64(uint64_t num) {
+  // Take a shortcut and just use 32 bit operations if the upper word is all
+  // clear. This will cause a slight off by one issue for numbers close to 2^32,
+  // but it probably isn't going to matter (and gives us a big performance win).
+  if ((num >> 32) == 0) {
+    return Sqrt32((uint32_t)num);
+  }
+  uint64_t res = 0;
+  int max_bit_number = 64 - MostSignificantBit64(num);
+  max_bit_number |= 1;
+  uint64_t bit = 1ULL << (63 - max_bit_number);
+  int iterations = (63 - max_bit_number) / 2 + 1;
+  while (iterations--) {
+    if (num >= res + bit) {
+      num -= res + bit;
+      res = (res >> 1U) + bit;
+    } else {
+      res >>= 1U;
+    }
+    bit >>= 2U;
+  }
+  // Do rounding - if we have the bits.
+  if (num > res && res != 0xFFFFFFFFLL) {
+    ++res;
+  }
+  return res;
+}
+
+uint32_t* FilterbankSqrt(struct FilterbankState* state, int scale_down_shift) {
+  const int num_channels = state->num_channels;
+  const uint64_t* work = state->work + 1;
+  // Reuse the work buffer since we're fine clobbering it at this point to hold
+  // the output.
+  uint32_t* output = (uint32_t*)state->work;
+  int i;
+  for (i = 0; i < num_channels; ++i) {
+    *output++ = Sqrt64(*work++) >> scale_down_shift;
+  }
+  return (uint32_t*)state->work;
+}
+
+void FilterbankReset(struct FilterbankState* state) {
+  memset(state->work, 0, (state->num_channels + 1) * sizeof(*state->work));
+}

tensorflow/lite/experimental/microfrontend/lib/filterbank.h

@@ -0,0 +1,63 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FILTERBANK_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FILTERBANK_H_
+
+#include <stdint.h>
+#include <stdlib.h>
+
+#include "tensorflow/lite/experimental/microfrontend/lib/fft.h"
+
+#define kFilterbankBits 12
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct FilterbankState {
+  int num_channels;
+  int start_index;
+  int end_index;
+  int16_t* channel_frequency_starts;
+  int16_t* channel_weight_starts;
+  int16_t* channel_widths;
+  int16_t* weights;
+  int16_t* unweights;
+  uint64_t* work;
+};
+
+// Converts the relevant complex values of an FFT output into energy (the
+// square magnitude).
+void FilterbankConvertFftComplexToEnergy(struct FilterbankState* state,
+                                         struct complex_int16_t* fft_output,
+                                         int32_t* energy);
+
+// Computes the mel-scale filterbank on the given energy array. Output is cached
+// internally - to fetch it, you need to call FilterbankSqrt.
+void FilterbankAccumulateChannels(struct FilterbankState* state,
+                                  const int32_t* energy);
+
+// Applies an integer square root to the 64 bit intermediate values of the
+// filterbank, and returns a pointer to them. Memory will be invalidated the
+// next time FilterbankAccumulateChannels is called.
+uint32_t* FilterbankSqrt(struct FilterbankState* state, int scale_down_shift);
+
+void FilterbankReset(struct FilterbankState* state);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FILTERBANK_H_

tensorflow/lite/experimental/microfrontend/lib/filterbank_io.c

@@ -0,0 +1,67 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/experimental/microfrontend/lib/filterbank_io.h"
+
+static void PrintArray(FILE* fp, const char* name, const int16_t* values,
+                       size_t size) {
+  fprintf(fp, "static int16_t filterbank_%s[] = {", name);
+  int i;
+  for (i = 0; i < size; ++i) {
+    fprintf(fp, "%d", values[i]);
+    if (i < size - 1) {
+      fprintf(fp, ", ");
+    }
+  }
+  fprintf(fp, "};\n");
+}
+
+void FilterbankWriteMemmapPreamble(FILE* fp,
+                                   const struct FilterbankState* state) {
+  const int num_channels_plus_1 = state->num_channels + 1;
+
+  PrintArray(fp, "channel_frequency_starts", state->channel_frequency_starts,
+             num_channels_plus_1);
+  PrintArray(fp, "channel_weight_starts", state->channel_weight_starts,
+             num_channels_plus_1);
+  PrintArray(fp, "channel_widths", state->channel_widths, num_channels_plus_1);
+  int num_weights = 0;
+  int i;
+  for (i = 0; i < num_channels_plus_1; ++i) {
+    num_weights += state->channel_widths[i];
+  }
+  PrintArray(fp, "weights", state->weights, num_weights);
+  PrintArray(fp, "unweights", state->unweights, num_weights);
+
+  fprintf(fp, "static uint64_t filterbank_work[%d];\n", num_channels_plus_1);
+  fprintf(fp, "\n");
+}
+
+void FilterbankWriteMemmap(FILE* fp, const struct FilterbankState* state,
+                           const char* variable) {
+  fprintf(fp, "%s->num_channels = %d;\n", variable, state->num_channels);
+  fprintf(fp, "%s->start_index = %d;\n", variable, state->start_index);
+  fprintf(fp, "%s->end_index = %d;\n", variable, state->end_index);
+
+  fprintf(
+      fp,
+      "%s->channel_frequency_starts = filterbank_channel_frequency_starts;\n",
+      variable);
+  fprintf(fp, "%s->channel_weight_starts = filterbank_channel_weight_starts;\n",
+          variable);
+  fprintf(fp, "%s->channel_widths = filterbank_channel_widths;\n", variable);
+  fprintf(fp, "%s->weights = filterbank_weights;\n", variable);
+  fprintf(fp, "%s->unweights = filterbank_unweights;\n", variable);
+  fprintf(fp, "%s->work = filterbank_work;\n", variable);
+}

tensorflow/lite/experimental/microfrontend/lib/filterbank_io.h

@@ -0,0 +1,35 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FILTERBANK_IO_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FILTERBANK_IO_H_
+
+#include <stdio.h>
+
+#include "tensorflow/lite/experimental/microfrontend/lib/filterbank.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void FilterbankWriteMemmapPreamble(FILE* fp,
+                                   const struct FilterbankState* state);
+void FilterbankWriteMemmap(FILE* fp, const struct FilterbankState* state,
+                           const char* variable);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FILTERBANK_IO_H_

tensorflow/lite/experimental/microfrontend/lib/filterbank_util.c

@@ -0,0 +1,220 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/experimental/microfrontend/lib/filterbank_util.h"
+
+#include <assert.h>
+#include <math.h>
+#include <stdio.h>
+
+#define kFilterbankIndexAlignment 4
+#define kFilterbankChannelBlockSize 4
+
+void FilterbankFillConfigWithDefaults(struct FilterbankConfig* config) {
+  config->num_channels = 32;
+  config->lower_band_limit = 125.0f;
+  config->upper_band_limit = 7500.0f;
+  config->output_scale_shift = 7;
+}
+
+static float FreqToMel(float freq) { return 1127.0 * log1p(freq / 700.0); }
+
+static void CalculateCenterFrequencies(const int num_channels,
+                                       const float lower_frequency_limit,
+                                       const float upper_frequency_limit,
+                                       float* center_frequencies) {
+  assert(lower_frequency_limit >= 0.0f);
+  assert(upper_frequency_limit > lower_frequency_limit);
+
+  const float mel_low = FreqToMel(lower_frequency_limit);
+  const float mel_hi = FreqToMel(upper_frequency_limit);
+  const float mel_span = mel_hi - mel_low;
+  const float mel_spacing = mel_span / ((float)num_channels);
+  int i;
+  for (i = 0; i < num_channels; ++i) {
+    center_frequencies[i] = mel_low + (mel_spacing * (i + 1));
+  }
+}
+
+static void QuantizeFilterbankWeights(const float float_weight, int16_t* weight,
+                                      int16_t* unweight) {
+  *weight = floor(float_weight * (1 << kFilterbankBits) + 0.5);
+  *unweight = floor((1.0 - float_weight) * (1 << kFilterbankBits) + 0.5);
+}
+
+int FilterbankPopulateState(const struct FilterbankConfig* config,
+                            struct FilterbankState* state, int sample_rate,
+                            int spectrum_size) {
+  state->num_channels = config->num_channels;
+  const int num_channels_plus_1 = config->num_channels + 1;
+
+  // How should we align things to index counts given the byte alignment?
+  const int index_alignment =
+      (kFilterbankIndexAlignment < sizeof(int16_t)
+           ? 1
+           : kFilterbankIndexAlignment / sizeof(int16_t));
+
+  state->channel_frequency_starts =
+      malloc(num_channels_plus_1 * sizeof(*state->channel_frequency_starts));
+  state->channel_weight_starts =
+      malloc(num_channels_plus_1 * sizeof(*state->channel_weight_starts));
+  state->channel_widths =
+      malloc(num_channels_plus_1 * sizeof(*state->channel_widths));
+  state->work = malloc(num_channels_plus_1 * sizeof(*state->work));
+
+  float* center_mel_freqs =
+      malloc(num_channels_plus_1 * sizeof(*center_mel_freqs));
+  int16_t* actual_channel_starts =
+      malloc(num_channels_plus_1 * sizeof(*actual_channel_starts));
+  int16_t* actual_channel_widths =
+      malloc(num_channels_plus_1 * sizeof(*actual_channel_widths));
+
+  if (state->channel_frequency_starts == NULL ||
+      state->channel_weight_starts == NULL || state->channel_widths == NULL ||
+      center_mel_freqs == NULL || actual_channel_starts == NULL ||
+      actual_channel_widths == NULL) {
+    free(center_mel_freqs);
+    free(actual_channel_starts);
+    free(actual_channel_widths);
+    fprintf(stderr, "Failed to allocate channel buffers\n");
+    return 0;
+  }
+
+  CalculateCenterFrequencies(num_channels_plus_1, config->lower_band_limit,
+                             config->upper_band_limit, center_mel_freqs);
+
+  // Always exclude DC.
+  const float hz_per_sbin = 0.5 * sample_rate / ((float)spectrum_size - 1);
+  state->start_index = 1.5 + config->lower_band_limit / hz_per_sbin;
+  state->end_index = 0;  // Initialized to zero here, but actually set below.
+
+  // For each channel, we need to figure out what frequencies belong to it, and
+  // how much padding we need to add so that we can efficiently multiply the
+  // weights and unweights for accumulation. To simplify the multiplication
+  // logic, all channels will have some multiplication to do (even if there are
+  // no frequencies that accumulate to that channel) - they will be directed to
+  // a set of zero weights.
+  int chan_freq_index_start = state->start_index;
+  int weight_index_start = 0;
+  int needs_zeros = 0;
+
+  int chan;
+  for (chan = 0; chan < num_channels_plus_1; ++chan) {
+    // Keep jumping frequencies until we overshoot the bound on this channel.
+    int freq_index = chan_freq_index_start;
+    while (FreqToMel((freq_index)*hz_per_sbin) <= center_mel_freqs[chan]) {
+      ++freq_index;
+    }
+
+    const int width = freq_index - chan_freq_index_start;
+    actual_channel_starts[chan] = chan_freq_index_start;
+    actual_channel_widths[chan] = width;
+
+    if (width == 0) {
+      // This channel doesn't actually get anything from the frequencies, it's
+      // always zero. We need then to insert some 'zero' weights into the
+      // output, and just redirect this channel to do a single multiplication at
+      // this point. For simplicity, the zeros are placed at the beginning of
+      // the weights arrays, so we have to go and update all the other
+      // weight_starts to reflect this shift (but only once).
+      state->channel_frequency_starts[chan] = 0;
+      state->channel_weight_starts[chan] = 0;
+      state->channel_widths[chan] = kFilterbankChannelBlockSize;
+      if (!needs_zeros) {
+        needs_zeros = 1;
+        int j;
+        for (j = 0; j < chan; ++j) {
+          state->channel_weight_starts[j] += kFilterbankChannelBlockSize;
+        }
+        weight_index_start += kFilterbankChannelBlockSize;
+      }
+    } else {
+      // How far back do we need to go to ensure that we have the proper
+      // alignment?
+      const int aligned_start =
+          (chan_freq_index_start / index_alignment) * index_alignment;
+      const int aligned_width = (chan_freq_index_start - aligned_start + width);
+      const int padded_width =
+          (((aligned_width - 1) / kFilterbankChannelBlockSize) + 1) *
+          kFilterbankChannelBlockSize;
+
+      state->channel_frequency_starts[chan] = aligned_start;
+      state->channel_weight_starts[chan] = weight_index_start;
+      state->channel_widths[chan] = padded_width;
+      weight_index_start += padded_width;
+    }
+    chan_freq_index_start = freq_index;
+  }
+
+  // Allocate the two arrays to store the weights - weight_index_start contains
+  // the index of what would be the next set of weights that we would need to
+  // add, so that's how many weights we need to allocate.
+  state->weights = calloc(weight_index_start, sizeof(*state->weights));
+  state->unweights = calloc(weight_index_start, sizeof(*state->unweights));
+
+  // If the alloc failed, we also need to nuke the arrays.
+  if (state->weights == NULL || state->unweights == NULL) {
+    free(center_mel_freqs);
+    free(actual_channel_starts);
+    free(actual_channel_widths);
+    fprintf(stderr, "Failed to allocate weights or unweights\n");
+    return 0;
+  }
+
+  // Next pass, compute all the weights. Since everything has been memset to
+  // zero, we only need to fill in the weights that correspond to some frequency
+  // for a channel.
+  const float mel_low = FreqToMel(config->lower_band_limit);
+  for (chan = 0; chan < num_channels_plus_1; ++chan) {
+    int frequency = actual_channel_starts[chan];
+    const int num_frequencies = actual_channel_widths[chan];
+    const int frequency_offset =
+        frequency - state->channel_frequency_starts[chan];
+    const int weight_start = state->channel_weight_starts[chan];
+    const float denom_val = (chan == 0) ? mel_low : center_mel_freqs[chan - 1];
+
+    int j;
+    for (j = 0; j < num_frequencies; ++j, ++frequency) {
+      const float weight =
+          (center_mel_freqs[chan] - FreqToMel(frequency * hz_per_sbin)) /
+          (center_mel_freqs[chan] - denom_val);
+
+      // Make the float into an integer for the weights (and unweights).
+      const int weight_index = weight_start + frequency_offset + j;
+      QuantizeFilterbankWeights(weight, state->weights + weight_index,
+                                state->unweights + weight_index);
+    }
+    if (frequency > state->end_index) {
+      state->end_index = frequency;
+    }
+  }
+
+  free(center_mel_freqs);
+  free(actual_channel_starts);
+  free(actual_channel_widths);
+  if (state->end_index >= spectrum_size) {
+    fprintf(stderr, "Filterbank end_index is above spectrum size.\n");
+    return 0;
+  }
+  return 1;
+}
+
+void FilterbankFreeStateContents(struct FilterbankState* state) {
+  free(state->channel_frequency_starts);
+  free(state->channel_weight_starts);
+  free(state->channel_widths);
+  free(state->weights);
+  free(state->unweights);
+  free(state->work);
+}

tensorflow/lite/experimental/microfrontend/lib/filterbank_util.h

@@ -0,0 +1,50 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FILTERBANK_UTIL_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FILTERBANK_UTIL_H_
+
+#include "tensorflow/lite/experimental/microfrontend/lib/filterbank.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct FilterbankConfig {
+  // number of frequency channel buckets for filterbank
+  int num_channels;
+  // maximum frequency to include
+  float upper_band_limit;
+  // minimum frequency to include
+  float lower_band_limit;
+  // unused
+  int output_scale_shift;
+};
+
+// Fills the frontendConfig with "sane" defaults.
+void FilterbankFillConfigWithDefaults(struct FilterbankConfig* config);
+
+// Allocates any buffers.
+int FilterbankPopulateState(const struct FilterbankConfig* config,
+                            struct FilterbankState* state, int sample_rate,
+                            int spectrum_size);
+
+// Frees any allocated buffers.
+void FilterbankFreeStateContents(struct FilterbankState* state);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FILTERBANK_UTIL_H_

tensorflow/lite/experimental/microfrontend/lib/frontend.c

@@ -0,0 +1,72 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/experimental/microfrontend/lib/frontend.h"
+
+#include "tensorflow/lite/experimental/microfrontend/lib/bits.h"
+
+struct FrontendOutput FrontendProcessSamples(struct FrontendState* state,
+                                             const int16_t* samples,
+                                             size_t num_samples,
+                                             size_t* num_samples_read) {
+  struct FrontendOutput output;
+  output.values = NULL;
+  output.size = 0;
+
+  // Try to apply the window - if it fails, return and wait for more data.
+  if (!WindowProcessSamples(&state->window, samples, num_samples,
+                            num_samples_read)) {
+    return output;
+  }
+
+  // Apply the FFT to the window's output (and scale it so that the fixed point
+  // FFT can have as much resolution as possible).
+  int input_shift =
+      15 - MostSignificantBit32(state->window.max_abs_output_value);
+  FftCompute(&state->fft, state->window.output, input_shift);
+
+  // We can re-ruse the fft's output buffer to hold the energy.
+  int32_t* energy = (int32_t*)state->fft.output;
+
+  FilterbankConvertFftComplexToEnergy(&state->filterbank, state->fft.output,
+                                      energy);
+
+  FilterbankAccumulateChannels(&state->filterbank, energy);
+  uint32_t* scaled_filterbank = FilterbankSqrt(&state->filterbank, input_shift);
+
+  // Apply noise reduction.
+  NoiseReductionApply(&state->noise_reduction, scaled_filterbank);
+
+  if (state->pcan_gain_control.enable_pcan) {
+    PcanGainControlApply(&state->pcan_gain_control, scaled_filterbank);
+  }
+
+  // Apply the log and scale.
+  int correction_bits =
+      MostSignificantBit32(state->fft.fft_size) - 1 - (kFilterbankBits / 2);
+  uint16_t* logged_filterbank =
+      LogScaleApply(&state->log_scale, scaled_filterbank,
+                    state->filterbank.num_channels, correction_bits);
+
+  output.size = state->filterbank.num_channels;
+  output.values = logged_filterbank;
+  return output;
+}
+
+void FrontendReset(struct FrontendState* state) {
+  WindowReset(&state->window);
+  FftReset(&state->fft);
+  FilterbankReset(&state->filterbank);
+  NoiseReductionReset(&state->noise_reduction);
+}

tensorflow/lite/experimental/microfrontend/lib/frontend.h

@@ -0,0 +1,64 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FRONTEND_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FRONTEND_H_
+
+#include <stdint.h>
+#include <stdlib.h>
+
+#include "tensorflow/lite/experimental/microfrontend/lib/fft.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/filterbank.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/log_scale.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/noise_reduction.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/window.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct FrontendState {
+  struct WindowState window;
+  struct FftState fft;
+  struct FilterbankState filterbank;
+  struct NoiseReductionState noise_reduction;
+  struct PcanGainControlState pcan_gain_control;
+  struct LogScaleState log_scale;
+};
+
+struct FrontendOutput {
+  const uint16_t* values;
+  size_t size;
+};
+
+// Main entry point to processing frontend samples. Updates num_samples_read to
+// contain the number of samples that have been consumed from the input array.
+// Returns a struct containing the generated output. If not enough samples were
+// added to generate a feature vector, the returned size will be 0 and the
+// values pointer will be NULL. Note that the output pointer will be invalidated
+// as soon as FrontendProcessSamples is called again, so copy the contents
+// elsewhere if you need to use them later.
+struct FrontendOutput FrontendProcessSamples(struct FrontendState* state,
+                                             const int16_t* samples,
+                                             size_t num_samples,
+                                             size_t* num_samples_read);
+
+void FrontendReset(struct FrontendState* state);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FRONTEND_H_

tensorflow/lite/experimental/microfrontend/lib/frontend_io.c

@@ -0,0 +1,69 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/experimental/microfrontend/lib/frontend_io.h"
+
+#include <stdio.h>
+
+#include "tensorflow/lite/experimental/microfrontend/lib/fft_io.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/filterbank_io.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/log_scale_io.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/noise_reduction_io.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/window_io.h"
+
+int WriteFrontendStateMemmap(const char* header, const char* source,
+                             const struct FrontendState* state) {
+  // Write a header that just has our init function.
+  FILE* fp = fopen(header, "w");
+  if (!fp) {
+    fprintf(stderr, "Failed to open header '%s' for write\n", header);
+    return 0;
+  }
+  fprintf(fp, "#ifndef FRONTEND_STATE_MEMMAP_H_\n");
+  fprintf(fp, "#define FRONTEND_STATE_MEMMAP_H_\n");
+  fprintf(fp, "\n");
+  fprintf(fp, "#include \"frontend.h\"\n");
+  fprintf(fp, "\n");
+  fprintf(fp, "struct FrontendState* GetFrontendStateMemmap();\n");
+  fprintf(fp, "\n");
+  fprintf(fp, "#endif  // FRONTEND_STATE_MEMMAP_H_\n");
+  fclose(fp);
+
+  // Write out the source file that actually has everything in it.
+  fp = fopen(source, "w");
+  if (!fp) {
+    fprintf(stderr, "Failed to open source '%s' for write\n", source);
+    return 0;
+  }
+  fprintf(fp, "#include \"%s\"\n", header);
+  fprintf(fp, "\n");
+  WindowWriteMemmapPreamble(fp, &state->window);
+  FftWriteMemmapPreamble(fp, &state->fft);
+  FilterbankWriteMemmapPreamble(fp, &state->filterbank);
+  NoiseReductionWriteMemmapPreamble(fp, &state->noise_reduction);
+  fprintf(fp, "static struct FrontendState state;\n");
+  fprintf(fp, "struct FrontendState* GetFrontendStateMemmap() {\n");
+  WindowWriteMemmap(fp, &state->window, "  (&state.window)");
+  FftWriteMemmap(fp, &state->fft, "  (&state.fft)");
+  FilterbankWriteMemmap(fp, &state->filterbank, "  (&state.filterbank)");
+  NoiseReductionWriteMemmap(fp, &state->noise_reduction,
+                            "  (&state.noise_reduction)");
+  LogScaleWriteMemmap(fp, &state->log_scale, "  (&state.log_scale)");
+  fprintf(fp, "  FftInit(&state.fft);\n");
+  fprintf(fp, "  FrontendReset(&state);\n");
+  fprintf(fp, "  return &state;\n");
+  fprintf(fp, "}\n");
+  fclose(fp);
+  return 1;
+}

tensorflow/lite/experimental/microfrontend/lib/frontend_io.h

@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FRONTEND_IO_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FRONTEND_IO_H_
+
+#include "tensorflow/lite/experimental/microfrontend/lib/frontend.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int WriteFrontendStateMemmap(const char* header, const char* source,
+                             const struct FrontendState* state);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FRONTEND_IO_H_

tensorflow/lite/experimental/microfrontend/lib/frontend_main.c

@@ -0,0 +1,71 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <stdio.h>
+
+#include "tensorflow/lite/experimental/microfrontend/lib/frontend.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/frontend_util.h"
+
+int main(int argc, char** argv) {
+  struct FrontendConfig frontend_config;
+  FrontendFillConfigWithDefaults(&frontend_config);
+
+  char* filename = argv[1];
+  int sample_rate = 16000;
+
+  struct FrontendState frontend_state;
+  if (!FrontendPopulateState(&frontend_config, &frontend_state, sample_rate)) {
+    fprintf(stderr, "Failed to populate frontend state\n");
+    FrontendFreeStateContents(&frontend_state);
+    return 1;
+  }
+
+  FILE* fp = fopen(filename, "r");
+  if (fp == NULL) {
+    fprintf(stderr, "Failed to open %s for read\n", filename);
+    return 1;
+  }
+  fseek(fp, 0L, SEEK_END);
+  size_t audio_file_size = ftell(fp) / sizeof(int16_t);
+  fseek(fp, 0L, SEEK_SET);
+  int16_t* audio_data = malloc(audio_file_size * sizeof(int16_t));
+  int16_t* original_audio_data = audio_data;
+  if (audio_file_size !=
+      fread(audio_data, sizeof(int16_t), audio_file_size, fp)) {
+    fprintf(stderr, "Failed to read in all audio data\n");
+    fclose(fp);
+    return 1;
+  }
+
+  while (audio_file_size > 0) {
+    size_t num_samples_read;
+    struct FrontendOutput output = FrontendProcessSamples(
+        &frontend_state, audio_data, audio_file_size, &num_samples_read);
+    audio_data += num_samples_read;
+    audio_file_size -= num_samples_read;
+
+    if (output.values != NULL) {
+      int i;
+      for (i = 0; i < output.size; ++i) {
+        printf("%d ", output.values[i]);
+      }
+      printf("\n");
+    }
+  }
+
+  FrontendFreeStateContents(&frontend_state);
+  free(original_audio_data);
+  fclose(fp);
+  return 0;
+}

tensorflow/lite/experimental/microfrontend/lib/frontend_memmap_generator.c

@@ -0,0 +1,48 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <stdio.h>
+
+#include "tensorflow/lite/experimental/microfrontend/lib/frontend.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/frontend_util.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/frontend_io.h"
+
+int main(int argc, char** argv) {
+  if (argc != 3) {
+    fprintf(stderr,
+            "%s requires exactly two parameters - the names of the header and "
+            "source files to save\n",
+            argv[0]);
+    return 1;
+  }
+  struct FrontendConfig frontend_config;
+  FrontendFillConfigWithDefaults(&frontend_config);
+
+  int sample_rate = 16000;
+  struct FrontendState frontend_state;
+  if (!FrontendPopulateState(&frontend_config, &frontend_state, sample_rate)) {
+    fprintf(stderr, "Failed to populate frontend state\n");
+    FrontendFreeStateContents(&frontend_state);
+    return 1;
+  }
+
+  if (!WriteFrontendStateMemmap(argv[1], argv[2], &frontend_state)) {
+    fprintf(stderr, "Failed to write memmap\n");
+    FrontendFreeStateContents(&frontend_state);
+    return 1;
+  }
+
+  FrontendFreeStateContents(&frontend_state);
+  return 0;
+}

tensorflow/lite/experimental/microfrontend/lib/frontend_memmap_main.c

@@ -0,0 +1,60 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <stdio.h>
+
+#include "memmap.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/frontend.h"
+
+int main(int argc, char** argv) {
+  struct FrontendState* frontend_state = GetFrontendStateMemmap();
+
+  char* filename = argv[1];
+  FILE* fp = fopen(filename, "r");
+  if (fp == NULL) {
+    fprintf(stderr, "Failed to open %s for read\n", filename);
+    return 1;
+  }
+  fseek(fp, 0L, SEEK_END);
+  size_t audio_file_size = ftell(fp) / sizeof(int16_t);
+  fseek(fp, 0L, SEEK_SET);
+  int16_t* audio_data = malloc(audio_file_size * sizeof(int16_t));
+  int16_t* original_audio_data = audio_data;
+  if (audio_file_size !=
+      fread(audio_data, sizeof(int16_t), audio_file_size, fp)) {
+    fprintf(stderr, "Failed to read in all audio data\n");
+    fclose(fp);
+    return 1;
+  }
+
+  while (audio_file_size > 0) {
+    size_t num_samples_read;
+    struct FrontendOutput output = FrontendProcessSamples(
+        frontend_state, audio_data, audio_file_size, &num_samples_read);
+    audio_data += num_samples_read;
+    audio_file_size -= num_samples_read;
+
+    if (output.values != NULL) {
+      int i;
+      for (i = 0; i < output.size; ++i) {
+        printf("%d ", output.values[i]);
+      }
+      printf("\n");
+    }
+  }
+
+  free(original_audio_data);
+  fclose(fp);
+  return 0;
+}

tensorflow/lite/experimental/microfrontend/lib/frontend_util.c

@@ -0,0 +1,85 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/experimental/microfrontend/lib/frontend_util.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#include "tensorflow/lite/experimental/microfrontend/lib/bits.h"
+
+void FrontendFillConfigWithDefaults(struct FrontendConfig* config) {
+  WindowFillConfigWithDefaults(&config->window);
+  FilterbankFillConfigWithDefaults(&config->filterbank);
+  NoiseReductionFillConfigWithDefaults(&config->noise_reduction);
+  PcanGainControlFillConfigWithDefaults(&config->pcan_gain_control);
+  LogScaleFillConfigWithDefaults(&config->log_scale);
+}
+
+int FrontendPopulateState(const struct FrontendConfig* config,
+                          struct FrontendState* state, int sample_rate) {
+  memset(state, 0, sizeof(*state));
+
+  if (!WindowPopulateState(&config->window, &state->window, sample_rate)) {
+    fprintf(stderr, "Failed to populate window state\n");
+    return 0;
+  }
+
+  if (!FftPopulateState(&state->fft, state->window.size)) {
+    fprintf(stderr, "Failed to populate fft state\n");
+    return 0;
+  }
+  FftInit(&state->fft);
+
+  if (!FilterbankPopulateState(&config->filterbank, &state->filterbank,
+                               sample_rate, state->fft.fft_size / 2 + 1)) {
+    fprintf(stderr, "Failed to populate filterbank state\n");
+    return 0;
+  }
+
+  if (!NoiseReductionPopulateState(&config->noise_reduction,
+                                   &state->noise_reduction,
+                                   state->filterbank.num_channels)) {
+    fprintf(stderr, "Failed to populate noise reduction state\n");
+    return 0;
+  }
+
+  int input_correction_bits =
+      MostSignificantBit32(state->fft.fft_size) - 1 - (kFilterbankBits / 2);
+  if (!PcanGainControlPopulateState(
+          &config->pcan_gain_control, &state->pcan_gain_control,
+          state->noise_reduction.estimate, state->filterbank.num_channels,
+          state->noise_reduction.smoothing_bits, input_correction_bits)) {
+    fprintf(stderr, "Failed to populate pcan gain control state\n");
+    return 0;
+  }
+
+  if (!LogScalePopulateState(&config->log_scale, &state->log_scale)) {
+    fprintf(stderr, "Failed to populate log scale state\n");
+    return 0;
+  }
+
+  FrontendReset(state);
+
+  // All good, return a true value.
+  return 1;
+}
+
+void FrontendFreeStateContents(struct FrontendState* state) {
+  WindowFreeStateContents(&state->window);
+  FftFreeStateContents(&state->fft);
+  FilterbankFreeStateContents(&state->filterbank);
+  NoiseReductionFreeStateContents(&state->noise_reduction);
+  PcanGainControlFreeStateContents(&state->pcan_gain_control);
+}

tensorflow/lite/experimental/microfrontend/lib/frontend_util.h

@@ -0,0 +1,52 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FRONTEND_UTIL_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FRONTEND_UTIL_H_
+
+#include "tensorflow/lite/experimental/microfrontend/lib/fft_util.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/filterbank_util.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/frontend.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/log_scale_util.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/noise_reduction_util.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control_util.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/window_util.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct FrontendConfig {
+  struct WindowConfig window;
+  struct FilterbankConfig filterbank;
+  struct NoiseReductionConfig noise_reduction;
+  struct PcanGainControlConfig pcan_gain_control;
+  struct LogScaleConfig log_scale;
+};
+
+// Fills the frontendConfig with "sane" defaults.
+void FrontendFillConfigWithDefaults(struct FrontendConfig* config);
+
+// Allocates any buffers.
+int FrontendPopulateState(const struct FrontendConfig* config,
+                          struct FrontendState* state, int sample_rate);
+
+// Frees any allocated buffers.
+void FrontendFreeStateContents(struct FrontendState* state);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_FRONTEND_UTIL_H_

tensorflow/lite/experimental/microfrontend/lib/log_lut.c

@@ -0,0 +1,30 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/experimental/microfrontend/lib/log_lut.h"
+const uint16_t kLogLut[]
+#ifndef _MSC_VER
+    __attribute__((aligned(4)))
+#endif  // _MSV_VER
+    = {0,    224,  442,  654,  861,  1063, 1259, 1450, 1636, 1817, 1992, 2163,
+       2329, 2490, 2646, 2797, 2944, 3087, 3224, 3358, 3487, 3611, 3732, 3848,
+       3960, 4068, 4172, 4272, 4368, 4460, 4549, 4633, 4714, 4791, 4864, 4934,
+       5001, 5063, 5123, 5178, 5231, 5280, 5326, 5368, 5408, 5444, 5477, 5507,
+       5533, 5557, 5578, 5595, 5610, 5622, 5631, 5637, 5640, 5641, 5638, 5633,
+       5626, 5615, 5602, 5586, 5568, 5547, 5524, 5498, 5470, 5439, 5406, 5370,
+       5332, 5291, 5249, 5203, 5156, 5106, 5054, 5000, 4944, 4885, 4825, 4762,
+       4697, 4630, 4561, 4490, 4416, 4341, 4264, 4184, 4103, 4020, 3935, 3848,
+       3759, 3668, 3575, 3481, 3384, 3286, 3186, 3084, 2981, 2875, 2768, 2659,
+       2549, 2437, 2323, 2207, 2090, 1971, 1851, 1729, 1605, 1480, 1353, 1224,
+       1094, 963,  830,  695,  559,  421,  282,  142,  0,    0};

tensorflow/lite/experimental/microfrontend/lib/log_lut.h

@@ -0,0 +1,40 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_LUT_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_LUT_H_
+
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Number of segments in the log lookup table. The table will be kLogSegments+1
+// in length (with some padding).
+#define kLogSegments 128
+#define kLogSegmentsLog2 7
+
+// Scale used by lookup table.
+#define kLogScale 65536
+#define kLogScaleLog2 16
+#define kLogCoeff 45426
+
+extern const uint16_t kLogLut[];
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_LUT_H_

tensorflow/lite/experimental/microfrontend/lib/log_scale.c

@@ -0,0 +1,83 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/experimental/microfrontend/lib/log_scale.h"
+
+#include "tensorflow/lite/experimental/microfrontend/lib/bits.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/log_lut.h"
+
+#define kuint16max 0x0000FFFF
+
+// The following functions implement integer logarithms of various sizes. The
+// approximation is calculated according to method described in
+//       www.inti.gob.ar/electronicaeinformatica/instrumentacion/utic/
+//       publicaciones/SPL2007/Log10-spl07.pdf
+// It first calculates log2 of the input and then converts it to natural
+// logarithm.
+
+static uint32_t Log2FractionPart(const uint32_t x, const uint32_t log2x) {
+  // Part 1
+  int32_t frac = x - (1LL << log2x);
+  if (log2x < kLogScaleLog2) {
+    frac <<= kLogScaleLog2 - log2x;
+  } else {
+    frac >>= log2x - kLogScaleLog2;
+  }
+  // Part 2
+  const uint32_t base_seg = frac >> (kLogScaleLog2 - kLogSegmentsLog2);
+  const uint32_t seg_unit =
+      (((uint32_t)1) << kLogScaleLog2) >> kLogSegmentsLog2;
+
+  const int32_t c0 = kLogLut[base_seg];
+  const int32_t c1 = kLogLut[base_seg + 1];
+  const int32_t seg_base = seg_unit * base_seg;
+  const int32_t rel_pos = ((c1 - c0) * (frac - seg_base)) >> kLogScaleLog2;
+  return frac + c0 + rel_pos;
+}
+
+static uint32_t Log(const uint32_t x, const uint32_t scale_shift) {
+  const uint32_t integer = MostSignificantBit32(x) - 1;
+  const uint32_t fraction = Log2FractionPart(x, integer);
+  const uint32_t log2 = (integer << kLogScaleLog2) + fraction;
+  const uint32_t round = kLogScale / 2;
+  const uint32_t loge = (((uint64_t)kLogCoeff) * log2 + round) >> kLogScaleLog2;
+  // Finally scale to our output scale
+  const uint32_t loge_scaled = ((loge << scale_shift) + round) >> kLogScaleLog2;
+  return loge_scaled;
+}
+
+uint16_t* LogScaleApply(struct LogScaleState* state, uint32_t* signal,
+                        int signal_size, int correction_bits) {
+  const int scale_shift = state->scale_shift;
+  uint16_t* output = (uint16_t*)signal;
+  uint16_t* ret = output;
+  int i;
+  for (i = 0; i < signal_size; ++i) {
+    uint32_t value = *signal++;
+    if (state->enable_log) {
+      if (correction_bits < 0) {
+        value >>= -correction_bits;
+      } else {
+        value <<= correction_bits;
+      }
+      if (value > 1) {
+        value = Log(value, scale_shift);
+      } else {
+        value = 0;
+      }
+    }
+    *output++ = (value < kuint16max) ? value : kuint16max;
+  }
+  return ret;
+}

tensorflow/lite/experimental/microfrontend/lib/log_scale.h

@@ -0,0 +1,39 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_SCALE_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_SCALE_H_
+
+#include <stdint.h>
+#include <stdlib.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct LogScaleState {
+  int enable_log;
+  int scale_shift;
+};
+
+// Applies a fixed point logarithm to the signal and converts it to 16 bit. Note
+// that the signal array will be modified.
+uint16_t* LogScaleApply(struct LogScaleState* state, uint32_t* signal,
+                        int signal_size, int correction_bits);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_SCALE_H_

tensorflow/lite/experimental/microfrontend/lib/log_scale_io.c

@@ -0,0 +1,21 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/experimental/microfrontend/lib/log_scale_io.h"
+
+void LogScaleWriteMemmap(FILE* fp, const struct LogScaleState* state,
+                         const char* variable) {
+  fprintf(fp, "%s->enable_log = %d;\n", variable, state->enable_log);
+  fprintf(fp, "%s->scale_shift = %d;\n", variable, state->scale_shift);
+}

tensorflow/lite/experimental/microfrontend/lib/log_scale_io.h

@@ -0,0 +1,33 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_SCALE_IO_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_SCALE_IO_H_
+
+#include <stdio.h>
+
+#include "tensorflow/lite/experimental/microfrontend/lib/log_scale.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void LogScaleWriteMemmap(FILE* fp, const struct LogScaleState* state,
+                         const char* variable);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_SCALE_IO_H_

tensorflow/lite/experimental/microfrontend/lib/log_scale_util.c

@@ -0,0 +1,27 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/experimental/microfrontend/lib/log_scale_util.h"
+
+void LogScaleFillConfigWithDefaults(struct LogScaleConfig* config) {
+  config->enable_log = 1;
+  config->scale_shift = 6;
+}
+
+int LogScalePopulateState(const struct LogScaleConfig* config,
+                          struct LogScaleState* state) {
+  state->enable_log = config->enable_log;
+  state->scale_shift = config->scale_shift;
+  return 1;
+}

tensorflow/lite/experimental/microfrontend/lib/log_scale_util.h

@@ -0,0 +1,45 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_SCALE_UTIL_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_SCALE_UTIL_H_
+
+#include <stdint.h>
+#include <stdlib.h>
+
+#include "tensorflow/lite/experimental/microfrontend/lib/log_scale.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct LogScaleConfig {
+  // set to false (0) to disable this module
+  int enable_log;
+  // scale results by 2^(scale_shift)
+  int scale_shift;
+};
+
+// Populates the LogScaleConfig with "sane" default values.
+void LogScaleFillConfigWithDefaults(struct LogScaleConfig* config);
+
+// Allocates any buffers.
+int LogScalePopulateState(const struct LogScaleConfig* config,
+                          struct LogScaleState* state);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_LOG_SCALE_UTIL_H_

tensorflow/lite/experimental/microfrontend/lib/noise_reduction.c

@@ -0,0 +1,51 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/experimental/microfrontend/lib/noise_reduction.h"
+
+#include <string.h>
+
+void NoiseReductionApply(struct NoiseReductionState* state, uint32_t* signal) {
+  int i;
+  for (i = 0; i < state->num_channels; ++i) {
+    const uint32_t smoothing =
+        ((i & 1) == 0) ? state->even_smoothing : state->odd_smoothing;
+    const uint32_t one_minus_smoothing = (1 << kNoiseReductionBits) - smoothing;
+
+    // Update the estimate of the noise.
+    const uint32_t signal_scaled_up = signal[i] << state->smoothing_bits;
+    uint32_t estimate =
+        (((uint64_t)signal_scaled_up * smoothing) +
+         ((uint64_t)state->estimate[i] * one_minus_smoothing)) >>
+        kNoiseReductionBits;
+    state->estimate[i] = estimate;
+
+    // Make sure that we can't get a negative value for the signal - estimate.
+    if (estimate > signal_scaled_up) {
+      estimate = signal_scaled_up;
+    }
+
+    const uint32_t floor =
+        ((uint64_t)signal[i] * state->min_signal_remaining) >>
+        kNoiseReductionBits;
+    const uint32_t subtracted =
+        (signal_scaled_up - estimate) >> state->smoothing_bits;
+    const uint32_t output = subtracted > floor ? subtracted : floor;
+    signal[i] = output;
+  }
+}
+
+void NoiseReductionReset(struct NoiseReductionState* state) {
+  memset(state->estimate, 0, sizeof(*state->estimate) * state->num_channels);
+}

tensorflow/lite/experimental/microfrontend/lib/noise_reduction.h

@@ -0,0 +1,46 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_NOISE_REDUCTION_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_NOISE_REDUCTION_H_
+
+#define kNoiseReductionBits 14
+
+#include <stdint.h>
+#include <stdlib.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct NoiseReductionState {
+  int smoothing_bits;
+  uint16_t even_smoothing;
+  uint16_t odd_smoothing;
+  uint16_t min_signal_remaining;
+  int num_channels;
+  uint32_t* estimate;
+};
+
+// Removes stationary noise from each channel of the signal using a low pass
+// filter.
+void NoiseReductionApply(struct NoiseReductionState* state, uint32_t* signal);
+
+void NoiseReductionReset(struct NoiseReductionState* state);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_NOISE_REDUCTION_H_

tensorflow/lite/experimental/microfrontend/lib/noise_reduction_io.c

@@ -0,0 +1,34 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/experimental/microfrontend/lib/noise_reduction_io.h"
+
+void NoiseReductionWriteMemmapPreamble(
+    FILE* fp, const struct NoiseReductionState* state) {
+  fprintf(fp, "static uint32_t noise_reduction_estimate[%zu];\n",
+          state->num_channels);
+  fprintf(fp, "\n");
+}
+
+void NoiseReductionWriteMemmap(FILE* fp,
+                               const struct NoiseReductionState* state,
+                               const char* variable) {
+  fprintf(fp, "%s->even_smoothing = %d;\n", variable, state->even_smoothing);
+  fprintf(fp, "%s->odd_smoothing = %d;\n", variable, state->odd_smoothing);
+  fprintf(fp, "%s->min_signal_remaining = %d;\n", variable,
+          state->min_signal_remaining);
+  fprintf(fp, "%s->num_channels = %d;\n", variable, state->num_channels);
+
+  fprintf(fp, "%s->estimate = noise_reduction_estimate;\n", variable);
+}

tensorflow/lite/experimental/microfrontend/lib/noise_reduction_io.h

@@ -0,0 +1,36 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_NOISE_REDUCTION_IO_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_NOISE_REDUCTION_IO_H_
+
+#include <stdio.h>
+
+#include "tensorflow/lite/experimental/microfrontend/lib/noise_reduction.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void NoiseReductionWriteMemmapPreamble(FILE* fp,
+                                       const struct NoiseReductionState* state);
+void NoiseReductionWriteMemmap(FILE* fp,
+                               const struct NoiseReductionState* state,
+                               const char* variable);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_NOISE_REDUCTION_IO_H_

tensorflow/lite/experimental/microfrontend/lib/noise_reduction_util.c

@@ -0,0 +1,45 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/experimental/microfrontend/lib/noise_reduction_util.h"
+
+#include <stdio.h>
+
+void NoiseReductionFillConfigWithDefaults(struct NoiseReductionConfig* config) {
+  config->smoothing_bits = 10;
+  config->even_smoothing = 0.025;
+  config->odd_smoothing = 0.06;
+  config->min_signal_remaining = 0.05;
+}
+
+int NoiseReductionPopulateState(const struct NoiseReductionConfig* config,
+                                struct NoiseReductionState* state,
+                                int num_channels) {
+  state->smoothing_bits = config->smoothing_bits;
+  state->odd_smoothing = config->odd_smoothing * (1 << kNoiseReductionBits);
+  state->even_smoothing = config->even_smoothing * (1 << kNoiseReductionBits);
+  state->min_signal_remaining =
+      config->min_signal_remaining * (1 << kNoiseReductionBits);
+  state->num_channels = num_channels;
+  state->estimate = calloc(state->num_channels, sizeof(*state->estimate));
+  if (state->estimate == NULL) {
+    fprintf(stderr, "Failed to alloc estimate buffer\n");
+    return 0;
+  }
+  return 1;
+}
+
+void NoiseReductionFreeStateContents(struct NoiseReductionState* state) {
+  free(state->estimate);
+}

tensorflow/lite/experimental/microfrontend/lib/noise_reduction_util.h

@@ -0,0 +1,50 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_NOISE_REDUCTION_UTIL_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_NOISE_REDUCTION_UTIL_H_
+
+#include "tensorflow/lite/experimental/microfrontend/lib/noise_reduction.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct NoiseReductionConfig {
+  // scale the signal up by 2^(smoothing_bits) before reduction
+  int smoothing_bits;
+  // smoothing coefficient for even-numbered channels
+  float even_smoothing;
+  // smoothing coefficient for odd-numbered channels
+  float odd_smoothing;
+  // fraction of signal to preserve (1.0 disables this module)
+  float min_signal_remaining;
+};
+
+// Populates the NoiseReductionConfig with "sane" default values.
+void NoiseReductionFillConfigWithDefaults(struct NoiseReductionConfig* config);
+
+// Allocates any buffers.
+int NoiseReductionPopulateState(const struct NoiseReductionConfig* config,
+                                struct NoiseReductionState* state,
+                                int num_channels);
+
+// Frees any allocated buffers.
+void NoiseReductionFreeStateContents(struct NoiseReductionState* state);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_NOISE_REDUCTION_UTIL_H_

tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control.c

@@ -0,0 +1,56 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control.h"
+
+#include "tensorflow/lite/experimental/microfrontend/lib/bits.h"
+
+int16_t WideDynamicFunction(const uint32_t x, const int16_t* lut) {
+  if (x <= 2) {
+    return lut[x];
+  }
+
+  const int16_t interval = MostSignificantBit32(x);
+  lut += 4 * interval - 6;
+
+  const int16_t frac =
+      ((interval < 11) ? (x << (11 - interval)) : (x >> (interval - 11))) &
+      0x3FF;
+
+  int32_t result = ((int32_t)lut[2] * frac) >> 5;
+  result += (int32_t)((uint32_t)lut[1] << 5);
+  result *= frac;
+  result = (result + (1 << 14)) >> 15;
+  result += lut[0];
+  return (int16_t)result;
+}
+
+uint32_t PcanShrink(const uint32_t x) {
+  if (x < (2 << kPcanSnrBits)) {
+    return (x * x) >> (2 + 2 * kPcanSnrBits - kPcanOutputBits);
+  } else {
+    return (x >> (kPcanSnrBits - kPcanOutputBits)) - (1 << kPcanOutputBits);
+  }
+}
+
+void PcanGainControlApply(struct PcanGainControlState* state,
+                          uint32_t* signal) {
+  int i;
+  for (i = 0; i < state->num_channels; ++i) {
+    const uint32_t gain =
+        WideDynamicFunction(state->noise_estimate[i], state->gain_lut);
+    const uint32_t snr = ((uint64_t)signal[i] * gain) >> state->snr_shift;
+    signal[i] = PcanShrink(snr);
+  }
+}

tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control.h

@@ -0,0 +1,47 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_PCAN_GAIN_CONTROL_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_PCAN_GAIN_CONTROL_H_
+
+#include <stdint.h>
+#include <stdlib.h>
+
+#define kPcanSnrBits 12
+#define kPcanOutputBits 6
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Details at https://research.google/pubs/pub45911.pdf
+struct PcanGainControlState {
+  int enable_pcan;
+  uint32_t* noise_estimate;
+  int num_channels;
+  int16_t* gain_lut;
+  int32_t snr_shift;
+};
+
+int16_t WideDynamicFunction(const uint32_t x, const int16_t* lut);
+
+uint32_t PcanShrink(const uint32_t x);
+
+void PcanGainControlApply(struct PcanGainControlState* state, uint32_t* signal);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_PCAN_GAIN_CONTROL_H_

tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control_util.c

@@ -0,0 +1,92 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control_util.h"
+
+#include <math.h>
+#include <stdio.h>
+
+#define kint16max 0x00007FFF
+
+void PcanGainControlFillConfigWithDefaults(
+    struct PcanGainControlConfig* config) {
+  config->enable_pcan = 0;
+  config->strength = 0.95;
+  config->offset = 80.0;
+  config->gain_bits = 21;
+}
+
+int16_t PcanGainLookupFunction(const struct PcanGainControlConfig* config,
+                               int32_t input_bits, uint32_t x) {
+  const float x_as_float = ((float)x) / ((uint32_t)1 << input_bits);
+  const float gain_as_float =
+      ((uint32_t)1 << config->gain_bits) *
+      powf(x_as_float + config->offset, -config->strength);
+
+  if (gain_as_float > kint16max) {
+    return kint16max;
+  }
+  return (int16_t)(gain_as_float + 0.5f);
+}
+
+int PcanGainControlPopulateState(const struct PcanGainControlConfig* config,
+                                 struct PcanGainControlState* state,
+                                 uint32_t* noise_estimate,
+                                 const int num_channels,
+                                 const uint16_t smoothing_bits,
+                                 const int32_t input_correction_bits) {
+  state->enable_pcan = config->enable_pcan;
+  if (!state->enable_pcan) {
+    return 1;
+  }
+  state->noise_estimate = noise_estimate;
+  state->num_channels = num_channels;
+  state->gain_lut = malloc(kWideDynamicFunctionLUTSize * sizeof(int16_t));
+  if (state->gain_lut == NULL) {
+    fprintf(stderr, "Failed to allocate gain LUT\n");
+    return 0;
+  }
+  state->snr_shift = config->gain_bits - input_correction_bits - kPcanSnrBits;
+
+  const int32_t input_bits = smoothing_bits - input_correction_bits;
+  state->gain_lut[0] = PcanGainLookupFunction(config, input_bits, 0);
+  state->gain_lut[1] = PcanGainLookupFunction(config, input_bits, 1);
+  state->gain_lut -= 6;
+  int interval;
+  for (interval = 2; interval <= kWideDynamicFunctionBits; ++interval) {
+    const uint32_t x0 = (uint32_t)1 << (interval - 1);
+    const uint32_t x1 = x0 + (x0 >> 1);
+    const uint32_t x2 =
+        (interval == kWideDynamicFunctionBits) ? x0 + (x0 - 1) : 2 * x0;
+
+    const int16_t y0 = PcanGainLookupFunction(config, input_bits, x0);
+    const int16_t y1 = PcanGainLookupFunction(config, input_bits, x1);
+    const int16_t y2 = PcanGainLookupFunction(config, input_bits, x2);
+
+    const int32_t diff1 = (int32_t)y1 - y0;
+    const int32_t diff2 = (int32_t)y2 - y0;
+    const int32_t a1 = 4 * diff1 - diff2;
+    const int32_t a2 = diff2 - a1;
+
+    state->gain_lut[4 * interval] = y0;
+    state->gain_lut[4 * interval + 1] = (int16_t)a1;
+    state->gain_lut[4 * interval + 2] = (int16_t)a2;
+  }
+  state->gain_lut += 6;
+  return 1;
+}
+
+void PcanGainControlFreeStateContents(struct PcanGainControlState* state) {
+  free(state->gain_lut);
+}

tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control_util.h

@@ -0,0 +1,57 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_PCAN_GAIN_CONTROL_UTIL_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_PCAN_GAIN_CONTROL_UTIL_H_
+
+#include "tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control.h"
+
+#define kWideDynamicFunctionBits 32
+#define kWideDynamicFunctionLUTSize (4 * kWideDynamicFunctionBits - 3)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct PcanGainControlConfig {
+  // set to false (0) to disable this module
+  int enable_pcan;
+  // gain normalization exponent (0.0 disables, 1.0 full strength)
+  float strength;
+  // positive value added in the normalization denominator
+  float offset;
+  // number of fractional bits in the gain
+  int gain_bits;
+};
+
+void PcanGainControlFillConfigWithDefaults(
+    struct PcanGainControlConfig* config);
+
+int16_t PcanGainLookupFunction(const struct PcanGainControlConfig* config,
+                               int32_t input_bits, uint32_t x);
+
+int PcanGainControlPopulateState(const struct PcanGainControlConfig* config,
+                                 struct PcanGainControlState* state,
+                                 uint32_t* noise_estimate,
+                                 const int num_channels,
+                                 const uint16_t smoothing_bits,
+                                 const int32_t input_correction_bits);
+
+void PcanGainControlFreeStateContents(struct PcanGainControlState* state);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_PCAN_GAIN_CONTROL_UTIL_H_

tensorflow/lite/experimental/microfrontend/lib/window.c

@@ -0,0 +1,70 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/experimental/microfrontend/lib/window.h"
+
+#include <string.h>
+
+int WindowProcessSamples(struct WindowState* state, const int16_t* samples,
+                         size_t num_samples, size_t* num_samples_read) {
+  const int size = state->size;
+
+  // Copy samples from the samples buffer over to our local input.
+  size_t max_samples_to_copy = state->size - state->input_used;
+  if (max_samples_to_copy > num_samples) {
+    max_samples_to_copy = num_samples;
+  }
+  memcpy(state->input + state->input_used, samples,
+         max_samples_to_copy * sizeof(*samples));
+  *num_samples_read = max_samples_to_copy;
+  state->input_used += max_samples_to_copy;
+
+  if (state->input_used < state->size) {
+    // We don't have enough samples to compute a window.
+    return 0;
+  }
+
+  // Apply the window to the input.
+  const int16_t* coefficients = state->coefficients;
+  const int16_t* input = state->input;
+  int16_t* output = state->output;
+  int i;
+  int16_t max_abs_output_value = 0;
+  for (i = 0; i < size; ++i) {
+    int16_t new_value =
+        (((int32_t)*input++) * *coefficients++) >> kFrontendWindowBits;
+    *output++ = new_value;
+    if (new_value < 0) {
+      new_value = -new_value;
+    }
+    if (new_value > max_abs_output_value) {
+      max_abs_output_value = new_value;
+    }
+  }
+  // Shuffle the input down by the step size, and update how much we have used.
+  memmove(state->input, state->input + state->step,
+          sizeof(*state->input) * (state->size - state->step));
+  state->input_used -= state->step;
+  state->max_abs_output_value = max_abs_output_value;
+
+  // Indicate that the output buffer is valid for the next stage.
+  return 1;
+}
+
+void WindowReset(struct WindowState* state) {
+  memset(state->input, 0, state->size * sizeof(*state->input));
+  memset(state->output, 0, state->size * sizeof(*state->output));
+  state->input_used = 0;
+  state->max_abs_output_value = 0;
+}

tensorflow/lite/experimental/microfrontend/lib/window.h

@@ -0,0 +1,49 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_WINDOW_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_WINDOW_H_
+
+#include <stdint.h>
+#include <stdlib.h>
+
+#define kFrontendWindowBits 12
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct WindowState {
+  size_t size;
+  int16_t* coefficients;
+  size_t step;
+
+  int16_t* input;
+  size_t input_used;
+  int16_t* output;
+  int16_t max_abs_output_value;
+};
+
+// Applies a window to the samples coming in, stepping forward at the given
+// rate.
+int WindowProcessSamples(struct WindowState* state, const int16_t* samples,
+                         size_t num_samples, size_t* num_samples_read);
+
+void WindowReset(struct WindowState* state);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_WINDOW_H_

tensorflow/lite/experimental/microfrontend/lib/window_io.c

@@ -0,0 +1,43 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/experimental/microfrontend/lib/window_io.h"
+
+void WindowWriteMemmapPreamble(FILE* fp, const struct WindowState* state) {
+  fprintf(fp, "static int16_t window_coefficients[] = {\n");
+  int i;
+  for (i = 0; i < state->size; ++i) {
+    fprintf(fp, "%d", state->coefficients[i]);
+    if (i < state->size - 1) {
+      fprintf(fp, ", ");
+    }
+  }
+  fprintf(fp, "};\n");
+  fprintf(fp, "static int16_t window_input[%zu];\n", state->size);
+  fprintf(fp, "static int16_t window_output[%zu];\n", state->size);
+  fprintf(fp, "\n");
+}
+
+void WindowWriteMemmap(FILE* fp, const struct WindowState* state,
+                       const char* variable) {
+  fprintf(fp, "%s->size = %zu;\n", variable, state->size);
+  fprintf(fp, "%s->coefficients = window_coefficients;\n", variable);
+  fprintf(fp, "%s->step = %zu;\n", variable, state->step);
+
+  fprintf(fp, "%s->input = window_input;\n", variable);
+  fprintf(fp, "%s->input_used = %zu;\n", variable, state->input_used);
+  fprintf(fp, "%s->output = window_output;\n", variable);
+  fprintf(fp, "%s->max_abs_output_value = %d;\n", variable,
+          state->max_abs_output_value);
+}

tensorflow/lite/experimental/microfrontend/lib/window_io.h

@@ -0,0 +1,34 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_WINDOW_IO_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_WINDOW_IO_H_
+
+#include <stdio.h>
+
+#include "tensorflow/lite/experimental/microfrontend/lib/window.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void WindowWriteMemmapPreamble(FILE* fp, const struct WindowState* state);
+void WindowWriteMemmap(FILE* fp, const struct WindowState* state,
+                       const char* variable);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_WINDOW_IO_H_

tensorflow/lite/experimental/microfrontend/lib/window_util.c

@@ -0,0 +1,73 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/experimental/microfrontend/lib/window_util.h"
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// Some platforms don't have M_PI
+#ifndef M_PI
+#define M_PI 3.14159265358979323846
+#endif
+
+void WindowFillConfigWithDefaults(struct WindowConfig* config) {
+  config->size_ms = 25;
+  config->step_size_ms = 10;
+}
+
+int WindowPopulateState(const struct WindowConfig* config,
+                        struct WindowState* state, int sample_rate) {
+  state->size = config->size_ms * sample_rate / 1000;
+  state->step = config->step_size_ms * sample_rate / 1000;
+
+  state->coefficients = malloc(state->size * sizeof(*state->coefficients));
+  if (state->coefficients == NULL) {
+    fprintf(stderr, "Failed to allocate window coefficients\n");
+    return 0;
+  }
+
+  // Populate the window values.
+  const float arg = M_PI * 2.0 / ((float)state->size);
+  int i;
+  for (i = 0; i < state->size; ++i) {
+    float float_value = 0.5 - (0.5 * cos(arg * (i + 0.5)));
+    // Scale it to fixed point and round it.
+    state->coefficients[i] =
+        floor(float_value * (1 << kFrontendWindowBits) + 0.5);
+  }
+
+  state->input_used = 0;
+  state->input = malloc(state->size * sizeof(*state->input));
+  if (state->input == NULL) {
+    fprintf(stderr, "Failed to allocate window input\n");
+    return 0;
+  }
+
+  state->output = malloc(state->size * sizeof(*state->output));
+  if (state->output == NULL) {
+    fprintf(stderr, "Failed to allocate window output\n");
+    return 0;
+  }
+
+  return 1;
+}
+
+void WindowFreeStateContents(struct WindowState* state) {
+  free(state->coefficients);
+  free(state->input);
+  free(state->output);
+}

tensorflow/lite/experimental/microfrontend/lib/window_util.h

@@ -0,0 +1,45 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_WINDOW_UTIL_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_WINDOW_UTIL_H_
+
+#include "tensorflow/lite/experimental/microfrontend/lib/window.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct WindowConfig {
+  // length of window frame in milliseconds
+  size_t size_ms;
+  // length of step for next frame in milliseconds
+  size_t step_size_ms;
+};
+
+// Populates the WindowConfig with "sane" default values.
+void WindowFillConfigWithDefaults(struct WindowConfig* config);
+
+// Allocates any buffers.
+int WindowPopulateState(const struct WindowConfig* config,
+                        struct WindowState* state, int sample_rate);
+
+// Frees any allocated buffers.
+void WindowFreeStateContents(struct WindowState* state);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICROFRONTEND_LIB_WINDOW_UTIL_H_

tensorflow/lite/experimental/microfrontend/ops/audio_microfrontend_op.cc

@@ -0,0 +1,301 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/experimental/microfrontend/lib/frontend.h"
+#include "tensorflow/lite/experimental/microfrontend/lib/frontend_util.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/shape_inference.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/platform/macros.h"
+
+using tensorflow::errors::Internal;
+using tensorflow::errors::InvalidArgument;
+using tensorflow::shape_inference::DimensionHandle;
+using tensorflow::shape_inference::InferenceContext;
+using tensorflow::shape_inference::ShapeHandle;
+
+namespace tensorflow {
+REGISTER_OP("AudioMicrofrontend")
+    .Input("audio: int16")
+    .Output("filterbanks: out_type")
+    .Attr("sample_rate: int = 16000")
+    .Attr("window_size: int = 25")
+    .Attr("window_step: int = 10")
+    .Attr("num_channels: int = 32")
+    .Attr("upper_band_limit: float = 7500.0")
+    .Attr("lower_band_limit: float = 125.0")
+    .Attr("smoothing_bits: int = 10")
+    .Attr("even_smoothing: float = 0.025")
+    .Attr("odd_smoothing: float = 0.06")
+    .Attr("min_signal_remaining: float = 0.05")
+    .Attr("enable_pcan: bool = false")
+    .Attr("pcan_strength: float = 0.95")
+    .Attr("pcan_offset: float = 80.0")
+    .Attr("gain_bits: int = 21")
+    .Attr("enable_log: bool = true")
+    .Attr("scale_shift: int = 6")
+    .Attr("left_context: int = 0")
+    .Attr("right_context: int = 0")
+    .Attr("frame_stride: int = 1")
+    .Attr("zero_padding: bool = false")
+    .Attr("out_scale: int = 1")
+    .Attr("out_type: {uint16, float} = DT_UINT16")
+    .SetShapeFn([](InferenceContext* ctx) {
+      ShapeHandle input;
+      TF_RETURN_IF_ERROR(ctx->WithRank(ctx->input(0), 1, &input));
+
+      int sample_rate;
+      TF_RETURN_IF_ERROR(ctx->GetAttr("sample_rate", &sample_rate));
+      int window_size;
+      TF_RETURN_IF_ERROR(ctx->GetAttr("window_size", &window_size));
+      window_size *= sample_rate / 1000;
+      int window_step;
+      TF_RETURN_IF_ERROR(ctx->GetAttr("window_step", &window_step));
+      window_step *= sample_rate / 1000;
+
+      int num_channels;
+      TF_RETURN_IF_ERROR(ctx->GetAttr("num_channels", &num_channels));
+      int left_context;
+      TF_RETURN_IF_ERROR(ctx->GetAttr("left_context", &left_context));
+      int right_context;
+      TF_RETURN_IF_ERROR(ctx->GetAttr("right_context", &right_context));
+      int frame_stride;
+      TF_RETURN_IF_ERROR(ctx->GetAttr("frame_stride", &frame_stride));
+
+      DimensionHandle num_frames = ctx->Dim(input, 0);
+      if (ctx->Value(num_frames) < window_size) {
+        num_frames = ctx->MakeDim(0);
+      } else {
+        TF_RETURN_IF_ERROR(ctx->Subtract(num_frames, window_size, &num_frames));
+        TF_RETURN_IF_ERROR(
+            ctx->Divide(num_frames, window_step, false, &num_frames));
+        TF_RETURN_IF_ERROR(
+            ctx->Divide(num_frames, frame_stride, false, &num_frames));
+        TF_RETURN_IF_ERROR(ctx->Add(num_frames, 1, &num_frames));
+      }
+
+      int stack_size = 1 + left_context + right_context;
+      DimensionHandle num_features = ctx->MakeDim(num_channels);
+      TF_RETURN_IF_ERROR(
+          ctx->Multiply(num_features, stack_size, &num_features));
+
+      ShapeHandle output = ctx->MakeShape({num_frames, num_features});
+      ctx->set_output(0, output);
+      return absl::OkStatus();
+    })
+    .Doc(R"doc(
+Audio Microfrontend Op.
+
+This Op converts a sequence of audio data into one or more
+feature vectors containing filterbanks of the input. The
+conversion process uses a lightweight library to perform:
+
+1. A slicing window function
+2. Short-time FFTs
+3. Filterbank calculations
+4. Noise reduction
+5. PCAN Auto Gain Control
+6. Logarithmic scaling
+
+Arguments
+  audio: 1D Tensor, int16 audio data in temporal ordering.
+  sample_rate: Integer, the sample rate of the audio in Hz.
+  window_size: Integer, length of desired time frames in ms.
+  window_step: Integer, length of step size for the next frame in ms.
+  num_channels: Integer, the number of filterbank channels to use.
+  upper_band_limit: Float, the highest frequency included in the filterbanks.
+  lower_band_limit: Float, the lowest frequency included in the filterbanks.
+  smoothing_bits: Int, scale up signal by 2^(smoothing_bits) before reduction.
+  even_smoothing: Float, smoothing coefficient for even-numbered channels.
+  odd_smoothing: Float, smoothing coefficient for odd-numbered channels.
+  min_signal_remaining: Float, fraction of signal to preserve in smoothing.
+  enable_pcan: Bool, enable PCAN auto gain control.
+  pcan_strength: Float, gain normalization exponent.
+  pcan_offset: Float, positive value added in the normalization denominator.
+  gain_bits: Int, number of fractional bits in the gain.
+  enable_log: Bool, enable logarithmic scaling of filterbanks.
+  scale_shift: Integer, scale filterbanks by 2^(scale_shift).
+  left_context: Integer, number of preceding frames to attach to each frame.
+  right_context: Integer, number of preceding frames to attach to each frame.
+  frame_stride: Integer, M frames to skip over, where output[n] = frame[n*M].
+  zero_padding: Bool, if left/right context is out-of-bounds, attach frame of
+                zeroes. Otherwise, frame[0] or frame[size-1] will be copied.
+  out_scale: Integer, divide all filterbanks by this number.
+  out_type: DType, type of the output Tensor, defaults to UINT16.
+
+Returns
+  filterbanks: 2D Tensor, each row is a time frame, each column is a channel.
+)doc");
+
+template <typename T>
+class AudioMicrofrontendOp : public OpKernel {
+ public:
+  explicit AudioMicrofrontendOp(OpKernelConstruction* ctx) : OpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("sample_rate", &sample_rate_));
+
+    int window_size;
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("window_size", &window_size));
+    config_.window.size_ms = window_size;
+
+    int window_step;
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("window_step", &window_step));
+    config_.window.step_size_ms = window_step;
+
+    OP_REQUIRES_OK(
+        ctx, ctx->GetAttr("num_channels", &config_.filterbank.num_channels));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("upper_band_limit",
+                                     &config_.filterbank.upper_band_limit));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("lower_band_limit",
+                                     &config_.filterbank.lower_band_limit));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("smoothing_bits",
+                                     &config_.noise_reduction.smoothing_bits));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("even_smoothing",
+                                     &config_.noise_reduction.even_smoothing));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("odd_smoothing",
+                                     &config_.noise_reduction.odd_smoothing));
+    OP_REQUIRES_OK(ctx,
+                   ctx->GetAttr("min_signal_remaining",
+                                &config_.noise_reduction.min_signal_remaining));
+
+    bool enable_pcan;
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("enable_pcan", &enable_pcan));
+    config_.pcan_gain_control.enable_pcan = enable_pcan;
+
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("pcan_strength",
+                                     &config_.pcan_gain_control.strength));
+    OP_REQUIRES_OK(
+        ctx, ctx->GetAttr("pcan_offset", &config_.pcan_gain_control.offset));
+    OP_REQUIRES_OK(
+        ctx, ctx->GetAttr("gain_bits", &config_.pcan_gain_control.gain_bits));
+
+    bool enable_log;
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("enable_log", &enable_log));
+    config_.log_scale.enable_log = enable_log;
+
+    OP_REQUIRES_OK(ctx,
+                   ctx->GetAttr("scale_shift", &config_.log_scale.scale_shift));
+
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("left_context", &left_context_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("right_context", &right_context_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("frame_stride", &frame_stride_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("zero_padding", &zero_padding_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("out_scale", &out_scale_));
+  }
+
+  void Compute(OpKernelContext* ctx) override {
+    const Tensor* audio;
+    OP_REQUIRES_OK(ctx, ctx->input("audio", &audio));
+    OP_REQUIRES(ctx, TensorShapeUtils::IsVector(audio->shape()),
+                InvalidArgument("audio is not a vector"));
+
+    auto audio_data =
+        reinterpret_cast<const int16_t*>(audio->tensor_data().data());
+    int audio_size = audio->NumElements();
+
+    Tensor* filterbanks = nullptr;
+    int window_size = config_.window.size_ms * sample_rate_ / 1000;
+    int window_step = config_.window.step_size_ms * sample_rate_ / 1000;
+    int num_frames = 0;
+    int sampled_frames = 0;
+    if (audio_size >= window_size) {
+      num_frames = (audio_size - window_size) / window_step + 1;
+      sampled_frames = (num_frames - 1) / frame_stride_ + 1;
+    }
+    TensorShape filterbanks_shape{
+        sampled_frames,
+        config_.filterbank.num_channels * (1 + left_context_ + right_context_)};
+    OP_REQUIRES_OK(ctx,
+                   ctx->allocate_output(0, filterbanks_shape, &filterbanks));
+    auto filterbanks_flat = filterbanks->flat<T>();
+
+    struct FrontendState state;
+    if (!TF_PREDICT_TRUE(
+            FrontendPopulateState(&config_, &state, sample_rate_))) {
+      ctx->CtxFailure(__FILE__, __LINE__,
+                      Internal("failed to populate frontend state"));
+      FrontendFreeStateContents(&state);
+      return;
+    }
+
+    std::vector<std::vector<T>> frame_buffer(num_frames);
+    int frame_index = 0;
+    while (audio_size > 0) {
+      size_t num_samples_read;
+      struct FrontendOutput output = FrontendProcessSamples(
+          &state, audio_data, audio_size, &num_samples_read);
+      audio_data += num_samples_read;
+      audio_size -= num_samples_read;
+
+      if (output.values != nullptr) {
+        frame_buffer[frame_index].reserve(output.size);
+        int i;
+        for (i = 0; i < output.size; ++i) {
+          frame_buffer[frame_index].push_back(static_cast<T>(output.values[i]) /
+                                              out_scale_);
+        }
+        ++frame_index;
+      }
+    }
+    FrontendFreeStateContents(&state);
+
+    int index = 0;
+    std::vector<T> pad(config_.filterbank.num_channels, 0);
+    int anchor;
+    for (anchor = 0; anchor < frame_buffer.size(); anchor += frame_stride_) {
+      int frame;
+      for (frame = anchor - left_context_; frame <= anchor + right_context_;
+           ++frame) {
+        std::vector<T>* feature;
+        if (zero_padding_ && (frame < 0 || frame >= frame_buffer.size())) {
+          feature = &pad;
+        } else if (frame < 0) {
+          feature = &frame_buffer[0];
+        } else if (frame >= frame_buffer.size()) {
+          feature = &frame_buffer[frame_buffer.size() - 1];
+        } else {
+          feature = &frame_buffer[frame];
+        }
+        for (auto f : *feature) {
+          filterbanks_flat(index++) = f;
+        }
+      }
+    }
+  }
+
+ protected:
+  int sample_rate_;
+  struct FrontendConfig config_;
+  int left_context_;
+  int right_context_;
+  int frame_stride_;
+  bool zero_padding_;
+  int out_scale_;
+
+  AudioMicrofrontendOp(const AudioMicrofrontendOp&) = delete;
+  void operator=(const AudioMicrofrontendOp&) = delete;
+};
+
+REGISTER_KERNEL_BUILDER(Name("AudioMicrofrontend")
+                            .Device(tensorflow::DEVICE_CPU)
+                            .TypeConstraint<uint16_t>("out_type"),
+                        AudioMicrofrontendOp<uint16_t>);
+REGISTER_KERNEL_BUILDER(Name("AudioMicrofrontend")
+                            .Device(tensorflow::DEVICE_CPU)
+                            .TypeConstraint<float>("out_type"),
+                        AudioMicrofrontendOp<float>);
+}  // namespace tensorflow

tensorflow/lite/experimental/microfrontend/python/kernel_tests/audio_microfrontend_op_test.py

@@ -0,0 +1,167 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for AudioMicrofrontend."""
+
+import tensorflow as tf
+
+from tensorflow.lite.experimental.microfrontend.python.ops import audio_microfrontend_op as frontend_op
+from tensorflow.python.framework import ops
+
+SAMPLE_RATE = 1000
+WINDOW_SIZE = 25
+WINDOW_STEP = 10
+NUM_CHANNELS = 2
+UPPER_BAND_LIMIT = 450.0
+LOWER_BAND_LIMIT = 8.0
+SMOOTHING_BITS = 10
+
+
+class AudioFeatureGenerationTest(tf.test.TestCase):
+
+  def setUp(self):
+    super(AudioFeatureGenerationTest, self).setUp()
+    ops.disable_eager_execution()
+
+  def testSimple(self):
+    with self.test_session():
+      audio = tf.constant(
+          [0, 32767, 0, -32768] * ((WINDOW_SIZE + 4 * WINDOW_STEP) // 4),
+          tf.int16)
+      filterbanks = frontend_op.audio_microfrontend(
+          audio,
+          sample_rate=SAMPLE_RATE,
+          window_size=WINDOW_SIZE,
+          window_step=WINDOW_STEP,
+          num_channels=NUM_CHANNELS,
+          upper_band_limit=UPPER_BAND_LIMIT,
+          lower_band_limit=LOWER_BAND_LIMIT,
+          smoothing_bits=SMOOTHING_BITS,
+          enable_pcan=True)
+      self.assertAllEqual(
+          self.evaluate(filterbanks),
+          [[479, 425], [436, 378], [410, 350], [391, 325]])
+
+  def testSimpleFloatScaled(self):
+    with self.test_session():
+      audio = tf.constant(
+          [0, 32767, 0, -32768] * ((WINDOW_SIZE + 4 * WINDOW_STEP) // 4),
+          tf.int16)
+      filterbanks = frontend_op.audio_microfrontend(
+          audio,
+          sample_rate=SAMPLE_RATE,
+          window_size=WINDOW_SIZE,
+          window_step=WINDOW_STEP,
+          num_channels=NUM_CHANNELS,
+          upper_band_limit=UPPER_BAND_LIMIT,
+          lower_band_limit=LOWER_BAND_LIMIT,
+          smoothing_bits=SMOOTHING_BITS,
+          enable_pcan=True,
+          out_scale=64,
+          out_type=tf.float32)
+      self.assertAllEqual(
+          self.evaluate(filterbanks),
+          [[7.484375, 6.640625], [6.8125, 5.90625], [6.40625, 5.46875],
+           [6.109375, 5.078125]])
+
+  def testStacking(self):
+    with self.test_session():
+      audio = tf.constant(
+          [0, 32767, 0, -32768] * ((WINDOW_SIZE + 4 * WINDOW_STEP) // 4),
+          tf.int16)
+      filterbanks = frontend_op.audio_microfrontend(
+          audio,
+          sample_rate=SAMPLE_RATE,
+          window_size=WINDOW_SIZE,
+          window_step=WINDOW_STEP,
+          num_channels=NUM_CHANNELS,
+          upper_band_limit=UPPER_BAND_LIMIT,
+          lower_band_limit=LOWER_BAND_LIMIT,
+          smoothing_bits=SMOOTHING_BITS,
+          enable_pcan=True,
+          right_context=1,
+          frame_stride=2)
+      self.assertAllEqual(
+          self.evaluate(filterbanks),
+          [[479, 425, 436, 378], [410, 350, 391, 325]])
+
+  def testStackingWithOverlap(self):
+    with self.test_session():
+      audio = tf.constant(
+          [0, 32767, 0, -32768] * ((WINDOW_SIZE + 4 * WINDOW_STEP) // 4),
+          tf.int16)
+      filterbanks = frontend_op.audio_microfrontend(
+          audio,
+          sample_rate=SAMPLE_RATE,
+          window_size=WINDOW_SIZE,
+          window_step=WINDOW_STEP,
+          num_channels=NUM_CHANNELS,
+          upper_band_limit=UPPER_BAND_LIMIT,
+          lower_band_limit=LOWER_BAND_LIMIT,
+          smoothing_bits=SMOOTHING_BITS,
+          enable_pcan=True,
+          left_context=1,
+          right_context=1)
+      self.assertAllEqual(
+          self.evaluate(filterbanks),
+          [[479, 425, 479, 425, 436, 378], [479, 425, 436, 378, 410, 350],
+           [436, 378, 410, 350, 391, 325], [410, 350, 391, 325, 391, 325]])
+
+  def testStackingDropFrame(self):
+    with self.test_session():
+      audio = tf.constant(
+          [0, 32767, 0, -32768] * ((WINDOW_SIZE + 4 * WINDOW_STEP) // 4),
+          tf.int16)
+      filterbanks = frontend_op.audio_microfrontend(
+          audio,
+          sample_rate=SAMPLE_RATE,
+          window_size=WINDOW_SIZE,
+          window_step=WINDOW_STEP,
+          num_channels=NUM_CHANNELS,
+          upper_band_limit=UPPER_BAND_LIMIT,
+          lower_band_limit=LOWER_BAND_LIMIT,
+          smoothing_bits=SMOOTHING_BITS,
+          enable_pcan=True,
+          left_context=1,
+          frame_stride=2)
+      self.assertAllEqual(
+          self.evaluate(filterbanks),
+          [[479, 425, 479, 425], [436, 378, 410, 350]])
+
+  def testZeroPadding(self):
+    with self.test_session():
+      audio = tf.constant(
+          [0, 32767, 0, -32768] * ((WINDOW_SIZE + 7 * WINDOW_STEP) // 4),
+          tf.int16)
+      filterbanks = frontend_op.audio_microfrontend(
+          audio,
+          sample_rate=SAMPLE_RATE,
+          window_size=WINDOW_SIZE,
+          window_step=WINDOW_STEP,
+          num_channels=NUM_CHANNELS,
+          upper_band_limit=UPPER_BAND_LIMIT,
+          lower_band_limit=LOWER_BAND_LIMIT,
+          smoothing_bits=SMOOTHING_BITS,
+          enable_pcan=True,
+          left_context=2,
+          frame_stride=3,
+          zero_padding=True)
+      self.assertAllEqual(
+          self.evaluate(filterbanks),
+          [[0, 0, 0, 0, 479, 425], [436, 378, 410, 350, 391, 325],
+           [374, 308, 362, 292, 352, 275]])
+
+
+if __name__ == '__main__':
+  tf.test.main()

tensorflow/lite/experimental/microfrontend/python/ops/audio_microfrontend_op.py

@@ -0,0 +1,110 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""AudioMicrofrontend Op creates filterbanks from audio data."""
+
+from tensorflow.lite.experimental.microfrontend.ops import gen_audio_microfrontend_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import load_library
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.platform import resource_loader
+
+_audio_microfrontend_op = load_library.load_op_library(
+    resource_loader.get_path_to_datafile("_audio_microfrontend_op.so"))
+
+
+def audio_microfrontend(audio,
+                        sample_rate=16000,
+                        window_size=25,
+                        window_step=10,
+                        num_channels=32,
+                        upper_band_limit=7500.0,
+                        lower_band_limit=125.0,
+                        smoothing_bits=10,
+                        even_smoothing=0.025,
+                        odd_smoothing=0.06,
+                        min_signal_remaining=0.05,
+                        enable_pcan=True,
+                        pcan_strength=0.95,
+                        pcan_offset=80.0,
+                        gain_bits=21,
+                        enable_log=True,
+                        scale_shift=6,
+                        left_context=0,
+                        right_context=0,
+                        frame_stride=1,
+                        zero_padding=False,
+                        out_scale=1,
+                        out_type=dtypes.uint16):
+  """Audio Microfrontend Op.
+
+  This Op converts a sequence of audio data into one or more
+  feature vectors containing filterbanks of the input. The
+  conversion process uses a lightweight library to perform:
+
+  1. A slicing window function
+  2. Short-time FFTs
+  3. Filterbank calculations
+  4. Noise reduction
+  5. PCAN Auto Gain Control
+  6. Logarithmic scaling
+
+  Args:
+    audio: 1D Tensor, int16 audio data in temporal ordering.
+    sample_rate: Integer, the sample rate of the audio in Hz.
+    window_size: Integer, length of desired time frames in ms.
+    window_step: Integer, length of step size for the next frame in ms.
+    num_channels: Integer, the number of filterbank channels to use.
+    upper_band_limit: Float, the highest frequency included in the filterbanks.
+    lower_band_limit: Float, the lowest frequency included in the filterbanks.
+    smoothing_bits: Int, scale up signal by 2^(smoothing_bits) before reduction.
+    even_smoothing: Float, smoothing coefficient for even-numbered channels.
+    odd_smoothing: Float, smoothing coefficient for odd-numbered channels.
+    min_signal_remaining: Float, fraction of signal to preserve in smoothing.
+    enable_pcan: Bool, enable PCAN auto gain control.
+    pcan_strength: Float, gain normalization exponent.
+    pcan_offset: Float, positive value added in the normalization denominator.
+    gain_bits: Int, number of fractional bits in the gain.
+    enable_log: Bool, enable logarithmic scaling of filterbanks.
+    scale_shift: Integer, scale filterbanks by 2^(scale_shift).
+    left_context: Integer, number of preceding frames to attach to each frame.
+    right_context: Integer, number of preceding frames to attach to each frame.
+    frame_stride: Integer, M frames to skip over, where output[n] = frame[n*M].
+    zero_padding: Bool, if left/right context is out-of-bounds, attach frame of
+      zeroes. Otherwise, frame[0] or frame[size-1] will be copied.
+    out_scale: Integer, divide all filterbanks by this number.
+    out_type: DType, type of the output Tensor, defaults to UINT16.
+
+  Returns:
+    filterbanks: 2D Tensor, each row is a time frame, each column is a channel.
+
+  Raises:
+    ValueError: If the audio tensor is not explicitly a vector.
+  """
+  audio_shape = audio.shape
+  if audio_shape.ndims is None:
+    raise ValueError("Input to `AudioMicrofrontend` should have known rank.")
+  if len(audio_shape) > 1:
+    audio = array_ops.reshape(audio, [-1])
+
+  return gen_audio_microfrontend_op.audio_microfrontend(
+      audio, sample_rate, window_size, window_step, num_channels,
+      upper_band_limit, lower_band_limit, smoothing_bits, even_smoothing,
+      odd_smoothing, min_signal_remaining, enable_pcan, pcan_strength,
+      pcan_offset, gain_bits, enable_log, scale_shift, left_context,
+      right_context, frame_stride, zero_padding, out_scale, out_type)
+
+
+ops.NotDifferentiable("AudioMicrofrontend")