chore: import upstream snapshot with attribution

tensorflow/lite/experimental/microfrontend/python/kernel_tests/audio_microfrontend_op_test.py

@@ -0,0 +1,167 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for AudioMicrofrontend."""
+
+import tensorflow as tf
+
+from tensorflow.lite.experimental.microfrontend.python.ops import audio_microfrontend_op as frontend_op
+from tensorflow.python.framework import ops
+
+SAMPLE_RATE = 1000
+WINDOW_SIZE = 25
+WINDOW_STEP = 10
+NUM_CHANNELS = 2
+UPPER_BAND_LIMIT = 450.0
+LOWER_BAND_LIMIT = 8.0
+SMOOTHING_BITS = 10
+
+
+class AudioFeatureGenerationTest(tf.test.TestCase):
+
+  def setUp(self):
+    super(AudioFeatureGenerationTest, self).setUp()
+    ops.disable_eager_execution()
+
+  def testSimple(self):
+    with self.test_session():
+      audio = tf.constant(
+          [0, 32767, 0, -32768] * ((WINDOW_SIZE + 4 * WINDOW_STEP) // 4),
+          tf.int16)
+      filterbanks = frontend_op.audio_microfrontend(
+          audio,
+          sample_rate=SAMPLE_RATE,
+          window_size=WINDOW_SIZE,
+          window_step=WINDOW_STEP,
+          num_channels=NUM_CHANNELS,
+          upper_band_limit=UPPER_BAND_LIMIT,
+          lower_band_limit=LOWER_BAND_LIMIT,
+          smoothing_bits=SMOOTHING_BITS,
+          enable_pcan=True)
+      self.assertAllEqual(
+          self.evaluate(filterbanks),
+          [[479, 425], [436, 378], [410, 350], [391, 325]])
+
+  def testSimpleFloatScaled(self):
+    with self.test_session():
+      audio = tf.constant(
+          [0, 32767, 0, -32768] * ((WINDOW_SIZE + 4 * WINDOW_STEP) // 4),
+          tf.int16)
+      filterbanks = frontend_op.audio_microfrontend(
+          audio,
+          sample_rate=SAMPLE_RATE,
+          window_size=WINDOW_SIZE,
+          window_step=WINDOW_STEP,
+          num_channels=NUM_CHANNELS,
+          upper_band_limit=UPPER_BAND_LIMIT,
+          lower_band_limit=LOWER_BAND_LIMIT,
+          smoothing_bits=SMOOTHING_BITS,
+          enable_pcan=True,
+          out_scale=64,
+          out_type=tf.float32)
+      self.assertAllEqual(
+          self.evaluate(filterbanks),
+          [[7.484375, 6.640625], [6.8125, 5.90625], [6.40625, 5.46875],
+           [6.109375, 5.078125]])
+
+  def testStacking(self):
+    with self.test_session():
+      audio = tf.constant(
+          [0, 32767, 0, -32768] * ((WINDOW_SIZE + 4 * WINDOW_STEP) // 4),
+          tf.int16)
+      filterbanks = frontend_op.audio_microfrontend(
+          audio,
+          sample_rate=SAMPLE_RATE,
+          window_size=WINDOW_SIZE,
+          window_step=WINDOW_STEP,
+          num_channels=NUM_CHANNELS,
+          upper_band_limit=UPPER_BAND_LIMIT,
+          lower_band_limit=LOWER_BAND_LIMIT,
+          smoothing_bits=SMOOTHING_BITS,
+          enable_pcan=True,
+          right_context=1,
+          frame_stride=2)
+      self.assertAllEqual(
+          self.evaluate(filterbanks),
+          [[479, 425, 436, 378], [410, 350, 391, 325]])
+
+  def testStackingWithOverlap(self):
+    with self.test_session():
+      audio = tf.constant(
+          [0, 32767, 0, -32768] * ((WINDOW_SIZE + 4 * WINDOW_STEP) // 4),
+          tf.int16)
+      filterbanks = frontend_op.audio_microfrontend(
+          audio,
+          sample_rate=SAMPLE_RATE,
+          window_size=WINDOW_SIZE,
+          window_step=WINDOW_STEP,
+          num_channels=NUM_CHANNELS,
+          upper_band_limit=UPPER_BAND_LIMIT,
+          lower_band_limit=LOWER_BAND_LIMIT,
+          smoothing_bits=SMOOTHING_BITS,
+          enable_pcan=True,
+          left_context=1,
+          right_context=1)
+      self.assertAllEqual(
+          self.evaluate(filterbanks),
+          [[479, 425, 479, 425, 436, 378], [479, 425, 436, 378, 410, 350],
+           [436, 378, 410, 350, 391, 325], [410, 350, 391, 325, 391, 325]])
+
+  def testStackingDropFrame(self):
+    with self.test_session():
+      audio = tf.constant(
+          [0, 32767, 0, -32768] * ((WINDOW_SIZE + 4 * WINDOW_STEP) // 4),
+          tf.int16)
+      filterbanks = frontend_op.audio_microfrontend(
+          audio,
+          sample_rate=SAMPLE_RATE,
+          window_size=WINDOW_SIZE,
+          window_step=WINDOW_STEP,
+          num_channels=NUM_CHANNELS,
+          upper_band_limit=UPPER_BAND_LIMIT,
+          lower_band_limit=LOWER_BAND_LIMIT,
+          smoothing_bits=SMOOTHING_BITS,
+          enable_pcan=True,
+          left_context=1,
+          frame_stride=2)
+      self.assertAllEqual(
+          self.evaluate(filterbanks),
+          [[479, 425, 479, 425], [436, 378, 410, 350]])
+
+  def testZeroPadding(self):
+    with self.test_session():
+      audio = tf.constant(
+          [0, 32767, 0, -32768] * ((WINDOW_SIZE + 7 * WINDOW_STEP) // 4),
+          tf.int16)
+      filterbanks = frontend_op.audio_microfrontend(
+          audio,
+          sample_rate=SAMPLE_RATE,
+          window_size=WINDOW_SIZE,
+          window_step=WINDOW_STEP,
+          num_channels=NUM_CHANNELS,
+          upper_band_limit=UPPER_BAND_LIMIT,
+          lower_band_limit=LOWER_BAND_LIMIT,
+          smoothing_bits=SMOOTHING_BITS,
+          enable_pcan=True,
+          left_context=2,
+          frame_stride=3,
+          zero_padding=True)
+      self.assertAllEqual(
+          self.evaluate(filterbanks),
+          [[0, 0, 0, 0, 479, 425], [436, 378, 410, 350, 391, 325],
+           [374, 308, 362, 292, 352, 275]])
+
+
+if __name__ == '__main__':
+  tf.test.main()

tensorflow/lite/experimental/microfrontend/python/ops/audio_microfrontend_op.py

@@ -0,0 +1,110 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""AudioMicrofrontend Op creates filterbanks from audio data."""
+
+from tensorflow.lite.experimental.microfrontend.ops import gen_audio_microfrontend_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import load_library
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.platform import resource_loader
+
+_audio_microfrontend_op = load_library.load_op_library(
+    resource_loader.get_path_to_datafile("_audio_microfrontend_op.so"))
+
+
+def audio_microfrontend(audio,
+                        sample_rate=16000,
+                        window_size=25,
+                        window_step=10,
+                        num_channels=32,
+                        upper_band_limit=7500.0,
+                        lower_band_limit=125.0,
+                        smoothing_bits=10,
+                        even_smoothing=0.025,
+                        odd_smoothing=0.06,
+                        min_signal_remaining=0.05,
+                        enable_pcan=True,
+                        pcan_strength=0.95,
+                        pcan_offset=80.0,
+                        gain_bits=21,
+                        enable_log=True,
+                        scale_shift=6,
+                        left_context=0,
+                        right_context=0,
+                        frame_stride=1,
+                        zero_padding=False,
+                        out_scale=1,
+                        out_type=dtypes.uint16):
+  """Audio Microfrontend Op.
+
+  This Op converts a sequence of audio data into one or more
+  feature vectors containing filterbanks of the input. The
+  conversion process uses a lightweight library to perform:
+
+  1. A slicing window function
+  2. Short-time FFTs
+  3. Filterbank calculations
+  4. Noise reduction
+  5. PCAN Auto Gain Control
+  6. Logarithmic scaling
+
+  Args:
+    audio: 1D Tensor, int16 audio data in temporal ordering.
+    sample_rate: Integer, the sample rate of the audio in Hz.
+    window_size: Integer, length of desired time frames in ms.
+    window_step: Integer, length of step size for the next frame in ms.
+    num_channels: Integer, the number of filterbank channels to use.
+    upper_band_limit: Float, the highest frequency included in the filterbanks.
+    lower_band_limit: Float, the lowest frequency included in the filterbanks.
+    smoothing_bits: Int, scale up signal by 2^(smoothing_bits) before reduction.
+    even_smoothing: Float, smoothing coefficient for even-numbered channels.
+    odd_smoothing: Float, smoothing coefficient for odd-numbered channels.
+    min_signal_remaining: Float, fraction of signal to preserve in smoothing.
+    enable_pcan: Bool, enable PCAN auto gain control.
+    pcan_strength: Float, gain normalization exponent.
+    pcan_offset: Float, positive value added in the normalization denominator.
+    gain_bits: Int, number of fractional bits in the gain.
+    enable_log: Bool, enable logarithmic scaling of filterbanks.
+    scale_shift: Integer, scale filterbanks by 2^(scale_shift).
+    left_context: Integer, number of preceding frames to attach to each frame.
+    right_context: Integer, number of preceding frames to attach to each frame.
+    frame_stride: Integer, M frames to skip over, where output[n] = frame[n*M].
+    zero_padding: Bool, if left/right context is out-of-bounds, attach frame of
+      zeroes. Otherwise, frame[0] or frame[size-1] will be copied.
+    out_scale: Integer, divide all filterbanks by this number.
+    out_type: DType, type of the output Tensor, defaults to UINT16.
+
+  Returns:
+    filterbanks: 2D Tensor, each row is a time frame, each column is a channel.
+
+  Raises:
+    ValueError: If the audio tensor is not explicitly a vector.
+  """
+  audio_shape = audio.shape
+  if audio_shape.ndims is None:
+    raise ValueError("Input to `AudioMicrofrontend` should have known rank.")
+  if len(audio_shape) > 1:
+    audio = array_ops.reshape(audio, [-1])
+
+  return gen_audio_microfrontend_op.audio_microfrontend(
+      audio, sample_rate, window_size, window_step, num_channels,
+      upper_band_limit, lower_band_limit, smoothing_bits, even_smoothing,
+      odd_smoothing, min_signal_remaining, enable_pcan, pcan_strength,
+      pcan_offset, gain_bits, enable_log, scale_shift, left_context,
+      right_context, frame_stride, zero_padding, out_scale, out_type)
+
+
+ops.NotDifferentiable("AudioMicrofrontend")