chore: import upstream snapshot with attribution

python/paddle/audio/features/layers.py

@@ -0,0 +1,448 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from functools import partial
+from typing import TYPE_CHECKING, Literal, TypeAlias
+
+import paddle
+from paddle import nn
+
+from ..functional import compute_fbank_matrix, create_dct, power_to_db
+from ..functional.window import get_window
+
+if TYPE_CHECKING:
+    from paddle import Tensor
+
+_WindowLiteral: TypeAlias = Literal[
+    'hamming',
+    'hann',
+    'kaiser',
+    'bartlett',
+    'nuttall',
+    'gaussian',
+    'exponential',
+    'triang',
+    'bohman',
+    'blackman',
+    'cosine',
+    'tukey',
+    'taylor',
+]
+
+
+class Spectrogram(nn.Layer):
+    """Compute spectrogram of given signals, typically audio waveforms.
+    The spectrogram is defined as the complex norm of the short-time Fourier transformation.
+
+    Args:
+        n_fft (int, optional): The number of frequency components of the discrete Fourier transform. Defaults to 512.
+        hop_length (Optional[int], optional): The hop length of the short time FFT. If `None`, it is set to `win_length//4`. Defaults to None.
+        win_length (Optional[int], optional): The window length of the short time FFT. If `None`, it is set to same as `n_fft`. Defaults to None.
+        window (str, optional): The window function applied to the signal before the Fourier transform. Supported window functions: 'hamming', 'hann', 'gaussian', 'exponential', 'triang', 'bohman', 'blackman', 'cosine', 'tukey', 'taylor', 'bartlett', 'kaiser', 'nuttall'. Defaults to 'hann'.
+        power (float, optional): Exponent for the magnitude spectrogram. Defaults to 2.0.
+        center (bool, optional): Whether to pad `x` to make that the :math:`t \times hop\\_length` at the center of `t`-th frame. Defaults to True.
+        pad_mode (str, optional): Choose padding pattern when `center` is `True`. Defaults to 'reflect'.
+        dtype (str, optional): Data type of input and window. Defaults to 'float32'.
+
+    Returns:
+        :ref:`api_paddle_nn_Layer`. An instance of Spectrogram.
+
+
+
+    Examples:
+        .. code-block:: pycon
+
+            >>> import paddle
+            >>> from paddle.audio.features import Spectrogram
+
+            >>> sample_rate = 16000
+            >>> wav_duration = 0.5
+            >>> num_channels = 1
+            >>> num_frames = sample_rate * wav_duration
+            >>> wav_data = paddle.linspace(-1.0, 1.0, int(num_frames)) * 0.1
+            >>> waveform = wav_data.tile([num_channels, 1])
+
+            >>> feature_extractor = Spectrogram(n_fft=512, window='hann', power=1.0)
+            >>> feats = feature_extractor(waveform)
+    """
+
+    power: float
+    fft_window: Tensor
+
+    def __init__(
+        self,
+        n_fft: int = 512,
+        hop_length: int | None = 512,
+        win_length: int | None = None,
+        window: _WindowLiteral = 'hann',
+        power: float = 1.0,
+        center: bool = True,
+        pad_mode: Literal['reflect'] = 'reflect',
+        dtype: str = 'float32',
+    ) -> None:
+        super().__init__()
+
+        assert power > 0, 'Power of spectrogram must be > 0.'
+        self.power = power
+
+        if win_length is None:
+            win_length = n_fft
+
+        self.fft_window = get_window(
+            window, win_length, fftbins=True, dtype=dtype
+        )
+        self._stft = partial(
+            paddle.signal.stft,
+            n_fft=n_fft,
+            hop_length=hop_length,
+            win_length=win_length,
+            window=self.fft_window,
+            center=center,
+            pad_mode=pad_mode,
+        )
+        self.register_buffer('fft_window', self.fft_window)
+
+    def forward(self, x: Tensor) -> Tensor:
+        """
+        Args:
+            x (Tensor): Tensor of waveforms with shape `(N, T)`
+
+        Returns:
+            Tensor: Spectrograms with shape `(N, n_fft//2 + 1, num_frames)`.
+        """
+        stft = self._stft(x)
+        spectrogram = paddle.pow(paddle.abs(stft), self.power)
+        return spectrogram
+
+
+class MelSpectrogram(nn.Layer):
+    """Compute the melspectrogram of given signals, typically audio waveforms. It is computed by multiplying spectrogram with Mel filter bank matrix.
+
+    Args:
+        sr (int, optional): Sample rate. Defaults to 22050.
+        n_fft (int, optional): The number of frequency components of the discrete Fourier transform. Defaults to 512.
+        hop_length (Optional[int], optional): The hop length of the short time FFT. If `None`, it is set to `win_length//4`. Defaults to None.
+        win_length (Optional[int], optional): The window length of the short time FFT. If `None`, it is set to same as `n_fft`. Defaults to None.
+        window (str, optional): The window function applied to the signal before the Fourier transform. Supported window functions: 'hamming', 'hann', 'gaussian', 'exponential', 'triang', 'bohman', 'blackman', 'cosine', 'tukey', 'taylor', 'bartlett', 'kaiser', 'nuttall'. Defaults to 'hann'.
+        power (float, optional): Exponent for the magnitude spectrogram. Defaults to 2.0.
+        center (bool, optional): Whether to pad `x` to make that the :math:`t \times hop\\_length` at the center of `t`-th frame. Defaults to True.
+        pad_mode (str, optional): Choose padding pattern when `center` is `True`. Defaults to 'reflect'.
+        n_mels (int, optional): Number of mel bins. Defaults to 64.
+        f_min (float, optional): Minimum frequency in Hz. Defaults to 50.0.
+        f_max (Optional[float], optional): Maximum frequency in Hz. Defaults to None.
+        htk (bool, optional): Use HTK formula in computing fbank matrix. Defaults to False.
+        norm (Union[str, float], optional): Type of normalization in computing fbank matrix. Slaney-style is used by default. You can specify norm=1.0/2.0 to use customized p-norm normalization. Defaults to 'slaney'.
+        dtype (str, optional): Data type of input and window. Defaults to 'float32'.
+
+    Returns:
+        :ref:`api_paddle_nn_Layer`. An instance of MelSpectrogram.
+
+    Examples:
+        .. code-block:: pycon
+
+            >>> import paddle
+            >>> from paddle.audio.features import MelSpectrogram
+
+            >>> sample_rate = 16000
+            >>> wav_duration = 0.5
+            >>> num_channels = 1
+            >>> num_frames = sample_rate * wav_duration
+            >>> wav_data = paddle.linspace(-1.0, 1.0, int(num_frames)) * 0.1
+            >>> waveform = wav_data.tile([num_channels, 1])
+
+            >>> feature_extractor = MelSpectrogram(sr=sample_rate, n_fft=512, window='hann', power=1.0)
+            >>> feats = feature_extractor(waveform)
+    """
+
+    n_mels: int
+    f_min: float
+    f_max: float
+    htk: bool
+    norm: Literal['slaney'] | float
+    fbank_matrix: Tensor
+
+    def __init__(
+        self,
+        sr: int = 22050,
+        n_fft: int = 2048,
+        hop_length: int | None = 512,
+        win_length: int | None = None,
+        window: _WindowLiteral = 'hann',
+        power: float = 2.0,
+        center: bool = True,
+        pad_mode: Literal['reflect'] = 'reflect',
+        n_mels: int = 64,
+        f_min: float = 50.0,
+        f_max: float | None = None,
+        htk: bool = False,
+        norm: Literal['slaney'] | float = 'slaney',
+        dtype: str = 'float32',
+    ) -> None:
+        super().__init__()
+
+        self._spectrogram = Spectrogram(
+            n_fft=n_fft,
+            hop_length=hop_length,
+            win_length=win_length,
+            window=window,
+            power=power,
+            center=center,
+            pad_mode=pad_mode,
+            dtype=dtype,
+        )
+        self.n_mels = n_mels
+        self.f_min = f_min
+        self.f_max = f_max
+        self.htk = htk
+        self.norm = norm
+        if f_max is None:
+            f_max = sr // 2
+        self.fbank_matrix = compute_fbank_matrix(
+            sr=sr,
+            n_fft=n_fft,
+            n_mels=n_mels,
+            f_min=f_min,
+            f_max=f_max,
+            htk=htk,
+            norm=norm,
+            dtype=dtype,
+        )
+        self.register_buffer('fbank_matrix', self.fbank_matrix)
+
+    def forward(self, x: Tensor) -> Tensor:
+        """
+        Args:
+            x (Tensor): Tensor of waveforms with shape `(N, T)`
+
+        Returns:
+            Tensor: Mel spectrograms with shape `(N, n_mels, num_frames)`.
+        """
+        spect_feature = self._spectrogram(x)
+        mel_feature = paddle.matmul(self.fbank_matrix, spect_feature)
+        return mel_feature
+
+
+class LogMelSpectrogram(nn.Layer):
+    """Compute log-mel-spectrogram feature of given signals, typically audio waveforms.
+
+    Args:
+        sr (int, optional): Sample rate. Defaults to 22050.
+        n_fft (int, optional): The number of frequency components of the discrete Fourier transform. Defaults to 512.
+        hop_length (Optional[int], optional): The hop length of the short time FFT. If `None`, it is set to `win_length//4`. Defaults to None.
+        win_length (Optional[int], optional): The window length of the short time FFT. If `None`, it is set to same as `n_fft`. Defaults to None.
+        window (str, optional): The window function applied to the signal before the Fourier transform. Supported window functions: 'hamming', 'hann', 'gaussian', 'exponential', 'triang', 'bohman', 'blackman', 'cosine', 'tukey', 'taylor', 'bartlett', 'kaiser', 'nuttall'. Defaults to 'hann'.
+        power (float, optional): Exponent for the magnitude spectrogram. Defaults to 2.0.
+        center (bool, optional): Whether to pad `x` to make that the :math:`t \times hop\\_length` at the center of `t`-th frame. Defaults to True.
+        pad_mode (str, optional): Choose padding pattern when `center` is `True`. Defaults to 'reflect'.
+        n_mels (int, optional): Number of mel bins. Defaults to 64.
+        f_min (float, optional): Minimum frequency in Hz. Defaults to 50.0.
+        f_max (Optional[float], optional): Maximum frequency in Hz. Defaults to None.
+        htk (bool, optional): Use HTK formula in computing fbank matrix. Defaults to False.
+        norm (Union[str, float], optional): Type of normalization in computing fbank matrix. Slaney-style is used by default. You can specify norm=1.0/2.0 to use customized p-norm normalization. Defaults to 'slaney'.
+        ref_value (float, optional): The reference value. If smaller than 1.0, the db level of the signal will be pulled up accordingly. Otherwise, the db level is pushed down. Defaults to 1.0.
+        amin (float, optional): The minimum value of input magnitude. Defaults to 1e-10.
+        top_db (Optional[float], optional): The maximum db value of spectrogram. Defaults to None.
+        dtype (str, optional): Data type of input and window. Defaults to 'float32'.
+
+    Returns:
+        :ref:`api_paddle_nn_Layer`. An instance of LogMelSpectrogram.
+
+    Examples:
+        .. code-block:: pycon
+
+            >>> import paddle
+            >>> from paddle.audio.features import LogMelSpectrogram
+
+            >>> sample_rate = 16000
+            >>> wav_duration = 0.5
+            >>> num_channels = 1
+            >>> num_frames = sample_rate * wav_duration
+            >>> wav_data = paddle.linspace(-1.0, 1.0, int(num_frames)) * 0.1
+            >>> waveform = wav_data.tile([num_channels, 1])
+
+            >>> feature_extractor = LogMelSpectrogram(sr=sample_rate, n_fft=512, window='hann', power=1.0)
+            >>> feats = feature_extractor(waveform)
+    """
+
+    ref_value: float
+    amin: float
+    top_db: float | None
+
+    def __init__(
+        self,
+        sr: int = 22050,
+        n_fft: int = 512,
+        hop_length: int | None = None,
+        win_length: int | None = None,
+        window: _WindowLiteral = 'hann',
+        power: float = 2.0,
+        center: bool = True,
+        pad_mode: Literal['reflect'] = 'reflect',
+        n_mels: int = 64,
+        f_min: float = 50.0,
+        f_max: float | None = None,
+        htk: bool = False,
+        norm: Literal['slaney'] | float = 'slaney',
+        ref_value: float = 1.0,
+        amin: float = 1e-10,
+        top_db: float | None = None,
+        dtype: str = 'float32',
+    ) -> None:
+        super().__init__()
+
+        self._melspectrogram = MelSpectrogram(
+            sr=sr,
+            n_fft=n_fft,
+            hop_length=hop_length,
+            win_length=win_length,
+            window=window,
+            power=power,
+            center=center,
+            pad_mode=pad_mode,
+            n_mels=n_mels,
+            f_min=f_min,
+            f_max=f_max,
+            htk=htk,
+            norm=norm,
+            dtype=dtype,
+        )
+
+        self.ref_value = ref_value
+        self.amin = amin
+        self.top_db = top_db
+
+    def forward(self, x: Tensor) -> Tensor:
+        """
+        Args:
+            x (Tensor): Tensor of waveforms with shape `(N, T)`
+
+        Returns:
+            Tensor: Log mel spectrograms with shape `(N, n_mels, num_frames)`.
+        """
+        mel_feature = self._melspectrogram(x)
+        log_mel_feature = power_to_db(
+            mel_feature,
+            ref_value=self.ref_value,
+            amin=self.amin,
+            top_db=self.top_db,
+        )
+        return log_mel_feature
+
+
+class MFCC(nn.Layer):
+    """Compute mel frequency cepstral coefficients(MFCCs) feature of given waveforms.
+
+    Args:
+        sr (int, optional): Sample rate. Defaults to 22050.
+        n_mfcc (int, optional): [description]. Defaults to 40.
+        n_fft (int, optional): The number of frequency components of the discrete Fourier transform. Defaults to 512.
+        hop_length (Optional[int], optional): The hop length of the short time FFT. If `None`, it is set to `win_length//4`. Defaults to None.
+        win_length (Optional[int], optional): The window length of the short time FFT. If `None`, it is set to same as `n_fft`. Defaults to None.
+        window (str, optional): The window function applied to the signal before the Fourier transform. Supported window functions: 'hamming', 'hann', 'gaussian', 'exponential', 'triang', 'bohman', 'blackman', 'cosine', 'tukey', 'taylor', 'bartlett', 'kaiser', 'nuttall'. Defaults to 'hann'.
+        power (float, optional): Exponent for the magnitude spectrogram. Defaults to 2.0.
+        center (bool, optional): Whether to pad `x` to make that the :math:`t \times hop\\_length` at the center of `t`-th frame. Defaults to True.
+        pad_mode (str, optional): Choose padding pattern when `center` is `True`. Defaults to 'reflect'.
+        n_mels (int, optional): Number of mel bins. Defaults to 64.
+        f_min (float, optional): Minimum frequency in Hz. Defaults to 50.0.
+        f_max (Optional[float], optional): Maximum frequency in Hz. Defaults to None.
+        htk (bool, optional): Use HTK formula in computing fbank matrix. Defaults to False.
+        norm (Union[str, float], optional): Type of normalization in computing fbank matrix. Slaney-style is used by default. You can specify norm=1.0/2.0 to use customized p-norm normalization. Defaults to 'slaney'.
+        ref_value (float, optional): The reference value. If smaller than 1.0, the db level of the signal will be pulled up accordingly. Otherwise, the db level is pushed down. Defaults to 1.0.
+        amin (float, optional): The minimum value of input magnitude. Defaults to 1e-10.
+        top_db (Optional[float], optional): The maximum db value of spectrogram. Defaults to None.
+        dtype (str, optional): Data type of input and window. Defaults to 'float32'.
+
+    Returns:
+        :ref:`api_paddle_nn_Layer`. An instance of MFCC.
+
+    Examples:
+        .. code-block:: pycon
+
+            >>> import paddle
+            >>> from paddle.audio.features import MFCC
+
+            >>> sample_rate = 16000
+            >>> wav_duration = 0.5
+            >>> num_channels = 1
+            >>> num_frames = sample_rate * wav_duration
+            >>> wav_data = paddle.linspace(-1.0, 1.0, int(num_frames)) * 0.1
+            >>> waveform = wav_data.tile([num_channels, 1])
+
+            >>> feature_extractor = MFCC(sr=sample_rate, n_fft=512, window='hann')
+            >>> feats = feature_extractor(waveform)
+    """
+
+    dct_matrix: Tensor
+
+    def __init__(
+        self,
+        sr: int = 22050,
+        n_mfcc: int = 40,
+        n_fft: int = 512,
+        hop_length: int | None = None,
+        win_length: int | None = None,
+        window: _WindowLiteral = 'hann',
+        power: float = 2.0,
+        center: bool = True,
+        pad_mode: Literal['reflect'] = 'reflect',
+        n_mels: int = 64,
+        f_min: float = 50.0,
+        f_max: float | None = None,
+        htk: bool = False,
+        norm: Literal['slaney'] | float = 'slaney',
+        ref_value: float = 1.0,
+        amin: float = 1e-10,
+        top_db: float | None = None,
+        dtype: str = 'float32',
+    ) -> None:
+        super().__init__()
+        assert n_mfcc <= n_mels, (
+            f'n_mfcc cannot be larger than n_mels: {n_mfcc} vs {n_mels}'
+        )
+        self._log_melspectrogram = LogMelSpectrogram(
+            sr=sr,
+            n_fft=n_fft,
+            hop_length=hop_length,
+            win_length=win_length,
+            window=window,
+            power=power,
+            center=center,
+            pad_mode=pad_mode,
+            n_mels=n_mels,
+            f_min=f_min,
+            f_max=f_max,
+            htk=htk,
+            norm=norm,
+            ref_value=ref_value,
+            amin=amin,
+            top_db=top_db,
+            dtype=dtype,
+        )
+        self.dct_matrix = create_dct(n_mfcc=n_mfcc, n_mels=n_mels, dtype=dtype)
+        self.register_buffer('dct_matrix', self.dct_matrix)
+
+    def forward(self, x: Tensor) -> Tensor:
+        """
+        Args:
+            x (Tensor): Tensor of waveforms with shape `(N, T)`
+
+        Returns:
+            Tensor: Mel frequency cepstral coefficients with shape `(N, n_mfcc, num_frames)`.
+        """
+        log_mel_feature = self._log_melspectrogram(x)
+        mfcc = paddle.matmul(
+            log_mel_feature.transpose((0, 2, 1)), self.dct_matrix
+        ).transpose((0, 2, 1))  # (B, n_mels, L)
+        return mfcc