wehub/paddlepaddle--paddle
1
0
Fork 0
Code Issues Pull Requests Actions 2 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:40:42 +08:00
commit e25996e7db
15472 changed files with 3536181 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/paddle/audio/__init__.py
+26
View File
@@ -0,0 +1,26 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from . import backends, datasets, features, functional
from .backends.backend import info, load, save
__all__ = [
"functional",
"features",
"datasets",
"backends",
"load",
"info",
"save",
]
python/paddle/audio/backends/__init__.py
+27
View File
@@ -0,0 +1,27 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from . import init_backend
from .init_backend import (
get_current_backend,
list_available_backends,
set_backend,
)
init_backend._init_set_audio_backend()
__all__ = [
'get_current_backend',
'list_available_backends',
'set_backend',
]
python/paddle/audio/backends/backend.py
+164
View File
@@ -0,0 +1,164 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License
from __future__ import annotations
from typing import TYPE_CHECKING, BinaryIO
if TYPE_CHECKING:
from pathlib import Path
from paddle import Tensor
class AudioInfo:
"""Audio info, return type of backend info function"""
sample_rate: int
num_samples: int
num_channels: int
bits_per_sample: int
encoding: str
def __init__(
self,
sample_rate: int,
num_samples: int,
num_channels: int,
bits_per_sample: int,
encoding: str,
) -> None:
self.sample_rate = sample_rate
self.num_samples = num_samples
self.num_channels = num_channels
self.bits_per_sample = bits_per_sample
self.encoding = encoding
def info(filepath: str | BinaryIO) -> AudioInfo:
"""Get signal information of input audio file.
Args:
filepath: audio path or file object.
Returns:
AudioInfo: info of the given audio.
Example:
.. code-block:: pycon
>>> import os
>>> import paddle
>>> sample_rate = 16000
>>> wav_duration = 0.5
>>> num_channels = 1
>>> num_frames = sample_rate * wav_duration
>>> wav_data = paddle.linspace(-1.0, 1.0, int(num_frames)) * 0.1
>>> waveform = wav_data.tile([num_channels, 1])
>>> base_dir = os.getcwd()
>>> filepath = os.path.join(base_dir, "test.wav")
>>> paddle.audio.save(filepath, waveform, sample_rate)
>>> wav_info = paddle.audio.info(filepath)
"""
# for API doc
raise NotImplementedError("please set audio backend")
def load(
filepath: str | Path,
frame_offset: int = 0,
num_frames: int = -1,
normalize: bool = True,
channels_first: bool = True,
) -> tuple[Tensor, int]:
"""Load audio data from file.Load the audio content start form frame_offset, and get num_frames.
Args:
frame_offset: from 0 to total frames,
num_frames: from -1 (means total frames) or number frames which want to read,
normalize:
if True: return audio which norm to (-1, 1), dtype=float32
if False: return audio with raw data, dtype=int16
channels_first:
if True: return audio with shape (channels, time)
Return:
Tuple[paddle.Tensor, int]: (audio_content, sample rate)
Examples:
.. code-block:: pycon
>>> import os
>>> import paddle
>>> sample_rate = 16000
>>> wav_duration = 0.5
>>> num_channels = 1
>>> num_frames = sample_rate * wav_duration
>>> wav_data = paddle.linspace(-1.0, 1.0, int(num_frames)) * 0.1
>>> waveform = wav_data.tile([num_channels, 1])
>>> base_dir = os.getcwd()
>>> filepath = os.path.join(base_dir, "test.wav")
>>> paddle.audio.save(filepath, waveform, sample_rate)
>>> wav_data_read, sr = paddle.audio.load(filepath)
"""
# for API doc
raise NotImplementedError("please set audio backend")
def save(
filepath: str,
src: Tensor,
sample_rate: int,
channels_first: bool = True,
encoding: str | None = None,
bits_per_sample: int | None = 16,
) -> None:
"""
Save audio tensor to file.
Args:
filepath: saved path
src: the audio tensor
sample_rate: the number of samples of audio per second.
channels_first: src channel information
if True, means input tensor is (channels, time)
if False, means input tensor is (time, channels)
encoding:encoding format, wave_backend only support PCM16 now.
bits_per_sample: bits per sample, wave_backend only support 16 bits now.
Returns:
None
Examples:
.. code-block:: pycon
>>> import paddle
>>> sample_rate = 16000
>>> wav_duration = 0.5
>>> num_channels = 1
>>> num_frames = sample_rate * wav_duration
>>> wav_data = paddle.linspace(-1.0, 1.0, int(num_frames)) * 0.1
>>> waveform = wav_data.tile([num_channels, 1])
>>> filepath = "./test.wav"
>>> paddle.audio.save(filepath, waveform, sample_rate)
"""
# for API doc
raise NotImplementedError("please set audio backend")
python/paddle/audio/backends/init_backend.py
+192
View File
@@ -0,0 +1,192 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import sys
import warnings
import paddle
from . import backend, wave_backend
def _check_version(version: str) -> bool:
# require paddleaudio >= 1.0.2
ver_arr = version.split('.')
v0 = int(ver_arr[0])
v1 = int(ver_arr[1])
v2 = int(ver_arr[2])
if v0 < 1:
return False
if v0 == 1 and v1 == 0 and v2 <= 1:
return False
return True
def list_available_backends() -> list[str]:
"""List available backends, the backends in paddleaudio and the default backend.
Returns:
list[str]: The list of available backends.
Examples:
.. code-block:: pycon
>>> import paddle
>>> sample_rate = 16000
>>> wav_duration = 0.5
>>> num_channels = 1
>>> num_frames = sample_rate * wav_duration
>>> wav_data = paddle.linspace(-1.0, 1.0, int(num_frames)) * 0.1
>>> waveform = wav_data.tile([num_channels, 1])
>>> wav_path = "./test.wav"
>>> current_backend = paddle.audio.backends.get_current_backend()
>>> print(current_backend)
wave_backend
>>> backends = paddle.audio.backends.list_available_backends()
>>> # default backends is ['wave_backend']
>>> # backends is ['wave_backend', 'soundfile'], if have installed paddleaudio >= 1.0.2
>>> if 'soundfile' in backends:
... paddle.audio.backends.set_backend('soundfile')
>>> paddle.audio.save(wav_path, waveform, sample_rate)
"""
backends = []
try:
import paddleaudio
except ImportError:
package = "paddleaudio"
warn_msg = (
f"Failed importing {package}. \n"
"only wave_backend(only can deal with PCM16 WAV) supported.\n"
"if want soundfile_backend(more audio type supported),\n"
f"please manually installed (usually with `pip install {package} >= 1.0.2`). "
)
warnings.warn(warn_msg)
if "paddleaudio" in sys.modules:
version = paddleaudio.__version__
if not _check_version(version):
err_msg = (
f"the version of paddleaudio installed is {version},\n"
"please ensure the paddleaudio >= 1.0.2."
)
raise ImportError(err_msg)
backends = paddleaudio.backends.list_audio_backends()
backends.append("wave_backend")
return backends
def get_current_backend() -> str:
"""Get the name of the current audio backend
Returns:
str: The name of the current backend,
the wave_backend or backend imported from paddleaudio
Examples:
.. code-block:: pycon
>>> import paddle
>>> sample_rate = 16000
>>> wav_duration = 0.5
>>> num_channels = 1
>>> num_frames = sample_rate * wav_duration
>>> wav_data = paddle.linspace(-1.0, 1.0, int(num_frames)) * 0.1
>>> waveform = wav_data.tile([num_channels, 1])
>>> wav_path = "./test.wav"
>>> current_backend = paddle.audio.backends.get_current_backend()
>>> print(current_backend)
wave_backend
>>> backends = paddle.audio.backends.list_available_backends()
>>> # default backends is ['wave_backend']
>>> # backends is ['wave_backend', 'soundfile'], if have installed paddleaudio >= 1.0.2
>>> if 'soundfile' in backends:
... paddle.audio.backends.set_backend('soundfile')
>>> paddle.audio.save(wav_path, waveform, sample_rate)
"""
current_backend = None
if "paddleaudio" in sys.modules:
import paddleaudio
current_backend = paddleaudio.backends.get_audio_backend()
if paddle.audio.load == paddleaudio.load:
return current_backend
return "wave_backend"
def set_backend(backend_name: str) -> None:
"""Set the backend by one of the list_audio_backend return.
Args:
backend (str): one of the list_audio_backend. "wave_backend" is the default. "soundfile" imported from paddleaudio.
Returns:
None
Examples:
.. code-block:: pycon
>>> import paddle
>>> sample_rate = 16000
>>> wav_duration = 0.5
>>> num_channels = 1
>>> num_frames = sample_rate * wav_duration
>>> wav_data = paddle.linspace(-1.0, 1.0, int(num_frames)) * 0.1
>>> waveform = wav_data.tile([num_channels, 1])
>>> wav_path = "./test.wav"
>>> current_backend = paddle.audio.backends.get_current_backend()
>>> print(current_backend)
wave_backend
>>> backends = paddle.audio.backends.list_available_backends()
>>> # default backends is ['wave_backend']
>>> # backends is ['wave_backend', 'soundfile'], if have installed paddleaudio >= 1.0.2
>>> if 'soundfile' in backends:
... paddle.audio.backends.set_backend('soundfile')
>>> paddle.audio.save(wav_path, waveform, sample_rate)
"""
if backend_name not in list_available_backends():
raise NotImplementedError
if backend_name == "wave_backend":
module = wave_backend
else:
import paddleaudio
paddleaudio.backends.set_audio_backend(backend_name)
module = paddleaudio
for func in ["save", "load", "info"]:
setattr(backend, func, getattr(module, func))
setattr(paddle.audio, func, getattr(module, func))
def _init_set_audio_backend() -> None:
# init the default wave_backend.
for func in ["save", "load", "info"]:
setattr(backend, func, getattr(wave_backend, func))
python/paddle/audio/backends/wave_backend.py
+236
View File
@@ -0,0 +1,236 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import wave
from typing import TYPE_CHECKING, BinaryIO
import numpy as np
import paddle
from .backend import AudioInfo
if TYPE_CHECKING:
from pathlib import Path
from paddle import Tensor
def _error_message():
package = "paddleaudio"
warn_msg = (
"only PCM16 WAV supported. \n"
"if want support more other audio types, please "
f"manually installed (usually with `pip install {package}`). \n "
"and use paddle.audio.backends.set_backend('soundfile') to set audio backend"
)
return warn_msg
def info(filepath: str | BinaryIO) -> AudioInfo:
"""Get signal information of input audio file.
Args:
filepath: audio path or file object.
Returns:
AudioInfo: info of the given audio.
Example:
.. code-block:: pycon
>>> import os
>>> import paddle
>>> sample_rate = 16000
>>> wav_duration = 0.5
>>> num_channels = 1
>>> num_frames = sample_rate * wav_duration
>>> wav_data = paddle.linspace(-1.0, 1.0, int(num_frames)) * 0.1
>>> waveform = wav_data.tile([num_channels, 1])
>>> base_dir = os.getcwd()
>>> filepath = os.path.join(base_dir, "test.wav")
>>> paddle.audio.save(filepath, waveform, sample_rate)
>>> wav_info = paddle.audio.info(filepath)
"""
if hasattr(filepath, 'read'):
file_obj = filepath
else:
file_obj = open(filepath, 'rb')
try:
file_ = wave.open(file_obj)
except wave.Error:
file_obj.seek(0)
file_obj.close()
err_msg = _error_message()
raise NotImplementedError(err_msg)
channels = file_.getnchannels()
sample_rate = file_.getframerate()
sample_frames = file_.getnframes() # audio frame
bits_per_sample = file_.getsampwidth() * 8
encoding = "PCM_S" # default WAV encoding, only support
file_obj.close()
return AudioInfo(
sample_rate, sample_frames, channels, bits_per_sample, encoding
)
def load(
filepath: str | Path,
frame_offset: int = 0,
num_frames: int = -1,
normalize: bool = True,
channels_first: bool = True,
) -> tuple[Tensor, int]:
"""Load audio data from file. load the audio content start form frame_offset, and get num_frames.
Args:
frame_offset: from 0 to total frames,
num_frames: from -1 (means total frames) or number frames which want to read,
normalize:
if True: return audio which norm to (-1, 1), dtype=float32
if False: return audio with raw data, dtype=int16
channels_first:
if True: return audio with shape (channels, time)
Return:
Tuple[paddle.Tensor, int]: (audio_content, sample rate)
Examples:
.. code-block:: pycon
>>> import os
>>> import paddle
>>> sample_rate = 16000
>>> wav_duration = 0.5
>>> num_channels = 1
>>> num_frames = sample_rate * wav_duration
>>> wav_data = paddle.linspace(-1.0, 1.0, int(num_frames)) * 0.1
>>> waveform = wav_data.tile([num_channels, 1])
>>> base_dir = os.getcwd()
>>> filepath = os.path.join(base_dir, "test.wav")
>>> paddle.audio.save(filepath, waveform, sample_rate)
>>> wav_data_read, sr = paddle.audio.load(filepath)
"""
if hasattr(filepath, 'read'):
file_obj = filepath
else:
file_obj = open(filepath, 'rb')
try:
file_ = wave.open(file_obj)
except wave.Error:
file_obj.seek(0)
file_obj.close()
err_msg = _error_message()
raise NotImplementedError(err_msg)
channels = file_.getnchannels()
sample_rate = file_.getframerate()
frames = file_.getnframes() # audio frame
audio_content = file_.readframes(frames)
file_obj.close()
# default_subtype = "PCM_16", only support PCM16 WAV
audio_as_np16 = np.frombuffer(audio_content, dtype=np.int16)
audio_as_np32 = audio_as_np16.astype(np.float32)
if normalize:
# dtype = "float32"
audio_norm = audio_as_np32 / (2**15)
else:
# dtype = "int16"
audio_norm = audio_as_np32
waveform = np.reshape(audio_norm, (frames, channels))
if num_frames != -1:
waveform = waveform[frame_offset : frame_offset + num_frames, :]
waveform = paddle.to_tensor(waveform)
if channels_first:
waveform = paddle.transpose(waveform, perm=[1, 0])
return waveform, sample_rate
def save(
filepath: str,
src: Tensor,
sample_rate: int,
channels_first: bool = True,
encoding: str | None = None,
bits_per_sample: int | None = 16,
) -> None:
"""
Save audio tensor to file.
Args:
filepath: saved path
src: the audio tensor
sample_rate: the number of samples of audio per second.
channels_first: src channel information
if True, means input tensor is (channels, time)
if False, means input tensor is (time, channels)
encoding: audio encoding format, wave_backend only support PCM16 now.
bits_per_sample: bits per sample, wave_backend only support 16 bits now.
Returns:
None
Examples:
.. code-block:: pycon
>>> import paddle
>>> sample_rate = 16000
>>> wav_duration = 0.5
>>> num_channels = 1
>>> num_frames = sample_rate * wav_duration
>>> wav_data = paddle.linspace(-1.0, 1.0, int(num_frames)) * 0.1
>>> waveform = wav_data.tile([num_channels, 1])
>>> filepath = "./test.wav"
>>> paddle.audio.save(filepath, waveform, sample_rate)
"""
assert src.ndim == 2, "Expected 2D tensor"
audio_numpy = src.numpy()
# change src shape to (time, channels)
if channels_first:
audio_numpy = np.transpose(audio_numpy)
channels = audio_numpy.shape[1]
# only support PCM16
if bits_per_sample not in (None, 16):
raise ValueError("Invalid bits_per_sample, only support 16 bit")
sample_width = int(bits_per_sample / 8) # 2
if src.dtype == paddle.float32:
audio_numpy = (audio_numpy * (2**15)).astype("<h")
with wave.open(filepath, 'w') as f:
f.setnchannels(channels)
f.setsampwidth(sample_width)
f.setframerate(sample_rate)
f.writeframes(audio_numpy.tobytes())
python/paddle/audio/datasets/__init__.py
+18
View File
@@ -0,0 +1,18 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from .esc50 import ESC50
from .tess import TESS
__all__ = ["ESC50", "TESS"]
python/paddle/audio/datasets/dataset.py
+98
View File
@@ -0,0 +1,98 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import paddle
from ..features import MFCC, LogMelSpectrogram, MelSpectrogram, Spectrogram
feat_funcs = {
'raw': None,
'melspectrogram': MelSpectrogram,
'mfcc': MFCC,
'logmelspectrogram': LogMelSpectrogram,
'spectrogram': Spectrogram,
}
class AudioClassificationDataset(paddle.io.Dataset):
"""
Base class of audio classification dataset.
"""
def __init__(
self,
files: list[str],
labels: list[int],
feat_type: str = 'raw',
sample_rate: int | None = None,
**kwargs,
):
"""
Args:
files (:obj:`List[str]`): A list of absolute path of audio files.
labels (:obj:`List[int]`): Labels of audio files.
feat_type (:obj:`str`, `optional`, defaults to `raw`):
It identifies the feature type that user wants to extract an audio file.
"""
super().__init__()
if feat_type not in feat_funcs.keys():
raise RuntimeError(
f"Unknown feat_type: {feat_type}, it must be one in {list(feat_funcs.keys())}"
)
self.files = files
self.labels = labels
self.feat_type = feat_type
self.sample_rate = sample_rate
self.feat_config = (
kwargs # Pass keyword arguments to customize feature config
)
def _get_data(self, input_file: str):
raise NotImplementedError
def _convert_to_record(self, idx):
file, label = self.files[idx], self.labels[idx]
waveform, sample_rate = paddle.audio.load(file)
self.sample_rate = sample_rate
feat_func = feat_funcs[self.feat_type]
record = {}
if len(waveform.shape) == 2:
waveform = waveform.squeeze(0) # 1D input
waveform = paddle.to_tensor(waveform, dtype=paddle.float32)
if feat_func is not None:
waveform = waveform.unsqueeze(0) # (batch_size, T)
if self.feat_type != 'spectrogram':
feature_extractor = feat_func(
sr=self.sample_rate, **self.feat_config
)
else:
feature_extractor = feat_func(**self.feat_config)
record['feat'] = feature_extractor(waveform).squeeze(0)
else:
record['feat'] = waveform
record['label'] = label
return record
def __getitem__(self, idx):
record = self._convert_to_record(idx)
return record['feat'], record['label']
def __len__(self):
return len(self.files)
python/paddle/audio/datasets/esc50.py
+227
View File
@@ -0,0 +1,227 @@
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import os
from typing import TYPE_CHECKING, Any, Literal, NamedTuple, TypeAlias
from paddle.dataset.common import DATA_HOME
from paddle.utils import download
from .dataset import AudioClassificationDataset
if TYPE_CHECKING:
_ModeLiteral: TypeAlias = Literal[
'train',
'dev',
]
_FeatTypeLiteral: TypeAlias = Literal[
'raw',
'melspectrogram',
'mfcc',
'logmelspectrogram',
'spectrogram',
]
__all__ = []
class ESC50(AudioClassificationDataset):
"""
The ESC-50 dataset is a labeled collection of 2000 environmental audio recordings
suitable for benchmarking methods of environmental sound classification. The dataset
consists of 5-second-long recordings organized into 50 semantical classes (with
40 examples per class)
Reference:
ESC: Dataset for Environmental Sound Classification
http://dx.doi.org/10.1145/2733373.2806390
Args:
mode (str, optional): It identifies the dataset mode (train or dev). Default:train.
split (int, optional): It specify the fold of dev dataset. Default:1.
feat_type (str, optional): It identifies the feature type that user wants to extract of an audio file. Default:raw.
archive(dict, optional): it tells where to download the audio archive. Default:None.
Returns:
:ref:`api_paddle_io_Dataset`. An instance of ESC50 dataset.
Examples:
.. code-block:: pycon
>>> # doctest: +TIMEOUT(60)
>>> import paddle
>>> esc50_dataset = paddle.audio.datasets.ESC50(
... mode='dev',
... feat_type='raw',
... )
>>> for idx in range(5):
... audio, label = esc50_dataset[idx]
... # do something with audio, label
... print(audio.shape, label)
... # [audio_data_length] , label_id
paddle.Size([220500]) 0
paddle.Size([220500]) 14
paddle.Size([220500]) 36
paddle.Size([220500]) 36
paddle.Size([220500]) 19
>>> esc50_dataset = paddle.audio.datasets.ESC50(
... mode='dev',
... feat_type='mfcc',
... n_mfcc=40,
... )
>>> for idx in range(5):
... audio, label = esc50_dataset[idx]
... # do something with mfcc feature, label
... print(audio.shape, label)
... # [feature_dim, length] , label_id
paddle.Size([40, 1723]) 0
paddle.Size([40, 1723]) 14
paddle.Size([40, 1723]) 36
paddle.Size([40, 1723]) 36
paddle.Size([40, 1723]) 19
"""
archive: dict[str, str] = {
'url': 'https://paddleaudio.bj.bcebos.com/datasets/ESC-50-master.zip',
'md5': '7771e4b9d86d0945acce719c7a59305a',
}
label_list: list[str] = [
# Animals
'Dog',
'Rooster',
'Pig',
'Cow',
'Frog',
'Cat',
'Hen',
'Insects (flying)',
'Sheep',
'Crow',
# Natural soundscapes & water sounds
'Rain',
'Sea waves',
'Crackling fire',
'Crickets',
'Chirping birds',
'Water drops',
'Wind',
'Pouring water',
'Toilet flush',
'Thunderstorm',
# Human, non-speech sounds
'Crying baby',
'Sneezing',
'Clapping',
'Breathing',
'Coughing',
'Footsteps',
'Laughing',
'Brushing teeth',
'Snoring',
'Drinking, sipping',
# Interior/domestic sounds
'Door knock',
'Mouse click',
'Keyboard typing',
'Door, wood creaks',
'Can opening',
'Washing machine',
'Vacuum cleaner',
'Clock alarm',
'Clock tick',
'Glass breaking',
# Exterior/urban noises
'Helicopter',
'Chainsaw',
'Siren',
'Car horn',
'Engine',
'Train',
'Church bells',
'Airplane',
'Fireworks',
'Hand saw',
]
meta: str = os.path.join('ESC-50-master', 'meta', 'esc50.csv')
audio_path: str = os.path.join('ESC-50-master', 'audio')
class meta_info(NamedTuple):
filename: str
fold: str
target: str
category: str
esc10: str
src_file: str
take: str
def __init__(
self,
mode: _ModeLiteral = 'train',
split: int = 1,
feat_type: _FeatTypeLiteral = 'raw',
archive: dict[str, str] | None = None,
**kwargs: Any,
) -> None:
assert split in range(1, 6), (
f'The selected split should be integer, and 1 <= split <= 5, but got {split}'
)
if archive is not None:
self.archive = archive
files, labels = self._get_data(mode, split)
super().__init__(
files=files, labels=labels, feat_type=feat_type, **kwargs
)
def _get_meta_info(self) -> list[meta_info]:
ret = []
with open(os.path.join(DATA_HOME, self.meta), 'r') as rf:
for line in rf.readlines()[1:]:
ret.append(self.meta_info(*line.strip().split(',')))
return ret
def _get_data(
self, mode: _ModeLiteral, split: int
) -> tuple[list[str], list[int]]:
if not os.path.isdir(
os.path.join(DATA_HOME, self.audio_path)
) or not os.path.isfile(os.path.join(DATA_HOME, self.meta)):
download.get_path_from_url(
self.archive['url'],
DATA_HOME,
self.archive['md5'],
decompress=True,
)
meta_info = self._get_meta_info()
files = []
labels = []
for sample in meta_info:
filename, fold, target, _, _, _, _ = sample
if mode == 'train' and int(fold) != split:
files.append(os.path.join(DATA_HOME, self.audio_path, filename))
labels.append(int(target))
if mode != 'train' and int(fold) == split:
files.append(os.path.join(DATA_HOME, self.audio_path, filename))
labels.append(int(target))
return files, labels
python/paddle/audio/datasets/tess.py
+166
View File
@@ -0,0 +1,166 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import os
from typing import TYPE_CHECKING, Any, NamedTuple
from paddle.dataset.common import DATA_HOME
from paddle.utils import download
from .dataset import AudioClassificationDataset
if TYPE_CHECKING:
from .esc50 import _FeatTypeLiteral, _ModeLiteral
__all__ = []
class TESS(AudioClassificationDataset):
"""
TESS is a set of 200 target words were spoken in the carrier phrase
"Say the word _____' by two actresses (aged 26 and 64 years) and
recordings were made of the set portraying each of seven emotions(anger,
disgust, fear, happiness, pleasant surprise, sadness, and neutral).
There are 2800 stimuli in total.
Reference:
Toronto emotional speech set (TESS) https://tspace.library.utoronto.ca/handle/1807/24487
https://doi.org/10.5683/SP2/E8H2MF
Args:
mode (str, optional): It identifies the dataset mode (train or dev). Defaults to train.
n_folds (int, optional): Split the dataset into n folds. 1 fold for dev dataset and n-1 for train dataset. Defaults to 5.
split (int, optional): It specify the fold of dev dataset. Defaults to 1.
feat_type (str, optional): It identifies the feature type that user wants to extract of an audio file. Defaults to raw.
archive(dict): it tells where to download the audio archive. Defaults to None.
Returns:
:ref:`api_paddle_io_Dataset`. An instance of TESS dataset.
Examples:
.. code-block:: pycon
>>> # doctest: +TIMEOUT(60)
>>> import paddle
>>> tess_dataset = paddle.audio.datasets.TESS(
... mode='dev',
... feat_type='raw',
... )
>>> for idx in range(5):
... audio, label = tess_dataset[idx]
... # do something with audio, label
... print(audio.shape, label)
... # [audio_data_length] , label_id
>>> tess_dataset = paddle.audio.datasets.TESS(
... mode='dev',
... feat_type='mfcc',
... n_mfcc=40,
... )
>>> for idx in range(5):
... audio, label = tess_dataset[idx]
... # do something with mfcc feature, label
... print(audio.shape, label)
... # [feature_dim, num_frames] , label_id
"""
archive: dict[str, str] = {
'url': 'https://bj.bcebos.com/paddleaudio/datasets/TESS_Toronto_emotional_speech_set.zip',
'md5': '1465311b24d1de704c4c63e4ccc470c7',
}
label_list: list[str] = [
'angry',
'disgust',
'fear',
'happy',
'neutral',
'ps', # pleasant surprise
'sad',
]
audio_path: str = 'TESS_Toronto_emotional_speech_set'
class meta_info(NamedTuple):
speaker: str
word: str
emotion: str
def __init__(
self,
mode: _ModeLiteral = 'train',
n_folds: int = 5,
split: int = 1,
feat_type: _FeatTypeLiteral = 'raw',
archive: dict[str, str] | None = None,
**kwargs: Any,
) -> None:
assert isinstance(n_folds, int) and (n_folds >= 1), (
f'the n_folds should be integer and n_folds >= 1, but got {n_folds}'
)
assert split in range(1, n_folds + 1), (
f'The selected split should be integer and should be 1 <= split <= {n_folds}, but got {split}'
)
if archive is not None:
self.archive = archive
files, labels = self._get_data(mode, n_folds, split)
super().__init__(
files=files, labels=labels, feat_type=feat_type, **kwargs
)
def _get_meta_info(self, files) -> list[meta_info]:
ret = []
for file in files:
basename_without_extend = os.path.basename(file)[:-4]
ret.append(self.meta_info(*basename_without_extend.split('_')))
return ret
def _get_data(
self, mode: str, n_folds: int, split: int
) -> tuple[list[str], list[int]]:
if not os.path.isdir(os.path.join(DATA_HOME, self.audio_path)):
download.get_path_from_url(
self.archive['url'],
DATA_HOME,
self.archive['md5'],
decompress=True,
)
wav_files = []
for root, _, files in os.walk(os.path.join(DATA_HOME, self.audio_path)):
for file in files:
if file.endswith('.wav'):
wav_files.append(os.path.join(root, file))
meta_info = self._get_meta_info(wav_files)
files = []
labels = []
for idx, sample in enumerate(meta_info):
_, _, emotion = sample
target = self.label_list.index(emotion)
fold = idx % n_folds + 1
if mode == 'train' and int(fold) != split:
files.append(wav_files[idx])
labels.append(target)
if mode != 'train' and int(fold) == split:
files.append(wav_files[idx])
labels.append(target)
return files, labels
python/paddle/audio/features/__init__.py
+26
View File
@@ -0,0 +1,26 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from .layers import (
MFCC,
LogMelSpectrogram,
MelSpectrogram,
Spectrogram,
)
__all__ = [
'LogMelSpectrogram',
'MelSpectrogram',
'MFCC',
'Spectrogram',
]
python/paddle/audio/features/layers.py
+448
View File
@@ -0,0 +1,448 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from functools import partial
from typing import TYPE_CHECKING, Literal, TypeAlias
import paddle
from paddle import nn
from ..functional import compute_fbank_matrix, create_dct, power_to_db
from ..functional.window import get_window
if TYPE_CHECKING:
from paddle import Tensor
_WindowLiteral: TypeAlias = Literal[
'hamming',
'hann',
'kaiser',
'bartlett',
'nuttall',
'gaussian',
'exponential',
'triang',
'bohman',
'blackman',
'cosine',
'tukey',
'taylor',
]
class Spectrogram(nn.Layer):
"""Compute spectrogram of given signals, typically audio waveforms.
The spectrogram is defined as the complex norm of the short-time Fourier transformation.
Args:
n_fft (int, optional): The number of frequency components of the discrete Fourier transform. Defaults to 512.
hop_length (Optional[int], optional): The hop length of the short time FFT. If `None`, it is set to `win_length//4`. Defaults to None.
win_length (Optional[int], optional): The window length of the short time FFT. If `None`, it is set to same as `n_fft`. Defaults to None.
window (str, optional): The window function applied to the signal before the Fourier transform. Supported window functions: 'hamming', 'hann', 'gaussian', 'exponential', 'triang', 'bohman', 'blackman', 'cosine', 'tukey', 'taylor', 'bartlett', 'kaiser', 'nuttall'. Defaults to 'hann'.
power (float, optional): Exponent for the magnitude spectrogram. Defaults to 2.0.
center (bool, optional): Whether to pad `x` to make that the :math:`t \times hop\\_length` at the center of `t`-th frame. Defaults to True.
pad_mode (str, optional): Choose padding pattern when `center` is `True`. Defaults to 'reflect'.
dtype (str, optional): Data type of input and window. Defaults to 'float32'.
Returns:
:ref:`api_paddle_nn_Layer`. An instance of Spectrogram.
Examples:
.. code-block:: pycon
>>> import paddle
>>> from paddle.audio.features import Spectrogram
>>> sample_rate = 16000
>>> wav_duration = 0.5
>>> num_channels = 1
>>> num_frames = sample_rate * wav_duration
>>> wav_data = paddle.linspace(-1.0, 1.0, int(num_frames)) * 0.1
>>> waveform = wav_data.tile([num_channels, 1])
>>> feature_extractor = Spectrogram(n_fft=512, window='hann', power=1.0)
>>> feats = feature_extractor(waveform)
"""
power: float
fft_window: Tensor
def __init__(
self,
n_fft: int = 512,
hop_length: int | None = 512,
win_length: int | None = None,
window: _WindowLiteral = 'hann',
power: float = 1.0,
center: bool = True,
pad_mode: Literal['reflect'] = 'reflect',
dtype: str = 'float32',
) -> None:
super().__init__()
assert power > 0, 'Power of spectrogram must be > 0.'
self.power = power
if win_length is None:
win_length = n_fft
self.fft_window = get_window(
window, win_length, fftbins=True, dtype=dtype
)
self._stft = partial(
paddle.signal.stft,
n_fft=n_fft,
hop_length=hop_length,
win_length=win_length,
window=self.fft_window,
center=center,
pad_mode=pad_mode,
)
self.register_buffer('fft_window', self.fft_window)
def forward(self, x: Tensor) -> Tensor:
"""
Args:
x (Tensor): Tensor of waveforms with shape `(N, T)`
Returns:
Tensor: Spectrograms with shape `(N, n_fft//2 + 1, num_frames)`.
"""
stft = self._stft(x)
spectrogram = paddle.pow(paddle.abs(stft), self.power)
return spectrogram
class MelSpectrogram(nn.Layer):
"""Compute the melspectrogram of given signals, typically audio waveforms. It is computed by multiplying spectrogram with Mel filter bank matrix.
Args:
sr (int, optional): Sample rate. Defaults to 22050.
n_fft (int, optional): The number of frequency components of the discrete Fourier transform. Defaults to 512.
hop_length (Optional[int], optional): The hop length of the short time FFT. If `None`, it is set to `win_length//4`. Defaults to None.
win_length (Optional[int], optional): The window length of the short time FFT. If `None`, it is set to same as `n_fft`. Defaults to None.
window (str, optional): The window function applied to the signal before the Fourier transform. Supported window functions: 'hamming', 'hann', 'gaussian', 'exponential', 'triang', 'bohman', 'blackman', 'cosine', 'tukey', 'taylor', 'bartlett', 'kaiser', 'nuttall'. Defaults to 'hann'.
power (float, optional): Exponent for the magnitude spectrogram. Defaults to 2.0.
center (bool, optional): Whether to pad `x` to make that the :math:`t \times hop\\_length` at the center of `t`-th frame. Defaults to True.
pad_mode (str, optional): Choose padding pattern when `center` is `True`. Defaults to 'reflect'.
n_mels (int, optional): Number of mel bins. Defaults to 64.
f_min (float, optional): Minimum frequency in Hz. Defaults to 50.0.
f_max (Optional[float], optional): Maximum frequency in Hz. Defaults to None.
htk (bool, optional): Use HTK formula in computing fbank matrix. Defaults to False.
norm (Union[str, float], optional): Type of normalization in computing fbank matrix. Slaney-style is used by default. You can specify norm=1.0/2.0 to use customized p-norm normalization. Defaults to 'slaney'.
dtype (str, optional): Data type of input and window. Defaults to 'float32'.
Returns:
:ref:`api_paddle_nn_Layer`. An instance of MelSpectrogram.
Examples:
.. code-block:: pycon
>>> import paddle
>>> from paddle.audio.features import MelSpectrogram
>>> sample_rate = 16000
>>> wav_duration = 0.5
>>> num_channels = 1
>>> num_frames = sample_rate * wav_duration
>>> wav_data = paddle.linspace(-1.0, 1.0, int(num_frames)) * 0.1
>>> waveform = wav_data.tile([num_channels, 1])
>>> feature_extractor = MelSpectrogram(sr=sample_rate, n_fft=512, window='hann', power=1.0)
>>> feats = feature_extractor(waveform)
"""
n_mels: int
f_min: float
f_max: float
htk: bool
norm: Literal['slaney'] | float
fbank_matrix: Tensor
def __init__(
self,
sr: int = 22050,
n_fft: int = 2048,
hop_length: int | None = 512,
win_length: int | None = None,
window: _WindowLiteral = 'hann',
power: float = 2.0,
center: bool = True,
pad_mode: Literal['reflect'] = 'reflect',
n_mels: int = 64,
f_min: float = 50.0,
f_max: float | None = None,
htk: bool = False,
norm: Literal['slaney'] | float = 'slaney',
dtype: str = 'float32',
) -> None:
super().__init__()
self._spectrogram = Spectrogram(
n_fft=n_fft,
hop_length=hop_length,
win_length=win_length,
window=window,
power=power,
center=center,
pad_mode=pad_mode,
dtype=dtype,
)
self.n_mels = n_mels
self.f_min = f_min
self.f_max = f_max
self.htk = htk
self.norm = norm
if f_max is None:
f_max = sr // 2
self.fbank_matrix = compute_fbank_matrix(
sr=sr,
n_fft=n_fft,
n_mels=n_mels,
f_min=f_min,
f_max=f_max,
htk=htk,
norm=norm,
dtype=dtype,
)
self.register_buffer('fbank_matrix', self.fbank_matrix)
def forward(self, x: Tensor) -> Tensor:
"""
Args:
x (Tensor): Tensor of waveforms with shape `(N, T)`
Returns:
Tensor: Mel spectrograms with shape `(N, n_mels, num_frames)`.
"""
spect_feature = self._spectrogram(x)
mel_feature = paddle.matmul(self.fbank_matrix, spect_feature)
return mel_feature
class LogMelSpectrogram(nn.Layer):
"""Compute log-mel-spectrogram feature of given signals, typically audio waveforms.
Args:
sr (int, optional): Sample rate. Defaults to 22050.
n_fft (int, optional): The number of frequency components of the discrete Fourier transform. Defaults to 512.
hop_length (Optional[int], optional): The hop length of the short time FFT. If `None`, it is set to `win_length//4`. Defaults to None.
win_length (Optional[int], optional): The window length of the short time FFT. If `None`, it is set to same as `n_fft`. Defaults to None.
window (str, optional): The window function applied to the signal before the Fourier transform. Supported window functions: 'hamming', 'hann', 'gaussian', 'exponential', 'triang', 'bohman', 'blackman', 'cosine', 'tukey', 'taylor', 'bartlett', 'kaiser', 'nuttall'. Defaults to 'hann'.
power (float, optional): Exponent for the magnitude spectrogram. Defaults to 2.0.
center (bool, optional): Whether to pad `x` to make that the :math:`t \times hop\\_length` at the center of `t`-th frame. Defaults to True.
pad_mode (str, optional): Choose padding pattern when `center` is `True`. Defaults to 'reflect'.
n_mels (int, optional): Number of mel bins. Defaults to 64.
f_min (float, optional): Minimum frequency in Hz. Defaults to 50.0.
f_max (Optional[float], optional): Maximum frequency in Hz. Defaults to None.
htk (bool, optional): Use HTK formula in computing fbank matrix. Defaults to False.
norm (Union[str, float], optional): Type of normalization in computing fbank matrix. Slaney-style is used by default. You can specify norm=1.0/2.0 to use customized p-norm normalization. Defaults to 'slaney'.
ref_value (float, optional): The reference value. If smaller than 1.0, the db level of the signal will be pulled up accordingly. Otherwise, the db level is pushed down. Defaults to 1.0.
amin (float, optional): The minimum value of input magnitude. Defaults to 1e-10.
top_db (Optional[float], optional): The maximum db value of spectrogram. Defaults to None.
dtype (str, optional): Data type of input and window. Defaults to 'float32'.
Returns:
:ref:`api_paddle_nn_Layer`. An instance of LogMelSpectrogram.
Examples:
.. code-block:: pycon
>>> import paddle
>>> from paddle.audio.features import LogMelSpectrogram
>>> sample_rate = 16000
>>> wav_duration = 0.5
>>> num_channels = 1
>>> num_frames = sample_rate * wav_duration
>>> wav_data = paddle.linspace(-1.0, 1.0, int(num_frames)) * 0.1
>>> waveform = wav_data.tile([num_channels, 1])
>>> feature_extractor = LogMelSpectrogram(sr=sample_rate, n_fft=512, window='hann', power=1.0)
>>> feats = feature_extractor(waveform)
"""
ref_value: float
amin: float
top_db: float | None
def __init__(
self,
sr: int = 22050,
n_fft: int = 512,
hop_length: int | None = None,
win_length: int | None = None,
window: _WindowLiteral = 'hann',
power: float = 2.0,
center: bool = True,
pad_mode: Literal['reflect'] = 'reflect',
n_mels: int = 64,
f_min: float = 50.0,
f_max: float | None = None,
htk: bool = False,
norm: Literal['slaney'] | float = 'slaney',
ref_value: float = 1.0,
amin: float = 1e-10,
top_db: float | None = None,
dtype: str = 'float32',
) -> None:
super().__init__()
self._melspectrogram = MelSpectrogram(
sr=sr,
n_fft=n_fft,
hop_length=hop_length,
win_length=win_length,
window=window,
power=power,
center=center,
pad_mode=pad_mode,
n_mels=n_mels,
f_min=f_min,
f_max=f_max,
htk=htk,
norm=norm,
dtype=dtype,
)
self.ref_value = ref_value
self.amin = amin
self.top_db = top_db
def forward(self, x: Tensor) -> Tensor:
"""
Args:
x (Tensor): Tensor of waveforms with shape `(N, T)`
Returns:
Tensor: Log mel spectrograms with shape `(N, n_mels, num_frames)`.
"""
mel_feature = self._melspectrogram(x)
log_mel_feature = power_to_db(
mel_feature,
ref_value=self.ref_value,
amin=self.amin,
top_db=self.top_db,
)
return log_mel_feature
class MFCC(nn.Layer):
"""Compute mel frequency cepstral coefficients(MFCCs) feature of given waveforms.
Args:
sr (int, optional): Sample rate. Defaults to 22050.
n_mfcc (int, optional): [description]. Defaults to 40.
n_fft (int, optional): The number of frequency components of the discrete Fourier transform. Defaults to 512.
hop_length (Optional[int], optional): The hop length of the short time FFT. If `None`, it is set to `win_length//4`. Defaults to None.
win_length (Optional[int], optional): The window length of the short time FFT. If `None`, it is set to same as `n_fft`. Defaults to None.
window (str, optional): The window function applied to the signal before the Fourier transform. Supported window functions: 'hamming', 'hann', 'gaussian', 'exponential', 'triang', 'bohman', 'blackman', 'cosine', 'tukey', 'taylor', 'bartlett', 'kaiser', 'nuttall'. Defaults to 'hann'.
power (float, optional): Exponent for the magnitude spectrogram. Defaults to 2.0.
center (bool, optional): Whether to pad `x` to make that the :math:`t \times hop\\_length` at the center of `t`-th frame. Defaults to True.
pad_mode (str, optional): Choose padding pattern when `center` is `True`. Defaults to 'reflect'.
n_mels (int, optional): Number of mel bins. Defaults to 64.
f_min (float, optional): Minimum frequency in Hz. Defaults to 50.0.
f_max (Optional[float], optional): Maximum frequency in Hz. Defaults to None.
htk (bool, optional): Use HTK formula in computing fbank matrix. Defaults to False.
norm (Union[str, float], optional): Type of normalization in computing fbank matrix. Slaney-style is used by default. You can specify norm=1.0/2.0 to use customized p-norm normalization. Defaults to 'slaney'.
ref_value (float, optional): The reference value. If smaller than 1.0, the db level of the signal will be pulled up accordingly. Otherwise, the db level is pushed down. Defaults to 1.0.
amin (float, optional): The minimum value of input magnitude. Defaults to 1e-10.
top_db (Optional[float], optional): The maximum db value of spectrogram. Defaults to None.
dtype (str, optional): Data type of input and window. Defaults to 'float32'.
Returns:
:ref:`api_paddle_nn_Layer`. An instance of MFCC.
Examples:
.. code-block:: pycon
>>> import paddle
>>> from paddle.audio.features import MFCC
>>> sample_rate = 16000
>>> wav_duration = 0.5
>>> num_channels = 1
>>> num_frames = sample_rate * wav_duration
>>> wav_data = paddle.linspace(-1.0, 1.0, int(num_frames)) * 0.1
>>> waveform = wav_data.tile([num_channels, 1])
>>> feature_extractor = MFCC(sr=sample_rate, n_fft=512, window='hann')
>>> feats = feature_extractor(waveform)
"""
dct_matrix: Tensor
def __init__(
self,
sr: int = 22050,
n_mfcc: int = 40,
n_fft: int = 512,
hop_length: int | None = None,
win_length: int | None = None,
window: _WindowLiteral = 'hann',
power: float = 2.0,
center: bool = True,
pad_mode: Literal['reflect'] = 'reflect',
n_mels: int = 64,
f_min: float = 50.0,
f_max: float | None = None,
htk: bool = False,
norm: Literal['slaney'] | float = 'slaney',
ref_value: float = 1.0,
amin: float = 1e-10,
top_db: float | None = None,
dtype: str = 'float32',
) -> None:
super().__init__()
assert n_mfcc <= n_mels, (
f'n_mfcc cannot be larger than n_mels: {n_mfcc} vs {n_mels}'
)
self._log_melspectrogram = LogMelSpectrogram(
sr=sr,
n_fft=n_fft,
hop_length=hop_length,
win_length=win_length,
window=window,
power=power,
center=center,
pad_mode=pad_mode,
n_mels=n_mels,
f_min=f_min,
f_max=f_max,
htk=htk,
norm=norm,
ref_value=ref_value,
amin=amin,
top_db=top_db,
dtype=dtype,
)
self.dct_matrix = create_dct(n_mfcc=n_mfcc, n_mels=n_mels, dtype=dtype)
self.register_buffer('dct_matrix', self.dct_matrix)
def forward(self, x: Tensor) -> Tensor:
"""
Args:
x (Tensor): Tensor of waveforms with shape `(N, T)`
Returns:
Tensor: Mel frequency cepstral coefficients with shape `(N, n_mfcc, num_frames)`.
"""
log_mel_feature = self._log_melspectrogram(x)
mfcc = paddle.matmul(
log_mel_feature.transpose((0, 2, 1)), self.dct_matrix
).transpose((0, 2, 1)) # (B, n_mels, L)
return mfcc
python/paddle/audio/functional/__init__.py
+36
View File
@@ -0,0 +1,36 @@
# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from .functional import (
compute_fbank_matrix,
create_dct,
fft_frequencies,
hz_to_mel,
mel_frequencies,
mel_to_hz,
power_to_db,
resample,
)
from .window import get_window
__all__ = [
'compute_fbank_matrix',
'create_dct',
'fft_frequencies',
'hz_to_mel',
'mel_frequencies',
'mel_to_hz',
'power_to_db',
'get_window',
'resample',
]
python/paddle/audio/functional/functional.py
+611
View File
@@ -0,0 +1,611 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Modified from librosa(https://github.com/librosa/librosa)
from __future__ import annotations
import math
from typing import TYPE_CHECKING, Literal, TypeVar
import paddle
from paddle import Tensor
from paddle.base.framework import Variable
from paddle.pir import Value
if TYPE_CHECKING:
_TensorOrFloat = TypeVar("_TensorOrFloat", Tensor, float)
def hz_to_mel(freq: _TensorOrFloat, htk: bool = False) -> _TensorOrFloat:
"""Convert Hz to Mels.
Args:
freq (Union[Tensor, float]): The input tensor with arbitrary shape.
htk (bool, optional): Use htk scaling. Defaults to False.
Returns:
Union[Tensor, float]: Frequency in mels.
Examples:
.. code-block:: pycon
>>> import paddle
>>> val = 3.0
>>> htk_flag = True
>>> mel_paddle_tensor = paddle.audio.functional.hz_to_mel(paddle.to_tensor(val), htk_flag)
"""
if htk:
if isinstance(freq, (Tensor, Variable, Value)):
return 2595.0 * paddle.log10(1.0 + freq / 700.0)
else:
return 2595.0 * math.log10(1.0 + freq / 700.0)
# Fill in the linear part
f_min = 0.0
f_sp = 200.0 / 3
mels = (freq - f_min) / f_sp
# Fill in the log-scale part
min_log_hz = 1000.0 # beginning of log region (Hz)
min_log_mel = (min_log_hz - f_min) / f_sp # same (Mels)
logstep = math.log(6.4) / 27.0 # step size for log region
if isinstance(freq, (Tensor, Variable, Value)):
target = (
min_log_mel + paddle.log(freq / min_log_hz + 1e-10) / logstep
) # prevent nan with 1e-10
mask = (freq > min_log_hz).astype(freq.dtype)
mels = target * mask + mels * (
1 - mask
) # will replace by masked_fill OP in future
else:
if freq >= min_log_hz:
mels = min_log_mel + math.log(freq / min_log_hz + 1e-10) / logstep
return mels
def mel_to_hz(mel: _TensorOrFloat, htk: bool = False) -> _TensorOrFloat:
"""Convert mel bin numbers to frequencies.
Args:
mel (Union[float, Tensor]): The mel frequency represented as a tensor with arbitrary shape.
htk (bool, optional): Use htk scaling. Defaults to False.
Returns:
Union[float, Tensor]: Frequencies in Hz.
Examples:
.. code-block:: pycon
>>> import paddle
>>> val = 3.0
>>> htk_flag = True
>>> mel_paddle_tensor = paddle.audio.functional.mel_to_hz(paddle.to_tensor(val), htk_flag)
"""
if htk:
return 700.0 * (10.0 ** (mel / 2595.0) - 1.0)
f_min = 0.0
f_sp = 200.0 / 3
freqs = f_min + f_sp * mel
# And now the nonlinear scale
min_log_hz = 1000.0 # beginning of log region (Hz)
min_log_mel = (min_log_hz - f_min) / f_sp # same (Mels)
logstep = math.log(6.4) / 27.0 # step size for log region
if isinstance(mel, (Tensor, Variable, Value)):
target = min_log_hz * paddle.exp(logstep * (mel - min_log_mel))
mask = (mel > min_log_mel).astype(mel.dtype)
freqs = target * mask + freqs * (
1 - mask
) # will replace by masked_fill OP in future
else:
if mel >= min_log_mel:
freqs = min_log_hz * math.exp(logstep * (mel - min_log_mel))
return freqs
def mel_frequencies(
n_mels: int = 64,
f_min: float = 0.0,
f_max: float = 11025.0,
htk: bool = False,
dtype: str = 'float32',
) -> Tensor:
"""Compute mel frequencies.
Args:
n_mels (int, optional): Number of mel bins. Defaults to 64.
f_min (float, optional): Minimum frequency in Hz. Defaults to 0.0.
fmax (float, optional): Maximum frequency in Hz. Defaults to 11025.0.
htk (bool, optional): Use htk scaling. Defaults to False.
dtype (str, optional): The data type of the return frequencies. Defaults to 'float32'.
Returns:
Tensor: Tensor of n_mels frequencies in Hz with shape `(n_mels,)`.
Examples:
.. code-block:: pycon
>>> import paddle
>>> n_mels = 64
>>> f_min = 0.5
>>> f_max = 10000
>>> htk_flag = True
>>> paddle_mel_freq = paddle.audio.functional.mel_frequencies(n_mels, f_min, f_max, htk_flag, 'float64')
"""
# 'Center freqs' of mel bands - uniformly spaced between limits
min_mel = hz_to_mel(f_min, htk=htk)
max_mel = hz_to_mel(f_max, htk=htk)
mels = paddle.linspace(min_mel, max_mel, n_mels, dtype=dtype)
freqs = mel_to_hz(mels, htk=htk)
return freqs
def fft_frequencies(sr: int, n_fft: int, dtype: str = 'float32') -> Tensor:
"""Compute fourier frequencies.
Args:
sr (int): Sample rate.
n_fft (int): Number of fft bins.
dtype (str, optional): The data type of the return frequencies. Defaults to 'float32'.
Returns:
Tensor: FFT frequencies in Hz with shape `(n_fft//2 + 1,)`.
Examples:
.. code-block:: pycon
>>> import paddle
>>> sr = 16000
>>> n_fft = 128
>>> fft_freq = paddle.audio.functional.fft_frequencies(sr, n_fft)
"""
return paddle.linspace(0, float(sr) / 2, int(1 + n_fft // 2), dtype=dtype)
def compute_fbank_matrix(
sr: int,
n_fft: int,
n_mels: int = 64,
f_min: float = 0.0,
f_max: float | None = None,
htk: bool = False,
norm: Literal['slaney'] | float = 'slaney',
dtype: str = 'float32',
) -> Tensor:
"""Compute fbank matrix.
Args:
sr (int): Sample rate.
n_fft (int): Number of fft bins.
n_mels (int, optional): Number of mel bins. Defaults to 64.
f_min (float, optional): Minimum frequency in Hz. Defaults to 0.0.
f_max (Optional[float], optional): Maximum frequency in Hz. Defaults to None.
htk (bool, optional): Use htk scaling. Defaults to False.
norm (Union[str, float], optional): Type of normalization. Defaults to 'slaney'.
dtype (str, optional): The data type of the return matrix. Defaults to 'float32'.
Returns:
Tensor: Mel transform matrix with shape `(n_mels, n_fft//2 + 1)`.
Examples:
.. code-block:: pycon
>>> import paddle
>>> sr = 23
>>> n_fft = 51
>>> fbank = paddle.audio.functional.compute_fbank_matrix(sr, n_fft)
"""
if f_max is None:
f_max = float(sr) / 2
# Initialize the weights
weights = paddle.zeros((n_mels, int(1 + n_fft // 2)), dtype=dtype)
# Center freqs of each FFT bin
fftfreqs = fft_frequencies(sr=sr, n_fft=n_fft, dtype=dtype)
# 'Center freqs' of mel bands - uniformly spaced between limits
mel_f = mel_frequencies(
n_mels + 2, f_min=f_min, f_max=f_max, htk=htk, dtype=dtype
)
fdiff = mel_f[1:] - mel_f[:-1] # np.diff(mel_f)
ramps = mel_f.unsqueeze(1) - fftfreqs.unsqueeze(0)
# ramps = np.subtract.outer(mel_f, fftfreqs)
for i in range(n_mels):
# lower and upper slopes for all bins
lower = -ramps[i] / fdiff[i]
upper = ramps[i + 2] / fdiff[i + 1]
# .. then intersect them with each other and zero
weights[i] = paddle.maximum(
paddle.zeros_like(lower), paddle.minimum(lower, upper)
)
# Slaney-style mel is scaled to be approx constant energy per channel
if norm == 'slaney':
enorm = 2.0 / (mel_f[2 : n_mels + 2] - mel_f[:n_mels])
weights *= enorm.unsqueeze(1)
elif isinstance(norm, (int, float)):
weights = paddle.nn.functional.normalize(weights, p=norm, axis=-1)
return weights
def power_to_db(
spect: Tensor,
ref_value: float = 1.0,
amin: float = 1e-10,
top_db: float | None = 80.0,
) -> Tensor:
"""Convert a power spectrogram (amplitude squared) to decibel (dB) units. The function computes the scaling `10 * log10(x / ref)` in a numerically stable way.
Args:
spect (Tensor): STFT power spectrogram.
ref_value (float, optional): The reference value. If smaller than 1.0, the db level of the signal will be pulled up accordingly. Otherwise, the db level is pushed down. Defaults to 1.0.
amin (float, optional): Minimum threshold. Defaults to 1e-10.
top_db (Optional[float], optional): Threshold the output at `top_db` below the peak. Defaults to None.
Returns:
Tensor: Power spectrogram in db scale.
Examples:
.. code-block:: pycon
>>> import paddle
>>> val = 3.0
>>> decibel_paddle = paddle.audio.functional.power_to_db(paddle.to_tensor(val))
"""
if amin <= 0:
raise Exception("amin must be strictly positive")
if ref_value <= 0:
raise Exception("ref_value must be strictly positive")
ones = paddle.ones_like(spect)
log_spec = 10.0 * paddle.log10(paddle.maximum(ones * amin, spect))
log_spec -= 10.0 * math.log10(max(ref_value, amin))
if top_db is not None:
if top_db < 0:
raise Exception("top_db must be non-negative")
log_spec = paddle.maximum(log_spec, ones * (log_spec.max() - top_db))
return log_spec
def create_dct(
n_mfcc: int,
n_mels: int,
norm: Literal['ortho'] | None = 'ortho',
dtype: str = 'float32',
) -> Tensor:
"""Create a discrete cosine transform(DCT) matrix.
Args:
n_mfcc (int): Number of mel frequency cepstral coefficients.
n_mels (int): Number of mel filterbanks.
norm (Optional[str], optional): Normalization type. Defaults to 'ortho'.
dtype (str, optional): The data type of the return matrix. Defaults to 'float32'.
Returns:
Tensor: The DCT matrix with shape `(n_mels, n_mfcc)`.
Examples:
.. code-block:: pycon
>>> import paddle
>>> n_mfcc = 23
>>> n_mels = 257
>>> dct = paddle.audio.functional.create_dct(n_mfcc, n_mels)
"""
n = paddle.arange(n_mels, dtype=dtype)
k = paddle.arange(n_mfcc, dtype=dtype).unsqueeze(1)
dct = paddle.cos(
math.pi / float(n_mels) * (n + 0.5) * k
) # size (n_mfcc, n_mels)
if norm is None:
dct *= 2.0
else:
assert norm == "ortho"
dct[0] *= 1.0 / math.sqrt(2.0)
dct *= math.sqrt(2.0 / float(n_mels))
return dct.T
def _get_sinc_resample_kernel(
orig_freq: int,
new_freq: int,
gcd: int,
lowpass_filter_width: int = 6,
rolloff: float = 0.99,
resampling_method: Literal[
"sinc_interp_hann", "sinc_interp_kaiser"
] = "sinc_interp_hann",
beta: float | None = None,
dtype: paddle.dtype | None = None,
):
"""
Generate the sinc interpolation kernel for resampling.
This internal function computes the resampling kernel based on the sinc
interpolation formula with windowing. The kernel is used by
_apply_sinc_resample_kernel to perform the actual resampling.
Args:
orig_freq (int): Original sampling frequency.
new_freq (int): Target sampling frequency.
gcd (int): Greatest common divisor of orig_freq and new_freq.
lowpass_filter_width (int, optional): Controls the sharpness of the filter,
larger value means sharper but less efficient. Default: 6.
rolloff (float, optional): Roll-off frequency as a fraction of the Nyquist.
Lower values reduce anti-aliasing but also attenuate high frequencies.
Default: 0.99.
resampling_method (str, optional): Window method for filter design.
Options: ["sinc_interp_hann", "sinc_interp_kaiser"]. Default: "sinc_interp_hann".
beta (float, optional): Shape parameter for Kaiser window. Required only
when resampling_method="sinc_interp_kaiser". Default: None.
dtype (paddle.dtype, optional): Data type for kernel computation.
If None, uses float64 for computation and converts to float32 for output.
Default: None.
Returns:
tuple: (kernel, width)
- kernel (Tensor): Resampling kernel of shape (1, 1, kernel_width)
- width (int): Half-width of the filter in terms of input samples
Raises:
Exception: If frequencies are not integers.
ValueError: If resampling_method is invalid or lowpass_filter_width <= 0.
"""
if not (int(orig_freq) == orig_freq and int(new_freq) == new_freq):
raise ValueError(
"Frequencies must be of integer type to ensure quality resampling computation. "
"To work around this, manually convert both frequencies to integer values "
"that maintain their resampling rate ratio before passing them into the function. "
"Example: To downsample a 44100 hz waveform by a factor of 8, use "
"`orig_freq=8` and `new_freq=1` instead of `orig_freq=44100` and `new_freq=5512.5`. "
)
if resampling_method not in ["sinc_interp_hann", "sinc_interp_kaiser"]:
raise ValueError(f"Invalid resampling method: {resampling_method}")
orig_freq = int(orig_freq) // gcd
new_freq = int(new_freq) // gcd
if lowpass_filter_width <= 0:
raise ValueError("Low pass filter width should be positive.")
base_freq = min(orig_freq, new_freq)
# Perform antialiasing filtering by removing the highest frequencies.
base_freq *= rolloff
# Calculate filter width based on lowpass_filter_width and frequency ratio
width = math.ceil(lowpass_filter_width * orig_freq / base_freq)
idx_dtype = dtype if dtype is not None else paddle.float64
idx = (
paddle.arange(-width, width + orig_freq, dtype=idx_dtype)[None, None]
/ orig_freq
)
t = (
paddle.arange(0, -new_freq, -1, dtype=dtype)[:, None, None] / new_freq
+ idx
)
t *= base_freq
t = t.clip_(-lowpass_filter_width, lowpass_filter_width)
# we do not use built-in paddle windows here as we need to evaluate the window
# at specific positions, not over a regular grid.
if resampling_method == "sinc_interp_hann":
window = paddle.cos(t * math.pi / lowpass_filter_width / 2) ** 2
else:
# sinc_interp_kaiser
if beta is None:
beta = 14.769656459379492
beta_tensor = paddle.to_tensor(float(beta))
window = paddle.i0(
beta_tensor * paddle.sqrt(1 - (t / lowpass_filter_width) ** 2),
) / paddle.i0(beta_tensor)
t *= math.pi
scale = base_freq / orig_freq
kernels = paddle.where(
t == 0, paddle.to_tensor(1.0).cast(t.dtype), t.sin() / t
)
kernels *= window * scale
if dtype is None: # pragma: no cover
kernels = kernels.cast(paddle.float32)
return kernels, width
def _apply_sinc_resample_kernel(
waveform: Tensor,
orig_freq: int,
new_freq: int,
gcd: int,
kernel: Tensor,
width: int,
):
"""
Apply sinc interpolation resampling using precomputed kernel.
This internal function performs the actual resampling operation using the
kernel generated by _get_sinc_resample_kernel. It handles batch processing
and ensures correct output length.
Args:
waveform (Tensor): Input waveform of shape (..., time). Must be floating point.
orig_freq (int): Original sampling frequency.
new_freq (int): Target sampling frequency.
gcd (int): Greatest common divisor of orig_freq and new_freq.
kernel (Tensor): Resampling kernel from _get_sinc_resample_kernel.
width (int): Half-width of the filter from _get_sinc_resample_kernel.
Returns:
Tensor: Resampled waveform of shape (..., new_time).
"""
orig_freq = int(orig_freq) // gcd
new_freq = int(new_freq) // gcd
# pack batch
shape = waveform.shape
waveform = waveform.reshape([-1, shape[-1]])
num_wavs, length = waveform.shape
waveform = paddle.nn.functional.pad(waveform, (width, width + orig_freq))
resampled = paddle.nn.functional.conv1d(
waveform[:, None], kernel, stride=orig_freq
)
resampled = resampled.transpose([0, 2, 1]).reshape((num_wavs, -1))
target_length = paddle.ceil(
paddle.to_tensor(new_freq * length / orig_freq)
).astype(paddle.int64)
resampled = resampled[..., :target_length]
# unpack batch
resampled = resampled.reshape(shape[:-1] + resampled.shape[-1:])
return resampled
def resample(
waveform: Tensor,
orig_freq: int,
new_freq: int,
lowpass_filter_width: int = 6,
rolloff: float = 0.99,
resampling_method: Literal[
"sinc_interp_hann", "sinc_interp_kaiser"
] = "sinc_interp_hann",
beta: float | None = None,
) -> Tensor:
"""
Resample the waveform from orig_freq to new_freq using bandlimited interpolation.
This function implements resampling through sinc interpolation with windowing.
It first computes a resampling kernel based on the specified parameters, then
applies it to the input waveform using convolution. The algorithm handles both
upsampling and downsampling while minimizing aliasing artifacts.
Args:
waveform (Tensor): The input signal of dimension (..., time). Must be
floating point type (float32 or float64).
orig_freq (int): The original frequency of the signal. Must be positive.
new_freq (int): The desired target frequency. Must be positive.
lowpass_filter_width (int, optional): Controls the sharpness of the filter.
Larger values give sharper filtering but are less efficient.
Default: 6.
rolloff (float, optional): The roll-off frequency of the filter as a fraction
of the Nyquist frequency. Lower values reduce anti-aliasing but also
attenuate some high frequencies. Default: 0.99.
resampling_method (str, optional): The windowing method to use for filter
design. Options: "sinc_interp_hann" (Hann window) or "sinc_interp_kaiser"
(Kaiser window). Default: "sinc_interp_hann".
beta (float, optional): Shape parameter for the Kaiser window. Required only
when resampling_method="sinc_interp_kaiser". If not provided for Kaiser,
a default value of 14.769656459379492 is used. Default: None.
Returns:
Tensor: The waveform resampled to new_freq, with dimension (..., new_time).
Raises:
ValueError: If orig_freq or new_freq are not positive.
Exception: If frequencies are not integers (see note below).
TypeError: If waveform is not floating point.
Note:
- orig_freq and new_freq must be integers. For non-integer frequencies,
convert them to integers while maintaining the ratio.
- For repeated resampling with same parameters, use
:class:`paddle.audio.transforms.Resample` for better efficiency.
- Uses windowed sinc interpolation for high-quality audio resampling.
- This function does not support ONNX export now.
Examples:
.. code-block:: pycon
>>> import paddle
>>> from paddle.audio.functional import resample
>>> # Create a sample waveform (1 channel, 1000 samples at 16000 Hz)
>>> waveform = paddle.randn([1, 1000])
>>> # Downsample from 16000 Hz to 8000 Hz
>>> resampled = resample(waveform, 16000, 8000)
>>> print(resampled.shape)
paddle.Size([1, 500])
>>> # Upsample from 16000 Hz to 48000 Hz with custom filter width
>>> resampled = resample(waveform, 16000, 48000, lowpass_filter_width=12)
>>> print(resampled.shape)
paddle.Size([1, 3000])
>>> # Use Kaiser window resampling
>>> resampled = resample(waveform, 16000, 8000, resampling_method="sinc_interp_kaiser", beta=12.0)
>>> print(resampled.shape)
paddle.Size([1, 500])
>>> # Batch processing: multiple waveforms
>>> batch_waveforms = paddle.randn([4, 1, 1000]) # [batch, channels, time]
>>> resampled_batch = resample(batch_waveforms, 16000, 8000)
>>> print(resampled_batch.shape)
paddle.Size([4, 1, 500])
"""
if orig_freq <= 0.0 or new_freq <= 0.0:
raise ValueError(
"Original frequency and desired frequency should be positive integers"
)
if not waveform.is_floating_point():
raise TypeError(
f"Expected floating point type for waveform tensor, but received {waveform.dtype}."
)
if orig_freq == new_freq:
return waveform
gcd = math.gcd(int(orig_freq), int(new_freq))
kernel, width = _get_sinc_resample_kernel(
orig_freq,
new_freq,
gcd,
lowpass_filter_width,
rolloff,
resampling_method,
beta,
waveform.dtype,
)
resampled = _apply_sinc_resample_kernel(
waveform, orig_freq, new_freq, gcd, kernel, width
)
return resampled
python/paddle/audio/functional/window.py
+731
View File
@@ -0,0 +1,731 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
from __future__ import annotations
import math
import warnings
from typing import TYPE_CHECKING
import numpy as np
import paddle
if TYPE_CHECKING:
from paddle import Tensor
from paddle._typing import PlaceLike
from ..features.layers import _WindowLiteral
from paddle.base.framework import (
_current_expected_place,
_get_paddle_place,
_to_pinned_place,
in_dynamic_or_pir_mode,
)
class WindowFunctionRegister:
def __init__(self):
self._functions_dict = {}
def register(self, func=None):
def add_subfunction(func):
name = func.__name__
self._functions_dict[name] = func
return func
return add_subfunction
def get(self, name):
return self._functions_dict[name]
window_function_register = WindowFunctionRegister()
@window_function_register.register()
def _cat(x: list[Tensor], data_type: str) -> Tensor:
l = []
for t in x:
if np.isscalar(t) and not isinstance(t, str):
l.append(paddle.to_tensor([t], data_type))
else:
l.append(paddle.to_tensor(t, data_type))
return paddle.concat(l)
@window_function_register.register()
def _bartlett(M: int, sym: bool = True, dtype: str = 'float64') -> Tensor:
"""
Computes the Bartlett window.
This function is consistent with scipy.signal.windows.bartlett().
"""
if _len_guards(M):
return paddle.ones((M,), dtype=dtype)
M, needs_trunc = _extend(M, sym)
n = paddle.arange(0, M, dtype=dtype)
M = paddle.to_tensor(M, dtype=dtype)
w = paddle.where(
paddle.less_equal(n, (M - 1) / 2.0),
2.0 * n / (M - 1),
2.0 - 2.0 * n / (M - 1),
)
return _truncate(w, needs_trunc)
@window_function_register.register()
def _kaiser(
M: int, beta: float, sym: bool = True, dtype: str = 'float64'
) -> Tensor:
"""Compute the Kaiser window.
This function is consistent with scipy.signal.windows.kaiser().
"""
if _len_guards(M):
return paddle.ones((M,), dtype=dtype)
M, needs_trunc = _extend(M, sym)
beta = paddle.to_tensor(beta, dtype=dtype)
n = paddle.arange(0, M, dtype=dtype)
M = paddle.to_tensor(M, dtype=dtype)
alpha = (M - 1) / 2.0
w = paddle.i0(
beta * paddle.sqrt(1 - ((n - alpha) / alpha) ** 2.0)
) / paddle.i0(beta)
return _truncate(w, needs_trunc)
@window_function_register.register()
def _nuttall(M: int, sym: bool = True, dtype: str = 'float64') -> Tensor:
"""Nuttall window.
This function is consistent with scipy.signal.windows.nuttall().
"""
a = paddle.to_tensor(
[0.3635819, 0.4891775, 0.1365995, 0.0106411], dtype=dtype
)
return _general_cosine(M, a=a, sym=sym, dtype=dtype)
@window_function_register.register()
def _acosh(x: Tensor | float) -> Tensor:
if isinstance(x, float):
return math.log(x + math.sqrt(x**2 - 1))
return paddle.log(x + paddle.sqrt(paddle.square(x) - 1))
@window_function_register.register()
def _extend(M: int, sym: bool) -> bool:
"""Extend window by 1 sample if needed for DFT-even symmetry."""
if not sym:
return M + 1, True
else:
return M, False
@window_function_register.register()
def _len_guards(M: int) -> bool:
"""Handle small or incorrect window lengths."""
if int(M) != M or M < 0:
raise ValueError('Window length M must be a non-negative integer')
return M <= 1
@window_function_register.register()
def _truncate(w: Tensor, needed: bool) -> Tensor:
"""Truncate window by 1 sample if needed for DFT-even symmetry."""
if needed:
return w[:-1]
else:
return w
@window_function_register.register()
def _general_gaussian(
M: int, p, sig, sym: bool = True, dtype: str = 'float64'
) -> Tensor:
"""Compute a window with a generalized Gaussian shape.
This function is consistent with scipy.signal.windows.general_gaussian().
"""
if _len_guards(M):
return paddle.ones((M,), dtype=dtype)
M, needs_trunc = _extend(M, sym)
n = paddle.arange(0, M, dtype=dtype) - (M - 1.0) / 2.0
w = paddle.exp(-0.5 * paddle.abs(n / sig) ** (2 * p))
return _truncate(w, needs_trunc)
@window_function_register.register()
def _general_cosine(
M: int, a: list[float], sym: bool = True, dtype: str = 'float64'
) -> Tensor:
"""Compute a generic weighted sum of cosine terms window.
This function is consistent with scipy.signal.windows.general_cosine().
"""
if _len_guards(M):
return paddle.ones((M,), dtype=dtype)
M, needs_trunc = _extend(M, sym)
fac = paddle.linspace(-math.pi, math.pi, M, dtype=dtype)
w = paddle.zeros((M,), dtype=dtype)
for k in range(len(a)):
w += a[k] * paddle.cos(k * fac)
return _truncate(w, needs_trunc)
@window_function_register.register()
def _general_hamming(
M: int, alpha: float, sym: bool = True, dtype: str = 'float64'
) -> Tensor:
"""Compute a generalized Hamming window.
This function is consistent with scipy.signal.windows.general_hamming()
"""
return _general_cosine(M, [alpha, 1.0 - alpha], sym, dtype=dtype)
@window_function_register.register()
def _taylor(
M: int, nbar=4, sll=30, norm=True, sym: bool = True, dtype: str = 'float64'
) -> Tensor:
"""Compute a Taylor window.
The Taylor window taper function approximates the Dolph-Chebyshev window's
constant sidelobe level for a parameterized number of near-in sidelobes.
"""
if _len_guards(M):
return paddle.ones((M,), dtype=dtype)
M, needs_trunc = _extend(M, sym)
# Original text uses a negative sidelobe level parameter and then negates
# it in the calculation of B. To keep consistent with other methods we
# assume the sidelobe level parameter to be positive.
B = 10 ** (sll / 20)
A = _acosh(B) / math.pi
s2 = nbar**2 / (A**2 + (nbar - 0.5) ** 2)
ma = paddle.arange(1, nbar, dtype=dtype)
Fm = paddle.empty((nbar - 1,), dtype=dtype)
signs = paddle.empty_like(ma)
signs[::2] = 1
signs[1::2] = -1
m2 = ma * ma
for mi in range(len(ma)):
number = signs[mi] * paddle.prod(
1 - m2[mi] / s2 / (A**2 + (ma - 0.5) ** 2)
)
if mi == 0:
denom = 2 * paddle.prod(1 - m2[mi] / m2[mi + 1 :])
elif mi == len(ma) - 1:
denom = 2 * paddle.prod(1 - m2[mi] / m2[:mi])
else:
denom = (
2
* paddle.prod(1 - m2[mi] / m2[:mi])
* paddle.prod(1 - m2[mi] / m2[mi + 1 :])
)
Fm[mi] = number / denom
def W(n):
return 1 + 2 * paddle.matmul(
Fm.unsqueeze(0),
paddle.cos(2 * math.pi * ma.unsqueeze(1) * (n - M / 2.0 + 0.5) / M),
)
w = W(paddle.arange(0, M, dtype=dtype))
# normalize (Note that this is not described in the original text [1])
if norm:
scale = 1.0 / W((M - 1) / 2)
w *= scale
w = w.squeeze()
return _truncate(w, needs_trunc)
@window_function_register.register()
def _hamming(M: int, sym: bool = True, dtype: str = 'float64') -> Tensor:
"""Compute a Hamming window.
The Hamming window is a taper formed by using a raised cosine with
non-zero endpoints, optimized to minimize the nearest side lobe.
"""
return _general_hamming(M, 0.54, sym, dtype=dtype)
@window_function_register.register()
def _hann(M: int, sym: bool = True, dtype: str = 'float64') -> Tensor:
"""Compute a Hann window.
The Hann window is a taper formed by using a raised cosine or sine-squared
with ends that touch zero.
"""
return _general_hamming(M, 0.5, sym, dtype=dtype)
@window_function_register.register()
def _tukey(
M: int, alpha=0.5, sym: bool = True, dtype: str = 'float64'
) -> Tensor:
"""Compute a Tukey window.
The Tukey window is also known as a tapered cosine window.
"""
if _len_guards(M):
return paddle.ones((M,), dtype=dtype)
if alpha <= 0:
return paddle.ones((M,), dtype=dtype)
elif alpha >= 1.0:
return _hann(M, sym=sym)
M, needs_trunc = _extend(M, sym)
n = paddle.arange(0, M, dtype=dtype)
width = int(alpha * (M - 1) / 2.0)
n1 = n[0 : width + 1]
n2 = n[width + 1 : M - width - 1]
n3 = n[M - width - 1 :]
w1 = 0.5 * (1 + paddle.cos(math.pi * (-1 + 2.0 * n1 / alpha / (M - 1))))
w2 = paddle.ones(n2.shape, dtype=dtype)
w3 = 0.5 * (
1
+ paddle.cos(math.pi * (-2.0 / alpha + 1 + 2.0 * n3 / alpha / (M - 1)))
)
w = paddle.concat([w1, w2, w3])
return _truncate(w, needs_trunc)
@window_function_register.register()
def _gaussian(
M: int, std: float, sym: bool = True, dtype: str = 'float64'
) -> Tensor:
"""Compute a Gaussian window.
The Gaussian widows has a Gaussian shape defined by the standard deviation(std).
"""
if _len_guards(M):
return paddle.ones((M,), dtype=dtype)
M, needs_trunc = _extend(M, sym)
n = paddle.arange(0, M, dtype=dtype) - (M - 1.0) / 2.0
sig2 = 2 * std * std
w = paddle.exp(-(n**2) / sig2)
return _truncate(w, needs_trunc)
@window_function_register.register()
def _exponential(
M: int, center=None, tau=1.0, sym: bool = True, dtype: str = 'float64'
) -> Tensor:
"""Compute an exponential (or Poisson) window."""
if sym and center is not None:
raise ValueError("If sym==True, center must be None.")
if _len_guards(M):
return paddle.ones((M,), dtype=dtype)
M, needs_trunc = _extend(M, sym)
if center is None:
center = (M - 1) / 2
n = paddle.arange(0, M, dtype=dtype)
w = paddle.exp(-paddle.abs(n - center) / tau)
return _truncate(w, needs_trunc)
@window_function_register.register()
def _triang(M: int, sym: bool = True, dtype: str = 'float64') -> Tensor:
"""Compute a triangular window."""
if _len_guards(M):
return paddle.ones((M,), dtype=dtype)
M, needs_trunc = _extend(M, sym)
n = paddle.arange(1, (M + 1) // 2 + 1, dtype=dtype)
if M % 2 == 0:
w = (2 * n - 1.0) / M
w = paddle.concat([w, w[::-1]])
else:
w = 2 * n / (M + 1.0)
w = paddle.concat([w, w[-2::-1]])
return _truncate(w, needs_trunc)
@window_function_register.register()
def _bohman(M: int, sym: bool = True, dtype: str = 'float64') -> Tensor:
"""Compute a Bohman window.
The Bohman window is the autocorrelation of a cosine window.
"""
if _len_guards(M):
return paddle.ones((M,), dtype=dtype)
M, needs_trunc = _extend(M, sym)
fac = paddle.abs(paddle.linspace(-1, 1, M, dtype=dtype)[1:-1])
w = (1 - fac) * paddle.cos(math.pi * fac) + 1.0 / math.pi * paddle.sin(
math.pi * fac
)
w = _cat([0, w, 0], dtype)
return _truncate(w, needs_trunc)
@window_function_register.register()
def _blackman(M: int, sym: bool = True, dtype: str = 'float64') -> Tensor:
"""Compute a Blackman window.
The Blackman window is a taper formed by using the first three terms of
a summation of cosines. It was designed to have close to the minimal
leakage possible. It is close to optimal, only slightly worse than a
Kaiser window.
"""
return _general_cosine(M, [0.42, 0.50, 0.08], sym, dtype=dtype)
@window_function_register.register()
def _cosine(M: int, sym: bool = True, dtype: str = 'float64') -> Tensor:
"""Compute a window with a simple cosine shape."""
if _len_guards(M):
return paddle.ones((M,), dtype=dtype)
M, needs_trunc = _extend(M, sym)
w = paddle.sin(math.pi / M * (paddle.arange(0, M, dtype=dtype) + 0.5))
return _truncate(w, needs_trunc)
def get_window(
window: _WindowLiteral | tuple[_WindowLiteral, float],
win_length: int,
fftbins: bool = True,
dtype: str | None = 'float64',
) -> Tensor:
"""Return a window of a given length and type.
Args:
window (Union[str, Tuple[str, float]]): The window function applied to the signal before the Fourier transform. Supported window functions: 'hamming', 'hann', 'gaussian', 'general_gaussian', 'exponential', 'triang', 'bohman', 'blackman', 'cosine', 'tukey', 'taylor', 'bartlett', 'kaiser', 'nuttall'.
win_length (int): Number of samples.
fftbins (bool, optional): If True, create a "periodic" window. Otherwise, create a "symmetric" window, for use in filter design. Defaults to True.
dtype (str, optional): The data type of the return window. Defaults to 'float64'.
Returns:
Tensor: The window represented as a tensor.
Examples:
.. code-block:: pycon
>>> import paddle
>>> n_fft = 512
>>> cosine_window = paddle.audio.functional.get_window('cosine', n_fft)
>>> std = 7
>>> gaussian_window = paddle.audio.functional.get_window(('gaussian', std), n_fft)
"""
if dtype is None:
dtype = 'float32'
sym = not fftbins
args = ()
if isinstance(window, tuple):
winstr = window[0]
if len(window) > 1:
args = window[1:]
elif isinstance(window, str):
if window in ['gaussian', 'exponential', 'kaiser']:
raise ValueError(
"The '" + window + "' window needs one or "
"more parameters -- pass a tuple."
)
else:
winstr = window
else:
raise ValueError(f"{type(window)} as window type is not supported.")
try:
winfunc = window_function_register.get('_' + winstr)
except KeyError as e:
raise ValueError("Unknown window type.") from e
params = (win_length, *args)
kwargs = {'sym': sym}
return winfunc(*params, dtype=dtype, **kwargs)
def _apply_window_postprocess(
w: Tensor,
*,
layout: str | None = None,
device: PlaceLike | None = None,
pin_memory: bool = False,
requires_grad: bool = False,
) -> Tensor:
if layout not in (None, 'strided'):
raise RuntimeError(
"Window functions only support layout='strided' or None"
)
if layout is not None:
warnings.warn("layout only supports 'strided' in Paddle; ignored")
if in_dynamic_or_pir_mode():
device = (
_get_paddle_place(device)
if device is not None
else _current_expected_place()
)
if pin_memory and paddle.in_dynamic_mode() and device is not None:
device = _to_pinned_place(device)
w = w.to(device=device)
if pin_memory and paddle.in_dynamic_mode():
w = w.pin_memory()
if requires_grad is True:
w.stop_gradient = False
return w
def hamming_window(
window_length: int,
periodic: bool = True,
alpha: float = 0.54,
beta: float = 0.46,
*,
dtype: str = 'float32',
layout: str | None = None,
device: PlaceLike | None = None,
pin_memory: bool = False,
requires_grad: bool = False,
):
"""
Compute a generalized Hamming window.
Args:
window_length (int): The size of the returned window. Must be positive.
periodic (bool, optional): If True, returns a window for use as a periodic function; if False, returns a symmetric window. Defaults to True.
alpha (float, optional): The coefficient α in the equation above. Defaults to 0.54.
beta (float, optional): The coefficient β in the equation above. Defaults to 0.46.
dtype (str, optional): The data type of the returned tensor. Defaults to 'float32'.
layout (str, optional): Only included for API consistency with PyTorch; ignored in Paddle. Defaults to None.
device(PlaceLike|None, optional): The desired device of returned tensor.
if None, uses the current device for the default tensor type (see paddle.device.set_device()).
device will be the CPU for CPU tensor types and the current CUDA device for CUDA tensor types. Default: None.
pin_memory(bool, optional): If set, return tensor would be allocated in the pinned memory. Works only for CPU tensors. Default: False
requires_grad(bool, optional): If autograd should record operations on the returned tensor. Default: False.
Returns:
Tensor: A 1-D tensor of shape `(window_length,)` containing the Hamming window.
Examples:
.. code-block:: pycon
>>> import paddle
>>> win = paddle.hamming_window(400, requires_grad=True)
>>> win = paddle.hamming_window(256, alpha=0.5, beta=0.5)
"""
w0 = get_window('hamming', window_length, fftbins=periodic, dtype=dtype)
alpha0, beta0 = 0.54, 0.46
B = beta / beta0
A = alpha - B * alpha0
w = A + B * w0
return _apply_window_postprocess(
w,
layout=layout,
device=device,
pin_memory=pin_memory,
requires_grad=requires_grad,
)
def hann_window(
window_length: int,
periodic: bool = True,
*,
dtype: str = 'float32',
layout: str | None = None,
device: PlaceLike | None = None,
pin_memory: bool = False,
requires_grad: bool = False,
):
"""
Compute a Hann window.
Args:
window_length (int): The size of the returned window. Must be positive.
periodic (bool, optional): If True, returns a window for use as a periodic function; if False, returns a symmetric window. Defaults to True.
dtype (str, optional): The data type of the returned tensor. Defaults to 'float32'.
layout (str, optional): Only included for API consistency with PyTorch; ignored in Paddle. Defaults to None.
device(PlaceLike|None, optional): The desired device of returned tensor.
if None, uses the current device for the default tensor type (see paddle.device.set_device()).
device will be the CPU for CPU tensor types and the current CUDA device for CUDA tensor types. Default: None.
pin_memory(bool, optional): If set, return tensor would be allocated in the pinned memory. Works only for CPU tensors. Default: False
requires_grad(bool, optional): If autograd should record operations on the returned tensor. Default: False.
Returns:
Tensor: A 1-D tensor of shape `(window_length,)` containing the Hann window.
Examples:
.. code-block:: pycon
>>> import paddle
>>> win = paddle.hann_window(512)
>>> win = paddle.hann_window(512, requires_grad=True)
"""
w = get_window('hann', window_length, fftbins=periodic, dtype=dtype)
return _apply_window_postprocess(
w,
layout=layout,
device=device,
pin_memory=pin_memory,
requires_grad=requires_grad,
)
def kaiser_window(
window_length: int,
periodic: bool = True,
beta: float = 12.0,
*,
dtype: str | None = 'float32',
layout: str | None = None,
device: PlaceLike | None = None,
pin_memory: bool = False,
requires_grad: bool = False,
out: Tensor | None = None,
):
"""
Compute a Kaiser window.
Args:
window_length (int): The size of the returned window. Must be positive.
periodic (bool, optional): If True, returns a window for use as a periodic function; if False, returns a symmetric window. Defaults to True.
beta (float, optional): Shape parameter for the window. Defaults to 12.0.
dtype (str, optional): The data type of the returned tensor. Defaults to 'float32'.
layout (str, optional): Only included for API consistency with PyTorch; ignored in Paddle. Defaults to None.
device(PlaceLike|None, optional): The desired device of returned tensor.
if None, uses the current device for the default tensor type (see paddle.device.set_device()).
device will be the CPU for CPU tensor types and the current CUDA device for CUDA tensor types. Default: None.
pin_memory(bool, optional): If set, return tensor would be allocated in the pinned memory. Works only for CPU tensors. Default: False
requires_grad(bool, optional): If autograd should record operations on the returned tensor. Default: False.
out(Tensor|None, optional): The output tensor. Default: None.
Returns:
Tensor: A 1-D tensor of shape `(window_length,)` containing the Kaiser window.
Examples:
.. code-block:: pycon
>>> import paddle
>>> win = paddle.kaiser_window(400, beta=8.6)
>>> win = paddle.kaiser_window(400, requires_grad=True)
"""
w = get_window(
('kaiser', beta), window_length, fftbins=periodic, dtype=dtype
)
w = _apply_window_postprocess(
w,
layout=layout,
device=device,
pin_memory=pin_memory,
requires_grad=requires_grad,
)
return paddle.assign(w, out) if out is not None else w
def blackman_window(
window_length: int,
periodic: bool = True,
*,
dtype: str = 'float32',
layout: str | None = None,
device: PlaceLike | None = None,
pin_memory: bool = False,
requires_grad: bool = False,
):
"""
Compute a Blackman window.
Args:
window_length (int): The size of the returned window. Must be positive.
periodic (bool, optional): If True, returns a window for use as a periodic function; if False, returns a symmetric window. Defaults to True.
dtype (str, optional): The data type of the returned tensor. Defaults to 'float32'.
layout (str, optional): Only included for API consistency with PyTorch; ignored in Paddle. Defaults to None.
device(PlaceLike|None, optional): The desired device of returned tensor.
if None, uses the current device for the default tensor type (see paddle.device.set_device()).
device will be the CPU for CPU tensor types and the current CUDA device for CUDA tensor types. Default: None.
pin_memory(bool, optional): If set, return tensor would be allocated in the pinned memory. Works only for CPU tensors. Default: False
requires_grad(bool, optional): If autograd should record operations on the returned tensor. Default: False.
Returns:
Tensor: A 1-D tensor of shape `(window_length,)` containing the Blackman window.
Examples:
.. code-block:: pycon
>>> import paddle
>>> win = paddle.blackman_window(256)
>>> win = paddle.blackman_window(256, requires_grad=True)
"""
w = get_window('blackman', window_length, fftbins=periodic, dtype=dtype)
return _apply_window_postprocess(
w,
layout=layout,
device=device,
pin_memory=pin_memory,
requires_grad=requires_grad,
)
def bartlett_window(
window_length: int,
periodic: bool = True,
*,
dtype: str = 'float32',
layout: str | None = None,
device: PlaceLike | None = None,
pin_memory: bool = False,
requires_grad: bool = False,
):
"""
Compute a Bartlett window.
Args:
window_length (int): The size of the returned window. Must be positive.
periodic (bool, optional): If True, returns a window for use as a periodic function; if False, returns a symmetric window. Defaults to True.
dtype (str, optional): The data type of the returned tensor. Defaults to 'float32'.
layout (str, optional): Only included for API consistency with PyTorch; ignored in Paddle. Defaults to None.
device(PlaceLike|None, optional): The desired device of returned tensor.
if None, uses the current device for the default tensor type (see paddle.device.set_device()).
device will be the CPU for CPU tensor types and the current CUDA device for CUDA tensor types. Default: None.
pin_memory(bool, optional): If set, return tensor would be allocated in the pinned memory. Works only for CPU tensors. Default: False
requires_grad(bool, optional): If autograd should record operations on the returned tensor. Default: False.
Returns:
Tensor: A 1-D tensor of shape `(window_length,)` containing the Bartlett window.
Examples:
.. code-block:: pycon
>>> import paddle
>>> n_fft = 512
>>> win = paddle.bartlett_window(n_fft)
>>> win = paddle.bartlett_window(n_fft, requires_grad=True)
"""
w = get_window('bartlett', window_length, fftbins=periodic, dtype=dtype)
return _apply_window_postprocess(
w,
layout=layout,
device=device,
pin_memory=pin_memory,
requires_grad=requires_grad,
)