chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,20 @@
|
||||
import torch.nn
|
||||
from modules.backbones.wavenet import WaveNet
|
||||
from modules.backbones.lynxnet import LYNXNet
|
||||
from modules.backbones.lynxnet2 import LYNXNet2
|
||||
from utils import filter_kwargs
|
||||
|
||||
BACKBONES = {
|
||||
'wavenet': WaveNet,
|
||||
'lynxnet': LYNXNet,
|
||||
'lynxnet2': LYNXNet2,
|
||||
}
|
||||
|
||||
|
||||
def build_backbone(
|
||||
out_dims: int, num_feats: int,
|
||||
backbone_type: str, backbone_args: dict
|
||||
) -> torch.nn.Module:
|
||||
backbone = BACKBONES[backbone_type]
|
||||
kwargs = filter_kwargs(backbone_args, backbone)
|
||||
return BACKBONES[backbone_type](out_dims, num_feats, **kwargs)
|
||||
@@ -0,0 +1,147 @@
|
||||
# refer to：
|
||||
# https://github.com/CNChTu/Diffusion-SVC/blob/v2.0_dev/diffusion/naive_v2/model_conformer_naive.py
|
||||
# https://github.com/CNChTu/Diffusion-SVC/blob/v2.0_dev/diffusion/naive_v2/naive_v2_diff.py
|
||||
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from modules.commons.common_layers import SinusoidalPosEmb, SwiGLU, Transpose, AdamWConv1d
|
||||
from modules.commons.common_layers import KaimingNormalConv1d as Conv1d
|
||||
from utils.hparams import hparams
|
||||
|
||||
|
||||
class LYNXConvModule(nn.Module):
|
||||
@staticmethod
|
||||
def calc_same_padding(kernel_size):
|
||||
pad = kernel_size // 2
|
||||
return pad, pad - (kernel_size + 1) % 2
|
||||
|
||||
def __init__(self, dim, expansion_factor, kernel_size=31, activation='PReLU', dropout=0.0):
|
||||
super().__init__()
|
||||
inner_dim = dim * expansion_factor
|
||||
activation_classes = {
|
||||
'SiLU': nn.SiLU,
|
||||
'ReLU': nn.ReLU,
|
||||
'PReLU': lambda: nn.PReLU(inner_dim)
|
||||
}
|
||||
activation = activation if activation is not None else 'PReLU'
|
||||
if activation not in activation_classes:
|
||||
raise ValueError(f'{activation} is not a valid activation')
|
||||
_activation = activation_classes[activation]()
|
||||
padding = self.calc_same_padding(kernel_size)
|
||||
if float(dropout) > 0.:
|
||||
_dropout = nn.Dropout(dropout)
|
||||
else:
|
||||
_dropout = nn.Identity()
|
||||
self.net = nn.Sequential(
|
||||
nn.LayerNorm(dim),
|
||||
Transpose((1, 2)),
|
||||
nn.Conv1d(dim, inner_dim * 2, 1),
|
||||
SwiGLU(dim=1),
|
||||
nn.Conv1d(inner_dim, inner_dim, kernel_size=kernel_size, padding=padding[0], groups=inner_dim),
|
||||
_activation,
|
||||
nn.Conv1d(inner_dim, dim, 1),
|
||||
Transpose((1, 2)),
|
||||
_dropout
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
return self.net(x)
|
||||
|
||||
|
||||
class LYNXNetResidualLayer(nn.Module):
|
||||
def __init__(self, dim_cond, dim, expansion_factor, kernel_size=31, activation='PReLU', dropout=0.0):
|
||||
super().__init__()
|
||||
self.diffusion_projection = nn.Conv1d(dim, dim, 1)
|
||||
self.conditioner_projection = nn.Conv1d(dim_cond, dim, 1)
|
||||
self.convmodule = LYNXConvModule(dim=dim, expansion_factor=expansion_factor, kernel_size=kernel_size,
|
||||
activation=activation, dropout=dropout)
|
||||
|
||||
def forward(self, x, conditioner, diffusion_step, front_cond_inject=False):
|
||||
if front_cond_inject:
|
||||
x = x + self.conditioner_projection(conditioner)
|
||||
res_x = x
|
||||
else:
|
||||
res_x = x
|
||||
x = x + self.conditioner_projection(conditioner)
|
||||
x = x + self.diffusion_projection(diffusion_step)
|
||||
x = x.transpose(1, 2)
|
||||
x = self.convmodule(x) # (#batch, dim, length)
|
||||
x = x.transpose(1, 2) + res_x
|
||||
return x # (#batch, length, dim)
|
||||
|
||||
|
||||
class LYNXNet(nn.Module):
|
||||
def __init__(self, in_dims, n_feats, *, num_layers=6, num_channels=512, expansion_factor=2, kernel_size=31,
|
||||
activation='PReLU', dropout_rate=0.0, strong_cond=False):
|
||||
"""
|
||||
LYNXNet(Linear Gated Depthwise Separable Convolution Network)
|
||||
TIPS:You can control the style of the generated results by modifying the 'activation',
|
||||
- 'PReLU'(default) : Similar to WaveNet
|
||||
- 'SiLU' : Voice will be more pronounced, not recommended for use under DDPM
|
||||
- 'ReLU' : Contrary to 'SiLU', Voice will be weakened
|
||||
"""
|
||||
super().__init__()
|
||||
self.in_dims = in_dims
|
||||
self.n_feats = n_feats
|
||||
self.input_projection = Conv1d(in_dims * n_feats, num_channels, 1)
|
||||
self.diffusion_embedding = nn.Sequential(
|
||||
SinusoidalPosEmb(num_channels),
|
||||
nn.Linear(num_channels, num_channels * 4),
|
||||
nn.GELU(),
|
||||
nn.Linear(num_channels * 4, num_channels),
|
||||
)
|
||||
self.residual_layers = nn.ModuleList(
|
||||
[
|
||||
LYNXNetResidualLayer(
|
||||
dim_cond=hparams['hidden_size'],
|
||||
dim=num_channels,
|
||||
expansion_factor=expansion_factor,
|
||||
kernel_size=kernel_size,
|
||||
activation=activation,
|
||||
dropout=dropout_rate
|
||||
)
|
||||
for _ in range(num_layers)
|
||||
]
|
||||
)
|
||||
self.norm = nn.LayerNorm(num_channels)
|
||||
self.output_projection = AdamWConv1d(num_channels, in_dims * n_feats, kernel_size=1)
|
||||
self.strong_cond = strong_cond
|
||||
nn.init.zeros_(self.output_projection.weight)
|
||||
|
||||
def forward(self, spec, diffusion_step, cond):
|
||||
"""
|
||||
:param spec: [B, F, M, T]
|
||||
:param diffusion_step: [B, 1]
|
||||
:param cond: [B, H, T]
|
||||
:return:
|
||||
"""
|
||||
|
||||
if self.n_feats == 1:
|
||||
x = spec[:, 0] # [B, M, T]
|
||||
else:
|
||||
x = spec.flatten(start_dim=1, end_dim=2) # [B, F x M, T]
|
||||
|
||||
x = self.input_projection(x) # x [B, residual_channel, T]
|
||||
if not self.strong_cond:
|
||||
x = F.gelu(x)
|
||||
|
||||
diffusion_step = self.diffusion_embedding(diffusion_step).unsqueeze(-1)
|
||||
|
||||
for layer in self.residual_layers:
|
||||
x = layer(x, cond, diffusion_step, front_cond_inject=self.strong_cond)
|
||||
|
||||
# post-norm
|
||||
x = self.norm(x.transpose(1, 2)).transpose(1, 2)
|
||||
|
||||
# output_projection
|
||||
x = self.output_projection(x) # [B, 128, T]
|
||||
|
||||
if self.n_feats == 1:
|
||||
x = x[:, None, :, :]
|
||||
else:
|
||||
# This is the temporary solution since PyTorch 1.13
|
||||
# does not support exporting aten::unflatten to ONNX
|
||||
# x = x.unflatten(dim=1, sizes=(self.n_feats, self.in_dims))
|
||||
x = x.reshape(-1, self.n_feats, self.in_dims, x.shape[2])
|
||||
return x
|
||||
@@ -0,0 +1,115 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from modules.commons.common_layers import SinusoidalPosEmb, SwiGLU, ATanGLU, Transpose, AdamWLinear
|
||||
from utils.hparams import hparams
|
||||
|
||||
|
||||
class LYNXNet2Block(nn.Module):
|
||||
def __init__(self, dim, expansion_factor, kernel_size=31, dropout=0., glu_type='swiglu'):
|
||||
super().__init__()
|
||||
inner_dim = int(dim * expansion_factor)
|
||||
if glu_type == 'swiglu':
|
||||
_glu = SwiGLU()
|
||||
elif glu_type == 'atanglu':
|
||||
_glu = ATanGLU()
|
||||
else:
|
||||
raise ValueError(f'{glu_type} is not a valid activation')
|
||||
if float(dropout) > 0.:
|
||||
_dropout = nn.Dropout(dropout)
|
||||
else:
|
||||
_dropout = nn.Identity()
|
||||
self.net = nn.Sequential(
|
||||
nn.LayerNorm(dim),
|
||||
Transpose((1, 2)),
|
||||
nn.Conv1d(dim, dim, kernel_size=kernel_size, padding=kernel_size // 2, groups=dim),
|
||||
Transpose((1, 2)),
|
||||
nn.Linear(dim, inner_dim * 2),
|
||||
_glu,
|
||||
nn.Linear(inner_dim, inner_dim * 2),
|
||||
_glu,
|
||||
nn.Linear(inner_dim, dim),
|
||||
_dropout
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
return x + self.net(x)
|
||||
|
||||
|
||||
class LYNXNet2(nn.Module):
|
||||
def __init__(self, in_dims, n_feats, *, num_layers=6, num_channels=512, expansion_factor=1, kernel_size=31,
|
||||
dropout_rate=0.0, use_conditioner_cache=False, glu_type='swiglu'):
|
||||
"""
|
||||
LYNXNet2(Linear Gated Depthwise Separable Convolution Network Version 2)
|
||||
"""
|
||||
super().__init__()
|
||||
self.in_dims = in_dims
|
||||
self.n_feats = n_feats
|
||||
self.input_projection = nn.Linear(in_dims * n_feats, num_channels)
|
||||
self.use_conditioner_cache = use_conditioner_cache
|
||||
if self.use_conditioner_cache:
|
||||
# Conv1d is used for condition cache compatibility
|
||||
self.conditioner_projection = nn.Conv1d(hparams['hidden_size'], num_channels, 1)
|
||||
else:
|
||||
self.conditioner_projection = nn.Linear(hparams['hidden_size'], num_channels)
|
||||
self.diffusion_embedding = nn.Sequential(
|
||||
SinusoidalPosEmb(num_channels),
|
||||
nn.Linear(num_channels, num_channels * 4),
|
||||
nn.GELU(),
|
||||
nn.Linear(num_channels * 4, num_channels),
|
||||
)
|
||||
self.residual_layers = nn.ModuleList(
|
||||
[
|
||||
LYNXNet2Block(
|
||||
dim=num_channels,
|
||||
expansion_factor=expansion_factor,
|
||||
kernel_size=kernel_size,
|
||||
dropout=dropout_rate,
|
||||
glu_type=glu_type
|
||||
)
|
||||
for _ in range(num_layers)
|
||||
]
|
||||
)
|
||||
self.norm = nn.LayerNorm(num_channels)
|
||||
self.output_projection = AdamWLinear(num_channels, in_dims * n_feats)
|
||||
nn.init.kaiming_normal_(self.input_projection.weight)
|
||||
nn.init.kaiming_normal_(self.conditioner_projection.weight)
|
||||
nn.init.zeros_(self.output_projection.weight)
|
||||
|
||||
def forward(self, spec, diffusion_step, cond):
|
||||
"""
|
||||
:param spec: [B, F, M, T]
|
||||
:param diffusion_step: [B, 1]
|
||||
:param cond: [B, H, T]
|
||||
:return:
|
||||
"""
|
||||
|
||||
if self.n_feats == 1:
|
||||
x = spec[:, 0] # [B, M, T]
|
||||
else:
|
||||
x = spec.flatten(start_dim=1, end_dim=2) # [B, F x M, T]
|
||||
|
||||
x = self.input_projection(x.transpose(1, 2)) # [B, T, F x M]
|
||||
if self.use_conditioner_cache:
|
||||
x = x + self.conditioner_projection(cond).transpose(1, 2)
|
||||
else:
|
||||
x = x + self.conditioner_projection(cond.transpose(1, 2))
|
||||
x = x + self.diffusion_embedding(diffusion_step).unsqueeze(1)
|
||||
|
||||
for layer in self.residual_layers:
|
||||
x = layer(x)
|
||||
|
||||
# post-norm
|
||||
x = self.norm(x)
|
||||
|
||||
# output projection
|
||||
x = self.output_projection(x).transpose(1, 2) # [B, 128, T]
|
||||
|
||||
if self.n_feats == 1:
|
||||
x = x[:, None, :, :]
|
||||
else:
|
||||
# Using reshape instead of unflatten for ONNX export compatibility
|
||||
# x = x.unflatten(dim=1, sizes=(self.n_feats, self.in_dims))
|
||||
x = x.reshape(-1, self.n_feats, self.in_dims, x.shape[2])
|
||||
return x
|
||||
@@ -0,0 +1,104 @@
|
||||
import math
|
||||
from math import sqrt
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from modules.commons.common_layers import SinusoidalPosEmb, AdamWConv1d
|
||||
from modules.commons.common_layers import KaimingNormalConv1d as Conv1d
|
||||
from utils.hparams import hparams
|
||||
|
||||
|
||||
class ResidualBlock(nn.Module):
|
||||
def __init__(self, encoder_hidden, residual_channels, dilation):
|
||||
super().__init__()
|
||||
self.residual_channels = residual_channels
|
||||
self.dilated_conv = nn.Conv1d(
|
||||
residual_channels,
|
||||
2 * residual_channels,
|
||||
kernel_size=3,
|
||||
padding=dilation,
|
||||
dilation=dilation
|
||||
)
|
||||
self.diffusion_projection = nn.Linear(residual_channels, residual_channels)
|
||||
self.conditioner_projection = nn.Conv1d(encoder_hidden, 2 * residual_channels, 1)
|
||||
self.output_projection = nn.Conv1d(residual_channels, 2 * residual_channels, 1)
|
||||
|
||||
def forward(self, x, conditioner, diffusion_step):
|
||||
diffusion_step = self.diffusion_projection(diffusion_step).unsqueeze(-1)
|
||||
conditioner = self.conditioner_projection(conditioner)
|
||||
y = x + diffusion_step
|
||||
|
||||
y = self.dilated_conv(y) + conditioner
|
||||
|
||||
# Using torch.split instead of torch.chunk to avoid using onnx::Slice
|
||||
gate, filter = torch.split(y, [self.residual_channels, self.residual_channels], dim=1)
|
||||
y = torch.sigmoid(gate) * torch.tanh(filter)
|
||||
|
||||
y = self.output_projection(y)
|
||||
|
||||
# Using torch.split instead of torch.chunk to avoid using onnx::Slice
|
||||
residual, skip = torch.split(y, [self.residual_channels, self.residual_channels], dim=1)
|
||||
return (x + residual) / math.sqrt(2.0), skip
|
||||
|
||||
|
||||
class WaveNet(nn.Module):
|
||||
def __init__(self, in_dims, n_feats, *, num_layers=20, num_channels=256, dilation_cycle_length=4):
|
||||
super().__init__()
|
||||
self.in_dims = in_dims
|
||||
self.n_feats = n_feats
|
||||
self.input_projection = Conv1d(in_dims * n_feats, num_channels, 1)
|
||||
self.diffusion_embedding = SinusoidalPosEmb(num_channels)
|
||||
self.mlp = nn.Sequential(
|
||||
nn.Linear(num_channels, num_channels * 4),
|
||||
nn.Mish(),
|
||||
nn.Linear(num_channels * 4, num_channels)
|
||||
)
|
||||
self.residual_layers = nn.ModuleList([
|
||||
ResidualBlock(
|
||||
encoder_hidden=hparams['hidden_size'],
|
||||
residual_channels=num_channels,
|
||||
dilation=2 ** (i % dilation_cycle_length)
|
||||
)
|
||||
for i in range(num_layers)
|
||||
])
|
||||
self.skip_projection = Conv1d(num_channels, num_channels, 1)
|
||||
self.output_projection = AdamWConv1d(num_channels, in_dims * n_feats, 1)
|
||||
nn.init.zeros_(self.output_projection.weight)
|
||||
|
||||
def forward(self, spec, diffusion_step, cond):
|
||||
"""
|
||||
:param spec: [B, F, M, T]
|
||||
:param diffusion_step: [B, 1]
|
||||
:param cond: [B, H, T]
|
||||
:return:
|
||||
"""
|
||||
if self.n_feats == 1:
|
||||
# Use indexing instead of squeeze to avoid emitting an onnx::If
|
||||
# whose branches have different rank, which breaks shape inference
|
||||
# for the downstream Conv on PyTorch >= 2.0.
|
||||
x = spec[:, 0] # [B, M, T]
|
||||
else:
|
||||
x = spec.flatten(start_dim=1, end_dim=2) # [B, F x M, T]
|
||||
x = self.input_projection(x) # [B, C, T]
|
||||
|
||||
x = F.relu(x)
|
||||
diffusion_step = self.diffusion_embedding(diffusion_step)
|
||||
diffusion_step = self.mlp(diffusion_step)
|
||||
skip = []
|
||||
for layer in self.residual_layers:
|
||||
x, skip_connection = layer(x, cond, diffusion_step)
|
||||
skip.append(skip_connection)
|
||||
|
||||
x = torch.sum(torch.stack(skip), dim=0) / sqrt(len(self.residual_layers))
|
||||
x = self.skip_projection(x)
|
||||
x = F.relu(x)
|
||||
x = self.output_projection(x) # [B, M, T]
|
||||
if self.n_feats == 1:
|
||||
x = x[:, None, :, :]
|
||||
else:
|
||||
# Using reshape instead of unflatten for ONNX export compatibility
|
||||
# x = x.unflatten(dim=1, sizes=(self.n_feats, self.in_dims))
|
||||
x = x.reshape(-1, self.n_feats, self.in_dims, x.shape[2])
|
||||
return x
|
||||
Reference in New Issue
Block a user