chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,220 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List, Tuple
|
||||
|
||||
import torch
|
||||
from torch import Tensor
|
||||
|
||||
from modules.core import (
|
||||
GaussianDiffusion, PitchDiffusion, MultiVarianceDiffusion
|
||||
)
|
||||
|
||||
|
||||
def extract(a, t):
|
||||
return a[t].reshape((1, 1, 1, 1))
|
||||
|
||||
|
||||
# noinspection PyMethodOverriding
|
||||
class GaussianDiffusionONNX(GaussianDiffusion):
|
||||
@property
|
||||
def backbone(self):
|
||||
return self.denoise_fn
|
||||
|
||||
# We give up the setter for the property `backbone` because this will cause TorchScript to fail
|
||||
# @backbone.setter
|
||||
@torch.jit.unused
|
||||
def set_backbone(self, value):
|
||||
self.denoise_fn = value
|
||||
|
||||
def q_sample(self, x_start, t, noise):
|
||||
return (
|
||||
extract(self.sqrt_alphas_cumprod, t) * x_start +
|
||||
extract(self.sqrt_one_minus_alphas_cumprod, t) * noise
|
||||
)
|
||||
|
||||
def p_sample(self, x, t, cond):
|
||||
x_pred = self.denoise_fn(x, t, cond)
|
||||
x_recon = (
|
||||
extract(self.sqrt_recip_alphas_cumprod, t) * x -
|
||||
extract(self.sqrt_recipm1_alphas_cumprod, t) * x_pred
|
||||
)
|
||||
# This is previously inherited from original DiffSinger repository
|
||||
# and disabled due to some loudness issues when speedup = 1.
|
||||
# x_recon = torch.clamp(x_recon, min=-1., max=1.)
|
||||
|
||||
model_mean = (
|
||||
extract(self.posterior_mean_coef1, t) * x_recon +
|
||||
extract(self.posterior_mean_coef2, t) * x
|
||||
)
|
||||
model_log_variance = extract(self.posterior_log_variance_clipped, t)
|
||||
noise = torch.randn_like(x)
|
||||
# no noise when t == 0
|
||||
nonzero_mask = ((t > 0).float()).reshape(1, 1, 1, 1)
|
||||
return model_mean + nonzero_mask * (0.5 * model_log_variance).exp() * noise
|
||||
|
||||
def p_sample_ddim(self, x, t, interval: int, cond):
|
||||
a_t = extract(self.alphas_cumprod, t)
|
||||
t_prev = t - interval
|
||||
a_prev = extract(self.alphas_cumprod, t_prev * (t_prev > 0))
|
||||
|
||||
noise_pred = self.denoise_fn(x, t, cond=cond)
|
||||
x_prev = a_prev.sqrt() * (
|
||||
x / a_t.sqrt() + (((1 - a_prev) / a_prev).sqrt() - ((1 - a_t) / a_t).sqrt()) * noise_pred
|
||||
)
|
||||
return x_prev
|
||||
|
||||
def plms_get_x_pred(self, x, noise_t, t, t_prev):
|
||||
a_t = extract(self.alphas_cumprod, t)
|
||||
a_prev = extract(self.alphas_cumprod, t_prev)
|
||||
a_t_sq, a_prev_sq = a_t.sqrt(), a_prev.sqrt()
|
||||
|
||||
x_delta = (a_prev - a_t) * ((1 / (a_t_sq * (a_t_sq + a_prev_sq))) * x - 1 / (
|
||||
a_t_sq * (((1 - a_prev) * a_t).sqrt() + ((1 - a_t) * a_prev).sqrt())) * noise_t)
|
||||
x_pred = x + x_delta
|
||||
|
||||
return x_pred
|
||||
|
||||
def p_sample_plms(self, x_prev, t, interval: int, cond, noise_list: List[Tensor], stage: int):
|
||||
noise_pred = self.denoise_fn(x_prev, t, cond)
|
||||
t_prev = t - interval
|
||||
t_prev = t_prev * (t_prev > 0)
|
||||
if stage == 0:
|
||||
x_pred = self.plms_get_x_pred(x_prev, noise_pred, t, t_prev)
|
||||
noise_pred_prev = self.denoise_fn(x_pred, t_prev, cond)
|
||||
noise_pred_prime = (noise_pred + noise_pred_prev) / 2.
|
||||
elif stage == 1:
|
||||
noise_pred_prime = (3. * noise_pred - noise_list[-1]) / 2.
|
||||
elif stage == 2:
|
||||
noise_pred_prime = (23. * noise_pred - 16. * noise_list[-1] + 5. * noise_list[-2]) / 12.
|
||||
else:
|
||||
noise_pred_prime = (55. * noise_pred - 59. * noise_list[-1] + 37.
|
||||
* noise_list[-2] - 9. * noise_list[-3]) / 24.
|
||||
x_prev = self.plms_get_x_pred(x_prev, noise_pred_prime, t, t_prev)
|
||||
return noise_pred, x_prev
|
||||
|
||||
def norm_spec(self, x):
|
||||
k = (self.spec_max - self.spec_min) / 2.
|
||||
b = (self.spec_max + self.spec_min) / 2.
|
||||
return (x - b) / k
|
||||
|
||||
def denorm_spec(self, x):
|
||||
k = (self.spec_max - self.spec_min) / 2.
|
||||
b = (self.spec_max + self.spec_min) / 2.
|
||||
return x * k + b
|
||||
|
||||
def forward(self, condition, x_start=None, depth=None, steps: int = 10):
|
||||
condition = condition.transpose(1, 2) # [1, T, H] => [1, H, T]
|
||||
device = condition.device
|
||||
n_frames = condition.shape[2]
|
||||
|
||||
noise = torch.randn((1, self.num_feats, self.out_dims, n_frames), device=device)
|
||||
if x_start is None:
|
||||
speedup = max(1, self.timesteps // steps)
|
||||
speedup = self.timestep_factors[torch.sum(self.timestep_factors <= speedup) - 1]
|
||||
step_range = torch.arange(0, self.k_step, speedup, dtype=torch.long, device=device).flip(0)[:, None]
|
||||
x = noise
|
||||
else:
|
||||
depth_int64 = min(torch.round(depth * self.timesteps).long(), self.k_step)
|
||||
speedup = max(1, depth_int64 // steps)
|
||||
depth_int64 = depth_int64 // speedup * speedup # make depth_int64 a multiple of speedup
|
||||
step_range = torch.arange(0, depth_int64, speedup, dtype=torch.long, device=device).flip(0)[:, None]
|
||||
x_start = self.norm_spec(x_start).transpose(-2, -1)
|
||||
if self.num_feats == 1:
|
||||
x_start = x_start[:, None, :, :]
|
||||
if depth_int64 >= self.timesteps:
|
||||
x = noise
|
||||
elif depth_int64 > 0:
|
||||
x = self.q_sample(
|
||||
x_start, torch.full((1,), depth_int64 - 1, device=device, dtype=torch.long), noise
|
||||
)
|
||||
else:
|
||||
x = x_start
|
||||
|
||||
if speedup > 1:
|
||||
for t in step_range:
|
||||
x = self.p_sample_ddim(x, t, interval=speedup, cond=condition)
|
||||
# plms_noise_stage: int = 0
|
||||
# noise_list: List[Tensor] = []
|
||||
# for t in step_range:
|
||||
# noise_pred, x = self.p_sample_plms(
|
||||
# x, t, interval=speedup, cond=condition,
|
||||
# noise_list=noise_list, stage=plms_noise_stage
|
||||
# )
|
||||
# if plms_noise_stage == 0:
|
||||
# noise_list = [noise_pred]
|
||||
# plms_noise_stage = plms_noise_stage + 1
|
||||
# else:
|
||||
# if plms_noise_stage >= 3:
|
||||
# noise_list.pop(0)
|
||||
# else:
|
||||
# plms_noise_stage = plms_noise_stage + 1
|
||||
# noise_list.append(noise_pred)
|
||||
else:
|
||||
for t in step_range:
|
||||
x = self.p_sample(x, t, cond=condition)
|
||||
|
||||
if self.num_feats == 1:
|
||||
x = x.squeeze(1).permute(0, 2, 1) # [B, 1, M, T] => [B, T, M]
|
||||
else:
|
||||
x = x.permute(0, 1, 3, 2) # [B, F, M, T] => [B, F, T, M]
|
||||
x = self.denorm_spec(x)
|
||||
return x
|
||||
|
||||
|
||||
class PitchDiffusionONNX(GaussianDiffusionONNX, PitchDiffusion):
|
||||
def __init__(self, vmin: float, vmax: float,
|
||||
cmin: float, cmax: float, repeat_bins,
|
||||
timesteps=1000, k_step=1000,
|
||||
backbone_type=None, backbone_args=None,
|
||||
betas=None):
|
||||
self.vmin = vmin
|
||||
self.vmax = vmax
|
||||
self.cmin = cmin
|
||||
self.cmax = cmax
|
||||
super(PitchDiffusion, self).__init__(
|
||||
vmin=vmin, vmax=vmax, repeat_bins=repeat_bins,
|
||||
timesteps=timesteps, k_step=k_step,
|
||||
backbone_type=backbone_type, backbone_args=backbone_args,
|
||||
betas=betas
|
||||
)
|
||||
|
||||
def clamp_spec(self, x):
|
||||
return x.clamp(min=self.cmin, max=self.cmax)
|
||||
|
||||
def denorm_spec(self, x):
|
||||
d = (self.spec_max - self.spec_min) / 2.
|
||||
m = (self.spec_max + self.spec_min) / 2.
|
||||
x = x * d + m
|
||||
x = x.mean(dim=-1)
|
||||
return x
|
||||
|
||||
|
||||
class MultiVarianceDiffusionONNX(GaussianDiffusionONNX, MultiVarianceDiffusion):
|
||||
def __init__(
|
||||
self, ranges: List[Tuple[float, float]],
|
||||
clamps: List[Tuple[float | None, float | None] | None],
|
||||
repeat_bins, timesteps=1000, k_step=1000,
|
||||
backbone_type=None, backbone_args=None,
|
||||
betas=None
|
||||
):
|
||||
assert len(ranges) == len(clamps)
|
||||
self.clamps = clamps
|
||||
vmin = [r[0] for r in ranges]
|
||||
vmax = [r[1] for r in ranges]
|
||||
if len(vmin) == 1:
|
||||
vmin = vmin[0]
|
||||
if len(vmax) == 1:
|
||||
vmax = vmax[0]
|
||||
super(MultiVarianceDiffusion, self).__init__(
|
||||
vmin=vmin, vmax=vmax, repeat_bins=repeat_bins,
|
||||
timesteps=timesteps, k_step=k_step,
|
||||
backbone_type=backbone_type, backbone_args=backbone_args,
|
||||
betas=betas
|
||||
)
|
||||
|
||||
def denorm_spec(self, x):
|
||||
d = (self.spec_max - self.spec_min) / 2.
|
||||
m = (self.spec_max + self.spec_min) / 2.
|
||||
x = x * d + m
|
||||
x = x.mean(dim=-1)
|
||||
return x
|
||||
@@ -0,0 +1,235 @@
|
||||
import copy
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from modules.commons.common_layers import NormalInitEmbedding as Embedding
|
||||
from modules.fastspeech.acoustic_encoder import FastSpeech2Acoustic
|
||||
from modules.fastspeech.variance_encoder import FastSpeech2Variance
|
||||
from utils.hparams import hparams
|
||||
from utils.phoneme_utils import PAD_INDEX
|
||||
|
||||
f0_bin = 256
|
||||
f0_max = 1100.0
|
||||
f0_min = 50.0
|
||||
f0_mel_min = 1127 * np.log(1 + f0_min / 700)
|
||||
f0_mel_max = 1127 * np.log(1 + f0_max / 700)
|
||||
|
||||
|
||||
def uniform_attention_pooling(spk_embed, durations):
|
||||
_, T_mel, _ = spk_embed.shape
|
||||
ph_starts = torch.cumsum(torch.cat([torch.zeros_like(durations[:, :1]), durations[:, :-1]], dim=1), dim=1)
|
||||
ph_ends = ph_starts + durations
|
||||
mel_indices = torch.arange(T_mel, device=spk_embed.device).view(1, 1, T_mel)
|
||||
phoneme_to_mel_mask = (mel_indices >= ph_starts.unsqueeze(-1)) & (mel_indices < ph_ends.unsqueeze(-1))
|
||||
uniform_scores = phoneme_to_mel_mask.float()
|
||||
sum_scores = uniform_scores.sum(dim=2, keepdim=True)
|
||||
attn_weights = uniform_scores / (sum_scores + (sum_scores == 0).float()) # [B, T_ph, T_mel]
|
||||
ph_spk_embed = torch.bmm(attn_weights, spk_embed)
|
||||
|
||||
return ph_spk_embed
|
||||
|
||||
|
||||
def f0_to_coarse(f0):
|
||||
f0_mel = 1127 * (1 + f0 / 700).log()
|
||||
a = (f0_bin - 2) / (f0_mel_max - f0_mel_min)
|
||||
b = f0_mel_min * a - 1.
|
||||
f0_mel = torch.where(f0_mel > 0, f0_mel * a - b, f0_mel)
|
||||
torch.clip_(f0_mel, min=1., max=float(f0_bin - 1))
|
||||
f0_coarse = torch.round(f0_mel).long()
|
||||
return f0_coarse
|
||||
|
||||
|
||||
class LengthRegulator(nn.Module):
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def forward(self, dur):
|
||||
token_idx = torch.arange(1, dur.shape[1] + 1, device=dur.device)[None, :, None]
|
||||
dur_cumsum = torch.cumsum(dur, dim=1)
|
||||
dur_cumsum_prev = F.pad(dur_cumsum, (1, -1), mode='constant', value=0)
|
||||
pos_idx = torch.arange(dur.sum(dim=1).max(), device=dur.device)[None, None]
|
||||
token_mask = (pos_idx >= dur_cumsum_prev[:, :, None]) & (pos_idx < dur_cumsum[:, :, None])
|
||||
mel2ph = (token_idx * token_mask).sum(dim=1)
|
||||
return mel2ph
|
||||
|
||||
|
||||
class FastSpeech2AcousticONNX(FastSpeech2Acoustic):
|
||||
def __init__(self, vocab_size, cross_lingual_token_idx=None):
|
||||
super().__init__(vocab_size=vocab_size)
|
||||
self.register_buffer(
|
||||
'cross_lingual_token_idx',
|
||||
torch.LongTensor(cross_lingual_token_idx),
|
||||
persistent=False
|
||||
) # [N,]
|
||||
if len(cross_lingual_token_idx) == 0:
|
||||
self.use_lang_id = False
|
||||
|
||||
# for temporary compatibility; will be completely removed in the future
|
||||
self.f0_embed_type = hparams.get('f0_embed_type', 'continuous')
|
||||
if self.f0_embed_type == 'discrete':
|
||||
self.pitch_embed = Embedding(300, hparams['hidden_size'], PAD_INDEX)
|
||||
|
||||
self.lr = LengthRegulator()
|
||||
if hparams['use_key_shift_embed']:
|
||||
self.shift_min, self.shift_max = hparams['augmentation_args']['random_pitch_shifting']['range']
|
||||
if hparams['use_speed_embed']:
|
||||
self.speed_min, self.speed_max = hparams['augmentation_args']['random_time_stretching']['range']
|
||||
|
||||
# noinspection PyMethodOverriding
|
||||
def forward(
|
||||
self, tokens, durations,
|
||||
f0, variances: dict,
|
||||
gender=None, velocity=None,
|
||||
spk_embed=None,
|
||||
languages=None
|
||||
):
|
||||
txt_embed = self.txt_embed(tokens)
|
||||
durations = durations * (tokens > 0)
|
||||
mel2ph = self.lr(durations)
|
||||
_mel2ph = mel2ph
|
||||
f0 = f0 * (mel2ph > 0)
|
||||
mel2ph = mel2ph[..., None].repeat((1, 1, hparams['hidden_size']))
|
||||
if self.use_variance_scaling:
|
||||
dur_embed = self.dur_embed(torch.log(1 + durations.float())[:, :, None])
|
||||
else:
|
||||
dur_embed = self.dur_embed(durations.float()[:, :, None])
|
||||
if self.use_lang_id:
|
||||
lang_mask = torch.any(
|
||||
tokens[..., None] == self.cross_lingual_token_idx[None, None],
|
||||
dim=-1
|
||||
)
|
||||
lang_embed = self.lang_embed(languages * lang_mask)
|
||||
extra_embed = dur_embed + lang_embed
|
||||
else:
|
||||
extra_embed = dur_embed
|
||||
if hparams.get('use_mix_ln', False):
|
||||
if hasattr(self, 'frozen_spk_embed'):
|
||||
ph_spk_embed = self.frozen_spk_embed.repeat(1, tokens.shape[1], 1)
|
||||
else:
|
||||
ph_spk_embed = uniform_attention_pooling(spk_embed, durations)
|
||||
else:
|
||||
ph_spk_embed = None
|
||||
encoded = self.encoder(txt_embed, extra_embed, tokens == PAD_INDEX, spk_embed=ph_spk_embed)
|
||||
encoded = F.pad(encoded, (0, 0, 1, 0))
|
||||
condition = torch.gather(encoded, 1, mel2ph)
|
||||
|
||||
if self.use_stretch_embed:
|
||||
stretch = torch.round(1000 * self.sr(_mel2ph, durations))
|
||||
table = self.stretch_embed(torch.arange(0, 1001, device=stretch.device))
|
||||
stretch_embed = torch.index_select(table, 0, stretch.view(-1).long()).view_as(condition)
|
||||
condition += stretch_embed
|
||||
stretch_embed_rnn_out, _ = self.stretch_embed_rnn(condition)
|
||||
condition += stretch_embed_rnn_out
|
||||
|
||||
if self.f0_embed_type == 'discrete':
|
||||
pitch = f0_to_coarse(f0)
|
||||
pitch_embed = self.pitch_embed(pitch)
|
||||
else:
|
||||
f0_mel = (1 + f0 / 700).log()
|
||||
pitch_embed = self.pitch_embed(f0_mel[:, :, None])
|
||||
condition += pitch_embed
|
||||
|
||||
if self.use_variance_embeds:
|
||||
variance_embeds = torch.stack([
|
||||
self.variance_embeds[v_name](variances[v_name][:, :, None] * self.variance_scaling_factor[v_name])
|
||||
for v_name in self.variance_embed_list
|
||||
], dim=-1).sum(-1)
|
||||
condition += variance_embeds
|
||||
|
||||
if hparams['use_key_shift_embed']:
|
||||
if hasattr(self, 'frozen_key_shift'):
|
||||
key_shift_embed = self.key_shift_embed(self.frozen_key_shift[:, None, None] * self.variance_scaling_factor['key_shift'])
|
||||
else:
|
||||
gender = torch.clip(gender, min=-1., max=1.)
|
||||
gender_mask = (gender < 0.).float()
|
||||
key_shift = gender * ((1. - gender_mask) * self.shift_max + gender_mask * abs(self.shift_min))
|
||||
key_shift_embed = self.key_shift_embed(key_shift[:, :, None] * self.variance_scaling_factor['key_shift'])
|
||||
condition += key_shift_embed
|
||||
|
||||
if hparams['use_speed_embed']:
|
||||
if velocity is not None:
|
||||
velocity = torch.clip(velocity, min=self.speed_min, max=self.speed_max)
|
||||
speed_embed = self.speed_embed(velocity[:, :, None] * self.variance_scaling_factor['speed'])
|
||||
else:
|
||||
speed_embed = self.speed_embed(torch.FloatTensor([1.]).to(condition.device)[:, None, None] * self.variance_scaling_factor['speed'])
|
||||
condition += speed_embed
|
||||
|
||||
if hparams['use_spk_id']:
|
||||
if hasattr(self, 'frozen_spk_embed'):
|
||||
condition += self.frozen_spk_embed
|
||||
else:
|
||||
condition += spk_embed
|
||||
return condition
|
||||
|
||||
|
||||
class FastSpeech2VarianceONNX(FastSpeech2Variance):
|
||||
def __init__(self, vocab_size, cross_lingual_token_idx=None):
|
||||
super().__init__(vocab_size=vocab_size)
|
||||
self.register_buffer(
|
||||
'cross_lingual_token_idx',
|
||||
torch.LongTensor(cross_lingual_token_idx),
|
||||
persistent=False
|
||||
)
|
||||
if len(cross_lingual_token_idx) == 0:
|
||||
self.use_lang_id = False
|
||||
self.lr = LengthRegulator()
|
||||
|
||||
def forward_encoder_word(self, tokens, word_div, word_dur, languages=None):
|
||||
txt_embed = self.txt_embed(tokens)
|
||||
ph2word = self.lr(word_div)
|
||||
onset = ph2word > F.pad(ph2word, [1, -1])
|
||||
onset_embed = self.onset_embed(onset.long())
|
||||
ph_word_dur = torch.gather(F.pad(word_dur, [1, 0]), 1, ph2word)
|
||||
word_dur_embed = self.word_dur_embed(ph_word_dur.float()[:, :, None])
|
||||
extra_embed = onset_embed + word_dur_embed
|
||||
if self.use_lang_id:
|
||||
lang_mask = torch.any(
|
||||
tokens[..., None] == self.cross_lingual_token_idx[None, None],
|
||||
dim=-1
|
||||
)
|
||||
lang_embed = self.lang_embed(languages * lang_mask)
|
||||
extra_embed += lang_embed
|
||||
x_masks = tokens == PAD_INDEX
|
||||
return self.encoder(txt_embed, extra_embed, x_masks), x_masks
|
||||
|
||||
def forward_encoder_phoneme(self, tokens, ph_dur, languages=None):
|
||||
txt_embed = self.txt_embed(tokens)
|
||||
if self.use_variance_scaling:
|
||||
ph_dur_embed = self.ph_dur_embed(torch.log(1 + ph_dur.float())[:, :, None])
|
||||
else:
|
||||
ph_dur_embed = self.ph_dur_embed(ph_dur.float()[:, :, None])
|
||||
if self.use_lang_id:
|
||||
lang_mask = torch.any(
|
||||
tokens[..., None] == self.cross_lingual_token_idx[None, None],
|
||||
dim=-1
|
||||
)
|
||||
lang_embed = self.lang_embed(languages * lang_mask)
|
||||
extra_embed = ph_dur_embed + lang_embed
|
||||
else:
|
||||
extra_embed = ph_dur_embed
|
||||
x_masks = tokens == PAD_INDEX
|
||||
return self.encoder(txt_embed, extra_embed, x_masks), x_masks
|
||||
|
||||
def forward_dur_predictor(self, encoder_out, x_masks, ph_midi, spk_embed=None):
|
||||
midi_embed = self.midi_embed(ph_midi)
|
||||
dur_cond = encoder_out + midi_embed
|
||||
if hparams['use_spk_id'] and spk_embed is not None:
|
||||
dur_cond += spk_embed
|
||||
ph_dur = self.dur_predictor(dur_cond, x_masks=x_masks)
|
||||
return ph_dur
|
||||
|
||||
def view_as_encoder(self):
|
||||
model = copy.deepcopy(self)
|
||||
if self.predict_dur:
|
||||
del model.dur_predictor
|
||||
model.forward = model.forward_encoder_word
|
||||
else:
|
||||
model.forward = model.forward_encoder_phoneme
|
||||
return model
|
||||
|
||||
def view_as_dur_predictor(self):
|
||||
model = copy.deepcopy(self)
|
||||
del model.encoder
|
||||
model.forward = model.forward_dur_predictor
|
||||
return model
|
||||
@@ -0,0 +1,16 @@
|
||||
import torch
|
||||
|
||||
from modules.nsf_hifigan.env import AttrDict
|
||||
from modules.nsf_hifigan.models import Generator
|
||||
|
||||
|
||||
# noinspection SpellCheckingInspection
|
||||
class NSFHiFiGANONNX(torch.nn.Module):
|
||||
def __init__(self, attrs: dict):
|
||||
super().__init__()
|
||||
self.generator = Generator(AttrDict(attrs))
|
||||
|
||||
def forward(self, mel: torch.Tensor, f0: torch.Tensor):
|
||||
mel = mel.transpose(1, 2)
|
||||
wav = self.generator(mel, f0)
|
||||
return wav.squeeze(1)
|
||||
@@ -0,0 +1,123 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from modules.core import (
|
||||
RectifiedFlow, PitchRectifiedFlow, MultiVarianceRectifiedFlow
|
||||
)
|
||||
|
||||
|
||||
class RectifiedFlowONNX(RectifiedFlow):
|
||||
@property
|
||||
def backbone(self):
|
||||
return self.velocity_fn
|
||||
|
||||
# We give up the setter for the property `backbone` because this will cause TorchScript to fail
|
||||
# @backbone.setter
|
||||
@torch.jit.unused
|
||||
def set_backbone(self, value):
|
||||
self.velocity_fn = value
|
||||
|
||||
def sample_euler(self, x, t, dt: float, cond):
|
||||
x += self.velocity_fn(x, t * self.time_scale_factor, cond) * dt
|
||||
return x
|
||||
|
||||
def norm_spec(self, x):
|
||||
k = (self.spec_max - self.spec_min) / 2.
|
||||
b = (self.spec_max + self.spec_min) / 2.
|
||||
return (x - b) / k
|
||||
|
||||
def denorm_spec(self, x):
|
||||
k = (self.spec_max - self.spec_min) / 2.
|
||||
b = (self.spec_max + self.spec_min) / 2.
|
||||
return x * k + b
|
||||
|
||||
def forward(self, condition, x_end=None, depth=None, steps: int = 10):
|
||||
condition = condition.transpose(1, 2) # [1, T, H] => [1, H, T]
|
||||
device = condition.device
|
||||
n_frames = condition.shape[2]
|
||||
noise = torch.randn((1, self.num_feats, self.out_dims, n_frames), device=device)
|
||||
if x_end is None:
|
||||
t_start = 0.
|
||||
x = noise
|
||||
else:
|
||||
t_start = torch.max(1 - depth, torch.tensor(self.t_start, dtype=torch.float32, device=device))
|
||||
x_end = self.norm_spec(x_end).transpose(-2, -1)
|
||||
if self.num_feats == 1:
|
||||
x_end = x_end[:, None, :, :]
|
||||
if t_start <= 0.:
|
||||
x = noise
|
||||
elif t_start >= 1.:
|
||||
x = x_end
|
||||
else:
|
||||
x = t_start * x_end + (1 - t_start) * noise
|
||||
|
||||
t_width = 1. - t_start
|
||||
if t_width >= 0.:
|
||||
dt = t_width / max(1, steps)
|
||||
for t in torch.arange(steps, dtype=torch.long, device=device)[:, None].float() * dt + t_start:
|
||||
x = self.sample_euler(x, t, dt, condition)
|
||||
|
||||
if self.num_feats == 1:
|
||||
x = x.squeeze(1).permute(0, 2, 1) # [B, 1, M, T] => [B, T, M]
|
||||
else:
|
||||
x = x.permute(0, 1, 3, 2) # [B, F, M, T] => [B, F, T, M]
|
||||
x = self.denorm_spec(x)
|
||||
return x
|
||||
|
||||
|
||||
class PitchRectifiedFlowONNX(RectifiedFlowONNX, PitchRectifiedFlow):
|
||||
def __init__(self, vmin: float, vmax: float,
|
||||
cmin: float, cmax: float, repeat_bins,
|
||||
time_scale_factor=1000,
|
||||
backbone_type=None, backbone_args=None):
|
||||
self.vmin = vmin
|
||||
self.vmax = vmax
|
||||
self.cmin = cmin
|
||||
self.cmax = cmax
|
||||
super(PitchRectifiedFlow, self).__init__(
|
||||
vmin=vmin, vmax=vmax, repeat_bins=repeat_bins,
|
||||
time_scale_factor=time_scale_factor,
|
||||
backbone_type=backbone_type, backbone_args=backbone_args
|
||||
)
|
||||
|
||||
def clamp_spec(self, x):
|
||||
return x.clamp(min=self.cmin, max=self.cmax)
|
||||
|
||||
def denorm_spec(self, x):
|
||||
d = (self.spec_max - self.spec_min) / 2.
|
||||
m = (self.spec_max + self.spec_min) / 2.
|
||||
x = x * d + m
|
||||
x = x.mean(dim=-1)
|
||||
return x
|
||||
|
||||
|
||||
class MultiVarianceRectifiedFlowONNX(RectifiedFlowONNX, MultiVarianceRectifiedFlow):
|
||||
def __init__(
|
||||
self, ranges: List[Tuple[float, float]],
|
||||
clamps: List[Tuple[float | None, float | None] | None],
|
||||
repeat_bins, time_scale_factor=1000,
|
||||
backbone_type=None, backbone_args=None
|
||||
):
|
||||
assert len(ranges) == len(clamps)
|
||||
self.clamps = clamps
|
||||
vmin = [r[0] for r in ranges]
|
||||
vmax = [r[1] for r in ranges]
|
||||
if len(vmin) == 1:
|
||||
vmin = vmin[0]
|
||||
if len(vmax) == 1:
|
||||
vmax = vmax[0]
|
||||
super(MultiVarianceRectifiedFlow, self).__init__(
|
||||
vmin=vmin, vmax=vmax, repeat_bins=repeat_bins,
|
||||
time_scale_factor=time_scale_factor,
|
||||
backbone_type=backbone_type, backbone_args=backbone_args
|
||||
)
|
||||
|
||||
def denorm_spec(self, x):
|
||||
d = (self.spec_max - self.spec_min) / 2.
|
||||
m = (self.spec_max + self.spec_min) / 2.
|
||||
x = x * d + m
|
||||
x = x.mean(dim=-1)
|
||||
return x
|
||||
@@ -0,0 +1,413 @@
|
||||
import copy
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from torch import Tensor
|
||||
|
||||
from deployment.modules.diffusion import (
|
||||
GaussianDiffusionONNX, PitchDiffusionONNX, MultiVarianceDiffusionONNX
|
||||
)
|
||||
from deployment.modules.rectified_flow import (
|
||||
RectifiedFlowONNX, PitchRectifiedFlowONNX, MultiVarianceRectifiedFlowONNX
|
||||
)
|
||||
from deployment.modules.fastspeech2 import FastSpeech2AcousticONNX, FastSpeech2VarianceONNX
|
||||
from modules.toplevel import DiffSingerAcoustic, DiffSingerVariance
|
||||
from utils.hparams import hparams
|
||||
|
||||
|
||||
class DiffSingerAcousticONNX(DiffSingerAcoustic):
|
||||
def __init__(self, vocab_size, out_dims, cross_lingual_token_idx=None):
|
||||
super().__init__(vocab_size, out_dims)
|
||||
del self.fs2
|
||||
del self.diffusion
|
||||
self.fs2 = FastSpeech2AcousticONNX(
|
||||
vocab_size=vocab_size,
|
||||
cross_lingual_token_idx=cross_lingual_token_idx
|
||||
)
|
||||
if self.diffusion_type == 'ddpm':
|
||||
self.diffusion = GaussianDiffusionONNX(
|
||||
out_dims=out_dims,
|
||||
num_feats=1,
|
||||
timesteps=hparams['timesteps'],
|
||||
k_step=hparams['K_step'],
|
||||
backbone_type=self.backbone_type,
|
||||
backbone_args=self.backbone_args,
|
||||
spec_min=hparams['spec_min'],
|
||||
spec_max=hparams['spec_max']
|
||||
)
|
||||
elif self.diffusion_type == 'reflow':
|
||||
self.diffusion = RectifiedFlowONNX(
|
||||
out_dims=out_dims,
|
||||
num_feats=1,
|
||||
t_start=hparams['T_start'],
|
||||
time_scale_factor=hparams['time_scale_factor'],
|
||||
backbone_type=self.backbone_type,
|
||||
backbone_args=self.backbone_args,
|
||||
spec_min=hparams['spec_min'],
|
||||
spec_max=hparams['spec_max']
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Invalid diffusion type: {self.diffusion_type}")
|
||||
self.mel_base = hparams.get('mel_base', '10')
|
||||
|
||||
def ensure_mel_base(self, mel):
|
||||
if self.mel_base != 'e':
|
||||
# log10 mel to log mel
|
||||
mel = mel * 2.30259
|
||||
return mel
|
||||
|
||||
def forward_fs2_aux(
|
||||
self,
|
||||
tokens: Tensor,
|
||||
durations: Tensor,
|
||||
f0: Tensor,
|
||||
variances: dict,
|
||||
gender: Tensor = None,
|
||||
velocity: Tensor = None,
|
||||
spk_embed: Tensor = None,
|
||||
languages: Tensor = None
|
||||
):
|
||||
condition = self.fs2(
|
||||
tokens, durations, f0, variances=variances,
|
||||
gender=gender, velocity=velocity, spk_embed=spk_embed,
|
||||
languages=languages
|
||||
)
|
||||
if self.use_shallow_diffusion:
|
||||
aux_mel_pred = self.aux_decoder(condition, infer=True)
|
||||
return condition, aux_mel_pred
|
||||
else:
|
||||
return condition
|
||||
|
||||
def forward_shallow_diffusion(
|
||||
self, condition: Tensor, x_start: Tensor,
|
||||
depth, steps: int
|
||||
) -> Tensor:
|
||||
mel_pred = self.diffusion(condition, x_start=x_start, depth=depth, steps=steps)
|
||||
return self.ensure_mel_base(mel_pred)
|
||||
|
||||
def forward_diffusion(self, condition: Tensor, steps: int):
|
||||
mel_pred = self.diffusion(condition, steps=steps)
|
||||
return self.ensure_mel_base(mel_pred)
|
||||
|
||||
def forward_shallow_reflow(
|
||||
self, condition: Tensor, x_end: Tensor,
|
||||
depth, steps: int
|
||||
):
|
||||
mel_pred = self.diffusion(condition, x_end=x_end, depth=depth, steps=steps)
|
||||
return self.ensure_mel_base(mel_pred)
|
||||
|
||||
def forward_reflow(self, condition: Tensor, steps: int):
|
||||
mel_pred = self.diffusion(condition, steps=steps)
|
||||
return self.ensure_mel_base(mel_pred)
|
||||
|
||||
def view_as_fs2_aux(self) -> nn.Module:
|
||||
model = copy.deepcopy(self)
|
||||
del model.diffusion
|
||||
model.forward = model.forward_fs2_aux
|
||||
return model
|
||||
|
||||
def view_as_diffusion(self) -> nn.Module:
|
||||
model = copy.deepcopy(self)
|
||||
del model.fs2
|
||||
if self.use_shallow_diffusion:
|
||||
del model.aux_decoder
|
||||
model.forward = model.forward_shallow_diffusion
|
||||
else:
|
||||
model.forward = model.forward_diffusion
|
||||
return model
|
||||
|
||||
def view_as_reflow(self) -> nn.Module:
|
||||
model = copy.deepcopy(self)
|
||||
del model.fs2
|
||||
if self.use_shallow_diffusion:
|
||||
del model.aux_decoder
|
||||
model.forward = model.forward_shallow_reflow
|
||||
else:
|
||||
model.forward = model.forward_reflow
|
||||
return model
|
||||
|
||||
|
||||
class DiffSingerVarianceONNX(DiffSingerVariance):
|
||||
def __init__(self, vocab_size, cross_lingual_token_idx=None):
|
||||
super().__init__(vocab_size=vocab_size)
|
||||
del self.fs2
|
||||
self.fs2 = FastSpeech2VarianceONNX(
|
||||
vocab_size=vocab_size,
|
||||
cross_lingual_token_idx=cross_lingual_token_idx
|
||||
)
|
||||
self.hidden_size = hparams['hidden_size']
|
||||
if self.predict_pitch:
|
||||
del self.pitch_predictor
|
||||
self.smooth: nn.Conv1d = None
|
||||
pitch_hparams = hparams['pitch_prediction_args']
|
||||
if self.diffusion_type == 'ddpm':
|
||||
self.pitch_predictor = PitchDiffusionONNX(
|
||||
vmin=pitch_hparams['pitd_norm_min'],
|
||||
vmax=pitch_hparams['pitd_norm_max'],
|
||||
cmin=pitch_hparams['pitd_clip_min'],
|
||||
cmax=pitch_hparams['pitd_clip_max'],
|
||||
repeat_bins=pitch_hparams['repeat_bins'],
|
||||
timesteps=hparams['timesteps'],
|
||||
k_step=hparams['K_step'],
|
||||
backbone_type=self.pitch_backbone_type,
|
||||
backbone_args=self.pitch_backbone_args
|
||||
)
|
||||
elif self.diffusion_type == 'reflow':
|
||||
self.pitch_predictor = PitchRectifiedFlowONNX(
|
||||
vmin=pitch_hparams['pitd_norm_min'],
|
||||
vmax=pitch_hparams['pitd_norm_max'],
|
||||
cmin=pitch_hparams['pitd_clip_min'],
|
||||
cmax=pitch_hparams['pitd_clip_max'],
|
||||
repeat_bins=pitch_hparams['repeat_bins'],
|
||||
time_scale_factor=hparams['time_scale_factor'],
|
||||
backbone_type=self.pitch_backbone_type,
|
||||
backbone_args=self.pitch_backbone_args
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Invalid diffusion type: {self.diffusion_type}")
|
||||
if self.predict_variances:
|
||||
del self.variance_predictor
|
||||
if self.diffusion_type == 'ddpm':
|
||||
self.variance_predictor = self.build_adaptor(cls=MultiVarianceDiffusionONNX)
|
||||
elif self.diffusion_type == 'reflow':
|
||||
self.variance_predictor = self.build_adaptor(cls=MultiVarianceRectifiedFlowONNX)
|
||||
else:
|
||||
raise NotImplementedError(self.diffusion_type)
|
||||
|
||||
def build_smooth_op(self, device):
|
||||
smooth_kernel_size = round(hparams['midi_smooth_width'] * hparams['audio_sample_rate'] / hparams['hop_size'])
|
||||
smooth = nn.Conv1d(
|
||||
in_channels=1,
|
||||
out_channels=1,
|
||||
kernel_size=smooth_kernel_size,
|
||||
bias=False,
|
||||
padding='same',
|
||||
padding_mode='replicate'
|
||||
).eval()
|
||||
smooth_kernel = torch.sin(torch.from_numpy(
|
||||
np.linspace(0, 1, smooth_kernel_size).astype(np.float32) * np.pi
|
||||
))
|
||||
smooth_kernel /= smooth_kernel.sum()
|
||||
smooth.weight.data = smooth_kernel[None, None]
|
||||
self.smooth = smooth.to(device)
|
||||
|
||||
def embed_frozen_spk(self, encoder_out):
|
||||
if hparams['use_spk_id'] and hasattr(self, 'frozen_spk_embed'):
|
||||
encoder_out += self.frozen_spk_embed
|
||||
return encoder_out
|
||||
|
||||
def forward_linguistic_encoder_word(self, tokens, word_div, word_dur, languages=None):
|
||||
encoder_out, x_masks = self.fs2.forward_encoder_word(tokens, word_div, word_dur, languages=languages)
|
||||
encoder_out = self.embed_frozen_spk(encoder_out)
|
||||
return encoder_out, x_masks
|
||||
|
||||
def forward_linguistic_encoder_phoneme(self, tokens, ph_dur, languages=None):
|
||||
encoder_out, x_masks = self.fs2.forward_encoder_phoneme(tokens, ph_dur, languages=languages)
|
||||
encoder_out = self.embed_frozen_spk(encoder_out)
|
||||
return encoder_out, x_masks
|
||||
|
||||
def forward_dur_predictor(self, encoder_out, x_masks, ph_midi, spk_embed=None):
|
||||
return self.fs2.forward_dur_predictor(encoder_out, x_masks, ph_midi, spk_embed=spk_embed)
|
||||
|
||||
def forward_mel2x_gather(self, x_src, x_dur, x_dim=None, check_stretch_embed=False):
|
||||
mel2x = self.lr(x_dur)
|
||||
_mel2x = mel2x
|
||||
if x_dim is not None:
|
||||
x_src = F.pad(x_src, [0, 0, 1, 0])
|
||||
mel2x = mel2x[..., None].repeat([1, 1, x_dim])
|
||||
else:
|
||||
x_src = F.pad(x_src, [1, 0])
|
||||
x_cond = torch.gather(x_src, 1, mel2x)
|
||||
if self.use_stretch_embed and check_stretch_embed:
|
||||
stretch = torch.round(1000 * self.sr(_mel2x, x_dur))
|
||||
table = self.stretch_embed(torch.arange(0, 1001, device=stretch.device))
|
||||
stretch_embed = torch.index_select(table, 0, stretch.view(-1).long()).view_as(x_cond)
|
||||
x_cond += stretch_embed
|
||||
stretch_embed_rnn_out, _ = self.stretch_embed_rnn(x_cond)
|
||||
x_cond += stretch_embed_rnn_out
|
||||
return x_cond
|
||||
|
||||
def forward_pitch_preprocess(
|
||||
self, encoder_out, ph_dur,
|
||||
note_midi=None, note_rest=None, note_dur=None, note_glide=None,
|
||||
pitch=None, expr=None, retake=None, spk_embed=None
|
||||
):
|
||||
condition = self.forward_mel2x_gather(encoder_out, ph_dur, x_dim=self.hidden_size, check_stretch_embed=True)
|
||||
if self.use_melody_encoder:
|
||||
if self.melody_encoder.use_glide_embed and note_glide is None:
|
||||
note_glide = torch.LongTensor([[0]]).to(encoder_out.device)
|
||||
melody_encoder_out = self.melody_encoder(
|
||||
note_midi, note_rest, note_dur,
|
||||
glide=note_glide
|
||||
)
|
||||
melody_encoder_out = self.forward_mel2x_gather(melody_encoder_out, note_dur, x_dim=self.hidden_size)
|
||||
condition += melody_encoder_out
|
||||
if expr is None:
|
||||
retake_embed = self.pitch_retake_embed(retake.long())
|
||||
else:
|
||||
retake_true_embed = self.pitch_retake_embed(
|
||||
torch.ones(1, 1, dtype=torch.long, device=encoder_out.device)
|
||||
) # [B=1, T=1] => [B=1, T=1, H]
|
||||
retake_false_embed = self.pitch_retake_embed(
|
||||
torch.zeros(1, 1, dtype=torch.long, device=encoder_out.device)
|
||||
) # [B=1, T=1] => [B=1, T=1, H]
|
||||
expr = (expr * retake)[:, :, None] # [B, T, 1]
|
||||
retake_embed = expr * retake_true_embed + (1. - expr) * retake_false_embed
|
||||
pitch_cond = condition + retake_embed
|
||||
frame_midi_pitch = self.forward_mel2x_gather(note_midi, note_dur, x_dim=None)
|
||||
base_pitch = self.smooth(frame_midi_pitch)
|
||||
if self.use_melody_encoder:
|
||||
delta_pitch = (pitch - base_pitch) * ~retake
|
||||
if self.use_variance_scaling:
|
||||
pitch_cond += self.delta_pitch_embed(delta_pitch[:, :, None] / 12)
|
||||
else:
|
||||
pitch_cond += self.delta_pitch_embed(delta_pitch[:, :, None])
|
||||
else:
|
||||
base_pitch = base_pitch * retake + pitch * ~retake
|
||||
if self.use_variance_scaling:
|
||||
pitch_cond += self.base_pitch_embed(base_pitch[:, :, None] / 128)
|
||||
else:
|
||||
pitch_cond += self.base_pitch_embed(base_pitch[:, :, None])
|
||||
if hparams['use_spk_id'] and spk_embed is not None:
|
||||
pitch_cond += spk_embed
|
||||
return pitch_cond, base_pitch
|
||||
|
||||
def forward_pitch_reflow(
|
||||
self, pitch_cond, steps: int = 10
|
||||
):
|
||||
x_pred = self.pitch_predictor(pitch_cond, steps=steps)
|
||||
return x_pred
|
||||
|
||||
def forward_pitch_postprocess(self, x_pred, base_pitch):
|
||||
pitch_pred = self.pitch_predictor.clamp_spec(x_pred) + base_pitch
|
||||
return pitch_pred
|
||||
|
||||
def forward_variance_preprocess(
|
||||
self, encoder_out, ph_dur, pitch,
|
||||
variances: dict = None, retake=None, spk_embed=None
|
||||
):
|
||||
condition = self.forward_mel2x_gather(encoder_out, ph_dur, x_dim=self.hidden_size, check_stretch_embed=True)
|
||||
if self.use_variance_scaling:
|
||||
variance_cond = condition + self.pitch_embed(pitch[:, :, None] / 12)
|
||||
else:
|
||||
variance_cond = condition + self.pitch_embed(pitch[:, :, None])
|
||||
non_retake_masks = [
|
||||
v_retake.float() # [B, T, 1]
|
||||
for v_retake in (~retake).split(1, dim=2)
|
||||
]
|
||||
variance_embeds = [
|
||||
self.variance_embeds[v_name](variances[v_name][:, :, None] * self.variance_retake_scaling[v_name]) * v_masks
|
||||
for v_name, v_masks in zip(self.variance_prediction_list, non_retake_masks)
|
||||
]
|
||||
variance_cond += torch.stack(variance_embeds, dim=-1).sum(-1)
|
||||
if hparams['use_spk_id'] and spk_embed is not None:
|
||||
variance_cond += spk_embed
|
||||
return variance_cond
|
||||
|
||||
def forward_variance_reflow(self, variance_cond, steps: int = 10):
|
||||
xs_pred = self.variance_predictor(variance_cond, steps=steps)
|
||||
return xs_pred
|
||||
|
||||
def forward_variance_postprocess(self, xs_pred):
|
||||
if self.variance_predictor.num_feats == 1:
|
||||
xs_pred = [xs_pred]
|
||||
else:
|
||||
xs_pred = xs_pred.unbind(dim=1)
|
||||
variance_pred = self.variance_predictor.clamp_spec(xs_pred)
|
||||
return tuple(variance_pred)
|
||||
|
||||
def view_as_linguistic_encoder(self):
|
||||
model = copy.deepcopy(self)
|
||||
if self.predict_pitch:
|
||||
del model.pitch_predictor
|
||||
if self.use_melody_encoder:
|
||||
del model.melody_encoder
|
||||
if self.predict_variances:
|
||||
del model.variance_predictor
|
||||
model.fs2 = model.fs2.view_as_encoder()
|
||||
if self.predict_dur:
|
||||
model.forward = model.forward_linguistic_encoder_word
|
||||
else:
|
||||
model.forward = model.forward_linguistic_encoder_phoneme
|
||||
return model
|
||||
|
||||
def view_as_dur_predictor(self):
|
||||
assert self.predict_dur
|
||||
model = copy.deepcopy(self)
|
||||
if self.predict_pitch:
|
||||
del model.pitch_predictor
|
||||
if self.use_melody_encoder:
|
||||
del model.melody_encoder
|
||||
if self.predict_variances:
|
||||
del model.variance_predictor
|
||||
model.fs2 = model.fs2.view_as_dur_predictor()
|
||||
model.forward = model.forward_dur_predictor
|
||||
return model
|
||||
|
||||
def view_as_pitch_preprocess(self):
|
||||
model = copy.deepcopy(self)
|
||||
del model.fs2
|
||||
if self.predict_pitch:
|
||||
del model.pitch_predictor
|
||||
if self.predict_variances:
|
||||
del model.variance_predictor
|
||||
model.forward = model.forward_pitch_preprocess
|
||||
return model
|
||||
|
||||
def view_as_pitch_predictor(self):
|
||||
assert self.predict_pitch
|
||||
model = copy.deepcopy(self)
|
||||
del model.fs2
|
||||
del model.lr
|
||||
if self.use_melody_encoder:
|
||||
del model.melody_encoder
|
||||
if self.predict_variances:
|
||||
del model.variance_predictor
|
||||
model.forward = model.forward_pitch_reflow
|
||||
return model
|
||||
|
||||
def view_as_pitch_postprocess(self):
|
||||
model = copy.deepcopy(self)
|
||||
del model.fs2
|
||||
if self.use_melody_encoder:
|
||||
del model.melody_encoder
|
||||
if self.predict_variances:
|
||||
del model.variance_predictor
|
||||
model.forward = model.forward_pitch_postprocess
|
||||
return model
|
||||
|
||||
def view_as_variance_preprocess(self):
|
||||
model = copy.deepcopy(self)
|
||||
del model.fs2
|
||||
if self.predict_pitch:
|
||||
del model.pitch_predictor
|
||||
if self.use_melody_encoder:
|
||||
del model.melody_encoder
|
||||
if self.predict_variances:
|
||||
del model.variance_predictor
|
||||
model.forward = model.forward_variance_preprocess
|
||||
return model
|
||||
|
||||
def view_as_variance_predictor(self):
|
||||
assert self.predict_variances
|
||||
model = copy.deepcopy(self)
|
||||
del model.fs2
|
||||
del model.lr
|
||||
if self.predict_pitch:
|
||||
del model.pitch_predictor
|
||||
if self.use_melody_encoder:
|
||||
del model.melody_encoder
|
||||
model.forward = model.forward_variance_reflow
|
||||
return model
|
||||
|
||||
def view_as_variance_postprocess(self):
|
||||
model = copy.deepcopy(self)
|
||||
del model.fs2
|
||||
if self.predict_pitch:
|
||||
del model.pitch_predictor
|
||||
if self.use_melody_encoder:
|
||||
del model.melody_encoder
|
||||
model.forward = model.forward_variance_postprocess
|
||||
return model
|
||||
Reference in New Issue
Block a user