File size: 4,499 Bytes
09b2bbf |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 |
import librosa.display as lbd
import matplotlib.pyplot as plt
import soundfile
import torch
from .InferenceArchitectures.InferenceFastSpeech2 import FastSpeech2
from .InferenceArchitectures.InferenceHiFiGAN import HiFiGANGenerator
from ..Preprocessing.ArticulatoryCombinedTextFrontend import ArticulatoryCombinedTextFrontend
from ..Preprocessing.ArticulatoryCombinedTextFrontend import get_language_id
class AnonFastSpeech2(torch.nn.Module):
def __init__(self, device: str, path_to_hifigan_model: str, path_to_fastspeech_model: str):
"""
Args:
device: Device to run on. CPU is feasible, still faster than real-time, but a GPU is significantly faster.
path_to_hifigan_model: Path to the vocoder model, including filename and suffix.
path_to_fastspeech_model: Path to the synthesis model, including filename and suffix.
"""
super().__init__()
language = "en"
self.device = device
self.text2phone = ArticulatoryCombinedTextFrontend(language=language, add_silence_to_end=True)
checkpoint = torch.load(path_to_fastspeech_model, map_location='cpu')
self.phone2mel = FastSpeech2(weights=checkpoint["model"], lang_embs=None).to(torch.device(device))
self.mel2wav = HiFiGANGenerator(path_to_weights=path_to_hifigan_model).to(torch.device(device))
self.default_utterance_embedding = checkpoint["default_emb"].to(self.device)
self.phone2mel.eval()
self.mel2wav.eval()
self.lang_id = get_language_id(language)
self.to(torch.device(device))
def forward(self, text, view=False, text_is_phonemes=False):
"""
Args:
text: The text that the TTS should convert to speech
view: Boolean flag whether to produce and display a graphic showing the generated audio
text_is_phonemes: Boolean flag whether the text parameter contains phonemes (True) or graphemes (False)
Returns:
48kHz waveform as 1d tensor
"""
with torch.no_grad():
phones = self.text2phone.string_to_tensor(text, input_phonemes=text_is_phonemes).to(torch.device(self.device))
mel, durations, pitch, energy = self.phone2mel(phones,
return_duration_pitch_energy=True,
utterance_embedding=self.default_utterance_embedding)
mel = mel.transpose(0, 1)
wave = self.mel2wav(mel)
if view:
from Utility.utils import cumsum_durations
fig, ax = plt.subplots(nrows=2, ncols=1)
ax[0].plot(wave.cpu().numpy())
lbd.specshow(mel.cpu().numpy(),
ax=ax[1],
sr=16000,
cmap='GnBu',
y_axis='mel',
x_axis=None,
hop_length=256)
ax[0].yaxis.set_visible(False)
ax[1].yaxis.set_visible(False)
duration_splits, label_positions = cumsum_durations(durations.cpu().numpy())
ax[1].set_xticks(duration_splits, minor=True)
ax[1].xaxis.grid(True, which='minor')
ax[1].set_xticks(label_positions, minor=False)
ax[1].set_xticklabels(self.text2phone.get_phone_string(text))
ax[0].set_title(text)
plt.subplots_adjust(left=0.05, bottom=0.1, right=0.95, top=.9, wspace=0.0, hspace=0.0)
plt.show()
return wave
def anonymize_to_file(self, text: str, text_is_phonemes: bool, target_speaker_embedding: torch.tensor, path_to_result_file: str):
"""
Args:
text: The text that the TTS should convert to speech
text_is_phonemes: Boolean flag whether the text parameter contains phonemes (True) or graphemes (False)
target_speaker_embedding: The speaker embedding that should be used for the produced speech
path_to_result_file: The path to the location where the resulting speech should be saved (including the filename and .wav suffix)
"""
assert text.strip() != ""
assert path_to_result_file.endswith(".wav")
self.default_utterance_embedding = target_speaker_embedding.to(self.device)
wav = self(text=text, text_is_phonemes=text_is_phonemes)
soundfile.write(file=path_to_result_file, data=wav.cpu().numpy(), samplerate=48000)
|