Spaces:
Running
Running
| # -*- coding: utf-8 -*- | |
| import typing | |
| import types # fusion of forward() of Wav2Vec2 | |
| import gradio as gr | |
| import matplotlib.pyplot as plt | |
| import numpy as np | |
| import os | |
| import torch | |
| import torch.nn as nn | |
| from transformers import Wav2Vec2Processor | |
| from transformers.models.wav2vec2.modeling_wav2vec2 import Wav2Vec2Model | |
| from transformers.models.wav2vec2.modeling_wav2vec2 import Wav2Vec2PreTrainedModel | |
| import audiofile | |
| from tts import StyleTTS2 | |
| import audresample | |
| import json | |
| import re | |
| import unicodedata | |
| import textwrap | |
| import nltk | |
| from num2words import num2words | |
| from num2word_greek.numbers2words import convert_numbers | |
| from audionar import VitsModel, VitsTokenizer | |
| from audiocraft import AudioGen | |
| audiogen = AudioGen().eval().to('cpu') | |
| nltk.download('punkt', download_dir='./') | |
| nltk.download('punkt_tab', download_dir='./') | |
| nltk.data.path.append('.') | |
| device = 'cpu' | |
| def fix_vocals(text, lang='ron'): | |
| # Longer phrases should come before shorter ones to prevent partial matches. | |
| ron_replacements = { | |
| 'ţ': 'ț', | |
| 'ț': 'ts', | |
| 'î': 'u', | |
| 'â': 'a', | |
| 'ş': 's', | |
| 'w': 'oui', | |
| 'k': 'c', | |
| 'l': 'll', | |
| # Math symbols | |
| 'sqrt': ' rădăcina pătrată din ', | |
| '^': ' la puterea ', | |
| '+': ' plus ', | |
| ' - ': ' minus ', # only replace if standalone so to not say minus if is a-b-c | |
| '*': ' ori ', # times | |
| '/': ' împărțit la ', # divided by | |
| '=': ' egal cu ', # equals | |
| 'pi': ' pi ', | |
| '<': ' mai mic decât ', | |
| '>': ' mai mare decât', | |
| '%': ' la sută ', # percent (from previous) | |
| '(': ' paranteză deschisă ', | |
| ')': ' paranteză închisă ', | |
| '[': ' paranteză pătrată deschisă ', | |
| ']': ' paranteză pătrată închisă ', | |
| '{': ' acoladă deschisă ', | |
| '}': ' acoladă închisă ', | |
| '≠': ' nu este egal cu ', | |
| '≤': ' mai mic sau egal cu ', | |
| '≥': ' mai mare sau egal cu ', | |
| '≈': ' aproximativ ', | |
| '∞': ' infinit ', | |
| '€': ' euro ', | |
| '$': ' dolar ', | |
| '£': ' liră ', | |
| '&': ' și ', # and | |
| '@': ' la ', # at | |
| '#': ' diez ', # hash | |
| '∑': ' sumă ', | |
| '∫': ' integrală ', | |
| '√': ' rădăcina pătrată a ', # more generic square root | |
| } | |
| eng_replacements = { | |
| 'wik': 'weaky', | |
| 'sh': 'ss', | |
| 'ch': 'ttss', | |
| 'oo': 'oeo', | |
| # Math symbols for English | |
| 'sqrt': ' square root of ', | |
| '^': ' to the power of ', | |
| '+': ' plus ', | |
| ' - ': ' minus ', | |
| '*': ' times ', | |
| ' / ': ' divided by ', | |
| '=': ' equals ', | |
| 'pi': ' pi ', | |
| '<': ' less than ', | |
| '>': ' greater than ', | |
| # Additional common math symbols from previous list | |
| '%': ' percent ', | |
| '(': ' open parenthesis ', | |
| ')': ' close parenthesis ', | |
| '[': ' open bracket ', | |
| ']': ' close bracket ', | |
| '{': ' open curly brace ', | |
| '}': ' close curly brace ', | |
| '∑': ' sum ', | |
| '∫': ' integral ', | |
| '√': ' square root of ', | |
| '≠': ' not equals ', | |
| '≤': ' less than or equals ', | |
| '≥': ' greater than or equals ', | |
| '≈': ' approximately ', | |
| '∞': ' infinity ', | |
| '€': ' euro ', | |
| '$': ' dollar ', | |
| '£': ' pound ', | |
| '&': ' and ', | |
| '@': ' at ', | |
| '#': ' hash ', | |
| } | |
| serbian_replacements = { | |
| 'rn': 'rrn', | |
| 'ć': 'č', | |
| 'c': 'č', | |
| 'đ': 'd', | |
| 'j': 'i', | |
| 'l': 'lll', | |
| 'w': 'v', | |
| # https://huggingface.co/facebook/mms-tts-rmc-script_latin | |
| 'sqrt': 'kvadratni koren iz', | |
| '^': ' na stepen ', | |
| '+': ' plus ', | |
| ' - ': ' minus ', | |
| '*': ' puta ', | |
| ' / ': ' podeljeno sa ', | |
| '=': ' jednako ', | |
| 'pi': ' pi ', | |
| '<': ' manje od ', | |
| '>': ' veće od ', | |
| '%': ' procenat ', | |
| '(': ' otvorena zagrada ', | |
| ')': ' zatvorena zagrada ', | |
| '[': ' otvorena uglasta zagrada ', | |
| ']': ' zatvorena uglasta zagrada ', | |
| '{': ' otvorena vitičasta zagrada ', | |
| '}': ' zatvorena vitičasta zagrada ', | |
| '∑': ' suma ', | |
| '∫': ' integral ', | |
| '√': ' kvadratni koren ', | |
| '≠': ' nije jednako ', | |
| '≤': ' manje ili jednako od ', | |
| '≥': ' veće ili jednako od ', | |
| '≈': ' približno ', | |
| '∞': ' beskonačnost ', | |
| '€': ' evro ', | |
| '$': ' dolar ', | |
| '£': ' funta ', | |
| '&': ' i ', | |
| '@': ' et ', | |
| '#': ' taraba ', | |
| # Others | |
| # 'rn': 'rrn', | |
| # 'ć': 'č', | |
| # 'c': 'č', | |
| # 'đ': 'd', | |
| # 'l': 'le', | |
| # 'ij': 'i', | |
| # 'ji': 'i', | |
| # 'j': 'i', | |
| # 'služ': 'sloooozz', # 'službeno' | |
| # 'suver': 'siuveeerra', # 'suverena' | |
| # 'država': 'dirrezav', # 'država' | |
| # 'iči': 'ici', # 'Graniči' | |
| # 's ': 'se', # a s with space | |
| # 'q': 'ku', | |
| # 'w': 'aou', | |
| # 'z': 's', | |
| # "š": "s", | |
| # 'th': 'ta', | |
| # 'v': 'vv', | |
| # "ć": "č", | |
| # "đ": "ď", | |
| # "lj": "ľ", | |
| # "nj": "ň", | |
| # "ž": "z", | |
| # "c": "č" | |
| } | |
| deu_replacements = { | |
| 'sch': 'sh', | |
| 'ch': 'kh', | |
| 'ie': 'ee', | |
| 'ei': 'ai', | |
| 'ä': 'ae', | |
| 'ö': 'oe', | |
| 'ü': 'ue', | |
| 'ß': 'ss', | |
| # Math symbols for German | |
| 'sqrt': ' Quadratwurzel aus ', | |
| '^': ' hoch ', | |
| '+': ' plus ', | |
| ' - ': ' minus ', | |
| '*': ' mal ', | |
| ' / ': ' geteilt durch ', | |
| '=': ' gleich ', | |
| 'pi': ' pi ', | |
| '<': ' kleiner als ', | |
| '>': ' größer als', | |
| # Additional common math symbols from previous list | |
| '%': ' prozent ', | |
| '(': ' Klammer auf ', | |
| ')': ' Klammer zu ', | |
| '[': ' eckige Klammer auf ', | |
| ']': ' eckige Klammer zu ', | |
| '{': ' geschweifte Klammer auf ', | |
| '}': ' geschweifte Klammer zu ', | |
| '∑': ' Summe ', | |
| '∫': ' Integral ', | |
| '√': ' Quadratwurzel ', | |
| '≠': ' ungleich ', | |
| '≤': ' kleiner oder gleich ', | |
| '≥': ' größer oder gleich ', | |
| '≈': ' ungefähr ', | |
| '∞': ' unendlich ', | |
| '€': ' euro ', | |
| '$': ' dollar ', | |
| '£': ' pfund ', | |
| '&': ' und ', | |
| '@': ' at ', # 'Klammeraffe' is also common but 'at' is simpler | |
| '#': ' raute ', | |
| } | |
| fra_replacements = { | |
| # French specific phonetic replacements (add as needed) | |
| # e.g., 'ç': 's', 'é': 'e', etc. | |
| 'w': 'v', | |
| # Math symbols for French | |
| 'sqrt': ' racine carrée de ', | |
| '^': ' à la puissance ', | |
| '+': ' plus ', | |
| ' - ': ' moins ', # tiré ; | |
| '*': ' fois ', | |
| ' / ': ' divisé par ', | |
| '=': ' égale ', | |
| 'pi': ' pi ', | |
| '<': ' inférieur à ', | |
| '>': ' supérieur à ', | |
| # Add more common math symbols as needed for French | |
| '%': ' pour cent ', | |
| '(': ' parenthèse ouverte ', | |
| ')': ' parenthèse fermée ', | |
| '[': ' crochet ouvert ', | |
| ']': ' crochet fermé ', | |
| '{': ' accolade ouverte ', | |
| '}': ' accolade fermée ', | |
| '∑': ' somme ', | |
| '∫': ' intégrale ', | |
| '√': ' racine carrée ', | |
| '≠': ' n\'égale pas ', | |
| '≤': ' inférieur ou égal à ', | |
| '≥': ' supérieur ou égal à ', | |
| '≈': ' approximativement ', | |
| '∞': ' infini ', | |
| '€': ' euro ', | |
| '$': ' dollar ', | |
| '£': ' livre ', | |
| '&': ' et ', | |
| '@': ' arobase ', | |
| '#': ' dièse ', | |
| } | |
| hun_replacements = { | |
| # Hungarian specific phonetic replacements (add as needed) | |
| # e.g., 'á': 'a', 'é': 'e', etc. | |
| 'ch': 'ts', | |
| 'cs': 'tz', | |
| 'g': 'gk', | |
| 'w': 'v', | |
| 'z': 'zz', | |
| # Math symbols for Hungarian | |
| 'sqrt': ' négyzetgyök ', | |
| '^': ' hatvány ', | |
| '+': ' plusz ', | |
| ' - ': ' mínusz ', | |
| '*': ' szorozva ', | |
| ' / ': ' osztva ', | |
| '=': ' egyenlő ', | |
| 'pi': ' pi ', | |
| '<': ' kisebb mint ', | |
| '>': ' nagyobb mint ', | |
| # Add more common math symbols as needed for Hungarian | |
| '%': ' százalék ', | |
| '(': ' nyitó zárójel ', | |
| ')': ' záró zárójel ', | |
| '[': ' nyitó szögletes zárójel ', | |
| ']': ' záró szögletes zárójel ', | |
| '{': ' nyitó kapcsos zárójel ', | |
| '}': ' záró kapcsos zárójel ', | |
| '∑': ' szumma ', | |
| '∫': ' integrál ', | |
| '√': ' négyzetgyök ', | |
| '≠': ' nem egyenlő ', | |
| '≤': ' kisebb vagy egyenlő ', | |
| '≥': ' nagyobb vagy egyenlő ', | |
| '≈': ' körülbelül ', | |
| '∞': ' végtelen ', | |
| '€': ' euró ', | |
| '$': ' dollár ', | |
| '£': ' font ', | |
| '&': ' és ', | |
| '@': ' kukac ', | |
| '#': ' kettőskereszt ', | |
| } | |
| grc_replacements = { | |
| # Ancient Greek specific phonetic replacements (add as needed) | |
| # These are more about transliterating Greek letters if they are in the input text. | |
| # Math symbols for Ancient Greek (literal translations) | |
| 'sqrt': ' τετραγωνικὴ ῥίζα ', | |
| '^': ' εἰς τὴν δύναμιν ', | |
| '+': ' σὺν ', | |
| ' - ': ' χωρὶς ', | |
| '*': ' πολλάκις ', | |
| ' / ': ' διαιρέω ', | |
| '=': ' ἴσον ', | |
| 'pi': ' πῖ ', | |
| '<': ' ἔλαττον ', | |
| '>': ' μεῖζον ', | |
| # Add more common math symbols as needed for Ancient Greek | |
| '%': ' τοῖς ἑκατόν ', # tois hekaton - 'of the hundred' | |
| '(': ' ἀνοικτὴ παρένθεσις ', | |
| ')': ' κλειστὴ παρένθεσις ', | |
| '[': ' ἀνοικτὴ ἀγκύλη ', | |
| ']': ' κλειστὴ ἀγκύλη ', | |
| '{': ' ἀνοικτὴ σγουρὴ ἀγκύλη ', | |
| '}': ' κλειστὴ σγουρὴ ἀγκύλη ', | |
| '∑': ' ἄθροισμα ', | |
| '∫': ' ὁλοκλήρωμα ', | |
| '√': ' τετραγωνικὴ ῥίζα ', | |
| '≠': ' οὐκ ἴσον ', | |
| '≤': ' ἔλαττον ἢ ἴσον ', | |
| '≥': ' μεῖζον ἢ ἴσον ', | |
| '≈': ' περίπου ', | |
| '∞': ' ἄπειρον ', | |
| '€': ' εὐρώ ', | |
| '$': ' δολάριον ', | |
| '£': ' λίρα ', | |
| '&': ' καὶ ', | |
| '@': ' ἀτ ', # at | |
| '#': ' δίεση ', # hash | |
| } | |
| # Select the appropriate replacement dictionary based on the language | |
| replacements_map = { | |
| 'grc': grc_replacements, | |
| 'ron': ron_replacements, | |
| 'eng': eng_replacements, | |
| 'deu': deu_replacements, | |
| 'fra': fra_replacements, | |
| 'hun': hun_replacements, | |
| 'rmc-script_latin': serbian_replacements, | |
| } | |
| current_replacements = replacements_map.get(lang) | |
| if current_replacements: | |
| # Sort replacements by length of the key in descending order. | |
| # This is crucial for correctly replacing multi-character strings (like 'sqrt', 'sch') | |
| # before their shorter substrings ('s', 'ch', 'q', 'r', 't'). | |
| sorted_replacements = sorted(current_replacements.items(), key=lambda item: len(item[0]), reverse=True) | |
| for old, new in sorted_replacements: | |
| text = text.replace(old, new) | |
| return text | |
| else: | |
| # If the language is not supported, return the original text | |
| print(f"Warning: Language '{lang}' not supported for text replacement. Returning original text.") | |
| return text | |
| def _num2words(text='01234', lang=None): | |
| if lang == 'grc': | |
| return convert_numbers(text) | |
| return num2words(text, lang=lang) # HAS TO BE kwarg lang=lang | |
| def transliterate_number(number_string, | |
| lang=None): | |
| if lang == 'rmc-script_latin': | |
| lang = 'sr' | |
| exponential_pronoun = ' puta deset na stepen od ' | |
| comma = ' tačka ' | |
| elif lang == 'ron': | |
| lang = 'ro' | |
| exponential_pronoun = ' tízszer a erejéig ' | |
| comma = ' virgulă ' | |
| elif lang == 'hun': | |
| lang = 'hu' | |
| exponential_pronoun = ' tízszer a erejéig ' | |
| comma = ' virgula ' | |
| elif lang == 'deu': | |
| exponential_pronoun = ' mal zehn hoch ' | |
| comma = ' komma ' | |
| elif lang == 'fra': | |
| lang = 'fr' | |
| exponential_pronoun = ' puissance ' | |
| comma = 'virgule' | |
| elif lang == 'grc': | |
| exponential_pronoun = ' εις την δυναμην του ' | |
| comma = 'κομμα' | |
| else: | |
| lang = lang[:2] | |
| exponential_pronoun = ' times ten to the power of ' | |
| comma = ' point ' | |
| def replace_number(match): | |
| prefix = match.group(1) or "" | |
| number_part = match.group(2) | |
| suffix = match.group(5) or "" | |
| try: | |
| if 'e' in number_part.lower(): | |
| base, exponent = number_part.lower().split('e') | |
| words = _num2words(base, lang=lang) + exponential_pronoun + _num2words(exponent, lang=lang) | |
| elif '.' in number_part: | |
| integer_part, decimal_part = number_part.split('.') | |
| words = _num2words(integer_part, lang=lang) + comma + " ".join( | |
| [_num2words(digit, lang=lang) for digit in decimal_part]) | |
| else: | |
| words = _num2words(number_part, lang=lang) | |
| return prefix + words + suffix | |
| except ValueError: | |
| return match.group(0) # Return original if conversion fails | |
| pattern = r'([^\d]*)(\d+(\.\d+)?([Ee][+-]?\d+)?)([^\d]*)' | |
| return re.sub(pattern, replace_number, number_string) | |
| language_names = ['Ancient greek', | |
| 'English', | |
| 'Deutsch', | |
| 'French', | |
| 'Hungarian', | |
| 'Romanian', | |
| 'Serbian (Approx.)'] | |
| def audionar_tts(text=None, | |
| lang='romanian', | |
| soundscape='', | |
| cache_lim=24): | |
| # https://huggingface.co/dkounadis/artificial-styletts2/blob/main/msinference.py | |
| lang_map = { | |
| 'ancient greek': 'grc', | |
| 'english': 'eng', | |
| 'deutsch': 'deu', | |
| 'french': 'fra', | |
| 'hungarian': 'hun', | |
| 'romanian': 'ron', | |
| 'serbian (approx.)': 'rmc-script_latin', | |
| } | |
| if text and text.strip(): | |
| if lang not in language_names: | |
| speech_audio = _styletts2(text=text, # Eng. | |
| ref_s='wav/' + lang + '.wav') | |
| else: # VITS | |
| lang_code = lang_map.get(lang.lower(), lang.lower().split()[0].strip()) | |
| global cached_lang_code, cached_net_g, cached_tokenizer | |
| if 'cached_lang_code' not in globals() or cached_lang_code != lang_code: | |
| cached_lang_code = lang_code | |
| cached_net_g = VitsModel.from_pretrained(f'facebook/mms-tts-{lang_code}').eval() | |
| cached_tokenizer = VitsTokenizer.from_pretrained(f'facebook/mms-tts-{lang_code}') | |
| net_g = cached_net_g | |
| tokenizer = cached_tokenizer | |
| text = only_greek_or_only_latin(text, lang=lang_code) | |
| text = transliterate_number(text, lang=lang_code) | |
| text = fix_vocals(text, lang=lang_code) | |
| sentences = textwrap.wrap(text, width=439) | |
| total_audio_parts = [] | |
| for sentence in sentences: | |
| inputs = cached_tokenizer(sentence, return_tensors="pt") | |
| with torch.no_grad(): | |
| audio_part = cached_net_g( | |
| input_ids=inputs.input_ids.to(device), | |
| attention_mask=inputs.attention_mask.to(device), | |
| lang_code=lang_code, | |
| )[0, :] | |
| total_audio_parts.append(audio_part) | |
| speech_audio = torch.cat(total_audio_parts).cpu().numpy() | |
| # AudioGen | |
| if soundscape and soundscape.strip(): | |
| speech_duration_secs = len(speech_audio) / 16000 if speech_audio is not None else 0 | |
| target_duration = max(speech_duration_secs + 0.74, 2.0) | |
| background_audio = audiogen.generate( | |
| soundscape, | |
| duration=target_duration, | |
| cache_lim=max(4, int(cache_lim)) # at least allow 10 A/R stEps | |
| ).numpy() | |
| if speech_audio is not None: | |
| len_speech = len(speech_audio) | |
| len_background = len(background_audio) | |
| if len_background > len_speech: | |
| padding = np.zeros(len_background - len_speech, | |
| dtype=np.float32) | |
| speech_audio = np.concatenate([speech_audio, padding]) | |
| elif len_speech > len_background: | |
| padding = np.zeros(len_speech - len_background, | |
| dtype=np.float32) | |
| background_audio = np.concatenate([background_audio, padding]) | |
| speech_audio_stereo = speech_audio[None, :] | |
| background_audio_stereo = background_audio[None, :] | |
| final_audio = np.concatenate([ | |
| 0.49 * speech_audio_stereo + 0.51 * background_audio_stereo, | |
| 0.51 * background_audio_stereo + 0.49 * speech_audio_stereo | |
| ], 0) | |
| else: | |
| final_audio = background_audio | |
| # If no soundscape, use the speech audio as is. | |
| elif speech_audio is not None: | |
| final_audio = speech_audio | |
| # If both inputs are empty, create a 2s silent audio file. | |
| if final_audio is None: | |
| final_audio = np.zeros(16000 * 2, dtype=np.float32) | |
| wavfile = '_vits_.wav' | |
| audiofile.write(wavfile, final_audio, 16000) | |
| return wavfile, wavfile # 2x file for [audio out & state to pass to the Emotion reco tAB] | |
| # -- EXPRESSIO | |
| device = 0 if torch.cuda.is_available() else "cpu" | |
| duration = 2 # limit processing of audio | |
| age_gender_model_name = "audeering/wav2vec2-large-robust-6-ft-age-gender" | |
| expression_model_name = "audeering/wav2vec2-large-robust-12-ft-emotion-msp-dim" | |
| class AgeGenderHead(nn.Module): | |
| r"""Age-gender model head.""" | |
| def __init__(self, config, num_labels): | |
| super().__init__() | |
| self.dense = nn.Linear(config.hidden_size, config.hidden_size) | |
| self.dropout = nn.Dropout(config.final_dropout) | |
| self.out_proj = nn.Linear(config.hidden_size, num_labels) | |
| def forward(self, features, **kwargs): | |
| x = features | |
| x = self.dropout(x) | |
| x = self.dense(x) | |
| x = torch.tanh(x) | |
| x = self.dropout(x) | |
| x = self.out_proj(x) | |
| return x | |
| class AgeGenderModel(Wav2Vec2PreTrainedModel): | |
| r"""Age-gender recognition model.""" | |
| def __init__(self, config): | |
| super().__init__(config) | |
| self.config = config | |
| self.wav2vec2 = Wav2Vec2Model(config) | |
| self.age = AgeGenderHead(config, 1) | |
| self.gender = AgeGenderHead(config, 3) | |
| self.init_weights() | |
| def forward( | |
| self, | |
| frozen_cnn7, | |
| ): | |
| hidden_states = self.wav2vec2(frozen_cnn7=frozen_cnn7) # runs only Transformer layers | |
| hidden_states = torch.mean(hidden_states, dim=1) | |
| logits_age = self.age(hidden_states) | |
| logits_gender = torch.softmax(self.gender(hidden_states), dim=1) | |
| return hidden_states, logits_age, logits_gender | |
| # AgeGenderModel.forward() is switched to accept computed frozen CNN7 features from ExpressioNmodel | |
| def _forward( | |
| self, | |
| frozen_cnn7=None, # CNN7 fetures of wav2vec2 calc. from CNN7 feature extractor (once) | |
| attention_mask=None): | |
| if attention_mask is not None: | |
| # compute reduced attention_mask corresponding to feature vectors | |
| attention_mask = self._get_feature_vector_attention_mask( | |
| frozen_cnn7.shape[1], attention_mask, add_adapter=False | |
| ) | |
| hidden_states, _ = self.wav2vec2.feature_projection(frozen_cnn7) | |
| hidden_states = self.wav2vec2.encoder( | |
| hidden_states, | |
| attention_mask=attention_mask, | |
| output_attentions=None, | |
| output_hidden_states=None, | |
| return_dict=None, | |
| )[0] | |
| return hidden_states | |
| def _forward_and_cnn7( | |
| self, | |
| input_values, | |
| attention_mask=None): | |
| frozen_cnn7 = self.wav2vec2.feature_extractor(input_values) | |
| frozen_cnn7 = frozen_cnn7.transpose(1, 2) | |
| if attention_mask is not None: | |
| # compute reduced attention_mask corresponding to feature vectors | |
| attention_mask = self.wav2vec2._get_feature_vector_attention_mask( | |
| frozen_cnn7.shape[1], attention_mask, add_adapter=False | |
| ) | |
| hidden_states, _ = self.wav2vec2.feature_projection(frozen_cnn7) # grad=True non frozen | |
| hidden_states = self.wav2vec2.encoder( | |
| hidden_states, | |
| attention_mask=attention_mask, | |
| output_attentions=None, | |
| output_hidden_states=None, | |
| return_dict=None, | |
| )[0] | |
| return hidden_states, frozen_cnn7 #feature_proj is trainable thus we have to access the frozen_cnn7 before projection layer | |
| class ExpressionHead(nn.Module): | |
| r"""Expression model head.""" | |
| def __init__(self, config): | |
| super().__init__() | |
| self.dense = nn.Linear(config.hidden_size, config.hidden_size) | |
| self.dropout = nn.Dropout(config.final_dropout) | |
| self.out_proj = nn.Linear(config.hidden_size, config.num_labels) | |
| def forward(self, features, **kwargs): | |
| x = features | |
| x = self.dropout(x) | |
| x = self.dense(x) | |
| x = torch.tanh(x) | |
| x = self.dropout(x) | |
| x = self.out_proj(x) | |
| return x | |
| class ExpressionModel(Wav2Vec2PreTrainedModel): | |
| r"""speech expression model.""" | |
| def __init__(self, config): | |
| super().__init__(config) | |
| self.config = config | |
| self.wav2vec2 = Wav2Vec2Model(config) | |
| self.classifier = ExpressionHead(config) | |
| self.init_weights() | |
| def forward(self, input_values): | |
| hidden_states, frozen_cnn7 = self.wav2vec2(input_values) | |
| hidden_states = torch.mean(hidden_states, dim=1) | |
| logits = self.classifier(hidden_states) | |
| return hidden_states, logits, frozen_cnn7 | |
| # Load models from hub | |
| age_gender_model = AgeGenderModel.from_pretrained(age_gender_model_name) | |
| expression_processor = Wav2Vec2Processor.from_pretrained(expression_model_name) | |
| expression_model = ExpressionModel.from_pretrained(expression_model_name) | |
| # Emotion Calc. CNN features | |
| age_gender_model.wav2vec2.forward = types.MethodType(_forward, age_gender_model) | |
| expression_model.wav2vec2.forward = types.MethodType(_forward_and_cnn7, expression_model) | |
| def process_func(x: np.ndarray, sampling_rate: int) -> typing.Tuple[str, dict, str]: | |
| # batch audio | |
| y = expression_processor(x, sampling_rate=sampling_rate) | |
| y = y['input_values'][0] | |
| y = y.reshape(1, -1) | |
| y = torch.from_numpy(y).to(device) | |
| # run through expression model | |
| with torch.no_grad(): | |
| _, logits_expression, frozen_cnn7 = expression_model(y) | |
| _, logits_age, logits_gender = age_gender_model(frozen_cnn7=frozen_cnn7) | |
| # Plot A/D/V values | |
| plot_expression(logits_expression[0, 0].item(), # implicit detach().cpu().numpy() | |
| logits_expression[0, 1].item(), | |
| logits_expression[0, 2].item()) | |
| expression_file = "expression.png" | |
| plt.savefig(expression_file) | |
| return ( | |
| f"{round(100 * logits_age[0, 0].item())} years", # age | |
| { | |
| "female": logits_gender[0, 0].item(), | |
| "male": logits_gender[0, 1].item(), | |
| "child": logits_gender[0, 2].item(), | |
| }, | |
| expression_file, | |
| ) | |
| def recognize(input_file): | |
| if input_file is None: | |
| raise gr.Error( | |
| "No audio file submitted! " | |
| "Please upload or record an audio file " | |
| "before submitting your request." | |
| ) | |
| signal, sampling_rate = audiofile.read(input_file, duration=duration) | |
| # Resample to sampling rate supported byu the models | |
| target_rate = 16000 | |
| signal = audresample.resample(signal, sampling_rate, target_rate) | |
| return process_func(signal, target_rate) | |
| def explode(data): | |
| """ | |
| Expands a 3D array by creating gaps between voxels. | |
| This function is used to create the visual separation between the voxels. | |
| """ | |
| shape_orig = np.array(data.shape) | |
| shape_new = shape_orig * 2 - 1 | |
| retval = np.zeros(shape_new, dtype=data.dtype) | |
| retval[::2, ::2, ::2] = data | |
| return retval | |
| def explode(data): | |
| """ | |
| Expands a 3D array by adding new voxels between existing ones. | |
| This is used to create the gaps in the 3D plot. | |
| """ | |
| shape = data.shape | |
| new_shape = (2 * shape[0] - 1, 2 * shape[1] - 1, 2 * shape[2] - 1) | |
| new_data = np.zeros(new_shape, dtype=data.dtype) | |
| new_data[::2, ::2, ::2] = data | |
| return new_data | |
| def plot_expression(arousal, dominance, valence): | |
| '''_h = cuda tensor (N_PIX, N_PIX, N_PIX)''' | |
| N_PIX = 5 | |
| _h = np.random.rand(N_PIX, N_PIX, N_PIX) * 1e-3 | |
| adv = np.array([arousal, .994 - dominance, valence]).clip(0, .99) | |
| arousal, dominance, valence = (adv * N_PIX).astype(np.int64) # find voxel | |
| _h[arousal, dominance, valence] = .22 | |
| filled = np.ones((N_PIX, N_PIX, N_PIX), dtype=bool) | |
| # upscale the above voxel image, leaving gaps | |
| filled_2 = explode(filled) | |
| # Shrink the gaps | |
| x, y, z = np.indices(np.array(filled_2.shape) + 1).astype(float) // 2 | |
| x[1::2, :, :] += 1 | |
| y[:, 1::2, :] += 1 | |
| z[:, :, 1::2] += 1 | |
| fig = plt.figure() | |
| ax = fig.add_subplot(projection='3d') | |
| f_2 = np.ones([2 * N_PIX - 1, | |
| 2 * N_PIX - 1, | |
| 2 * N_PIX - 1, 4], dtype=np.float64) | |
| f_2[:, :, :, 3] = explode(_h) | |
| cm = plt.get_cmap('cool') | |
| f_2[:, :, :, :3] = cm(f_2[:, :, :, 3])[..., :3] | |
| f_2[:, :, :, 3] = f_2[:, :, :, 3].clip(.01, .74) | |
| ecolors_2 = f_2 | |
| ax.voxels(x, y, z, filled_2, facecolors=f_2, edgecolors=.006 * ecolors_2) | |
| ax.set_aspect('equal') | |
| ax.set_zticks([0, N_PIX]) | |
| ax.set_xticks([0, N_PIX]) | |
| ax.set_yticks([0, N_PIX]) | |
| ax.set_zticklabels([f'{n/N_PIX:.2f}'[0:] for n in ax.get_zticks()]) | |
| ax.set_zlabel('valence', fontsize=10, labelpad=0) | |
| ax.set_xticklabels([f'{n/N_PIX:.2f}' for n in ax.get_xticks()]) | |
| ax.set_xlabel('arousal', fontsize=10, labelpad=7) | |
| # The y-axis rotation is corrected here from 275 to 90 degrees | |
| ax.set_yticklabels([f'{1-n/N_PIX:.2f}' for n in ax.get_yticks()], rotation=90) | |
| ax.set_ylabel('dominance', fontsize=10, labelpad=10) | |
| ax.grid(False) | |
| ax.plot([N_PIX, N_PIX], [0, N_PIX + .2], [N_PIX, N_PIX], 'g', linewidth=1) | |
| ax.plot([0, N_PIX], [N_PIX, N_PIX + .24], [N_PIX, N_PIX], 'k', linewidth=1) | |
| # Missing lines on the top face | |
| ax.plot([0, 0], [0, N_PIX], [N_PIX, N_PIX], 'darkred', linewidth=1) | |
| ax.plot([0, N_PIX], [0, 0], [N_PIX, N_PIX], 'darkblue', linewidth=1) | |
| # Set pane colors after plotting the lines | |
| # UPDATED: Replaced `w_xaxis` with `xaxis` and `w_yaxis` with `yaxis`. | |
| ax.xaxis.set_pane_color((0.8, 0.8, 0.8, 0.5)) | |
| ax.yaxis.set_pane_color((0.8, 0.8, 0.8, 0.5)) | |
| ax.zaxis.set_pane_color((0.8, 0.8, 0.8, 0.0)) | |
| # Restore the limits to prevent the plot from expanding | |
| ax.set_xlim(0, N_PIX) | |
| ax.set_ylim(0, N_PIX) | |
| ax.set_zlim(0, N_PIX) | |
| # plt.show() | |
| # TTS | |
| # VOICES = [f'wav/{vox}' for vox in os.listdir('wav')] | |
| # add unidecode (to parse non-roman characters for the StyleTTS2 | |
| # # for the VITS it should better skip the unknown letters - dont use unidecode()) | |
| # at generation fill the state of "last tts" | |
| # at record fill the state of "last record" and place in list of voice/langs for TTS | |
| VOICES = ['jv_ID_google-gmu_04982.wav', | |
| 'it_IT_mls_1595.wav', | |
| 'en_US_vctk_p303.wav', | |
| 'en_US_vctk_p306.wav', | |
| 'it_IT_mls_8842.wav', | |
| 'en_US_cmu_arctic_ksp.wav', | |
| 'jv_ID_google-gmu_05970.wav', | |
| 'en_US_vctk_p318.wav', | |
| 'ha_NE_openbible.wav', | |
| 'ne_NP_ne-google_0883.wav', | |
| 'en_US_vctk_p280.wav', | |
| 'bn_multi_1010.wav', | |
| 'en_US_vctk_p259.wav', | |
| 'it_IT_mls_844.wav', | |
| 'en_US_vctk_p269.wav', | |
| 'en_US_vctk_p285.wav', | |
| 'de_DE_m-ailabs_angela_merkel.wav', | |
| 'en_US_vctk_p316.wav', | |
| 'en_US_vctk_p362.wav', | |
| 'jv_ID_google-gmu_06207.wav', | |
| 'tn_ZA_google-nwu_9061.wav', | |
| 'fr_FR_tom.wav', | |
| 'en_US_vctk_p233.wav', | |
| 'it_IT_mls_4975.wav', | |
| 'en_US_vctk_p236.wav', | |
| 'bn_multi_01232.wav', | |
| 'bn_multi_5958.wav', | |
| 'it_IT_mls_9185.wav', | |
| 'en_US_vctk_p248.wav', | |
| 'en_US_vctk_p287.wav', | |
| 'it_IT_mls_9772.wav', | |
| 'te_IN_cmu-indic_sk.wav', | |
| 'tn_ZA_google-nwu_8333.wav', | |
| 'en_US_vctk_p260.wav', | |
| 'en_US_vctk_p247.wav', | |
| 'en_US_vctk_p329.wav', | |
| 'en_US_cmu_arctic_fem.wav', | |
| 'en_US_cmu_arctic_rms.wav', | |
| 'en_US_vctk_p308.wav', | |
| 'jv_ID_google-gmu_08736.wav', | |
| 'en_US_vctk_p245.wav', | |
| 'fr_FR_m-ailabs_nadine_eckert_boulet.wav', | |
| 'jv_ID_google-gmu_03314.wav', | |
| 'en_US_vctk_p239.wav', | |
| 'jv_ID_google-gmu_05540.wav', | |
| 'it_IT_mls_7440.wav', | |
| 'en_US_vctk_p310.wav', | |
| 'en_US_vctk_p237.wav', | |
| 'en_US_hifi-tts_92.wav', | |
| 'en_US_cmu_arctic_aew.wav', | |
| 'ne_NP_ne-google_2099.wav', | |
| 'en_US_vctk_p226.wav', | |
| 'af_ZA_google-nwu_1919.wav', | |
| 'jv_ID_google-gmu_03727.wav', | |
| 'en_US_vctk_p317.wav', | |
| 'tn_ZA_google-nwu_0378.wav', | |
| 'nl_pmk.wav', | |
| 'en_US_vctk_p286.wav', | |
| 'tn_ZA_google-nwu_3342.wav', | |
| # 'en_US_vctk_p343.wav', | |
| 'de_DE_m-ailabs_ramona_deininger.wav', | |
| 'jv_ID_google-gmu_03424.wav', | |
| 'en_US_vctk_p341.wav', | |
| 'jv_ID_google-gmu_03187.wav', | |
| 'ne_NP_ne-google_3960.wav', | |
| 'jv_ID_google-gmu_06080.wav', | |
| 'ne_NP_ne-google_3997.wav', | |
| # 'en_US_vctk_p267.wav', | |
| 'en_US_vctk_p240.wav', | |
| 'ne_NP_ne-google_5687.wav', | |
| 'ne_NP_ne-google_9407.wav', | |
| 'jv_ID_google-gmu_05667.wav', | |
| 'jv_ID_google-gmu_01519.wav', | |
| 'ne_NP_ne-google_7957.wav', | |
| 'it_IT_mls_4705.wav', | |
| 'ne_NP_ne-google_6329.wav', | |
| 'it_IT_mls_1725.wav', | |
| 'tn_ZA_google-nwu_8914.wav', | |
| 'en_US_ljspeech.wav', | |
| 'tn_ZA_google-nwu_4850.wav', | |
| 'en_US_vctk_p238.wav', | |
| 'en_US_vctk_p302.wav', | |
| 'jv_ID_google-gmu_08178.wav', | |
| 'en_US_vctk_p313.wav', | |
| 'af_ZA_google-nwu_2418.wav', | |
| 'bn_multi_00737.wav', | |
| 'en_US_vctk_p275.wav', # y | |
| 'af_ZA_google-nwu_0184.wav', | |
| 'jv_ID_google-gmu_07638.wav', | |
| 'ne_NP_ne-google_6587.wav', | |
| 'ne_NP_ne-google_0258.wav', | |
| 'en_US_vctk_p232.wav', | |
| 'en_US_vctk_p336.wav', | |
| 'jv_ID_google-gmu_09039.wav', | |
| 'en_US_vctk_p312.wav', | |
| 'af_ZA_google-nwu_8148.wav', | |
| 'en_US_vctk_p326.wav', | |
| 'en_US_vctk_p264.wav', | |
| 'en_US_vctk_p295.wav', | |
| # 'en_US_vctk_p298.wav', | |
| 'es_ES_m-ailabs_victor_villarraza.wav', | |
| 'pl_PL_m-ailabs_nina_brown.wav', | |
| 'tn_ZA_google-nwu_9365.wav', | |
| 'en_US_vctk_p294.wav', | |
| 'jv_ID_google-gmu_00658.wav', | |
| 'jv_ID_google-gmu_08305.wav', | |
| 'en_US_vctk_p330.wav', | |
| 'gu_IN_cmu-indic_cmu_indic_guj_dp.wav', | |
| 'jv_ID_google-gmu_05219.wav', | |
| 'en_US_vctk_p284.wav', | |
| 'de_DE_m-ailabs_eva_k.wav', | |
| # 'bn_multi_00779.wav', | |
| 'en_UK_apope.wav', | |
| 'en_US_vctk_p345.wav', | |
| 'it_IT_mls_6744.wav', | |
| 'en_US_vctk_p347.wav', | |
| 'en_US_m-ailabs_mary_ann.wav', | |
| 'en_US_m-ailabs_elliot_miller.wav', | |
| 'en_US_vctk_p279.wav', | |
| 'ru_RU_multi_nikolaev.wav', | |
| 'bn_multi_4811.wav', | |
| 'tn_ZA_google-nwu_7693.wav', | |
| 'bn_multi_01701.wav', | |
| 'en_US_vctk_p262.wav', | |
| # 'en_US_vctk_p266.wav', | |
| 'en_US_vctk_p243.wav', | |
| 'en_US_vctk_p297.wav', | |
| 'en_US_vctk_p278.wav', | |
| 'jv_ID_google-gmu_02059.wav', | |
| 'en_US_vctk_p231.wav', | |
| 'te_IN_cmu-indic_kpn.wav', | |
| 'en_US_vctk_p250.wav', | |
| 'it_IT_mls_4974.wav', | |
| 'en_US_cmu_arctic_awbrms.wav', | |
| # 'en_US_vctk_p263.wav', | |
| 'nl_femal.wav', | |
| 'tn_ZA_google-nwu_6116.wav', | |
| 'jv_ID_google-gmu_06383.wav', | |
| 'en_US_vctk_p225.wav', | |
| 'en_US_vctk_p228.wav', | |
| 'it_IT_mls_277.wav', | |
| 'tn_ZA_google-nwu_7866.wav', | |
| 'en_US_vctk_p300.wav', | |
| 'ne_NP_ne-google_0649.wav', | |
| 'es_ES_carlfm.wav', | |
| 'jv_ID_google-gmu_06510.wav', | |
| 'de_DE_m-ailabs_rebecca_braunert_plunkett.wav', | |
| 'en_US_vctk_p340.wav', | |
| 'en_US_cmu_arctic_gka.wav', | |
| 'ne_NP_ne-google_2027.wav', | |
| 'jv_ID_google-gmu_09724.wav', | |
| 'en_US_vctk_p361.wav', | |
| 'ne_NP_ne-google_6834.wav', | |
| 'jv_ID_google-gmu_02326.wav', | |
| 'fr_FR_m-ailabs_zeckou.wav', | |
| 'tn_ZA_google-nwu_1932.wav', | |
| # 'female-20-happy.wav', | |
| 'tn_ZA_google-nwu_1483.wav', | |
| 'de_DE_thorsten-emotion_amused.wav', | |
| 'ru_RU_multi_minaev.wav', | |
| 'sw_lanfrica.wav', | |
| 'en_US_vctk_p271.wav', | |
| 'tn_ZA_google-nwu_0441.wav', | |
| 'it_IT_mls_6001.wav', | |
| 'en_US_vctk_p305.wav', | |
| 'it_IT_mls_8828.wav', | |
| 'jv_ID_google-gmu_08002.wav', | |
| 'it_IT_mls_2033.wav', | |
| 'tn_ZA_google-nwu_3629.wav', | |
| 'it_IT_mls_6348.wav', | |
| 'en_US_cmu_arctic_axb.wav', | |
| 'it_IT_mls_8181.wav', | |
| 'en_US_vctk_p230.wav', | |
| 'af_ZA_google-nwu_7214.wav', | |
| 'nl_nathalie.wav', | |
| 'it_IT_mls_8207.wav', | |
| 'ko_KO_kss.wav', | |
| 'af_ZA_google-nwu_6590.wav', | |
| 'jv_ID_google-gmu_00264.wav', | |
| 'tn_ZA_google-nwu_6234.wav', | |
| 'jv_ID_google-gmu_05522.wav', | |
| 'en_US_cmu_arctic_lnh.wav', | |
| 'en_US_vctk_p272.wav', | |
| 'en_US_cmu_arctic_slp.wav', | |
| 'en_US_vctk_p299.wav', | |
| 'en_US_hifi-tts_9017.wav', | |
| 'it_IT_mls_4998.wav', | |
| 'it_IT_mls_6299.wav', | |
| 'en_US_cmu_arctic_rxr.wav', | |
| 'female-46-neutral.wav', | |
| 'jv_ID_google-gmu_01392.wav', | |
| 'tn_ZA_google-nwu_8512.wav', | |
| 'en_US_vctk_p244.wav', | |
| # 'bn_multi_3108.wav', | |
| # 'it_IT_mls_7405.wav', | |
| # 'bn_multi_3713.wav', | |
| # 'yo_openbible.wav', | |
| # 'jv_ID_google-gmu_01932.wav', | |
| 'en_US_vctk_p270.wav', | |
| 'tn_ZA_google-nwu_6459.wav', | |
| 'bn_multi_4046.wav', | |
| 'en_US_vctk_p288.wav', | |
| 'en_US_vctk_p251.wav', | |
| 'es_ES_m-ailabs_tux.wav', | |
| 'tn_ZA_google-nwu_6206.wav', | |
| 'bn_multi_9169.wav', | |
| # 'en_US_vctk_p293.wav', | |
| # 'en_US_vctk_p255.wav', | |
| 'af_ZA_google-nwu_8963.wav', | |
| # 'en_US_vctk_p265.wav', | |
| 'gu_IN_cmu-indic_cmu_indic_guj_ad.wav', | |
| 'jv_ID_google-gmu_07335.wav', | |
| 'en_US_vctk_p323.wav', | |
| 'en_US_vctk_p281.wav', | |
| 'en_US_cmu_arctic_bdl.wav', | |
| 'en_US_m-ailabs_judy_bieber.wav', | |
| 'it_IT_mls_10446.wav', | |
| 'en_US_vctk_p261.wav', | |
| 'en_US_vctk_p292.wav', | |
| 'te_IN_cmu-indic_ss.wav', | |
| 'en_US_vctk_p311.wav', | |
| 'it_IT_mls_12428.wav', | |
| 'en_US_cmu_arctic_aup.wav', | |
| 'jv_ID_google-gmu_04679.wav', | |
| 'it_IT_mls_4971.wav', | |
| 'en_US_cmu_arctic_ljm.wav', | |
| 'fa_haaniye.wav', | |
| 'en_US_vctk_p339.wav', | |
| 'tn_ZA_google-nwu_7896.wav', | |
| 'en_US_vctk_p253.wav', | |
| 'it_IT_mls_5421.wav', | |
| # 'ne_NP_ne-google_0546.wav', | |
| 'vi_VN_vais1000.wav', | |
| 'en_US_vctk_p229.wav', | |
| 'en_US_vctk_p254.wav', | |
| 'en_US_vctk_p258.wav', | |
| 'it_IT_mls_7936.wav', | |
| 'en_US_vctk_p301.wav', | |
| 'tn_ZA_google-nwu_0045.wav', | |
| 'it_IT_mls_659.wav', | |
| 'tn_ZA_google-nwu_7674.wav', | |
| 'it_IT_mls_12804.wav', | |
| 'el_GR_rapunzelina.wav', | |
| 'en_US_hifi-tts_6097.wav', | |
| 'en_US_vctk_p257.wav', | |
| 'jv_ID_google-gmu_07875.wav', | |
| 'it_IT_mls_1157.wav', | |
| 'it_IT_mls_643.wav', | |
| 'en_US_vctk_p304.wav', | |
| 'ru_RU_multi_hajdurova.wav', | |
| 'it_IT_mls_8461.wav', | |
| 'bn_multi_3958.wav', | |
| 'it_IT_mls_1989.wav', | |
| 'en_US_vctk_p249.wav', | |
| # 'bn_multi_0834.wav', | |
| 'en_US_vctk_p307.wav', | |
| 'es_ES_m-ailabs_karen_savage.wav', | |
| 'fr_FR_m-ailabs_bernard.wav', | |
| 'en_US_vctk_p252.wav', | |
| 'en_US_cmu_arctic_jmk.wav', | |
| 'en_US_vctk_p333.wav', | |
| 'tn_ZA_google-nwu_4506.wav', | |
| 'ne_NP_ne-google_0283.wav', | |
| 'de_DE_m-ailabs_karlsson.wav', | |
| 'en_US_cmu_arctic_awb.wav', | |
| 'en_US_vctk_p246.wav', | |
| 'en_US_cmu_arctic_clb.wav', | |
| 'en_US_vctk_p364.wav', | |
| 'nl_flemishguy.wav', | |
| 'en_US_vctk_p276.wav', # y | |
| # 'en_US_vctk_p274.wav', | |
| 'fr_FR_m-ailabs_gilles_g_le_blanc.wav', | |
| 'it_IT_mls_7444.wav', | |
| 'style_o22050.wav', | |
| 'en_US_vctk_s5.wav', | |
| 'en_US_vctk_p268.wav', | |
| 'it_IT_mls_6807.wav', | |
| 'it_IT_mls_2019.wav', | |
| 'male-60-angry.wav', | |
| 'af_ZA_google-nwu_8924.wav', | |
| 'en_US_vctk_p374.wav', | |
| 'en_US_vctk_p363.wav', | |
| 'it_IT_mls_644.wav', | |
| 'ne_NP_ne-google_3614.wav', | |
| 'en_US_vctk_p241.wav', | |
| 'ne_NP_ne-google_3154.wav', | |
| 'en_US_vctk_p234.wav', | |
| 'it_IT_mls_8384.wav', | |
| 'fr_FR_m-ailabs_ezwa.wav', | |
| 'it_IT_mls_5010.wav', | |
| 'en_US_vctk_p351.wav', | |
| 'en_US_cmu_arctic_eey.wav', | |
| 'jv_ID_google-gmu_04285.wav', | |
| 'jv_ID_google-gmu_06941.wav', | |
| 'hu_HU_diana-majlinger.wav', | |
| 'tn_ZA_google-nwu_2839.wav', | |
| 'bn_multi_03042.wav', | |
| 'tn_ZA_google-nwu_5628.wav', | |
| 'it_IT_mls_4649.wav', | |
| 'af_ZA_google-nwu_7130.wav', | |
| 'en_US_cmu_arctic_slt.wav', | |
| 'jv_ID_google-gmu_04175.wav', | |
| 'gu_IN_cmu-indic_cmu_indic_guj_kt.wav', | |
| 'jv_ID_google-gmu_00027.wav', | |
| 'jv_ID_google-gmu_02884.wav', | |
| 'en_US_vctk_p360.wav', | |
| 'en_US_vctk_p334.wav', | |
| 'male-27-sad.wav', | |
| 'tn_ZA_google-nwu_1498.wav', | |
| 'fi_FI_harri-tapani-ylilammi.wav', | |
| 'bn_multi_rm.wav', | |
| 'ne_NP_ne-google_2139.wav', | |
| 'pl_PL_m-ailabs_piotr_nater.wav', | |
| 'fr_FR_siwis.wav', | |
| 'nl_bart-de-leeuw.wav', | |
| 'jv_ID_google-gmu_04715.wav', | |
| 'en_US_vctk_p283.wav', | |
| 'en_US_vctk_p314.wav', | |
| 'en_US_vctk_p335.wav', | |
| 'jv_ID_google-gmu_07765.wav', | |
| 'en_US_vctk_p273.wav' | |
| ] | |
| VOICES = [t[:-4] for t in VOICES] # crop .wav for visuals in gr.DropDown | |
| _tts = StyleTTS2().to('cpu') | |
| def only_greek_or_only_latin(text, lang='grc'): | |
| ''' | |
| str: The converted string in the specified target script. | |
| Characters not found in any mapping are preserved as is. | |
| Latin accented characters in the input (e.g., 'É', 'ü') will | |
| be preserved in their lowercase form (e.g., 'é', 'ü') if | |
| converting to Latin. | |
| ''' | |
| # --- Mapping Dictionaries --- | |
| # Keys are in lowercase as input text is case-folded. | |
| # If the output needs to maintain original casing, additional logic is required. | |
| latin_to_greek_map = { | |
| 'a': 'α', 'b': 'β', 'g': 'γ', 'd': 'δ', 'e': 'ε', | |
| 'ch': 'τσο', # Example of a multi-character Latin sequence | |
| 'z': 'ζ', 'h': 'χ', 'i': 'ι', 'k': 'κ', 'l': 'λ', | |
| 'm': 'μ', 'n': 'ν', 'x': 'ξ', 'o': 'ο', 'p': 'π', | |
| 'v': 'β', 'sc': 'σκ', 'r': 'ρ', 's': 'σ', 't': 'τ', | |
| 'u': 'ου', 'f': 'φ', 'c': 'σ', 'w': 'β', 'y': 'γ', | |
| } | |
| greek_to_latin_map = { | |
| 'ου': 'ou', # Prioritize common diphthongs/digraphs | |
| 'α': 'a', 'β': 'v', 'γ': 'g', 'δ': 'd', 'ε': 'e', | |
| 'ζ': 'z', 'η': 'i', 'θ': 'th', 'ι': 'i', 'κ': 'k', | |
| 'λ': 'l', 'μ': 'm', 'ν': 'n', 'ξ': 'x', 'ο': 'o', | |
| 'π': 'p', 'ρ': 'r', 'σ': 's', 'τ': 't', 'υ': 'y', # 'y' is a common transliteration for upsilon | |
| 'φ': 'f', 'χ': 'ch', 'ψ': 'ps', 'ω': 'o', | |
| 'ς': 's', # Final sigma | |
| } | |
| cyrillic_to_latin_map = { | |
| 'а': 'a', 'б': 'b', 'в': 'v', 'г': 'g', 'д': 'd', 'е': 'e', 'ё': 'yo', 'ж': 'zh', | |
| 'з': 'z', 'и': 'i', 'й': 'y', 'к': 'k', 'л': 'l', 'м': 'm', 'н': 'n', 'о': 'o', | |
| 'п': 'p', 'р': 'r', 'с': 's', 'т': 't', 'у': 'u', 'ф': 'f', 'х': 'kh', 'ц': 'ts', | |
| 'ч': 'ch', 'ш': 'sh', 'щ': 'shch', 'ъ': '', 'ы': 'y', 'ь': '', 'э': 'e', 'ю': 'yu', | |
| 'я': 'ya', | |
| } | |
| # Direct Cyrillic to Greek mapping based on phonetic similarity. | |
| # These are approximations and may not be universally accepted transliterations. | |
| cyrillic_to_greek_map = { | |
| 'а': 'α', 'б': 'β', 'в': 'β', 'г': 'γ', 'д': 'δ', 'е': 'ε', 'ё': 'ιο', 'ж': 'ζ', | |
| 'з': 'ζ', 'и': 'ι', 'й': 'ι', 'κ': 'κ', 'λ': 'λ', 'м': 'μ', 'н': 'ν', 'о': 'ο', | |
| 'π': 'π', 'ρ': 'ρ', 'σ': 'σ', 'τ': 'τ', 'у': 'ου', 'ф': 'φ', 'х': 'χ', 'ц': 'τσ', | |
| 'ч': 'τσ', # or τζ depending on desired sound | |
| 'ш': 'σ', 'щ': 'σ', # approximations | |
| 'ъ': '', 'ы': 'ι', 'ь': '', 'э': 'ε', 'ю': 'ιου', | |
| 'я': 'ια', | |
| } | |
| # Convert the input text to lowercase, preserving accents for Latin characters. | |
| # casefold() is used for more robust caseless matching across Unicode characters. | |
| lowercased_text = text.lower() #casefold() | |
| output_chars = [] | |
| current_index = 0 | |
| if lang == 'grc': | |
| # Combine all relevant maps for direct lookup to Greek | |
| conversion_map = {**latin_to_greek_map, **cyrillic_to_greek_map} | |
| # Sort keys by length in reverse order to handle multi-character sequences first | |
| sorted_source_keys = sorted( | |
| list(latin_to_greek_map.keys()) + list(cyrillic_to_greek_map.keys()), | |
| key=len, | |
| reverse=True | |
| ) | |
| while current_index < len(lowercased_text): | |
| found_conversion = False | |
| for key in sorted_source_keys: | |
| if lowercased_text.startswith(key, current_index): | |
| output_chars.append(conversion_map[key]) | |
| current_index += len(key) | |
| found_conversion = True | |
| break | |
| if not found_conversion: | |
| # If no specific mapping found, append the character as is. | |
| # This handles unmapped characters and already Greek characters. | |
| output_chars.append(lowercased_text[current_index]) | |
| current_index += 1 | |
| return ''.join(output_chars) | |
| else: # Default to 'lat' conversion | |
| # Combine Greek to Latin and Cyrillic to Latin maps. | |
| # Cyrillic map keys will take precedence in case of overlap if defined after Greek. | |
| combined_to_latin_map = {**greek_to_latin_map, **cyrillic_to_latin_map} | |
| # Sort all relevant source keys by length in reverse for replacement | |
| sorted_source_keys = sorted( | |
| list(greek_to_latin_map.keys()) + list(cyrillic_to_latin_map.keys()), | |
| key=len, | |
| reverse=True | |
| ) | |
| while current_index < len(lowercased_text): | |
| found_conversion = False | |
| for key in sorted_source_keys: | |
| if lowercased_text.startswith(key, current_index): | |
| latin_equivalent = combined_to_latin_map[key] | |
| # Strip accents ONLY if the source character was from the Greek map. | |
| # This preserves accents on original Latin characters (like 'é') | |
| # and allows for intentional accent stripping from Greek transliterations. | |
| if key in greek_to_latin_map: | |
| normalized_latin = unicodedata.normalize('NFD', latin_equivalent) | |
| stripped_latin = ''.join(c for c in normalized_latin if not unicodedata.combining(c)) | |
| output_chars.append(stripped_latin) | |
| else: | |
| output_chars.append(latin_equivalent) | |
| current_index += len(key) | |
| found_conversion = True | |
| break | |
| if not found_conversion: | |
| # If no conversion happened from Greek or Cyrillic, append the character as is. | |
| # This preserves existing Latin characters (including accented ones from input), | |
| # numbers, punctuation, and other symbols. | |
| output_chars.append(lowercased_text[current_index]) | |
| current_index += 1 | |
| return ''.join(output_chars) | |
| def _stylett2(text='Hallov worlds Far over the', | |
| ref_s='wav/af_ZA_google-nwu_0184.wav'): | |
| if text and text.strip(): | |
| text = only_greek_or_only_latin(text, lang='eng') | |
| speech_audio = _tts.inference(text, | |
| ref_s=re_s)[0, 0, :].numpy() # 24 Khz | |
| if speech_audio.shape[0] > 10: | |
| speech_audio = audresample.resample(signal=speech_audio.astype(np.float32), | |
| original_rate=24000, | |
| target_rate=16000)[0, :] # 16 KHz | |
| return speech_audio | |
| import gradio as gr | |
| # Dummy functions to make the code runnable for demonstration | |
| def audionar_tts(text, choice, soundscape, kv): | |
| # This function would generate an audio file and return its path | |
| return "dummy_audio.wav" | |
| def recognize(audio_input_path): | |
| # This function would analyze the audio and return results | |
| return "30", "Male", {"Angry": 0.9} | |
| # Assuming these are defined elsewhere in the user's code | |
| language_names = ["English", "Spanish"] | |
| VOICES = ["Voice 1", "Voice 2"] | |
| with gr.Blocks(theme='huggingface') as demo: | |
| tts_file = gr.State(value=None) | |
| audio_examples_state = gr.State( | |
| value=[ | |
| ["wav/female-46-neutral.wav"], | |
| ["wav/female-20-happy.wav"], | |
| ["wav/male-60-angry.wav"], | |
| ["wav/male-27-sad.wav"], | |
| ] | |
| ) | |
| with gr.Tab(label="TTS"): | |
| with gr.Row(): | |
| text_input = gr.Textbox( | |
| label="Type text for TTS:", | |
| placeholder="Type Text for TTS", | |
| lines=4, | |
| value="Farover the misty mountains cold too dungeons deep and caverns old.", | |
| ) | |
| choice_dropdown = gr.Dropdown( | |
| choices=language_names + VOICES, | |
| label="Select Voice or Language", | |
| value=VOICES[0] | |
| ) | |
| soundscape_input = gr.Textbox( | |
| lines=1, | |
| value="frogs", | |
| label="AudioGen Txt" | |
| ) | |
| kv_input = gr.Number( | |
| label="kv Period", | |
| value=24, | |
| ) | |
| generate_button = gr.Button("Generate Audio", variant="primary") | |
| output_audio = gr.Audio(label="TTS Output") | |
| def generate_and_update_state(text, choice, soundscape, kv, current_examples): | |
| audio_path = audionar_tts(text, choice, soundscape, kv) | |
| updated_examples = current_examples + [[audio_path]] | |
| return audio_path, updated_examples | |
| generate_button.click( | |
| fn=generate_and_update_state, | |
| inputs=[text_input, choice_dropdown, soundscape_input, kv_input, audio_examples_state], | |
| outputs=[output_audio, audio_examples_state] | |
| ) | |
| with gr.Tab(label="Speech Analysis"): | |
| with gr.Row(): | |
| with gr.Column(): | |
| input_audio_analysis = gr.Audio( | |
| sources=["upload", "microphone"], | |
| type="filepath", | |
| label="Audio input", | |
| min_length=0.025, | |
| ) | |
| audio_examples = gr.Examples( | |
| examples=[], # Initialize with an empty list | |
| inputs=[input_audio_analysis], | |
| label="Examples from CREMA-D, ODbL v1.0 license", | |
| ) | |
| gr.Markdown("Only the first two seconds of the audio will be processed.") | |
| submit_btn = gr.Button(value="Submit", variant="primary") | |
| with gr.Column(): | |
| output_age = gr.Textbox(label="Age") | |
| output_gender = gr.Label(label="Gender") | |
| output_expression = gr.Image(label="Expression") | |
| outputs = [output_age, output_gender, output_expression] | |
| # Fix: This function should not update gr.Examples directly. | |
| # Instead, it should just return the updated examples list. | |
| # The `demo.load` event will handle the update. | |
| def load_examples_from_state(examples_list): | |
| return gr.Examples.update(examples=examples_list) | |
| demo.load( | |
| fn=load_examples_from_state, | |
| inputs=[audio_examples_state], | |
| outputs=[audio_examples], | |
| queue=False, | |
| ) | |
| submit_btn.click(recognize, input_audio_analysis, outputs) | |
| demo.launch(debug=True) |