# -*- coding: utf-8 -*- import typing import types # fusion of forward() of Wav2Vec2 import gradio as gr import matplotlib.pyplot as plt import numpy as np import os import torch import torch.nn as nn from transformers import Wav2Vec2Processor from transformers.models.wav2vec2.modeling_wav2vec2 import Wav2Vec2Model from transformers.models.wav2vec2.modeling_wav2vec2 import Wav2Vec2PreTrainedModel import audiofile from tts import StyleTTS2 import audresample import json import re import unicodedata import textwrap import nltk from num2words import num2words from num2word_greek.numbers2words import convert_numbers from audionar import VitsModel, VitsTokenizer from audiocraft import AudioGen audiogen = AudioGen().eval().to('cpu') nltk.download('punkt', download_dir='./') nltk.download('punkt_tab', download_dir='./') nltk.data.path.append('.') device = 'cpu' def fix_vocals(text, lang='ron'): # Longer phrases should come before shorter ones to prevent partial matches. ron_replacements = { 'ţ': 'ț', 'ț': 'ts', 'î': 'u', 'â': 'a', 'ş': 's', 'w': 'oui', 'k': 'c', 'l': 'll', # Math symbols 'sqrt': ' rădăcina pătrată din ', '^': ' la puterea ', '+': ' plus ', ' - ': ' minus ', # only replace if standalone so to not say minus if is a-b-c '*': ' ori ', # times '/': ' împărțit la ', # divided by '=': ' egal cu ', # equals 'pi': ' pi ', '<': ' mai mic decât ', '>': ' mai mare decât', '%': ' la sută ', # percent (from previous) '(': ' paranteză deschisă ', ')': ' paranteză închisă ', '[': ' paranteză pătrată deschisă ', ']': ' paranteză pătrată închisă ', '{': ' acoladă deschisă ', '}': ' acoladă închisă ', '≠': ' nu este egal cu ', '≤': ' mai mic sau egal cu ', '≥': ' mai mare sau egal cu ', '≈': ' aproximativ ', '∞': ' infinit ', '€': ' euro ', '$': ' dolar ', '£': ' liră ', '&': ' și ', # and '@': ' la ', # at '#': ' diez ', # hash '∑': ' sumă ', '∫': ' integrală ', '√': ' rădăcina pătrată a ', # more generic square root } eng_replacements = { 'wik': 'weaky', 'sh': 'ss', 'ch': 'ttss', 'oo': 'oeo', # Math symbols for English 'sqrt': ' square root of ', '^': ' to the power of ', '+': ' plus ', ' - ': ' minus ', '*': ' times ', ' / ': ' divided by ', '=': ' equals ', 'pi': ' pi ', '<': ' less than ', '>': ' greater than ', # Additional common math symbols from previous list '%': ' percent ', '(': ' open parenthesis ', ')': ' close parenthesis ', '[': ' open bracket ', ']': ' close bracket ', '{': ' open curly brace ', '}': ' close curly brace ', '∑': ' sum ', '∫': ' integral ', '√': ' square root of ', '≠': ' not equals ', '≤': ' less than or equals ', '≥': ' greater than or equals ', '≈': ' approximately ', '∞': ' infinity ', '€': ' euro ', '$': ' dollar ', '£': ' pound ', '&': ' and ', '@': ' at ', '#': ' hash ', } serbian_replacements = { 'rn': 'rrn', 'ć': 'č', 'c': 'č', 'đ': 'd', 'j': 'i', 'l': 'lll', 'w': 'v', # https://huggingface.co/facebook/mms-tts-rmc-script_latin 'sqrt': 'kvadratni koren iz', '^': ' na stepen ', '+': ' plus ', ' - ': ' minus ', '*': ' puta ', ' / ': ' podeljeno sa ', '=': ' jednako ', 'pi': ' pi ', '<': ' manje od ', '>': ' veće od ', '%': ' procenat ', '(': ' otvorena zagrada ', ')': ' zatvorena zagrada ', '[': ' otvorena uglasta zagrada ', ']': ' zatvorena uglasta zagrada ', '{': ' otvorena vitičasta zagrada ', '}': ' zatvorena vitičasta zagrada ', '∑': ' suma ', '∫': ' integral ', '√': ' kvadratni koren ', '≠': ' nije jednako ', '≤': ' manje ili jednako od ', '≥': ' veće ili jednako od ', '≈': ' približno ', '∞': ' beskonačnost ', '€': ' evro ', '$': ' dolar ', '£': ' funta ', '&': ' i ', '@': ' et ', '#': ' taraba ', # Others # 'rn': 'rrn', # 'ć': 'č', # 'c': 'č', # 'đ': 'd', # 'l': 'le', # 'ij': 'i', # 'ji': 'i', # 'j': 'i', # 'služ': 'sloooozz', # 'službeno' # 'suver': 'siuveeerra', # 'suverena' # 'država': 'dirrezav', # 'država' # 'iči': 'ici', # 'Graniči' # 's ': 'se', # a s with space # 'q': 'ku', # 'w': 'aou', # 'z': 's', # "š": "s", # 'th': 'ta', # 'v': 'vv', # "ć": "č", # "đ": "ď", # "lj": "ľ", # "nj": "ň", # "ž": "z", # "c": "č" } deu_replacements = { 'sch': 'sh', 'ch': 'kh', 'ie': 'ee', 'ei': 'ai', 'ä': 'ae', 'ö': 'oe', 'ü': 'ue', 'ß': 'ss', # Math symbols for German 'sqrt': ' Quadratwurzel aus ', '^': ' hoch ', '+': ' plus ', ' - ': ' minus ', '*': ' mal ', ' / ': ' geteilt durch ', '=': ' gleich ', 'pi': ' pi ', '<': ' kleiner als ', '>': ' größer als', # Additional common math symbols from previous list '%': ' prozent ', '(': ' Klammer auf ', ')': ' Klammer zu ', '[': ' eckige Klammer auf ', ']': ' eckige Klammer zu ', '{': ' geschweifte Klammer auf ', '}': ' geschweifte Klammer zu ', '∑': ' Summe ', '∫': ' Integral ', '√': ' Quadratwurzel ', '≠': ' ungleich ', '≤': ' kleiner oder gleich ', '≥': ' größer oder gleich ', '≈': ' ungefähr ', '∞': ' unendlich ', '€': ' euro ', '$': ' dollar ', '£': ' pfund ', '&': ' und ', '@': ' at ', # 'Klammeraffe' is also common but 'at' is simpler '#': ' raute ', } fra_replacements = { # French specific phonetic replacements (add as needed) # e.g., 'ç': 's', 'é': 'e', etc. 'w': 'v', # Math symbols for French 'sqrt': ' racine carrée de ', '^': ' à la puissance ', '+': ' plus ', ' - ': ' moins ', # tiré ; '*': ' fois ', ' / ': ' divisé par ', '=': ' égale ', 'pi': ' pi ', '<': ' inférieur à ', '>': ' supérieur à ', # Add more common math symbols as needed for French '%': ' pour cent ', '(': ' parenthèse ouverte ', ')': ' parenthèse fermée ', '[': ' crochet ouvert ', ']': ' crochet fermé ', '{': ' accolade ouverte ', '}': ' accolade fermée ', '∑': ' somme ', '∫': ' intégrale ', '√': ' racine carrée ', '≠': ' n\'égale pas ', '≤': ' inférieur ou égal à ', '≥': ' supérieur ou égal à ', '≈': ' approximativement ', '∞': ' infini ', '€': ' euro ', '$': ' dollar ', '£': ' livre ', '&': ' et ', '@': ' arobase ', '#': ' dièse ', } hun_replacements = { # Hungarian specific phonetic replacements (add as needed) # e.g., 'á': 'a', 'é': 'e', etc. 'ch': 'ts', 'cs': 'tz', 'g': 'gk', 'w': 'v', 'z': 'zz', # Math symbols for Hungarian 'sqrt': ' négyzetgyök ', '^': ' hatvány ', '+': ' plusz ', ' - ': ' mínusz ', '*': ' szorozva ', ' / ': ' osztva ', '=': ' egyenlő ', 'pi': ' pi ', '<': ' kisebb mint ', '>': ' nagyobb mint ', # Add more common math symbols as needed for Hungarian '%': ' százalék ', '(': ' nyitó zárójel ', ')': ' záró zárójel ', '[': ' nyitó szögletes zárójel ', ']': ' záró szögletes zárójel ', '{': ' nyitó kapcsos zárójel ', '}': ' záró kapcsos zárójel ', '∑': ' szumma ', '∫': ' integrál ', '√': ' négyzetgyök ', '≠': ' nem egyenlő ', '≤': ' kisebb vagy egyenlő ', '≥': ' nagyobb vagy egyenlő ', '≈': ' körülbelül ', '∞': ' végtelen ', '€': ' euró ', '$': ' dollár ', '£': ' font ', '&': ' és ', '@': ' kukac ', '#': ' kettőskereszt ', } grc_replacements = { # Ancient Greek specific phonetic replacements (add as needed) # These are more about transliterating Greek letters if they are in the input text. # Math symbols for Ancient Greek (literal translations) 'sqrt': ' τετραγωνικὴ ῥίζα ', '^': ' εἰς τὴν δύναμιν ', '+': ' σὺν ', ' - ': ' χωρὶς ', '*': ' πολλάκις ', ' / ': ' διαιρέω ', '=': ' ἴσον ', 'pi': ' πῖ ', '<': ' ἔλαττον ', '>': ' μεῖζον ', # Add more common math symbols as needed for Ancient Greek '%': ' τοῖς ἑκατόν ', # tois hekaton - 'of the hundred' '(': ' ἀνοικτὴ παρένθεσις ', ')': ' κλειστὴ παρένθεσις ', '[': ' ἀνοικτὴ ἀγκύλη ', ']': ' κλειστὴ ἀγκύλη ', '{': ' ἀνοικτὴ σγουρὴ ἀγκύλη ', '}': ' κλειστὴ σγουρὴ ἀγκύλη ', '∑': ' ἄθροισμα ', '∫': ' ὁλοκλήρωμα ', '√': ' τετραγωνικὴ ῥίζα ', '≠': ' οὐκ ἴσον ', '≤': ' ἔλαττον ἢ ἴσον ', '≥': ' μεῖζον ἢ ἴσον ', '≈': ' περίπου ', '∞': ' ἄπειρον ', '€': ' εὐρώ ', '$': ' δολάριον ', '£': ' λίρα ', '&': ' καὶ ', '@': ' ἀτ ', # at '#': ' δίεση ', # hash } # Select the appropriate replacement dictionary based on the language replacements_map = { 'grc': grc_replacements, 'ron': ron_replacements, 'eng': eng_replacements, 'deu': deu_replacements, 'fra': fra_replacements, 'hun': hun_replacements, 'rmc-script_latin': serbian_replacements, } current_replacements = replacements_map.get(lang) if current_replacements: # Sort replacements by length of the key in descending order. # This is crucial for correctly replacing multi-character strings (like 'sqrt', 'sch') # before their shorter substrings ('s', 'ch', 'q', 'r', 't'). sorted_replacements = sorted(current_replacements.items(), key=lambda item: len(item[0]), reverse=True) for old, new in sorted_replacements: text = text.replace(old, new) return text else: # If the language is not supported, return the original text print(f"Warning: Language '{lang}' not supported for text replacement. Returning original text.") return text def _num2words(text='01234', lang=None): if lang == 'grc': return convert_numbers(text) return num2words(text, lang=lang) # HAS TO BE kwarg lang=lang def transliterate_number(number_string, lang=None): if lang == 'rmc-script_latin': lang = 'sr' exponential_pronoun = ' puta deset na stepen od ' comma = ' tačka ' elif lang == 'ron': lang = 'ro' exponential_pronoun = ' tízszer a erejéig ' comma = ' virgulă ' elif lang == 'hun': lang = 'hu' exponential_pronoun = ' tízszer a erejéig ' comma = ' virgula ' elif lang == 'deu': exponential_pronoun = ' mal zehn hoch ' comma = ' komma ' elif lang == 'fra': lang = 'fr' exponential_pronoun = ' puissance ' comma = 'virgule' elif lang == 'grc': exponential_pronoun = ' εις την δυναμην του ' comma = 'κομμα' else: lang = lang[:2] exponential_pronoun = ' times ten to the power of ' comma = ' point ' def replace_number(match): prefix = match.group(1) or "" number_part = match.group(2) suffix = match.group(5) or "" try: if 'e' in number_part.lower(): base, exponent = number_part.lower().split('e') words = _num2words(base, lang=lang) + exponential_pronoun + _num2words(exponent, lang=lang) elif '.' in number_part: integer_part, decimal_part = number_part.split('.') words = _num2words(integer_part, lang=lang) + comma + " ".join( [_num2words(digit, lang=lang) for digit in decimal_part]) else: words = _num2words(number_part, lang=lang) return prefix + words + suffix except ValueError: return match.group(0) # Return original if conversion fails pattern = r'([^\d]*)(\d+(\.\d+)?([Ee][+-]?\d+)?)([^\d]*)' return re.sub(pattern, replace_number, number_string) language_names = ['Ancient greek', 'English', 'Deutsch', 'French', 'Hungarian', 'Romanian', 'Serbian (Approx.)'] def audionar_tts(text=None, lang='romanian', soundscape='', cache_lim=24): # https://huggingface.co/dkounadis/artificial-styletts2/blob/main/msinference.py lang_map = { 'ancient greek': 'grc', 'english': 'eng', 'deutsch': 'deu', 'french': 'fra', 'hungarian': 'hun', 'romanian': 'ron', 'serbian (approx.)': 'rmc-script_latin', } if text and text.strip(): if lang not in language_names: speech_audio = _styletts2(text=text, # Eng. ref_s='wav/' + lang + '.wav') else: # VITS lang_code = lang_map.get(lang.lower(), lang.lower().split()[0].strip()) global cached_lang_code, cached_net_g, cached_tokenizer if 'cached_lang_code' not in globals() or cached_lang_code != lang_code: cached_lang_code = lang_code cached_net_g = VitsModel.from_pretrained(f'facebook/mms-tts-{lang_code}').eval() cached_tokenizer = VitsTokenizer.from_pretrained(f'facebook/mms-tts-{lang_code}') net_g = cached_net_g tokenizer = cached_tokenizer text = only_greek_or_only_latin(text, lang=lang_code) text = transliterate_number(text, lang=lang_code) text = fix_vocals(text, lang=lang_code) sentences = textwrap.wrap(text, width=439) total_audio_parts = [] for sentence in sentences: inputs = cached_tokenizer(sentence, return_tensors="pt") with torch.no_grad(): audio_part = cached_net_g( input_ids=inputs.input_ids.to(device), attention_mask=inputs.attention_mask.to(device), lang_code=lang_code, )[0, :] total_audio_parts.append(audio_part) speech_audio = torch.cat(total_audio_parts).cpu().numpy() # AudioGen if soundscape and soundscape.strip(): speech_duration_secs = len(speech_audio) / 16000 if speech_audio is not None else 0 target_duration = max(speech_duration_secs + 0.74, 2.0) background_audio = audiogen.generate( soundscape, duration=target_duration, cache_lim=max(4, int(cache_lim)) # at least allow 10 A/R stEps ).numpy() if speech_audio is not None: len_speech = len(speech_audio) len_background = len(background_audio) if len_background > len_speech: padding = np.zeros(len_background - len_speech, dtype=np.float32) speech_audio = np.concatenate([speech_audio, padding]) elif len_speech > len_background: padding = np.zeros(len_speech - len_background, dtype=np.float32) background_audio = np.concatenate([background_audio, padding]) speech_audio_stereo = speech_audio[None, :] background_audio_stereo = background_audio[None, :] final_audio = np.concatenate([ 0.49 * speech_audio_stereo + 0.51 * background_audio_stereo, 0.51 * background_audio_stereo + 0.49 * speech_audio_stereo ], 0) else: final_audio = background_audio # If no soundscape, use the speech audio as is. elif speech_audio is not None: final_audio = speech_audio # If both inputs are empty, create a 2s silent audio file. if final_audio is None: final_audio = np.zeros(16000 * 2, dtype=np.float32) wavfile = '_vits_.wav' audiofile.write(wavfile, final_audio, 16000) return wavfile, wavfile # 2x file for [audio out & state to pass to the Emotion reco tAB] # -- EXPRESSIO device = 0 if torch.cuda.is_available() else "cpu" duration = 2 # limit processing of audio age_gender_model_name = "audeering/wav2vec2-large-robust-6-ft-age-gender" expression_model_name = "audeering/wav2vec2-large-robust-12-ft-emotion-msp-dim" class AgeGenderHead(nn.Module): r"""Age-gender model head.""" def __init__(self, config, num_labels): super().__init__() self.dense = nn.Linear(config.hidden_size, config.hidden_size) self.dropout = nn.Dropout(config.final_dropout) self.out_proj = nn.Linear(config.hidden_size, num_labels) def forward(self, features, **kwargs): x = features x = self.dropout(x) x = self.dense(x) x = torch.tanh(x) x = self.dropout(x) x = self.out_proj(x) return x class AgeGenderModel(Wav2Vec2PreTrainedModel): r"""Age-gender recognition model.""" def __init__(self, config): super().__init__(config) self.config = config self.wav2vec2 = Wav2Vec2Model(config) self.age = AgeGenderHead(config, 1) self.gender = AgeGenderHead(config, 3) self.init_weights() def forward( self, frozen_cnn7, ): hidden_states = self.wav2vec2(frozen_cnn7=frozen_cnn7) # runs only Transformer layers hidden_states = torch.mean(hidden_states, dim=1) logits_age = self.age(hidden_states) logits_gender = torch.softmax(self.gender(hidden_states), dim=1) return hidden_states, logits_age, logits_gender # AgeGenderModel.forward() is switched to accept computed frozen CNN7 features from ExpressioNmodel def _forward( self, frozen_cnn7=None, # CNN7 fetures of wav2vec2 calc. from CNN7 feature extractor (once) attention_mask=None): if attention_mask is not None: # compute reduced attention_mask corresponding to feature vectors attention_mask = self._get_feature_vector_attention_mask( frozen_cnn7.shape[1], attention_mask, add_adapter=False ) hidden_states, _ = self.wav2vec2.feature_projection(frozen_cnn7) hidden_states = self.wav2vec2.encoder( hidden_states, attention_mask=attention_mask, output_attentions=None, output_hidden_states=None, return_dict=None, )[0] return hidden_states def _forward_and_cnn7( self, input_values, attention_mask=None): frozen_cnn7 = self.wav2vec2.feature_extractor(input_values) frozen_cnn7 = frozen_cnn7.transpose(1, 2) if attention_mask is not None: # compute reduced attention_mask corresponding to feature vectors attention_mask = self.wav2vec2._get_feature_vector_attention_mask( frozen_cnn7.shape[1], attention_mask, add_adapter=False ) hidden_states, _ = self.wav2vec2.feature_projection(frozen_cnn7) # grad=True non frozen hidden_states = self.wav2vec2.encoder( hidden_states, attention_mask=attention_mask, output_attentions=None, output_hidden_states=None, return_dict=None, )[0] return hidden_states, frozen_cnn7 #feature_proj is trainable thus we have to access the frozen_cnn7 before projection layer class ExpressionHead(nn.Module): r"""Expression model head.""" def __init__(self, config): super().__init__() self.dense = nn.Linear(config.hidden_size, config.hidden_size) self.dropout = nn.Dropout(config.final_dropout) self.out_proj = nn.Linear(config.hidden_size, config.num_labels) def forward(self, features, **kwargs): x = features x = self.dropout(x) x = self.dense(x) x = torch.tanh(x) x = self.dropout(x) x = self.out_proj(x) return x class ExpressionModel(Wav2Vec2PreTrainedModel): r"""speech expression model.""" def __init__(self, config): super().__init__(config) self.config = config self.wav2vec2 = Wav2Vec2Model(config) self.classifier = ExpressionHead(config) self.init_weights() def forward(self, input_values): hidden_states, frozen_cnn7 = self.wav2vec2(input_values) hidden_states = torch.mean(hidden_states, dim=1) logits = self.classifier(hidden_states) return hidden_states, logits, frozen_cnn7 # Load models from hub age_gender_model = AgeGenderModel.from_pretrained(age_gender_model_name) expression_processor = Wav2Vec2Processor.from_pretrained(expression_model_name) expression_model = ExpressionModel.from_pretrained(expression_model_name) # Emotion Calc. CNN features age_gender_model.wav2vec2.forward = types.MethodType(_forward, age_gender_model) expression_model.wav2vec2.forward = types.MethodType(_forward_and_cnn7, expression_model) def process_func(x: np.ndarray, sampling_rate: int) -> typing.Tuple[str, dict, str]: # batch audio y = expression_processor(x, sampling_rate=sampling_rate) y = y['input_values'][0] y = y.reshape(1, -1) y = torch.from_numpy(y).to(device) # run through expression model with torch.no_grad(): _, logits_expression, frozen_cnn7 = expression_model(y) _, logits_age, logits_gender = age_gender_model(frozen_cnn7=frozen_cnn7) # Plot A/D/V values plot_expression(logits_expression[0, 0].item(), # implicit detach().cpu().numpy() logits_expression[0, 1].item(), logits_expression[0, 2].item()) expression_file = "expression.png" plt.savefig(expression_file) return ( f"{round(100 * logits_age[0, 0].item())} years", # age { "female": logits_gender[0, 0].item(), "male": logits_gender[0, 1].item(), "child": logits_gender[0, 2].item(), }, expression_file, ) def recognize(input_file): if input_file is None: raise gr.Error( "No audio file submitted! " "Please upload or record an audio file " "before submitting your request." ) signal, sampling_rate = audiofile.read(input_file, duration=duration) # Resample to sampling rate supported byu the models target_rate = 16000 signal = audresample.resample(signal, sampling_rate, target_rate) return process_func(signal, target_rate) def explode(data): """ Expands a 3D array by creating gaps between voxels. This function is used to create the visual separation between the voxels. """ shape_orig = np.array(data.shape) shape_new = shape_orig * 2 - 1 retval = np.zeros(shape_new, dtype=data.dtype) retval[::2, ::2, ::2] = data return retval def explode(data): """ Expands a 3D array by adding new voxels between existing ones. This is used to create the gaps in the 3D plot. """ shape = data.shape new_shape = (2 * shape[0] - 1, 2 * shape[1] - 1, 2 * shape[2] - 1) new_data = np.zeros(new_shape, dtype=data.dtype) new_data[::2, ::2, ::2] = data return new_data def plot_expression(arousal, dominance, valence): '''_h = cuda tensor (N_PIX, N_PIX, N_PIX)''' N_PIX = 5 _h = np.random.rand(N_PIX, N_PIX, N_PIX) * 1e-3 adv = np.array([arousal, .994 - dominance, valence]).clip(0, .99) arousal, dominance, valence = (adv * N_PIX).astype(np.int64) # find voxel _h[arousal, dominance, valence] = .22 filled = np.ones((N_PIX, N_PIX, N_PIX), dtype=bool) # upscale the above voxel image, leaving gaps filled_2 = explode(filled) # Shrink the gaps x, y, z = np.indices(np.array(filled_2.shape) + 1).astype(float) // 2 x[1::2, :, :] += 1 y[:, 1::2, :] += 1 z[:, :, 1::2] += 1 fig = plt.figure() ax = fig.add_subplot(projection='3d') f_2 = np.ones([2 * N_PIX - 1, 2 * N_PIX - 1, 2 * N_PIX - 1, 4], dtype=np.float64) f_2[:, :, :, 3] = explode(_h) cm = plt.get_cmap('cool') f_2[:, :, :, :3] = cm(f_2[:, :, :, 3])[..., :3] f_2[:, :, :, 3] = f_2[:, :, :, 3].clip(.01, .74) ecolors_2 = f_2 ax.voxels(x, y, z, filled_2, facecolors=f_2, edgecolors=.006 * ecolors_2) ax.set_aspect('equal') ax.set_zticks([0, N_PIX]) ax.set_xticks([0, N_PIX]) ax.set_yticks([0, N_PIX]) ax.set_zticklabels([f'{n/N_PIX:.2f}'[0:] for n in ax.get_zticks()]) ax.set_zlabel('valence', fontsize=10, labelpad=0) ax.set_xticklabels([f'{n/N_PIX:.2f}' for n in ax.get_xticks()]) ax.set_xlabel('arousal', fontsize=10, labelpad=7) # The y-axis rotation is corrected here from 275 to 90 degrees ax.set_yticklabels([f'{1-n/N_PIX:.2f}' for n in ax.get_yticks()], rotation=90) ax.set_ylabel('dominance', fontsize=10, labelpad=10) ax.grid(False) ax.plot([N_PIX, N_PIX], [0, N_PIX + .2], [N_PIX, N_PIX], 'g', linewidth=1) ax.plot([0, N_PIX], [N_PIX, N_PIX + .24], [N_PIX, N_PIX], 'k', linewidth=1) # Missing lines on the top face ax.plot([0, 0], [0, N_PIX], [N_PIX, N_PIX], 'darkred', linewidth=1) ax.plot([0, N_PIX], [0, 0], [N_PIX, N_PIX], 'darkblue', linewidth=1) # Set pane colors after plotting the lines # UPDATED: Replaced `w_xaxis` with `xaxis` and `w_yaxis` with `yaxis`. ax.xaxis.set_pane_color((0.8, 0.8, 0.8, 0.5)) ax.yaxis.set_pane_color((0.8, 0.8, 0.8, 0.5)) ax.zaxis.set_pane_color((0.8, 0.8, 0.8, 0.0)) # Restore the limits to prevent the plot from expanding ax.set_xlim(0, N_PIX) ax.set_ylim(0, N_PIX) ax.set_zlim(0, N_PIX) # plt.show() # TTS # VOICES = [f'wav/{vox}' for vox in os.listdir('wav')] # add unidecode (to parse non-roman characters for the StyleTTS2 # # for the VITS it should better skip the unknown letters - dont use unidecode()) # at generation fill the state of "last tts" # at record fill the state of "last record" and place in list of voice/langs for TTS VOICES = ['jv_ID_google-gmu_04982.wav', 'it_IT_mls_1595.wav', 'en_US_vctk_p303.wav', 'en_US_vctk_p306.wav', 'it_IT_mls_8842.wav', 'en_US_cmu_arctic_ksp.wav', 'jv_ID_google-gmu_05970.wav', 'en_US_vctk_p318.wav', 'ha_NE_openbible.wav', 'ne_NP_ne-google_0883.wav', 'en_US_vctk_p280.wav', 'bn_multi_1010.wav', 'en_US_vctk_p259.wav', 'it_IT_mls_844.wav', 'en_US_vctk_p269.wav', 'en_US_vctk_p285.wav', 'de_DE_m-ailabs_angela_merkel.wav', 'en_US_vctk_p316.wav', 'en_US_vctk_p362.wav', 'jv_ID_google-gmu_06207.wav', 'tn_ZA_google-nwu_9061.wav', 'fr_FR_tom.wav', 'en_US_vctk_p233.wav', 'it_IT_mls_4975.wav', 'en_US_vctk_p236.wav', 'bn_multi_01232.wav', 'bn_multi_5958.wav', 'it_IT_mls_9185.wav', 'en_US_vctk_p248.wav', 'en_US_vctk_p287.wav', 'it_IT_mls_9772.wav', 'te_IN_cmu-indic_sk.wav', 'tn_ZA_google-nwu_8333.wav', 'en_US_vctk_p260.wav', 'en_US_vctk_p247.wav', 'en_US_vctk_p329.wav', 'en_US_cmu_arctic_fem.wav', 'en_US_cmu_arctic_rms.wav', 'en_US_vctk_p308.wav', 'jv_ID_google-gmu_08736.wav', 'en_US_vctk_p245.wav', 'fr_FR_m-ailabs_nadine_eckert_boulet.wav', 'jv_ID_google-gmu_03314.wav', 'en_US_vctk_p239.wav', 'jv_ID_google-gmu_05540.wav', 'it_IT_mls_7440.wav', 'en_US_vctk_p310.wav', 'en_US_vctk_p237.wav', 'en_US_hifi-tts_92.wav', 'en_US_cmu_arctic_aew.wav', 'ne_NP_ne-google_2099.wav', 'en_US_vctk_p226.wav', 'af_ZA_google-nwu_1919.wav', 'jv_ID_google-gmu_03727.wav', 'en_US_vctk_p317.wav', 'tn_ZA_google-nwu_0378.wav', 'nl_pmk.wav', 'en_US_vctk_p286.wav', 'tn_ZA_google-nwu_3342.wav', # 'en_US_vctk_p343.wav', 'de_DE_m-ailabs_ramona_deininger.wav', 'jv_ID_google-gmu_03424.wav', 'en_US_vctk_p341.wav', 'jv_ID_google-gmu_03187.wav', 'ne_NP_ne-google_3960.wav', 'jv_ID_google-gmu_06080.wav', 'ne_NP_ne-google_3997.wav', # 'en_US_vctk_p267.wav', 'en_US_vctk_p240.wav', 'ne_NP_ne-google_5687.wav', 'ne_NP_ne-google_9407.wav', 'jv_ID_google-gmu_05667.wav', 'jv_ID_google-gmu_01519.wav', 'ne_NP_ne-google_7957.wav', 'it_IT_mls_4705.wav', 'ne_NP_ne-google_6329.wav', 'it_IT_mls_1725.wav', 'tn_ZA_google-nwu_8914.wav', 'en_US_ljspeech.wav', 'tn_ZA_google-nwu_4850.wav', 'en_US_vctk_p238.wav', 'en_US_vctk_p302.wav', 'jv_ID_google-gmu_08178.wav', 'en_US_vctk_p313.wav', 'af_ZA_google-nwu_2418.wav', 'bn_multi_00737.wav', 'en_US_vctk_p275.wav', # y 'af_ZA_google-nwu_0184.wav', 'jv_ID_google-gmu_07638.wav', 'ne_NP_ne-google_6587.wav', 'ne_NP_ne-google_0258.wav', 'en_US_vctk_p232.wav', 'en_US_vctk_p336.wav', 'jv_ID_google-gmu_09039.wav', 'en_US_vctk_p312.wav', 'af_ZA_google-nwu_8148.wav', 'en_US_vctk_p326.wav', 'en_US_vctk_p264.wav', 'en_US_vctk_p295.wav', # 'en_US_vctk_p298.wav', 'es_ES_m-ailabs_victor_villarraza.wav', 'pl_PL_m-ailabs_nina_brown.wav', 'tn_ZA_google-nwu_9365.wav', 'en_US_vctk_p294.wav', 'jv_ID_google-gmu_00658.wav', 'jv_ID_google-gmu_08305.wav', 'en_US_vctk_p330.wav', 'gu_IN_cmu-indic_cmu_indic_guj_dp.wav', 'jv_ID_google-gmu_05219.wav', 'en_US_vctk_p284.wav', 'de_DE_m-ailabs_eva_k.wav', # 'bn_multi_00779.wav', 'en_UK_apope.wav', 'en_US_vctk_p345.wav', 'it_IT_mls_6744.wav', 'en_US_vctk_p347.wav', 'en_US_m-ailabs_mary_ann.wav', 'en_US_m-ailabs_elliot_miller.wav', 'en_US_vctk_p279.wav', 'ru_RU_multi_nikolaev.wav', 'bn_multi_4811.wav', 'tn_ZA_google-nwu_7693.wav', 'bn_multi_01701.wav', 'en_US_vctk_p262.wav', # 'en_US_vctk_p266.wav', 'en_US_vctk_p243.wav', 'en_US_vctk_p297.wav', 'en_US_vctk_p278.wav', 'jv_ID_google-gmu_02059.wav', 'en_US_vctk_p231.wav', 'te_IN_cmu-indic_kpn.wav', 'en_US_vctk_p250.wav', 'it_IT_mls_4974.wav', 'en_US_cmu_arctic_awbrms.wav', # 'en_US_vctk_p263.wav', 'nl_femal.wav', 'tn_ZA_google-nwu_6116.wav', 'jv_ID_google-gmu_06383.wav', 'en_US_vctk_p225.wav', 'en_US_vctk_p228.wav', 'it_IT_mls_277.wav', 'tn_ZA_google-nwu_7866.wav', 'en_US_vctk_p300.wav', 'ne_NP_ne-google_0649.wav', 'es_ES_carlfm.wav', 'jv_ID_google-gmu_06510.wav', 'de_DE_m-ailabs_rebecca_braunert_plunkett.wav', 'en_US_vctk_p340.wav', 'en_US_cmu_arctic_gka.wav', 'ne_NP_ne-google_2027.wav', 'jv_ID_google-gmu_09724.wav', 'en_US_vctk_p361.wav', 'ne_NP_ne-google_6834.wav', 'jv_ID_google-gmu_02326.wav', 'fr_FR_m-ailabs_zeckou.wav', 'tn_ZA_google-nwu_1932.wav', # 'female-20-happy.wav', 'tn_ZA_google-nwu_1483.wav', 'de_DE_thorsten-emotion_amused.wav', 'ru_RU_multi_minaev.wav', 'sw_lanfrica.wav', 'en_US_vctk_p271.wav', 'tn_ZA_google-nwu_0441.wav', 'it_IT_mls_6001.wav', 'en_US_vctk_p305.wav', 'it_IT_mls_8828.wav', 'jv_ID_google-gmu_08002.wav', 'it_IT_mls_2033.wav', 'tn_ZA_google-nwu_3629.wav', 'it_IT_mls_6348.wav', 'en_US_cmu_arctic_axb.wav', 'it_IT_mls_8181.wav', 'en_US_vctk_p230.wav', 'af_ZA_google-nwu_7214.wav', 'nl_nathalie.wav', 'it_IT_mls_8207.wav', 'ko_KO_kss.wav', 'af_ZA_google-nwu_6590.wav', 'jv_ID_google-gmu_00264.wav', 'tn_ZA_google-nwu_6234.wav', 'jv_ID_google-gmu_05522.wav', 'en_US_cmu_arctic_lnh.wav', 'en_US_vctk_p272.wav', 'en_US_cmu_arctic_slp.wav', 'en_US_vctk_p299.wav', 'en_US_hifi-tts_9017.wav', 'it_IT_mls_4998.wav', 'it_IT_mls_6299.wav', 'en_US_cmu_arctic_rxr.wav', 'female-46-neutral.wav', 'jv_ID_google-gmu_01392.wav', 'tn_ZA_google-nwu_8512.wav', 'en_US_vctk_p244.wav', # 'bn_multi_3108.wav', # 'it_IT_mls_7405.wav', # 'bn_multi_3713.wav', # 'yo_openbible.wav', # 'jv_ID_google-gmu_01932.wav', 'en_US_vctk_p270.wav', 'tn_ZA_google-nwu_6459.wav', 'bn_multi_4046.wav', 'en_US_vctk_p288.wav', 'en_US_vctk_p251.wav', 'es_ES_m-ailabs_tux.wav', 'tn_ZA_google-nwu_6206.wav', 'bn_multi_9169.wav', # 'en_US_vctk_p293.wav', # 'en_US_vctk_p255.wav', 'af_ZA_google-nwu_8963.wav', # 'en_US_vctk_p265.wav', 'gu_IN_cmu-indic_cmu_indic_guj_ad.wav', 'jv_ID_google-gmu_07335.wav', 'en_US_vctk_p323.wav', 'en_US_vctk_p281.wav', 'en_US_cmu_arctic_bdl.wav', 'en_US_m-ailabs_judy_bieber.wav', 'it_IT_mls_10446.wav', 'en_US_vctk_p261.wav', 'en_US_vctk_p292.wav', 'te_IN_cmu-indic_ss.wav', 'en_US_vctk_p311.wav', 'it_IT_mls_12428.wav', 'en_US_cmu_arctic_aup.wav', 'jv_ID_google-gmu_04679.wav', 'it_IT_mls_4971.wav', 'en_US_cmu_arctic_ljm.wav', 'fa_haaniye.wav', 'en_US_vctk_p339.wav', 'tn_ZA_google-nwu_7896.wav', 'en_US_vctk_p253.wav', 'it_IT_mls_5421.wav', # 'ne_NP_ne-google_0546.wav', 'vi_VN_vais1000.wav', 'en_US_vctk_p229.wav', 'en_US_vctk_p254.wav', 'en_US_vctk_p258.wav', 'it_IT_mls_7936.wav', 'en_US_vctk_p301.wav', 'tn_ZA_google-nwu_0045.wav', 'it_IT_mls_659.wav', 'tn_ZA_google-nwu_7674.wav', 'it_IT_mls_12804.wav', 'el_GR_rapunzelina.wav', 'en_US_hifi-tts_6097.wav', 'en_US_vctk_p257.wav', 'jv_ID_google-gmu_07875.wav', 'it_IT_mls_1157.wav', 'it_IT_mls_643.wav', 'en_US_vctk_p304.wav', 'ru_RU_multi_hajdurova.wav', 'it_IT_mls_8461.wav', 'bn_multi_3958.wav', 'it_IT_mls_1989.wav', 'en_US_vctk_p249.wav', # 'bn_multi_0834.wav', 'en_US_vctk_p307.wav', 'es_ES_m-ailabs_karen_savage.wav', 'fr_FR_m-ailabs_bernard.wav', 'en_US_vctk_p252.wav', 'en_US_cmu_arctic_jmk.wav', 'en_US_vctk_p333.wav', 'tn_ZA_google-nwu_4506.wav', 'ne_NP_ne-google_0283.wav', 'de_DE_m-ailabs_karlsson.wav', 'en_US_cmu_arctic_awb.wav', 'en_US_vctk_p246.wav', 'en_US_cmu_arctic_clb.wav', 'en_US_vctk_p364.wav', 'nl_flemishguy.wav', 'en_US_vctk_p276.wav', # y # 'en_US_vctk_p274.wav', 'fr_FR_m-ailabs_gilles_g_le_blanc.wav', 'it_IT_mls_7444.wav', 'style_o22050.wav', 'en_US_vctk_s5.wav', 'en_US_vctk_p268.wav', 'it_IT_mls_6807.wav', 'it_IT_mls_2019.wav', 'male-60-angry.wav', 'af_ZA_google-nwu_8924.wav', 'en_US_vctk_p374.wav', 'en_US_vctk_p363.wav', 'it_IT_mls_644.wav', 'ne_NP_ne-google_3614.wav', 'en_US_vctk_p241.wav', 'ne_NP_ne-google_3154.wav', 'en_US_vctk_p234.wav', 'it_IT_mls_8384.wav', 'fr_FR_m-ailabs_ezwa.wav', 'it_IT_mls_5010.wav', 'en_US_vctk_p351.wav', 'en_US_cmu_arctic_eey.wav', 'jv_ID_google-gmu_04285.wav', 'jv_ID_google-gmu_06941.wav', 'hu_HU_diana-majlinger.wav', 'tn_ZA_google-nwu_2839.wav', 'bn_multi_03042.wav', 'tn_ZA_google-nwu_5628.wav', 'it_IT_mls_4649.wav', 'af_ZA_google-nwu_7130.wav', 'en_US_cmu_arctic_slt.wav', 'jv_ID_google-gmu_04175.wav', 'gu_IN_cmu-indic_cmu_indic_guj_kt.wav', 'jv_ID_google-gmu_00027.wav', 'jv_ID_google-gmu_02884.wav', 'en_US_vctk_p360.wav', 'en_US_vctk_p334.wav', 'male-27-sad.wav', 'tn_ZA_google-nwu_1498.wav', 'fi_FI_harri-tapani-ylilammi.wav', 'bn_multi_rm.wav', 'ne_NP_ne-google_2139.wav', 'pl_PL_m-ailabs_piotr_nater.wav', 'fr_FR_siwis.wav', 'nl_bart-de-leeuw.wav', 'jv_ID_google-gmu_04715.wav', 'en_US_vctk_p283.wav', 'en_US_vctk_p314.wav', 'en_US_vctk_p335.wav', 'jv_ID_google-gmu_07765.wav', 'en_US_vctk_p273.wav' ] VOICES = [t[:-4] for t in VOICES] # crop .wav for visuals in gr.DropDown _tts = StyleTTS2().to('cpu') def only_greek_or_only_latin(text, lang='grc'): ''' str: The converted string in the specified target script. Characters not found in any mapping are preserved as is. Latin accented characters in the input (e.g., 'É', 'ü') will be preserved in their lowercase form (e.g., 'é', 'ü') if converting to Latin. ''' # --- Mapping Dictionaries --- # Keys are in lowercase as input text is case-folded. # If the output needs to maintain original casing, additional logic is required. latin_to_greek_map = { 'a': 'α', 'b': 'β', 'g': 'γ', 'd': 'δ', 'e': 'ε', 'ch': 'τσο', # Example of a multi-character Latin sequence 'z': 'ζ', 'h': 'χ', 'i': 'ι', 'k': 'κ', 'l': 'λ', 'm': 'μ', 'n': 'ν', 'x': 'ξ', 'o': 'ο', 'p': 'π', 'v': 'β', 'sc': 'σκ', 'r': 'ρ', 's': 'σ', 't': 'τ', 'u': 'ου', 'f': 'φ', 'c': 'σ', 'w': 'β', 'y': 'γ', } greek_to_latin_map = { 'ου': 'ou', # Prioritize common diphthongs/digraphs 'α': 'a', 'β': 'v', 'γ': 'g', 'δ': 'd', 'ε': 'e', 'ζ': 'z', 'η': 'i', 'θ': 'th', 'ι': 'i', 'κ': 'k', 'λ': 'l', 'μ': 'm', 'ν': 'n', 'ξ': 'x', 'ο': 'o', 'π': 'p', 'ρ': 'r', 'σ': 's', 'τ': 't', 'υ': 'y', # 'y' is a common transliteration for upsilon 'φ': 'f', 'χ': 'ch', 'ψ': 'ps', 'ω': 'o', 'ς': 's', # Final sigma } cyrillic_to_latin_map = { 'а': 'a', 'б': 'b', 'в': 'v', 'г': 'g', 'д': 'd', 'е': 'e', 'ё': 'yo', 'ж': 'zh', 'з': 'z', 'и': 'i', 'й': 'y', 'к': 'k', 'л': 'l', 'м': 'm', 'н': 'n', 'о': 'o', 'п': 'p', 'р': 'r', 'с': 's', 'т': 't', 'у': 'u', 'ф': 'f', 'х': 'kh', 'ц': 'ts', 'ч': 'ch', 'ш': 'sh', 'щ': 'shch', 'ъ': '', 'ы': 'y', 'ь': '', 'э': 'e', 'ю': 'yu', 'я': 'ya', } # Direct Cyrillic to Greek mapping based on phonetic similarity. # These are approximations and may not be universally accepted transliterations. cyrillic_to_greek_map = { 'а': 'α', 'б': 'β', 'в': 'β', 'г': 'γ', 'д': 'δ', 'е': 'ε', 'ё': 'ιο', 'ж': 'ζ', 'з': 'ζ', 'и': 'ι', 'й': 'ι', 'κ': 'κ', 'λ': 'λ', 'м': 'μ', 'н': 'ν', 'о': 'ο', 'π': 'π', 'ρ': 'ρ', 'σ': 'σ', 'τ': 'τ', 'у': 'ου', 'ф': 'φ', 'х': 'χ', 'ц': 'τσ', 'ч': 'τσ', # or τζ depending on desired sound 'ш': 'σ', 'щ': 'σ', # approximations 'ъ': '', 'ы': 'ι', 'ь': '', 'э': 'ε', 'ю': 'ιου', 'я': 'ια', } # Convert the input text to lowercase, preserving accents for Latin characters. # casefold() is used for more robust caseless matching across Unicode characters. lowercased_text = text.lower() #casefold() output_chars = [] current_index = 0 if lang == 'grc': # Combine all relevant maps for direct lookup to Greek conversion_map = {**latin_to_greek_map, **cyrillic_to_greek_map} # Sort keys by length in reverse order to handle multi-character sequences first sorted_source_keys = sorted( list(latin_to_greek_map.keys()) + list(cyrillic_to_greek_map.keys()), key=len, reverse=True ) while current_index < len(lowercased_text): found_conversion = False for key in sorted_source_keys: if lowercased_text.startswith(key, current_index): output_chars.append(conversion_map[key]) current_index += len(key) found_conversion = True break if not found_conversion: # If no specific mapping found, append the character as is. # This handles unmapped characters and already Greek characters. output_chars.append(lowercased_text[current_index]) current_index += 1 return ''.join(output_chars) else: # Default to 'lat' conversion # Combine Greek to Latin and Cyrillic to Latin maps. # Cyrillic map keys will take precedence in case of overlap if defined after Greek. combined_to_latin_map = {**greek_to_latin_map, **cyrillic_to_latin_map} # Sort all relevant source keys by length in reverse for replacement sorted_source_keys = sorted( list(greek_to_latin_map.keys()) + list(cyrillic_to_latin_map.keys()), key=len, reverse=True ) while current_index < len(lowercased_text): found_conversion = False for key in sorted_source_keys: if lowercased_text.startswith(key, current_index): latin_equivalent = combined_to_latin_map[key] # Strip accents ONLY if the source character was from the Greek map. # This preserves accents on original Latin characters (like 'é') # and allows for intentional accent stripping from Greek transliterations. if key in greek_to_latin_map: normalized_latin = unicodedata.normalize('NFD', latin_equivalent) stripped_latin = ''.join(c for c in normalized_latin if not unicodedata.combining(c)) output_chars.append(stripped_latin) else: output_chars.append(latin_equivalent) current_index += len(key) found_conversion = True break if not found_conversion: # If no conversion happened from Greek or Cyrillic, append the character as is. # This preserves existing Latin characters (including accented ones from input), # numbers, punctuation, and other symbols. output_chars.append(lowercased_text[current_index]) current_index += 1 return ''.join(output_chars) def _stylett2(text='Hallov worlds Far over the', ref_s='wav/af_ZA_google-nwu_0184.wav'): if text and text.strip(): text = only_greek_or_only_latin(text, lang='eng') speech_audio = _tts.inference(text, ref_s=re_s)[0, 0, :].numpy() # 24 Khz if speech_audio.shape[0] > 10: speech_audio = audresample.resample(signal=speech_audio.astype(np.float32), original_rate=24000, target_rate=16000)[0, :] # 16 KHz return speech_audio import gradio as gr # Dummy functions to make the code runnable for demonstration def audionar_tts(text, choice, soundscape, kv): # This function would generate an audio file and return its path return "dummy_audio.wav" def recognize(audio_input_path): # This function would analyze the audio and return results return "30", "Male", {"Angry": 0.9} # Assuming these are defined elsewhere in the user's code language_names = ["English", "Spanish"] VOICES = ["Voice 1", "Voice 2"] with gr.Blocks(theme='huggingface') as demo: tts_file = gr.State(value=None) audio_examples_state = gr.State( value=[ ["wav/female-46-neutral.wav"], ["wav/female-20-happy.wav"], ["wav/male-60-angry.wav"], ["wav/male-27-sad.wav"], ] ) with gr.Tab(label="TTS"): with gr.Row(): text_input = gr.Textbox( label="Type text for TTS:", placeholder="Type Text for TTS", lines=4, value="Farover the misty mountains cold too dungeons deep and caverns old.", ) choice_dropdown = gr.Dropdown( choices=language_names + VOICES, label="Select Voice or Language", value=VOICES[0] ) soundscape_input = gr.Textbox( lines=1, value="frogs", label="AudioGen Txt" ) kv_input = gr.Number( label="kv Period", value=24, ) generate_button = gr.Button("Generate Audio", variant="primary") output_audio = gr.Audio(label="TTS Output") def generate_and_update_state(text, choice, soundscape, kv, current_examples): audio_path = audionar_tts(text, choice, soundscape, kv) updated_examples = current_examples + [[audio_path]] return audio_path, updated_examples generate_button.click( fn=generate_and_update_state, inputs=[text_input, choice_dropdown, soundscape_input, kv_input, audio_examples_state], outputs=[output_audio, audio_examples_state] ) with gr.Tab(label="Speech Analysis"): with gr.Row(): with gr.Column(): input_audio_analysis = gr.Audio( sources=["upload", "microphone"], type="filepath", label="Audio input", min_length=0.025, ) audio_examples = gr.Examples( examples=[], # Initialize with an empty list inputs=[input_audio_analysis], label="Examples from CREMA-D, ODbL v1.0 license", ) gr.Markdown("Only the first two seconds of the audio will be processed.") submit_btn = gr.Button(value="Submit", variant="primary") with gr.Column(): output_age = gr.Textbox(label="Age") output_gender = gr.Label(label="Gender") output_expression = gr.Image(label="Expression") outputs = [output_age, output_gender, output_expression] # Fix: This function should not update gr.Examples directly. # Instead, it should just return the updated examples list. # The `demo.load` event will handle the update. def load_examples_from_state(examples_list): return gr.Examples.update(examples=examples_list) demo.load( fn=load_examples_from_state, inputs=[audio_examples_state], outputs=[audio_examples], queue=False, ) submit_btn.click(recognize, input_audio_analysis, outputs) demo.launch(debug=True)