# imports import os import sys os.system("pip install git+https://github.com/openai/whisper.git") import gradio as gr import whisper # the model we are using for ASR, options are small, medium, large and largev2 (large and largev2 don't fit on huggingface cpu) model = whisper.load_model("medium") import torch # A table to look up all the languages language_id_lookup = { "Arabic" : "ar", "English" : "en", "Chinese" : "zh", "Spanish" : "es", "Russian" : "ru", "French" : "fr", } # load mRASP2 os.system("git clone https://github.com/PANXiao1994/mRASP2.git") os.system('mv -n mRASP2/* ./') os.system("rm -rf mRASP2") os.system("pip install -r requirements.txt") os.system("git clone https://github.com/pytorch/fairseq") os.system("cd fairseq; pip install ./; cd ..") model_name = "12e12d_last.pt" # os.system("wget https://lf3-nlp-opensource.bytetos.com/obj/nlp-opensource/acl2021/mrasp2/" + model_name) os.system("wget https://lf3-nlp-opensource.bytetos.com/obj/nlp-opensource/acl2021/mrasp2/bpe_vocab") os.system("wget https://lf3-nlp-opensource.bytetos.com/obj/nlp-opensource/emnlp2020/mrasp/pretrain/dataset/codes.bpe.32000") # load tts # os.system("git clone https://github.com/Kyubyong/g2pC.git") # os.system("cd g2pC; sed -i 's/pkuseg/spacy_pkuseg/g' setup.py; \ # sed -i 's/import pkuseg/import spacy_pkuseg as pkuseg/g' g2pc/g2pc.py; \ # sed -i 's/package_data={/# package_data={/g' setup.py; \ # pip install ./; cd ..") # os.system("git clone https://github.com/DigitalPhonetics/IMS-Toucan.git") # sys.path.append('./IMS-Toucan') # os.system("cd IMS-Toucan; pip install --no-cache-dir -r requirements.txt") # os.system("python run_model_downloader.py; cd ..") # from InferenceInterfaces.PortaSpeechInterface import PortaSpeechInterface # cwd = os.getcwd() # os.chdir('./IMS-Toucan') # tts = PortaSpeechInterface(device='cpu', tts_model_path='Meta') # os.chdir(cwd) # azure tts # os.system("2d1847f4151f4f94ae06d0b620533936") # os.system("eastus") lang2voice = { "zh": "zh-CN-XiaoxiaoNeural", "ar": "ar-EG-SalmaNeural", "en": "en-US-JennyNeural", "es": "es-ES-AbrilNeural", "ru": "ru-RU-DariyaNeural", "fr": "fr-FR-AlainNeural", } # The predict function. audio, language and mic_audio are all parameters directly passed by gradio # which means they are user inputted. They are specified in gr.inputs[] block at the bottom. The # gr.outputs[] block will specify the output type. def predict(audio, src_language, tgt_language, mic_audio=None): # checks if mic_audio is used, otherwise feeds model uploaded audio if mic_audio is not None: input_audio = mic_audio elif audio is not None: input_audio = audio else: return "(please provide audio)" # Uses the model's preprocessing methods to preprocess audio audio = whisper.load_audio(input_audio) audio = whisper.pad_or_trim(audio) # Calculates the mel frequency spectogram mel = whisper.log_mel_spectrogram(audio).to(model.device) # if model is supposed to detect language, set outLanguage to None # otherwise set to specified language if(src_language == "Detect Language"): src_language = None else: src_language = language_id_lookup[src_language.split()[0]] tgt_language = language_id_lookup[tgt_language.split()[0]] # Runs the audio through the whisper model and gets the DecodingResult object, which has the features: # audio_features (Tensor), language, language_probs, tokens, text, avg_logprob, no_speech_prob, temperature, compression_ratio # asr options = whisper.DecodingOptions(fp16 = True, language = src_language) result = whisper.decode(model, mel, options) if src_language is None: src_language = result.language transcript = result.text # mt with open("input." + src_language, 'w') as w: w.write(result.text) with open("input." + tgt_language, 'w') as w: w.write('LANG_TOK_' + src_language.upper()) os.system("python fairseq/fairseq_cli/preprocess.py --dataset-impl raw \ --srcdict bpe_vocab --tgtdict bpe_vocab --testpref input -s {} -t {}".format( \ src_language, tgt_language)) os.system("python fairseq/fairseq_cli/interactive.py ./data-bin \ --user-dir mcolt \ -s zh \ -t en \ --skip-invalid-size-inputs-valid-test \ --path {} \ --max-tokens 1024 \ --task translation_w_langtok \ --lang-prefix-tok \"LANG_TOK_{}\" \ --max-source-positions 1024 \ --max-target-positions 1024 \ --nbest 1 \ --bpe subword_nmt \ --bpe-codes codes.bpe.32000 \ --post-process --tokenizer moses \ --input input.{} | grep -E '[D]-[0-9]+' > output".format( model_name, tgt_language.upper(), src_language)) with open("output", 'r') as r: translation = (' '.join(r.readline().split(' ')[3:])).strip() # tts # tts.set_language(tgt_language) # tts.read_to_file(text_list=[translation], file_location='output.wav') # azure tts import azure.cognitiveservices.speech as speechsdk speech_key = "2d1847f4151f4f94ae06d0b620533936" service_region = "eastus" speech_config = speechsdk.SpeechConfig(subscription=speech_key, region=service_region) # Note: the voice setting will not overwrite the voice element in input SSML. speech_config.speech_synthesis_voice_name = lang2voice[tgt_language] audio_config = speechsdk.audio.AudioOutputConfig(filename="output.wav") speech_synthesizer = speechsdk.SpeechSynthesizer(speech_config=speech_config, audio_config=audio_config) speech_synthesizer.speak_text(translation) # Returns the text return transcript, translation, "output.wav" title = "Demo for Whisper (ASR) -> Something -> IMS Toucan (TTS)" description = """ How to use: Upload an audio file or record using the microphone. The audio is into the whisper model developed by openai. The output is the text transcription of the audio in the language you inputted. If you asked the model to detect a language, it will tell you what language it detected. """ # The gradio interface gr.Interface( fn=predict, inputs=[ gr.Audio(label="Upload Speech", source="upload", type="filepath"), gr.inputs.Dropdown(['Arabic', 'Chinese', 'English', 'Spanish', 'Russian', 'French', 'Detect Language'], type="value", default='English', label="Select the language of input"), gr.inputs.Dropdown(['Arabic', 'Chinese', 'English', 'Spanish', 'Russian', 'French', 'Detect Language'], type="value", default='English', label="Select the language of output"), gr.Audio(label="Record Speech", source="microphone", type="filepath"), ], # To change to output audio, replace the outputs line with # outputs=gr.outputs.Audio(type="numpy", label=None) outputs=[ gr.Text(label="Transcript"), gr.Text(label="Translation"), gr.Audio(label="Translation Speech") ], title=title, description=description, ).launch()