app.py

import os
import torch
from transformers import pipeline

# Loading the TTS and Vocoder ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
import sys
import re
import json
import inflect
import random
import uroman as ur
import numpy as np
import torchaudio
import subprocess
import requests
from transformers import AutoModelForCausalLM, AutoTokenizer
from outetts.wav_tokenizer.decoder import WavTokenizer

# Function to execute shell commands safely
def run_command(command):
    try:
        process = subprocess.Popen(
            command,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            shell=True,
            universal_newlines=True
        )
        stdout, stderr = process.communicate()
        
        if process.returncode != 0:
            print(f"Error executing: {command}")
            print(stderr)
            return False
        else:
            print(stdout)
            return True
    except Exception as e:
        print(f"Exception during execution of {command}: {e}")
        return False

# Clone the YarnGPT repository
if not os.path.exists('yarngpt'):
    print("Cloning YarnGPT repository...")
    run_command("git clone https://github.com/saheedniyi02/yarngpt.git")
else:
    print("YarnGPT repository already exists")

# # Install required packages with specific versions
# print("Installing required packages with specific versions...")
# run_command("pip install -q outetts==0.3.3 uroman==1.3.1.1")

# Add the yarngpt directory to Python path instead of installing it
yarngpt_path = os.path.join(os.getcwd(), 'yarngpt')
if os.path.exists(yarngpt_path) and yarngpt_path not in sys.path:
    sys.path.append(yarngpt_path)
    print(f"Added {yarngpt_path} to Python path")

# Now you should be able to import from yarngpt
# Import this after adding to path
try:
    from yarngpt.audiotokenizer import AudioTokenizerV2
    print("Successfully imported AudioTokenizerV2 from yarngpt")
except ImportError as e:
    print(f"Error importing from yarngpt: {e}")
    # Check the content of the directory to debug
    if os.path.exists(yarngpt_path):
        print("Contents of yarngpt directory:")
        print(os.listdir(yarngpt_path))

# Download files using Python's requests library instead of !wget
def download_file(url, save_path):
    response = requests.get(url, stream=True)
    if response.status_code == 200:
        with open(save_path, 'wb') as f:
            f.write(response.content)
        print(f"Downloaded {save_path}")
    else:
        print(f"Failed to download {url}")

# Download required files
download_file(
    "https://huggingface.co/novateur/WavTokenizer-medium-speech-75token/resolve/main/wavtokenizer_mediumdata_frame75_3s_nq1_code4096_dim512_kmeans200_attn.yaml",
    "wavtokenizer_mediumdata_frame75_3s_nq1_code4096_dim512_kmeans200_attn.yaml"
)
download_file(
    "https://huggingface.co/novateur/WavTokenizer-large-speech-75token/resolve/main/wavtokenizer_large_speech_320_24k.ckpt",
    "wavtokenizer_large_speech_320_24k.ckpt"
)


from yarngpt.audiotokenizer import AudioTokenizerV2

device = "cuda:0" if torch.cuda.is_available() else "cpu"

tokenizer_path="saheedniyi/YarnGPT2"
wav_tokenizer_config_path = "./wavtokenizer_mediumdata_frame75_3s_nq1_code4096_dim512_kmeans200_attn.yaml"
wav_tokenizer_model_path = "./wavtokenizer_large_speech_320_24k.ckpt"

audio_tokenizer=AudioTokenizerV2(tokenizer_path,wav_tokenizer_model_path,wav_tokenizer_config_path)
tts_model = AutoModelForCausalLM.from_pretrained(tokenizer_path,torch_dtype="auto").to(audio_tokenizer.device)

# The LLM Model ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
from huggingface_hub import HfFolder
from openai import OpenAI

api_key = os.getenv("API_KEY")
if api_key is None:
    raise ValueError("API_KEY is not set in the environment variables.")
print("API key successfully loaded.")

# Initialize OpenAI client for Hugging Face Inference Endpoint
client = OpenAI(
	base_url="https://y1ztgv8tu09nay6u.us-east-1.aws.endpoints.huggingface.cloud/v1/", #https://f2iozzwigntrzkve.us-east-1.aws.endpoints.huggingface.cloud/v1/",
	api_key=api_key
)

def generate_llm_response(text, model_id="ccibeekeoc42/Llama3.1-8b-base-SFT-2024-11-09"):
    full_response = []
    try:
        chat_completion = client.chat.completions.create(
            model="tgi",
            messages=[
                {"role": "system", "content": "You are HypaAI a very BRIEF AND DIRECT assistant. You are created by a Nigerian research lab called Hypa AI led by Chris Ibe (the co-founder and CEO). As part of a speech pipeline so keep your responses short (under 60 words), fluent, and straight to the point. Avoid markdown or digits in responses."},
                {"role": "user", "content": text}
            ],
            top_p=0.3,
            temperature=1,
            max_tokens=150,
            stream=True,
            seed=None,
            stop=None,
            frequency_penalty=None,
            presence_penalty=None
        )
        for chunk in chat_completion:
            if chunk.choices[0].delta.content:
                full_response.append(chunk.choices[0].delta.content)
        return "".join(full_response)
    except Exception as e:
        # If the error has a response with status code 503, assume the GPU is booting up.
        if hasattr(e, 'response') and e.response is not None and e.response.status_code == 503:
            return "The GPU is currently booting up. Please wait about 10 minutes and try again."
        else:
            raise e

# generate_llm_response("Explain Deep Learning in Igbo")


# Loading the ST Model (Whisper) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
pipe = pipeline("automatic-speech-recognition", model="okezieowen/whisper-small-multilingual-naija-11-03-2024", device=device)

# Take audio and return translated text
def transcribe(audio):
    outputs = pipe(audio, max_new_tokens=256, generate_kwargs={"task": "transcribe"})
    return outputs["text"]


# putting the ST and TTS system together ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
import numpy as np
def synthesise_yarn2(text):
    # change the language and voice
    prompt=audio_tokenizer.create_prompt(text,lang="english",speaker_name="idera")
    input_ids=audio_tokenizer.tokenize_prompt(prompt)
    output  = tts_model.generate(
                input_ids=input_ids,
                temperature=0.1,
                repetition_penalty=1.1,
                max_length=4000,
                num_beams=5,# using a beam size helps for the local languages but not english
            )

    codes=audio_tokenizer.get_codes(output)
    audio=audio_tokenizer.get_audio(codes)
    return audio.cpu()

target_dtype = np.int16
max_range = np.iinfo(target_dtype).max # Maximum value for 16-bit PCM audio conversion

def speech_to_speech_translation(audio, language="english"):
    # Speech to Text
    transcribed_text = transcribe(audio)
    print(f"Transcribed: {transcribed_text}")

    # Generate LLM Response
    print("Now making LLM Call ~~~~~~~~~~~~~~~~~~~~~~~~")
    llm_response = generate_llm_response(transcribed_text)
    print(f"LLM Response: {llm_response}")

    # Select a random voice based on the chosen language
    voice_mapping = {
        "english": ["idera", "chinenye", "jude", "emma", "umar", "joke", "zainab", "osagie", "remi", "tayo"],
        "yoruba": ["yoruba_male2", "yoruba_female2", "yoruba_feamle1"],
        "igbo": ["igbo_female2", "igbo_male2", "igbo_female1"],
        "hausa": ["hausa_feamle1", "hausa_female2", "hausa_male2", "hausa_male1"]
    }

    selected_voice = random.choice(voice_mapping.get(language.lower(), voice_mapping["english"]))
    print(f"Selected {language} voice: {selected_voice}")

    # Text to Speech
    print("Synthesizing Speech ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
    # Use the selected language and voice
    prompt = audio_tokenizer.create_prompt(llm_response, lang=language.lower(), speaker_name=selected_voice)
    input_ids = audio_tokenizer.tokenize_prompt(prompt)
    output = tts_model.generate(
        input_ids=input_ids,
        temperature=0.1,
        repetition_penalty=1.1,
        max_length=4000,
    )

    codes = audio_tokenizer.get_codes(output)
    synthesised_speech = audio_tokenizer.get_audio(codes)

    # Make sure we have a NumPy array, not a tensor
    if hasattr(synthesised_speech, 'numpy'):
        audio_np = synthesised_speech.numpy()
    else:
        audio_np = synthesised_speech

    # Handle NaN and Inf values
    audio_np = np.nan_to_num(audio_np)

    # Ensure audio is in [-1, 1] range
    if np.max(np.abs(audio_np)) > 0:
        audio_np = audio_np / np.max(np.abs(audio_np))

    # Convert to signed int16 (-32768 to 32767)
    int16_max = 32767  # Max value for signed 16-bit
    audio_int16 = np.clip(audio_np * int16_max, -int16_max, int16_max).astype(np.int16)

    # Ensure the audio is mono channel if needed
    if len(audio_int16.shape) > 1 and audio_int16.shape[0] == 1:
        audio_int16 = audio_int16[0]  # Convert from [1, samples] to [samples]

    # Debug info
    print(f"Audio stats - Min: {np.min(audio_int16)}, Max: {np.max(audio_int16)}, Shape: {audio_int16.shape}")

    # Ensure sample rate is within valid range (1-192000)
    sample_rate = min(max(24000, 1), 192000)

    print("Speech Synthesis Completed~~~~~~~~~~~~~~~~~~~")
    return transcribed_text, llm_response, (sample_rate, audio_int16)


# Gradio Demo
import gradio as gr

demo = gr.Blocks()

with demo:
    gr.Markdown("# Aware Speech-to-Speech Demo")

    with gr.Tab("Microphone"):
        with gr.Row():
            mic_input = gr.Audio(sources="microphone", type="filepath", label="Speak")
            lang_dropdown_mic = gr.Dropdown(
                choices=["English", "Yoruba", "Igbo", "Hausa"],
                value="English",
                label="Select Language"
            )

        mic_submit = gr.Button("Submit")

        with gr.Row():
            mic_transcribed = gr.Textbox(label="Transcribed Text", interactive=False)
            mic_response = gr.Textbox(label="HypaAI's Response", interactive=False)

        mic_audio_output = gr.Audio(label="Generated Speech", type="numpy")

        mic_submit.click(
            fn=speech_to_speech_translation,
            inputs=[mic_input, lang_dropdown_mic],
            outputs=[mic_transcribed, mic_response, mic_audio_output]
        )

    with gr.Tab("Audio File"):
        with gr.Row():
            file_input = gr.Audio(sources="upload", type="filepath", label="Upload Audio")
            lang_dropdown_file = gr.Dropdown(
                choices=["English", "Yoruba", "Igbo", "Hausa"],
                value="English",
                label="Select Language"
            )

        file_submit = gr.Button("Submit")

        with gr.Row():
            file_transcribed = gr.Textbox(label="Transcribed Text", interactive=False)
            file_response = gr.Textbox(label="HypaAI's Response", interactive=False)

        file_audio_output = gr.Audio(label="Generated Speech", type="numpy")

        file_submit.click(
            fn=speech_to_speech_translation,
            inputs=[file_input, lang_dropdown_file],
            outputs=[file_transcribed, file_response, file_audio_output]
        )

demo.launch(share=True)