Update app.py

app.py

@@ -45,108 +45,83 @@ class VoiceSynthesizer:
         except Exception as e:
             print(f"Bark model loading error: {e}")
     
-    def process_reference_audio(self, reference_audio):
-        """Process and store reference audio for voice cloning"""
-        try:
-            # Ensure audio is in the right format
-            if reference_audio is None:
-                return "No audio provided"
-            
-            # Convert to numpy array if needed
-            if isinstance(reference_audio, tuple):
-                reference_audio = reference_audio[0]
-            
-            # Ensure the audio is mono and normalized
-            if reference_audio.ndim > 1:
-                reference_audio = reference_audio.mean(axis=1)
-            
-            # Resample or trim if necessary
-            if len(reference_audio) > SAMPLE_RATE * 10:  # Limit to 10 seconds
-                reference_audio = reference_audio[:SAMPLE_RATE * 10]
-            
-            # Save reference audio
-            ref_filename = os.path.join(self.working_dir, "reference_voice.wav")
-            sf.write(ref_filename, reference_audio, SAMPLE_RATE)
-            
-            # Store reference voice
-            self.reference_voice = reference_audio
-            
-            return "Reference voice processed successfully"
+def process_reference_audio(self, reference_audio):
+    """Process and store reference audio for voice cloning"""
+    try:
+        # Gradio can pass audio in different formats
+        if reference_audio is None:
+            return "No audio provided"
         
-        except Exception as e:
-            print(f"Reference audio processing error: {e}")
-            return f"Error processing reference audio: {str(e)}"
-    
-    def _initialize_bark(self):
-        """Bark model initialization (already done in __init__)"""
-        return None
-    
-    def _initialize_speecht5(self):
-        """Initialize SpeechT5 model from Hugging Face"""
-        try:
-            # Load SpeechT5 model and processor
-            model = SpeechT5ForTextToSpeech.from_pretrained("microsoft/speecht5_tts")
-            processor = SpeechT5Processor.from_pretrained("microsoft/speecht5_tts")
-            vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
-            
-            # Load speaker embeddings
-            embeddings_dataset = datasets.load_dataset("Matthijs/cmu-arctic-xvectors", split="validation")
-            speaker_embeddings = torch.tensor(embeddings_dataset[0]["xvector"]).unsqueeze(0)
-            
-            return {
-                "model": model,
-                "processor": processor,
-                "vocoder": vocoder,
-                "speaker_embeddings": speaker_embeddings
-            }
-        except Exception as e:
-            print(f"SpeechT5 model loading error: {e}")
-            return None
-    
-    def generate_speech(self, text, model_name=None, voice_preset=None):
-        """Generate speech using selected model"""
-        if not text or not text.strip():
-            return None, "Please enter some text to speak"
+        # Handle different input types
+        if isinstance(reference_audio, tuple):
+            # Gradio typically returns (sample_rate, audio_array)
+            if len(reference_audio) == 2:
+                sample_rate, audio_data = reference_audio
+            else:
+                audio_data = reference_audio[0]
+                sample_rate = SAMPLE_RATE  # Default to Bark sample rate
+        elif isinstance(reference_audio, np.ndarray):
+            audio_data = reference_audio
+            sample_rate = SAMPLE_RATE
+        else:
+            return "Invalid audio format"
         
-        # Use specified model or current model
-        current_model = model_name or self.current_model
+        # Ensure audio is numpy array
+        audio_data = np.asarray(audio_data)
         
-        try:
-            if current_model == "bark":
-                return self._generate_bark_speech(text, voice_preset)
-            elif current_model == "speecht5":
-                return self._generate_speecht5_speech(text, voice_preset)
-            else:
-                raise ValueError(f"Unsupported model: {current_model}")
+        # Handle multi-channel audio
+        if audio_data.ndim > 1:
+            audio_data = audio_data.mean(axis=1)
         
-        except Exception as e:
-            print(f"Speech generation error: {e}")
-            import traceback
-            traceback.print_exc()
-            return None, f"Error generating speech: {str(e)}"
-    
-    def _generate_bark_speech(self, text, voice_preset=None):
-        """Generate speech using Bark"""
-        # Default Bark voice presets
-        voice_presets = [
-            "v2/en_speaker_6",  # Female
-            "v2/en_speaker_3",  # Male
-            "v2/en_speaker_9",  # Neutral
-        ]
+        # Trim or pad to standard length
+        max_duration = 10  # 10 seconds
+        max_samples = max_duration * sample_rate
+        
+        if len(audio_data) > max_samples:
+            audio_data = audio_data[:max_samples]
         
-        # Prepare history prompt
-        history_prompt = None
+        # Resample if necessary
+        if sample_rate != SAMPLE_RATE:
+            from scipy.signal import resample
+            audio_data = resample(audio_data, int(len(audio_data) * SAMPLE_RATE / sample_rate))
         
-        # Check if a reference voice is available
-        if self.reference_voice is not None:
-            # Save reference voice for Bark
-            ref_filename = os.path.join(self.working_dir, "reference_voice.wav")
-            history_prompt = ref_filename
-        elif voice_preset:
-            # Use predefined voice preset
-            history_prompt = voice_presets[0] if "v2/en_speaker" not in voice_preset else voice_preset
+        # Save reference audio
+        ref_filename = os.path.join(self.working_dir, "reference_voice.wav")
+        sf.write(ref_filename, audio_data, SAMPLE_RATE)
         
-        # Generate audio with or without history prompt
+        # Store reference voice
+        self.reference_voice = ref_filename
+        
+        return "Reference voice processed successfully"
+    
+    except Exception as e:
+        print(f"Reference audio processing error: {e}")
+        import traceback
+        traceback.print_exc()
+        return f"Error processing reference audio: {str(e)}"
+
+def _generate_bark_speech(self, text, voice_preset=None):
+    """Generate speech using Bark"""
+    # Default Bark voice presets
+    voice_presets = [
+        "v2/en_speaker_6",  # Female
+        "v2/en_speaker_3",  # Male
+        "v2/en_speaker_9",  # Neutral
+    ]
+    
+    # Prepare history prompt
+    history_prompt = None
+    
+    # Check if a reference voice is available
+    if self.reference_voice is not None:
+        # Use saved reference voice file
+        history_prompt = self.reference_voice
+    elif voice_preset:
+        # Use predefined voice preset
+        history_prompt = voice_presets[0] if "v2/en_speaker" not in voice_preset else voice_preset
+    
+    # Generate audio with or without history prompt
+    try:
         if history_prompt:
             audio_array = generate_audio(
                 text, 
@@ -163,37 +138,11 @@ class VoiceSynthesizer:
         
         return filepath, None
     
-    def _generate_speecht5_speech(self, text, speaker_id=None):
-        """Generate speech using SpeechT5"""
-        # Ensure model is initialized
-        speecht5_models = self.models["speecht5"]()
-        if not speecht5_models:
-            return None, "SpeechT5 model not loaded"
-        
-        model = speecht5_models["model"]
-        processor = speecht5_models["processor"]
-        vocoder = speecht5_models["vocoder"]
-        speaker_embeddings = speecht5_models["speaker_embeddings"]
-        
-        # Prepare inputs
-        inputs = processor(text=text, return_tensors="pt")
-        
-        # Generate speech
-        speech = model.generate_speech(
-            inputs["input_ids"], 
-            speaker_embeddings
-        )
-        
-        # Convert to numpy array
-        audio_array = speech.numpy()
-        
-        # Save generated audio
-        filename = f"speecht5_speech_{int(time.time())}.wav"
-        filepath = os.path.join(self.working_dir, filename)
-        wavfile.write(filepath, 16000, audio_array)
-        
-        return filepath, None
-
+    except Exception as e:
+        print(f"Bark speech generation error: {e}")
+        import traceback
+        traceback.print_exc()
+        return None, f"Error in Bark speech generation: {str(e)}"
 def create_interface():
     synthesizer = VoiceSynthesizer()