Update app.py

app.py

@@ -1,127 +1,63 @@
 import gradio as gr
-import numpy as np
-import cv2
-import librosa
-import tempfile
-import wave
-import os
-import speech_recognition as sr
-import pickle
-import json
-from tensorflow.keras.models import load_model
-from tensorflow.keras.preprocessing.text import tokenizer_from_json
-from tensorflow.keras.preprocessing.sequence import pad_sequences
-import nltk
-from collections import Counter
 from transformers import LlamaTokenizer, LlamaForCausalLM
+import tempfile
+import numpy as np
 
-# Initialize necessary models and tools
-# Load the tokenizer and model for text-based emotion prediction
-with open('tokenizer.json') as json_file:
-    tokenizer_json = json.load(json_file)
-tokenizer = tokenizer_from_json(tokenizer_json)
-text_model = load_model('model_for_text_emotion_updated(1).keras')
-
-# Load the audio emotion model and scaler
-with open('encoder.pkl', 'rb') as file:
-    encoder = pickle.load(file)
-with open('scaler.pkl', 'rb') as file:
-    scaler = pickle.load(file)
-audio_model = load_model('my_model.h5')
-
-# Load the LLaMA model for question answering
+# Initialize LLaMA Model for Question Answering
 llama_tokenizer = LlamaTokenizer.from_pretrained('huggingface/llama-7b')
 llama_model = LlamaForCausalLM.from_pretrained('huggingface/llama-7b')
 
-# Initialize NLTK tools
-nltk.download('punkt')
-nltk.download('wordnet')
-nltk.download('stopwords')
-lemmatizer = nltk.WordNetLemmatizer()
-stop_words = set(nltk.corpus.stopwords.words('english'))
-
-# Preprocess text for emotion prediction
-def preprocess_text(text):
-    tokens = nltk.word_tokenize(text.lower())
-    tokens = [word for word in tokens if word.isalnum() and word not in stop_words]
-    lemmatized_tokens = [lemmatizer.lemmatize(word) for word in tokens]
-    return ' '.join(lemmatized_tokens)
-
-# Extract audio features and predict emotion
-def extract_audio_features(data, sample_rate):
-    result = np.array([])
-    zcr = np.mean(librosa.feature.zero_crossing_rate(y=data).T, axis=0)
-    result = np.hstack((result, zcr))
-    mfcc = np.mean(librosa.feature.mfcc(y=data, sr=sample_rate).T, axis=0)
-    result = np.hstack((result, mfcc))
-    return result
-
-def predict_emotion_from_audio(audio_data):
-    sample_rate, data = audio_data
-    features = extract_audio_features(data, sample_rate)
-    features = np.expand_dims(features, axis=0)
-    scaled_features = scaler.transform(features)
-    prediction = audio_model.predict(scaled_features)
-    emotion_index = np.argmax(prediction)
-    emotion_array = np.zeros((1, len(encoder.categories_[0])))
-    emotion_array[0, emotion_index] = 1
-    emotion_label = encoder.inverse_transform(emotion_array)[0]
-    return emotion_label
-
-# Extract text from audio (speech recognition)
-def extract_text_from_audio(audio_path):
-    recognizer = sr.Recognizer()
-    with sr.AudioFile(audio_path) as source:
-        audio_data = recognizer.record(source)
-    text = recognizer.recognize_google(audio_data)
-    return text
-
-# Use LLaMA to answer questions based on the text
-def ask_llama(question, context):
-    inputs = llama_tokenizer(question, context, return_tensors="pt")
-    outputs = llama_model.generate(inputs['input_ids'], max_length=150)
-    answer = llama_tokenizer.decode(outputs[0], skip_special_tokens=True)
-    return answer
-
-# Process the video and extract text, emotion, and context for LLaMA
-def process_video(video_path):
-    # Extract audio from the video
-    video = mp.VideoFileClip(video_path)
-    if video.audio is None:
-        raise ValueError("No audio found in the video.")
+# Updated transcribe_and_predict_video function from your code
+def transcribe_and_predict_video(video):
+    # Process video frames for image-based emotion recognition
+    image_emotion = process_video(video)
     
-    audio = video.audio
-    with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as temp_audio_file:
-        temp_audio_path = temp_audio_file.name
-        audio.write_audiofile(temp_audio_path)
-
-    # Extract text from the audio
-    video_text = extract_text_from_audio(temp_audio_path)
+    # Process audio for text and audio-based emotion recognition
+    text_emotion, audio_emotion = process_audio_from_video(video)
     
-    # Predict emotions from the text and audio
-    preprocessed_text = preprocess_text(video_text)
-    title_seq = tokenizer.texts_to_sequences([preprocessed_text])
-    padded_title_seq = pad_sequences(title_seq, maxlen=35, padding='post', truncating='post')
-    text_emotion_prediction = text_model.predict(np.array(padded_title_seq))
-    text_emotion = ['anger', 'disgust', 'fear', 'joy', 'neutral', 'sadness', 'surprise'][np.argmax(text_emotion_prediction)]
+    # Determine the overall emotion (could be based on majority vote or some other logic)
+    overall_emotion = Counter([text_emotion, audio_emotion, image_emotion]).most_common(1)[0][0]
     
-    audio_data = audio.to_soundarray(fps=audio.fps)
-    audio_emotion = predict_emotion_from_audio((audio.fps, audio_data))
+    return overall_emotion
 
-    # Answer user queries based on the video text
-    context = video_text
-    return context, text_emotion, audio_emotion
+# Emotion-aware Question Answering with LLM
+def emotion_aware_qa(question, video):
+    # Get the emotion from the video (this uses the emotion detection you already implemented)
+    detected_emotion = transcribe_and_predict_video(video)
+    
+    # Create a custom response context based on the detected emotion
+    if detected_emotion == 'joy':
+        emotion_context = "You're in a good mood! Let's keep the positivity going."
+    elif detected_emotion == 'sadness':
+        emotion_context = "It seems like you're feeling a bit down. Let me help with that."
+    elif detected_emotion == 'anger':
+        emotion_context = "I sense some frustration. Let's work through it together."
+    elif detected_emotion == 'fear':
+        emotion_context = "It sounds like you're anxious. How can I assist in calming things down?"
+    elif detected_emotion == 'neutral':
+        emotion_context = "You're feeling neutral. How can I help you today?"
+    else:
+        emotion_context = "You're in an uncertain emotional state. Let me guide you."
+    
+    # Prepare the prompt for LLaMA, including emotion context and user question
+    prompt = f"{emotion_context} User asks: {question}"
+    
+    # Tokenize and generate response from LLaMA
+    inputs = llama_tokenizer(prompt, return_tensors="pt")
+    outputs = llama_model.generate(inputs['input_ids'], max_length=150)
+    answer = llama_tokenizer.decode(outputs[0], skip_special_tokens=True)
+    
+    return answer
 
-# Define Gradio Interface
-def video_query_interface(video, question):
-    context, text_emotion, audio_emotion = process_video(video)
-    answer = ask_llama(question, context)
-    return f"Text Emotion: {text_emotion}, Audio Emotion: {audio_emotion}\nAnswer: {answer}"
+# Create Gradio interface to interact with the LLM and video emotion detection
+def gradio_interface(question, video):
+    response = emotion_aware_qa(question, video)
+    return response
 
-iface = gr.Interface(fn=video_query_interface, 
-                     inputs=[gr.Video(), gr.Textbox()], 
+iface = gr.Interface(fn=gradio_interface, 
+                     inputs=["text", gr.Video()], 
                      outputs="text", 
-                     title="Video Emotion and Q&A",
-                     description="Upload a video and ask a question based on the audio content.")
+                     title="Emotion-Aware Question Answering",
+                     description="Ask a question and get an emotion-aware response based on the video.")
 
 iface.launch()