project added

app.py

@@ -1,4 +1,3 @@
-# Import necessary libraries
 from groq import Groq
 import gradio as gr
 from gtts import gTTS
@@ -10,11 +9,11 @@ import logging
 import spacy
 from transformers import pipeline
 import torch
-import cv2
-import numpy as np
+from PIL import Image
 from torchvision import transforms
 import pathlib
-
+import cv2  # Import OpenCV
+import numpy as np
 
 # Pathlib adjustment for Windows compatibility
 temp = pathlib.PosixPath
@@ -31,60 +30,29 @@ file_handler.setFormatter(formatter)
 logger.addHandler(console_handler)
 logger.addHandler(file_handler)
 
-# Initialize Groq Client
+#Initialize Groq Client
 client = Groq(api_key=os.getenv("GROQ_API_KEY_2"))
 
-# Initialize Groq Client
+# # Initialize Groq Client
 #client = Groq(api_key="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")
 
 # Initialize spaCy NLP model for named entity recognition (NER)
-import spacy
-
-# Download the model if it's not already installed
-try:
-    nlp = spacy.load("en_core_web_sm")
-except OSError:
-    print("Downloading 'en_core_web_sm' model...")
-    import os
-    os.system("python -m spacy download en_core_web_sm")
-    nlp = spacy.load("en_core_web_sm")
-
-# Your code continues here
-print("Model loaded successfully!")
+nlp = spacy.load("en_core_web_sm")
 
 # Initialize sentiment analysis model using Hugging Face
 sentiment_analyzer = pipeline("sentiment-analysis")
 
-import torch
-import os
-
+# Load pre-trained YOLOv5 model
 def load_yolov5_model():
-    # Load model from Hugging Face Hub or local path
     model = torch.hub.load(
-        'ultralytics/yolov5',  # Use the official YOLOv5 repo
+        r'C:\Users\RESHMA R B\OneDrive\Documents\Desktop\project_without_malayalam\chatbot2\yolov5',
         'custom',
-        path='models/best.pt',  # Relative path to the model file
-        source='local'  # Change to 'github' if loading from the official repo
+        path=r"C:\Users\RESHMA R B\OneDrive\Documents\Desktop\project_without_malayalam\chatbot2\models\best.pt",
+        source="local"
     )
+    model.eval()
     return model
 
-# Example usage
-if __name__ == "__main__":
-    model = load_yolov5_model()
-    print("Model loaded successfully!")
-
-
-# Load pre-trained YOLOv5 model
-# def load_yolov5_model():
-#     model = torch.hub.load(
-#         r'C:\Users\RESHMA R B\OneDrive\Documents\Desktop\project_without_malayalam\chatbot2\yolov5',
-#         'custom',
-#         path=r"C:\Users\RESHMA R B\OneDrive\Documents\Desktop\project_without_malayalam\chatbot2\models\best.pt",
-#         source="local"
-#     )
-#     model.eval()
-#     return model
-
 model = load_yolov5_model()
 
 # Function to preprocess user input for better NLP understanding
@@ -157,13 +125,9 @@ def predict_image(image):
         if image is None:
             return "Error: No image uploaded.", "No description available."
 
-        # Convert PIL image to NumPy array
+        # Convert PIL image to NumPy array (OpenCV format)
         image_np = np.array(image)  # Convert PIL image to NumPy array
 
-        # Handle grayscale images
-        if len(image_np.shape) == 2:  # Grayscale image
-            image_np = cv2.cvtColor(image_np, cv2.COLOR_GRAY2RGB)
-
         # Convert RGB to BGR (OpenCV uses BGR by default)
         image_np = cv2.cvtColor(image_np, cv2.COLOR_RGB2BGR)
 
@@ -172,8 +136,8 @@ def predict_image(image):
 
         # Transform the image for the model
         transform = transforms.Compose([
-            transforms.ToTensor(),
-            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+            transforms.ToTensor(),  # Convert image to tensor
+            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),  # Normalize
         ])
         im = transform(image_resized).unsqueeze(0)  # Add batch dimension (BCHW)
 
@@ -198,6 +162,10 @@ def predict_image(image):
             prediction_result = f"Predicted Class ID: {predicted_class_id}\nConfidence: {confidence_score:.4f}"
             description = "No description available."
 
+        # Display the image with OpenCV (optional)
+        cv2.imshow("Processed Image", image_resized)
+        cv2.waitKey(1)  # Wait for 1 ms to display the image
+
         return prediction_result, description
 
     except Exception as e:
@@ -214,6 +182,83 @@ def get_description(class_name):
     }
     return descriptions.get(class_name.lower(), "No description available.")
 
+# Custom LLM Bot Function
+def customLLMBot(user_input, uploaded_image, chat_history):
+    try:
+        global messages
+        logger.info("Processing input...")
+
+        # Preprocess the user input
+        user_input = preprocess_input(user_input)
+        
+        # Analyze sentiment (Optional)
+        sentiment = analyze_sentiment(user_input)
+        logger.info(f"Sentiment detected: {sentiment}")
+
+        # Extract medical entities (Optional)
+        medical_entities = extract_medical_entities(user_input)
+        logger.info(f"Extracted medical entities: {medical_entities}")
+
+        # Append user input to the chat history
+        chat_history.append(("user", user_input))
+
+        if uploaded_image is not None:
+            # Encode the image to base64
+            base64_image = encode_image(uploaded_image)
+
+            logger.debug(f"Image received, size: {len(base64_image)} bytes")
+
+            # Create a message for the image prompt
+            messages_image = [
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": "What's in this image?"},
+                        {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{base64_image}"}}
+                    ]
+                }
+            ]
+
+            logger.info("Sending image to Groq API for processing...")
+            response = client.chat.completions.create(
+                model="llama-3.2-11b-vision-preview",
+                messages=messages_image,
+            )
+            logger.info("Image processed successfully.")
+        else:
+            # Process text input
+            logger.info("Processing text input...")
+            messages.append({
+                "role": "user",
+                "content": user_input
+            })
+            response = client.chat.completions.create(
+                model="llama-3.2-11b-vision-preview",
+                messages=messages,
+            )
+            logger.info("Text processed successfully.")
+
+        # Extract the reply
+        LLM_reply = response.choices[0].message.content
+        logger.debug(f"LLM reply: {LLM_reply}")
+
+        # Append the bot's response to the chat history
+        chat_history.append(("bot", LLM_reply))
+        messages.append({"role": "assistant", "content": LLM_reply})
+
+        # Generate audio for response
+        audio_file = f"response_{uuid.uuid4().hex}.mp3"
+        tts = gTTS(LLM_reply, lang='en')
+        tts.save(audio_file)
+        logger.info(f"Audio response saved as {audio_file}")
+
+        # Return chat history and audio file
+        return chat_history, audio_file
+
+    except Exception as e:
+        logger.error(f"Error in customLLMBot function: {e}")
+        return [("user", user_input or "Image uploaded"), ("bot", f"An error occurred: {e}")], None
+
 # Gradio Interface
 def chatbot_ui():
     with gr.Blocks() as demo:

requirements.txt

@@ -1,30 +1,29 @@
 # Core Libraries
-numpy==1.26.4
-pandas==2.2.2
-scipy==1.13.0
+numpy
+pandas
+scipy
 
 # Machine Learning & Deep Learning
-torch==2.2.1
-torchvision==0.17.1
-transformers==4.39.3
-scikit-learn==1.4.1.post1
-ultralytics==8.1.27
+torch
+torchvision
+transformers
+scikit-learn
+ultralytics
 
 # Image Processing
-pillow==10.2.0
-opencv-python==4.9.0.80
+pillow
+opencv-python
 
 # NLP
-spacy==3.7.4
-https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.0/en_core_web_sm-3.7.0-py3-none-any.whl
+spacy
 
 # Visualization
-matplotlib==3.8.3
+matplotlib
 
 # Gradio & Audio
-gradio==4.24.0
-gtts==2.5.1
+gradio
+gtts
 
 # API Integration
-groq==0.15.0
-requests==2.31.0
+groq
+requests